From 99fb78d0f75395fb378c4631b57e6d97adc64718 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Wed, 23 Sep 2026 15:07:57 +0700 Subject: [PATCH 1/2] feat(plugin-dynamodb): rewrite the DynamoDB driver with index-aware reads, typed edits and table management --- .github/plugin-registry.json | 2 +- CHANGELOG.md | 28 + .../SQLLexicalProfile.swift | 11 +- .../SQLLexicalReadingsTests.swift | 34 + .../DynamoDBAccessPlanner.swift | 432 ++++++ .../DynamoDBAttributeValue.swift | 178 +++ .../DynamoDBBrowseRequest.swift | 106 ++ .../DynamoDBCatalog.swift | 168 +++ .../DynamoDBCellCodec.swift | 268 ++++ .../DynamoDBDriverPlugin/DynamoDBClient.swift | 264 ++++ .../DynamoDBConnection.swift | 655 -------- .../DynamoDBEndpoint.swift | 115 ++ .../DynamoDBDriverPlugin/DynamoDBError.swift | 169 +++ .../DynamoDBExpression.swift | 146 ++ .../DynamoDBFilterTranslator.swift | 355 +++++ .../DynamoDBItemFlattener.swift | 278 ---- .../DynamoDBItemTable.swift | 141 ++ .../DynamoDBDriverPlugin/DynamoDBJSON.swift | 321 ++++ .../DynamoDBDriverPlugin/DynamoDBNumber.swift | 110 ++ .../DynamoDBOperations.swift | 52 - .../DynamoDBPartiQL.swift | 397 +++++ .../DynamoDBPartiQLParser.swift | 173 --- .../DynamoDBDriverPlugin/DynamoDBPlugin.swift | 266 ++-- .../DynamoDBPluginDriver+API.swift | 254 ++++ .../DynamoDBPluginDriver+Execution.swift | 607 ++++++++ .../DynamoDBPluginDriver+Reading.swift | 279 ++++ .../DynamoDBPluginDriver+Schema.swift | 321 ++++ ...DynamoDBPluginDriver+TableManagement.swift | 190 +++ .../DynamoDBPluginDriver+Writes.swift | 192 +++ .../DynamoDBPluginDriver.swift | 1317 +++-------------- .../DynamoDBQueryBuilder.swift | 275 ---- .../DynamoDBRetryPolicy.swift | 82 + .../DynamoDBDriverPlugin/DynamoDBSigner.swift | 114 ++ .../DynamoDBStatement.swift | 203 +++ .../DynamoDBStatementGenerator.swift | 255 ---- .../DynamoDBTableDefinition.swift | 405 +++++ .../DynamoDBTableSchema.swift | 208 +++ .../DynamoDBWriteStatements.swift | 233 +++ .../PluginCreateTableForm.swift | 140 ++ .../PluginDatabaseDriver.swift | 11 + .../PluginSchemaOperation.swift | 3 + .../Core/Coordinators/ExactRowCounter.swift | 18 +- .../Coordinators/PaginationCoordinator.swift | 44 +- .../QueryExecutionCoordinator+Helpers.swift | 6 +- TablePro/Core/Database/DatabaseDriver.swift | 10 + .../Core/Plugins/DatabaseType+Registry.swift | 4 + .../PluginDriverAdapter+CreateTableForm.swift | 17 + .../Core/Plugins/PluginDriverAdapter.swift | 26 +- .../Plugins/PluginManager+Registration.swift | 10 + TablePro/Core/Plugins/PluginManager.swift | 6 + ...PluginMetadataRegistry+CloudDefaults.swift | 242 +-- .../Core/Plugins/PluginMetadataRegistry.swift | 5 + .../Plugins/PluginResultColumnHints.swift | 43 + .../SchemaOperationRefusal.swift | 10 +- .../SchemaStatementGenerator.swift | 4 +- .../Export/ForeignApp/TablePlusImporter.swift | 23 + .../SQL/CatalogChangeClassifier.swift | 12 + .../Utilities/SQL/DynamoDBRequestJSON.swift | 228 +++ .../SQL/DynamoDBRequestStatement.swift | 222 +++ .../SQL/QueryClassifier+DynamoDB.swift | 130 ++ .../Core/Utilities/SQL/QueryClassifier.swift | 14 +- TablePro/Models/Query/QueryTabState.swift | 3 + .../CreateTableFormState+Validation.swift | 157 ++ .../Models/Schema/CreateTableFormState.swift | 156 ++ .../Views/Structure/CreateTableDraft.swift | 18 +- .../Structure/CreateTableFormEditor.swift | 85 ++ .../Structure/CreateTableFormFieldRow.swift | 59 + .../Structure/CreateTableFormPreview.swift | 34 + .../CreateTableFormSectionView.swift | 95 ++ .../Views/Structure/CreateTableFormView.swift | 29 + .../Views/Structure/CreateTableView.swift | 137 +- .../Coordinators/ExactCountOutcomeTests.swift | 53 + .../Coordinators/ExactRowCounterTests.swift | 68 +- .../Core/Coordinators/RowCountPlanTests.swift | 25 + .../Core/Plugins/PasswordHidingTests.swift | 1 + ...ginDriverAdapterCreateTableFormTests.swift | 116 ++ ...tadataRegistryCuratedCapabilityTests.swift | 29 + .../PluginResultColumnHintsTests.swift | 159 ++ .../SchemaOperationRefusalTests.swift | 16 + .../ForeignApp/TablePlusImporterTests.swift | 40 + .../SQL/QueryClassifierDynamoDBTests.swift | 247 ++++ .../Schema/CreateTableFormStateTests.swift | 461 ++++++ .../DocumentStoreCaseSensitivityTests.swift | 30 - .../DynamoDB/DynamoDBAccessPlannerTests.swift | 722 +++++++++ .../DynamoDBAttributeValueTests.swift | 189 +++ .../DynamoDB/DynamoDBCatalogTests.swift | 293 ++++ .../DynamoDB/DynamoDBCellCodecTests.swift | 357 +++++ .../DynamoDB/DynamoDBClientTests.swift | 427 ++++++ .../DynamoDB/DynamoDBDriverTests.swift | 1018 +++++++++++++ .../DynamoDB/DynamoDBEndpointTests.swift | 246 +++ .../Plugins/DynamoDB/DynamoDBErrorTests.swift | 244 +++ .../DynamoDB/DynamoDBExpressionTests.swift | 259 ++++ .../DynamoDBFilterTranslatorTests.swift | 642 ++++++++ .../DynamoDB/DynamoDBItemTableTests.swift | 316 ++++ .../Plugins/DynamoDB/DynamoDBJSONTests.swift | 256 ++++ .../DynamoDBLocalIntegrationTests.swift | 367 +++++ .../DynamoDB/DynamoDBNumberTests.swift | 170 +++ .../DynamoDBParameterBinderTests.swift | 243 +++ .../DynamoDB/DynamoDBPartiQLTests.swift | 403 +++++ .../DynamoDB/DynamoDBRetryPolicyTests.swift | 264 ++++ .../DynamoDB/DynamoDBSignerTests.swift | 189 +++ .../DynamoDB/DynamoDBStatementTests.swift | 423 ++++++ .../DynamoDBTableManagementTests.swift | 348 +++++ .../DynamoDBWriteStatementsTests.swift | 279 ++++ .../Plugins/DynamoDBMetadataParityTests.swift | 114 ++ .../Plugins/DynamoDBOperationsTests.swift | 57 - .../Plugins/DynamoDBQueryBuilderTests.swift | 336 ----- docs/databases/dynamodb.mdx | 179 ++- docs/development/plugin-development.mdx | 1 + docs/features/table-operations.mdx | 3 +- docs/images/dynamodb-create-table-dark.png | Bin 0 -> 6565 bytes docs/images/dynamodb-create-table.png | Bin 0 -> 6565 bytes docs/scripts/check-docs-against-source.py | 3 +- project.yml | 31 +- scripts/dynamodb-test-local.sh | 55 + 115 files changed, 17707 insertions(+), 3577 deletions(-) create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBAccessPlanner.swift create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBAttributeValue.swift create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBBrowseRequest.swift create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBCatalog.swift create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBCellCodec.swift create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBClient.swift delete mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBConnection.swift create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBEndpoint.swift create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBError.swift create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBExpression.swift create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBFilterTranslator.swift delete mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBItemFlattener.swift create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBItemTable.swift create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBJSON.swift create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBNumber.swift delete mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBOperations.swift create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBPartiQL.swift delete mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBPartiQLParser.swift create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+API.swift create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Execution.swift create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Reading.swift create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Schema.swift create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+TableManagement.swift create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Writes.swift delete mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBQueryBuilder.swift create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBRetryPolicy.swift create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBSigner.swift create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBStatement.swift delete mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBStatementGenerator.swift create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBTableDefinition.swift create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBTableSchema.swift create mode 100644 Plugins/DynamoDBDriverPlugin/DynamoDBWriteStatements.swift create mode 100644 Plugins/TableProPluginKit/PluginCreateTableForm.swift create mode 100644 TablePro/Core/Plugins/PluginDriverAdapter+CreateTableForm.swift create mode 100644 TablePro/Core/Plugins/PluginResultColumnHints.swift create mode 100644 TablePro/Core/Utilities/SQL/DynamoDBRequestJSON.swift create mode 100644 TablePro/Core/Utilities/SQL/DynamoDBRequestStatement.swift create mode 100644 TablePro/Core/Utilities/SQL/QueryClassifier+DynamoDB.swift create mode 100644 TablePro/Models/Schema/CreateTableFormState+Validation.swift create mode 100644 TablePro/Models/Schema/CreateTableFormState.swift create mode 100644 TablePro/Views/Structure/CreateTableFormEditor.swift create mode 100644 TablePro/Views/Structure/CreateTableFormFieldRow.swift create mode 100644 TablePro/Views/Structure/CreateTableFormPreview.swift create mode 100644 TablePro/Views/Structure/CreateTableFormSectionView.swift create mode 100644 TablePro/Views/Structure/CreateTableFormView.swift create mode 100644 TableProTests/Core/Coordinators/ExactCountOutcomeTests.swift create mode 100644 TableProTests/Core/Plugins/PluginDriverAdapterCreateTableFormTests.swift create mode 100644 TableProTests/Core/Plugins/PluginResultColumnHintsTests.swift create mode 100644 TableProTests/Core/Utilities/SQL/QueryClassifierDynamoDBTests.swift create mode 100644 TableProTests/Models/Schema/CreateTableFormStateTests.swift create mode 100644 TableProTests/Plugins/DynamoDB/DynamoDBAccessPlannerTests.swift create mode 100644 TableProTests/Plugins/DynamoDB/DynamoDBAttributeValueTests.swift create mode 100644 TableProTests/Plugins/DynamoDB/DynamoDBCatalogTests.swift create mode 100644 TableProTests/Plugins/DynamoDB/DynamoDBCellCodecTests.swift create mode 100644 TableProTests/Plugins/DynamoDB/DynamoDBClientTests.swift create mode 100644 TableProTests/Plugins/DynamoDB/DynamoDBDriverTests.swift create mode 100644 TableProTests/Plugins/DynamoDB/DynamoDBEndpointTests.swift create mode 100644 TableProTests/Plugins/DynamoDB/DynamoDBErrorTests.swift create mode 100644 TableProTests/Plugins/DynamoDB/DynamoDBExpressionTests.swift create mode 100644 TableProTests/Plugins/DynamoDB/DynamoDBFilterTranslatorTests.swift create mode 100644 TableProTests/Plugins/DynamoDB/DynamoDBItemTableTests.swift create mode 100644 TableProTests/Plugins/DynamoDB/DynamoDBJSONTests.swift create mode 100644 TableProTests/Plugins/DynamoDB/DynamoDBLocalIntegrationTests.swift create mode 100644 TableProTests/Plugins/DynamoDB/DynamoDBNumberTests.swift create mode 100644 TableProTests/Plugins/DynamoDB/DynamoDBParameterBinderTests.swift create mode 100644 TableProTests/Plugins/DynamoDB/DynamoDBPartiQLTests.swift create mode 100644 TableProTests/Plugins/DynamoDB/DynamoDBRetryPolicyTests.swift create mode 100644 TableProTests/Plugins/DynamoDB/DynamoDBSignerTests.swift create mode 100644 TableProTests/Plugins/DynamoDB/DynamoDBStatementTests.swift create mode 100644 TableProTests/Plugins/DynamoDB/DynamoDBTableManagementTests.swift create mode 100644 TableProTests/Plugins/DynamoDB/DynamoDBWriteStatementsTests.swift create mode 100644 TableProTests/Plugins/DynamoDBMetadataParityTests.swift delete mode 100644 TableProTests/Plugins/DynamoDBOperationsTests.swift delete mode 100644 TableProTests/Plugins/DynamoDBQueryBuilderTests.swift create mode 100644 docs/images/dynamodb-create-table-dark.png create mode 100644 docs/images/dynamodb-create-table.png create mode 100755 scripts/dynamodb-test-local.sh diff --git a/.github/plugin-registry.json b/.github/plugin-registry.json index 89cd97b446..9af5393f64 100644 --- a/.github/plugin-registry.json +++ b/.github/plugin-registry.json @@ -136,7 +136,7 @@ "bundleId": "com.TablePro.DynamoDBDriverPlugin", "bundled": false, "displayName": "DynamoDB Driver", - "summary": "Amazon DynamoDB driver with PartiQL queries and AWS IAM/Profile/SSO authentication", + "summary": "Amazon DynamoDB: index-aware browsing, typed editing, PartiQL, the DynamoDB API and table management", "databaseTypeIds": [ "DynamoDB" ], diff --git a/CHANGELOG.md b/CHANGELOG.md index e2e438bd59..93967e4772 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **File > Session**, with the agent session commands and the assistant's conversation commands. - Eight more rebindable commands in **Settings > Keyboard**, among them the sidebar's lists and the session commands. - **Global** on a saved query folder's menu, for a folder every connection shows. +- DynamoDB reads that query the table, a local index or a global index when the grid's filters allow it. +- Create Table form for DynamoDB: keys, capacity, table class, deletion protection and secondary indexes. +- DynamoDB API requests in the editor, such as `CreateTable {…}`, `UpdateTimeToLive {…}` and `BatchWriteItem {…}`. +- DynamoDB point-in-time recovery, deletion protection, stream, class and billing under **Maintenance**. +- Adding and dropping a DynamoDB global secondary index from the **Structure** tab. +- Nested DynamoDB attribute paths in the filter bar and autocomplete. +- **DynamoDB Local (no credentials)** auth method. +- Items read and read units for a DynamoDB browse in the result status bar. ### Changed @@ -59,6 +67,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Middle-dot separators dropped from the CSV inspector's status bar and the query history rows. - Connection marked with a tinted symbol rather than a color dot in the query history rows. - Safe Mode list offering only the levels a connection allows, with the reason under it and in the toolbar tooltip. +- DynamoDB maps, lists and sets shown as plain JSON and edited in the JSON editor. +- DynamoDB column types named as the AWS console names them: String, Number, Map, String Set. +- DynamoDB region taken from the AWS profile when the connection names none. +- DynamoDB table counts left to **Count Exactly**, with no automatic full-table count. +- DynamoDB table DDL shown as the `CreateTable` request that recreates it. +- Plain HTTP DynamoDB endpoint refused for any host but this Mac, instead of switched to HTTPS. ### Removed @@ -321,6 +335,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Destination folder and the first database reading as one path in the backup result sheet. (#3046) - Only the last line of a failed backup's error shown, which on `pg_dump` is the hint rather than the cause. - Backup failure reported as an exit code alone when the tool wrote its message and exited at once. +- DynamoDB edits saving numbers, booleans, maps and sets as strings. +- DynamoDB binary, map and long-text edits lost or saved as a fragment, and **Set NULL** leaving a NULL attribute. +- Duplicated DynamoDB row saved with `__DEFAULT__` as its key. +- Nine DynamoDB filter operators matching nothing, and an OR filter dropping other partitions' items. +- DynamoDB column sort ignored, and each page re-reading every page before it. +- DynamoDB **Count Exactly** never finishing. +- DynamoDB export dropping attributes first seen after the first page. +- DynamoDB PartiQL result cut to its first 1 MB. +- DynamoDB **Stop** cancelling the wrong request, and throttled requests failing instead of retrying. +- AWS SSO sign-in prompt never shown for DynamoDB. +- DynamoDB connections to China and European Sovereign Cloud regions failing. +- DynamoDB connection sampling every table on connect. +- Imported DynamoDB connection losing its AWS Region. +- Failed **Count Exactly** showing no error. ### Security diff --git a/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift index e9670bc16b..f47fc6bda4 100644 --- a/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift +++ b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift @@ -170,6 +170,11 @@ public struct SQLLexicalProfile: Sendable, Hashable { .doubleSlashLineComments, ] + /// DynamoDB, from the driver's statement forms: a request is ` {JSON}`, and a backslash escapes inside a + /// JSON string. PartiQL, the other form, documents only a doubled quote, so plain ANSI is kept as an alternative + /// and a gate reads both. Not measured. + static let dynamoDB: SQLLexicalGrammar = [.backslashEscapesInDoubleQuotes] + /// Engines whose statements are commands or JSON documents rather than SQL. Their splitting is what it has always /// been: a backslash escapes inside any quote, as it does in JSON and in `redis-cli`. static let commandLine: SQLLexicalGrammar = [ @@ -250,7 +255,11 @@ public struct SQLLexicalProfile: Sendable, Hashable { ), "Cassandra": cqlFamily, "ScyllaDB": cqlFamily, - "DynamoDB": SQLLexicalProfile(grammar: .ansi, undetermined: [.carriageReturnEndsLineComments]), + "DynamoDB": SQLLexicalProfile( + grammar: dynamoDB, + alternatives: [.ansi], + undetermined: [.carriageReturnEndsLineComments] + ), "SurrealDB": SQLLexicalProfile(grammar: surrealQL, undetermined: [.carriageReturnEndsLineComments]), "Redis": commandLineFamily, "MongoDB": commandLineFamily, diff --git a/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLLexicalReadingsTests.swift b/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLLexicalReadingsTests.swift index b83d9f76ea..0ee3eea13b 100644 --- a/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLLexicalReadingsTests.swift +++ b/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLLexicalReadingsTests.swift @@ -79,6 +79,40 @@ struct SQLLexicalReadingsTests { #expect(readings.contains { $0.contains(.taggedDollarQuotes) }) } + @Test("A DynamoDB request whose JSON string escapes a quote is one statement to the driver") + func dynamoDBRequestKeepsEscapedQuotesInsideTheString() { + let readings = SQLLexicalReadings.resolve(databaseTypeId: "DynamoDB", declared: nil, session: nil) + let text = #"PutItem {"TableName": "t", "Item": {"a": {"S": "x\";y"}}}; SELECT * FROM "t""# + + let executed = SQLStatementScanner.executableStatements(in: text, grammar: readings.execution).map(\.sql) + + #expect(executed == [#"PutItem {"TableName": "t", "Item": {"a": {"S": "x\";y"}}}"#, #"SELECT * FROM "t""#]) + } + + @Test("DynamoDB keeps plain ANSI as a reading, so a gate still counts what PartiQL could split") + func dynamoDBGateReadsTheANSIReading() { + let readings = SQLLexicalReadings.resolve(databaseTypeId: "DynamoDB", declared: nil, session: nil) + let text = #"PutItem {"TableName": "t", "Item": {"a": {"S": "x\";y\";z"}}}; SELECT * FROM "t""# + + let counts = readings.distinct(for: text).map { + SQLStatementScanner.executableStatements(in: text, grammar: $0).count + } + + #expect(readings.all.contains(.ansi)) + #expect(SQLStatementScanner.executableStatements(in: text, grammar: readings.execution).count == 2) + #expect(counts.max() == 3) + } + + @Test("PartiQL's doubled quote still closes nothing under DynamoDB's execution grammar") + func dynamoDBPartiQLDoubledQuoteStillLexes() { + let readings = SQLLexicalReadings.resolve(databaseTypeId: "DynamoDB", declared: nil, session: nil) + let text = #"SELECT * FROM "a""b"; DELETE FROM "t""# + + let executed = SQLStatementScanner.executableStatements(in: text, grammar: readings.execution).map(\.sql) + + #expect(executed == [#"SELECT * FROM "a""b""#, #"DELETE FROM "t""#]) + } + @Test("Every combination of the undetermined facts is a reading") func undeterminedFactsExpand() { let profile = SQLLexicalProfile(grammar: .ansi, undetermined: [.hashLineComments, .nestedBlockComments]) diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBAccessPlanner.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBAccessPlanner.swift new file mode 100644 index 0000000000..c8331aa022 --- /dev/null +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBAccessPlanner.swift @@ -0,0 +1,432 @@ +import Foundation + +/// How a Browse request reads DynamoDB. +struct DynamoDBReadPlan: Sendable, Equatable { + enum Access: Sendable, Equatable { + case scan + case query + case batchGet + case nothing + } + + let table: String + let access: Access + let indexName: String? + /// The request bodies, read one after another, each paged to its end. + let requests: [[String: DynamoDBJSON]] + let clientPredicates: [DynamoDBClientPredicate] + let clientMatchAll: Bool + /// Sort terms DynamoDB could not apply, which the reader applies itself when it holds the + /// whole result. + let unsatisfiedOrder: [DynamoDBOrderTerm] + + var hasFilters: Bool { + !clientPredicates.isEmpty || requests.contains { $0["FilterExpression"] != nil } + } + + func clientMatches(_ item: DynamoDBItem) -> Bool { + guard !clientPredicates.isEmpty else { return true } + if clientMatchAll { + return clientPredicates.allSatisfy { $0.matches(item) } + } + return clientPredicates.contains { $0.matches(item) } + } + + /// Which request a remembered page position belongs to. A position is the key of an item in + /// one index, so it cannot start a read of another: a filter that scanned while an index was + /// still being built queries that index once it is ready. + var positionKey: String { + switch access { + case .scan: return "|scan" + case .query: return "|query:\(indexName ?? "")" + case .batchGet: return "|batchGet" + case .nothing: return "|nothing" + } + } + + var summary: String { + switch access { + case .query: + guard let indexName else { return String(localized: "Query on the table") } + return String(format: String(localized: "Query on index %@"), indexName) + case .scan: + return hasFilters ? String(localized: "Scan with filters") : String(localized: "Scan") + case .batchGet: + return String(localized: "Get by key") + case .nothing: + return String(localized: "No read needed") + } + } +} + +struct DynamoDBAccessPlanner { + let schema: DynamoDBTableSchema + + private var translator: DynamoDBFilterTranslator { DynamoDBFilterTranslator(schema: schema) } + + func plan(_ request: DynamoDBBrowseRequest, order: [DynamoDBOrderTerm]) throws -> DynamoDBReadPlan { + if let reason = request.filters.lazy.compactMap(DynamoDBFilterTranslator.unsupportedReason).first { + throw DynamoDBError.invalidStatement(reason) + } + let known = Set(request.columns).union(schema.allKeyAttributes) + let paths = request.filters.map { filter -> DynamoDBAttributePath? in + guard filter.attribute != DynamoDBFilterTranslator.anyAttributeColumn else { return nil } + return DynamoDBAttributePath.parse(filter.attribute, knownAttributes: known) + } + if request.matchAll { + return planAll(request, paths: paths, order: order) + } + return planAny(request, paths: paths, order: order) + } + + // MARK: - Match All + + private struct Candidate { + let index: DynamoDBIndex? + let keys: DynamoDBKeySchema + let partitionFilters: [Int] + let sortFilters: [Int] + let partitionValues: [DynamoDBAttributeValue]? + let score: Int + } + + private func planAll( + _ request: DynamoDBBrowseRequest, + paths: [DynamoDBAttributePath?], + order: [DynamoDBOrderTerm] + ) -> DynamoDBReadPlan { + guard let candidate = bestCandidate(request, paths: paths, order: order) else { + return scanPlan(request, paths: paths, order: order) + } + let orderOnSortKey = candidate.keys.sort.count == 1 + && order.count == 1 + && order[0].attribute == candidate.keys.sort[0] + && (candidate.partitionValues?.count ?? 1) == 1 + + let partitionValueSets = candidate.partitionValues.map { values in values.map { [$0] } } + ?? [[]] + var requests: [[String: DynamoDBJSON]] = [] + var clientPredicates: [DynamoDBClientPredicate] = [] + var isImpossible = false + + for partitionValue in partitionValueSets { + var context = DynamoDBExpressionContext() + var keyTerms: [String] = [] + for (position, filterIndex) in candidate.partitionFilters.enumerated() { + let attribute = candidate.keys.partition[position] + if let value = partitionValue.first, candidate.partitionValues != nil { + let name = context.name(attribute) + keyTerms.append("\(name) = \(context.value(value, hint: attribute))") + } else if let term = translator.keyCondition(request.filters[filterIndex], attribute: attribute, context: &context) { + keyTerms.append(term) + } else { + isImpossible = true + } + } + for (position, filterIndex) in candidate.sortFilters.enumerated() { + let attribute = candidate.keys.sort[position] + guard let term = translator.keyCondition(request.filters[filterIndex], attribute: attribute, context: &context) + else { + isImpossible = true + continue + } + keyTerms.append(term) + } + + let usedFilters = Set(candidate.partitionFilters + candidate.sortFilters) + let pathKeys = Set(candidate.keys.attributes) + var filterTerms: [String] = [] + clientPredicates = [] + for (index, filter) in request.filters.enumerated() where !usedFilters.contains(index) { + let path = paths[index] + if let path, path.isTopLevel, pathKeys.contains(path.root) { + clientPredicates.append(translator.clientPredicate(filter, path: path)) + continue + } + var trial = context + switch translator.translate(filter, path: path, context: &trial) { + case .server(let term): + context = trial + filterTerms.append(term) + case .client(let predicate): clientPredicates.append(predicate) + case .never: isImpossible = true + } + } + + var body: [String: DynamoDBJSON] = [ + "TableName": .string(schema.name), + "KeyConditionExpression": .string(keyTerms.joined(separator: " AND ")) + ] + if let index = candidate.index { + body["IndexName"] = .string(index.name) + if index.kind == .local, index.projection != .all { + body["Select"] = .string("ALL_ATTRIBUTES") + } + } + if !filterTerms.isEmpty { + body["FilterExpression"] = .string(filterTerms.joined(separator: " AND ")) + } + if orderOnSortKey, order[0].descending { + body["ScanIndexForward"] = .bool(false) + } + context.apply(to: &body) + requests.append(body) + } + + guard !isImpossible else { return nothingPlan() } + return DynamoDBReadPlan( + table: schema.name, + access: .query, + indexName: candidate.index?.name, + requests: requests, + clientPredicates: clientPredicates, + clientMatchAll: true, + unsatisfiedOrder: orderOnSortKey ? [] : order + ) + } + + private func bestCandidate( + _ request: DynamoDBBrowseRequest, + paths: [DynamoDBAttributePath?], + order: [DynamoDBOrderTerm] + ) -> Candidate? { + var candidates: [(DynamoDBIndex?, DynamoDBKeySchema, Int)] = [(nil, schema.keys, 3)] + for index in schema.indexes where index.isQueryable { + if index.kind == .global, index.projection != .all { continue } + candidates.append((index, index.keys, index.kind == .local ? 2 : 1)) + } + return candidates.compactMap { index, keys, preference in + candidate(request, paths: paths, index: index, keys: keys, preference: preference, order: order) + }.max { $0.score < $1.score } + } + + private func candidate( + _ request: DynamoDBBrowseRequest, + paths: [DynamoDBAttributePath?], + index: DynamoDBIndex?, + keys: DynamoDBKeySchema, + preference: Int, + order: [DynamoDBOrderTerm] + ) -> Candidate? { + guard !keys.partition.isEmpty else { return nil } + var partitionFilters: [Int] = [] + var partitionValues: [DynamoDBAttributeValue]? + for attribute in keys.partition { + if let found = filterIndex(request, paths: paths, attribute: attribute, operators: ["="]) { + partitionFilters.append(found) + continue + } + guard keys.partition.count == 1, + let found = filterIndex(request, paths: paths, attribute: attribute, operators: ["IN"]), + let type = schema.keyType(of: attribute) + else { return nil } + var values: [DynamoDBAttributeValue] = [] + for item in DynamoDBClientPredicate.listItems(request.filters[found].value) { + guard let typed = translator.typed(item, as: type) else { continue } + if !values.contains(where: { DynamoDBAccessPlanner.sameKeyValue($0, typed) }) { + values.append(typed) + } + } + guard !values.isEmpty, values.count <= 100 else { return nil } + partitionFilters.append(found) + partitionValues = values + } + + var sortFilters: [Int] = [] + for (position, attribute) in keys.sort.enumerated() { + let isLast = position == keys.sort.count - 1 + let operators = isLast ? DynamoDBFilterTranslator.keyConditionOperators : ["="] + guard let found = filterIndex(request, paths: paths, attribute: attribute, operators: operators) else { break } + sortFilters.append(found) + guard request.filters[found].op == "=" else { break } + } + + if index != nil, !keys.sort.allSatisfy({ excludesItemsMissing($0, request, paths: paths) }) { + return nil + } + let orderMatches = keys.sort.count == 1 && order.count == 1 && order[0].attribute == keys.sort[0] + let score = preference + sortFilters.count * 10 + (orderMatches ? 5 : 0) + return Candidate( + index: index, + keys: keys, + partitionFilters: partitionFilters, + sortFilters: sortFilters, + partitionValues: partitionValues, + score: score + ) + } + + /// A secondary index holds only the items that carry every one of its key attributes, so it + /// answers a read only when the filters already reject an item missing `attribute`. Every + /// operator but `IS NULL` is false on a missing attribute. + private func excludesItemsMissing( + _ attribute: String, + _ request: DynamoDBBrowseRequest, + paths: [DynamoDBAttributePath?] + ) -> Bool { + request.filters.indices.contains { index in + guard let path = paths[index], path.isTopLevel, path.root == attribute else { return false } + return request.filters[index].op != "IS NULL" + } + } + + private func filterIndex( + _ request: DynamoDBBrowseRequest, + paths: [DynamoDBAttributePath?], + attribute: String, + operators: Set + ) -> Int? { + request.filters.indices.first { index in + guard let path = paths[index], path.isTopLevel, path.root == attribute else { return false } + let filter = request.filters[index] + guard operators.contains(filter.op), !translator.needsClient(filter) else { return false } + guard let type = schema.keyType(of: attribute) else { return false } + if filter.op == "STARTS WITH", type == .number { return false } + if filter.op == "BETWEEN" { + guard let bounds = DynamoDBFilterTranslator.bounds(value: filter.value, secondValue: filter.secondValue) + else { return false } + return translator.typed(bounds.lower, as: type) != nil && translator.typed(bounds.upper, as: type) != nil + } + if filter.op == "IN" { return true } + return translator.typed(filter.value, as: type) != nil + } + } + + // MARK: - Match Any + + private func planAny( + _ request: DynamoDBBrowseRequest, + paths: [DynamoDBAttributePath?], + order: [DynamoDBOrderTerm] + ) -> DynamoDBReadPlan { + if let keyed = keyListPlan(request, paths: paths, order: order) { + return keyed + } + if request.filters.contains(where: translator.needsClient) { + let predicates = request.filters.enumerated().map { index, filter in + translator.clientPredicate(filter, path: paths[index]) + } + return DynamoDBReadPlan( + table: schema.name, access: .scan, indexName: nil, + requests: [["TableName": .string(schema.name)]], + clientPredicates: predicates, clientMatchAll: false, unsatisfiedOrder: order + ) + } + var context = DynamoDBExpressionContext() + var terms: [String] = [] + for (index, filter) in request.filters.enumerated() { + var trial = context + if case .server(let term) = translator.translate(filter, path: paths[index], context: &trial) { + context = trial + terms.append(term) + } + } + guard !terms.isEmpty else { return nothingPlan() } + var body: [String: DynamoDBJSON] = [ + "TableName": .string(schema.name), + "FilterExpression": .string(terms.joined(separator: " OR ")) + ] + context.apply(to: &body) + return DynamoDBReadPlan( + table: schema.name, access: .scan, indexName: nil, requests: [body], + clientPredicates: [], clientMatchAll: true, unsatisfiedOrder: order + ) + } + + /// `pk = a OR pk = b`, which is how Data Rewind reads rows back: a batch of GetItems when the + /// table has no sort key, one Query per partition when it has one. A Scan would read the table. + private func keyListPlan( + _ request: DynamoDBBrowseRequest, + paths: [DynamoDBAttributePath?], + order: [DynamoDBOrderTerm] + ) -> DynamoDBReadPlan? { + guard schema.keys.partition.count == 1, + let partition = schema.keys.partition.first, + let type = schema.keyType(of: partition), + !request.filters.isEmpty + else { return nil } + var values: [DynamoDBAttributeValue] = [] + for (index, filter) in request.filters.enumerated() { + guard let path = paths[index], path.isTopLevel, path.root == partition, + filter.op == "=", !translator.needsClient(filter), + let value = translator.typed(filter.value, as: type) + else { return nil } + if !values.contains(where: { Self.sameKeyValue($0, value) }) { values.append(value) } + } + + if schema.keys.sort.isEmpty { + let requests = stride(from: 0, to: values.count, by: 100).map { start -> [String: DynamoDBJSON] in + let keys = values[start.. [String: DynamoDBJSON] in + var context = DynamoDBExpressionContext() + let name = context.name(partition) + var body: [String: DynamoDBJSON] = [ + "TableName": .string(schema.name), + "KeyConditionExpression": .string("\(name) = \(context.value(value, hint: partition))"), + "ConsistentRead": .bool(true) + ] + context.apply(to: &body) + return body + } + return DynamoDBReadPlan( + table: schema.name, access: .query, indexName: nil, requests: requests, + clientPredicates: [], clientMatchAll: true, unsatisfiedOrder: order + ) + } + + // MARK: - Scan + + private func scanPlan( + _ request: DynamoDBBrowseRequest, + paths: [DynamoDBAttributePath?], + order: [DynamoDBOrderTerm] + ) -> DynamoDBReadPlan { + var context = DynamoDBExpressionContext() + var terms: [String] = [] + var predicates: [DynamoDBClientPredicate] = [] + for (index, filter) in request.filters.enumerated() { + var trial = context + switch translator.translate(filter, path: paths[index], context: &trial) { + case .server(let term): + context = trial + terms.append(term) + case .client(let predicate): predicates.append(predicate) + case .never: return nothingPlan() + } + } + var body: [String: DynamoDBJSON] = ["TableName": .string(schema.name)] + if !terms.isEmpty { + body["FilterExpression"] = .string(terms.joined(separator: " AND ")) + } + context.apply(to: &body) + return DynamoDBReadPlan( + table: schema.name, access: .scan, indexName: nil, requests: [body], + clientPredicates: predicates, clientMatchAll: true, unsatisfiedOrder: order + ) + } + + /// Key values DynamoDB treats as one item: `5` and `5.0` are the same Number. + static func sameKeyValue(_ lhs: DynamoDBAttributeValue, _ rhs: DynamoDBAttributeValue) -> Bool { + if case .number(let left) = lhs, case .number(let right) = rhs { + return DynamoDBNumber.areEqual(left, right) + } + return lhs == rhs + } + + private func nothingPlan() -> DynamoDBReadPlan { + DynamoDBReadPlan( + table: schema.name, access: .nothing, indexName: nil, requests: [], + clientPredicates: [], clientMatchAll: true, unsatisfiedOrder: [] + ) + } +} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBAttributeValue.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBAttributeValue.swift new file mode 100644 index 0000000000..3e8af77d3f --- /dev/null +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBAttributeValue.swift @@ -0,0 +1,178 @@ +import Foundation + +enum DynamoDBAttributeType: String, CaseIterable, Sendable { + case string = "S" + case number = "N" + case binary = "B" + case boolean = "BOOL" + case null = "NULL" + case list = "L" + case map = "M" + case stringSet = "SS" + case numberSet = "NS" + case binarySet = "BS" + + /// The names the DynamoDB console uses, which is what the grid and the Structure tab show. + var displayName: String { + switch self { + case .string: return "String" + case .number: return "Number" + case .binary: return "Binary" + case .boolean: return "Boolean" + case .null: return "Null" + case .list: return "List" + case .map: return "Map" + case .stringSet: return "String Set" + case .numberSet: return "Number Set" + case .binarySet: return "Binary Set" + } + } + + /// The type name the app classifies a column by. A list, a map and every set show as JSON, so + /// the grid offers the JSON editor for them. + var classificationName: String { + switch self { + case .string, .null: return "TEXT" + case .number: return "NUMERIC" + case .binary: return "BLOB" + case .boolean: return "BOOLEAN" + case .list, .map, .stringSet, .numberSet, .binarySet: return "JSON" + } + } + + var isKeyType: Bool { + self == .string || self == .number || self == .binary + } + + init?(displayName: String) { + guard let match = Self.allCases.first(where: { + $0.displayName.caseInsensitiveCompare(displayName) == .orderedSame + || $0.rawValue.caseInsensitiveCompare(displayName) == .orderedSame + }) else { return nil } + self = match + } +} + +indirect enum DynamoDBAttributeValue: Sendable, Equatable { + case string(String) + case number(String) + case binary(Data) + case bool(Bool) + case null + case list([DynamoDBAttributeValue]) + case map([String: DynamoDBAttributeValue]) + case stringSet([String]) + case numberSet([String]) + case binarySet([Data]) + + var type: DynamoDBAttributeType { + switch self { + case .string: return .string + case .number: return .number + case .binary: return .binary + case .bool: return .boolean + case .null: return .null + case .list: return .list + case .map: return .map + case .stringSet: return .stringSet + case .numberSet: return .numberSet + case .binarySet: return .binarySet + } + } + + /// The wire form, `{"S": "x"}`, as a JSON tree. + var wireJSON: DynamoDBJSON { + switch self { + case .string(let value): + return .object(["S": .string(value)]) + case .number(let value): + return .object(["N": .string(value)]) + case .binary(let value): + return .object(["B": .string(value.base64EncodedString())]) + case .bool(let value): + return .object(["BOOL": .bool(value)]) + case .null: + return .object(["NULL": .bool(true)]) + case .list(let items): + return .object(["L": .array(items.map(\.wireJSON))]) + case .map(let entries): + return .object(["M": .object(entries.mapValues(\.wireJSON))]) + case .stringSet(let values): + return .object(["SS": .array(values.map(DynamoDBJSON.string))]) + case .numberSet(let values): + return .object(["NS": .array(values.map(DynamoDBJSON.string))]) + case .binarySet(let values): + return .object(["BS": .array(values.map { .string($0.base64EncodedString()) })]) + } + } + + init(wireJSON json: DynamoDBJSON) throws { + guard case .object(let entries) = json, entries.count == 1, let (tag, payload) = entries.first else { + throw DynamoDBError.invalidResponse(String(localized: "An attribute value must name exactly one type")) + } + guard let type = DynamoDBAttributeType(rawValue: tag) else { + throw DynamoDBError.invalidResponse( + String(format: String(localized: "Unknown attribute type \"%@\""), tag) + ) + } + self = try Self.decode(type: type, payload: payload) + } + + private static func decode(type: DynamoDBAttributeType, payload: DynamoDBJSON) throws -> DynamoDBAttributeValue { + switch (type, payload) { + case (.string, .string(let value)): + return .string(value) + case (.number, .string(let value)): + return .number(value) + case (.binary, .string(let value)): + return .binary(try decodeBase64(value)) + case (.boolean, .bool(let value)): + return .bool(value) + case (.null, .bool): + return .null + case (.list, .array(let items)): + return .list(try items.map(DynamoDBAttributeValue.init(wireJSON:))) + case (.map, .object(let entries)): + return .map(try entries.mapValues(DynamoDBAttributeValue.init(wireJSON:))) + case (.stringSet, .array(let items)): + return .stringSet(try items.map(stringPayload)) + case (.numberSet, .array(let items)): + return .numberSet(try items.map(stringPayload)) + case (.binarySet, .array(let items)): + return .binarySet(try items.map { try decodeBase64(try stringPayload($0)) }) + default: + throw DynamoDBError.invalidResponse( + String(format: String(localized: "A %@ attribute has a payload of the wrong shape"), type.rawValue) + ) + } + } + + private static func stringPayload(_ json: DynamoDBJSON) throws -> String { + guard case .string(let value) = json else { + throw DynamoDBError.invalidResponse(String(localized: "A set member must be a string")) + } + return value + } + + private static func decodeBase64(_ text: String) throws -> Data { + guard let data = Data(base64Encoded: text) else { + throw DynamoDBError.invalidResponse(String(localized: "A binary value is not valid base64")) + } + return data + } +} + +typealias DynamoDBItem = [String: DynamoDBAttributeValue] + +extension Dictionary where Key == String, Value == DynamoDBAttributeValue { + var wireJSON: DynamoDBJSON { + .object(mapValues(\.wireJSON)) + } + + init(wireItem json: DynamoDBJSON) throws { + guard case .object(let entries) = json else { + throw DynamoDBError.invalidResponse(String(localized: "An item must be a JSON object")) + } + self = try entries.mapValues(DynamoDBAttributeValue.init(wireJSON:)) + } +} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBBrowseRequest.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBBrowseRequest.swift new file mode 100644 index 0000000000..f522687a08 --- /dev/null +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBBrowseRequest.swift @@ -0,0 +1,106 @@ +import Foundation +import TableProPluginKit + +struct DynamoDBBrowseFilter: Sendable, Equatable { + let attribute: String + let op: String + let value: String + let secondValue: String? + /// The grid's kind for the column (`text`, `integer`, `decimal`, `boolean`), a hint only. + let kind: String? + let caseSensitive: Bool +} + +/// What a table tab asks for: the table, the grid's filters and its columns. +/// +/// This is the request, not the plan. The driver picks the table, a local index or a global index +/// and the key condition when the statement runs, against the table as it is then, so the grid +/// never builds one page with an old key schema and the next with a new one. +struct DynamoDBBrowseRequest: Sendable, Equatable { + let table: String + let filters: [DynamoDBBrowseFilter] + let matchAll: Bool + let columns: [String] + + init(table: String, filters: [DynamoDBBrowseFilter], matchAll: Bool, columns: [String]) { + self.table = table + self.filters = filters + self.matchAll = matchAll + self.columns = columns + } + + init( + table: String, + queryFilters: [PluginQueryFilter], + logicMode: String, + columns: [String], + columnKinds: [String: PluginColumnKind] + ) { + self.table = table + self.filters = queryFilters.map { filter in + DynamoDBBrowseFilter( + attribute: filter.column, + op: filter.op.uppercased(), + value: filter.value, + secondValue: filter.secondValue, + kind: columnKinds[filter.column]?.rawValue, + caseSensitive: filter.isCaseSensitive + ) + } + self.matchAll = logicMode.lowercased() != "or" + self.columns = columns + } + + init(json: DynamoDBJSON) throws { + guard let table = json["TableName"]?.stringValue, !table.isEmpty else { + throw DynamoDBError.invalidStatement(String(localized: "Browse needs a TableName")) + } + self.table = table + self.filters = try (json["Filters"]?.arrayValue ?? []).map { entry in + guard let attribute = entry["Attribute"]?.stringValue, let op = entry["Operator"]?.stringValue else { + throw DynamoDBError.invalidStatement(String(localized: "Each Browse filter needs an Attribute and an Operator")) + } + return DynamoDBBrowseFilter( + attribute: attribute, + op: op.uppercased(), + value: Self.text(entry["Value"]) ?? "", + secondValue: Self.text(entry["SecondValue"]), + kind: entry["Kind"]?.stringValue, + caseSensitive: entry["CaseSensitive"]?.boolValue ?? true + ) + } + self.matchAll = json["Match"]?.stringValue?.lowercased() != "any" + self.columns = (json["Columns"]?.arrayValue ?? []).compactMap(\.stringValue) + } + + private static func text(_ json: DynamoDBJSON?) -> String? { + switch json { + case .string(let value)?: return value + case .number(let value)?: return value + case .bool(let value)?: return value ? "true" : "false" + default: return nil + } + } + + var json: DynamoDBJSON { + var object: [String: DynamoDBJSON] = ["TableName": .string(table)] + if !filters.isEmpty { + object["Filters"] = .array(filters.map { filter in + var entry: [String: DynamoDBJSON] = [ + "Attribute": .string(filter.attribute), + "Operator": .string(filter.op), + "Value": .string(filter.value) + ] + if let second = filter.secondValue { entry["SecondValue"] = .string(second) } + if let kind = filter.kind { entry["Kind"] = .string(kind) } + if !filter.caseSensitive { entry["CaseSensitive"] = .bool(false) } + return .object(entry) + }) + object["Match"] = .string(matchAll ? "All" : "Any") + } + if !columns.isEmpty { + object["Columns"] = .array(columns.map(DynamoDBJSON.string)) + } + return .object(object) + } +} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBCatalog.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBCatalog.swift new file mode 100644 index 0000000000..fece1ae344 --- /dev/null +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBCatalog.swift @@ -0,0 +1,168 @@ +import Foundation + +/// Where a page of a read resumes: the request of the plan, the key to start after, and for a +/// batch of GetItems, which has no key to start after, how many of its items to skip. +struct DynamoDBResumePoint: Sendable, Equatable { + let requestIndex: Int + let startKey: DynamoDBItem? + let skip: Int + + static let start = DynamoDBResumePoint(requestIndex: 0, startKey: nil, skip: 0) +} + +/// What the driver knows about a connection's tables, shared by every driver instance that +/// connection builds. +/// +/// The app runs one connection on several driver instances: the session's, pooled ones for +/// metadata reads, and an unconnected one that only builds statements. A cache on one instance is +/// invisible to the others, so a type seen by the Structure tab never reached the save, and a +/// table changed on a pooled instance stayed stale on the session's. +final class DynamoDBCatalog: @unchecked Sendable { + static let shared = DynamoDBCatalog() + + static let schemaLifetime: TimeInterval = 60 + static let maximumFingerprints = 64 + static let maximumResumePoints = 4_096 + + struct Scope: Hashable, Sendable { + let endpoint: String + let region: String + let identity: String + } + + private struct TableKey: Hashable { + let scope: Scope + let table: String + } + + private struct CachedSchema { + let schema: DynamoDBTableSchema + let fetchedAt: Date + } + + private struct ResumeKey: Hashable { + let scope: Scope + let table: String + let fingerprint: String + } + + private let lock = NSLock() + private var schemas: [TableKey: CachedSchema] = [:] + private var columnTypes: [TableKey: [String: DynamoDBAttributeType]] = [:] + private var resumePoints: [ResumeKey: [Int: DynamoDBResumePoint]] = [:] + private var resumeOrder: [ResumeKey] = [] + + func schema(for table: String, in scope: Scope, now: Date = Date()) -> DynamoDBTableSchema? { + lock.withLock { + guard let cached = schemas[TableKey(scope: scope, table: table)], + now.timeIntervalSince(cached.fetchedAt) < Self.schemaLifetime + else { return nil } + return cached.schema + } + } + + func cachedSchemas(in scope: Scope) -> [DynamoDBTableSchema] { + lock.withLock { + schemas.filter { $0.key.scope == scope }.map(\.value.schema).sorted { $0.name < $1.name } + } + } + + func store(_ schema: DynamoDBTableSchema, in scope: Scope, now: Date = Date()) { + lock.withLock { + schemas[TableKey(scope: scope, table: schema.name)] = CachedSchema(schema: schema, fetchedAt: now) + } + } + + /// Forgets a table after DDL: its description, the types seen in its items, and every read + /// position of it. A table dropped and created again under the same name shares none of them. + func invalidate(table: String, in scope: Scope) { + lock.withLock { + schemas[TableKey(scope: scope, table: table)] = nil + columnTypes[TableKey(scope: scope, table: table)] = nil + } + forgetReadPositions { $0.scope == scope && $0.table == table } + } + + /// Forgets where each page of a table starts, after a write. A position is the key of the item + /// before it, so an item added or removed ahead of it moves every page after it by one. + func forgetReadPositions(table: String, in scope: Scope) { + forgetReadPositions { $0.scope == scope && $0.table == table } + } + + /// Forgets every read position in a scope, for a write that names no single table. + func forgetReadPositions(in scope: Scope) { + forgetReadPositions { $0.scope == scope } + } + + private func forgetReadPositions(where isStale: (ResumeKey) -> Bool) { + lock.withLock { + let stale = resumePoints.keys.filter(isStale) + for key in stale { + resumePoints[key] = nil + } + resumeOrder.removeAll { stale.contains($0) } + } + } + + func invalidateAll(in scope: Scope) { + lock.withLock { + schemas = schemas.filter { $0.key.scope != scope } + columnTypes = columnTypes.filter { $0.key.scope != scope } + resumePoints = resumePoints.filter { $0.key.scope != scope } + resumeOrder.removeAll { $0.scope == scope } + } + } + + func columnTypes(for table: String, in scope: Scope) -> [String: DynamoDBAttributeType] { + lock.withLock { columnTypes[TableKey(scope: scope, table: table)] ?? [:] } + } + + func mergeColumnTypes(_ types: [String: DynamoDBAttributeType], for table: String, in scope: Scope) { + guard !types.isEmpty else { return } + lock.withLock { + columnTypes[TableKey(scope: scope, table: table), default: [:]].merge(types) { _, new in new } + } + } + + // MARK: - Resume points + + func nearestResumePoint( + table: String, + fingerprint: String, + atOrBefore offset: Int, + in scope: Scope + ) -> (offset: Int, point: DynamoDBResumePoint)? { + lock.withLock { + guard let points = resumePoints[ResumeKey(scope: scope, table: table, fingerprint: fingerprint)] else { + return nil + } + guard let best = points.keys.filter({ $0 <= offset }).max(), let point = points[best] else { return nil } + return (best, point) + } + } + + func storeResumePoint( + _ point: DynamoDBResumePoint, + table: String, + fingerprint: String, + offset: Int, + in scope: Scope + ) { + let key = ResumeKey(scope: scope, table: table, fingerprint: fingerprint) + lock.withLock { + if resumePoints[key] == nil { + resumeOrder.append(key) + if resumeOrder.count > Self.maximumFingerprints { + let evicted = resumeOrder.removeFirst() + resumePoints[evicted] = nil + } + } + var points = resumePoints[key] ?? [:] + if points.count >= Self.maximumResumePoints, points[offset] == nil { + return + } + points[offset] = point + resumePoints[key] = points + } + } +} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBCellCodec.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBCellCodec.swift new file mode 100644 index 0000000000..8d6fc42495 --- /dev/null +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBCellCodec.swift @@ -0,0 +1,268 @@ +import Foundation +import TableProPluginKit + +/// Moves attribute values into grid cells and edited cells back into attribute values. +/// +/// A cell carries text, bytes or nothing, never a type, so an edit is decoded against a template: +/// the attribute's current value when the item has one, otherwise the column's type. That is what +/// keeps `02134` a String, a String Set a set and a nested Binary binary when a map is edited as +/// plain JSON. +enum DynamoDBCellCodec { + static func cell(for value: DynamoDBAttributeValue?) -> PluginCellValue { + guard let value else { return .null } + switch value { + case .string(let text): + return .text(text) + case .number(let text): + return .text(text) + case .bool(let flag): + return .text(flag ? "true" : "false") + case .null: + return .null + case .binary(let data): + return .bytes(data) + case .list, .map, .stringSet, .numberSet, .binarySet: + return .text(plainJSON(value).serialized()) + } + } + + /// The JSON a person reads and edits: no type envelopes, map keys and set members in a fixed + /// order so the same value always renders as the same text. + static func plainJSON(_ value: DynamoDBAttributeValue) -> DynamoDBJSON { + switch value { + case .string(let text): + return .string(text) + case .number(let text): + return .number(jsonNumberText(text)) + case .binary(let data): + return .string(data.base64EncodedString()) + case .bool(let flag): + return .bool(flag) + case .null: + return .null + case .list(let items): + return .array(items.map(plainJSON)) + case .map(let entries): + return .object(entries.mapValues(plainJSON)) + case .stringSet(let members): + return .array(members.sorted().map(DynamoDBJSON.string)) + case .numberSet(let members): + return .array( + members.sorted { DynamoDBNumber.compare($0, $1) == .orderedAscending } + .map { .number(jsonNumberText($0)) } + ) + case .binarySet(let members): + return .array(members.map { $0.base64EncodedString() }.sorted().map(DynamoDBJSON.string)) + } + } + + /// Decodes an edited cell. Nil means the attribute is removed. + static func decode( + _ cell: PluginCellValue, + template: DynamoDBAttributeValue?, + columnType: DynamoDBAttributeType?, + attribute: String + ) throws -> DynamoDBAttributeValue? { + switch cell { + case .null: + return nil + case .bytes(let data): + return .binary(data) + case .text(let text): + let targetType = template?.type ?? columnType + guard let targetType else { return inferred(from: text) } + return try decode(text: text, as: targetType, template: template, attribute: attribute) + } + } + + static func decode( + text: String, + as type: DynamoDBAttributeType, + template: DynamoDBAttributeValue?, + attribute: String + ) throws -> DynamoDBAttributeValue { + switch type { + case .string: + return .string(text) + case .number: + let trimmed = text.trimmingCharacters(in: .whitespaces) + guard DynamoDBNumber.isValid(trimmed) else { + throw DynamoDBError.invalidValue( + attribute: attribute, + reason: String(format: String(localized: "\"%@\" is not a DynamoDB number"), text) + ) + } + return .number(trimmed) + case .boolean: + switch text.trimmingCharacters(in: .whitespaces).lowercased() { + case "true", "1": return .bool(true) + case "false", "0": return .bool(false) + default: + throw DynamoDBError.invalidValue( + attribute: attribute, + reason: String(format: String(localized: "\"%@\" is not true or false"), text) + ) + } + case .null: + let trimmed = text.trimmingCharacters(in: .whitespaces) + if trimmed.isEmpty || trimmed.caseInsensitiveCompare("null") == .orderedSame { + return .null + } + return inferred(from: text) + case .binary: + guard let data = Data(base64Encoded: text.trimmingCharacters(in: .whitespacesAndNewlines)) else { + throw DynamoDBError.invalidValue( + attribute: attribute, reason: String(localized: "A Binary value must be base64") + ) + } + return .binary(data) + case .list, .map, .stringSet, .numberSet, .binarySet: + let json: DynamoDBJSON + do { + json = try DynamoDBJSON.parse(text) + } catch { + throw DynamoDBError.invalidValue( + attribute: attribute, + reason: String(format: String(localized: "A %@ value must be JSON: %@"), + type.displayName, error.localizedDescription) + ) + } + return try value(fromPlainJSON: json, template: template ?? emptyTemplate(for: type), attribute: attribute) + } + } + + static func value( + fromPlainJSON json: DynamoDBJSON, + template: DynamoDBAttributeValue?, + attribute: String + ) throws -> DynamoDBAttributeValue { + switch json { + case .object(let entries): + var templates: [String: DynamoDBAttributeValue] = [:] + if case .map(let existing) = template { templates = existing } + var converted: [String: DynamoDBAttributeValue] = [:] + for (key, element) in entries { + converted[key] = try value(fromPlainJSON: element, template: templates[key], attribute: attribute) + } + return .map(converted) + case .array(let elements): + return try arrayValue(elements, template: template, attribute: attribute) + case .string(let text): + if case .binary = template, let data = Data(base64Encoded: text) { + return .binary(data) + } + return .string(text) + case .number(let text): + guard DynamoDBNumber.isValid(text) else { + throw DynamoDBError.invalidValue( + attribute: attribute, + reason: String(format: String(localized: "\"%@\" is not a DynamoDB number"), text) + ) + } + return .number(text) + case .bool(let flag): + return .bool(flag) + case .null: + return .null + } + } + + private static func arrayValue( + _ elements: [DynamoDBJSON], + template: DynamoDBAttributeValue?, + attribute: String + ) throws -> DynamoDBAttributeValue { + switch template { + case .stringSet: + let members = elements.compactMap(\.stringValue) + if members.count == elements.count { + return .stringSet(try validatedSet(members, attribute: attribute) { Array($0.utf8) == Array($1.utf8) }) + } + case .numberSet: + let members = elements.compactMap { $0.numberText ?? $0.stringValue } + if members.count == elements.count, members.allSatisfy(DynamoDBNumber.isValid) { + return .numberSet(try validatedSet(members, attribute: attribute, sameMember: DynamoDBNumber.areEqual)) + } + case .binarySet: + let members = elements.compactMap { $0.stringValue.flatMap { Data(base64Encoded: $0) } } + if members.count == elements.count { + return .binarySet(try validatedSet(members, attribute: attribute, sameMember: ==)) + } + default: + break + } + var itemTemplates: [DynamoDBAttributeValue] = [] + if case .list(let existing) = template { itemTemplates = existing } + return .list(try elements.enumerated().map { index, element in + let elementTemplate = index < itemTemplates.count ? itemTemplates[index] : nil + return try value(fromPlainJSON: element, template: elementTemplate, attribute: attribute) + }) + } + + private static func validatedSet( + _ members: [Member], + attribute: String, + sameMember: (Member, Member) -> Bool + ) throws -> [Member] { + guard !members.isEmpty else { + throw DynamoDBError.invalidValue( + attribute: attribute, reason: String(localized: "A set can't be empty. Clear the cell to remove it.") + ) + } + for (index, member) in members.enumerated() where members[.. DynamoDBAttributeValue { + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard let first = trimmed.first, first == "{" || first == "[", + let json = try? DynamoDBJSON.parse(trimmed), + let converted = try? value(fromPlainJSON: json, template: nil, attribute: "") + else { return .string(text) } + return converted + } + + private static func emptyTemplate(for type: DynamoDBAttributeType) -> DynamoDBAttributeValue? { + switch type { + case .stringSet: return .stringSet([]) + case .numberSet: return .numberSet([]) + case .binarySet: return .binarySet([]) + case .map: return .map([:]) + case .list: return .list([]) + default: return nil + } + } + + /// DynamoDB accepts `+5`, `.5`, `5.` and `007`, none of which is a JSON number. + static func jsonNumberText(_ text: String) -> String { + var body = text.trimmingCharacters(in: .whitespaces) + var sign = "" + if let first = body.first, first == "+" || first == "-" { + sign = first == "-" ? "-" : "" + body.removeFirst() + } + var mantissa = body + var exponent = "" + if let marker = body.firstIndex(where: { $0 == "e" || $0 == "E" }) { + mantissa = String(body[.. (Data, HTTPURLResponse) + func invalidate() +} + +/// URLSession, with every redirect refused. +/// +/// URLSession follows a 307 to another host by default and sends the body and +/// `X-Amz-Security-Token` along, which would carry a request past the rule that plain HTTP goes +/// only to this Mac. DynamoDB never redirects, so a redirect is an error. +final class DynamoDBURLSessionTransport: NSObject, DynamoDBTransport, URLSessionTaskDelegate, @unchecked Sendable { + private lazy var session: URLSession = { + let configuration = URLSessionConfiguration.ephemeral + configuration.timeoutIntervalForRequest = HttpQueryTimeout.sessionBootstrapRequestTimeout + configuration.timeoutIntervalForResource = HttpQueryTimeout.sessionResourceTimeout + configuration.urlCache = nil + configuration.httpCookieStorage = nil + return URLSession(configuration: configuration, delegate: self, delegateQueue: nil) + }() + + func send(_ request: URLRequest) async throws -> (Data, HTTPURLResponse) { + let (data, response) = try await session.data(for: request) + guard let http = response as? HTTPURLResponse else { + throw DynamoDBError.invalidResponse(String(localized: "The endpoint did not answer over HTTP")) + } + return (data, http) + } + + func invalidate() { + session.invalidateAndCancel() + } + + func urlSession( + _ session: URLSession, + task: URLSessionTask, + willPerformHTTPRedirection response: HTTPURLResponse, + newRequest request: URLRequest, + completionHandler: @escaping (URLRequest?) -> Void + ) { + completionHandler(nil) + } +} + +/// Resolves and caches the AWS credentials a connection signs with. +final class DynamoDBCredentialsProvider: @unchecked Sendable { + static let localAccessKey = "local" + + private let fields: [String: String] + private let method: DynamoDBAuthMethod + private let lock = NSLock() + private var cached: AWSCredentials? + + init(fields: [String: String], username: String, password: String) { + var resolved = fields + if (resolved["awsAccessKeyId"] ?? "").isEmpty, !username.isEmpty { + resolved["awsAccessKeyId"] = username + } + if (resolved["awsSecretAccessKey"] ?? "").isEmpty, !password.isEmpty { + resolved["awsSecretAccessKey"] = password + } + self.fields = resolved + self.method = DynamoDBAuthMethod(fieldValue: fields["awsAuthMethod"]) + } + + var identity: String { + switch method { + case .local: return "local" + case .profile, .singleSignOn: return "profile:" + (fields["awsProfileName"] ?? "default") + case .accessKey: return "key:" + (fields["awsAccessKeyId"] ?? "") + } + } + + /// `AWSSSOError` and `AWSAuthError` leave here unwrapped: the app offers its SSO sign-in + /// prompt only when it can see an `AWSSSOError`. + func credentials(forceRefresh: Bool = false) async throws -> AWSCredentials { + if method == .local { + return AWSCredentials(accessKeyId: Self.localAccessKey, secretAccessKey: Self.localAccessKey, sessionToken: nil) + } + if !forceRefresh, let current = lock.withLock({ cached }), !current.isExpired() { + return current + } + let fresh = try await AWSCredentialResolver.resolve(source: method.credentialSource, fields: fields) + lock.withLock { cached = fresh } + return fresh + } +} + +/// Sends DynamoDB requests: signs them, classifies failures, and retries what may be retried. +final class DynamoDBClient: @unchecked Sendable { + private static let logger = Logger(subsystem: "com.TablePro", category: "DynamoDBClient") + + let endpoint: DynamoDBEndpoint + private let transport: DynamoDBTransport + private let credentials: DynamoDBCredentialsProvider + private let retryPolicy: DynamoDBRetryPolicy + private let now: @Sendable () -> Date + private let sleep: @Sendable (TimeInterval) async throws -> Void + private let lock = NSLock() + private var clockOffset: TimeInterval = 0 + private let timeout = HttpQueryTimeoutBox() + + init( + endpoint: DynamoDBEndpoint, + credentials: DynamoDBCredentialsProvider, + transport: DynamoDBTransport = DynamoDBURLSessionTransport(), + retryPolicy: DynamoDBRetryPolicy = DynamoDBRetryPolicy(), + now: @escaping @Sendable () -> Date = { Date() }, + sleep: @escaping @Sendable (TimeInterval) async throws -> Void = { seconds in + try await Task.sleep(nanoseconds: UInt64(max(seconds, 0) * 1_000_000_000)) + } + ) { + self.endpoint = endpoint + self.credentials = credentials + self.transport = transport + self.retryPolicy = retryPolicy + self.now = now + self.sleep = sleep + } + + func setQueryTimeout(_ seconds: Int) { + timeout.set(serverTimeoutSeconds: seconds) + } + + var queryTimeoutSeconds: Int { + timeout.current.serverTimeoutSeconds + } + + func invalidate() { + transport.invalidate() + } + + func send(_ operation: DynamoDBOperation, _ body: [String: DynamoDBJSON]) async throws -> DynamoDBJSON { + try await send(operation, .object(body)) + } + + /// The wait before resending the part of a batch DynamoDB left unprocessed, which it does when + /// the table is throttled, so it backs off like a throttled request. + func backOff(afterAttempt attempt: Int) async throws { + do { + try await sleep(retryPolicy.delay(base: DynamoDBRetryPolicy.throttlingBase, attempt: attempt)) + } catch { + throw DynamoDBError.cancelled + } + } + + func send(_ operation: DynamoDBOperation, _ body: DynamoDBJSON) async throws -> DynamoDBJSON { + var attempt = 0 + var refreshedCredentials = false + var correctedClock = false + var refreshBeforeNextAttempt = false + while true { + attempt += 1 + try checkCancellation() + do { + let forceRefresh = refreshBeforeNextAttempt + refreshBeforeNextAttempt = false + return try await sendOnce(operation, body: body, forceRefresh: forceRefresh) + } catch let error as DynamoDBError { + let decision = retryPolicy.decision( + for: error, + attempt: attempt, + operation: operation, + body: body, + alreadyRefreshedCredentials: refreshedCredentials, + alreadyCorrectedClock: correctedClock + ) + switch decision { + case .retry(let delay): + Self.logger.info("\(operation.rawValue, privacy: .public) retry \(attempt) after \(delay)s") + do { + try await sleep(delay) + } catch { + throw DynamoDBError.cancelled + } + case .refreshCredentialsAndRetry: + refreshedCredentials = true + refreshBeforeNextAttempt = true + case .correctClockAndRetry: + correctedClock = true + case .fail: + throw error + } + } + } + } + + private func sendOnce(_ operation: DynamoDBOperation, body: DynamoDBJSON, forceRefresh: Bool) async throws -> DynamoDBJSON { + let signingCredentials = try await credentials.credentials(forceRefresh: forceRefresh) + let payload = body.serializedData + var request = URLRequest(url: endpoint.url) + request.httpMethod = "POST" + request.httpBody = payload + request.timeoutInterval = timeout.requestTimeoutInterval + request.setValue("application/x-amz-json-1.0", forHTTPHeaderField: "Content-Type") + request.setValue(operation.target, forHTTPHeaderField: "X-Amz-Target") + let signingDate = now().addingTimeInterval(lock.withLock { clockOffset }) + DynamoDBSigner.sign( + &request, body: payload, credentials: signingCredentials, + region: endpoint.signingRegion, date: signingDate + ) + + let data: Data + let response: HTTPURLResponse + do { + (data, response) = try await transport.send(request) + } catch is CancellationError { + throw DynamoDBError.cancelled + } catch let error as URLError where error.code == .cancelled { + throw DynamoDBError.cancelled + } catch let error as DynamoDBError { + throw error + } catch { + throw DynamoDBError.transport(error.localizedDescription) + } + + switch response.statusCode { + case 200: + guard !data.isEmpty else { return .object([:]) } + do { + return try DynamoDBJSON.parse(data) + } catch { + Self.logger.error("\(operation.rawValue, privacy: .public) response did not parse, \(data.count) bytes") + throw DynamoDBError.invalidResponse(error.localizedDescription) + } + case 300..<400: + throw DynamoDBError.configuration(String( + localized: "The endpoint answered with a redirect, which DynamoDB never sends. Check the Custom Endpoint." + )) + default: + let serviceError = DynamoDBServiceError.parse(body: data, httpStatus: response.statusCode) + if serviceError.category == .clockSkew { + adoptServerClock(from: response) + } + Self.logger.info( + "\(operation.rawValue, privacy: .public) failed \(response.statusCode) \(serviceError.code, privacy: .public)" + ) + throw DynamoDBError.service(serviceError) + } + } + + private func adoptServerClock(from response: HTTPURLResponse) { + guard let header = response.value(forHTTPHeaderField: "Date"), + let serverDate = Self.httpDateFormatter.date(from: header) + else { return } + let offset = serverDate.timeIntervalSince(now()) + lock.withLock { clockOffset = offset } + } + + private func checkCancellation() throws { + if Task.isCancelled { throw DynamoDBError.cancelled } + } + + private static let httpDateFormatter: DateFormatter = { + let formatter = DateFormatter() + formatter.locale = Locale(identifier: "en_US_POSIX") + formatter.timeZone = TimeZone(identifier: "GMT") + formatter.dateFormat = "EEE, dd MMM yyyy HH:mm:ss zzz" + return formatter + }() +} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBConnection.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBConnection.swift deleted file mode 100644 index b29ef7803f..0000000000 --- a/Plugins/DynamoDBDriverPlugin/DynamoDBConnection.swift +++ /dev/null @@ -1,655 +0,0 @@ -// -// DynamoDBConnection.swift -// DynamoDBDriverPlugin -// -// AWS DynamoDB HTTP client with Signature V4 authentication. -// - -import CommonCrypto -import Foundation -import os -import TableProPluginKit - -// MARK: - DynamoDB Attribute Value - -indirect enum DynamoDBAttributeValue: Sendable, Equatable { - case string(String) - case number(String) - case binary(Data) - case bool(Bool) - case null - case list([DynamoDBAttributeValue]) - case map([String: DynamoDBAttributeValue]) - case stringSet([String]) - case numberSet([String]) - case binarySet([Data]) -} - -extension DynamoDBAttributeValue: Codable { - init(from decoder: Decoder) throws { - let container = try decoder.container(keyedBy: DynamoDBTypeCodingKey.self) - - if let value = try container.decodeIfPresent(String.self, forKey: .s) { - self = .string(value) - } else if let value = try container.decodeIfPresent(String.self, forKey: .n) { - self = .number(value) - } else if let value = try container.decodeIfPresent(String.self, forKey: .b) { - guard let data = Data(base64Encoded: value) else { - throw DecodingError.dataCorruptedError(forKey: .b, in: container, debugDescription: "Invalid base64 string") - } - self = .binary(data) - } else if let value = try container.decodeIfPresent(Bool.self, forKey: .bool) { - self = .bool(value) - } else if let value = try container.decodeIfPresent(Bool.self, forKey: .null), value { - self = .null - } else if let items = try container.decodeIfPresent([DynamoDBAttributeValue].self, forKey: .l) { - self = .list(items) - } else if let map = try container.decodeIfPresent([String: DynamoDBAttributeValue].self, forKey: .m) { - self = .map(map) - } else if let values = try container.decodeIfPresent([String].self, forKey: .ss) { - self = .stringSet(values) - } else if let values = try container.decodeIfPresent([String].self, forKey: .ns) { - self = .numberSet(values) - } else if let values = try container.decodeIfPresent([String].self, forKey: .bs) { - let decoded = try values.map { str -> Data in - guard let data = Data(base64Encoded: str) else { - throw DecodingError.dataCorruptedError( - forKey: .bs, in: container, - debugDescription: "Invalid base64 string in binary set" - ) - } - return data - } - self = .binarySet(decoded) - } else { - throw DecodingError.dataCorrupted( - DecodingError.Context( - codingPath: decoder.codingPath, - debugDescription: "Unknown DynamoDB attribute type" - ) - ) - } - } - - func encode(to encoder: Encoder) throws { - var container = encoder.container(keyedBy: DynamoDBTypeCodingKey.self) - - switch self { - case .string(let value): - try container.encode(value, forKey: .s) - case .number(let value): - try container.encode(value, forKey: .n) - case .binary(let value): - try container.encode(value.base64EncodedString(), forKey: .b) - case .bool(let value): - try container.encode(value, forKey: .bool) - case .null: - try container.encode(true, forKey: .null) - case .list(let items): - try container.encode(items, forKey: .l) - case .map(let map): - try container.encode(map, forKey: .m) - case .stringSet(let values): - try container.encode(values, forKey: .ss) - case .numberSet(let values): - try container.encode(values, forKey: .ns) - case .binarySet(let values): - try container.encode(values.map { $0.base64EncodedString() }, forKey: .bs) - } - } -} - -private enum DynamoDBTypeCodingKey: String, CodingKey { - case s = "S" - case n = "N" - case b = "B" - case bool = "BOOL" - case null = "NULL" - case l = "L" - case m = "M" - case ss = "SS" - case ns = "NS" - case bs = "BS" -} - -// MARK: - DynamoDB Error - -internal enum DynamoDBError: Error, LocalizedError { - case notConnected - case connectionFailed(String) - case serverError(String) - case authFailed(String) - case requestCancelled - case invalidResponse(String) - - var errorDescription: String? { - switch self { - case .notConnected: - return String(localized: "Not connected to DynamoDB") - case .connectionFailed(let detail): - return String(format: String(localized: "Connection failed: %@"), detail) - case .serverError(let detail): - return String(format: String(localized: "DynamoDB error: %@"), detail) - case .authFailed(let detail): - return String(format: String(localized: "Authentication failed: %@"), detail) - case .requestCancelled: - return String(localized: "Request was cancelled") - case .invalidResponse(let detail): - return String(format: String(localized: "Invalid response: %@"), detail) - } - } -} - -// MARK: - Response Types - -internal struct ListTablesResponse: Decodable { - let TableNames: [String]? - let LastEvaluatedTableName: String? -} - -internal struct DescribeTableResponse: Decodable { - let Table: TableDescription -} - -/// DeleteTable answers with the table's description as it enters DELETING. Nothing here reads it: -/// the sidebar refreshes from ListTables, and the useful part of the reply is that it succeeded. -internal struct DeleteTableResponse: Decodable {} - -internal struct TableDescription: Decodable { - let TableName: String - let KeySchema: [KeySchemaElement]? - let AttributeDefinitions: [AttributeDefinition]? - let GlobalSecondaryIndexes: [GlobalSecondaryIndexDescription]? - let LocalSecondaryIndexes: [LocalSecondaryIndexDescription]? - let ProvisionedThroughput: ProvisionedThroughputDescription? - let BillingModeSummary: BillingModeSummary? - let ItemCount: Int64? - let TableSizeBytes: Int64? - let TableStatus: String? - let TableArn: String? - let CreationDateTime: Double? -} - -internal struct KeySchemaElement: Decodable { - let AttributeName: String - let KeyType: String -} - -internal struct AttributeDefinition: Decodable { - let AttributeName: String - let AttributeType: String -} - -internal struct GlobalSecondaryIndexDescription: Decodable { - let IndexName: String - let KeySchema: [KeySchemaElement]? - let Projection: Projection? - let IndexStatus: String? - let ProvisionedThroughput: ProvisionedThroughputDescription? - let ItemCount: Int64? - let IndexSizeBytes: Int64? -} - -internal struct LocalSecondaryIndexDescription: Decodable { - let IndexName: String - let KeySchema: [KeySchemaElement]? - let Projection: Projection? - let ItemCount: Int64? - let IndexSizeBytes: Int64? -} - -internal struct ProvisionedThroughputDescription: Decodable { - let ReadCapacityUnits: Int64? - let WriteCapacityUnits: Int64? -} - -internal struct BillingModeSummary: Decodable { - let BillingMode: String? -} - -internal struct Projection: Decodable { - let ProjectionType: String? - let NonKeyAttributes: [String]? -} - -internal struct ScanResponse: Decodable { - let Items: [[String: DynamoDBAttributeValue]]? - let Count: Int? - let ScannedCount: Int? - let LastEvaluatedKey: [String: DynamoDBAttributeValue]? -} - -internal struct QueryResponse: Decodable { - let Items: [[String: DynamoDBAttributeValue]]? - let Count: Int? - let ScannedCount: Int? - let LastEvaluatedKey: [String: DynamoDBAttributeValue]? -} - -internal struct ExecuteStatementResponse: Decodable { - let Items: [[String: DynamoDBAttributeValue]]? - let NextToken: String? - let LastEvaluatedKey: [String: DynamoDBAttributeValue]? -} - -private struct DynamoDBErrorResponse: Decodable { - let __type: String? - let message: String? - let Message: String? - - var errorMessage: String { - message ?? Message ?? __type ?? "Unknown error" - } -} - -// MARK: - DynamoDB Connection - -internal final class DynamoDBConnection: @unchecked Sendable { - private let config: DriverConnectionConfig - private let lock = NSLock() - private var _session: URLSession? - private var _credentials: AWSCredentials? - private var _currentTask: URLSessionDataTask? - private let _queryTimeout = HttpQueryTimeoutBox() - private let region: String - private let endpointUrl: String - private static let logger = Logger(subsystem: "com.TablePro", category: "DynamoDBConnection") - private static let service = "dynamodb" - - var session: URLSession? { - lock.withLock { _session } - } - - func setQueryTimeout(_ seconds: Int) { - _queryTimeout.set(serverTimeoutSeconds: seconds) - } - - init(config: DriverConnectionConfig) { - self.config = config - self.region = config.additionalFields["awsRegion"] ?? "us-east-1" - - if let customEndpoint = config.additionalFields["awsEndpointUrl"], !customEndpoint.isEmpty { - if customEndpoint.lowercased().hasPrefix("http://") { - let loopbackHosts: Set = ["localhost", "127.0.0.1", "::1"] - let isLoopback = URL(string: customEndpoint).flatMap(\.host).map { - loopbackHosts.contains($0.lowercased()) - } ?? false - if isLoopback { - self.endpointUrl = customEndpoint - } else { - let upgraded = "https://" + customEndpoint.dropFirst("http://".count) - Self.logger.warning("Insecure endpoint for non-loopback host, upgrading to HTTPS") - self.endpointUrl = upgraded - } - } else { - self.endpointUrl = customEndpoint - } - } else { - self.endpointUrl = "https://dynamodb.\(region).amazonaws.com" - } - } - - func connect() async throws { - let credentials = try await resolveCredentials() - let sessionConfig = URLSessionConfiguration.default - sessionConfig.timeoutIntervalForRequest = HttpQueryTimeout.sessionBootstrapRequestTimeout - sessionConfig.timeoutIntervalForResource = HttpQueryTimeout.sessionResourceTimeout - let urlSession = URLSession(configuration: sessionConfig) - - lock.withLock { - _credentials = credentials - _session = urlSession - } - - // Verify connectivity by listing tables with limit 1 - _ = try await listTables(limit: 1) - } - - func disconnect() { - lock.withLock { - _currentTask?.cancel() - _currentTask = nil - // Don't invalidate the session — in-flight health monitor pings may still - // hold a reference. Just nil it out; URLSession cleans up on dealloc. - _session = nil - _credentials = nil - } - } - - func ping() async throws { - _ = try await listTables(limit: 1) - } - - func cancelCurrentRequest() { - lock.withLock { - _currentTask?.cancel() - _currentTask = nil - } - } - - // MARK: - DynamoDB API Operations - - func listTables(limit: Int = 100, exclusiveStartTableName: String? = nil) async throws -> ListTablesResponse { - var body: [String: Any] = ["Limit": limit] - if let startName = exclusiveStartTableName { - body["ExclusiveStartTableName"] = startName - } - return try await request(target: "DynamoDB_20120810.ListTables", body: body) - } - - func describeTable(tableName: String) async throws -> DescribeTableResponse { - let body: [String: Any] = ["TableName": tableName] - return try await request(target: "DynamoDB_20120810.DescribeTable", body: body) - } - - /// DeleteTable answers as soon as the table enters DELETING, not once it is gone, so a listing - /// taken straight afterwards can still show it. - func deleteTable(tableName: String) async throws -> DeleteTableResponse { - let body: [String: Any] = ["TableName": tableName] - return try await request(target: "DynamoDB_20120810.DeleteTable", body: body) - } - - func scan( - tableName: String, - limit: Int? = nil, - exclusiveStartKey: [String: DynamoDBAttributeValue]? = nil, - select: String? = nil - ) async throws -> ScanResponse { - var body: [String: Any] = ["TableName": tableName] - if let limit = limit { - body["Limit"] = limit - } - if let startKey = exclusiveStartKey { - body["ExclusiveStartKey"] = try encodedAttributeMap(startKey) - } - if let select = select { - body["Select"] = select - } - return try await request(target: "DynamoDB_20120810.Scan", body: body) - } - - func query( - tableName: String, - keyConditionExpression: String, - expressionAttributeValues: [String: DynamoDBAttributeValue], - limit: Int? = nil, - exclusiveStartKey: [String: DynamoDBAttributeValue]? = nil, - scanIndexForward: Bool = true, - select: String? = nil - ) async throws -> QueryResponse { - var body: [String: Any] = [ - "TableName": tableName, - "KeyConditionExpression": keyConditionExpression, - "ExpressionAttributeValues": try encodedAttributeMap(expressionAttributeValues) - ] - if let limit = limit { - body["Limit"] = limit - } - if let startKey = exclusiveStartKey { - body["ExclusiveStartKey"] = try encodedAttributeMap(startKey) - } - body["ScanIndexForward"] = scanIndexForward - if let select = select { - body["Select"] = select - } - return try await request(target: "DynamoDB_20120810.Query", body: body) - } - - func executeStatement( - statement: String, - parameters: [[String: Any]]? = nil, - limit: Int? = nil, - nextToken: String? = nil - ) async throws -> ExecuteStatementResponse { - var body: [String: Any] = ["Statement": statement] - if let parameters = parameters, !parameters.isEmpty { - body["Parameters"] = parameters - } - if let limit = limit { - body["Limit"] = limit - } - if let nextToken = nextToken { - body["NextToken"] = nextToken - } - return try await request(target: "DynamoDB_20120810.ExecuteStatement", body: body) - } - - // MARK: - Internal Request Handling - - private func request(target: String, body: [String: Any]) async throws -> T { - let urlSession: URLSession = try lock.withLock { - guard let s = _session else { throw DynamoDBError.notConnected } - return s - } - let credentials = try await validCredentials() - - let bodyData = try JSONSerialization.data(withJSONObject: body, options: [.sortedKeys]) - - guard let url = URL(string: endpointUrl) else { - throw DynamoDBError.connectionFailed("Invalid endpoint URL: \(endpointUrl)") - } - - var urlRequest = URLRequest(url: url) - urlRequest.httpMethod = "POST" - urlRequest.httpBody = bodyData - urlRequest.setValue("application/x-amz-json-1.0", forHTTPHeaderField: "Content-Type") - urlRequest.setValue(target, forHTTPHeaderField: "X-Amz-Target") - let hostHeader: String - if let host = url.host, let port = url.port { - hostHeader = "\(host):\(port)" - } else { - hostHeader = url.host ?? "" - } - urlRequest.setValue(hostHeader, forHTTPHeaderField: "Host") - - signRequest(&urlRequest, body: bodyData, credentials: credentials) - urlRequest.timeoutInterval = _queryTimeout.requestTimeoutInterval - - let (data, response) = try await withCheckedThrowingContinuation { - (continuation: CheckedContinuation<(Data, URLResponse), Error>) in - let task = urlSession.dataTask(with: urlRequest) { [weak self] data, response, error in - self?.lock.withLock { self?._currentTask = nil } - if let error { - if (error as? URLError)?.code == .cancelled { - continuation.resume(throwing: DynamoDBError.requestCancelled) - } else { - continuation.resume(throwing: DynamoDBError.connectionFailed(error.localizedDescription)) - } - return - } - guard let data, let response else { - continuation.resume(throwing: DynamoDBError.invalidResponse("Empty response")) - return - } - continuation.resume(returning: (data, response)) - } - self.lock.withLock { self._currentTask = task } - task.resume() - } - - guard let httpResponse = response as? HTTPURLResponse else { - throw DynamoDBError.invalidResponse("Not an HTTP response") - } - - if httpResponse.statusCode != 200 { - if let errorResponse = try? JSONDecoder().decode(DynamoDBErrorResponse.self, from: data) { - let errorType = errorResponse.__type ?? "UnknownError" - if errorType.contains("UnrecognizedClientException") || - errorType.contains("InvalidSignatureException") || - errorType.contains("AccessDeniedException") - { - throw DynamoDBError.authFailed(errorResponse.errorMessage) - } - throw DynamoDBError.serverError("[\(errorType)] \(errorResponse.errorMessage)") - } - throw DynamoDBError.serverError("HTTP \(httpResponse.statusCode): Response body redacted (length: \(data.count))") - } - - do { - let decoded = try JSONDecoder().decode(T.self, from: data) - return decoded - } catch { - Self.logger.error("Decode failed for \(target): responseLength=\(data.count), error=\(error.localizedDescription)") - throw DynamoDBError.invalidResponse("Failed to decode response: \(error.localizedDescription)") - } - } - - // MARK: - AWS Signature V4 - - private func signRequest(_ request: inout URLRequest, body: Data, credentials: AWSCredentials) { - let now = Date() - let dateFormatter = DateFormatter() - dateFormatter.locale = Locale(identifier: "en_US_POSIX") - dateFormatter.timeZone = TimeZone(identifier: "UTC") - dateFormatter.dateFormat = "yyyyMMdd'T'HHmmss'Z'" - let amzDate = dateFormatter.string(from: now) - - dateFormatter.dateFormat = "yyyyMMdd" - let dateStamp = dateFormatter.string(from: now) - - request.setValue(amzDate, forHTTPHeaderField: "X-Amz-Date") - - if let sessionToken = credentials.sessionToken, !sessionToken.isEmpty { - request.setValue(sessionToken, forHTTPHeaderField: "X-Amz-Security-Token") - } - - let host = request.value(forHTTPHeaderField: "Host") ?? request.url?.host ?? "" - let method = request.httpMethod ?? "POST" - let uri = request.url?.path ?? "/" - let canonicalUri = uri.isEmpty ? "/" : uri - let canonicalQuerystring = request.url?.query ?? "" - - // Signed headers: content-type, host, x-amz-date, and optionally x-amz-security-token - var signedHeaderNames = ["content-type", "host", "x-amz-date"] - var canonicalHeaders = "content-type:\(request.value(forHTTPHeaderField: "Content-Type") ?? "")\n" - canonicalHeaders += "host:\(host)\n" - canonicalHeaders += "x-amz-date:\(amzDate)\n" - - if let sessionToken = credentials.sessionToken, !sessionToken.isEmpty { - signedHeaderNames.append("x-amz-security-token") - canonicalHeaders += "x-amz-security-token:\(sessionToken)\n" - } - - let signedHeaders = signedHeaderNames.joined(separator: ";") - let payloadHash = sha256Hex(body) - - let canonicalRequest = [ - method, - canonicalUri, - canonicalQuerystring, - canonicalHeaders, - signedHeaders, - payloadHash - ].joined(separator: "\n") - - let credentialScope = "\(dateStamp)/\(region)/\(Self.service)/aws4_request" - let stringToSign = [ - "AWS4-HMAC-SHA256", - amzDate, - credentialScope, - sha256Hex(Data(canonicalRequest.utf8)) - ].joined(separator: "\n") - - let signingKey = deriveSigningKey( - secretKey: credentials.secretAccessKey, - dateStamp: dateStamp, - region: region, - service: Self.service - ) - let signature = hmacSHA256Hex(key: signingKey, data: Data(stringToSign.utf8)) - - let authorization = "AWS4-HMAC-SHA256 Credential=\(credentials.accessKeyId)/\(credentialScope), " + - "SignedHeaders=\(signedHeaders), Signature=\(signature)" - request.setValue(authorization, forHTTPHeaderField: "Authorization") - } - - private func deriveSigningKey(secretKey: String, dateStamp: String, region: String, service: String) -> Data { - let kDate = hmacSHA256(key: Data("AWS4\(secretKey)".utf8), data: Data(dateStamp.utf8)) - let kRegion = hmacSHA256(key: kDate, data: Data(region.utf8)) - let kService = hmacSHA256(key: kRegion, data: Data(service.utf8)) - let kSigning = hmacSHA256(key: kService, data: Data("aws4_request".utf8)) - return kSigning - } - - private func hmacSHA256(key: Data, data: Data) -> Data { - var result = [UInt8](repeating: 0, count: Int(CC_SHA256_DIGEST_LENGTH)) - key.withUnsafeBytes { keyPtr in - data.withUnsafeBytes { dataPtr in - CCHmac( - CCHmacAlgorithm(kCCHmacAlgSHA256), - keyPtr.baseAddress, key.count, - dataPtr.baseAddress, data.count, - &result - ) - } - } - return Data(result) - } - - private func hmacSHA256Hex(key: Data, data: Data) -> String { - hmacSHA256(key: key, data: data).map { String(format: "%02x", $0) }.joined() - } - - private func sha256Hex(_ data: Data) -> String { - var hash = [UInt8](repeating: 0, count: Int(CC_SHA256_DIGEST_LENGTH)) - data.withUnsafeBytes { ptr in - _ = CC_SHA256(ptr.baseAddress, CC_LONG(data.count), &hash) - } - return hash.map { String(format: "%02x", $0) }.joined() - } - - // MARK: - Credential Resolution - - private func resolveCredentials() async throws -> AWSCredentials { - let source = Self.credentialSource(forAuthMethod: config.additionalFields["awsAuthMethod"]) - var fields = config.additionalFields - if (fields["awsAccessKeyId"] ?? "").isEmpty, !config.username.isEmpty { - fields["awsAccessKeyId"] = config.username - } - if (fields["awsSecretAccessKey"] ?? "").isEmpty, !config.password.isEmpty { - fields["awsSecretAccessKey"] = config.password - } - - do { - return try await AWSCredentialResolver.resolve(source: source, fields: fields) - } catch let error as AWSAuthError { - throw DynamoDBError.authFailed(error.localizedDescription) - } catch let error as AWSSSOError { - throw DynamoDBError.authFailed(error.localizedDescription) - } - } - - static func credentialSource(forAuthMethod authMethod: String?) -> String { - switch authMethod { - case "profile": - return "profile" - case "sso": - return "sso" - default: - return "accessKey" - } - } - - private func validCredentials() async throws -> AWSCredentials { - if let current = lock.withLock({ _credentials }), !current.isExpired() { - return current - } - let refreshed = try await resolveCredentials() - lock.withLock { _credentials = refreshed } - return refreshed - } - - // MARK: - Helpers - - private func encodedAttributeMap(_ map: [String: DynamoDBAttributeValue]) throws -> [String: Any] { - let encoder = JSONEncoder() - var result: [String: Any] = [:] - for (key, value) in map { - let data = try encoder.encode(value) - if let json = try JSONSerialization.jsonObject(with: data) as? [String: Any] { - result[key] = json - } - } - return result - } -} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBEndpoint.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBEndpoint.swift new file mode 100644 index 0000000000..b32a24984c --- /dev/null +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBEndpoint.swift @@ -0,0 +1,115 @@ +import Foundation +import TableProPluginKit + +enum DynamoDBAuthMethod: String, Sendable { + case accessKey = "credentials" + case profile + case singleSignOn = "sso" + case local + + init(fieldValue: String?) { + self = fieldValue.flatMap(DynamoDBAuthMethod.init(rawValue:)) ?? .accessKey + } + + var credentialSource: String { + switch self { + case .profile: return "profile" + case .singleSignOn: return "sso" + case .accessKey, .local: return "accessKey" + } + } +} + +/// Where requests go and how they are signed. +struct DynamoDBEndpoint: Sendable, Equatable { + static let defaultRegion = "us-east-1" + static let localDefaultURL = "http://localhost:8000" + + let url: URL + let signingRegion: String + let isLocal: Bool + + /// Resolves the endpoint from the connection's fields. + /// + /// The region is the one the form names, else the profile's own `region`, else us-east-1; the + /// form used to save us-east-1 for every profile connection, which listed another region's + /// tables with no error. The host follows the region's partition, so `cn-north-1` reaches + /// `amazonaws.com.cn`. Plain HTTP is refused for any host but this Mac: a custom endpoint sees + /// the signed request, including a session token. + static func resolve( + fields: [String: String], + profileRegion: (String) -> String? = { AWSCredentialResolver.profileRegion(named: $0) } + ) throws -> DynamoDBEndpoint { + let method = DynamoDBAuthMethod(fieldValue: fields["awsAuthMethod"]) + let region = resolvedRegion(fields: fields, method: method, profileRegion: profileRegion) + guard isValidRegion(region) else { + throw DynamoDBError.configuration( + String(format: String(localized: "\"%@\" is not an AWS region"), region) + ) + } + let custom = fields["awsEndpointUrl"]?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + + guard !custom.isEmpty || method == .local else { + let host = AWSPartition.host(service: "dynamodb", region: region) + guard let url = URL(string: "https://\(host)/") else { + throw DynamoDBError.configuration( + String(format: String(localized: "\"%@\" is not an AWS region"), region) + ) + } + return DynamoDBEndpoint(url: url, signingRegion: region, isLocal: false) + } + + let text = custom.isEmpty ? localDefaultURL : custom + guard let url = URL(string: text), let scheme = url.scheme?.lowercased(), let host = url.host, !host.isEmpty else { + throw DynamoDBError.configuration( + String(format: String(localized: "\"%@\" is not a URL. Enter an endpoint such as http://localhost:8000."), text) + ) + } + guard scheme == "https" || scheme == "http" else { + throw DynamoDBError.configuration(String(localized: "The endpoint must start with https:// or http://")) + } + let isLoopback = isLoopbackHost(host) + guard scheme == "https" || isLoopback else { + throw DynamoDBError.configuration(String( + localized: "Plain HTTP is only allowed for an endpoint on this Mac (localhost). Use https:// for any other host." + )) + } + return DynamoDBEndpoint(url: url, signingRegion: region, isLocal: isLoopback) + } + + static func resolvedRegion( + fields: [String: String], + method: DynamoDBAuthMethod, + profileRegion: (String) -> String? + ) -> String { + let typed = AWSPartition.canonicalRegion(fields["awsRegion"] ?? "") + if !typed.isEmpty { return typed } + if method == .profile || method == .singleSignOn { + let profile = fields["awsProfileName"].flatMap { $0.isEmpty ? nil : $0 } ?? "default" + if let region = profileRegion(profile).map(AWSPartition.canonicalRegion), !region.isEmpty { + return region + } + } + return defaultRegion + } + + /// The region becomes part of the endpoint's host name and of the signature, so it may hold only + /// what a region name holds. A region from an imported connection or a profile could otherwise + /// carry `/`, `#` or `@` and move the signed request to another host. + static func isValidRegion(_ region: String) -> Bool { + guard !region.isEmpty, region.count <= 64, !region.hasPrefix("-"), !region.hasSuffix("-") else { return false } + return region.unicodeScalars.allSatisfy { scalar in + ("a"..."z").contains(scalar) || ("0"..."9").contains(scalar) || scalar == "-" + } + } + + /// Compares the literal host, with no DNS lookup, so a name cannot resolve its way onto the + /// loopback allowance. + static func isLoopbackHost(_ host: String) -> Bool { + let lowered = host.lowercased().trimmingCharacters(in: CharacterSet(charactersIn: "[]")) + if lowered == "localhost" || lowered == "::1" { return true } + let octets = lowered.split(separator: ".", omittingEmptySubsequences: false) + guard octets.count == 4, octets.allSatisfy({ UInt8($0) != nil }) else { return false } + return octets.first == "127" + } +} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBError.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBError.swift new file mode 100644 index 0000000000..7e8628fc7f --- /dev/null +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBError.swift @@ -0,0 +1,169 @@ +import Foundation + +struct DynamoDBCancellationReason: Sendable, Equatable { + let code: String + let message: String? +} + +enum DynamoDBError: Error, LocalizedError, Sendable, Equatable { + case notConnected + case configuration(String) + case service(DynamoDBServiceError) + case transport(String) + case cancelled + case timedOut(seconds: Int) + case invalidStatement(String) + case invalidValue(attribute: String, reason: String) + case invalidResponse(String) + case itemChanged(key: String) + case itemMissing(key: String) + case partialBatch(applied: Int, total: Int, failures: [String]) + + var errorDescription: String? { + switch self { + case .notConnected: + return String(localized: "Not connected to DynamoDB") + case .configuration(let message): + return message + case .service(let error): + return error.userMessage + case .transport(let detail): + return String(format: String(localized: "Connection failed: %@"), detail) + case .cancelled: + return String(localized: "Request was cancelled") + case .timedOut(let seconds): + return String(format: String(localized: "Stopped after %d seconds, the query timeout"), seconds) + case .invalidStatement(let message): + return message + case .invalidValue(let attribute, let reason): + guard !attribute.isEmpty else { return reason } + return String(format: String(localized: "%@: %@"), attribute, reason) + case .invalidResponse(let detail): + return String(format: String(localized: "Invalid response: %@"), detail) + case .itemChanged(let key): + return String(format: String( + localized: "The item %@ changed after it was loaded. Refresh the table and edit it again." + ), key) + case .itemMissing(let key): + return String(format: String(localized: "The item %@ no longer exists."), key) + case .partialBatch(let applied, let total, let failures): + let summary = String(format: String(localized: "%1$d of %2$d were applied."), applied, total) + return ([summary] + failures).joined(separator: "\n") + } + } +} + +/// An error DynamoDB answered with. `code` is the part of `__type` after `#`, so +/// `com.amazonaws.dynamodb.v20120810#ConditionalCheckFailedException` and +/// `com.amazon.coral.validate#ValidationException` both compare by their short name. +struct DynamoDBServiceError: Sendable, Equatable { + let code: String + let message: String + let httpStatus: Int + let cancellationReasons: [DynamoDBCancellationReason] + + init(code: String, message: String, httpStatus: Int, cancellationReasons: [DynamoDBCancellationReason] = []) { + self.code = code + self.message = message + self.httpStatus = httpStatus + self.cancellationReasons = cancellationReasons + } + + static func parse(body: Data, httpStatus: Int) -> DynamoDBServiceError { + guard let json = try? DynamoDBJSON.parse(body) else { + return DynamoDBServiceError( + code: "HTTP\(httpStatus)", + message: String(format: String(localized: "HTTP %d with a body of %d bytes"), httpStatus, body.count), + httpStatus: httpStatus + ) + } + let rawType = json["__type"]?.stringValue ?? "HTTP\(httpStatus)" + let code = rawType.split(separator: "#").last.map(String.init) ?? rawType + let message = json["message"]?.stringValue ?? json["Message"]?.stringValue ?? code + let reasons = (json["CancellationReasons"]?.arrayValue ?? []).map { reason in + DynamoDBCancellationReason( + code: reason["Code"]?.stringValue ?? "None", + message: reason["Message"]?.stringValue + ) + } + return DynamoDBServiceError(code: code, message: message, httpStatus: httpStatus, cancellationReasons: reasons) + } + + enum Category: Equatable { + case throttling + case transient + case expiredCredentials + case clockSkew + case authentication + case fatal + } + + var category: Category { + if Self.throttlingCodes.contains(code) { return .throttling } + if Self.transientCodes.contains(code) || httpStatus >= 500 { return .transient } + if code == "ExpiredTokenException" || code == "ExpiredToken" { return .expiredCredentials } + if Self.skewCodes.contains(code) || isSignatureExpiry { return .clockSkew } + if Self.authenticationCodes.contains(code) { return .authentication } + return .fatal + } + + var isConditionalCheckFailure: Bool { + code == "ConditionalCheckFailedException" + } + + private var isSignatureExpiry: Bool { + guard code == "InvalidSignatureException" else { return false } + let lowered = message.lowercased() + return lowered.contains("signature expired") || lowered.contains("signature not yet current") + } + + private static let throttlingCodes: Set = [ + "ProvisionedThroughputExceededException", + "ThrottlingException", + "RequestLimitExceeded", + "LimitExceededException", + "TransactionInProgressException", + "ReplicatedWriteConflictException", + "ItemCollectionSizeLimitExceededException" + ] + + private static let transientCodes: Set = [ + "InternalServerError", + "InternalFailure", + "ServiceUnavailable", + "ServiceUnavailableException" + ] + + private static let skewCodes: Set = [ + "RequestTimeTooSkewed", + "RequestExpired", + "RequestInTheFuture" + ] + + private static let authenticationCodes: Set = [ + "UnrecognizedClientException", + "InvalidSignatureException", + "MissingAuthenticationTokenException", + "MissingAuthenticationToken", + "IncompleteSignatureException", + "InvalidClientTokenId" + ] + + var userMessage: String { + var text: String + if category == .authentication { + text = String(format: String(localized: "Authentication failed: %@"), message) + } else { + text = String(format: String(localized: "DynamoDB error: [%1$@] %2$@"), code, message) + } + let failures = cancellationReasons.enumerated().compactMap { index, reason -> String? in + guard reason.code != "None" else { return nil } + let detail = reason.message.map { " \($0)" } ?? "" + return String(format: String(localized: "Action %1$d: %2$@%3$@"), index + 1, reason.code, detail) + } + if !failures.isEmpty { + text += "\n" + failures.joined(separator: "\n") + } + return text + } +} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBExpression.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBExpression.swift new file mode 100644 index 0000000000..c1c90286cd --- /dev/null +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBExpression.swift @@ -0,0 +1,146 @@ +import Foundation + +/// A document path such as `address.city` or `items[0].sku`. +struct DynamoDBAttributePath: Sendable, Equatable, Hashable { + enum Segment: Sendable, Equatable, Hashable { + case name(String) + case index(Int) + } + + let segments: [Segment] + + var root: String { + guard case .name(let name) = segments.first else { return "" } + return name + } + + var isTopLevel: Bool { segments.count == 1 } + + init(segments: [Segment]) { + self.segments = segments + } + + init(attribute: String) { + self.segments = [.name(attribute)] + } + + /// Reads `text` as a path, unless it is the name of an attribute the grid shows: an attribute + /// may be named `a.b`, and the filter bar sends that name, not a path. + static func parse(_ text: String, knownAttributes: Set) -> DynamoDBAttributePath { + if knownAttributes.contains(text) || !(text.contains(".") || text.contains("[")) { + return DynamoDBAttributePath(attribute: text) + } + var segments: [Segment] = [] + var current = "" + var index = text.startIndex + while index < text.endIndex { + let character = text[index] + switch character { + case ".": + if !current.isEmpty { segments.append(.name(current)) } + current = "" + index = text.index(after: index) + case "[": + if !current.isEmpty { segments.append(.name(current)) } + current = "" + guard let close = text[index...].firstIndex(of: "]"), + let position = Int(text[text.index(after: index)..= 0 + else { return DynamoDBAttributePath(attribute: text) } + segments.append(.index(position)) + index = text.index(after: close) + default: + current.append(character) + index = text.index(after: index) + } + } + if !current.isEmpty { segments.append(.name(current)) } + guard case .name = segments.first else { return DynamoDBAttributePath(attribute: text) } + return DynamoDBAttributePath(segments: segments) + } + + func value(in item: DynamoDBItem) -> DynamoDBAttributeValue? { + var current: DynamoDBAttributeValue? = item[root] + for segment in segments.dropFirst() { + switch (segment, current) { + case (.name(let name), .map(let entries)?): + current = entries[name] + case (.index(let position), .list(let items)?): + current = items.indices.contains(position) ? items[position] : nil + default: + return nil + } + } + return current + } +} + +/// Collects `#name` and `:value` placeholders while an expression is written. +/// +/// Every name goes through a placeholder. DynamoDB reserves 573 words (`name`, `status`, `data`, +/// `date` among them) and rejects `-` in a raw name, and it rejects a placeholder that is declared +/// but unused, so a placeholder exists only once something refers to it. +struct DynamoDBExpressionContext: Sendable, Equatable { + private(set) var names: [String: String] = [:] + private(set) var values: [String: DynamoDBAttributeValue] = [:] + private var placeholderByName: [String: String] = [:] + + mutating func name(_ attribute: String) -> String { + if let existing = placeholderByName[attribute] { return existing } + let placeholder = unique("#" + Self.sanitized(attribute), in: Set(names.keys)) + names[placeholder] = attribute + placeholderByName[attribute] = placeholder + return placeholder + } + + mutating func path(_ path: DynamoDBAttributePath) -> String { + var rendered = "" + for segment in path.segments { + switch segment { + case .name(let attribute): + rendered += (rendered.isEmpty ? "" : ".") + name(attribute) + case .index(let position): + rendered += "[\(position)]" + } + } + return rendered + } + + mutating func value(_ value: DynamoDBAttributeValue, hint: String) -> String { + if let existing = values.first(where: { $0.value == value && $0.key.hasPrefix(":" + Self.sanitized(hint)) }) { + return existing.key + } + let placeholder = unique(":" + Self.sanitized(hint), in: Set(values.keys)) + values[placeholder] = value + return placeholder + } + + /// Adds the placeholders to a request body, and only the ones in use. + func apply(to body: inout [String: DynamoDBJSON]) { + if !names.isEmpty { + body["ExpressionAttributeNames"] = .object(names.mapValues(DynamoDBJSON.string)) + } + if !values.isEmpty { + body["ExpressionAttributeValues"] = .object(values.mapValues(\.wireJSON)) + } + } + + private func unique(_ base: String, in taken: Set) -> String { + guard taken.contains(base) else { return base } + var counter = 2 + while taken.contains("\(base)\(counter)") { counter += 1 } + return "\(base)\(counter)" + } + + private static func sanitized(_ text: String) -> String { + let allowed = text.unicodeScalars.map { scalar -> Character in + let isAlphanumeric = (scalar.value < 128) && (CharacterSet.alphanumerics.contains(scalar)) + return isAlphanumeric ? Character(scalar) : "_" + } + var result = String(allowed.prefix(40)) + if result.isEmpty || result.first?.isNumber == true { + result = "a" + result + } + return result + } +} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBFilterTranslator.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBFilterTranslator.swift new file mode 100644 index 0000000000..258376592e --- /dev/null +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBFilterTranslator.swift @@ -0,0 +1,355 @@ +import Foundation + +/// A condition DynamoDB cannot evaluate, applied to the items it returns: `ENDS WITH`, a regular +/// expression, a match that ignores case, and a search across every attribute. +struct DynamoDBClientPredicate: Sendable, Equatable { + let path: DynamoDBAttributePath? + let op: String + let value: String + let secondValue: String? + let caseSensitive: Bool + + func matches(_ item: DynamoDBItem) -> Bool { + guard let path else { + return item.values.contains { matches(value: $0) } + } + return matches(value: path.value(in: item)) + } + + private func matches(value attribute: DynamoDBAttributeValue?) -> Bool { + switch op { + case "IS NULL": + return attribute == nil || attribute == .null + case "IS NOT NULL": + return attribute != nil && attribute != .null + default: + break + } + guard let attribute else { return false } + let text = DynamoDBCellCodec.displayText(for: attribute) + let subject = caseSensitive ? text : text.lowercased() + let needle = caseSensitive ? value : value.lowercased() + switch op { + case "=": return Self.equal(subject, needle, attribute) + case "!=", "<>": return !Self.equal(subject, needle, attribute) + case "CONTAINS": return contains(attribute) + case "NOT CONTAINS": return !contains(attribute) + case "STARTS WITH": return startsWith(attribute) + case "ENDS WITH": return subject.hasSuffix(needle) + case "IS EMPTY": return text.isEmpty + case "IS NOT EMPTY": return !text.isEmpty + case "IN": + return Self.listItems(value).contains { Self.equal(subject, caseSensitive ? $0 : $0.lowercased(), attribute) } + case "NOT IN": + return !Self.listItems(value).contains { Self.equal(subject, caseSensitive ? $0 : $0.lowercased(), attribute) } + case "REGEX": return regexMatches(text) + case ">", ">=", "<", "<=": return compares(text, op: op) + case "BETWEEN": + guard let bounds = DynamoDBFilterTranslator.bounds(value: value, secondValue: secondValue) else { + return false + } + return Self.order(text, bounds.lower) != .orderedAscending + && Self.order(text, bounds.upper) != .orderedDescending + default: + return false + } + } + + private func fold(_ text: String) -> String { + caseSensitive ? text : text.lowercased() + } + + /// DynamoDB's `contains`: text inside a String, or a member of a set or a list. A Number, a + /// Boolean, a Map and Binary never contain the filter's text, so the result is the same + /// whichever side runs the filter. + private func contains(_ attribute: DynamoDBAttributeValue) -> Bool { + let needle = fold(value) + switch attribute { + case .string(let text): + return fold(text).contains(needle) + case .stringSet(let members): + return members.contains { fold($0) == needle } + case .numberSet(let members): + return DynamoDBNumber.isValid(value) && members.contains { DynamoDBNumber.areEqual($0, value) } + case .list(let elements): + return elements.contains { element in + switch element { + case .string(let text): return fold(text) == needle + case .number(let text): return DynamoDBNumber.isValid(value) && DynamoDBNumber.areEqual(text, value) + default: return false + } + } + default: + return false + } + } + + /// DynamoDB's `begins_with`, which only a String answers for text. + private func startsWith(_ attribute: DynamoDBAttributeValue) -> Bool { + guard case .string(let text) = attribute else { return false } + return fold(text).hasPrefix(fold(value)) + } + + private static func equal(_ subject: String, _ needle: String, _ attribute: DynamoDBAttributeValue) -> Bool { + if case .number = attribute, DynamoDBNumber.isValid(subject), DynamoDBNumber.isValid(needle) { + return DynamoDBNumber.areEqual(subject, needle) + } + return subject == needle + } + + private func regexMatches(_ text: String) -> Bool { + let options: NSRegularExpression.Options = caseSensitive ? [] : [.caseInsensitive] + guard let regex = try? NSRegularExpression(pattern: value, options: options) else { return false } + return regex.firstMatch(in: text, range: NSRange(location: 0, length: (text as NSString).length)) != nil + } + + private func compares(_ text: String, op: String) -> Bool { + let order = Self.order(text, value) + switch op { + case ">": return order == .orderedDescending + case ">=": return order != .orderedAscending + case "<": return order == .orderedAscending + default: return order != .orderedDescending + } + } + + static func order(_ lhs: String, _ rhs: String) -> ComparisonResult { + if DynamoDBNumber.isValid(lhs), DynamoDBNumber.isValid(rhs) { + return DynamoDBNumber.compare(lhs, rhs) + } + return lhs < rhs ? .orderedAscending : (lhs == rhs ? .orderedSame : .orderedDescending) + } + + static func listItems(_ value: String) -> [String] { + value.split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty } + } +} + +/// Turns one grid filter into a FilterExpression term, a key condition, or a client predicate. +struct DynamoDBFilterTranslator { + enum Outcome: Equatable { + case server(String) + case client(DynamoDBClientPredicate) + /// The filter can match no item: a Number key compared with text that is not a number. + case never + } + + static let rawFilterColumn = "__RAW__" + static let anyAttributeColumn = "*" + + static let keyConditionOperators: Set = ["=", "<", "<=", ">", ">=", "BETWEEN", "STARTS WITH"] + + let schema: DynamoDBTableSchema + + static func unsupportedReason(for filter: DynamoDBBrowseFilter) -> String? { + guard filter.attribute == rawFilterColumn else { return nil } + return String(localized: "Raw filters aren't available for DynamoDB. Write the condition in PartiQL in the editor.") + } + + func translate( + _ filter: DynamoDBBrowseFilter, + path: DynamoDBAttributePath?, + context: inout DynamoDBExpressionContext + ) -> Outcome { + guard let path else { return .client(clientPredicate(filter, path: nil)) } + if needsClient(filter) { return .client(clientPredicate(filter, path: path)) } + + let candidates = { (text: String) in self.candidates(for: path, text: text, kind: filter.kind) } + let rendered = context.path(path) + + switch filter.op { + case "IS NULL": + let nullType = context.value(.string("NULL"), hint: "null") + return .server("(attribute_not_exists(\(rendered)) OR attribute_type(\(rendered), \(nullType)))") + case "IS NOT NULL": + let nullType = context.value(.string("NULL"), hint: "null") + return .server("(attribute_exists(\(rendered)) AND NOT attribute_type(\(rendered), \(nullType)))") + case "IS EMPTY": + let empty = context.value(.string(""), hint: "empty") + return .server("\(rendered) = \(empty)") + case "IS NOT EMPTY": + let empty = context.value(.string(""), hint: "empty") + return .server("(attribute_exists(\(rendered)) AND \(rendered) <> \(empty))") + case "=", ">", ">=", "<", "<=": + let values = candidates(filter.value).filter { filter.op == "=" || $0.type != .boolean } + guard !values.isEmpty else { return .never } + let terms = values.map { "\(rendered) \(filter.op) \(context.value($0, hint: path.root))" } + return .server(Self.any(terms)) + case "!=", "<>": + let values = candidates(filter.value) + let terms = values.map { "\(rendered) <> \(context.value($0, hint: path.root))" } + return .server(Self.all(["attribute_exists(\(rendered))"] + terms)) + case "CONTAINS", "NOT CONTAINS": + let values = containsCandidates(filter.value, path: path) + let terms = values.map { "contains(\(rendered), \(context.value($0, hint: path.root)))" } + let contained = Self.any(terms) + guard filter.op == "NOT CONTAINS" else { return .server(contained) } + return .server("(attribute_exists(\(rendered)) AND NOT \(contained))") + case "STARTS WITH": + let keyType = schema.keyType(of: path.root) + guard path.isTopLevel == false || keyType == nil || keyType == .string || keyType == .binary else { + return .never + } + let prefix = context.value(.string(filter.value), hint: path.root) + return .server("begins_with(\(rendered), \(prefix))") + case "ENDS WITH", "REGEX": + return .client(clientPredicate(filter, path: path)) + case "IN", "NOT IN": + let values = DynamoDBClientPredicate.listItems(filter.value).flatMap(candidates) + guard !values.isEmpty else { return filter.op == "IN" ? .never : .server("attribute_exists(\(rendered))") } + let placeholders = values.map { context.value($0, hint: path.root) } + let chunks = stride(from: 0, to: placeholders.count, by: 100).map { + "\(rendered) IN (\(placeholders[$0.. String? in + guard lower.type != .boolean else { return nil } + if lower.type == .number, DynamoDBNumber.compare(bounds.lower, bounds.upper) == .orderedDescending { + return nil + } + if lower.type == .string, bounds.lower > bounds.upper { return nil } + let lowerPlaceholder = context.value(lower, hint: path.root) + let upperPlaceholder = context.value(upper, hint: path.root) + return "\(rendered) BETWEEN \(lowerPlaceholder) AND \(upperPlaceholder)" + } + guard !terms.isEmpty else { return .never } + return .server(Self.any(terms)) + default: + return .client(clientPredicate(filter, path: path)) + } + } + + /// A key condition term for a key attribute of the access path, typed by the table's schema. + func keyCondition( + _ filter: DynamoDBBrowseFilter, + attribute: String, + context: inout DynamoDBExpressionContext + ) -> String? { + guard let type = schema.keyType(of: attribute), !needsClient(filter) else { return nil } + let name = context.name(attribute) + switch filter.op { + case "=", "<", "<=", ">", ">=": + guard let value = typed(filter.value, as: type) else { return nil } + return "\(name) \(filter.op) \(context.value(value, hint: attribute))" + case "BETWEEN": + guard let bounds = Self.bounds(value: filter.value, secondValue: filter.secondValue), + let lower = typed(bounds.lower, as: type), let upper = typed(bounds.upper, as: type), + !Self.isReversed(lower, upper) + else { return nil } + return "\(name) BETWEEN \(context.value(lower, hint: attribute)) AND \(context.value(upper, hint: attribute))" + case "STARTS WITH": + guard type == .string || type == .binary, let prefix = typed(filter.value, as: type) else { return nil } + return "begins_with(\(name), \(context.value(prefix, hint: attribute)))" + default: + return nil + } + } + + /// Whether the filter can be evaluated by DynamoDB at all. + func needsClient(_ filter: DynamoDBBrowseFilter) -> Bool { + if filter.attribute == Self.anyAttributeColumn { return true } + if filter.op == "ENDS WITH" || filter.op == "REGEX" { return true } + guard !filter.caseSensitive else { return false } + return ["=", "!=", "<>", "CONTAINS", "NOT CONTAINS", "STARTS WITH", "IN", "NOT IN"].contains(filter.op) + && filter.value.lowercased() != filter.value.uppercased() + } + + func clientPredicate(_ filter: DynamoDBBrowseFilter, path: DynamoDBAttributePath?) -> DynamoDBClientPredicate { + DynamoDBClientPredicate( + path: path, + op: filter.op, + value: filter.value, + secondValue: filter.secondValue, + caseSensitive: filter.caseSensitive + ) + } + + // MARK: - Typing + + /// Every typed value a filter's text could mean. A key attribute has one type; anything else + /// may hold a String in one item and a Number in the next, so text that is a valid number is + /// compared as both rather than guessed. + func candidates(for path: DynamoDBAttributePath, text: String, kind: String?) -> [DynamoDBAttributeValue] { + if path.isTopLevel, let keyType = schema.keyType(of: path.root) { + return typed(text, as: keyType).map { [$0] } ?? [] + } + var values: [DynamoDBAttributeValue] = [.string(text)] + let trimmed = text.trimmingCharacters(in: .whitespaces) + if DynamoDBNumber.isValid(trimmed), !trimmed.isEmpty { + values.append(.number(trimmed)) + } + switch trimmed.lowercased() { + case "true": values.append(.bool(true)) + case "false": values.append(.bool(false)) + default: break + } + if kind == "integer" || kind == "decimal" { + values.sort { $0.type == .number && $1.type != .number } + } + return values + } + + private func containsCandidates(_ text: String, path: DynamoDBAttributePath) -> [DynamoDBAttributeValue] { + var values: [DynamoDBAttributeValue] = [.string(text)] + let trimmed = text.trimmingCharacters(in: .whitespaces) + if DynamoDBNumber.isValid(trimmed), !trimmed.isEmpty, !(path.isTopLevel && schema.keyType(of: path.root) != nil) { + values.append(.number(trimmed)) + } + return values + } + + func typed(_ text: String, as type: DynamoDBAttributeType) -> DynamoDBAttributeValue? { + switch type { + case .number: + let trimmed = text.trimmingCharacters(in: .whitespaces) + return DynamoDBNumber.isValid(trimmed) && !trimmed.isEmpty ? .number(trimmed) : nil + case .binary: + return Data(base64Encoded: text).map(DynamoDBAttributeValue.binary) + default: + return .string(text) + } + } + + static func bounds(value: String, secondValue: String?) -> (lower: String, upper: String)? { + if let secondValue { + let upper = secondValue.trimmingCharacters(in: .whitespaces) + let suffix = ",\(secondValue)" + let lowerSource = value.hasSuffix(suffix) ? String(value.dropLast(suffix.count)) : value + let lower = lowerSource.trimmingCharacters(in: .whitespaces) + guard !lower.isEmpty, !upper.isEmpty else { return nil } + return (lower, upper) + } + let parts = value.split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) } + guard parts.count == 2, !parts[0].isEmpty, !parts[1].isEmpty else { return nil } + return (parts[0], parts[1]) + } + + static func isReversed(_ lower: DynamoDBAttributeValue, _ upper: DynamoDBAttributeValue) -> Bool { + switch (lower, upper) { + case (.number(let low), .number(let high)): + return DynamoDBNumber.compare(low, high) == .orderedDescending + case (.string(let low), .string(let high)): + return Array(low.utf8).lexicographicallyPrecedes(Array(high.utf8)) == false && low != high + case (.binary(let low), .binary(let high)): + return Array(low).lexicographicallyPrecedes(Array(high)) == false && low != high + default: + return false + } + } + + static func any(_ terms: [String]) -> String { + terms.count == 1 ? terms[0] : "(" + terms.joined(separator: " OR ") + ")" + } + + static func all(_ terms: [String]) -> String { + terms.count == 1 ? terms[0] : "(" + terms.joined(separator: " AND ") + ")" + } +} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBItemFlattener.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBItemFlattener.swift deleted file mode 100644 index 9638b2b0f9..0000000000 --- a/Plugins/DynamoDBDriverPlugin/DynamoDBItemFlattener.swift +++ /dev/null @@ -1,278 +0,0 @@ -// -// DynamoDBItemFlattener.swift -// DynamoDBDriverPlugin -// -// Converts DynamoDB items to flat tabular rows for display. -// - -import Foundation -import TableProNumberFormatting -import TableProPluginKit - -struct DynamoDBItemFlattener { - /// Maximum serialized JSON length for nested values - private static let maxNestedJsonLength = 10_000 - - // MARK: - Column Discovery - - /// Union of all attribute names across items. - /// Key schema columns come first, then remaining columns sorted alphabetically. - static func unionColumns( - from items: [[String: DynamoDBAttributeValue]], - keySchema: [(name: String, keyType: String)] - ) -> [String] { - var seen = Set() - var ordered: [String] = [] - - // Key columns first: HASH then RANGE - let sortedKeys = keySchema.sorted { lhs, _ in lhs.keyType == "HASH" } - for key in sortedKeys { - if !seen.contains(key.name) { - seen.insert(key.name) - ordered.append(key.name) - } - } - - var remaining = Set() - for item in items { - for key in item.keys where !seen.contains(key) { - remaining.insert(key) - } - } - - ordered.append(contentsOf: remaining.sorted()) - - return ordered - } - - // MARK: - Flattening - - /// Convert items to a 2D grid of cell values. Missing attributes become null. - static func flatten(items: [[String: DynamoDBAttributeValue]], columns: [String]) -> [[PluginCellValue]] { - items.map { item in - columns.map { column in - guard let value = item[column] else { return PluginCellValue.null } - if case .binary(let data) = value { - return .bytes(data) - } - return .text(attributeValueToString(value)) - } - } - } - - // MARK: - Type Inference - - /// Majority-vote type name for each column across all items. - static func columnTypeNames(for columns: [String], items: [[String: DynamoDBAttributeValue]]) -> [String] { - columns.map { column in - var typeCounts: [String: Int] = [:] - for item in items { - guard let value = item[column] else { continue } - let typeName = typeNameForValue(value) - typeCounts[typeName, default: 0] += 1 - } - return typeCounts.max(by: { $0.value < $1.value })?.key ?? "S" - } - } - - // MARK: - Value Serialization - - /// Serialize a single DynamoDB attribute value to its display string. - static func attributeValueToString(_ value: DynamoDBAttributeValue) -> String { - switch value { - case .string(let s): - return s - case .number(let n): - return n - case .binary(let data): - return data.base64EncodedString() - case .bool(let b): - return b ? "true" : "false" - case .null: - return "NULL" - case .list(let items): - return serializeToJson(listToTypedEnvelopes(items)) - case .map(let map): - return serializeToJson(mapToTypedEnvelopes(map)) - case .stringSet(let values): - return serializeToJson(values) - case .numberSet(let values): - return serializeToJson(values) - case .binarySet(let values): - return serializeToJson(values.map { $0.base64EncodedString() }) - } - } - - /// Reverse conversion: parse a display string back to a DynamoDBAttributeValue, - /// using the type hint to determine the correct type. - static func stringToAttributeValue(_ string: String?, typeHint: String) -> DynamoDBAttributeValue? { - guard let string = string else { return .null } - - switch typeHint { - case "S": - return .string(string) - case "N": - return .number(string) - case "B": - if let data = Data(base64Encoded: string) { - return .binary(data) - } - return .binary(Data(string.utf8)) - case "BOOL": - let lower = string.lowercased() - return .bool(lower == "true" || lower == "1") - case "NULL": - return .null - case "L": - if let data = string.data(using: .utf8), - let array = try? JSONDecoder().decode([DynamoDBAttributeValue].self, from: data) - { - return .list(array) - } - return .string(string) - case "M": - if let data = string.data(using: .utf8), - let map = try? JSONDecoder().decode([String: DynamoDBAttributeValue].self, from: data) - { - return .map(map) - } - return .string(string) - case "SS": - if let data = string.data(using: .utf8), - let values = try? JSONSerialization.jsonObject(with: data) as? [String] - { - return .stringSet(values) - } - return .stringSet([string]) - case "NS": - if let data = string.data(using: .utf8), - let values = try? JSONSerialization.jsonObject(with: data) as? [String] - { - return .numberSet(values) - } - return .numberSet([string]) - case "BS": - if let data = string.data(using: .utf8), - let values = try? JSONSerialization.jsonObject(with: data) as? [String] - { - return .binarySet(values.compactMap { Data(base64Encoded: $0) }) - } - return .string(string) - default: - return .string(string) - } - } - - // MARK: - Private Helpers - - private static func typeNameForValue(_ value: DynamoDBAttributeValue) -> String { - switch value { - case .string: return "S" - case .number: return "N" - case .binary: return "B" - case .bool: return "BOOL" - case .null: return "NULL" - case .list: return "L" - case .map: return "M" - case .stringSet: return "SS" - case .numberSet: return "NS" - case .binarySet: return "BS" - } - } - - private static func listToJson(_ items: [DynamoDBAttributeValue]) -> [Any] { - items.map { valueToJsonPrimitive($0) } - } - - private static func mapToJson(_ map: [String: DynamoDBAttributeValue]) -> [String: Any] { - var result: [String: Any] = [:] - for (key, value) in map { - result[key] = valueToJsonPrimitive(value) - } - return result - } - - /// Convert a list to DynamoDB-typed envelope format (e.g., [{"S":"val"},{"N":"123"}]) - /// so that `stringToAttributeValue` can round-trip correctly. - private static func listToTypedEnvelopes(_ items: [DynamoDBAttributeValue]) -> [Any] { - items.map { valueToTypedEnvelope($0) } - } - - /// Convert a map to DynamoDB-typed envelope format (e.g., {"k":{"S":"val"}}) - /// so that `stringToAttributeValue` can round-trip correctly. - private static func mapToTypedEnvelopes(_ map: [String: DynamoDBAttributeValue]) -> [String: Any] { - var result: [String: Any] = [:] - for (key, value) in map { - result[key] = valueToTypedEnvelope(value) - } - return result - } - - /// Wrap a single DynamoDBAttributeValue in its DynamoDB JSON type envelope. - private static func valueToTypedEnvelope(_ value: DynamoDBAttributeValue) -> [String: Any] { - switch value { - case .string(let s): - return ["S": s] - case .number(let n): - return ["N": n] - case .binary(let data): - return ["B": data.base64EncodedString()] - case .bool(let b): - return ["BOOL": b] - case .null: - return ["NULL": true] - case .list(let items): - return ["L": listToTypedEnvelopes(items)] - case .map(let map): - return ["M": mapToTypedEnvelopes(map)] - case .stringSet(let values): - return ["SS": values] - case .numberSet(let values): - return ["NS": values] - case .binarySet(let values): - return ["BS": values.map { $0.base64EncodedString() }] - } - } - - private static func valueToJsonPrimitive(_ value: DynamoDBAttributeValue) -> Any { - switch value { - case .string(let s): - return s - case .number(let n): - if let intVal = Int64(n) { - return intVal - } - if let dblVal = Double(n) { - return dblVal - } - return n - case .binary(let data): - return data.base64EncodedString() - case .bool(let b): - return b - case .null: - return NSNull() - case .list(let items): - return listToJson(items) - case .map(let map): - return mapToJson(map) - case .stringSet(let values): - return values - case .numberSet(let values): - return values.map { str -> Any in - if let intVal = Int64(str) { return intVal } - if let dblVal = Double(str) { return dblVal } - return str - } - case .binarySet(let values): - return values.map { $0.base64EncodedString() } - } - } - - private static func serializeToJson(_ value: Any) -> String { - guard let json = NumberText.json(from: value) else { - return String(describing: value) - } - return JSONTruncation.truncate(json, maxLength: maxNestedJsonLength) - } -} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBItemTable.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBItemTable.swift new file mode 100644 index 0000000000..d366adceac --- /dev/null +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBItemTable.swift @@ -0,0 +1,141 @@ +import Foundation +import TableProPluginKit + +/// Lays schemaless items out as a grid. +/// +/// Columns are the columns the grid already shows, in its order, then the table's key attributes, +/// then every other attribute any item carries, alphabetically. Keeping the grid's order is what +/// lets Data Rewind read a row back by position. +struct DynamoDBItemTable: Sendable { + let columns: [String] + let types: [DynamoDBAttributeType?] + let rows: [[PluginCellValue]] + + init( + items: [DynamoDBItem], + schema: DynamoDBTableSchema?, + preferredColumns: [String] = [], + includeAllKeys: Bool = true + ) { + var columns: [String] = [] + var seen = Set() + func add(_ name: String) { + guard seen.insert(name).inserted else { return } + columns.append(name) + } + preferredColumns.forEach(add) + schema?.keys.attributes + .filter { key in includeAllKeys || items.contains { $0[key] != nil } } + .forEach(add) + var remaining = Set() + for item in items { + for name in item.keys where !seen.contains(name) { + remaining.insert(name) + } + } + remaining.sorted().forEach(add) + + self.columns = columns + self.types = columns.map { column in + if let keyType = schema?.keyType(of: column), schema?.keys.attributes.contains(column) == true { + return keyType + } + return Self.majorityType(of: column, in: items) + } + self.rows = items.map { item in + columns.map { DynamoDBCellCodec.cell(for: item[$0]) } + } + } + + var typeNames: [String] { + types.map { $0?.displayName ?? DynamoDBAttributeType.string.displayName } + } + + /// Carries the type the app classifies each column by, so a Map, a List or a set opens in the + /// JSON editor while the header still reads Map. + func columnMeta(schema: DynamoDBTableSchema?) -> [PluginColumnInfo] { + zip(columns, types).map { column, type in + let resolved = type ?? .string + let isKey = schema?.keys.attributes.contains(column) ?? false + return PluginColumnInfo( + name: column, + dataType: resolved.displayName, + isNullable: !isKey, + isPrimaryKey: isKey, + defaultValue: nil, + extra: nil, + charset: nil, + collation: nil, + comment: nil, + identityKind: nil, + isGenerated: false, + allowedValues: nil, + generationExpression: nil, + generationKind: nil, + ddlSpelling: nil, + ddlDefault: nil, + ddlGenerationExpression: nil, + ddlCollation: nil, + classificationTypeName: resolved.classificationName + ) + } + } + + var observedTypes: [String: DynamoDBAttributeType] { + var result: [String: DynamoDBAttributeType] = [:] + for (column, type) in zip(columns, types) { + if let type { result[column] = type } + } + return result + } + + static func majorityType(of attribute: String, in items: [DynamoDBItem]) -> DynamoDBAttributeType? { + var counts: [DynamoDBAttributeType: Int] = [:] + for item in items { + guard let value = item[attribute], value != .null else { continue } + counts[value.type, default: 0] += 1 + } + return counts.max { lhs, rhs in + lhs.value == rhs.value ? lhs.key.rawValue > rhs.key.rawValue : lhs.value < rhs.value + }?.key + } + + /// Sorts items for an ORDER BY DynamoDB could not apply. Numbers compare as numbers, missing + /// values sort first, and ties keep DynamoDB's order. + static func sorted(_ items: [DynamoDBItem], by order: [DynamoDBOrderTerm]) -> [DynamoDBItem] { + items.enumerated().sorted { lhs, rhs in + for term in order { + let comparison = compare(lhs.element[term.attribute], rhs.element[term.attribute]) + guard comparison != .orderedSame else { continue } + return term.descending ? comparison == .orderedDescending : comparison == .orderedAscending + } + return lhs.offset < rhs.offset + }.map(\.element) + } + + private static func typeRank(_ value: DynamoDBAttributeValue) -> Int { + switch value.type { + case .null: return 0 + case .boolean: return 1 + case .number: return 2 + case .string: return 3 + case .binary: return 4 + default: return 5 + } + } + + private static func compare(_ lhs: DynamoDBAttributeValue?, _ rhs: DynamoDBAttributeValue?) -> ComparisonResult { + switch (lhs, rhs) { + case (nil, nil): return .orderedSame + case (nil, _): return .orderedAscending + case (_, nil): return .orderedDescending + case (.number(let left)?, .number(let right)?): return DynamoDBNumber.compare(left, right) + case (let left?, let right?) where typeRank(left) != typeRank(right): + return typeRank(left) < typeRank(right) ? .orderedAscending : .orderedDescending + case (let left?, let right?): + let leftText = DynamoDBCellCodec.displayText(for: left) + let rightText = DynamoDBCellCodec.displayText(for: right) + return leftText < rightText ? .orderedAscending : (leftText == rightText ? .orderedSame : .orderedDescending) + } + } +} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBJSON.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBJSON.swift new file mode 100644 index 0000000000..9b35de5245 --- /dev/null +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBJSON.swift @@ -0,0 +1,321 @@ +import Foundation +import TableProNumberFormatting + +/// A JSON tree that keeps every number as the text it was written with. +/// +/// A DynamoDB Number carries 38 significant digits, which no `Double` holds, so every JSON this +/// driver reads or writes goes through here rather than `JSONSerialization`: the text of a map cell, +/// a request the editor sends, and every response. +enum DynamoDBJSON: Sendable, Equatable { + case object([String: DynamoDBJSON]) + case array([DynamoDBJSON]) + case string(String) + case number(String) + case bool(Bool) + case null + + subscript(key: String) -> DynamoDBJSON? { + guard case .object(let entries) = self else { return nil } + return entries[key] + } + + var stringValue: String? { + guard case .string(let value) = self else { return nil } + return value + } + + var numberText: String? { + guard case .number(let value) = self else { return nil } + return value + } + + var intValue: Int? { + numberText.flatMap { Int($0) } + } + + var doubleValue: Double? { + numberText.flatMap { Double($0) } + } + + var boolValue: Bool? { + guard case .bool(let value) = self else { return nil } + return value + } + + var arrayValue: [DynamoDBJSON]? { + guard case .array(let items) = self else { return nil } + return items + } + + var objectValue: [String: DynamoDBJSON]? { + guard case .object(let entries) = self else { return nil } + return entries + } + + func serialized(pretty: Bool = false) -> String { + NumberText.json(from: foundationValue, sortedKeys: true, prettyPrinted: pretty) ?? "null" + } + + var serializedData: Data { + Data(serialized().utf8) + } + + private var foundationValue: Any { + switch self { + case .object(let entries): + return entries.mapValues(\.foundationValue) + case .array(let items): + return items.map(\.foundationValue) + case .string(let value): + return value + case .number(let text): + return NumberText.RawNumber(text) ?? NSNull() + case .bool(let value): + return NSNumber(value: value) + case .null: + return NSNull() + } + } +} + +extension DynamoDBJSON { + enum ParseError: Error, LocalizedError, Equatable { + case unexpected(offset: Int) + case duplicateKey(String) + case tooDeep + case trailingText(offset: Int) + + var errorDescription: String? { + switch self { + case .unexpected(let offset): + return String(format: String(localized: "Not valid JSON at character %d"), offset + 1) + case .duplicateKey(let key): + return String(format: String(localized: "The key \"%@\" appears twice in one JSON object"), key) + case .tooDeep: + return String(localized: "The JSON is nested more deeply than DynamoDB allows") + case .trailingText(let offset): + return String(format: String(localized: "Unexpected text after the JSON at character %d"), offset + 1) + } + } + } + + static func parse(_ text: String) throws -> DynamoDBJSON { + var parser = Parser(scalars: Array(text.unicodeScalars)) + return try parser.parseDocument() + } + + static func parse(_ data: Data) throws -> DynamoDBJSON { + guard let text = String(data: data, encoding: .utf8) else { + throw ParseError.unexpected(offset: 0) + } + return try parse(text) + } + + /// Parses the JSON value at the start of `text` and reports where it ended, for statements + /// that carry clauses after their JSON body. + static func parsePrefix(_ text: String) throws -> (value: DynamoDBJSON, remainder: String) { + let scalars = Array(text.unicodeScalars) + var parser = Parser(scalars: scalars) + parser.skipWhitespace() + let value = try parser.parseValue(depth: 0) + var remainder = String.UnicodeScalarView() + remainder.append(contentsOf: scalars[parser.position...]) + return (value, String(remainder)) + } + + private struct Parser { + static let maximumDepth = 128 + + let scalars: [Unicode.Scalar] + var position = 0 + + init(scalars: [Unicode.Scalar]) { + self.scalars = scalars + } + + mutating func parseDocument() throws -> DynamoDBJSON { + skipWhitespace() + let value = try parseValue(depth: 0) + skipWhitespace() + guard position == scalars.count else { throw ParseError.trailingText(offset: position) } + return value + } + + mutating func skipWhitespace() { + while position < scalars.count, [" ", "\n", "\r", "\t"].contains(scalars[position]) { + position += 1 + } + } + + mutating func parseValue(depth: Int) throws -> DynamoDBJSON { + guard depth <= Self.maximumDepth else { throw ParseError.tooDeep } + guard position < scalars.count else { throw ParseError.unexpected(offset: position) } + switch scalars[position] { + case "{": + return try parseObject(depth: depth) + case "[": + return try parseArray(depth: depth) + case "\"": + return .string(try parseString()) + case "t": + try expectLiteral("true") + return .bool(true) + case "f": + try expectLiteral("false") + return .bool(false) + case "n": + try expectLiteral("null") + return .null + default: + return .number(try parseNumber()) + } + } + + private mutating func parseObject(depth: Int) throws -> DynamoDBJSON { + position += 1 + var entries: [String: DynamoDBJSON] = [:] + skipWhitespace() + if position < scalars.count, scalars[position] == "}" { + position += 1 + return .object(entries) + } + while true { + skipWhitespace() + guard position < scalars.count, scalars[position] == "\"" else { + throw ParseError.unexpected(offset: position) + } + let key = try parseString() + skipWhitespace() + try expect(":") + skipWhitespace() + let value = try parseValue(depth: depth + 1) + guard entries[key] == nil else { throw ParseError.duplicateKey(key) } + entries[key] = value + skipWhitespace() + guard position < scalars.count else { throw ParseError.unexpected(offset: position) } + if scalars[position] == "," { + position += 1 + continue + } + try expect("}") + return .object(entries) + } + } + + private mutating func parseArray(depth: Int) throws -> DynamoDBJSON { + position += 1 + var items: [DynamoDBJSON] = [] + skipWhitespace() + if position < scalars.count, scalars[position] == "]" { + position += 1 + return .array(items) + } + while true { + skipWhitespace() + items.append(try parseValue(depth: depth + 1)) + skipWhitespace() + guard position < scalars.count else { throw ParseError.unexpected(offset: position) } + if scalars[position] == "," { + position += 1 + continue + } + try expect("]") + return .array(items) + } + } + + private mutating func parseString() throws -> String { + position += 1 + var result = String.UnicodeScalarView() + while position < scalars.count { + let scalar = scalars[position] + position += 1 + switch scalar { + case "\"": + return String(result) + case "\\": + result.append(try parseEscape()) + default: + guard scalar.value >= 0x20 else { throw ParseError.unexpected(offset: position - 1) } + result.append(scalar) + } + } + throw ParseError.unexpected(offset: position) + } + + private mutating func parseEscape() throws -> Unicode.Scalar { + guard position < scalars.count else { throw ParseError.unexpected(offset: position) } + let marker = scalars[position] + position += 1 + switch marker { + case "\"": return "\"" + case "\\": return "\\" + case "/": return "/" + case "b": return "\u{08}" + case "f": return "\u{0C}" + case "n": return "\n" + case "r": return "\r" + case "t": return "\t" + case "u": + let high = try parseHexQuad() + guard (0xD800...0xDBFF).contains(high) else { + guard let scalar = Unicode.Scalar(high) else { throw ParseError.unexpected(offset: position) } + return scalar + } + guard position + 1 < scalars.count, scalars[position] == "\\", scalars[position + 1] == "u" else { + throw ParseError.unexpected(offset: position) + } + position += 2 + let low = try parseHexQuad() + guard (0xDC00...0xDFFF).contains(low) else { throw ParseError.unexpected(offset: position) } + let combined = 0x10000 + ((high - 0xD800) << 10) + (low - 0xDC00) + guard let scalar = Unicode.Scalar(combined) else { throw ParseError.unexpected(offset: position) } + return scalar + default: + throw ParseError.unexpected(offset: position - 1) + } + } + + private mutating func parseHexQuad() throws -> UInt32 { + guard position + 4 <= scalars.count else { throw ParseError.unexpected(offset: position) } + var value: UInt32 = 0 + for _ in 0..<4 { + guard let digit = UInt32(String(scalars[position]), radix: 16) else { + throw ParseError.unexpected(offset: position) + } + value = value * 16 + digit + position += 1 + } + return value + } + + private mutating func parseNumber() throws -> String { + let start = position + while position < scalars.count, Self.numberScalars.contains(scalars[position]) { + position += 1 + } + var text = String.UnicodeScalarView() + text.append(contentsOf: scalars[start.. = [ + "-", "+", ".", "e", "E", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9" + ] + + private mutating func expect(_ scalar: Unicode.Scalar) throws { + guard position < scalars.count, scalars[position] == scalar else { + throw ParseError.unexpected(offset: position) + } + position += 1 + } + + private mutating func expectLiteral(_ literal: String) throws { + for scalar in literal.unicodeScalars { + try expect(scalar) + } + } + } +} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBNumber.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBNumber.swift new file mode 100644 index 0000000000..98e718b6c7 --- /dev/null +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBNumber.swift @@ -0,0 +1,110 @@ +import Foundation + +/// DynamoDB's Number type: up to 38 significant digits, magnitude from 1E-130 to below 1E+126. +/// +/// Carried as text end to end. A `Double` holds about 15 digits, so reading a Number through one +/// turns `12345678901234567890123456789012345678` into `...7525491324606797053952`. +enum DynamoDBNumber { + static let maximumSignificantDigits = 38 + static let smallestExponent = -130 + static let largestExponent = 125 + + struct Parts: Equatable { + let isNegative: Bool + let significantDigits: [UInt8] + let leadingExponent: Int + } + + static func isValid(_ text: String) -> Bool { + guard let parts = parts(of: text) else { return false } + guard !parts.significantDigits.isEmpty else { return true } + guard parts.significantDigits.count <= maximumSignificantDigits else { return false } + return (smallestExponent...largestExponent).contains(parts.leadingExponent) + } + + /// Splits a number into sign, significant digits and the power of ten of its first digit, or nil + /// when the text is not a number at all. Zero has no significant digits. + static func parts(of text: String) -> Parts? { + let trimmed = text.trimmingCharacters(in: .whitespaces) + var scalars = Substring(trimmed) + var isNegative = false + if let sign = scalars.first, sign == "-" || sign == "+" { + isNegative = sign == "-" + scalars = scalars.dropFirst() + } + var mantissa = scalars + var exponent = 0 + if let marker = scalars.firstIndex(where: { $0 == "e" || $0 == "E" }) { + mantissa = scalars[.. ComparisonResult { + guard let left = parts(of: lhs), let right = parts(of: rhs) else { + return lhs < rhs ? .orderedAscending : (lhs == rhs ? .orderedSame : .orderedDescending) + } + let leftSign = left.significantDigits.isEmpty ? 0 : (left.isNegative ? -1 : 1) + let rightSign = right.significantDigits.isEmpty ? 0 : (right.isNegative ? -1 : 1) + if leftSign != rightSign { + return leftSign < rightSign ? .orderedAscending : .orderedDescending + } + guard leftSign != 0 else { return .orderedSame } + let magnitude = compareMagnitude(left, right) + return leftSign > 0 ? magnitude : magnitude.reversed + } + + static func areEqual(_ lhs: String, _ rhs: String) -> Bool { + compare(lhs, rhs) == .orderedSame + } + + private static func compareMagnitude(_ left: Parts, _ right: Parts) -> ComparisonResult { + if left.leadingExponent != right.leadingExponent { + return left.leadingExponent < right.leadingExponent ? .orderedAscending : .orderedDescending + } + let count = max(left.significantDigits.count, right.significantDigits.count) + for index in 0.. String? { - guard isTableObject(objectType), let quoted = quotedName(name) else { return nil } - return "DROP TABLE \(quoted)" - } - - /// DynamoDB has only tables, so any other kind the app asks about is not something this engine - /// drops, and answering anyway would delete the table of that name instead. - static func isTableObject(_ objectType: String) -> Bool { - objectType.uppercased() == "TABLE" - } - - /// The table named by a drop statement, or nil when the text is not one. - static func droppedTableName(in statement: String) -> String? { - let trimmed = statement.trimmingCharacters(in: .whitespacesAndNewlines) - guard trimmed.uppercased().hasPrefix("DROP TABLE ") else { return nil } - let rest = trimmed.dropFirst("DROP TABLE ".count).trimmingCharacters(in: .whitespaces) - guard rest.hasPrefix("\""), rest.hasSuffix("\""), rest.count > 2 else { return nil } - let unquoted = String(rest.dropFirst().dropLast()).replacingOccurrences(of: "\"\"", with: "\"") - return isValidTableName(unquoted) ? unquoted : nil - } - - /// A DynamoDB table name is 3 to 255 characters of `a-z A-Z 0-9 _ - .` and nothing else, so a - /// name carrying anything more did not come from the table listing and is refused rather than - /// sent as a delete. - static func isValidTableName(_ name: String) -> Bool { - guard (3...255).contains(name.count) else { return false } - return name.allSatisfy { $0.isLetter || $0.isNumber || $0 == "_" || $0 == "-" || $0 == "." } - } - - private static func quotedName(_ name: String) -> String? { - guard isValidTableName(name) else { return nil } - return "\"\(name)\"" - } -} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBPartiQL.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBPartiQL.swift new file mode 100644 index 0000000000..5c4b84d4e8 --- /dev/null +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBPartiQL.swift @@ -0,0 +1,397 @@ +import Foundation + +struct DynamoDBPartiQLToken: Sendable, Equatable { + enum Kind: Sendable, Equatable { + case word + case quotedIdentifier + case string + case number + case parameter + case symbol + } + + let kind: Kind + /// The source spelling for words, numbers and symbols; the unescaped content for quoted + /// identifiers and string literals. + let text: String + let range: Range + let depth: Int + + func isKeyword(_ keyword: String) -> Bool { + kind == .word && text.caseInsensitiveCompare(keyword) == .orderedSame + } + + var identifierValue: String? { + switch kind { + case .word, .quotedIdentifier: return text + default: return nil + } + } +} + +enum DynamoDBPartiQL { + enum Kind: Equatable { + case select + case insert + case update + case delete + case other + } + + /// What a `?` stands for, read from the words around it. + enum ParameterRole: Equatable { + case assigned(DynamoDBAttributePath) + case compared(DynamoDBAttributePath) + case inserted(String) + case unknown + } + + static func kind(of statement: String) -> Kind { + let first = tokens(of: statement).first + if first?.isKeyword("SELECT") == true { return .select } + if first?.isKeyword("INSERT") == true { return .insert } + if first?.isKeyword("UPDATE") == true { return .update } + if first?.isKeyword("DELETE") == true { return .delete } + return .other + } + + /// The table and, for `"Table"."Index"`, the index a statement names. + static func target(of statement: String) -> (table: String, index: String?)? { + let tokens = tokens(of: statement) + guard let first = tokens.first else { return nil } + let nameStart: Int? + if first.isKeyword("SELECT") || first.isKeyword("DELETE") { + nameStart = tokens.firstIndex { $0.isKeyword("FROM") && $0.depth == 0 }.map { $0 + 1 } + } else if first.isKeyword("INSERT") { + nameStart = tokens.firstIndex { $0.isKeyword("INTO") }.map { $0 + 1 } + } else if first.isKeyword("UPDATE") { + nameStart = 1 + } else { + nameStart = nil + } + guard let start = nameStart, start < tokens.count, let table = tokens[start].identifierValue else { + return nil + } + if start + 2 < tokens.count, tokens[start + 1].text == ".", let index = tokens[start + 2].identifierValue { + return (table, index) + } + return (table, nil) + } + + static func hasReturning(_ statement: String) -> Bool { + tokens(of: statement).contains { $0.depth == 0 && $0.isKeyword("RETURNING") } + } + + /// Whether the WHERE clause pins the top-level `attribute` with `=`, or with `IN` when + /// `acceptsIn`, in a term every row must satisfy: one joined to the rest by AND alone, at the + /// top level or inside parentheses, and never negated. That is what lets DynamoDB answer a SELECT + /// with a Query rather than a Scan; its ORDER BY needs the `=` form, because with `IN` it orders + /// each partition on its own. + static func whereFixes(_ attribute: String, in statement: String, acceptsIn: Bool = true) -> Bool { + let tokens = tokens(of: statement) + guard let whereIndex = tokens.firstIndex(where: { $0.depth == 0 && $0.isKeyword("WHERE") }) else { + return false + } + let clause = Array(tokens[(whereIndex + 1)...]) + let clauseEnd = clause.firstIndex { $0.depth == 0 && ($0.isKeyword("ORDER") || $0.isKeyword("RETURNING")) } + ?? clause.count + let condition = Array(clause[.. Bool { + guard offset + 1 < condition.count else { return false } + let next = condition[offset + 1] + if offset > 0 { + let previous = condition[offset - 1] + if previous.text == "." || previous.isKeyword("NOT") { return false } + } + if next.text == "." || next.text == "[" { return false } + return next.text == "=" || (acceptsIn && next.isKeyword("IN")) + } + + /// Whether every group around a term is a plain parenthesized condition, neither negated nor a + /// function's arguments, and no level from the term out to the WHERE clause holds an OR. + private static func isRequired(openers: [Int], in condition: [DynamoDBPartiQLToken]) -> Bool { + for opener in openers { + guard condition[opener].text == "(" else { return false } + guard opener > 0 else { continue } + let before = condition[opener - 1] + if before.isKeyword("NOT") || before.kind == .quotedIdentifier { return false } + if before.kind == .word, !before.isKeyword("AND"), !before.isKeyword("OR") { return false } + } + if condition.contains(where: { $0.depth == 0 && $0.isKeyword("OR") }) { return false } + for opener in openers { + let innerDepth = condition[opener].depth + 1 + let closer = condition[(opener + 1)...].firstIndex { token in + token.kind == .symbol && token.text == ")" && token.depth == condition[opener].depth + } ?? condition.count + let group = condition[(opener + 1).. (statement: String, window: DynamoDBReadWindow) { + guard kind(of: statement) == .select else { return (statement, DynamoDBReadWindow()) } + var tokens = tokens(of: statement).filter { $0.depth == 0 } + var window = DynamoDBReadWindow() + var cut = statement.endIndex + + if tokens.count >= 2, tokens[tokens.count - 2].isKeyword("OFFSET"), + let offset = Int(tokens.last?.text ?? ""), offset >= 0 { + window.offset = offset + cut = tokens[tokens.count - 2].range.lowerBound + tokens.removeLast(2) + } + if tokens.count >= 2, tokens[tokens.count - 2].isKeyword("LIMIT"), + let limit = Int(tokens.last?.text ?? ""), limit >= 0 { + window.limit = limit + cut = tokens[tokens.count - 2].range.lowerBound + tokens.removeLast(2) + } + if let orderIndex = trailingOrderByIndex(tokens) { + var terms: [DynamoDBOrderTerm] = [] + var cursor = orderIndex + 2 + while cursor < tokens.count, let attribute = tokens[cursor].identifierValue { + cursor += 1 + var descending = false + if cursor < tokens.count, tokens[cursor].isKeyword("DESC") { + descending = true + cursor += 1 + } else if cursor < tokens.count, tokens[cursor].isKeyword("ASC") { + cursor += 1 + } + terms.append(DynamoDBOrderTerm(attribute: attribute, descending: descending)) + guard cursor < tokens.count, tokens[cursor].text == "," else { break } + cursor += 1 + } + if cursor == tokens.count, !terms.isEmpty { + window.order = terms + cut = tokens[orderIndex].range.lowerBound + } + } + let remaining = String(statement[.. Int? { + guard let index = tokens.lastIndex(where: { $0.isKeyword("ORDER") }), + index + 1 < tokens.count, tokens[index + 1].isKeyword("BY") + else { return nil } + return index + } + + /// The role of each `?` in `statement`, in order. + /// The top-level attribute names an INSERT's `VALUE {...}` sets, whether the value is a literal + /// or a `?`. + static func insertedAttributes(in statement: String) -> Set { + let tokens = tokens(of: statement) + guard let valueIndex = tokens.firstIndex(where: { $0.depth == 0 && $0.isKeyword("VALUE") }) else { return [] } + var names = Set() + for index in tokens.indices where index > valueIndex && index + 1 < tokens.count { + let token = tokens[index] + guard token.kind == .string, token.depth == 1, tokens[index + 1].text == ":" else { continue } + names.insert(token.text) + } + return names + } + + static func parameterRoles(in statement: String) -> [ParameterRole] { + let tokens = tokens(of: statement) + var roles: [ParameterRole] = [] + var clause = "" + var comparedPath: DynamoDBAttributePath? + var pendingKey: String? + + for (index, token) in tokens.enumerated() { + if token.kind == .word, token.depth == 0 { + let upper = token.text.uppercased() + if ["SET", "WHERE", "VALUE", "REMOVE", "RETURNING", "FROM", "INTO"].contains(upper) { + clause = upper + } + } + switch token.kind { + case .string where clause == "VALUE": + if index + 1 < tokens.count, tokens[index + 1].text == ":" { + pendingKey = token.text + } + case .word, .quotedIdentifier: + let isOperatorWord = ["AND", "OR", "NOT", "IN", "BETWEEN", "IS"].contains(token.text.uppercased()) + if !isOperatorWord || token.kind == .quotedIdentifier { + if index + 1 < tokens.count, tokens[index + 1].text != "(" { + comparedPath = path(tokens, endingAt: index) + } + } + case .parameter: + if clause == "VALUE", let key = pendingKey { + roles.append(.inserted(key)) + pendingKey = nil + } else if clause == "SET", let path = comparedPath { + roles.append(.assigned(path)) + } else if let path = comparedPath { + roles.append(.compared(path)) + } else { + roles.append(.unknown) + } + default: + break + } + } + return roles + } + + /// The document path, such as `"a"."b"[0]`, whose last name ends at `index`, followed by any + /// list indexes after it. + private static func path(_ tokens: [DynamoDBPartiQLToken], endingAt index: Int) -> DynamoDBAttributePath? { + var cursor = index + while cursor >= 2, tokens[cursor - 1].text == ".", tokens[cursor - 2].identifierValue != nil { + cursor -= 2 + } + var segments: [DynamoDBAttributePath.Segment] = [] + var position = cursor + while position <= index { + if let name = tokens[position].identifierValue { + segments.append(.name(name)) + } + position += 2 + } + var trailing = index + 1 + while trailing + 2 < tokens.count, tokens[trailing].text == "[", tokens[trailing + 2].text == "]", + let element = Int(tokens[trailing + 1].text), element >= 0 { + segments.append(.index(element)) + trailing += 3 + } + return segments.isEmpty ? nil : DynamoDBAttributePath(segments: segments) + } + + // MARK: - Tokenizer + + static func tokens(of text: String) -> [DynamoDBPartiQLToken] { + var tokens: [DynamoDBPartiQLToken] = [] + var index = text.startIndex + var depth = 0 + + while index < text.endIndex { + let character = text[index] + if character.isWhitespace { + index = text.index(after: index) + continue + } + if text[index...].hasPrefix("--") { + index = text[index...].firstIndex(where: \.isNewline) ?? text.endIndex + continue + } + if text[index...].hasPrefix("/*") { + let bodyStart = text.index(index, offsetBy: 2) + index = text[bodyStart...].range(of: "*/")?.upperBound ?? text.endIndex + continue + } + let start = index + switch character { + case "\"", "'": + let (content, end) = quoted(text, from: index, quote: character) + tokens.append(DynamoDBPartiQLToken( + kind: character == "\"" ? .quotedIdentifier : .string, + text: content, range: start.. (String, String.Index) { + var content = "" + var index = text.index(after: start) + while index < text.endIndex { + let character = text[index] + let next = text.index(after: index) + if character == quote { + if next < text.endIndex, text[next] == quote { + content.append(quote) + index = text.index(after: next) + continue + } + return (content, next) + } + content.append(character) + index = next + } + return (content, text.endIndex) + } + + private static func nextIsDigit(_ text: String, after index: String.Index) -> Bool { + let next = text.index(after: index) + return next < text.endIndex && text[next].isNumber + } + + private static func symbolEnd(_ text: String, from index: String.Index) -> String.Index { + let pairs = ["<>", "<=", ">=", "!=", "<<", ">>"] + for pair in pairs where text[index...].hasPrefix(pair) { + return text.index(index, offsetBy: 2) + } + return text.index(after: index) + } +} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBPartiQLParser.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBPartiQLParser.swift deleted file mode 100644 index 13e20550e2..0000000000 --- a/Plugins/DynamoDBDriverPlugin/DynamoDBPartiQLParser.swift +++ /dev/null @@ -1,173 +0,0 @@ -// -// DynamoDBPartiQLParser.swift -// DynamoDBDriverPlugin -// -// Lightweight PartiQL statement classifier. -// - -import Foundation - -internal enum DynamoDBQueryType { - case select - case insert - case update - case delete - case unknown -} - -internal struct DynamoDBPartiQLParser { - /// Classify a PartiQL statement by its first keyword. - static func queryType(_ statement: String) -> DynamoDBQueryType { - let trimmed = statement.trimmingCharacters(in: .whitespacesAndNewlines) - let firstWord = trimmed.components(separatedBy: .whitespacesAndNewlines).first?.uppercased() ?? "" - - switch firstWord { - case "SELECT": - return .select - case "INSERT": - return .insert - case "UPDATE": - return .update - case "DELETE": - return .delete - default: - return .unknown - } - } - - /// Extract the table name from a PartiQL statement. - /// Handles quoted ("TableName") and unquoted table names. - /// - /// Patterns: - /// - SELECT ... FROM "TableName" ... - /// - INSERT INTO "TableName" ... - /// - UPDATE "TableName" ... - /// - DELETE FROM "TableName" ... - static func extractTableName(_ statement: String) -> String? { - let trimmed = statement.trimmingCharacters(in: .whitespacesAndNewlines) - let tokens = tokenize(trimmed) - - guard !tokens.isEmpty else { return nil } - - let firstUpper = tokens[0].uppercased() - - switch firstUpper { - case "SELECT": - if let fromIndex = tokens.firstIndex(where: { $0.uppercased() == "FROM" }), - fromIndex + 1 < tokens.count - { - return normalizeIdentifierToken(tokens[fromIndex + 1]) - } - case "INSERT": - if tokens.count >= 3, tokens[1].uppercased() == "INTO" { - return normalizeIdentifierToken(tokens[2]) - } - case "UPDATE": - if tokens.count >= 2 { - return normalizeIdentifierToken(tokens[1]) - } - case "DELETE": - if tokens.count >= 3, tokens[1].uppercased() == "FROM" { - return normalizeIdentifierToken(tokens[2]) - } - default: - break - } - - return nil - } - - // MARK: - Private - - /// Simple tokenizer that respects quoted identifiers and string literals. - /// Handles PartiQL doubled single-quote escaping (e.g., `'O''Brien'`). - private static func tokenize(_ sql: String) -> [String] { - var tokens: [String] = [] - var current = "" - var inDoubleQuote = false - var inSingleQuote = false - var isEscaped = false - - let chars = Array(sql) - var i = 0 - - while i < chars.count { - let char = chars[i] - - if isEscaped { - current.append(char) - isEscaped = false - i += 1 - continue - } - - if char == "\\" { - current.append(char) - isEscaped = true - i += 1 - continue - } - - if char == "\"" && !inSingleQuote { - inDoubleQuote.toggle() - current.append(char) - i += 1 - continue - } - - if char == "'" && !inDoubleQuote { - if inSingleQuote { - // Check for doubled single-quote escape ('') - if i + 1 < chars.count && chars[i + 1] == "'" { - current.append(char) - current.append(chars[i + 1]) - i += 2 - continue - } - inSingleQuote = false - } else { - inSingleQuote = true - } - current.append(char) - i += 1 - continue - } - - if char.isWhitespace && !inDoubleQuote && !inSingleQuote { - if !current.isEmpty { - tokens.append(current) - current = "" - } - i += 1 - continue - } - - current.append(char) - i += 1 - } - - if !current.isEmpty { - tokens.append(current) - } - - return tokens - } - - /// Strip trailing punctuation (`;`, `,`) from a token before unquoting. - private static func normalizeIdentifierToken(_ token: String) -> String { - var cleaned = token - while cleaned.hasSuffix(";") || cleaned.hasSuffix(",") { - cleaned = String(cleaned.dropLast()) - } - return unquoteIdentifier(cleaned) - } - - /// Remove surrounding double quotes from an identifier if present. - private static func unquoteIdentifier(_ identifier: String) -> String { - if identifier.hasPrefix("\"") && identifier.hasSuffix("\"") && identifier.count >= 2 { - let inner = String(identifier.dropFirst().dropLast()) - return inner.replacingOccurrences(of: "\"\"", with: "\"") - } - return identifier - } -} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBPlugin.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBPlugin.swift index 3239c61335..3452763fde 100644 --- a/Plugins/DynamoDBDriverPlugin/DynamoDBPlugin.swift +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBPlugin.swift @@ -1,18 +1,11 @@ -// -// DynamoDBPlugin.swift -// DynamoDBDriverPlugin -// -// Amazon DynamoDB driver plugin via AWS HTTP API with PartiQL support -// - import Foundation import os import TableProPluginKit -final class DynamoDBPlugin: NSObject, TableProPlugin, DriverPlugin { +final class DynamoDBPlugin: NSObject, TableProPlugin, DriverPlugin, PluginDefaultSortProvider { static let pluginName = "DynamoDB Driver" - static let pluginVersion = "1.0.0" - static let pluginDescription = "Amazon DynamoDB support via AWS HTTP API with PartiQL" + static let pluginVersion = "2.0.0" + static let pluginDescription = "Amazon DynamoDB: tables, indexes, PartiQL and the DynamoDB API" static let capabilities: [PluginCapability] = [.databaseDriver] static let databaseTypeId = "DynamoDB" @@ -29,8 +22,16 @@ final class DynamoDBPlugin: NSObject, TableProPlugin, DriverPlugin { static let brandColorHex = "#4053D6" static let queryLanguageName = "PartiQL" static let editorLanguage: EditorLanguage = .sql + static let parameterStyle: ParameterStyle = .questionMark static let supportsForeignKeys = false - static let supportsSchemaEditing = false + static let supportsSchemaEditing = true + static let supportsAddColumn = false + static let supportsModifyColumn = false + static let supportsDropColumn = false + static let supportsRenameColumn = false + static let supportsModifyPrimaryKey = false + static let supportsAddIndex = true + static let supportsDropIndex = true static let supportsDatabaseSwitching = false static let supportsImport = false static let supportsExport = true @@ -42,123 +43,160 @@ final class DynamoDBPlugin: NSObject, TableProPlugin, DriverPlugin { static let databaseGroupingStrategy: GroupingStrategy = .flat static let defaultGroupName = "main" static let defaultPrimaryKeyColumn: String? = nil - static let structureColumnFields: [StructureColumnField] = [.name, .type] + static let structureColumnFields: [StructureColumnField] = [.name, .type, .primaryKey] + static let caseSensitivityStyle: SQLDialectDescriptor.CaseSensitivityStyle = .driverManaged static let sqlDialect: SQLDialectDescriptor? = SQLDialectDescriptor( identifierQuote: "\"", - keywords: [ - "SELECT", "FROM", "WHERE", "INSERT", "INTO", "VALUE", "SET", - "UPDATE", "DELETE", "AND", "OR", "NOT", "IN", "BETWEEN", - "EXISTS", "MISSING", "IS", "NULL", "LIMIT" - ], - functions: [ - "begins_with", "contains", "size", "attribute_type", - "attribute_exists", "attribute_not_exists" - ], - dataTypes: ["S", "N", "B", "BOOL", "NULL", "L", "M", "SS", "NS", "BS"], + keywords: DynamoDBEditorVocabulary.keywords, + functions: DynamoDBEditorVocabulary.functions, + dataTypes: Set(DynamoDBAttributeType.allCases.map(\.displayName)), + booleanLiteralStyle: .truefalse, + autoLimitStyle: .none, caseSensitivityStyle: .driverManaged ) static let columnTypesByCategory: [String: [String]] = [ - "String": ["S"], - "Number": ["N"], - "Binary": ["B"], - "Boolean": ["BOOL"], - "Null": ["NULL"], - "List": ["L"], - "Map": ["M"], - "String Set": ["SS"], - "Number Set": ["NS"], - "Binary Set": ["BS"] + "Key": [ + DynamoDBAttributeType.string.displayName, + DynamoDBAttributeType.number.displayName, + DynamoDBAttributeType.binary.displayName + ], + "Scalar": [ + DynamoDBAttributeType.boolean.displayName, + DynamoDBAttributeType.null.displayName + ], + "Document": [ + DynamoDBAttributeType.list.displayName, + DynamoDBAttributeType.map.displayName + ], + "Set": [ + DynamoDBAttributeType.stringSet.displayName, + DynamoDBAttributeType.numberSet.displayName, + DynamoDBAttributeType.binarySet.displayName + ] ] - static let additionalConnectionFields: [ConnectionField] = [ - ConnectionField( - id: "awsAuthMethod", - label: String(localized: "Auth Method"), - defaultValue: "credentials", - fieldType: .dropdown(options: [ - .init(value: "credentials", label: "Access Key + Secret Key"), - .init(value: "profile", label: "AWS Profile"), - .init(value: "sso", label: "AWS SSO") - ]), - section: .authentication - ), - ConnectionField( - id: "awsAccessKeyId", - label: String(localized: "Access Key ID"), - placeholder: "AKIA...", - section: .authentication, - visibleWhen: FieldVisibilityRule(fieldId: "awsAuthMethod", values: ["credentials"]) - ), - ConnectionField( - id: "awsSecretAccessKey", - label: String(localized: "Secret Access Key"), - placeholder: "wJalr...", - fieldType: .secure, - section: .authentication, - hidesPassword: true, - visibleWhen: FieldVisibilityRule(fieldId: "awsAuthMethod", values: ["credentials"]) - ), - ConnectionField( - id: "awsSessionToken", - label: String(localized: "Session Token"), - placeholder: "Optional (for temporary credentials)", - fieldType: .secure, - section: .authentication, - visibleWhen: FieldVisibilityRule(fieldId: "awsAuthMethod", values: ["credentials"]) - ), - ConnectionField( - id: "awsProfileName", - label: String(localized: "Profile Name"), - placeholder: "default", - section: .authentication, - visibleWhen: FieldVisibilityRule(fieldId: "awsAuthMethod", values: ["profile", "sso"]) - ).withDynamicOptions(.awsProfiles), - ConnectionField( - id: "awsRegion", - label: String(localized: "AWS Region"), - placeholder: "us-east-1", - defaultValue: "us-east-1", - fieldType: .text, - section: .authentication - ), - ConnectionField( - id: "awsEndpointUrl", - label: String(localized: "Custom Endpoint"), - placeholder: "http://localhost:8000 (DynamoDB Local)", - section: .authentication - ), - ] + static var additionalConnectionFields: [ConnectionField] { + DynamoDBConnectionFields.all + } static var statementCompletions: [CompletionEntry] { - [ - CompletionEntry(label: "SELECT", insertText: "SELECT"), - CompletionEntry(label: "INSERT INTO", insertText: "INSERT INTO"), - CompletionEntry(label: "UPDATE", insertText: "UPDATE"), - CompletionEntry(label: "DELETE FROM", insertText: "DELETE FROM"), - CompletionEntry(label: "VALUE", insertText: "VALUE"), - CompletionEntry(label: "SET", insertText: "SET"), - CompletionEntry(label: "WHERE", insertText: "WHERE"), - CompletionEntry(label: "AND", insertText: "AND"), - CompletionEntry(label: "OR", insertText: "OR"), - CompletionEntry(label: "BETWEEN", insertText: "BETWEEN"), - CompletionEntry(label: "EXISTS", insertText: "EXISTS"), - CompletionEntry(label: "MISSING", insertText: "MISSING"), - CompletionEntry(label: "IN", insertText: "IN"), - CompletionEntry(label: "IS", insertText: "IS"), - CompletionEntry(label: "NOT", insertText: "NOT"), - CompletionEntry(label: "NULL", insertText: "NULL"), - CompletionEntry(label: "begins_with", insertText: "begins_with"), - CompletionEntry(label: "contains", insertText: "contains"), - CompletionEntry(label: "size", insertText: "size"), - CompletionEntry(label: "attribute_type", insertText: "attribute_type"), - CompletionEntry(label: "attribute_exists", insertText: "attribute_exists"), - CompletionEntry(label: "attribute_not_exists", insertText: "attribute_not_exists") - ] + DynamoDBEditorVocabulary.completions + } + + func defaultSortHint(forTable table: String) -> DefaultSortHint { + .suppress } func createDriver(config: DriverConnectionConfig) -> any PluginDatabaseDriver { DynamoDBPluginDriver(config: config) } } + +enum DynamoDBConnectionFields { + static var all: [ConnectionField] { + [ + ConnectionField( + id: "awsAuthMethod", + label: String(localized: "Auth Method"), + defaultValue: DynamoDBAuthMethod.accessKey.rawValue, + fieldType: .dropdown(options: [ + .init(value: DynamoDBAuthMethod.accessKey.rawValue, label: String(localized: "Access Key + Secret Key")), + .init(value: DynamoDBAuthMethod.profile.rawValue, label: String(localized: "AWS Profile")), + .init(value: DynamoDBAuthMethod.singleSignOn.rawValue, label: String(localized: "AWS SSO")), + .init(value: DynamoDBAuthMethod.local.rawValue, label: String(localized: "DynamoDB Local (no credentials)")) + ]), + section: .authentication + ), + ConnectionField( + id: "awsAccessKeyId", + label: String(localized: "Access Key ID"), + placeholder: "AKIA...", + section: .authentication, + visibleWhen: FieldVisibilityRule(fieldId: "awsAuthMethod", values: [DynamoDBAuthMethod.accessKey.rawValue]) + ), + ConnectionField( + id: "awsSecretAccessKey", + label: String(localized: "Secret Access Key"), + placeholder: "wJalr...", + fieldType: .secure, + section: .authentication, + hidesPassword: true, + visibleWhen: FieldVisibilityRule(fieldId: "awsAuthMethod", values: [DynamoDBAuthMethod.accessKey.rawValue]) + ), + ConnectionField( + id: "awsSessionToken", + label: String(localized: "Session Token"), + placeholder: String(localized: "Optional, for temporary credentials"), + fieldType: .secure, + section: .authentication, + visibleWhen: FieldVisibilityRule(fieldId: "awsAuthMethod", values: [DynamoDBAuthMethod.accessKey.rawValue]) + ), + ConnectionField( + id: "awsProfileName", + label: String(localized: "Profile Name"), + placeholder: "default", + section: .authentication, + visibleWhen: FieldVisibilityRule( + fieldId: "awsAuthMethod", + values: [DynamoDBAuthMethod.profile.rawValue, DynamoDBAuthMethod.singleSignOn.rawValue] + ) + ).withDynamicOptions(.awsProfiles), + ConnectionField( + id: "awsRegion", + label: String(localized: "AWS Region"), + placeholder: String(localized: "The profile's region, or us-east-1"), + fieldType: .text, + section: .authentication + ), + ConnectionField( + id: "awsEndpointUrl", + label: String(localized: "Custom Endpoint"), + placeholder: String(localized: "Optional, such as http://localhost:8000"), + section: .authentication + ) + ] + } +} + +enum DynamoDBEditorVocabulary { + static let keywords: Set = [ + "SELECT", "FROM", "WHERE", "INSERT", "INTO", "VALUE", "SET", "REMOVE", "UPDATE", "DELETE", + "AND", "OR", "NOT", "IN", "BETWEEN", "EXISTS", "MISSING", "IS", "NULL", "TRUE", "FALSE", + "ORDER", "BY", "ASC", "DESC", "RETURNING", "ALL", "OLD", "NEW", "MODIFIED" + ] + + static let functions: Set = [ + "begins_with", "contains", "size", "attribute_type", "attribute_exists", "attribute_not_exists", "EXISTS" + ] + + static var completions: [CompletionEntry] { + let partiQL = [ + "SELECT", "INSERT INTO", "UPDATE", "DELETE FROM", "VALUE", "SET", "REMOVE", "WHERE", "AND", "OR", + "BETWEEN", "IN", "IS", "NOT", "NULL", "MISSING", "EXISTS", "ORDER BY", "RETURNING ALL OLD *", + "begins_with", "contains", "size", "attribute_type" + ].map { CompletionEntry(label: $0, insertText: $0) } + let requests: [CompletionEntry] = [ + template("Scan", #"{"TableName": ""}"#), + template("Query", ##"{"TableName": "", "KeyConditionExpression": "#pk = :pk", "##, + ##""ExpressionAttributeNames": {"#pk": ""}, "ExpressionAttributeValues": {":pk": {"S": ""}}}"##), + template("GetItem", #"{"TableName": "", "Key": {"": {"S": ""}}}"#), + template("PutItem", #"{"TableName": "", "Item": {"": {"S": ""}}}"#), + template("UpdateItem", #"{"TableName": "", "Key": {"": {"S": ""}}, "UpdateExpression": ""}"#), + template("DeleteItem", #"{"TableName": "", "Key": {"": {"S": ""}}}"#), + template("DescribeTable", #"{"TableName": ""}"#), + template("CreateTable", #"{"TableName": "", "BillingMode": "PAY_PER_REQUEST", "#, + #""AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], "#, + #""KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}]}"#), + template("UpdateTable", #"{"TableName": ""}"#), + template("UpdateTimeToLive", #"{"TableName": "", "TimeToLiveSpecification": "#, + #"{"Enabled": true, "AttributeName": "expiresAt"}}"#) + ] + return partiQL + requests + } + + private static func template(_ operation: String, _ parts: String...) -> CompletionEntry { + CompletionEntry(label: "\(operation) {…}", insertText: "\(operation) " + parts.joined()) + } +} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+API.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+API.swift new file mode 100644 index 0000000000..d247a7619f --- /dev/null +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+API.swift @@ -0,0 +1,254 @@ +import Foundation +import TableProPluginKit + +extension DynamoDBPluginDriver { + /// Runs one DynamoDB action from the editor or from a statement the driver built. + func callAPI(_ call: DynamoDBAPICall, session: Session) async throws -> PluginQueryResult { + let started = Date() + guard var body = call.body.objectValue else { + throw DynamoDBError.invalidStatement(String(localized: "The request must be a JSON object")) + } + let table = body["TableName"]?.stringValue + switch call.operation { + case .updateTable: + if let table { body = try await completingAttributeDefinitions(body, table: table, session: session) } + case .updateTimeToLive: + if let table { body = try await completingTimeToLive(body, table: table, session: session) } + case .transactWriteItems, .executeTransaction: + if body["ClientRequestToken"] == nil { + body["ClientRequestToken"] = .string(UUID().uuidString) + } + default: + break + } + + switch call.operation { + case .batchWriteItem: + return try await batchWrite(body, session: session, started: started) + case .batchGetItem: + let tables = body["RequestItems"]?.objectValue?.keys.sorted() ?? [] + var items: [DynamoDBItem] = [] + for name in tables { + let request: [String: DynamoDBJSON] = ["RequestItems": .object([name: body["RequestItems"]?[name] ?? .null])] + items += try await batchGet(body: request, table: name, session: session) + } + return Self.queryResult( + items: items, schema: nil, preferredColumns: [], includeAllKeys: false, + started: started, isTruncated: false, statusMessage: nil + ) + case .batchExecuteStatement: + defer { catalog.forgetReadPositions(in: session.scope) } + return try await batchExecute(body, session: session, started: started) + default: + break + } + + let response = try await session.client.send(call.operation, body) + if !call.operation.isRead { + if let table { + catalog.forgetReadPositions(table: table, in: session.scope) + } else { + catalog.forgetReadPositions(in: session.scope) + } + } + if call.operation.changesCatalog || call.operation == .updateTimeToLive || call.operation == .updateContinuousBackups, + let table { + catalog.invalidate(table: table, in: session.scope) + } + return try present(response, operation: call.operation, started: started, session: session, table: table) + } + + private func present( + _ response: DynamoDBJSON, + operation: DynamoDBOperation, + started: Date, + session: Session, + table: String? + ) throws -> PluginQueryResult { + switch operation { + case .getItem: + let items = try response["Item"].map { [try DynamoDBItem(wireItem: $0)] } ?? [] + return Self.queryResult( + items: items, schema: table.flatMap { catalog.schema(for: $0, in: session.scope) }, + preferredColumns: [], includeAllKeys: false, started: started, isTruncated: false, statusMessage: nil + ) + case .transactGetItems: + let items = try (response["Responses"]?.arrayValue ?? []).compactMap { entry in + try entry["Item"].map(DynamoDBItem.init(wireItem:)) + } + return Self.queryResult( + items: items, schema: nil, preferredColumns: [], includeAllKeys: false, + started: started, isTruncated: false, statusMessage: nil + ) + case .executeTransaction: + let items = try (response["Responses"]?.arrayValue ?? []).compactMap { entry in + try entry["Item"].map(DynamoDBItem.init(wireItem:)) + } + guard !items.isEmpty else { + return Self.messageResult(String(localized: "The transaction was applied"), started: started) + } + return Self.queryResult( + items: items, schema: nil, preferredColumns: [], includeAllKeys: false, + started: started, isTruncated: false, statusMessage: nil + ) + case .listTables: + let names = (response["TableNames"]?.arrayValue ?? []).compactMap(\.stringValue) + return PluginQueryResult( + columns: ["TableName"], columnTypeNames: ["String"], rows: names.map { [.text($0)] }, + rowsAffected: 0, timing: PluginQueryTiming(total: Date().timeIntervalSince(started)), + statusMessage: response["LastEvaluatedTableName"] != nil + ? String(localized: "More tables follow. Pass LastEvaluatedTableName as ExclusiveStartTableName.") + : nil + ) + case .putItem, .updateItem, .deleteItem: + let attributes = try response["Attributes"].map { [try DynamoDBItem(wireItem: $0)] } ?? [] + guard attributes.isEmpty else { + return Self.queryResult( + items: attributes, schema: nil, preferredColumns: [], includeAllKeys: false, + started: started, isTruncated: false, statusMessage: nil, rowsAffected: 1 + ) + } + return Self.messageResult( + String(format: String(localized: "%@ succeeded"), operation.rawValue), started: started, rowsAffected: 1 + ) + case .transactWriteItems: + return Self.messageResult(String(localized: "The transaction was applied"), started: started) + case .createTable, .updateTable, .deleteTable: + let status = response["TableDescription"]?["TableStatus"]?.stringValue + let message = status.map { String(format: String(localized: "%1$@ accepted. Table status: %2$@"), operation.rawValue, $0) } + return Self.responseResult(response, started: started, statusMessage: message) + default: + return Self.responseResult(response, started: started, statusMessage: nil) + } + } + + // MARK: - Batches + + /// BatchWriteItem applies what it can and returns the rest as `UnprocessedItems`, inside an HTTP + /// 200. The rest is sent again with backoff; whatever is still left is reported as a failure, + /// never as success. + private func batchWrite(_ body: [String: DynamoDBJSON], session: Session, started: Date) async throws -> PluginQueryResult { + let total = (body["RequestItems"]?.objectValue ?? [:]).values.reduce(0) { $0 + ($1.arrayValue?.count ?? 0) } + defer { + for name in (body["RequestItems"]?.objectValue ?? [:]).keys { + catalog.forgetReadPositions(table: name, in: session.scope) + } + } + var pending = body["RequestItems"] + var attempt = 0 + while let requestItems = pending, requestItems.objectValue?.isEmpty == false { + let response = try await session.client.send(.batchWriteItem, ["RequestItems": requestItems]) + pending = response["UnprocessedItems"] + let left = (pending?.objectValue ?? [:]).values.reduce(0) { $0 + ($1.arrayValue?.count ?? 0) } + guard left > 0 else { break } + attempt += 1 + guard attempt < 10 else { + throw DynamoDBError.partialBatch( + applied: total - left, total: total, + failures: [String(format: String(localized: "%d requests were not processed after 10 attempts"), left)] + ) + } + try await session.client.backOff(afterAttempt: attempt) + } + return Self.messageResult( + String(format: String(localized: "%d requests applied"), total), started: started, rowsAffected: total + ) + } + + /// BatchExecuteStatement answers each statement on its own inside an HTTP 200, so a statement + /// that failed is found in the response, not in the status. + private func batchExecute(_ body: [String: DynamoDBJSON], session: Session, started: Date) async throws -> PluginQueryResult { + let response = try await session.client.send(.batchExecuteStatement, body) + let responses = response["Responses"]?.arrayValue ?? [] + let failures = responses.enumerated().compactMap { index, entry -> String? in + guard let error = entry["Error"] else { return nil } + let code = error["Code"]?.stringValue ?? "Error" + let message = error["Message"]?.stringValue ?? "" + return String(format: String(localized: "Statement %1$d: %2$@ %3$@"), index + 1, code, message) + } + guard failures.isEmpty else { + throw DynamoDBError.partialBatch(applied: responses.count - failures.count, total: responses.count, failures: failures) + } + let items = try responses.compactMap { try $0["Item"].map(DynamoDBItem.init(wireItem:)) } + guard items.isEmpty else { + return Self.queryResult( + items: items, schema: nil, preferredColumns: [], includeAllKeys: false, + started: started, isTruncated: false, statusMessage: nil + ) + } + return Self.messageResult( + String(format: String(localized: "%d statements applied"), responses.count), + started: started, rowsAffected: responses.count + ) + } + + // MARK: - Request completion + + /// An UpdateTable that creates a global secondary index has to declare the type of each key + /// attribute the table does not already declare. The Structure tab's index editor has no type + /// to give, so a missing declaration is filled from the table and from the attribute's type in + /// the items, and a String when nothing says otherwise. + func completingAttributeDefinitions( + _ body: [String: DynamoDBJSON], + table: String, + session: Session + ) async throws -> [String: DynamoDBJSON] { + let creates = (body["GlobalSecondaryIndexUpdates"]?.arrayValue ?? []).compactMap { $0["Create"] } + guard !creates.isEmpty else { return body } + var declared = Set((body["AttributeDefinitions"]?.arrayValue ?? []).compactMap { $0["AttributeName"]?.stringValue }) + var definitions = body["AttributeDefinitions"]?.arrayValue ?? [] + let schema = try await tableSchema(table, session: session) + var observed = catalog.columnTypes(for: table, in: session.scope) + let needed = creates.flatMap { ($0["KeySchema"]?.arrayValue ?? []).compactMap { $0["AttributeName"]?.stringValue } } + if needed.contains(where: { schema.keyType(of: $0) == nil && observed[$0] == nil }) { + observed.merge(try await sampleTypes(table: table, schema: schema, session: session)) { current, _ in current } + } + for attribute in needed where !declared.contains(attribute) { + let known = schema.keyType(of: attribute) ?? observed[attribute] + guard let type = known, type.isKeyType else { + throw DynamoDBError.invalidStatement(String(format: String( + localized: "No item read so far holds \"%@\" as a key type. Create this index from the editor with its AttributeDefinitions." + ), attribute)) + } + definitions.append(.object([ + "AttributeName": .string(attribute), + "AttributeType": .string(type.rawValue) + ])) + declared.insert(attribute) + } + var completed = body + completed["AttributeDefinitions"] = .array(definitions) + if !schema.isOnDemand, let updates = body["GlobalSecondaryIndexUpdates"]?.arrayValue { + let capacity: DynamoDBJSON = .object([ + "ReadCapacityUnits": .number(String(schema.readCapacity ?? 1)), + "WriteCapacityUnits": .number(String(schema.writeCapacity ?? 1)) + ]) + completed["GlobalSecondaryIndexUpdates"] = .array(updates.map { update in + guard var create = update["Create"]?.objectValue, create["ProvisionedThroughput"] == nil else { return update } + create["ProvisionedThroughput"] = capacity + return .object(["Create": .object(create)]) + }) + } + return completed + } + + /// Turning Time to Live off still has to name the attribute it was on, which the Maintenance + /// menu cannot know when it builds the statement. + func completingTimeToLive( + _ body: [String: DynamoDBJSON], + table: String, + session: Session + ) async throws -> [String: DynamoDBJSON] { + guard var specification = body["TimeToLiveSpecification"]?.objectValue, + specification["AttributeName"] == nil + else { return body } + let response = try await session.client.send(.describeTimeToLive, ["TableName": .string(table)]) + guard let attribute = response["TimeToLiveDescription"]?["AttributeName"]?.stringValue else { + throw DynamoDBError.invalidStatement(String(localized: "Time to Live is not on for this table")) + } + specification["AttributeName"] = .string(attribute) + var completed = body + completed["TimeToLiveSpecification"] = .object(specification) + return completed + } +} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Execution.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Execution.swift new file mode 100644 index 0000000000..d0a591fcf9 --- /dev/null +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Execution.swift @@ -0,0 +1,607 @@ +import Foundation +import TableProPluginKit + +extension DynamoDBPluginDriver { + func execute(query: String) async throws -> PluginQueryResult { + try await runStatement(query, parameters: [], rowCap: PluginRowLimits.emergencyMax) + } + + func executeParameterized(query: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { + try await runStatement(query, parameters: parameters, rowCap: PluginRowLimits.emergencyMax) + } + + func executeUserQuery(query: String, rowCap: Int?, parameters: [PluginCellValue]?) async throws -> PluginQueryResult { + try await runStatement(query, parameters: parameters ?? [], rowCap: rowCap ?? PluginRowLimits.emergencyMax) + } + + func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { + try await runStatement(query, parameters: [], rowCap: rowCap) + } + + private func runStatement(_ text: String, parameters: [PluginCellValue], rowCap: Int) async throws -> PluginQueryResult { + let statement = try DynamoDBStatement.parse(text) + switch statement { + case .browse(let request, let window): + return try await run { session in + try await self.readBrowse(request, window: window, rowCap: rowCap, session: session) + } + case .apiCall(let call, let window): + guard parameters.isEmpty else { throw Self.parametersNeedPartiQL } + if call.operation == .scan || call.operation == .query { + return try await run { session in + try await self.readRequest(call, window: window, rowCap: rowCap, session: session) + } + } + return try await run(boundedByQueryTimeout: call.operation.isRead) { session in + try await self.callAPI(call, session: session) + } + case .partiQL(let statementText, let window): + if DynamoDBPartiQL.kind(of: statementText) == .select { + return try await run { session in + try await self.readPartiQL( + statementText, window: window, parameters: parameters, rowCap: rowCap, session: session + ) + } + } + return try await run(boundedByQueryTimeout: false) { session in + try await self.writePartiQL(statementText, parameters: parameters, session: session) + } + } + } + + static var parametersNeedPartiQL: DynamoDBError { + .invalidStatement(String(localized: "Parameters apply only to PartiQL statements")) + } + + // MARK: - Browse + + func readBrowse( + _ request: DynamoDBBrowseRequest, + window: DynamoDBReadWindow, + rowCap: Int, + session: Session + ) async throws -> PluginQueryResult { + let started = Date() + let schema = try await tableSchema(request.table, session: session) + guard !schema.isBeingCreated else { + return Self.queryResult( + items: [], schema: schema, preferredColumns: request.columns, includeAllKeys: true, + started: started, isTruncated: false, + statusMessage: String(localized: "DynamoDB is still creating this table. Refresh in a moment.") + ) + } + let plan = try DynamoDBAccessPlanner(schema: schema).plan(request, order: window.order) + let (wanted, readLimit) = Self.readLimits(window: window, rowCap: rowCap) + var items: [DynamoDBItem] = [] + let stats = try await readPlan( + plan, schema: schema, offset: window.offset, limit: readLimit, + fingerprint: fingerprint(of: .browse(request, window: window)) + plan.positionKey, session: session + ) { item in + items.append(item) + return true + } + let ordered = order(items, by: plan.unsatisfiedOrder, isComplete: window.offset == 0 && stats.reachedEnd) + .truncated(to: wanted) + let isTruncated = items.count > wanted + rememberTypes(of: ordered.items, table: schema.name, schema: schema, session: session) + var visibleStats = stats + visibleStats.returned = ordered.items.count + return Self.queryResult( + items: ordered.items, + schema: schema, + preferredColumns: request.columns, + includeAllKeys: true, + started: started, + isTruncated: isTruncated, + statusMessage: Self.statusMessage( + access: plan.summary, stats: visibleStats, extra: ordered.note.map { [$0] } ?? [] + ) + ) + } + + /// Applies sort terms DynamoDB could not. Only a result held in full can be sorted: the first + /// page of a sorted table is not the first page re-ordered. + struct OrderedItems { + var items: [DynamoDBItem] + let note: String? + + func truncated(to count: Int) -> OrderedItems { + OrderedItems(items: Array(items.prefix(max(count, 0))), note: note) + } + } + + /// Applies sort terms DynamoDB could not, before any row is cut: only a result held in full can + /// be sorted, because the first page of a sorted table is not the first page re-ordered. + func order(_ items: [DynamoDBItem], by terms: [DynamoDBOrderTerm], isComplete: Bool) -> OrderedItems { + guard !terms.isEmpty else { return OrderedItems(items: items, note: nil) } + guard isComplete else { + let names = terms.map(\.attribute).joined(separator: ", ") + return OrderedItems(items: items, note: String(format: String( + localized: "In DynamoDB order: sorting by %@ needs the whole result, or a Query on that sort key" + ), names)) + } + return OrderedItems(items: DynamoDBItemTable.sorted(items, by: terms), note: nil) + } + + /// How many rows to show and how many to read: one more than the cap, so a result that does not + /// fit is known to be truncated. + static func readLimits(window: DynamoDBReadWindow, rowCap: Int) -> (wanted: Int, read: Int) { + let cap = max(rowCap, 0) + guard let limit = window.limit else { return (cap, cap.addingReportingOverflow(1).overflow ? cap : cap + 1) } + let clamped = max(limit, 0) + return clamped <= cap ? (clamped, clamped) : (cap, cap + 1) + } + + func fingerprint(of statement: DynamoDBStatement) -> String { + switch statement { + case .browse(let request, let window): + let planOnly = DynamoDBBrowseRequest( + table: request.table, filters: request.filters, matchAll: request.matchAll, columns: [] + ) + return DynamoDBStatement.browse(planOnly, window: DynamoDBReadWindow(order: window.order)).text + case .apiCall(let call, let window): + return DynamoDBStatement.apiCall(call, window: DynamoDBReadWindow(order: window.order)).text + case .partiQL(let text, let window): + return DynamoDBStatement.partiQL(text: text, window: DynamoDBReadWindow(order: window.order)).text + } + } + + func rememberTypes(of items: [DynamoDBItem], table: String, schema: DynamoDBTableSchema?, session: Session) { + let observed = DynamoDBItemTable(items: items, schema: schema).observedTypes + catalog.mergeColumnTypes(observed, for: table, in: session.scope) + } + + // MARK: - Scan and Query requests + + func readRequest( + _ call: DynamoDBAPICall, + window: DynamoDBReadWindow, + rowCap: Int, + session: Session + ) async throws -> PluginQueryResult { + let started = Date() + guard let body = call.body.objectValue, let table = body["TableName"]?.stringValue else { + throw DynamoDBError.invalidStatement(String(localized: "The request needs a TableName")) + } + if body["Select"]?.stringValue == "COUNT" { + return try await countRequest(call.operation, body: body, session: session, started: started) + } + let schema = try? await tableSchema(table, session: session) + let plan = DynamoDBReadPlan( + table: table, + access: call.operation == .query ? .query : .scan, + indexName: body["IndexName"]?.stringValue, + requests: [body], + clientPredicates: [], + clientMatchAll: true, + unsatisfiedOrder: window.order + ) + let (wanted, readLimit) = Self.readLimits(window: window, rowCap: rowCap) + var items: [DynamoDBItem] = [] + let stats = try await readPlan( + plan, schema: schema, offset: window.offset, limit: readLimit, + fingerprint: fingerprint(of: .apiCall(call, window: window)), session: session + ) { item in + items.append(item) + return true + } + let ordered = order(items, by: window.order, isComplete: window.offset == 0 && stats.reachedEnd) + .truncated(to: wanted) + let isTruncated = items.count > wanted + var visibleStats = stats + visibleStats.returned = ordered.items.count + return Self.queryResult( + items: ordered.items, schema: schema, preferredColumns: [], includeAllKeys: false, + started: started, isTruncated: isTruncated, + statusMessage: Self.statusMessage( + access: call.operation.rawValue, stats: visibleStats, extra: ordered.note.map { [$0] } ?? [] + ) + ) + } + + private func countRequest( + _ operation: DynamoDBOperation, + body: [String: DynamoDBJSON], + session: Session, + started: Date + ) async throws -> PluginQueryResult { + var total = 0 + var scanned = 0 + var startKey: DynamoDBJSON? + repeat { + try session.checkDeadline() + var page = body + if let startKey { page["ExclusiveStartKey"] = startKey } + let response = try await session.client.send(operation, page) + total += response["Count"]?.intValue ?? 0 + scanned += response["ScannedCount"]?.intValue ?? 0 + startKey = response["LastEvaluatedKey"] + } while startKey != nil + return PluginQueryResult( + columns: ["Count", "ScannedCount"], + columnTypeNames: ["Number", "Number"], + rows: [[.text(String(total)), .text(String(scanned))]], + rowsAffected: 0, + timing: PluginQueryTiming(total: Date().timeIntervalSince(started)) + ) + } + + // MARK: - PartiQL reads + + func readPartiQL( + _ text: String, + window: DynamoDBReadWindow, + parameters: [PluginCellValue], + rowCap: Int, + session: Session + ) async throws -> PluginQueryResult { + let started = Date() + let target = DynamoDBPartiQL.target(of: text) + let schema = await target.asyncMap { try? await tableSchema($0.table, session: session) } ?? nil + var statement = text + var clientOrder = window.order + if let serverOrder = Self.serverOrder(for: text, target: target, schema: schema, order: window.order) { + statement += "\n" + serverOrder + clientOrder = [] + } + + var body: [String: DynamoDBJSON] = [ + "Statement": .string(statement), + "ReturnConsumedCapacity": .string("TOTAL") + ] + if !parameters.isEmpty { + let binder = DynamoDBParameterBinder( + schema: schema, + observedTypes: target.map { catalog.columnTypes(for: $0.table, in: session.scope) } ?? [:], + currentItem: nil + ) + let bound = try binder.bind(parameters, roles: DynamoDBPartiQL.parameterRoles(in: text)) + body["Parameters"] = .array(bound.map(\.wireJSON)) + } + + let (wanted, readLimit) = Self.readLimits(window: window, rowCap: rowCap) + var items: [DynamoDBItem] = [] + var stats = DynamoDBReadStats() + var skipped = 0 + var nextToken: String? + readLoop: while readLimit > 0 { + try session.checkDeadline() + var page = body + if let nextToken { page["NextToken"] = .string(nextToken) } + let (remaining, overflowed) = max(window.offset - skipped, 0).addingReportingOverflow(readLimit - items.count) + page["Limit"] = .number(String(overflowed ? 1_000 : min(max(remaining, 1), 1_000))) + let response = try await session.client.send(.executeStatement, page) + let pageItems = try (response["Items"]?.arrayValue ?? []).map(DynamoDBItem.init(wireItem:)) + let followingToken = response["NextToken"]?.stringValue + stats.scanned += pageItems.count + if let units = Self.readUnits(in: response) { + stats.readUnits += units + stats.reportedReadUnits = true + } + for (position, item) in pageItems.enumerated() { + if skipped < window.offset { + skipped += 1 + continue + } + items.append(item) + if items.count >= readLimit { + stats.reachedEnd = followingToken == nil && position == pageItems.count - 1 + break readLoop + } + } + guard let followingToken else { + stats.reachedEnd = true + break + } + nextToken = followingToken + } + let ordered = order(items, by: clientOrder, isComplete: window.offset == 0 && stats.reachedEnd) + .truncated(to: wanted) + let isTruncated = items.count > wanted + stats.returned = ordered.items.count + stats.scanned = stats.returned + if let target { + rememberTypes(of: ordered.items, table: target.table, schema: schema, session: session) + } + let readsWholeTable = target?.index == nil && schema.map { schema in + !schema.keys.partition.allSatisfy { DynamoDBPartiQL.whereFixes($0, in: text) } + } == true + let scanNote = readsWholeTable ? [String(localized: "This SELECT reads the whole table")] : [] + return Self.queryResult( + items: ordered.items, schema: schema, preferredColumns: [], includeAllKeys: false, + started: started, isTruncated: isTruncated, + statusMessage: Self.statusMessage( + access: nil, stats: stats, extra: scanNote + (ordered.note.map { [$0] } ?? []) + ) + ) + } + + /// The ORDER BY DynamoDB can run itself: one term naming the single sort key of the table or of + /// the index the statement reads, with every partition key attribute fixed by `=`. + static func serverOrder( + for text: String, + target: (table: String, index: String?)?, + schema: DynamoDBTableSchema?, + order: [DynamoDBOrderTerm] + ) -> String? { + guard let schema, let target, order.count == 1 else { return nil } + let keys: DynamoDBKeySchema + if let indexName = target.index { + guard let index = schema.index(named: indexName) else { return nil } + keys = index.keys + } else { + keys = schema.keys + } + guard keys.sort.count == 1, order[0].attribute == keys.sort[0], + keys.partition.allSatisfy({ DynamoDBPartiQL.whereFixes($0, in: text, acceptsIn: false) }) + else { return nil } + return "ORDER BY \(DynamoDBStatement.quote(keys.sort[0]))\(order[0].descending ? " DESC" : " ASC")" + } + + // MARK: - Streaming + + /// Streams every item a read returns, for an export, a copy or a compare. + /// + /// A stream states its columns once, before its first row, and a DynamoDB item may carry an + /// attribute no earlier item had. The items are therefore written to a temporary file while the + /// union of their attributes is collected, and the rows are sent once the read has finished, so + /// an attribute first seen on the last page still gets its column. + func streamRows(query: String) -> AsyncThrowingStream { + AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + let task = Task { + do { + try await self.run { session in + try await self.spool(query, session: session, continuation: continuation) + } + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + continuation.onTermination = { _ in task.cancel() } + } + } + + private func spool( + _ text: String, + session: Session, + continuation: AsyncThrowingStream.Continuation + ) async throws { + let statement = try DynamoDBStatement.parse(text) + let source: SpoolSource + do { + source = try await spoolSource(for: statement, session: session) + } catch DynamoDBError.invalidStatement(let message) where message == Self.notAReadMarker { + let result = try await runStatement(text, parameters: [], rowCap: PluginRowLimits.emergencyMax) + continuation.yield(.header(PluginStreamHeader(columns: result.columns, columnTypeNames: result.columnTypeNames))) + if !result.rows.isEmpty { continuation.yield(.rows(result.rows)) } + return + } + + let spoolURL = FileManager.default.temporaryDirectory + .appendingPathComponent("TablePro-DynamoDB-\(UUID().uuidString).jsonl") + guard FileManager.default.createFile(atPath: spoolURL.path, contents: nil) else { + throw DynamoDBError.transport(String(localized: "Could not create a temporary file for the export")) + } + defer { try? FileManager.default.removeItem(at: spoolURL) } + + let sortsLocally = !source.clientOrder.isEmpty + let readOffset = sortsLocally ? 0 : source.window.offset + let readLimit = sortsLocally ? Int.max : max(source.window.limit ?? Int.max, 0) + var collector = SpoolCollector(handle: try FileHandle(forWritingTo: spoolURL)) + try await source.read(readOffset, readLimit) { item in + try collector.add(item) + return true + } + try collector.handle.close() + + let columns = collector.columns(preferred: source.preferredColumns, schema: source.schema) + continuation.yield(.header(PluginStreamHeader(columns: columns, columnTypeNames: collector.typeNames(for: columns, schema: source.schema)))) + + if sortsLocally { + var items: [DynamoDBItem] = [] + var lines = try SpoolLines(url: spoolURL) + defer { lines.close() } + while let line = try lines.next() { + try Task.checkCancellation() + items.append(try DynamoDBItem(wireItem: DynamoDBJSON.parse(line))) + } + let sorted = DynamoDBItemTable.sorted(items, by: source.clientOrder) + let end = source.window.limit.map { limit in + let (sum, overflowed) = source.window.offset.addingReportingOverflow(max(limit, 0)) + return overflowed ? sorted.count : min(sorted.count, sum) + } ?? sorted.count + let slice = source.window.offset < end ? Array(sorted[source.window.offset.. Bool) async throws -> Void + } + + private func spoolSource(for statement: DynamoDBStatement, session: Session) async throws -> SpoolSource { + switch statement { + case .browse(let request, let window): + let schema = try await tableSchema(request.table, session: session) + let plan = try DynamoDBAccessPlanner(schema: schema).plan(request, order: window.order) + let fingerprint = fingerprint(of: statement) + plan.positionKey + return SpoolSource( + window: window, clientOrder: plan.unsatisfiedOrder, schema: schema, preferredColumns: request.columns + ) { offset, limit, sink in + _ = try await self.readPlan( + plan, schema: schema, offset: offset, limit: limit, fingerprint: fingerprint, session: session, sink: sink + ) + } + case .apiCall(let call, let window) where call.operation == .scan || call.operation == .query: + guard let body = call.body.objectValue, let table = body["TableName"]?.stringValue else { + throw DynamoDBError.invalidStatement(String(localized: "The request needs a TableName")) + } + let schema = try? await tableSchema(table, session: session) + let plan = DynamoDBReadPlan( + table: table, access: call.operation == .query ? .query : .scan, + indexName: body["IndexName"]?.stringValue, requests: [body], + clientPredicates: [], clientMatchAll: true, unsatisfiedOrder: window.order + ) + let fingerprint = fingerprint(of: statement) + return SpoolSource(window: window, clientOrder: window.order, schema: schema, preferredColumns: []) { offset, limit, sink in + _ = try await self.readPlan( + plan, schema: schema, offset: offset, limit: limit, fingerprint: fingerprint, session: session, sink: sink + ) + } + case .partiQL(let statementText, let window) where DynamoDBPartiQL.kind(of: statementText) == .select: + let target = DynamoDBPartiQL.target(of: statementText) + let schema = await target.asyncMap { try? await tableSchema($0.table, session: session) } ?? nil + let serverOrder = Self.serverOrder(for: statementText, target: target, schema: schema, order: window.order) + let sent = serverOrder.map { statementText + "\n" + $0 } ?? statementText + return SpoolSource( + window: window, clientOrder: serverOrder == nil ? window.order : [], schema: schema, preferredColumns: [] + ) { offset, limit, sink in + try await self.readPartiQLPages(sent, offset: offset, limit: limit, session: session, sink: sink) + } + default: + throw DynamoDBError.invalidStatement(Self.notAReadMarker) + } + } + + private func readPartiQLPages( + _ statement: String, + offset: Int, + limit: Int, + session: Session, + sink: (DynamoDBItem) throws -> Bool + ) async throws { + var skipped = 0 + var sent = 0 + var nextToken: String? + repeat { + try session.checkDeadline() + var body: [String: DynamoDBJSON] = ["Statement": .string(statement)] + if let nextToken { body["NextToken"] = .string(nextToken) } + let response = try await session.client.send(.executeStatement, body) + for itemJSON in response["Items"]?.arrayValue ?? [] { + if skipped < offset { + skipped += 1 + continue + } + guard sent < limit, try sink(try DynamoDBItem(wireItem: itemJSON)) else { return } + sent += 1 + } + nextToken = sent < limit ? response["NextToken"]?.stringValue : nil + } while nextToken != nil + } +} + +/// Reads the spool file back one line at a time with plain reads. +/// +/// `URL.lines` would read it through `FileHandle.AsyncBytes`, and Foundation serves every AsyncBytes reader in the +/// process from one serial queue. A reader blocked on a pipe that stays quiet, such as a language server's output, +/// holds that queue, and the export then waits for it. +private struct SpoolLines { + private static let chunkSize = 1 << 20 + + private let handle: FileHandle + private var buffer = Data() + private var start = 0 + private var reachedEnd = false + + init(url: URL) throws { + handle = try FileHandle(forReadingFrom: url) + } + + mutating func next() throws -> Data? { + while true { + if let newline = buffer[start...].firstIndex(of: UInt8(ascii: "\n")) { + let line = buffer[start..() + private var typeCounts: [String: [DynamoDBAttributeType: Int]] = [:] + + init(handle: FileHandle) { + self.handle = handle + } + + mutating func add(_ item: DynamoDBItem) throws { + try handle.write(contentsOf: Data((item.wireJSON.serialized() + "\n").utf8)) + for (name, value) in item { + attributes.insert(name) + guard value != .null else { continue } + typeCounts[name, default: [:]][value.type, default: 0] += 1 + } + } + + func columns(preferred: [String], schema: DynamoDBTableSchema?) -> [String] { + var columns = preferred + for name in schema?.keys.attributes ?? [] where !columns.contains(name) && attributes.contains(name) { + columns.append(name) + } + return columns + attributes.subtracting(columns).sorted() + } + + func typeNames(for columns: [String], schema: DynamoDBTableSchema?) -> [String] { + columns.map { column in + if schema?.keys.attributes.contains(column) == true, let keyType = schema?.keyType(of: column) { + return keyType.displayName + } + let counts = typeCounts[column] ?? [:] + return counts.max { $0.value < $1.value }?.key.displayName ?? DynamoDBAttributeType.string.displayName + } + } +} + +extension Optional { + func asyncMap(_ transform: (Wrapped) async throws -> T) async rethrows -> T? { + guard let value = self else { return nil } + return try await transform(value) + } +} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Reading.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Reading.swift new file mode 100644 index 0000000000..99d8f10f28 --- /dev/null +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Reading.swift @@ -0,0 +1,279 @@ +import Foundation +import TableProPluginKit + +struct DynamoDBReadStats: Sendable, Equatable { + var returned = 0 + var scanned = 0 + var readUnits: Double = 0 + var reportedReadUnits = false + var reachedEnd = false +} + +extension DynamoDBPluginDriver { + /// Reads the items of `plan` from match number `offset`, up to `limit` of them, handing each to + /// `sink`. The sink returns false to stop early. + /// + /// DynamoDB applies `Limit` before a FilterExpression, so an empty page with a + /// `LastEvaluatedKey` is not the end: the reader keeps reading until it has the items it was + /// asked for or DynamoDB has nothing left. Every page boundary it passes is remembered as the + /// key of the item before it, so the next page starts there instead of from the first item. + func readPlan( + _ plan: DynamoDBReadPlan, + schema: DynamoDBTableSchema?, + offset: Int, + limit: Int, + fingerprint: String, + session: Session, + sink: (DynamoDBItem) throws -> Bool + ) async throws -> DynamoDBReadStats { + var stats = DynamoDBReadStats() + guard plan.access != .nothing, limit > 0 else { + stats.reachedEnd = true + return stats + } + let index = plan.indexName.flatMap { schema?.index(named: $0) } + var point = DynamoDBResumePoint.start + var matched = 0 + if offset > 0, let cached = catalog.nearestResumePoint( + table: plan.table, fingerprint: fingerprint, atOrBefore: offset, in: session.scope + ) { + point = cached.point + matched = cached.offset + } + let pageSize = max(limit, 1) + + while point.requestIndex < plan.requests.count { + try session.checkDeadline() + var body = plan.requests[point.requestIndex] + if let startKey = point.startKey { + body["ExclusiveStartKey"] = startKey.wireJSON + } + if plan.access != .batchGet { + body["ReturnConsumedCapacity"] = .string("TOTAL") + if !plan.hasFilters, body["Limit"] == nil { + let (wanted, overflowed) = max(offset - matched, 0).addingReportingOverflow(limit - stats.returned) + body["Limit"] = .number(String(overflowed ? 1_000 : min(max(wanted, 1), 1_000))) + } + } + let page = try await fetchPage(plan: plan, body: body, session: session) + stats.scanned += page.scanned + stats.readUnits += page.readUnits + stats.reportedReadUnits = stats.reportedReadUnits || page.reportedReadUnits + + for (position, item) in page.items.enumerated() where position >= point.skip { + guard plan.clientMatches(item) else { continue } + let isLastOfResponse = position == page.items.count - 1 + if matched >= offset { + guard try sink(item) else { return stats } + stats.returned += 1 + } + matched += 1 + let next = resumePoint( + after: item, position: position, isLastOfResponse: isLastOfResponse, + page: page, plan: plan, point: point, schema: schema, index: index + ) + if let next, matched % pageSize == 0 { + catalog.storeResumePoint( + next, table: plan.table, fingerprint: fingerprint, offset: matched, in: session.scope + ) + } + if stats.returned >= limit { + stats.reachedEnd = isLastOfResponse + && page.lastEvaluatedKey == nil + && point.requestIndex == plan.requests.count - 1 + return stats + } + } + if let lastKey = page.lastEvaluatedKey { + point = DynamoDBResumePoint(requestIndex: point.requestIndex, startKey: lastKey, skip: 0) + } else { + point = DynamoDBResumePoint(requestIndex: point.requestIndex + 1, startKey: nil, skip: 0) + } + } + stats.reachedEnd = true + return stats + } + + private func resumePoint( + after item: DynamoDBItem, + position: Int, + isLastOfResponse: Bool, + page: DynamoDBPage, + plan: DynamoDBReadPlan, + point: DynamoDBResumePoint, + schema: DynamoDBTableSchema?, + index: DynamoDBIndex? + ) -> DynamoDBResumePoint? { + if plan.access == .batchGet { + guard isLastOfResponse else { + return DynamoDBResumePoint(requestIndex: point.requestIndex, startKey: nil, skip: position + 1) + } + return DynamoDBResumePoint(requestIndex: point.requestIndex + 1, startKey: nil, skip: 0) + } + if isLastOfResponse, page.lastEvaluatedKey == nil { + return DynamoDBResumePoint(requestIndex: point.requestIndex + 1, startKey: nil, skip: 0) + } + guard let key = schema?.startKey(for: item, index: index) else { return nil } + return DynamoDBResumePoint(requestIndex: point.requestIndex, startKey: key, skip: 0) + } + + struct DynamoDBPage { + let items: [DynamoDBItem] + let lastEvaluatedKey: DynamoDBItem? + let scanned: Int + let readUnits: Double + let reportedReadUnits: Bool + } + + private func fetchPage(plan: DynamoDBReadPlan, body: [String: DynamoDBJSON], session: Session) async throws -> DynamoDBPage { + switch plan.access { + case .batchGet: + let items = try await batchGet(body: body, table: plan.table, session: session) + return DynamoDBPage(items: items, lastEvaluatedKey: nil, scanned: items.count, readUnits: 0, reportedReadUnits: false) + case .query, .scan, .nothing: + let operation: DynamoDBOperation = plan.access == .query ? .query : .scan + let response = try await session.client.send(operation, body) + let items = try (response["Items"]?.arrayValue ?? []).map(DynamoDBItem.init(wireItem:)) + let lastKey = try response["LastEvaluatedKey"].map(DynamoDBItem.init(wireItem:)) + let capacity = Self.readUnits(in: response) + return DynamoDBPage( + items: items, + lastEvaluatedKey: lastKey, + scanned: response["ScannedCount"]?.intValue ?? items.count, + readUnits: capacity ?? 0, + reportedReadUnits: capacity != nil + ) + } + } + + /// BatchGetItem returns what it could read and names the rest in `UnprocessedKeys`, which are + /// sent again until none are left. Items come back in no order, so they are put back in the + /// order their keys were asked for. + func batchGet(body: [String: DynamoDBJSON], table: String, session: Session) async throws -> [DynamoDBItem] { + var pending: DynamoDBJSON? = body["RequestItems"] + var collected: [DynamoDBItem] = [] + var attempt = 0 + while let requestItems = pending, requestItems.objectValue?.isEmpty == false { + try session.checkDeadline() + let response = try await session.client.send(.batchGetItem, ["RequestItems": requestItems]) + for (_, items) in response["Responses"]?.objectValue ?? [:] { + collected += try (items.arrayValue ?? []).map(DynamoDBItem.init(wireItem:)) + } + pending = response["UnprocessedKeys"] + guard pending?.objectValue?.isEmpty == false else { break } + attempt += 1 + guard attempt < 10 else { + let requested = (body["RequestItems"]?.objectValue ?? [:]).values.reduce(0) { + $0 + ($1["Keys"]?.arrayValue?.count ?? 0) + } + throw DynamoDBError.partialBatch( + applied: collected.count, total: requested, + failures: [String(localized: "DynamoDB left some keys unread after 10 attempts")] + ) + } + try await session.client.backOff(afterAttempt: attempt) + } + let requested = body["RequestItems"]?[table]?["Keys"]?.arrayValue ?? [] + let order = try requested.map(DynamoDBItem.init(wireItem:)) + return collected.sorted { lhs, rhs in + let left = order.firstIndex { key in key.allSatisfy { lhs[$0.key] == $0.value } } ?? Int.max + let right = order.firstIndex { key in key.allSatisfy { rhs[$0.key] == $0.value } } ?? Int.max + return left < right + } + } + + static func readUnits(in response: DynamoDBJSON) -> Double? { + if let single = response["ConsumedCapacity"]?["CapacityUnits"]?.doubleValue { + return single + } + guard let list = response["ConsumedCapacity"]?.arrayValue, !list.isEmpty else { return nil } + return list.compactMap { $0["CapacityUnits"]?.doubleValue }.reduce(0, +) + } + + // MARK: - Results + + static func statusMessage( + access: String?, + stats: DynamoDBReadStats, + extra: [String] = [] + ) -> String { + var parts: [String] = [] + if let access { parts.append(access) } + let formatter = NumberFormatter() + formatter.numberStyle = .decimal + formatter.maximumFractionDigits = 1 + let returned = formatter.string(from: NSNumber(value: stats.returned)) ?? "\(stats.returned)" + let scanned = formatter.string(from: NSNumber(value: stats.scanned)) ?? "\(stats.scanned)" + parts.append(String(format: String(localized: "%@ returned"), returned)) + if stats.scanned != stats.returned { + parts.append(String(format: String(localized: "%@ read"), scanned)) + } + if stats.reportedReadUnits { + let units = formatter.string(from: NSNumber(value: stats.readUnits)) ?? "\(stats.readUnits)" + parts.append(String(format: String(localized: "%@ RCU"), units)) + } + return (parts + extra).joined(separator: " · ") + } + + static func queryResult( + items: [DynamoDBItem], + schema: DynamoDBTableSchema?, + preferredColumns: [String], + includeAllKeys: Bool, + started: Date, + isTruncated: Bool, + statusMessage: String?, + rowsAffected: Int = 0 + ) -> PluginQueryResult { + let table = DynamoDBItemTable( + items: items, + schema: schema, + preferredColumns: preferredColumns, + includeAllKeys: includeAllKeys + ) + guard !table.columns.isEmpty else { + return PluginQueryResult( + columns: [], columnTypeNames: [], rows: [], rowsAffected: rowsAffected, + timing: PluginQueryTiming(total: Date().timeIntervalSince(started)), + isTruncated: isTruncated, statusMessage: statusMessage + ) + } + return PluginQueryResult( + columns: table.columns, + columnTypeNames: table.typeNames, + rows: table.rows, + rowsAffected: rowsAffected, + timing: PluginQueryTiming(total: Date().timeIntervalSince(started)), + isTruncated: isTruncated, + statusMessage: statusMessage, + columnMeta: table.columnMeta(schema: schema) + ) + } + + static func messageResult(_ message: String, started: Date, rowsAffected: Int = 0) -> PluginQueryResult { + PluginQueryResult( + columns: [], columnTypeNames: [], rows: [], rowsAffected: rowsAffected, + timing: PluginQueryTiming(total: Date().timeIntervalSince(started)), + statusMessage: message + ) + } + + /// A response with no items, shown as its JSON in one cell the JSON viewer opens. + static func responseResult(_ response: DynamoDBJSON, started: Date, statusMessage: String?) -> PluginQueryResult { + let column = PluginColumnInfo( + name: "Response", dataType: "JSON", isNullable: true, isPrimaryKey: false, defaultValue: nil, + extra: nil, charset: nil, collation: nil, comment: nil, identityKind: nil, isGenerated: false, + allowedValues: nil, generationExpression: nil, generationKind: nil, ddlSpelling: nil, + ddlDefault: nil, ddlGenerationExpression: nil, ddlCollation: nil, classificationTypeName: "JSON" + ) + return PluginQueryResult( + columns: ["Response"], + columnTypeNames: ["JSON"], + rows: [[.text(response.serialized(pretty: true))]], + rowsAffected: 0, + timing: PluginQueryTiming(total: Date().timeIntervalSince(started)), + statusMessage: statusMessage, + columnMeta: [column] + ) + } +} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Schema.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Schema.swift new file mode 100644 index 0000000000..e81816c05b --- /dev/null +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Schema.swift @@ -0,0 +1,321 @@ +import Foundation +import TableProPluginKit + +extension DynamoDBPluginDriver { + static let columnSampleSize = 100 + + func fetchTables(schema: String?) async throws -> [PluginTableInfo] { + try await run(isUserOperation: false) { session in + var names: [String] = [] + var start: String? + repeat { + try session.checkDeadline() + var body: [String: DynamoDBJSON] = ["Limit": .number("100")] + if let start { body["ExclusiveStartTableName"] = .string(start) } + let response = try await session.client.send(.listTables, body) + names += (response["TableNames"]?.arrayValue ?? []).compactMap(\.stringValue) + start = response["LastEvaluatedTableName"]?.stringValue + } while start != nil + return names + .sorted { $0.localizedCaseInsensitiveCompare($1) == .orderedAscending } + .map { PluginTableInfo(name: $0, type: "TABLE") } + } + } + + /// The key attributes of the table and its indexes, typed as the table declares them, then the + /// attributes found in one sampled page. A DynamoDB table declares nothing else. + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { + try await run(isUserOperation: false) { session in + let description = try await self.tableSchema(table, session: session, refresh: true) + let sample = try await self.sample(table: table, session: session) + let observed = DynamoDBItemTable(items: sample, schema: description) + self.catalog.mergeColumnTypes(observed.observedTypes, for: table, in: session.scope) + let indexKeys = description.indexes.flatMap(\.keys.attributes) + var columns = observed.columns + for key in indexKeys where !columns.contains(key) { + columns.insert(key, at: min(description.keys.attributes.count, columns.count)) + } + return columns.map { name in + let type = description.keyType(of: name) ?? observed.observedTypes[name] ?? .string + return Self.columnInfo(name: name, type: type, schema: description) + } + } + } + + static func columnInfo(name: String, type: DynamoDBAttributeType, schema: DynamoDBTableSchema) -> PluginColumnInfo { + let isTableKey = schema.keys.attributes.contains(name) + var roles: [String] = [] + if schema.keys.partition.contains(name) { roles.append(String(localized: "Partition key")) } + if schema.keys.sort.contains(name) { roles.append(String(localized: "Sort key")) } + for index in schema.indexes where index.keys.attributes.contains(name) { + roles.append(String(format: String(localized: "Key of %@"), index.name)) + } + return PluginColumnInfo( + name: name, + dataType: type.displayName, + isNullable: !isTableKey, + isPrimaryKey: isTableKey, + defaultValue: nil, + extra: roles.isEmpty ? nil : roles.joined(separator: ", "), + charset: nil, + collation: nil, + comment: nil, + identityKind: nil, + isGenerated: false, + allowedValues: nil, + generationExpression: nil, + generationKind: nil, + ddlSpelling: nil, + ddlDefault: nil, + ddlGenerationExpression: nil, + ddlCollation: nil, + classificationTypeName: type.classificationName + ) + } + + /// Columns for autocomplete, from the tables already described, with no request at all. The + /// app asks for every table on connect, and answering by sampling each one read up to 300 + /// tables in the background. + func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { + guard let scope else { return [:] } + var result: [String: [PluginColumnInfo]] = [:] + for description in catalog.cachedSchemas(in: scope) { + let observed = catalog.columnTypes(for: description.name, in: scope) + var names = description.keys.attributes + names += description.indexes.flatMap(\.keys.attributes).filter { !names.contains($0) } + names += observed.keys.sorted().filter { !names.contains($0) } + result[description.name] = names.map { name in + Self.columnInfo( + name: name, type: description.keyType(of: name) ?? observed[name] ?? .string, schema: description + ) + } + } + return result + } + + func sample(table: String, session: Session) async throws -> [DynamoDBItem] { + let response = try await session.client.send( + .scan, ["TableName": .string(table), "Limit": .number(String(Self.columnSampleSize))] + ) + return try (response["Items"]?.arrayValue ?? []).map(DynamoDBItem.init(wireItem:)) + } + + /// Nested attribute paths for the filter bar and autocomplete, such as `address.city`. + func sampleFieldPaths(table: String, schema: String?, limit: Int) async throws -> [PluginFieldPath] { + try await run(isUserOperation: false) { session in + let response = try await session.client.send( + .scan, ["TableName": .string(table), "Limit": .number(String(min(max(limit, 1), Self.columnSampleSize)))] + ) + let items = try (response["Items"]?.arrayValue ?? []).map(DynamoDBItem.init(wireItem:)) + return DynamoDBFieldPaths.collect(from: items) + } + } + + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { + try await run(isUserOperation: false) { session in + let description = try await self.tableSchema(table, session: session) + var indexes = [ + PluginIndexInfo( + name: "PRIMARY", columns: description.keys.attributes, + isUnique: true, isPrimary: true, type: DynamoDBIndexTypeName.primary + ) + ] + for index in description.indexes { + let kind = index.kind == .global ? DynamoDBIndexTypeName.global : DynamoDBIndexTypeName.local + var type = "\(kind) \(index.projection.displayName)" + if let status = index.status, status != "ACTIVE" { type += " \(status)" } + if index.isBackfilling { type += " BACKFILLING" } + indexes.append(PluginIndexInfo( + name: index.name, + columns: index.keys.attributes, + isUnique: false, + isPrimary: false, + type: type, + columnPrefixes: nil, + whereClause: nil, + expressions: nil, + includedColumns: index.projection.nonKeyAttributes.isEmpty ? nil : index.projection.nonKeyAttributes, + ddlMethodAndKeys: nil, + ddlWhereClause: nil + )) + } + return indexes + } + } + + /// DynamoDB's own item count, refreshed by AWS about every six hours. + func fetchApproximateRowCount(table: String, schema: String?) async throws -> Int? { + try await run(isUserOperation: false) { session in + let description = try await self.tableSchema(table, session: session, refresh: true) + return description.itemCount.map { Int(clamping: $0) } + } + } + + func fetchFilteredRowCount(table: String, queryFilters: [PluginQueryFilter], logicMode: String) async throws -> Int? { + nil + } + + func fetchExactRowCount( + table: String, + schema: String?, + filters: [(column: String, op: String, value: String)], + logicMode: String + ) async throws -> Int? { + try await fetchExactRowCount( + table: table, schema: schema, + queryFilters: filters.map { PluginQueryFilter(column: $0.column, op: $0.op, value: $0.value) }, + logicMode: logicMode + ) + } + + /// Count Exactly: the same read the grid runs, answered with `Select: COUNT` so no item comes + /// back, and paged to the end. It is billed like reading every item it counts. + func fetchExactRowCount( + table: String, + schema: String?, + queryFilters: [PluginQueryFilter], + logicMode: String + ) async throws -> Int? { + try await run(boundedByQueryTimeout: false) { session in + let description = try await self.tableSchema(table, session: session) + let knownAttributes = self.catalog.columnTypes(for: table, in: session.scope).keys.sorted() + let request = DynamoDBBrowseRequest( + table: table, queryFilters: queryFilters, logicMode: logicMode, + columns: knownAttributes, columnKinds: [:] + ) + let plan = try DynamoDBAccessPlanner(schema: description).plan(request, order: []) + return try await self.count(plan, schema: description, session: session) + } + } + + private func count(_ plan: DynamoDBReadPlan, schema: DynamoDBTableSchema, session: Session) async throws -> Int { + guard plan.access == .query || plan.access == .scan, plan.clientPredicates.isEmpty else { + var total = 0 + _ = try await readPlan( + plan, schema: schema, offset: 0, limit: Int.max, + fingerprint: "count:\(UUID().uuidString)", session: session + ) { _ in + total += 1 + return true + } + return total + } + var total = 0 + let operation: DynamoDBOperation = plan.access == .query ? .query : .scan + for request in plan.requests { + var startKey: DynamoDBJSON? + repeat { + try session.checkDeadline() + var body = request + body["Select"] = .string("COUNT") + if let startKey { body["ExclusiveStartKey"] = startKey } + let response = try await session.client.send(operation, body) + total += response["Count"]?.intValue ?? 0 + startKey = response["LastEvaluatedKey"] + } while startKey != nil + } + return total + } + + // MARK: - DDL and metadata + + /// Statements that recreate the table: its `CreateTable` request, then Time to Live and + /// point-in-time recovery when they are on. Settings DynamoDB Local does not have are left out. + func fetchTableDDL(table: String, schema: String?) async throws -> String { + try await run(isUserOperation: false) { session in + let description = try await self.tableSchema(table, session: session) + var statements = [ + DynamoDBStatement.apiCall( + DynamoDBAPICall(operation: .createTable, body: DynamoDBTableDefinition.createTableRequest(description)), + window: DynamoDBReadWindow() + ).prettyText + ] + if let ttl = try? await session.client.send(.describeTimeToLive, ["TableName": .string(table)]), + ttl["TimeToLiveDescription"]?["TimeToLiveStatus"]?.stringValue == "ENABLED", + let attribute = ttl["TimeToLiveDescription"]?["AttributeName"]?.stringValue { + statements.append(DynamoDBStatement.apiCall( + DynamoDBAPICall(operation: .updateTimeToLive, body: .object([ + "TableName": .string(table), + "TimeToLiveSpecification": .object(["Enabled": .bool(true), "AttributeName": .string(attribute)]) + ])), + window: DynamoDBReadWindow() + ).prettyText) + } + if let backups = try? await session.client.send(.describeContinuousBackups, ["TableName": .string(table)]), + backups["ContinuousBackupsDescription"]?["PointInTimeRecoveryDescription"]?["PointInTimeRecoveryStatus"]? + .stringValue == "ENABLED" { + statements.append(DynamoDBStatement.apiCall( + DynamoDBAPICall(operation: .updateContinuousBackups, body: .object([ + "TableName": .string(table), + "PointInTimeRecoverySpecification": .object(["PointInTimeRecoveryEnabled": .bool(true)]) + ])), + window: DynamoDBReadWindow() + ).prettyText) + } + return statements.joined(separator: ";\n\n") + ";" + } + } + + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + try await run(isUserOperation: false) { session in + let description = try await self.tableSchema(table, session: session, refresh: true) + return PluginTableMetadata( + tableName: description.name, + dataSize: description.sizeBytes, + indexSize: description.indexes.compactMap(\.sizeBytes).reduce(0, +), + totalSize: description.sizeBytes, + rowCount: description.itemCount, + comment: DynamoDBTableDefinition.summary(description), + engine: "DynamoDB", + createTime: description.createdAt + ) + } + } +} + +extension DynamoDBStatement { + var prettyText: String { + guard case .apiCall(let call, _) = self else { return text } + return "\(call.operation.rawValue) \(call.body.serialized(pretty: true))" + } +} + +/// The `type` a DynamoDB index reports in the Structure tab. It comes back on an edited row, which +/// is how a refusal tells a local index and the primary key from a global index. +enum DynamoDBIndexTypeName { + static let primary = "PRIMARY KEY" + static let global = "GLOBAL" + static let local = "LOCAL" +} + +/// Nested attribute paths for the filter bar and autocomplete, through maps only. A path through a +/// list names no element DynamoDB can compare, so `items.sku` over a list of maps would filter every +/// item out. +enum DynamoDBFieldPaths { + static let maximumDepth = 4 + + static func collect(from items: [DynamoDBItem]) -> [PluginFieldPath] { + var found: [String: PluginFieldPath] = [:] + for item in items { + for (name, value) in item { + visit(value, path: name, depth: 1, into: &found) + } + } + return found.values.sorted { $0.path < $1.path } + } + + private static func visit( + _ value: DynamoDBAttributeValue, + path: String, + depth: Int, + into found: inout [String: PluginFieldPath] + ) { + if found[path] == nil { + found[path] = PluginFieldPath(path: path, typeName: value.type.displayName, depth: depth) + } + guard depth < maximumDepth, case .map(let entries) = value else { return } + for (key, nested) in entries { + visit(nested, path: "\(path).\(key)", depth: depth + 1, into: &found) + } + } +} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+TableManagement.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+TableManagement.swift new file mode 100644 index 0000000000..9761ac3ec9 --- /dev/null +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+TableManagement.swift @@ -0,0 +1,190 @@ +import Foundation +import TableProPluginKit + +extension DynamoDBPluginDriver { + // MARK: - Drop + + func dropObjectStatement(name: String, objectType: String, schema: String?, cascade: Bool) -> String? { + guard objectType.uppercased() == "TABLE", DynamoDBTableDefinition.isValidTableName(name) else { return nil } + return Self.apiStatement(.deleteTable, ["TableName": .string(name)]) + } + + // MARK: - Create Table form + + func createTableFormSpec(schema: String?) -> PluginCreateTableFormSpec? { + DynamoDBTableDefinition.formSpec + } + + func createTableStatements(for request: PluginCreateTableRequest, schema: String?) throws -> [String] { + let body = try DynamoDBTableDefinition.createTableRequest(from: request) + return [ + DynamoDBStatement.apiCall(DynamoDBAPICall(operation: .createTable, body: body), window: DynamoDBReadWindow()) + .prettyText + ] + } + + // MARK: - Global secondary indexes from the Structure tab + + /// The first column is the partition key and the second, when there is one, the sort key. The + /// key types and a provisioned table's capacity are filled in when the statement runs. + func generateAddIndexSQL(table: String, index: PluginIndexDefinition) -> String? { + guard !index.isUnique, (1...2).contains(index.columns.count) else { return nil } + var keys: [DynamoDBJSON] = [.object(["AttributeName": .string(index.columns[0]), "KeyType": .string("HASH")])] + if index.columns.count == 2 { + keys.append(.object(["AttributeName": .string(index.columns[1]), "KeyType": .string("RANGE")])) + } + var projection: [String: DynamoDBJSON] = ["ProjectionType": .string("ALL")] + if let included = index.includedColumns, !included.isEmpty { + projection = [ + "ProjectionType": .string("INCLUDE"), + "NonKeyAttributes": .array(included.map(DynamoDBJSON.string)) + ] + } + return Self.apiStatement(.updateTable, [ + "TableName": .string(table), + "GlobalSecondaryIndexUpdates": .array([.object(["Create": .object([ + "IndexName": .string(index.name), + "KeySchema": .array(keys), + "Projection": .object(projection) + ])])]) + ]) + } + + func generateDropIndexSQL(table: String, indexName: String) -> String? { + guard indexName != "PRIMARY" else { return nil } + return Self.apiStatement(.updateTable, [ + "TableName": .string(table), + "GlobalSecondaryIndexUpdates": .array([.object(["Delete": .object(["IndexName": .string(indexName)])])]) + ]) + } + + func schemaOperationRefusal(_ operation: PluginSchemaOperation) -> String? { + switch operation { + case .addIndex(let index): + if index.isUnique { + return String(localized: "DynamoDB indexes are never unique") + } + guard (1...2).contains(index.columns.count) else { + return String(localized: "A global secondary index takes a partition key and an optional sort key") + } + return nil + case .addColumn: + return String(localized: "A DynamoDB table declares only its keys. Add the attribute by writing it to an item.") + case .modifyIndex: + return String( + localized: "A DynamoDB index can't be changed. Delete it and save, then add the new one once the old one is gone." + ) + case .dropIndex(let index): + let type = index.indexType ?? "" + if index.name == "PRIMARY" || type == DynamoDBIndexTypeName.primary { + return String(localized: "The primary key is part of the table and can't be dropped.") + } + guard type.hasPrefix(DynamoDBIndexTypeName.local) else { return nil } + return String(localized: "A local secondary index is part of its table and is removed only with the table.") + default: + return nil + } + } + + // MARK: - Maintenance + + enum MaintenanceName { + static var pointInTimeRecovery: String { String(localized: "Point-in-Time Recovery") } + static var deletionProtection: String { String(localized: "Deletion Protection") } + static var stream: String { String(localized: "Stream") } + static var tableClass: String { String(localized: "Table Class") } + static var onDemand: String { String(localized: "Switch to On-Demand Capacity") } + static var timeToLiveOff: String { String(localized: "Turn Off Time to Live") } + } + + private enum MaintenanceOption { + static let enabled = "enabled" + static let choice = "choice" + } + + private static var streamChoices: [(label: String, viewType: String?)] { + [ + (String(localized: "Off"), nil), + (String(localized: "Keys only"), "KEYS_ONLY"), + (String(localized: "New image"), "NEW_IMAGE"), + (String(localized: "Old image"), "OLD_IMAGE"), + (String(localized: "New and old images"), "NEW_AND_OLD_IMAGES") + ] + } + + private static var tableClassChoices: [(label: String, value: String)] { + [ + (String(localized: "Standard"), "STANDARD"), + (String(localized: "Standard-Infrequent Access"), "STANDARD_INFREQUENT_ACCESS") + ] + } + + func maintenanceOperations() -> [PluginMaintenanceOperation]? { + let tables: Set = [.table] + return [ + PluginMaintenanceOperation( + name: MaintenanceName.pointInTimeRecovery, appliesTo: tables, scope: .object, + options: [PluginMaintenanceOption(key: MaintenanceOption.enabled, label: String(localized: "On"), defaultValue: "true")] + ), + PluginMaintenanceOperation( + name: MaintenanceName.deletionProtection, appliesTo: tables, scope: .object, + options: [PluginMaintenanceOption(key: MaintenanceOption.enabled, label: String(localized: "On"), defaultValue: "true")] + ), + PluginMaintenanceOperation( + name: MaintenanceName.stream, appliesTo: tables, scope: .object, + options: [PluginMaintenanceOption( + key: MaintenanceOption.choice, label: String(localized: "Stream"), + defaultValue: Self.streamChoices[4].label, choices: Self.streamChoices.map(\.label) + )] + ), + PluginMaintenanceOperation( + name: MaintenanceName.tableClass, appliesTo: tables, scope: .object, + options: [PluginMaintenanceOption( + key: MaintenanceOption.choice, label: String(localized: "Class"), + defaultValue: Self.tableClassChoices[0].label, choices: Self.tableClassChoices.map(\.label) + )] + ), + PluginMaintenanceOperation(name: MaintenanceName.onDemand, appliesTo: tables, scope: .object), + PluginMaintenanceOperation(name: MaintenanceName.timeToLiveOff, appliesTo: tables, scope: .object) + ] + } + + func maintenanceStatements(operation: String, table: String?, schema: String?, options: [String: String]) -> [String]? { + guard let table else { return nil } + let name: DynamoDBJSON = .string(table) + let enabled = options[MaintenanceOption.enabled] != "false" + switch operation { + case MaintenanceName.pointInTimeRecovery: + return [Self.apiStatement(.updateContinuousBackups, [ + "TableName": name, + "PointInTimeRecoverySpecification": .object(["PointInTimeRecoveryEnabled": .bool(enabled)]) + ])] + case MaintenanceName.deletionProtection: + return [Self.apiStatement(.updateTable, ["TableName": name, "DeletionProtectionEnabled": .bool(enabled)])] + case MaintenanceName.stream: + let choice = Self.streamChoices.first { $0.label == options[MaintenanceOption.choice] } + guard let choice else { return nil } + var specification: [String: DynamoDBJSON] = ["StreamEnabled": .bool(choice.viewType != nil)] + if let viewType = choice.viewType { specification["StreamViewType"] = .string(viewType) } + return [Self.apiStatement(.updateTable, ["TableName": name, "StreamSpecification": .object(specification)])] + case MaintenanceName.tableClass: + guard let choice = Self.tableClassChoices.first(where: { $0.label == options[MaintenanceOption.choice] }) else { + return nil + } + return [Self.apiStatement(.updateTable, ["TableName": name, "TableClass": .string(choice.value)])] + case MaintenanceName.onDemand: + return [Self.apiStatement(.updateTable, ["TableName": name, "BillingMode": .string("PAY_PER_REQUEST")])] + case MaintenanceName.timeToLiveOff: + return [Self.apiStatement(.updateTimeToLive, [ + "TableName": name, + "TimeToLiveSpecification": .object(["Enabled": .bool(false)]) + ])] + default: + return nil + } + } + + static func apiStatement(_ operation: DynamoDBOperation, _ body: [String: DynamoDBJSON]) -> String { + DynamoDBStatement.apiCall(DynamoDBAPICall(operation: operation, body: .object(body)), window: DynamoDBReadWindow()).text + } +} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Writes.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Writes.swift new file mode 100644 index 0000000000..8f628b29ba --- /dev/null +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Writes.swift @@ -0,0 +1,192 @@ +import Foundation +import TableProPluginKit + +extension DynamoDBPluginDriver { + func generateStatements( + table: String, + columns: [String], + primaryKeyColumns: [String], + changes: [PluginRowChange], + insertedRowData: [Int: [PluginCellValue]], + deletedRowIndices: Set, + insertedRowIndices: Set + ) -> [(statement: String, parameters: [PluginCellValue])]? { + DynamoDBWriteStatements(table: table, columns: columns, keyColumns: primaryKeyColumns).statements( + for: changes, + insertedRowData: insertedRowData, + deletedRowIndices: deletedRowIndices, + insertedRowIndices: insertedRowIndices + ) + } + + func generateIdentityPreservingInsert( + table: String, + schema: String?, + columns: [String], + primaryKeyColumns: [String], + rows: [[PluginCellValue]] + ) -> [(statement: String, parameters: [PluginCellValue])]? { + let writer = DynamoDBWriteStatements(table: table, columns: columns, keyColumns: primaryKeyColumns) + return rows.map { writer.insert(row: $0) } + } + + // MARK: - PartiQL writes + + func writePartiQL(_ text: String, parameters: [PluginCellValue], session: Session) async throws -> PluginQueryResult { + let started = Date() + let kind = DynamoDBPartiQL.kind(of: text) + let target = DynamoDBPartiQL.target(of: text) + let schema = await target.asyncMap { try? await tableSchema($0.table, session: session) } ?? nil + var body: [String: DynamoDBJSON] = [ + "Statement": .string(text), + "ReturnConsumedCapacity": .string("TOTAL") + ] + var boundKey: DynamoDBItem? + + if !parameters.isEmpty { + let roles = DynamoDBPartiQL.parameterRoles(in: text) + let table = target?.table + var observed = table.map { catalog.columnTypes(for: $0, in: session.scope) } ?? [:] + if kind == .insert, observed.isEmpty, let table { + observed = try await sampleTypes(table: table, schema: schema, session: session) + } + let keyOnly = DynamoDBParameterBinder(schema: schema, observedTypes: observed, currentItem: nil) + boundKey = try key(from: parameters, roles: roles, schema: schema, binder: keyOnly) + var current: DynamoDBItem? + if kind == .update || kind == .delete, let boundKey, let table { + current = try await currentItem(table: table, key: boundKey, roles: roles, session: session) + if kind == .update, current == nil { + throw DynamoDBError.itemMissing(key: schema?.describeKey(boundKey) ?? "") + } + } + let binder = DynamoDBParameterBinder(schema: schema, observedTypes: observed, currentItem: current) + let bound = try binder.bind(parameters, roles: roles) + if kind == .insert, let schema { + try requireKeys(schema: schema, statement: text, roles: roles, values: bound) + } + body["Parameters"] = .array(bound.map(\.wireJSON)) + } + + let response: DynamoDBJSON + do { + response = try await session.client.send(.executeStatement, body) + } catch DynamoDBError.service(let error) where error.isConditionalCheckFailure && kind == .update { + throw DynamoDBError.itemChanged(key: boundKey.flatMap { schema?.describeKey($0) } ?? "") + } catch DynamoDBError.service(let error) where error.code.hasPrefix("DuplicateItem") && kind == .insert { + throw DynamoDBError.invalidValue(attribute: "", reason: String(localized: "An item with this key already exists")) + } + + if let table = target?.table { + catalog.forgetReadPositions(table: table, in: session.scope) + } + let items = try (response["Items"]?.arrayValue ?? []).map(DynamoDBItem.init(wireItem:)) + let hasReturning = DynamoDBPartiQL.hasReturning(text) + let rowsAffected: Int + var message: String? + switch kind { + case .insert, .update: + rowsAffected = 1 + case .delete where hasReturning: + rowsAffected = items.count + case .delete: + rowsAffected = 0 + message = String(localized: "DELETE ran. DynamoDB reports whether an item was deleted only with RETURNING ALL OLD *.") + default: + rowsAffected = 0 + } + if hasReturning, !items.isEmpty { + return Self.queryResult( + items: items, schema: schema, preferredColumns: [], includeAllKeys: false, + started: started, isTruncated: false, statusMessage: message, rowsAffected: rowsAffected + ) + } + return Self.messageResult( + message ?? String(format: String(localized: "%d item(s) affected"), rowsAffected), + started: started, + rowsAffected: rowsAffected + ) + } + + /// The item's key, read out of the `"key" = ?` parameters of a statement that names every key + /// attribute of the table. + private func key( + from parameters: [PluginCellValue], + roles: [DynamoDBPartiQL.ParameterRole], + schema: DynamoDBTableSchema?, + binder: DynamoDBParameterBinder + ) throws -> DynamoDBItem? { + guard let schema else { return nil } + var key: DynamoDBItem = [:] + for (index, role) in roles.enumerated() where index < parameters.count { + guard case .compared(let path) = role, path.isTopLevel, schema.keys.attributes.contains(path.root), + key[path.root] == nil + else { continue } + key[path.root] = try binder.bind([parameters[index]], roles: [role]).first + } + return key.count == schema.keys.attributes.count ? key : nil + } + + /// The attributes an UPDATE or DELETE touches, as the item holds them now: the types its + /// parameters are decoded against. + private func currentItem( + table: String, + key: DynamoDBItem, + roles: [DynamoDBPartiQL.ParameterRole], + session: Session + ) async throws -> DynamoDBItem? { + var context = DynamoDBExpressionContext() + var projected: [String] = [] + for role in roles { + switch role { + case .assigned(let path), .compared(let path): + let placeholder = context.name(path.root) + if !projected.contains(placeholder) { projected.append(placeholder) } + default: + continue + } + } + var body: [String: DynamoDBJSON] = [ + "TableName": .string(table), + "Key": key.wireJSON, + "ConsistentRead": .bool(true) + ] + if !projected.isEmpty { + body["ProjectionExpression"] = .string(projected.joined(separator: ", ")) + } + context.apply(to: &body) + let response = try await session.client.send(.getItem, body) + return try response["Item"].map(DynamoDBItem.init(wireItem:)) + } + + /// Every key attribute of the table is in the INSERT, as a literal or as a `?` bound to a value. + private func requireKeys( + schema: DynamoDBTableSchema, + statement: String, + roles: [DynamoDBPartiQL.ParameterRole], + values: [DynamoDBAttributeValue] + ) throws { + let named = DynamoDBPartiQL.insertedAttributes(in: statement) + for attribute in schema.keys.attributes { + let boundToNull = roles.enumerated().contains { index, role in + guard case .inserted(let name) = role, name == attribute else { return false } + return index >= values.count || values[index] == .null + } + guard named.contains(attribute), !boundToNull else { + throw DynamoDBError.invalidValue(attribute: attribute, reason: String(localized: "A key attribute needs a value")) + } + } + } + + /// Types for the attributes of a table nothing has read yet, from one small page. + func sampleTypes( + table: String, + schema: DynamoDBTableSchema?, + session: Session + ) async throws -> [String: DynamoDBAttributeType] { + let response = try await session.client.send(.scan, ["TableName": .string(table), "Limit": .number("100")]) + let items = try (response["Items"]?.arrayValue ?? []).map(DynamoDBItem.init(wireItem:)) + let observed = DynamoDBItemTable(items: items, schema: schema).observedTypes + catalog.mergeColumnTypes(observed, for: table, in: session.scope) + return observed + } +} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver.swift index 17de5eb710..570d5d7a49 100644 --- a/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver.swift +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver.swift @@ -1,451 +1,167 @@ -// -// DynamoDBPluginDriver.swift -// DynamoDBDriverPlugin -// -// PluginDatabaseDriver implementation for Amazon DynamoDB. -// Routes both NoSQL browsing hooks and PartiQL commands through DynamoDBConnection. -// - import Foundation import os -import OSLog import TableProPluginKit -internal final class DynamoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { - private let config: DriverConnectionConfig - private var _connection: DynamoDBConnection? - private let lock = NSLock() - private var _serverVersion: String? +final class DynamoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { + static let logger = Logger(subsystem: "com.TablePro", category: "DynamoDBPluginDriver") - // Table description cache to avoid repeated DescribeTable calls - private var _tableDescriptionCache: [String: TableDescription] = [:] + let config: DriverConnectionConfig + let catalog: DynamoDBCatalog + private let clientFactory: @Sendable (DynamoDBEndpoint, DynamoDBCredentialsProvider) -> DynamoDBClient + private let lock = NSLock() + private var _client: DynamoDBClient? + private var _scope: DynamoDBCatalog.Scope? + private var runningOperations: [UUID: @Sendable () -> Void] = [:] - private var connection: DynamoDBConnection? { - lock.withLock { _connection } + init( + config: DriverConnectionConfig, + catalog: DynamoDBCatalog = .shared, + clientFactory: @escaping @Sendable (DynamoDBEndpoint, DynamoDBCredentialsProvider) -> DynamoDBClient = { + DynamoDBClient(endpoint: $0, credentials: $1) + } + ) { + self.config = config + self.catalog = catalog + self.clientFactory = clientFactory } - private static let logger = Logger(subsystem: "com.TablePro", category: "DynamoDBPluginDriver") - private static let maxItems = PluginRowLimits.emergencyMax + var client: DynamoDBClient? { lock.withLock { _client } } + var scope: DynamoDBCatalog.Scope? { lock.withLock { _scope } } var serverVersion: String? { - lock.withLock { _serverVersion } + guard let client else { return nil } + return client.endpoint.isLocal ? "DynamoDB Local" : "DynamoDB \(client.endpoint.signingRegion)" } + var capabilities: PluginCapabilities { [.parameterizedQueries, .cancelQuery] } var supportsTransactions: Bool { false } - - var capabilities: PluginCapabilities { - [ - .parameterizedQueries, - .cancelQuery, - ] - } + var parameterStyle: ParameterStyle { .questionMark } func beginTransaction() async throws {} func commitTransaction() async throws {} func rollbackTransaction() async throws {} func quoteIdentifier(_ name: String) -> String { - let escaped = name.replacingOccurrences(of: "\"", with: "\"\"") - return "\"\(escaped)\"" + DynamoDBStatement.quote(name) } func escapeStringLiteral(_ value: String) -> String { value.replacingOccurrences(of: "'", with: "''") } - func defaultExportQuery(table: String) -> String? { - "SELECT * FROM \(quoteIdentifier(table))" - } - - /// DynamoDB has no truncate. Emptying a table means scanning it and deleting every item in - /// batches, which is a long billed job rather than a statement, and DeleteTable plus - /// CreateTable loses the table's settings. Neither is what Truncate promises, so it stays - /// unoffered rather than offered as something else. - func truncateTableStatements(table: String, schema: String?, cascade: Bool) -> [String]? { - nil - } - - func dropObjectStatement(name: String, objectType: String, schema: String?, cascade: Bool) -> String? { - DynamoDBOperations.dropTable(named: name, objectType: objectType) - } - - init(config: DriverConnectionConfig) { - self.config = config - } - - // MARK: - Connection Management + // MARK: - Connection func connect() async throws { - let conn = DynamoDBConnection(config: config) - try await conn.connect() - - lock.withLock { - _connection = conn - _serverVersion = "DynamoDB" + let endpoint = try DynamoDBEndpoint.resolve(fields: config.additionalFields) + let credentials = DynamoDBCredentialsProvider( + fields: config.additionalFields, username: config.username, password: config.password + ) + let client = clientFactory(endpoint, credentials) + do { + _ = try await client.send(.listTables, ["Limit": .number("1")]) + } catch { + client.invalidate() + throw error + } + let scope = DynamoDBCatalog.Scope( + endpoint: endpoint.url.absoluteString, region: endpoint.signingRegion, identity: credentials.identity + ) + let previous = lock.withLock { () -> DynamoDBClient? in + let old = _client + _client = client + _scope = scope + return old } + previous?.invalidate() } func disconnect() { - lock.withLock { - _connection?.disconnect() - _connection = nil - _tableDescriptionCache.removeAll() + let (client, cancellations) = lock.withLock { () -> (DynamoDBClient?, [@Sendable () -> Void]) in + let current = _client + _client = nil + return (current, Array(runningOperations.values)) } + cancellations.forEach { $0() } + client?.invalidate() } func ping() async throws { - guard let conn = connection else { - throw DynamoDBError.notConnected - } - try await conn.ping() - } - - // MARK: - Query Execution - - func execute(query: String) async throws -> PluginQueryResult { - let startTime = Date() - - guard let conn = connection else { - throw DynamoDBError.notConnected - } - - let trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines) - - // Health monitor sends "SELECT 1" as a ping - if trimmed.lowercased() == "select 1" { - try await conn.ping() - return PluginQueryResult( - columns: ["ok"], - columnTypeNames: ["Int32"], - rows: [[.text("1")]], - rowsAffected: 0, - executionTime: Date().timeIntervalSince(startTime) - ) - } - - if DynamoDBQueryBuilder.isTaggedQuery(trimmed) { - return try await executeTaggedQuery(trimmed, conn: conn, startTime: startTime) - } - - if let table = DynamoDBOperations.droppedTableName(in: trimmed) { - return try await executeDropTable(table, conn: conn, startTime: startTime) - } - - return try await executePartiQL(trimmed, conn: conn, startTime: startTime) + guard let client else { throw DynamoDBError.notConnected } + _ = try await client.send(.listTables, ["Limit": .number("1")]) } - func executeParameterized(query: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { - let startTime = Date() - - guard let conn = connection else { - throw DynamoDBError.notConnected - } - - let trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines) - - guard !parameters.isEmpty else { - return try await execute(query: trimmed) - } - - let dynamoParams: [[String: Any]] = parameters.map { param -> [String: Any] in - switch param { - case .null: - return ["NULL": true] - case .bytes(let data): - return ["B": data.base64EncodedString()] - case .text(let value): - if Double(value) != nil { - return ["N": value] - } - return ["S": value] - } - } - - let response = try await conn.executeStatement(statement: trimmed, parameters: dynamoParams) - let items = response.Items ?? [] - - if items.isEmpty { - return PluginQueryResult( - columns: ["Result"], - columnTypeNames: ["String"], - rows: [[.text("Statement executed")]], - rowsAffected: 0, - executionTime: Date().timeIntervalSince(startTime) - ) - } - - let tableName = DynamoDBPartiQLParser.extractTableName(trimmed) - let keySchema: [(name: String, keyType: String)] - if let name = tableName { - keySchema = try await cachedKeySchema(name, conn: conn) - } else { - keySchema = [] - } - - let columns = DynamoDBItemFlattener.unionColumns(from: items, keySchema: keySchema) - let typeNames = DynamoDBItemFlattener.columnTypeNames(for: columns, items: items) - let rows = DynamoDBItemFlattener.flatten(items: items, columns: columns) - - return PluginQueryResult( - columns: columns, - columnTypeNames: typeNames, - rows: rows, - rowsAffected: 0, - executionTime: Date().timeIntervalSince(startTime) - ) - } - - // MARK: - Query Cancellation - func cancelQuery() throws { - connection?.cancelCurrentRequest() + let cancellations = lock.withLock { Array(runningOperations.values) } + cancellations.forEach { $0() } } func applyQueryTimeout(_ seconds: Int) async throws { - connection?.setQueryTimeout(seconds) - } - - // MARK: - Schema Operations - - func fetchTables(schema: String?) async throws -> [PluginTableInfo] { - guard let conn = connection else { - Self.logger.error("fetchTables: not connected") - throw DynamoDBError.notConnected - } - + client?.setQueryTimeout(seconds) + } + + // MARK: - Operations + + struct Session: Sendable { + let client: DynamoDBClient + let scope: DynamoDBCatalog.Scope + let deadline: Date? + let timeoutSeconds: Int + + func checkDeadline() throws { + if Task.isCancelled { throw DynamoDBError.cancelled } + guard let deadline, Date() >= deadline else { return } + throw DynamoDBError.timedOut(seconds: timeoutSeconds) + } + } + + /// Runs one operation. A read gets the query timeout as a deadline for all of its pages; a + /// write gets none, because stopping a write halfway is worse than waiting, and neither does + /// Count Exactly, a full read the user asked for and stops with Cancel. A user operation is one + /// Stop reaches; a metadata read for the sidebar or the Structure tab is not. A cancelled + /// operation ends in `CancellationError`, which the app reads as a stop rather than a failure. + func run( + boundedByQueryTimeout: Bool = true, + isUserOperation: Bool = true, + _ body: @escaping @Sendable (Session) async throws -> T + ) async throws -> T { + guard let client, let scope else { throw DynamoDBError.notConnected } + let timeout = client.queryTimeoutSeconds + let deadline = boundedByQueryTimeout && timeout > 0 ? Date().addingTimeInterval(TimeInterval(timeout)) : nil + let session = Session(client: client, scope: scope, deadline: deadline, timeoutSeconds: timeout) + let task = Task { try await body(session) } + let id = UUID() + if isUserOperation { + lock.withLock { runningOperations[id] = { task.cancel() } } + } + defer { lock.withLock { runningOperations[id] = nil } } do { - var allTableNames: [String] = [] - var lastEvaluated: String? - - repeat { - let response = try await conn.listTables(limit: 100, exclusiveStartTableName: lastEvaluated) - let names = response.TableNames ?? [] - allTableNames.append(contentsOf: names) - lastEvaluated = response.LastEvaluatedTableName - } while lastEvaluated != nil - - Self.logger.debug("fetchTables found \(allTableNames.count) tables") - return allTableNames.sorted { $0.localizedCaseInsensitiveCompare($1) == .orderedAscending } - .map { name in - PluginTableInfo(name: name, type: "TABLE") - } - } catch { - Self.logger.error("fetchTables error: \(error.localizedDescription)") - throw error - } - } - - func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { - guard let conn = connection else { - throw DynamoDBError.notConnected - } - - let tableDesc = try await cachedDescribeTable(table, conn: conn) - let keySchema = extractKeySchema(from: tableDesc) - - // Sample items to discover all columns - let sampleResponse = try await conn.scan(tableName: table, limit: 100) - let items = sampleResponse.Items ?? [] - - let columns = DynamoDBItemFlattener.unionColumns(from: items, keySchema: keySchema) - let typeNames = DynamoDBItemFlattener.columnTypeNames(for: columns, items: items) - - let keyNames = Set(keySchema.map(\.name)) - let hashKey = keySchema.first(where: { $0.keyType == "HASH" })?.name - - return zip(columns, typeNames).map { column, typeName in - PluginColumnInfo( - name: column, - dataType: typeName, - isNullable: !keyNames.contains(column), - isPrimaryKey: keyNames.contains(column), - defaultValue: nil, - extra: column == hashKey ? "HASH" : keySchema.first(where: { $0.name == column })?.keyType - ) - } - } - - func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { - let tables = try await fetchTables(schema: schema) - var result: [String: [PluginColumnInfo]] = [:] - for table in tables { - result[table.name] = try await fetchColumns(table: table.name, schema: schema) - } - return result - } - - func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { - guard let conn = connection else { - throw DynamoDBError.notConnected - } - - let tableDesc = try await cachedDescribeTable(table, conn: conn) - var indexes: [PluginIndexInfo] = [] - - if let keySchema = tableDesc.KeySchema { - let columns = keySchema.map(\.AttributeName) - indexes.append(PluginIndexInfo( - name: "PRIMARY", - columns: columns, - isUnique: true, - isPrimary: true, - type: "PRIMARY KEY" - )) - } - - if let gsis = tableDesc.GlobalSecondaryIndexes { - for gsi in gsis { - let columns = (gsi.KeySchema ?? []).map(\.AttributeName) - let projectionType = gsi.Projection?.ProjectionType ?? "ALL" - indexes.append(PluginIndexInfo( - name: gsi.IndexName, - columns: columns, - isUnique: false, - isPrimary: false, - type: "GSI (\(projectionType))" - )) - } - } - - if let lsis = tableDesc.LocalSecondaryIndexes { - for lsi in lsis { - let columns = (lsi.KeySchema ?? []).map(\.AttributeName) - let projectionType = lsi.Projection?.ProjectionType ?? "ALL" - indexes.append(PluginIndexInfo( - name: lsi.IndexName, - columns: columns, - isUnique: false, - isPrimary: false, - type: "LSI (\(projectionType))" - )) + return try await withTaskCancellationHandler { + try await task.value + } onCancel: { + task.cancel() } + } catch is CancellationError { + throw CancellationError() + } catch DynamoDBError.cancelled { + throw CancellationError() } - - return indexes - } - - func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { - [] } - func fetchApproximateRowCount(table: String, schema: String?) async throws -> Int? { - guard let conn = connection else { - throw DynamoDBError.notConnected - } - let tableDesc = try await cachedDescribeTable(table, conn: conn) - if let count = tableDesc.ItemCount { - return Int(count) - } - return nil - } - - func fetchTableDDL(table: String, schema: String?) async throws -> String { - guard let conn = connection else { - throw DynamoDBError.notConnected - } - - let tableDesc = try await cachedDescribeTable(table, conn: conn) - - var lines: [String] = [] - lines.append("Table: \(tableDesc.TableName)") - lines.append("Status: \(tableDesc.TableStatus ?? "UNKNOWN")") - - if let arn = tableDesc.TableArn { - lines.append("ARN: \(arn)") - } - - if let keySchema = tableDesc.KeySchema { - lines.append("") - lines.append("Key Schema:") - for key in keySchema { - let attrType = tableDesc.AttributeDefinitions?.first(where: { - $0.AttributeName == key.AttributeName - })?.AttributeType ?? "?" - lines.append(" \(key.AttributeName) (\(attrType)) - \(key.KeyType)") - } - } - - if let attrs = tableDesc.AttributeDefinitions { - lines.append("") - lines.append("Attribute Definitions:") - for attr in attrs { - lines.append(" \(attr.AttributeName): \(attr.AttributeType)") - } - } - - let billingMode = tableDesc.BillingModeSummary?.BillingMode ?? "PROVISIONED" - lines.append("") - lines.append("Billing Mode: \(billingMode)") - - if billingMode == "PROVISIONED", let throughput = tableDesc.ProvisionedThroughput { - lines.append("Read Capacity: \(throughput.ReadCapacityUnits ?? 0)") - lines.append("Write Capacity: \(throughput.WriteCapacityUnits ?? 0)") - } - - if let itemCount = tableDesc.ItemCount { - lines.append("") - lines.append("Item Count: \(itemCount)") - } - if let sizeBytes = tableDesc.TableSizeBytes { - lines.append("Table Size: \(formatBytes(sizeBytes))") - } - - if let gsis = tableDesc.GlobalSecondaryIndexes, !gsis.isEmpty { - lines.append("") - lines.append("Global Secondary Indexes:") - for gsi in gsis { - let keys = (gsi.KeySchema ?? []).map { "\($0.AttributeName) (\($0.KeyType))" }.joined(separator: ", ") - let projection = gsi.Projection?.ProjectionType ?? "ALL" - lines.append(" \(gsi.IndexName): [\(keys)] Projection=\(projection)") - if let status = gsi.IndexStatus { - lines.append(" Status: \(status)") - } - } - } + // MARK: - Table descriptions - if let lsis = tableDesc.LocalSecondaryIndexes, !lsis.isEmpty { - lines.append("") - lines.append("Local Secondary Indexes:") - for lsi in lsis { - let keys = (lsi.KeySchema ?? []).map { "\($0.AttributeName) (\($0.KeyType))" }.joined(separator: ", ") - let projection = lsi.Projection?.ProjectionType ?? "ALL" - lines.append(" \(lsi.IndexName): [\(keys)] Projection=\(projection)") - } + func tableSchema(_ table: String, session: Session, refresh: Bool = false) async throws -> DynamoDBTableSchema { + if !refresh, let cached = catalog.schema(for: table, in: session.scope) { + return cached } - - return lines.joined(separator: "\n") - } - - func fetchViewDefinition(view: String, schema: String?) async throws -> String { - throw DynamoDBError.serverError("DynamoDB does not support views") - } - - func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { - guard let conn = connection else { - throw DynamoDBError.notConnected + let response = try await session.client.send(.describeTable, ["TableName": .string(table)]) + let schema = try DynamoDBTableSchema(describeTableResponse: response) + if !schema.isBeingCreated { + catalog.store(schema, in: session.scope) } - - let tableDesc = try await cachedDescribeTable(table, conn: conn) - let billingMode = tableDesc.BillingModeSummary?.BillingMode ?? "PROVISIONED" - - return PluginTableMetadata( - tableName: tableDesc.TableName, - dataSize: tableDesc.TableSizeBytes, - rowCount: tableDesc.ItemCount, - comment: "Billing: \(billingMode)", - engine: "DynamoDB" - ) - } - - func fetchDatabases() async throws -> [String] { - ["default"] + return schema } - func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { - PluginDatabaseMetadata(name: database) - } - - // MARK: - NoSQL Query Building Hooks + // MARK: - Statement building func buildBrowseQuery( table: String, @@ -454,15 +170,16 @@ internal final class DynamoDBPluginDriver: PluginDatabaseDriver, @unchecked Send limit: Int, offset: Int ) -> String? { - DynamoDBQueryBuilder().buildBrowseQuery( - table: table, sortColumns: sortColumns, limit: limit, offset: offset + browseStatement( + DynamoDBBrowseRequest(table: table, filters: [], matchAll: true, columns: columns), + sortColumns: sortColumns, columns: columns, limit: limit, offset: offset ) } func buildFilteredQuery( table: String, schema: String?, - queryFilters filters: [PluginQueryFilter], + queryFilters: [PluginQueryFilter], logicMode: String, sortColumns: [(columnIndex: Int, ascending: Bool)], columns: [String], @@ -470,773 +187,89 @@ internal final class DynamoDBPluginDriver: PluginDatabaseDriver, @unchecked Send offset: Int, columnKinds: [String: PluginColumnKind] ) -> String? { - let (keySchema, attrTypes) = lock.withLock { - let desc = _tableDescriptionCache[table] - return (extractKeySchema(from: desc), extractAttributeTypes(from: desc)) - } - return DynamoDBQueryBuilder().buildFilteredQuery( - table: table, filters: filters, logicMode: logicMode, - sortColumns: sortColumns, columns: columns, limit: limit, offset: offset, - keySchema: keySchema, attributeTypes: attrTypes + let request = DynamoDBBrowseRequest( + table: table, queryFilters: queryFilters, logicMode: logicMode, + columns: columns, columnKinds: columnKinds ) + return browseStatement(request, sortColumns: sortColumns, columns: columns, limit: limit, offset: offset) } - // MARK: - Statement Generation - - func generateStatements( + func buildFilteredQuery( table: String, + filters: [(column: String, op: String, value: String)], + logicMode: String, + sortColumns: [(columnIndex: Int, ascending: Bool)], columns: [String], - primaryKeyColumns: [String], - changes: [PluginRowChange], - insertedRowData: [Int: [PluginCellValue]], - deletedRowIndices: Set, - insertedRowIndices: Set - ) -> [(statement: String, parameters: [PluginCellValue])]? { - let keySchema = lock.withLock { - extractKeySchema(from: _tableDescriptionCache[table]) - } - - let typeNames: [String] = columns.map { _ in "S" } - - let generator = DynamoDBStatementGenerator( - tableName: table, - columns: columns, - columnTypeNames: typeNames, - keySchema: keySchema - ) - do { - return try generator.generateStatements( - from: changes, - insertedRowData: insertedRowData, - deletedRowIndices: deletedRowIndices, - insertedRowIndices: insertedRowIndices - ) - } catch { - Self.logger.error("Statement generation failed: \(error.localizedDescription)") - return nil - } - } - - func allTablesMetadataSQL(schema: String?) -> String? { - nil - } - - // MARK: - Streaming - - func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { - try await boundedQueryFromStream(query: query, rowCap: rowCap) - } - - func streamRows(query: String) -> AsyncThrowingStream { - AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in - let streamTask = Task { - do { - try await self.performStreamRows(query: query, continuation: continuation) - } catch { - continuation.finish(throwing: error) - } - } - - continuation.onTermination = { @Sendable _ in - streamTask.cancel() - } - } - } - - private func performStreamRows( - query: String, - continuation: AsyncThrowingStream.Continuation - ) async throws { - guard let conn = connection else { - throw DynamoDBError.notConnected - } - - let trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines) - - if let parsed = DynamoDBQueryBuilder.parseScanQuery(trimmed) { - try await streamScan(parsed, conn: conn, continuation: continuation) - } else if let parsed = DynamoDBQueryBuilder.parseQueryQuery(trimmed) { - try await streamQuery(parsed, conn: conn, continuation: continuation) - } else { - try await streamPartiQL(trimmed, conn: conn, continuation: continuation) - } - - continuation.finish() - } - - private func streamScan( - _ parsed: DynamoDBParsedScanQuery, - conn: DynamoDBConnection, - continuation: AsyncThrowingStream.Continuation - ) async throws { - let keySchema = try await cachedKeySchema(parsed.tableName, conn: conn) - let hasFilters = !parsed.filters.isEmpty - var headerSent = false - var columns: [String] = [] - var lastEvaluatedKey: [String: DynamoDBAttributeValue]? - - repeat { - try Task.checkCancellation() - - let response = try await conn.scan( - tableName: parsed.tableName, - limit: 1000, - exclusiveStartKey: lastEvaluatedKey - ) - - var items = response.Items ?? [] - - if hasFilters { - items = applyClientFilters( - items: items, filters: parsed.filters, logicMode: parsed.logicMode - ) - } - - if !items.isEmpty { - if !headerSent { - columns = DynamoDBItemFlattener.unionColumns(from: items, keySchema: keySchema) - let typeNames = DynamoDBItemFlattener.columnTypeNames(for: columns, items: items) - continuation.yield(.header(PluginStreamHeader( - columns: columns, - columnTypeNames: typeNames, - estimatedRowCount: nil - ))) - headerSent = true - } - - let rows = DynamoDBItemFlattener.flatten(items: items, columns: columns) - if !rows.isEmpty { - continuation.yield(.rows(rows)) - } - } - - lastEvaluatedKey = response.LastEvaluatedKey - } while lastEvaluatedKey != nil - - if !headerSent { - let sampleResponse = try await conn.scan(tableName: parsed.tableName, limit: 1) - let sampleItems = sampleResponse.Items ?? [] - columns = DynamoDBItemFlattener.unionColumns(from: sampleItems, keySchema: keySchema) - let typeNames = DynamoDBItemFlattener.columnTypeNames(for: columns, items: sampleItems) - continuation.yield(.header(PluginStreamHeader( - columns: columns.isEmpty ? ["Result"] : columns, - columnTypeNames: typeNames.isEmpty ? ["String"] : typeNames, - estimatedRowCount: nil - ))) - } - } - - private func streamQuery( - _ parsed: DynamoDBParsedQueryQuery, - conn: DynamoDBConnection, - continuation: AsyncThrowingStream.Continuation - ) async throws { - let keySchema = try await cachedKeySchema(parsed.tableName, conn: conn) - - var expressionValues: [String: DynamoDBAttributeValue] = [:] - switch parsed.partitionKeyType { - case "N": - expressionValues[":pkval"] = .number(parsed.partitionKeyValue) - default: - expressionValues[":pkval"] = .string(parsed.partitionKeyValue) - } - let keyCondition = "\(parsed.partitionKeyName) = :pkval" - - var headerSent = false - var columns: [String] = [] - var lastEvaluatedKey: [String: DynamoDBAttributeValue]? - - repeat { - try Task.checkCancellation() - - let response = try await conn.query( - tableName: parsed.tableName, - keyConditionExpression: keyCondition, - expressionAttributeValues: expressionValues, - limit: 1000, - exclusiveStartKey: lastEvaluatedKey - ) - - var items = response.Items ?? [] - - if !parsed.filters.isEmpty { - items = applyClientFilters( - items: items, filters: parsed.filters, logicMode: parsed.logicMode - ) - } - - if !headerSent && !items.isEmpty { - columns = DynamoDBItemFlattener.unionColumns(from: items, keySchema: keySchema) - let typeNames = DynamoDBItemFlattener.columnTypeNames(for: columns, items: items) - continuation.yield(.header(PluginStreamHeader( - columns: columns, - columnTypeNames: typeNames, - estimatedRowCount: nil - ))) - headerSent = true - } - - if !items.isEmpty { - let rows = DynamoDBItemFlattener.flatten(items: items, columns: columns) - if !rows.isEmpty { - continuation.yield(.rows(rows)) - } - } - - lastEvaluatedKey = response.LastEvaluatedKey - } while lastEvaluatedKey != nil - - if !headerSent { - columns = DynamoDBItemFlattener.unionColumns(from: [], keySchema: keySchema) - let typeNames = DynamoDBItemFlattener.columnTypeNames(for: columns, items: []) - continuation.yield(.header(PluginStreamHeader( - columns: columns.isEmpty ? ["Result"] : columns, - columnTypeNames: typeNames.isEmpty ? ["String"] : typeNames, - estimatedRowCount: nil - ))) - } - } - - private func streamPartiQL( - _ statement: String, - conn: DynamoDBConnection, - continuation: AsyncThrowingStream.Continuation - ) async throws { - let tableName = DynamoDBPartiQLParser.extractTableName(statement) - let keySchema: [(name: String, keyType: String)] - if let name = tableName { - keySchema = try await cachedKeySchema(name, conn: conn) - } else { - keySchema = [] - } - - var headerSent = false - var columns: [String] = [] - var nextToken: String? - - let firstResponse = try await conn.executeStatement(statement: statement) - var items = firstResponse.Items ?? [] - - if !items.isEmpty { - columns = DynamoDBItemFlattener.unionColumns(from: items, keySchema: keySchema) - let typeNames = DynamoDBItemFlattener.columnTypeNames(for: columns, items: items) - continuation.yield(.header(PluginStreamHeader( - columns: columns, - columnTypeNames: typeNames, - estimatedRowCount: nil - ))) - headerSent = true - - let rows = DynamoDBItemFlattener.flatten(items: items, columns: columns) - for row in rows { - continuation.yield(.rows([row])) - } - } - - nextToken = firstResponse.NextToken - - while let token = nextToken { - try Task.checkCancellation() - - let response = try await conn.executeStatement( - statement: statement, nextToken: token - ) - items = response.Items ?? [] - - if !headerSent && !items.isEmpty { - columns = DynamoDBItemFlattener.unionColumns(from: items, keySchema: keySchema) - let typeNames = DynamoDBItemFlattener.columnTypeNames(for: columns, items: items) - continuation.yield(.header(PluginStreamHeader( - columns: columns, - columnTypeNames: typeNames, - estimatedRowCount: nil - ))) - headerSent = true - } - - if !items.isEmpty { - let rows = DynamoDBItemFlattener.flatten(items: items, columns: columns) - if !rows.isEmpty { - continuation.yield(.rows(rows)) - } - } - - nextToken = response.NextToken - } - - if !headerSent { - if let name = tableName { - let sampleResponse = try await conn.scan(tableName: name, limit: 1) - let sampleItems = sampleResponse.Items ?? [] - columns = DynamoDBItemFlattener.unionColumns(from: sampleItems, keySchema: keySchema) - let typeNames = DynamoDBItemFlattener.columnTypeNames(for: columns, items: sampleItems) - continuation.yield(.header(PluginStreamHeader( - columns: columns.isEmpty ? ["Result"] : columns, - columnTypeNames: typeNames.isEmpty ? ["String"] : typeNames, - estimatedRowCount: nil - ))) - } else { - continuation.yield(.header(PluginStreamHeader( - columns: ["Result"], - columnTypeNames: ["String"], - estimatedRowCount: nil - ))) - } - } - } - - // MARK: - Tagged Query Execution - - /// Issues DeleteTable for the driver's own `DROP TABLE "x"` statement. - /// - /// The cached description goes with it, or a table recreated under the same name would be read - /// through the old key schema. DeleteTable returns once the table is DELETING rather than gone, - /// so the row count is reported as the one table the request named, not as work completed. - private func executeDropTable( - _ table: String, conn: DynamoDBConnection, startTime: Date - ) async throws -> PluginQueryResult { - _ = try await conn.deleteTable(tableName: table) - lock.withLock { _tableDescriptionCache.removeValue(forKey: table) } - return PluginQueryResult( - columns: ["result"], - columnTypeNames: ["String"], - rows: [[.text("DELETING")]], - rowsAffected: 1, - executionTime: Date().timeIntervalSince(startTime) - ) - } - - private func executeTaggedQuery( - _ query: String, conn: DynamoDBConnection, startTime: Date - ) async throws -> PluginQueryResult { - Self.logger.debug("executeTaggedQuery called") - if let parsed = DynamoDBQueryBuilder.parseScanQuery(query) { - return try await executeScan(parsed, conn: conn, startTime: startTime) - } - - if let parsed = DynamoDBQueryBuilder.parseQueryQuery(query) { - return try await executeDynamoDBQuery(parsed, conn: conn, startTime: startTime) - } - - if let parsed = DynamoDBQueryBuilder.parseCountQuery(query) { - let count = try await countItems(tableName: parsed.tableName, conn: conn) - return PluginQueryResult( - columns: ["Count"], - columnTypeNames: ["Int64"], - rows: [[.text(String(count))]], - rowsAffected: 0, - executionTime: Date().timeIntervalSince(startTime) - ) - } - - throw DynamoDBError.serverError("Invalid tagged query format") - } - - // MARK: - Scan Execution - - private func executeScan( - _ parsed: DynamoDBParsedScanQuery, - conn: DynamoDBConnection, - startTime: Date - ) async throws -> PluginQueryResult { - let keySchema = try await cachedKeySchema(parsed.tableName, conn: conn) - - var allItems: [[String: DynamoDBAttributeValue]] = [] - var lastEvaluatedKey: [String: DynamoDBAttributeValue]? - let fetchLimit = min(parsed.limit + parsed.offset, Self.maxItems) - - let hasFilters = !parsed.filters.isEmpty - - repeat { - let batchLimit = min(fetchLimit - allItems.count, 1000) - let response = try await conn.scan( - tableName: parsed.tableName, - limit: batchLimit, - exclusiveStartKey: lastEvaluatedKey - ) - var items = response.Items ?? [] - - if hasFilters { - items = applyClientFilters( - items: items, filters: parsed.filters, logicMode: parsed.logicMode - ) - } - - allItems.append(contentsOf: items) - lastEvaluatedKey = response.LastEvaluatedKey - - if lastEvaluatedKey == nil || allItems.count >= fetchLimit { break } - } while true - - // Apply pagination - let total = allItems.count - let start = min(parsed.offset, total) - let end = min(start + parsed.limit, total) - let pageItems = start < end ? Array(allItems[start.. PluginQueryResult { - let keySchema = try await cachedKeySchema(parsed.tableName, conn: conn) - - var expressionValues: [String: DynamoDBAttributeValue] = [:] - switch parsed.partitionKeyType { - case "N": - expressionValues[":pkval"] = .number(parsed.partitionKeyValue) - default: - expressionValues[":pkval"] = .string(parsed.partitionKeyValue) - } - - let keyCondition = "\(parsed.partitionKeyName) = :pkval" - - var allItems: [[String: DynamoDBAttributeValue]] = [] - var lastEvaluatedKey: [String: DynamoDBAttributeValue]? - let fetchLimit = min(parsed.limit + parsed.offset, Self.maxItems) - - repeat { - let batchLimit = min(fetchLimit - allItems.count, 1000) - let response = try await conn.query( - tableName: parsed.tableName, - keyConditionExpression: keyCondition, - expressionAttributeValues: expressionValues, - limit: batchLimit, - exclusiveStartKey: lastEvaluatedKey - ) - let fetched = response.Items ?? [] - allItems.append(contentsOf: fetched) - lastEvaluatedKey = response.LastEvaluatedKey - - if lastEvaluatedKey == nil || allItems.count >= fetchLimit { break } - } while true - - if !parsed.filters.isEmpty { - allItems = applyClientFilters( - items: allItems, filters: parsed.filters, logicMode: parsed.logicMode - ) - } - - let start = min(parsed.offset, allItems.count) - let end = min(start + parsed.limit, allItems.count) - let pageItems = start < end ? Array(allItems[start.. String? { + let queryFilters = filters.map { PluginQueryFilter(column: $0.column, op: $0.op, value: $0.value) } + return buildFilteredQuery( + table: table, schema: nil, queryFilters: queryFilters, logicMode: logicMode, + sortColumns: sortColumns, columns: columns, limit: limit, offset: offset, columnKinds: [:] ) } - // MARK: - PartiQL Execution - - private func executePartiQL( - _ statement: String, conn: DynamoDBConnection, startTime: Date - ) async throws -> PluginQueryResult { - let queryType = DynamoDBPartiQLParser.queryType(statement) - let response = try await conn.executeStatement(statement: statement) - - switch queryType { - case .select: - let items = response.Items ?? [] - if items.isEmpty { - let tableName = DynamoDBPartiQLParser.extractTableName(statement) - var emptyColumns: [String] = [] - var emptyTypeNames: [String] = [] - if let name = tableName, - let conn = connection - { - let keySchema = try await cachedKeySchema(name, conn: conn) - let sampleResponse = try await conn.scan(tableName: name, limit: 1) - let sampleItems = sampleResponse.Items ?? [] - emptyColumns = DynamoDBItemFlattener.unionColumns(from: sampleItems, keySchema: keySchema) - emptyTypeNames = DynamoDBItemFlattener.columnTypeNames(for: emptyColumns, items: sampleItems) - } - return PluginQueryResult( - columns: emptyColumns.isEmpty ? ["Result"] : emptyColumns, - columnTypeNames: emptyTypeNames.isEmpty ? ["String"] : emptyTypeNames, - rows: [], - rowsAffected: 0, - executionTime: Date().timeIntervalSince(startTime) - ) - } - - let tableName = DynamoDBPartiQLParser.extractTableName(statement) - let keySchema: [(name: String, keyType: String)] - if let name = tableName { - keySchema = try await cachedKeySchema(name, conn: conn) - } else { - keySchema = [] - } - - let columns = DynamoDBItemFlattener.unionColumns(from: items, keySchema: keySchema) - let typeNames = DynamoDBItemFlattener.columnTypeNames(for: columns, items: items) - let rows = DynamoDBItemFlattener.flatten(items: items, columns: columns) - - return PluginQueryResult( - columns: columns, - columnTypeNames: typeNames, - rows: rows, - rowsAffected: 0, - executionTime: Date().timeIntervalSince(startTime) - ) - - case .insert: - return PluginQueryResult( - columns: ["Result"], - columnTypeNames: ["String"], - rows: [[.text("Item inserted successfully")]], - rowsAffected: 1, - executionTime: Date().timeIntervalSince(startTime) - ) - - case .update: - return PluginQueryResult( - columns: ["Result"], - columnTypeNames: ["String"], - rows: [[.text("Item updated successfully")]], - rowsAffected: 1, - executionTime: Date().timeIntervalSince(startTime) - ) - - case .delete: - return PluginQueryResult( - columns: ["Result"], - columnTypeNames: ["String"], - rows: [[.text("Item deleted successfully")]], - rowsAffected: 1, - executionTime: Date().timeIntervalSince(startTime) - ) - - case .unknown: - let items = response.Items ?? [] - if !items.isEmpty { - let columns = DynamoDBItemFlattener.unionColumns(from: items, keySchema: []) - let typeNames = DynamoDBItemFlattener.columnTypeNames(for: columns, items: items) - let rows = DynamoDBItemFlattener.flatten(items: items, columns: columns) - return PluginQueryResult( - columns: columns, - columnTypeNames: typeNames, - rows: rows, - rowsAffected: 0, - executionTime: Date().timeIntervalSince(startTime) - ) - } - return PluginQueryResult( - columns: ["Result"], - columnTypeNames: ["String"], - rows: [[.text("Statement executed")]], - rowsAffected: 0, - executionTime: Date().timeIntervalSince(startTime) - ) - } - } - - // MARK: - Helpers - - private func cachedDescribeTable(_ tableName: String, conn: DynamoDBConnection) async throws -> TableDescription { - if let cached = lock.withLock({ _tableDescriptionCache[tableName] }) { - return cached + private func browseStatement( + _ request: DynamoDBBrowseRequest, + sortColumns: [(columnIndex: Int, ascending: Bool)], + columns: [String], + limit: Int, + offset: Int + ) -> String { + let order = sortColumns.compactMap { sort -> DynamoDBOrderTerm? in + guard columns.indices.contains(sort.columnIndex) else { return nil } + return DynamoDBOrderTerm(attribute: columns[sort.columnIndex], descending: !sort.ascending) } - - let response = try await conn.describeTable(tableName: tableName) - let tableDesc = response.Table - lock.withLock { _tableDescriptionCache[tableName] = tableDesc } - return tableDesc - } - - private func cachedKeySchema( - _ tableName: String, conn: DynamoDBConnection - ) async throws -> [(name: String, keyType: String)] { - let tableDesc = try await cachedDescribeTable(tableName, conn: conn) - return extractKeySchema(from: tableDesc) - } - - private func extractKeySchema(from tableDesc: TableDescription?) -> [(name: String, keyType: String)] { - guard let keySchema = tableDesc?.KeySchema else { return [] } - return keySchema.map { (name: $0.AttributeName, keyType: $0.KeyType) } + let window = DynamoDBReadWindow(order: order, limit: limit, offset: offset) + return DynamoDBStatement.browse(request, window: window).text } - private func extractAttributeTypes(from tableDesc: TableDescription?) -> [String: String] { - guard let defs = tableDesc?.AttributeDefinitions else { return [:] } - var result: [String: String] = [:] - for def in defs { - result[def.AttributeName] = def.AttributeType - } - return result + func defaultExportQuery(table: String) -> String? { + DynamoDBStatement.browse( + DynamoDBBrowseRequest(table: table, filters: [], matchAll: true, columns: []), + window: DynamoDBReadWindow() + ).text } - private func countItems(tableName: String, conn: DynamoDBConnection) async throws -> Int { - // Use DescribeTable for approximate count (updated every ~6 hours) - let tableDesc = try await cachedDescribeTable(tableName, conn: conn) - if let count = tableDesc.ItemCount { - return Int(count) + func injectRowLimit(_ sql: String, limit: Int) -> String? { + guard let statement = try? DynamoDBStatement.parse(sql) else { return nil } + switch statement { + case .partiQL(let text, var window): + guard DynamoDBPartiQL.kind(of: text) == .select else { return nil } + window.limit = min(window.limit ?? limit, limit) + return DynamoDBStatement.partiQL(text: text, window: window).text + case .browse(let request, var window): + window.limit = min(window.limit ?? limit, limit) + return DynamoDBStatement.browse(request, window: window).text + case .apiCall(let call, var window): + guard call.operation == .scan || call.operation == .query else { return nil } + window.limit = min(window.limit ?? limit, limit) + return DynamoDBStatement.apiCall(call, window: window).text } - - // Fallback: do a count scan - var total = 0 - var lastKey: [String: DynamoDBAttributeValue]? - repeat { - let response = try await conn.scan( - tableName: tableName, - limit: 10000, - exclusiveStartKey: lastKey, - select: "COUNT" - ) - let batchCount = response.Count ?? 0 - total += batchCount - lastKey = response.LastEvaluatedKey - if lastKey == nil { break } - } while true - - return total } - private func countFilteredScanItems( - tableName: String, - conn: DynamoDBConnection, - filters: [DynamoDBFilterSpec], - logicMode: String - ) async throws -> Int { - var total = 0 - var lastKey: [String: DynamoDBAttributeValue]? - repeat { - let response = try await conn.scan( - tableName: tableName, - limit: 1000, - exclusiveStartKey: lastKey - ) - var items = response.Items ?? [] - if !filters.isEmpty { - items = applyClientFilters(items: items, filters: filters, logicMode: logicMode) - } - total += items.count - lastKey = response.LastEvaluatedKey - if lastKey == nil { break } - } while true - return total + func truncateTableStatements(table: String, schema: String?, cascade: Bool) -> [String]? { + nil } - private func countQueryItems( - parsed: DynamoDBParsedQueryQuery, - conn: DynamoDBConnection - ) async throws -> Int { - var expressionValues: [String: DynamoDBAttributeValue] = [:] - switch parsed.partitionKeyType { - case "N": - expressionValues[":pkval"] = .number(parsed.partitionKeyValue) - default: - expressionValues[":pkval"] = .string(parsed.partitionKeyValue) - } - let keyCondition = "\(parsed.partitionKeyName) = :pkval" - - var total = 0 - var lastKey: [String: DynamoDBAttributeValue]? - repeat { - let response = try await conn.query( - tableName: parsed.tableName, - keyConditionExpression: keyCondition, - expressionAttributeValues: expressionValues, - limit: 10000, - exclusiveStartKey: lastKey, - select: "COUNT" - ) - total += response.Count ?? 0 - lastKey = response.LastEvaluatedKey - if lastKey == nil { break } - } while true - return total + func allTablesMetadataSQL(schema: String?) -> String? { + nil } - private func applyClientFilters( - items: [[String: DynamoDBAttributeValue]], - filters: [DynamoDBFilterSpec], - logicMode: String - ) -> [[String: DynamoDBAttributeValue]] { - guard !filters.isEmpty else { return items } - return items.filter { item in - if logicMode.uppercased() == "OR" { - return filters.contains { matchesItemFilter(item, filter: $0) } - } - return filters.allSatisfy { matchesItemFilter(item, filter: $0) } - } + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { + [] } - private func matchesItemFilter( - _ item: [String: DynamoDBAttributeValue], - filter: DynamoDBFilterSpec - ) -> Bool { - if filter.column == "*" { - return item.values.contains { attrValue in - matchesFilter( - DynamoDBItemFlattener.attributeValueToString(attrValue), - op: filter.op, value: filter.value, ignoresCase: filter.ignoresCase - ) - } - } - - guard let attrValue = item[filter.column] else { return false } - let str = DynamoDBItemFlattener.attributeValueToString(attrValue) - return matchesFilter(str, op: filter.op, value: filter.value, ignoresCase: filter.ignoresCase) + func fetchViewDefinition(view: String, schema: String?) async throws -> String { + throw DynamoDBError.invalidStatement(String(localized: "DynamoDB has no views")) } - private func matchesFilter(_ str: String, op: String, value: String, ignoresCase: Bool) -> Bool { - let subject = ignoresCase ? str.lowercased() : str - let needle = ignoresCase ? value.lowercased() : value - switch op.uppercased() { - case "=": - return subject == needle - case "!=", "<>": - return subject != needle - case "CONTAINS": - return subject.contains(needle) - case "STARTS WITH": - return subject.hasPrefix(needle) - case "ENDS WITH": - return subject.hasSuffix(needle) - case ">": - if let d1 = Double(str), let d2 = Double(value) { return d1 > d2 } - return str > value - case "<": - if let d1 = Double(str), let d2 = Double(value) { return d1 < d2 } - return str < value - case ">=": - if let d1 = Double(str), let d2 = Double(value) { return d1 >= d2 } - return str >= value - case "<=": - if let d1 = Double(str), let d2 = Double(value) { return d1 <= d2 } - return str <= value - default: - Self.logger.warning("Unknown filter operator: \(op)") - return false - } + func fetchDatabases() async throws -> [String] { + ["default"] } - private func formatBytes(_ bytes: Int64) -> String { - let formatter = ByteCountFormatter() - formatter.countStyle = .binary - return formatter.string(fromByteCount: bytes) + func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { + PluginDatabaseMetadata(name: database) } } diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBQueryBuilder.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBQueryBuilder.swift deleted file mode 100644 index 33525dacf8..0000000000 --- a/Plugins/DynamoDBDriverPlugin/DynamoDBQueryBuilder.swift +++ /dev/null @@ -1,275 +0,0 @@ -// -// DynamoDBQueryBuilder.swift -// DynamoDBDriverPlugin -// -// Builds internal tagged query strings for DynamoDB table browsing and filtering. -// - -import Foundation -import TableProPluginKit - -// MARK: - Filter Encoding - -struct DynamoDBFilterSpec: Codable { - let column: String - let op: String - let value: String - var caseSensitive: Bool? - - /// Operators that matched without regard to case before a filter row could say otherwise. - /// A scan tag saved by an older build carries no setting and must keep behaving as it did. - private static let ignoreCaseByDefault: Set = ["CONTAINS", "STARTS WITH", "ENDS WITH"] - - var ignoresCase: Bool { - guard let caseSensitive else { return Self.ignoreCaseByDefault.contains(op.uppercased()) } - return !caseSensitive - } -} - -// MARK: - Parsed Query Types - -struct DynamoDBParsedScanQuery { - let tableName: String - let limit: Int - let offset: Int - let filters: [DynamoDBFilterSpec] - let logicMode: String -} - -struct DynamoDBParsedQueryQuery { - let tableName: String - let partitionKeyName: String - let partitionKeyValue: String - let partitionKeyType: String - let limit: Int - let offset: Int - let filters: [DynamoDBFilterSpec] - let logicMode: String -} - -struct DynamoDBParsedCountQuery { - let tableName: String - let filterColumn: String? - let filterOp: String? - let filterValue: String? -} - -// MARK: - Query Builder - -struct DynamoDBQueryBuilder { - static let scanTag = "DYNAMODB_SCAN:" - static let queryTag = "DYNAMODB_QUERY:" - static let countTag = "DYNAMODB_COUNT:" - - func buildBrowseQuery( - table: String, - sortColumns: [(columnIndex: Int, ascending: Bool)], - limit: Int, - offset: Int - ) -> String { - Self.encodeScanQuery(tableName: table, limit: limit, offset: offset, filters: [], logicMode: "AND") - } - - func buildFilteredQuery( - table: String, - filters: [PluginQueryFilter], - logicMode: String, - sortColumns: [(columnIndex: Int, ascending: Bool)], - columns: [String], - limit: Int, - offset: Int, - keySchema: [(name: String, keyType: String)], - attributeTypes: [String: String] = [:] - ) -> String? { - let partitionKey = keySchema.first(where: { $0.keyType == "HASH" }) - if let pk = partitionKey, - let pkFilter = filters.first(where: { $0.column == pk.name && $0.op == "=" }) - { - let pkType = attributeTypes[pk.name] ?? "S" - let remainingFilters = filters.filter { !($0.column == pk.name && $0.op == "=") } - let specs = remainingFilters.map { - DynamoDBFilterSpec(column: $0.column, op: $0.op, value: $0.value, caseSensitive: $0.isCaseSensitive) - } - return Self.encodeQueryQuery( - tableName: table, - partitionKeyName: pk.name, - partitionKeyValue: pkFilter.value, - partitionKeyType: pkType, - limit: limit, - offset: offset, - filters: specs, - logicMode: logicMode - ) - } - - let specs = filters.map { - DynamoDBFilterSpec(column: $0.column, op: $0.op, value: $0.value, caseSensitive: $0.isCaseSensitive) - } - return Self.encodeScanQuery( - tableName: table, limit: limit, offset: offset, - filters: specs, logicMode: logicMode - ) - } - - // MARK: - Encoding - - private static func encodeScanQuery( - tableName: String, - limit: Int, - offset: Int, - filters: [DynamoDBFilterSpec], - logicMode: String - ) -> String { - let b64Table = Data(tableName.utf8).base64EncodedString() - let filtersJson = (try? JSONEncoder().encode(filters)) ?? Data() - let b64Filters = filtersJson.base64EncodedString() - let b64Logic = Data(logicMode.utf8).base64EncodedString() - return "\(scanTag)\(b64Table):\(limit):\(offset):\(b64Filters):\(b64Logic)" - } - - private static func encodeQueryQuery( - tableName: String, - partitionKeyName: String, - partitionKeyValue: String, - partitionKeyType: String, - limit: Int, - offset: Int, - filters: [DynamoDBFilterSpec] = [], - logicMode: String = "AND" - ) -> String { - let b64Table = Data(tableName.utf8).base64EncodedString() - let b64PkName = Data(partitionKeyName.utf8).base64EncodedString() - let b64PkValue = Data(partitionKeyValue.utf8).base64EncodedString() - let b64PkType = Data(partitionKeyType.utf8).base64EncodedString() - let filtersJson = (try? JSONEncoder().encode(filters)) ?? Data() - let b64Filters = filtersJson.base64EncodedString() - let b64Logic = Data(logicMode.utf8).base64EncodedString() - return "\(queryTag)\(b64Table):\(limit):\(offset):\(b64PkName):\(b64PkValue):\(b64PkType):\(b64Filters):\(b64Logic)" - } - - static func encodeCountQuery( - tableName: String, - filterColumn: String? = nil, - filterOp: String? = nil, - filterValue: String? = nil - ) -> String { - let b64Table = Data(tableName.utf8).base64EncodedString() - let b64FilterCol = Data((filterColumn ?? "").utf8).base64EncodedString() - let b64FilterOp = Data((filterOp ?? "").utf8).base64EncodedString() - let b64FilterVal = Data((filterValue ?? "").utf8).base64EncodedString() - return "\(countTag)\(b64Table):\(b64FilterCol):\(b64FilterOp):\(b64FilterVal)" - } - - // MARK: - Decoding - - static func parseScanQuery(_ query: String) -> DynamoDBParsedScanQuery? { - guard query.hasPrefix(scanTag) else { return nil } - let body = String(query.dropFirst(scanTag.count)) - let parts = body.components(separatedBy: ":") - guard parts.count >= 5 else { return nil } - - guard let tableData = Data(base64Encoded: parts[0]), - let tableName = String(data: tableData, encoding: .utf8), - let limit = Int(parts[1]), - let offset = Int(parts[2]) - else { return nil } - - let filters: [DynamoDBFilterSpec] - if let filtersData = Data(base64Encoded: parts[3]), - let decoded = try? JSONDecoder().decode([DynamoDBFilterSpec].self, from: filtersData) - { - filters = decoded - } else { - filters = [] - } - - let logicMode = decodeBase64(parts[4]) ?? "AND" - - return DynamoDBParsedScanQuery( - tableName: tableName, - limit: limit, - offset: offset, - filters: filters, - logicMode: logicMode - ) - } - - static func parseQueryQuery(_ query: String) -> DynamoDBParsedQueryQuery? { - guard query.hasPrefix(queryTag) else { return nil } - let body = String(query.dropFirst(queryTag.count)) - let parts = body.components(separatedBy: ":") - guard parts.count >= 6 else { return nil } - - guard let tableData = Data(base64Encoded: parts[0]), - let tableName = String(data: tableData, encoding: .utf8), - let limit = Int(parts[1]), - let offset = Int(parts[2]), - let pkName = decodeBase64(parts[3]), - let pkValue = decodeBase64(parts[4]), - let pkType = decodeBase64(parts[5]) - else { return nil } - - let filters: [DynamoDBFilterSpec] - if parts.count >= 7, - let filtersData = Data(base64Encoded: parts[6]), - let decoded = try? JSONDecoder().decode([DynamoDBFilterSpec].self, from: filtersData) - { - filters = decoded - } else { - filters = [] - } - - let logicMode: String - if parts.count >= 8, let decoded = decodeBase64(parts[7]) { - logicMode = decoded - } else { - logicMode = "AND" - } - - return DynamoDBParsedQueryQuery( - tableName: tableName, - partitionKeyName: pkName, - partitionKeyValue: pkValue, - partitionKeyType: pkType, - limit: limit, - offset: offset, - filters: filters, - logicMode: logicMode - ) - } - - static func parseCountQuery(_ query: String) -> DynamoDBParsedCountQuery? { - guard query.hasPrefix(countTag) else { return nil } - let body = String(query.dropFirst(countTag.count)) - let parts = body.components(separatedBy: ":") - guard parts.count >= 4 else { return nil } - - guard let tableData = Data(base64Encoded: parts[0]), - let tableName = String(data: tableData, encoding: .utf8) - else { return nil } - - let filterColumn = decodeBase64(parts[1]) - let filterOp = decodeBase64(parts[2]) - let filterValue = decodeBase64(parts[3...].joined(separator: ":")) - - return DynamoDBParsedCountQuery( - tableName: tableName, - filterColumn: filterColumn?.isEmpty == true ? nil : filterColumn, - filterOp: filterOp?.isEmpty == true ? nil : filterOp, - filterValue: filterValue?.isEmpty == true ? nil : filterValue - ) - } - - static func isTaggedQuery(_ query: String) -> Bool { - query.hasPrefix(scanTag) || query.hasPrefix(queryTag) || query.hasPrefix(countTag) - } - - // MARK: - Helpers - - private static func decodeBase64(_ string: String) -> String? { - guard let data = Data(base64Encoded: string), - let decoded = String(data: data, encoding: .utf8) - else { return nil } - return decoded - } -} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBRetryPolicy.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBRetryPolicy.swift new file mode 100644 index 0000000000..68294a15ac --- /dev/null +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBRetryPolicy.swift @@ -0,0 +1,82 @@ +import Foundation + +/// When a failed request is sent again, and after how long. +/// +/// The delays follow the AWS SDKs' standard mode for DynamoDB: full jitter over an exponential +/// base, 25 ms for a transient failure and 1 s for throttling, capped at 20 s, four attempts. +/// Throttling means DynamoDB refused the request, so any request may go again. A 5xx or a dropped +/// connection may have been applied, so only a request that cannot apply twice goes again. +struct DynamoDBRetryPolicy: Sendable { + static let maximumAttempts = 4 + static let transientBase: TimeInterval = 0.025 + static let throttlingBase: TimeInterval = 1.0 + static let maximumDelay: TimeInterval = 20 + + enum Decision: Equatable { + case retry(after: TimeInterval) + case refreshCredentialsAndRetry + case correctClockAndRetry + case fail + } + + var random: @Sendable () -> Double = { Double.random(in: 0...1) } + + func decision( + for error: DynamoDBError, + attempt: Int, + operation: DynamoDBOperation, + body: DynamoDBJSON, + alreadyRefreshedCredentials: Bool, + alreadyCorrectedClock: Bool + ) -> Decision { + guard attempt < Self.maximumAttempts else { return .fail } + switch error { + case .service(let service): + switch service.category { + case .throttling: + return .retry(after: delay(base: Self.throttlingBase, attempt: attempt)) + case .transient: + guard Self.isIdempotent(operation, body: body) else { return .fail } + return .retry(after: delay(base: Self.transientBase, attempt: attempt)) + case .expiredCredentials: + return alreadyRefreshedCredentials ? .fail : .refreshCredentialsAndRetry + case .clockSkew: + return alreadyCorrectedClock ? .fail : .correctClockAndRetry + case .authentication, .fatal: + return .fail + } + case .transport: + guard Self.isIdempotent(operation, body: body) else { return .fail } + return .retry(after: delay(base: Self.transientBase, attempt: attempt)) + default: + return .fail + } + } + + func delay(base: TimeInterval, attempt: Int) -> TimeInterval { + let ceiling = min(Self.maximumDelay, base * pow(2, Double(max(attempt - 1, 0)))) + return random() * ceiling + } + + /// Whether sending the request twice leaves the table as sending it once would. + static func isIdempotent(_ operation: DynamoDBOperation, body: DynamoDBJSON) -> Bool { + if operation.isRead { return true } + switch operation { + case .putItem, .deleteItem: + return body["ConditionExpression"] == nil && body["Expected"] == nil + case .batchWriteItem: + return true + case .transactWriteItems, .executeTransaction: + return body["ClientRequestToken"]?.stringValue?.isEmpty == false + case .executeStatement: + return DynamoDBPartiQL.kind(of: body["Statement"]?.stringValue ?? "") == .select + case .batchExecuteStatement: + let statements = body["Statements"]?.arrayValue ?? [] + return !statements.isEmpty && statements.allSatisfy { + DynamoDBPartiQL.kind(of: $0["Statement"]?.stringValue ?? "") == .select + } + default: + return false + } + } +} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBSigner.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBSigner.swift new file mode 100644 index 0000000000..35df1b2829 --- /dev/null +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBSigner.swift @@ -0,0 +1,114 @@ +import Foundation +import TableProPluginKit + +/// Signature Version 4 for DynamoDB's JSON protocol. +enum DynamoDBSigner { + static let service = "dynamodb" + + struct Timestamps: Equatable { + let amzDate: String + let dateStamp: String + } + + static func timestamps(for date: Date) -> Timestamps { + let formatter = DateFormatter() + formatter.calendar = Calendar(identifier: .gregorian) + formatter.locale = Locale(identifier: "en_US_POSIX") + formatter.timeZone = TimeZone(identifier: "UTC") + formatter.dateFormat = "yyyyMMdd'T'HHmmss'Z'" + let amzDate = formatter.string(from: date) + return Timestamps(amzDate: amzDate, dateStamp: String(amzDate.prefix(8))) + } + + /// The `Host` value: the port only when it is not the scheme's default, as every AWS SDK sends it. + static func hostHeader(for url: URL) -> String { + let bare = url.host ?? "" + let host = bare.contains(":") ? "[\(bare)]" : bare + guard let port = url.port else { return host } + let scheme = url.scheme?.lowercased() + let isDefault = (scheme == "https" && port == 443) || (scheme == "http" && port == 80) + return isDefault ? host : "\(host):\(port)" + } + + /// The canonical URI keeps the path exactly as it is sent, percent-encoding and trailing slash + /// included. `URL.path` decodes `%20` and drops the slash, which signs a path the server never + /// sees. + static func canonicalURI(for url: URL) -> String { + let path = url.path(percentEncoded: true) + return path.isEmpty ? "/" : path + } + + static func sign( + _ request: inout URLRequest, + body: Data, + credentials: AWSCredentials, + region: String, + date: Date + ) { + guard let url = request.url else { return } + let stamps = timestamps(for: date) + var headers: [String: String] = [ + "content-type": request.value(forHTTPHeaderField: "Content-Type") ?? "", + "host": hostHeader(for: url), + "x-amz-date": stamps.amzDate, + "x-amz-target": request.value(forHTTPHeaderField: "X-Amz-Target") ?? "" + ] + if let token = credentials.sessionToken, !token.isEmpty { + headers["x-amz-security-token"] = token + } + let signedNames = headers.keys.sorted() + let canonicalHeaders = signedNames.map { "\($0):\(canonicalValue(headers[$0] ?? ""))\n" }.joined() + let signedHeaders = signedNames.joined(separator: ";") + let canonicalRequest = [ + request.httpMethod ?? "POST", + canonicalURI(for: url), + canonicalQuery(for: url), + canonicalHeaders, + signedHeaders, + AWSSigV4.sha256Hex(body) + ].joined(separator: "\n") + + let scope = "\(stamps.dateStamp)/\(region)/\(service)/aws4_request" + let stringToSign = [ + "AWS4-HMAC-SHA256", + stamps.amzDate, + scope, + AWSSigV4.sha256Hex(Data(canonicalRequest.utf8)) + ].joined(separator: "\n") + let key = AWSSigV4.deriveSigningKey( + secretKey: credentials.secretAccessKey, dateStamp: stamps.dateStamp, region: region, service: service + ) + let signature = AWSSigV4.hmacHex(key: key, data: Data(stringToSign.utf8)) + + request.setValue(headers["host"], forHTTPHeaderField: "Host") + request.setValue(stamps.amzDate, forHTTPHeaderField: "X-Amz-Date") + if let token = headers["x-amz-security-token"] { + request.setValue(token, forHTTPHeaderField: "X-Amz-Security-Token") + } + request.setValue( + "AWS4-HMAC-SHA256 Credential=\(credentials.accessKeyId)/\(scope), " + + "SignedHeaders=\(signedHeaders), Signature=\(signature)", + forHTTPHeaderField: "Authorization" + ) + } + + /// The query string as SigV4 signs it: every name and value encoded, sorted by name then value. + static func canonicalQuery(for url: URL) -> String { + guard let items = URLComponents(url: url, resolvingAgainstBaseURL: false)?.queryItems, !items.isEmpty else { + return "" + } + let pairs: [(name: String, value: String)] = items.map { item in + (AWSSigV4.uriEncode(item.name), AWSSigV4.uriEncode(item.value ?? "")) + } + let sorted = pairs.sorted { lhs, rhs in + lhs.name == rhs.name ? lhs.value < rhs.value : lhs.name < rhs.name + } + return sorted.map { "\($0.name)=\($0.value)" }.joined(separator: "&") + } + + private static func canonicalValue(_ value: String) -> String { + value.trimmingCharacters(in: .whitespaces) + .split(separator: " ", omittingEmptySubsequences: true) + .joined(separator: " ") + } +} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBStatement.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBStatement.swift new file mode 100644 index 0000000000..6689902c3b --- /dev/null +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBStatement.swift @@ -0,0 +1,203 @@ +import Foundation + +/// The DynamoDB actions the editor accepts as ` {request JSON}`. +enum DynamoDBOperation: String, CaseIterable, Sendable { + case scan = "Scan" + case query = "Query" + case getItem = "GetItem" + case batchGetItem = "BatchGetItem" + case transactGetItems = "TransactGetItems" + case describeTable = "DescribeTable" + case listTables = "ListTables" + case describeTimeToLive = "DescribeTimeToLive" + case describeContinuousBackups = "DescribeContinuousBackups" + case listTagsOfResource = "ListTagsOfResource" + case describeLimits = "DescribeLimits" + case putItem = "PutItem" + case updateItem = "UpdateItem" + case deleteItem = "DeleteItem" + case batchWriteItem = "BatchWriteItem" + case transactWriteItems = "TransactWriteItems" + case executeStatement = "ExecuteStatement" + case executeTransaction = "ExecuteTransaction" + case batchExecuteStatement = "BatchExecuteStatement" + case createTable = "CreateTable" + case updateTable = "UpdateTable" + case deleteTable = "DeleteTable" + case updateTimeToLive = "UpdateTimeToLive" + case updateContinuousBackups = "UpdateContinuousBackups" + case tagResource = "TagResource" + case untagResource = "UntagResource" + + var target: String { "DynamoDB_20120810.\(rawValue)" } + + init?(caseInsensitive name: String) { + guard let match = Self.allCases.first(where: { $0.rawValue.caseInsensitiveCompare(name) == .orderedSame }) + else { return nil } + self = match + } + + var isRead: Bool { + switch self { + case .scan, .query, .getItem, .batchGetItem, .transactGetItems, .describeTable, .listTables, + .describeTimeToLive, .describeContinuousBackups, .listTagsOfResource, .describeLimits: + return true + default: + return false + } + } + + var changesCatalog: Bool { + self == .createTable || self == .updateTable || self == .deleteTable + } +} + +struct DynamoDBOrderTerm: Sendable, Equatable { + let attribute: String + let descending: Bool +} + +/// The part of a read the grid, an export or a header click adds after the request itself. +/// `LIMIT` and `OFFSET` count the items returned, after every filter, not the items DynamoDB read. +struct DynamoDBReadWindow: Sendable, Equatable { + var order: [DynamoDBOrderTerm] = [] + var limit: Int? + var offset: Int = 0 + + var isEmpty: Bool { order.isEmpty && limit == nil && offset == 0 } + + var text: String { + var parts: [String] = [] + if !order.isEmpty { + let terms = order.map { "\(DynamoDBStatement.quote($0.attribute))\($0.descending ? " DESC" : " ASC")" } + parts.append("ORDER BY " + terms.joined(separator: ", ")) + } + if let limit { parts.append("LIMIT \(limit)") } + if offset > 0 { parts.append("OFFSET \(offset)") } + return parts.joined(separator: " ") + } +} + +struct DynamoDBAPICall: Sendable, Equatable { + let operation: DynamoDBOperation + let body: DynamoDBJSON +} + +enum DynamoDBStatement: Sendable, Equatable { + case partiQL(text: String, window: DynamoDBReadWindow) + case apiCall(DynamoDBAPICall, window: DynamoDBReadWindow) + case browse(DynamoDBBrowseRequest, window: DynamoDBReadWindow) + + static let browseVerb = "Browse" + + static func parse(_ raw: String) throws -> DynamoDBStatement { + let text = trimmed(raw) + let verb = String(text.prefix { $0.isLetter }) + let afterVerb = text.dropFirst(verb.count).drop(while: \.isWhitespace) + guard !verb.isEmpty, afterVerb.first == "{" else { return try partiQL(text) } + + let parsed: (value: DynamoDBJSON, remainder: String) + do { + parsed = try DynamoDBJSON.parsePrefix(String(afterVerb)) + } catch { + throw DynamoDBError.invalidStatement( + String(format: String(localized: "The %1$@ request is not valid JSON: %2$@"), verb, error.localizedDescription) + ) + } + guard case .object = parsed.value else { + throw DynamoDBError.invalidStatement( + String(format: String(localized: "The %@ request must be a JSON object"), verb) + ) + } + let window = try parseWindow(parsed.remainder, verb: verb) + + if verb.caseInsensitiveCompare(browseVerb) == .orderedSame { + return .browse(try DynamoDBBrowseRequest(json: parsed.value), window: window) + } + guard let operation = DynamoDBOperation(caseInsensitive: verb) else { + throw DynamoDBError.invalidStatement( + String(format: String(localized: "\"%@\" is not a DynamoDB action this editor runs"), verb) + ) + } + guard window.isEmpty || operation == .scan || operation == .query else { + throw DynamoDBError.invalidStatement( + String(format: String(localized: "%@ takes no ORDER BY, LIMIT or OFFSET"), operation.rawValue) + ) + } + return .apiCall(DynamoDBAPICall(operation: operation, body: parsed.value), window: window) + } + + var text: String { + switch self { + case .partiQL(let text, let window): + return window.isEmpty ? text : "\(text)\n\(window.text)" + case .apiCall(let call, let window): + let base = "\(call.operation.rawValue) \(call.body.serialized())" + return window.isEmpty ? base : "\(base) \(window.text)" + case .browse(let request, let window): + let base = "\(Self.browseVerb) \(request.json.serialized())" + return window.isEmpty ? base : "\(base) \(window.text)" + } + } + + static func quote(_ identifier: String) -> String { + "\"\(identifier.replacingOccurrences(of: "\"", with: "\"\""))\"" + } + + // MARK: - Parsing helpers + + private static func trimmed(_ raw: String) -> String { + var text = raw.trimmingCharacters(in: .whitespacesAndNewlines) + while text.hasSuffix(";") { + text.removeLast() + text = text.trimmingCharacters(in: .whitespacesAndNewlines) + } + return text + } + + private static func partiQL(_ text: String) throws -> DynamoDBStatement { + let split = DynamoDBPartiQL.splitTrailingWindow(text) + return .partiQL(text: split.statement, window: split.window) + } + + static func parseWindow(_ remainder: String, verb: String) throws -> DynamoDBReadWindow { + var tokens = DynamoDBPartiQL.tokens(of: remainder) + var window = DynamoDBReadWindow() + func unexpected() -> DynamoDBError { + let rest = remainder.trimmingCharacters(in: .whitespacesAndNewlines) + return .invalidStatement( + String(format: String(localized: "Unexpected text after the %1$@ request: %2$@"), verb, rest) + ) + } + if tokens.first?.isKeyword("ORDER") == true { + guard tokens.count >= 3, tokens[1].isKeyword("BY") else { throw unexpected() } + tokens.removeFirst(2) + while true { + guard let identifier = tokens.first?.identifierValue else { throw unexpected() } + tokens.removeFirst() + var descending = false + if tokens.first?.isKeyword("DESC") == true { + descending = true + tokens.removeFirst() + } else if tokens.first?.isKeyword("ASC") == true { + tokens.removeFirst() + } + window.order.append(DynamoDBOrderTerm(attribute: identifier, descending: descending)) + guard tokens.first?.text == "," else { break } + tokens.removeFirst() + } + } + if tokens.first?.isKeyword("LIMIT") == true { + guard tokens.count >= 2, let limit = Int(tokens[1].text), limit >= 0 else { throw unexpected() } + window.limit = limit + tokens.removeFirst(2) + } + if tokens.first?.isKeyword("OFFSET") == true { + guard tokens.count >= 2, let offset = Int(tokens[1].text), offset >= 0 else { throw unexpected() } + window.offset = offset + tokens.removeFirst(2) + } + guard tokens.isEmpty else { throw unexpected() } + return window + } +} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBStatementGenerator.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBStatementGenerator.swift deleted file mode 100644 index 62c155d81e..0000000000 --- a/Plugins/DynamoDBDriverPlugin/DynamoDBStatementGenerator.swift +++ /dev/null @@ -1,255 +0,0 @@ -// -// DynamoDBStatementGenerator.swift -// DynamoDBDriverPlugin -// -// Generates PartiQL statements from tracked cell changes. -// - -import Foundation -import os -import TableProPluginKit - -internal enum DynamoDBStatementError: LocalizedError { - case invalidNumber(value: String) - case invalidBoolean(value: String) - case unsupportedBinaryType - - var errorDescription: String? { - switch self { - case .invalidNumber(let value): - return "Invalid number value: '\(value)'" - case .invalidBoolean(let value): - return "Invalid boolean value: '\(value)'. Expected true/false/1/0." - case .unsupportedBinaryType: - return "Binary types (B, BS) cannot be expressed as PartiQL literals. Use parameter binding instead." - } - } -} - -internal struct DynamoDBStatementGenerator { - private static let logger = Logger(subsystem: "com.TablePro", category: "DynamoDBStatementGenerator") - - let tableName: String - let columns: [String] - let columnTypeNames: [String] - let keySchema: [(name: String, keyType: String)] - - private var keyColumnNames: Set { - Set(keySchema.map(\.name)) - } - - func generateStatements( - from changes: [PluginRowChange], - insertedRowData: [Int: [PluginCellValue]], - deletedRowIndices: Set, - insertedRowIndices: Set - ) throws -> [(statement: String, parameters: [PluginCellValue])] { - var statements: [(statement: String, parameters: [PluginCellValue])] = [] - - for change in changes { - switch change.type { - case .insert: - guard insertedRowIndices.contains(change.rowIndex) else { continue } - statements += try generateInsert(for: change, insertedRowData: insertedRowData) - case .update: - statements += try generateUpdate(for: change) - case .delete: - guard deletedRowIndices.contains(change.rowIndex) else { continue } - if let stmt = try generateDelete(for: change) { - statements.append(stmt) - } - } - } - - return statements - } - - // MARK: - INSERT - - private func generateInsert( - for change: PluginRowChange, - insertedRowData: [Int: [PluginCellValue]] - ) throws -> [(statement: String, parameters: [PluginCellValue])] { - var values: [String: String?] = [:] - - if let rowData = insertedRowData[change.rowIndex] { - for (index, column) in columns.enumerated() where index < rowData.count { - values[column] = rowData[index].asText - } - } else { - for cellChange in change.cellChanges { - values[cellChange.columnName] = cellChange.newValue.asText - } - } - - for key in keySchema { - guard let val = values[key.name], let unwrapped = val, !unwrapped.isEmpty else { - Self.logger.warning("Skipping INSERT - missing key column '\(key.name)'") - return [] - } - } - - var attrs: [String] = [] - for column in columns { - guard let value = values[column], let val = value else { continue } - let typeIndex = columns.firstIndex(of: column) ?? 0 - let typeName = typeIndex < columnTypeNames.count ? columnTypeNames[typeIndex] : "S" - attrs.append("'\(escapePartiQL(column))': \(try formatValue(val, typeName: typeName))") - } - - let quotedTable = "\"\(escapeIdentifier(tableName))\"" - let attrString = attrs.joined(separator: ", ") - let statement = "INSERT INTO \(quotedTable) VALUE { \(attrString) }" - - return [(statement: statement, parameters: [])] - } - - // MARK: - UPDATE - - private func generateUpdate( - for change: PluginRowChange - ) throws -> [(statement: String, parameters: [PluginCellValue])] { - guard !change.cellChanges.isEmpty else { return [] } - - let nonKeyChanges = change.cellChanges.filter { !keyColumnNames.contains($0.columnName) } - guard !nonKeyChanges.isEmpty else { - Self.logger.info("Skipping UPDATE - only key columns were changed (not allowed)") - return [] - } - - guard let whereClause = try buildWhereClause(from: change) else { - Self.logger.warning("Skipping UPDATE - cannot build WHERE clause") - return [] - } - - var setClauses: [String] = [] - for cellChange in nonKeyChanges { - let typeIndex = columns.firstIndex(of: cellChange.columnName) ?? 0 - let typeName = typeIndex < columnTypeNames.count ? columnTypeNames[typeIndex] : "S" - let formattedValue: String - if let newValue = cellChange.newValue.asText { - formattedValue = try formatValue(newValue, typeName: typeName) - } else { - formattedValue = "NULL" - } - setClauses.append("\"\(escapeIdentifier(cellChange.columnName))\" = \(formattedValue)") - } - - let quotedTable = "\"\(escapeIdentifier(tableName))\"" - let statement = "UPDATE \(quotedTable) SET \(setClauses.joined(separator: ", ")) WHERE \(whereClause)" - - return [(statement: statement, parameters: [])] - } - - // MARK: - DELETE - - private func generateDelete( - for change: PluginRowChange - ) throws -> (statement: String, parameters: [PluginCellValue])? { - guard let whereClause = try buildWhereClause(from: change) else { - Self.logger.warning("Skipping DELETE - cannot build WHERE clause") - return nil - } - - let quotedTable = "\"\(escapeIdentifier(tableName))\"" - let statement = "DELETE FROM \(quotedTable) WHERE \(whereClause)" - - return (statement: statement, parameters: []) - } - - // MARK: - Helpers - - private func buildWhereClause(from change: PluginRowChange) throws -> String? { - guard let originalRow = change.originalRow else { return nil } - - var conditions: [String] = [] - for key in keySchema { - guard let colIndex = columns.firstIndex(of: key.name), - colIndex < originalRow.count, - let value = originalRow[colIndex].asText - else { return nil } - - let typeName = colIndex < columnTypeNames.count ? columnTypeNames[colIndex] : "S" - conditions.append( - "\"\(escapeIdentifier(key.name))\" = \(try formatValue(value, typeName: typeName))" - ) - } - - guard !conditions.isEmpty else { return nil } - return conditions.joined(separator: " AND ") - } - - private func formatValue(_ value: String, typeName: String) throws -> String { - switch typeName { - case "N": - if Int64(value) != nil || Double(value) != nil { - return value - } - throw DynamoDBStatementError.invalidNumber(value: value) - case "BOOL": - let lower = value.lowercased() - switch lower { - case "true", "1": - return "true" - case "false", "0": - return "false" - default: - throw DynamoDBStatementError.invalidBoolean(value: value) - } - case "NULL": - let trimmed = value.trimmingCharacters(in: .whitespaces) - if trimmed.isEmpty || trimmed.lowercased() == "null" { - return "NULL" - } - return "'\(escapePartiQL(value))'" - case "SS": - return try formatStringSet(value) - case "NS": - return try formatNumberSet(value) - case "B", "BS": - throw DynamoDBStatementError.unsupportedBinaryType - case "S": - return "'\(escapePartiQL(value))'" - default: - if value.hasPrefix("[") || value.hasPrefix("{") { - return value - } - return "'\(escapePartiQL(value))'" - } - } - - private func formatStringSet(_ value: String) throws -> String { - guard let data = value.data(using: .utf8), - let array = try? JSONSerialization.jsonObject(with: data) as? [String] - else { - return "<<'\(escapePartiQL(value))'>>" - } - let elements = array.map { "'\(escapePartiQL($0))'" } - return "<<\(elements.joined(separator: ", "))>>" - } - - private func formatNumberSet(_ value: String) throws -> String { - guard let data = value.data(using: .utf8), - let array = try? JSONSerialization.jsonObject(with: data) as? [Any] - else { - throw DynamoDBStatementError.invalidNumber(value: value) - } - var elements: [String] = [] - for element in array { - let str = "\(element)" - guard Int64(str) != nil || Double(str) != nil else { - throw DynamoDBStatementError.invalidNumber(value: str) - } - elements.append(str) - } - return "<<\(elements.joined(separator: ", "))>>" - } - - private func escapePartiQL(_ value: String) -> String { - value.replacingOccurrences(of: "'", with: "''") - } - - private func escapeIdentifier(_ name: String) -> String { - name.replacingOccurrences(of: "\"", with: "\"\"") - } -} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBTableDefinition.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBTableDefinition.swift new file mode 100644 index 0000000000..971c0245a3 --- /dev/null +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBTableDefinition.swift @@ -0,0 +1,405 @@ +import Foundation +import TableProPluginKit + +/// Table definitions as `CreateTable` requests: rebuilt from a described table for the DDL view, +/// and built from the Create Table form. +enum DynamoDBTableDefinition { + static func isValidTableName(_ name: String) -> Bool { + guard (3...255).contains(name.count) else { return false } + return name.unicodeScalars.allSatisfy { scalar in + guard scalar.isASCII else { return false } + return CharacterSet.alphanumerics.contains(scalar) || scalar == "_" || scalar == "-" || scalar == "." + } + } + + // MARK: - From a described table + + static func createTableRequest(_ schema: DynamoDBTableSchema) -> DynamoDBJSON { + var body: [String: DynamoDBJSON] = [ + "TableName": .string(schema.name), + "KeySchema": keySchema(schema.keys), + "AttributeDefinitions": .array(schema.attributeTypes.keys.sorted().compactMap { name in + schema.attributeTypes[name].map { attributeDefinition(name, $0) } + }), + "BillingMode": .string(schema.isOnDemand ? "PAY_PER_REQUEST" : "PROVISIONED") + ] + if !schema.isOnDemand { + body["ProvisionedThroughput"] = throughput(read: schema.readCapacity, write: schema.writeCapacity) + } + let globals = schema.indexes.filter { $0.kind == .global } + if !globals.isEmpty { + body["GlobalSecondaryIndexes"] = .array(globals.map { index in + var entry = indexBody(index) + if !schema.isOnDemand { + entry["ProvisionedThroughput"] = throughput(read: index.readCapacity, write: index.writeCapacity) + } + return .object(entry) + }) + } + let locals = schema.indexes.filter { $0.kind == .local } + if !locals.isEmpty { + body["LocalSecondaryIndexes"] = .array(locals.map { .object(indexBody($0)) }) + } + if let tableClass = schema.tableClass { + body["TableClass"] = .string(tableClass) + } + if schema.deletionProtection { + body["DeletionProtectionEnabled"] = .bool(true) + } + if let viewType = schema.streamViewType { + body["StreamSpecification"] = .object(["StreamEnabled": .bool(true), "StreamViewType": .string(viewType)]) + } + if schema.sseType == "KMS" { + var sse: [String: DynamoDBJSON] = ["Enabled": .bool(true), "SSEType": .string("KMS")] + if let key = schema.sseKeyArn { sse["KMSMasterKeyId"] = .string(key) } + body["SSESpecification"] = .object(sse) + } + return .object(body) + } + + static func summary(_ schema: DynamoDBTableSchema) -> String { + var parts: [String] = [] + if schema.isOnDemand { + parts.append(String(localized: "On-demand capacity")) + } else { + parts.append(String( + format: String(localized: "Provisioned: %1$lld read, %2$lld write"), + schema.readCapacity ?? 0, schema.writeCapacity ?? 0 + )) + } + if let tableClass = schema.tableClass { + parts.append(tableClass == "STANDARD_INFREQUENT_ACCESS" + ? String(localized: "Standard-Infrequent Access") : String(localized: "Standard")) + } + if schema.deletionProtection { + parts.append(String(localized: "Deletion protection on")) + } + if let viewType = schema.streamViewType { + parts.append(String(format: String(localized: "Stream: %@"), viewType)) + } + if let status = schema.status, status != "ACTIVE" { + parts.append(String(format: String(localized: "Status: %@"), status)) + } + parts.append(String(localized: "Item count is approximate, updated about every six hours")) + return parts.joined(separator: " · ") + } + + private static func keySchema(_ keys: DynamoDBKeySchema) -> DynamoDBJSON { + .array( + keys.partition.map { .object(["AttributeName": .string($0), "KeyType": .string("HASH")]) } + + keys.sort.map { .object(["AttributeName": .string($0), "KeyType": .string("RANGE")]) } + ) + } + + private static func attributeDefinition(_ name: String, _ type: DynamoDBAttributeType) -> DynamoDBJSON { + .object(["AttributeName": .string(name), "AttributeType": .string(type.rawValue)]) + } + + private static func throughput(read: Int64?, write: Int64?) -> DynamoDBJSON { + .object([ + "ReadCapacityUnits": .number(String(read ?? 1)), + "WriteCapacityUnits": .number(String(write ?? 1)) + ]) + } + + private static func indexBody(_ index: DynamoDBIndex) -> [String: DynamoDBJSON] { + [ + "IndexName": .string(index.name), + "KeySchema": keySchema(index.keys), + "Projection": projection(index.projection) + ] + } + + private static func projection(_ projection: DynamoDBProjection) -> DynamoDBJSON { + switch projection { + case .all: + return .object(["ProjectionType": .string("ALL")]) + case .keysOnly: + return .object(["ProjectionType": .string("KEYS_ONLY")]) + case .include(let attributes): + return .object([ + "ProjectionType": .string("INCLUDE"), + "NonKeyAttributes": .array(attributes.map(DynamoDBJSON.string)) + ]) + } + } + + // MARK: - Create Table form + + enum Field { + static let partitionKeyName = "partitionKeyName" + static let partitionKeyType = "partitionKeyType" + static let sortKeyName = "sortKeyName" + static let sortKeyType = "sortKeyType" + static let billingMode = "billingMode" + static let readCapacity = "readCapacity" + static let writeCapacity = "writeCapacity" + static let tableClass = "tableClass" + static let deletionProtection = "deletionProtection" + static let indexName = "indexName" + static let projection = "projection" + static let includedAttributes = "includedAttributes" + static let globalIndexes = "globalIndexes" + static let localIndexes = "localIndexes" + } + + static var formSpec: PluginCreateTableFormSpec { + PluginCreateTableFormSpec( + sections: [ + PluginFormSection(id: "keys", title: String(localized: "Primary Key"), fields: [ + PluginFormField( + id: Field.partitionKeyName, label: String(localized: "Partition key"), + kind: .text(placeholder: "pk", isRequired: true) + ), + PluginFormField(id: Field.partitionKeyType, label: String(localized: "Type"), kind: keyTypePicker), + PluginFormField( + id: Field.sortKeyName, label: String(localized: "Sort key"), + kind: .text(placeholder: String(localized: "Optional"), isRequired: false) + ), + PluginFormField( + id: Field.sortKeyType, label: String(localized: "Type"), kind: keyTypePicker, + visibleWhen: PluginFormCondition(fieldId: Field.sortKeyName, values: nil) + ) + ]), + PluginFormSection(id: "capacity", title: String(localized: "Capacity"), fields: [ + PluginFormField( + id: Field.billingMode, label: String(localized: "Billing"), + kind: .picker(options: [ + PluginFormOption(value: "PAY_PER_REQUEST", label: String(localized: "On-demand")), + PluginFormOption(value: "PROVISIONED", label: String(localized: "Provisioned")) + ], defaultValue: "PAY_PER_REQUEST") + ), + PluginFormField( + id: Field.readCapacity, label: String(localized: "Read capacity units"), + kind: .integer(defaultValue: 5, minimum: 1, maximum: nil), + visibleWhen: PluginFormCondition(fieldId: Field.billingMode, values: ["PROVISIONED"]) + ), + PluginFormField( + id: Field.writeCapacity, label: String(localized: "Write capacity units"), + kind: .integer(defaultValue: 5, minimum: 1, maximum: nil), + visibleWhen: PluginFormCondition(fieldId: Field.billingMode, values: ["PROVISIONED"]) + ) + ]), + PluginFormSection(id: "settings", title: String(localized: "Settings"), fields: [ + PluginFormField( + id: Field.tableClass, label: String(localized: "Table class"), + kind: .picker(options: [ + PluginFormOption(value: "STANDARD", label: String(localized: "Standard")), + PluginFormOption( + value: "STANDARD_INFREQUENT_ACCESS", label: String(localized: "Standard-Infrequent Access") + ) + ], defaultValue: "STANDARD") + ), + PluginFormField( + id: Field.deletionProtection, label: String(localized: "Deletion protection"), + kind: .toggle(defaultValue: false) + ) + ]), + PluginFormSection( + id: Field.globalIndexes, title: String(localized: "Global Secondary Indexes"), + fields: indexFields(includesPartitionKey: true), + isRepeating: true, addLabel: String(localized: "Add Global Index"), maximumCount: 20 + ), + PluginFormSection( + id: Field.localIndexes, title: String(localized: "Local Secondary Indexes"), + fields: indexFields(includesPartitionKey: false), + isRepeating: true, addLabel: String(localized: "Add Local Index"), maximumCount: 5 + ) + ], + footnote: String(localized: "A DynamoDB table declares only its key attributes. The items you write add every other attribute.") + ) + } + + private static var keyTypePicker: PluginFormField.Kind { + .picker(options: [ + PluginFormOption(value: "S", label: DynamoDBAttributeType.string.displayName), + PluginFormOption(value: "N", label: DynamoDBAttributeType.number.displayName), + PluginFormOption(value: "B", label: DynamoDBAttributeType.binary.displayName) + ], defaultValue: "S") + } + + private static func indexFields(includesPartitionKey: Bool) -> [PluginFormField] { + var fields = [ + PluginFormField( + id: Field.indexName, label: String(localized: "Index name"), + kind: .text(placeholder: nil, isRequired: true) + ) + ] + if includesPartitionKey { + fields += [ + PluginFormField( + id: Field.partitionKeyName, label: String(localized: "Partition key"), + kind: .text(placeholder: nil, isRequired: true) + ), + PluginFormField(id: Field.partitionKeyType, label: String(localized: "Type"), kind: keyTypePicker) + ] + } + fields += [ + PluginFormField( + id: Field.sortKeyName, label: String(localized: "Sort key"), + kind: .text(placeholder: includesPartitionKey ? String(localized: "Optional") : nil, isRequired: !includesPartitionKey) + ), + PluginFormField( + id: Field.sortKeyType, label: String(localized: "Type"), kind: keyTypePicker, + visibleWhen: PluginFormCondition(fieldId: Field.sortKeyName, values: nil) + ), + PluginFormField( + id: Field.projection, label: String(localized: "Attributes"), + kind: .picker(options: [ + PluginFormOption(value: "ALL", label: String(localized: "All attributes")), + PluginFormOption(value: "KEYS_ONLY", label: String(localized: "Keys only")), + PluginFormOption(value: "INCLUDE", label: String(localized: "Keys and chosen attributes")) + ], defaultValue: "ALL") + ), + PluginFormField( + id: Field.includedAttributes, label: String(localized: "Chosen attributes"), + kind: .text(placeholder: String(localized: "Comma-separated names"), isRequired: true), + visibleWhen: PluginFormCondition(fieldId: Field.projection, values: ["INCLUDE"]) + ) + ] + return fields + } + + static func createTableRequest(from request: PluginCreateTableRequest) throws -> DynamoDBJSON { + let tableName = request.tableName.trimmingCharacters(in: .whitespaces) + guard isValidTableName(tableName) else { + throw PluginCreateTableFormError(message: String( + localized: "A table name is 3 to 255 characters of letters, digits, underscore, hyphen and period." + )) + } + var types: [String: DynamoDBAttributeType] = [:] + func declare(_ name: String, _ typeValue: String?, field: String) throws { + let type = typeValue.flatMap(DynamoDBAttributeType.init(rawValue:)) ?? .string + if let existing = types[name], existing != type { + throw PluginCreateTableFormError( + message: String(format: String(localized: "\"%@\" is declared as two different types"), name), + fieldId: field + ) + } + types[name] = type + } + + let values = request.values + let partition = trimmed(values[Field.partitionKeyName]) + guard !partition.isEmpty else { + throw PluginCreateTableFormError( + message: String(localized: "Enter the partition key's name"), fieldId: Field.partitionKeyName + ) + } + try declare(partition, values[Field.partitionKeyType], field: Field.partitionKeyType) + let sort = trimmed(values[Field.sortKeyName]) + if !sort.isEmpty { + try declare(sort, values[Field.sortKeyType], field: Field.sortKeyType) + } + let tableKeys = DynamoDBKeySchema(partition: [partition], sort: sort.isEmpty ? [] : [sort]) + + let isProvisioned = values[Field.billingMode] == "PROVISIONED" + let capacity = try isProvisioned ? provisionedThroughput(values) : nil + + let globals = try (request.repeatedValues[Field.globalIndexes] ?? []).map { entry in + try indexRequest(entry, tablePartition: nil, declare: declare, capacity: capacity) + } + let localEntries = request.repeatedValues[Field.localIndexes] ?? [] + if !localEntries.isEmpty, sort.isEmpty { + throw PluginCreateTableFormError( + message: String(localized: "A local secondary index needs a table with a sort key"), + fieldId: Field.sortKeyName + ) + } + let locals = try localEntries.map { entry in + try indexRequest(entry, tablePartition: partition, declare: declare, capacity: nil) + } + + var body: [String: DynamoDBJSON] = [ + "TableName": .string(tableName), + "KeySchema": keySchema(tableKeys), + "AttributeDefinitions": .array(types.keys.sorted().compactMap { name in + types[name].map { attributeDefinition(name, $0) } + }), + "BillingMode": .string(isProvisioned ? "PROVISIONED" : "PAY_PER_REQUEST") + ] + if let capacity { body["ProvisionedThroughput"] = capacity } + if !globals.isEmpty { body["GlobalSecondaryIndexes"] = .array(globals) } + if !locals.isEmpty { body["LocalSecondaryIndexes"] = .array(locals) } + if let tableClass = values[Field.tableClass], tableClass != "STANDARD" { + body["TableClass"] = .string(tableClass) + } + if values[Field.deletionProtection] == "true" { + body["DeletionProtectionEnabled"] = .bool(true) + } + return .object(body) + } + + private static func indexRequest( + _ entry: [String: String], + tablePartition: String?, + declare: (String, String?, String) throws -> Void, + capacity: DynamoDBJSON? + ) throws -> DynamoDBJSON { + let name = trimmed(entry[Field.indexName]) + guard isValidTableName(name) else { + throw PluginCreateTableFormError( + message: String(localized: "An index name is 3 to 255 characters of letters, digits, underscore, hyphen and period."), + fieldId: Field.indexName + ) + } + let partition = tablePartition ?? trimmed(entry[Field.partitionKeyName]) + guard !partition.isEmpty else { + throw PluginCreateTableFormError( + message: String(format: String(localized: "Enter the partition key of index %@"), name), + fieldId: Field.partitionKeyName + ) + } + if tablePartition == nil { + try declare(partition, entry[Field.partitionKeyType], Field.partitionKeyType) + } + let sort = trimmed(entry[Field.sortKeyName]) + if tablePartition != nil, sort.isEmpty { + throw PluginCreateTableFormError( + message: String(format: String(localized: "Enter the sort key of index %@"), name), + fieldId: Field.sortKeyName + ) + } + if !sort.isEmpty { + try declare(sort, entry[Field.sortKeyType], Field.sortKeyType) + } + let projectionType = entry[Field.projection] ?? "ALL" + var projection: [String: DynamoDBJSON] = ["ProjectionType": .string(projectionType)] + if projectionType == "INCLUDE" { + let attributes = (entry[Field.includedAttributes] ?? "") + .split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty } + guard !attributes.isEmpty else { + throw PluginCreateTableFormError( + message: String(format: String(localized: "Name the attributes index %@ includes"), name), + fieldId: Field.includedAttributes + ) + } + projection["NonKeyAttributes"] = .array(attributes.map(DynamoDBJSON.string)) + } + var body: [String: DynamoDBJSON] = [ + "IndexName": .string(name), + "KeySchema": keySchema(DynamoDBKeySchema(partition: [partition], sort: sort.isEmpty ? [] : [sort])), + "Projection": .object(projection) + ] + if let capacity { body["ProvisionedThroughput"] = capacity } + return .object(body) + } + + private static func provisionedThroughput(_ values: [String: String]) throws -> DynamoDBJSON { + guard let read = Int(trimmed(values[Field.readCapacity])), read >= 1 else { + throw PluginCreateTableFormError( + message: String(localized: "Read capacity must be a whole number of at least 1"), fieldId: Field.readCapacity + ) + } + guard let write = Int(trimmed(values[Field.writeCapacity])), write >= 1 else { + throw PluginCreateTableFormError( + message: String(localized: "Write capacity must be a whole number of at least 1"), fieldId: Field.writeCapacity + ) + } + return .object(["ReadCapacityUnits": .number(String(read)), "WriteCapacityUnits": .number(String(write))]) + } + + private static func trimmed(_ value: String?) -> String { + (value ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + } +} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBTableSchema.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBTableSchema.swift new file mode 100644 index 0000000000..66b44974f8 --- /dev/null +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBTableSchema.swift @@ -0,0 +1,208 @@ +import Foundation +import TableProPluginKit + +/// The key attributes of a table or an index, in declaration order. +/// +/// A global secondary index may name up to four partition and four sort attributes, so neither +/// half is a single name. +struct DynamoDBKeySchema: Sendable, Equatable { + let partition: [String] + let sort: [String] + + var attributes: [String] { partition + sort } + + init(partition: [String], sort: [String]) { + self.partition = partition + self.sort = sort + } + + init(json: DynamoDBJSON?) { + var partition: [String] = [] + var sort: [String] = [] + for element in json?.arrayValue ?? [] { + guard let name = element["AttributeName"]?.stringValue else { continue } + if element["KeyType"]?.stringValue == "RANGE" { + sort.append(name) + } else { + partition.append(name) + } + } + self.init(partition: partition, sort: sort) + } +} + +enum DynamoDBProjection: Sendable, Equatable { + case all + case keysOnly + case include([String]) + + init(json: DynamoDBJSON?) { + switch json?["ProjectionType"]?.stringValue { + case "KEYS_ONLY": + self = .keysOnly + case "INCLUDE": + self = .include((json?["NonKeyAttributes"]?.arrayValue ?? []).compactMap(\.stringValue)) + default: + self = .all + } + } + + var displayName: String { + switch self { + case .all: return "ALL" + case .keysOnly: return "KEYS_ONLY" + case .include: return "INCLUDE" + } + } + + var nonKeyAttributes: [String] { + guard case .include(let attributes) = self else { return [] } + return attributes + } +} + +struct DynamoDBIndex: Sendable, Equatable { + enum Kind: Sendable, Equatable { + case global + case local + } + + let name: String + let kind: Kind + let keys: DynamoDBKeySchema + let projection: DynamoDBProjection + let status: String? + let isBackfilling: Bool + let itemCount: Int64? + let sizeBytes: Int64? + let readCapacity: Int64? + let writeCapacity: Int64? + + /// A query can be answered from the index now. A global index being built or deleted is not, + /// and one still backfilling misses items. + var isQueryable: Bool { + guard kind == .global else { return true } + return (status == nil || status == "ACTIVE") && !isBackfilling + } +} + +struct DynamoDBTableSchema: Sendable, Equatable { + let name: String + let keys: DynamoDBKeySchema + let attributeTypes: [String: DynamoDBAttributeType] + let indexes: [DynamoDBIndex] + let status: String? + let arn: String? + let itemCount: Int64? + let sizeBytes: Int64? + let billingMode: String + let readCapacity: Int64? + let writeCapacity: Int64? + let tableClass: String? + let deletionProtection: Bool + let streamViewType: String? + let sseType: String? + let sseKeyArn: String? + let createdAt: Date? + + var isOnDemand: Bool { billingMode == "PAY_PER_REQUEST" } + + /// A table DynamoDB is still creating answers DescribeTable and nothing else yet. + var isBeingCreated: Bool { status == "CREATING" } + + var allKeyAttributes: Set { + Set(keys.attributes + indexes.flatMap(\.keys.attributes)) + } + + func index(named name: String) -> DynamoDBIndex? { + indexes.first { $0.name == name } + } + + func keyType(of attribute: String) -> DynamoDBAttributeType? { + attributeTypes[attribute] + } + + init(describeTableResponse json: DynamoDBJSON) throws { + guard let table = json["Table"], let name = table["TableName"]?.stringValue else { + throw DynamoDBError.invalidResponse(String(localized: "DescribeTable returned no table")) + } + self.name = name + self.keys = DynamoDBKeySchema(json: table["KeySchema"]) + var types: [String: DynamoDBAttributeType] = [:] + for definition in table["AttributeDefinitions"]?.arrayValue ?? [] { + guard let attribute = definition["AttributeName"]?.stringValue, + let type = definition["AttributeType"]?.stringValue.flatMap(DynamoDBAttributeType.init(rawValue:)) + else { continue } + types[attribute] = type + } + self.attributeTypes = types + let globals = (table["GlobalSecondaryIndexes"]?.arrayValue ?? []).map { Self.index($0, kind: .global) } + let locals = (table["LocalSecondaryIndexes"]?.arrayValue ?? []).map { Self.index($0, kind: .local) } + self.indexes = globals + locals + self.status = table["TableStatus"]?.stringValue + self.arn = table["TableArn"]?.stringValue + self.itemCount = table["ItemCount"]?.numberText.flatMap { Int64($0) } + self.sizeBytes = table["TableSizeBytes"]?.numberText.flatMap { Int64($0) } + self.billingMode = table["BillingModeSummary"]?["BillingMode"]?.stringValue ?? "PROVISIONED" + self.readCapacity = table["ProvisionedThroughput"]?["ReadCapacityUnits"]?.numberText.flatMap { Int64($0) } + self.writeCapacity = table["ProvisionedThroughput"]?["WriteCapacityUnits"]?.numberText.flatMap { Int64($0) } + self.tableClass = table["TableClassSummary"]?["TableClass"]?.stringValue + self.deletionProtection = table["DeletionProtectionEnabled"]?.boolValue ?? false + let streamEnabled = table["StreamSpecification"]?["StreamEnabled"]?.boolValue ?? false + self.streamViewType = streamEnabled ? table["StreamSpecification"]?["StreamViewType"]?.stringValue : nil + self.sseType = table["SSEDescription"]?["SSEType"]?.stringValue + self.sseKeyArn = table["SSEDescription"]?["KMSMasterKeyArn"]?.stringValue + self.createdAt = table["CreationDateTime"]?.doubleValue.map(Date.init(timeIntervalSince1970:)) + } + + private static func index(_ json: DynamoDBJSON, kind: DynamoDBIndex.Kind) -> DynamoDBIndex { + DynamoDBIndex( + name: json["IndexName"]?.stringValue ?? "", + kind: kind, + keys: DynamoDBKeySchema(json: json["KeySchema"]), + projection: DynamoDBProjection(json: json["Projection"]), + status: json["IndexStatus"]?.stringValue, + isBackfilling: json["Backfilling"]?.boolValue ?? false, + itemCount: json["ItemCount"]?.numberText.flatMap { Int64($0) }, + sizeBytes: json["IndexSizeBytes"]?.numberText.flatMap { Int64($0) }, + readCapacity: json["ProvisionedThroughput"]?["ReadCapacityUnits"]?.numberText.flatMap { Int64($0) }, + writeCapacity: json["ProvisionedThroughput"]?["WriteCapacityUnits"]?.numberText.flatMap { Int64($0) } + ) + } + + /// The key of `item` as an `ExclusiveStartKey` for a read on `index`, or on the table when nil: + /// the table's key attributes plus the index's own. + func startKey(for item: DynamoDBItem, index: DynamoDBIndex?) -> DynamoDBItem? { + var names = keys.attributes + if let index { + names += index.keys.attributes.filter { !names.contains($0) } + } + var key: DynamoDBItem = [:] + for name in names { + guard let value = item[name] else { return nil } + key[name] = value + } + return key + } + + func primaryKey(of item: DynamoDBItem) -> DynamoDBItem? { + startKey(for: item, index: nil) + } + + /// How the Structure tab and error messages name an item: `pk = a, sk = 3`. + func describeKey(_ key: DynamoDBItem) -> String { + keys.attributes.compactMap { name in + key[name].map { "\(name) = \(DynamoDBCellCodec.displayText(for: $0))" } + }.joined(separator: ", ") + } +} + +extension DynamoDBCellCodec { + static func displayText(for value: DynamoDBAttributeValue) -> String { + switch cell(for: value) { + case .text(let text): return text + case .bytes(let data): return data.base64EncodedString() + case .null: return "NULL" + } + } +} diff --git a/Plugins/DynamoDBDriverPlugin/DynamoDBWriteStatements.swift b/Plugins/DynamoDBDriverPlugin/DynamoDBWriteStatements.swift new file mode 100644 index 0000000000..ef3e8e0f14 --- /dev/null +++ b/Plugins/DynamoDBDriverPlugin/DynamoDBWriteStatements.swift @@ -0,0 +1,233 @@ +import Foundation +import TableProPluginKit + +/// Turns grid edits into PartiQL, one statement per changed row, every value a `?` parameter. +/// +/// The statements carry no types. A cell has none to give, and guessing one is how every edit used +/// to write a String; the driver types each parameter when the statement runs, from the key schema +/// and the item as it is then. An UPDATE also compares every attribute it changes with the value +/// the grid loaded, so it neither recreates an item someone deleted nor overwrites one someone +/// changed. +struct DynamoDBWriteStatements { + static let defaultSentinel = "__DEFAULT__" + + let table: String + let columns: [String] + let keyColumns: [String] + + func statements( + for changes: [PluginRowChange], + insertedRowData: [Int: [PluginCellValue]], + deletedRowIndices: Set, + insertedRowIndices: Set + ) -> [(statement: String, parameters: [PluginCellValue])] { + changes.compactMap { change in + switch change.type { + case .insert: + guard insertedRowIndices.contains(change.rowIndex) else { return nil } + return insert(change, rowData: insertedRowData[change.rowIndex]) + case .update: + return update(change) + case .delete: + guard deletedRowIndices.contains(change.rowIndex) else { return nil } + return delete(change) + } + } + } + + func insert(row: [PluginCellValue]) -> (statement: String, parameters: [PluginCellValue]) { + var names: [String] = [] + var parameters: [PluginCellValue] = [] + for (index, column) in columns.enumerated() where index < row.count { + let value = row[index] + if case .null = value { continue } + if case .text(let text) = value, text == Self.defaultSentinel { continue } + names.append(column) + parameters.append(value) + } + let entries = names.map { "\(Self.literal($0)): ?" }.joined(separator: ", ") + return ("INSERT INTO \(DynamoDBStatement.quote(table)) VALUE {\(entries)}", parameters) + } + + private func insert( + _ change: PluginRowChange, + rowData: [PluginCellValue]? + ) -> (statement: String, parameters: [PluginCellValue]) { + if let rowData { + return insert(row: rowData) + } + var row = [PluginCellValue](repeating: .null, count: columns.count) + for cell in change.cellChanges where cell.columnIndex < row.count { + row[cell.columnIndex] = cell.newValue + } + return insert(row: row) + } + + private func update(_ change: PluginRowChange) -> (statement: String, parameters: [PluginCellValue])? { + guard let original = change.originalRow, !change.cellChanges.isEmpty else { return nil } + var assignments: [String] = [] + var assignedValues: [PluginCellValue] = [] + var removals: [String] = [] + var guards: [String] = [] + var guardValues: [PluginCellValue] = [] + + for cell in change.cellChanges { + let name = DynamoDBStatement.quote(cell.columnName) + if case .null = cell.newValue { + removals.append(name) + } else { + assignments.append("\(name) = ?") + assignedValues.append(cell.newValue) + } + guard !keyColumns.contains(cell.columnName) else { continue } + if case .null = cell.oldValue { continue } + guards.append("\(name) = ?") + guardValues.append(cell.oldValue) + } + guard !assignments.isEmpty || !removals.isEmpty else { return nil } + + let key = keyCondition(original) + var statement = "UPDATE \(DynamoDBStatement.quote(table))" + if !assignments.isEmpty { + statement += " SET " + assignments.joined(separator: ", ") + } + if !removals.isEmpty { + statement += " REMOVE " + removals.joined(separator: ", ") + } + statement += " WHERE " + (key.terms + guards).joined(separator: " AND ") + return (statement, assignedValues + key.values + guardValues) + } + + private func delete(_ change: PluginRowChange) -> (statement: String, parameters: [PluginCellValue])? { + guard let original = change.originalRow else { return nil } + let key = keyCondition(original) + let statement = "DELETE FROM \(DynamoDBStatement.quote(table)) WHERE " + + key.terms.joined(separator: " AND ") + " RETURNING ALL OLD *" + return (statement, key.values) + } + + private func keyCondition(_ row: [PluginCellValue]) -> (terms: [String], values: [PluginCellValue]) { + var terms: [String] = [] + var values: [PluginCellValue] = [] + for column in keyColumns { + guard let index = columns.firstIndex(of: column), index < row.count else { continue } + terms.append("\(DynamoDBStatement.quote(column)) = ?") + values.append(row[index]) + } + return (terms, values) + } + + static func literal(_ name: String) -> String { + "'\(name.replacingOccurrences(of: "'", with: "''"))'" + } +} + +/// Types the `?` parameters of a PartiQL statement when it runs. +/// +/// A key attribute takes its declared type. Any other attribute takes the type of its current +/// value in the item, then the type its column shows, and only then a String: a Number is never +/// read out of text that happens to look like one. +struct DynamoDBParameterBinder { + let schema: DynamoDBTableSchema? + let observedTypes: [String: DynamoDBAttributeType] + let currentItem: DynamoDBItem? + + func bind( + _ parameters: [PluginCellValue], + roles: [DynamoDBPartiQL.ParameterRole] + ) throws -> [DynamoDBAttributeValue] { + try parameters.enumerated().map { index, cell in + let role = index < roles.count ? roles[index] : .unknown + return try bind(cell, role: role) + } + } + + private func bind(_ cell: PluginCellValue, role: DynamoDBPartiQL.ParameterRole) throws -> DynamoDBAttributeValue { + switch role { + case .assigned(let path): + if path.isTopLevel, schema?.keys.attributes.contains(path.root) == true { + throw DynamoDBError.invalidValue(attribute: path.root, reason: String( + localized: "DynamoDB can't change a key. Duplicate the row with the new key, then delete the old one." + )) + } + if path.isTopLevel, let keyType = indexKeyType(path.root) { + return try keyValue(cell, type: keyType, attribute: path.root) + } + return try typed(cell, path: path) + case .compared(let path): + if path.isTopLevel, let keyType = indexKeyType(path.root) { + return try keyValue(cell, type: keyType, attribute: path.root) + } + return try typed(cell, path: path) + case .inserted(let attribute): + if let keyType = indexKeyType(attribute) { + return try keyValue(cell, type: keyType, attribute: attribute) + } + return try typed(cell, path: DynamoDBAttributePath(attribute: attribute)) + case .unknown: + return untyped(cell) + } + } + + /// The declared type of a key attribute of the table or of any of its indexes. + private func indexKeyType(_ attribute: String) -> DynamoDBAttributeType? { + guard let schema, schema.allKeyAttributes.contains(attribute) else { return nil } + return schema.keyType(of: attribute) + } + + private func typed(_ cell: PluginCellValue, path: DynamoDBAttributePath) throws -> DynamoDBAttributeValue { + let template = currentItem.flatMap { path.value(in: $0) } + let decoded = try DynamoDBCellCodec.decode( + cell, + template: template, + columnType: path.isTopLevel ? observedTypes[path.root] : nil, + attribute: path.isTopLevel ? path.root : Self.describe(path) + ) + return decoded ?? .null + } + + private static func describe(_ path: DynamoDBAttributePath) -> String { + path.segments.map { segment -> String in + switch segment { + case .name(let name): return ".\(name)" + case .index(let position): return "[\(position)]" + } + }.joined().dropFirst().description + } + + private func keyValue( + _ cell: PluginCellValue, + type: DynamoDBAttributeType, + attribute: String + ) throws -> DynamoDBAttributeValue { + switch cell { + case .null: + throw DynamoDBError.invalidValue( + attribute: attribute, reason: String(localized: "A key attribute needs a value") + ) + case .bytes(let data): + guard type == .binary else { + throw DynamoDBError.invalidValue( + attribute: attribute, + reason: String(format: String(localized: "This key is a %@, not binary data"), type.displayName) + ) + } + return .binary(data) + case .text(let text): + guard !text.isEmpty, text != DynamoDBWriteStatements.defaultSentinel else { + throw DynamoDBError.invalidValue( + attribute: attribute, reason: String(localized: "A key attribute needs a value") + ) + } + return try DynamoDBCellCodec.decode(text: text, as: type, template: nil, attribute: attribute) + } + } + + private func untyped(_ cell: PluginCellValue) -> DynamoDBAttributeValue { + switch cell { + case .null: return .null + case .bytes(let data): return .binary(data) + case .text(let text): return .string(text) + } + } +} diff --git a/Plugins/TableProPluginKit/PluginCreateTableForm.swift b/Plugins/TableProPluginKit/PluginCreateTableForm.swift new file mode 100644 index 0000000000..75f1c1c86c --- /dev/null +++ b/Plugins/TableProPluginKit/PluginCreateTableForm.swift @@ -0,0 +1,140 @@ +import Foundation + +/// A Create Table form a driver describes and the app renders natively. +/// +/// For an engine whose tables are not a list of typed columns: a DynamoDB table declares its key +/// attributes, its capacity and its secondary indexes, and none of that fits the column grid the +/// app offers SQL engines. The driver turns the submitted values into the statements the app +/// previews and runs, so the form never executes anything on its own. +public struct PluginCreateTableFormSpec: Sendable, Hashable { + public let sections: [PluginFormSection] + public let footnote: String? + + public init(sections: [PluginFormSection], footnote: String? = nil) { + self.sections = sections + self.footnote = footnote + } +} + +public struct PluginFormSection: Sendable, Hashable { + public let id: String + public let title: String? + public let fields: [PluginFormField] + /// A repeating section is a list of entries, each with its own copy of `fields`, which the + /// user adds and removes. Its values arrive in `PluginCreateTableRequest.repeatedValues`. + public let isRepeating: Bool + public let addLabel: String? + public let maximumCount: Int? + + public init( + id: String, + title: String?, + fields: [PluginFormField], + isRepeating: Bool = false, + addLabel: String? = nil, + maximumCount: Int? = nil + ) { + self.id = id + self.title = title + self.fields = fields + self.isRepeating = isRepeating + self.addLabel = addLabel + self.maximumCount = maximumCount + } +} + +public struct PluginFormField: Sendable, Hashable { + public enum Kind: Sendable, Hashable { + case text(placeholder: String?, isRequired: Bool) + case integer(defaultValue: Int?, minimum: Int?, maximum: Int?) + case picker(options: [PluginFormOption], defaultValue: String) + case toggle(defaultValue: Bool) + } + + public let id: String + public let label: String + public let kind: Kind + public let visibleWhen: PluginFormCondition? + public let help: String? + + public init( + id: String, + label: String, + kind: Kind, + visibleWhen: PluginFormCondition? = nil, + help: String? = nil + ) { + self.id = id + self.label = label + self.kind = kind + self.visibleWhen = visibleWhen + self.help = help + } + + /// The value the field holds before the user touches it, spelled as it is submitted. + public var initialValue: String { + switch kind { + case .text: + return "" + case .integer(let defaultValue, _, _): + return defaultValue.map(String.init) ?? "" + case .picker(_, let defaultValue): + return defaultValue + case .toggle(let defaultValue): + return defaultValue ? "true" : "false" + } + } +} + +public struct PluginFormOption: Sendable, Hashable { + public let value: String + public let label: String + + public init(value: String, label: String) { + self.value = value + self.label = label + } +} + +/// When a field is shown. `values` nil means "whenever the other field is not empty". +public struct PluginFormCondition: Sendable, Hashable { + public let fieldId: String + public let values: [String]? + + public init(fieldId: String, values: [String]?) { + self.fieldId = fieldId + self.values = values + } + + public func isSatisfied(by entry: [String: String]) -> Bool { + let current = entry[fieldId]?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + guard let values else { return !current.isEmpty } + return values.contains(current) + } +} + +public struct PluginCreateTableRequest: Sendable, Hashable { + public let tableName: String + public let values: [String: String] + /// One array per repeating section, keyed by the section's id, one dictionary per entry. + public let repeatedValues: [String: [[String: String]]] + + public init(tableName: String, values: [String: String], repeatedValues: [String: [[String: String]]] = [:]) { + self.tableName = tableName + self.values = values + self.repeatedValues = repeatedValues + } +} + +/// Why a Create Table form cannot become statements yet, in words for the person filling it in. +public struct PluginCreateTableFormError: Error, LocalizedError, Sendable, Equatable { + public let message: String + public let fieldId: String? + + public init(message: String, fieldId: String? = nil) { + self.message = message + self.fieldId = fieldId + } + + public var errorDescription: String? { message } +} diff --git a/Plugins/TableProPluginKit/PluginDatabaseDriver.swift b/Plugins/TableProPluginKit/PluginDatabaseDriver.swift index d6b2a0308e..335b1ff94c 100644 --- a/Plugins/TableProPluginKit/PluginDatabaseDriver.swift +++ b/Plugins/TableProPluginKit/PluginDatabaseDriver.swift @@ -237,6 +237,11 @@ public protocol PluginDatabaseDriver: AnyObject, Sendable { func fetchDependentSequences(table: String, schema: String?) async throws -> [(name: String, ddl: String)] func createDatabaseFormSpec() async throws -> PluginCreateDatabaseFormSpec? func createDatabase(_ request: PluginCreateDatabaseRequest) async throws + /// The form the app shows for a new table in place of its column grid, or nil for the grid. + func createTableFormSpec(schema: String?) -> PluginCreateTableFormSpec? + /// The statements that create what the form describes. Throws `PluginCreateTableFormError` + /// with a message for the user when a value is missing or invalid. + func createTableStatements(for request: PluginCreateTableRequest, schema: String?) throws -> [String] func dropDatabase(name: String) async throws func dropSchema(name: String) async throws @@ -825,6 +830,12 @@ public extension PluginDatabaseDriver { ) } + func createTableFormSpec(schema: String?) -> PluginCreateTableFormSpec? { nil } + + func createTableStatements(for request: PluginCreateTableRequest, schema: String?) throws -> [String] { + throw PluginCreateTableFormError(message: String(localized: "This database has no Create Table form")) + } + func renameTable(name: String, schema: String?, to newName: String, objectType: String) async throws { throw PluginDriverUnsupportedOperation.renameTable } diff --git a/Plugins/TableProPluginKit/PluginSchemaOperation.swift b/Plugins/TableProPluginKit/PluginSchemaOperation.swift index 30f9136d97..1183e0efc7 100644 --- a/Plugins/TableProPluginKit/PluginSchemaOperation.swift +++ b/Plugins/TableProPluginKit/PluginSchemaOperation.swift @@ -9,4 +9,7 @@ public enum PluginSchemaOperation: Sendable { case addColumn(PluginColumnDefinition) case addIndex(PluginIndexDefinition) case renameCheckConstraint(from: String, to: String) + /// An index edited in place, which the app otherwise saves as a drop followed by an add. + case modifyIndex(old: PluginIndexDefinition, new: PluginIndexDefinition) + case dropIndex(PluginIndexDefinition) } diff --git a/TablePro/Core/Coordinators/ExactRowCounter.swift b/TablePro/Core/Coordinators/ExactRowCounter.swift index 481c4f052a..493cab09b2 100644 --- a/TablePro/Core/Coordinators/ExactRowCounter.swift +++ b/TablePro/Core/Coordinators/ExactRowCounter.swift @@ -11,8 +11,14 @@ internal enum ExactRowCounter { private static let logger = Logger(subsystem: "com.TablePro", category: "ExactRowCounter") - internal static func route(countSQL: String?, driverOwnsQueryBuilding: Bool) -> Route { - guard let countSQL else { return .driverCount } + /// An engine whose count is a billed scan has no `COUNT(*)` to fall back on, so its driver is the only source + /// and a failure there is the answer rather than a cue to try the host query. + internal static func route( + countSQL: String?, + driverOwnsQueryBuilding: Bool, + exactRowCountIsBilledScan: Bool + ) -> Route { + guard let countSQL, !exactRowCountIsBilledScan else { return .driverCount } return driverOwnsQueryBuilding ? .driverCountThenHostSQL(countSQL) : .hostCountSQL(countSQL) } @@ -23,8 +29,12 @@ internal enum ExactRowCounter { logicMode: FilterLogicMode, countSQL: String? ) async throws -> Int? { - let ownsQueryBuilding = driver.queryBuildingPluginDriver != nil - switch route(countSQL: countSQL, driverOwnsQueryBuilding: ownsQueryBuilding) { + let chosen = route( + countSQL: countSQL, + driverOwnsQueryBuilding: driver.queryBuildingPluginDriver != nil, + exactRowCountIsBilledScan: driver.connection.type.exactRowCountIsBilledScan + ) + switch chosen { case .driverCount: return try await driver.fetchExactRowCount(table: table, filters: filters, logicMode: logicMode) case .hostCountSQL(let sql): diff --git a/TablePro/Core/Coordinators/PaginationCoordinator.swift b/TablePro/Core/Coordinators/PaginationCoordinator.swift index dac34baee2..a006ddb9f6 100644 --- a/TablePro/Core/Coordinators/PaginationCoordinator.swift +++ b/TablePro/Core/Coordinators/PaginationCoordinator.swift @@ -10,6 +10,7 @@ import os import TableProPluginKit private let progressLog = Logger(subsystem: "com.TablePro", category: "ProgressiveLoad") +private let exactCountLog = Logger(subsystem: "com.TablePro", category: "ExactRowCount") @MainActor final class PaginationCoordinator: ObservableObject { @@ -189,7 +190,7 @@ final class PaginationCoordinator: ObservableObject { let token = UUID() parent.claimExactCount(for: tabId, token: token) let task = Task(priority: .userInitiated) { [parent] in - let count = await Self.exactRowCount( + let outcome = await Self.exactRowCount( scope: scope, tableName: tableName, filters: filters, @@ -210,25 +211,50 @@ final class PaginationCoordinator: ObservableObject { if ownsIndicator { tab.pagination.isCountingExact = false } - guard isCurrent, let count, count >= 0 else { return } - tab.pagination.totalRowCount = count - tab.pagination.isApproximateRowCount = false + guard isCurrent else { return } + Self.applyExactCount(outcome, to: &tab) } } parent.setRowCountTask(task, token: token, for: tabId) } + static func applyExactCount(_ outcome: Result, to tab: inout QueryTab) { + switch outcome { + case .success(let count): + if let shown = tab.pagination.exactCountError, tab.execution.errorMessage == shown { + tab.execution.errorMessage = nil + } + tab.pagination.exactCountError = nil + guard let count, count >= 0 else { return } + tab.pagination.totalRowCount = count + tab.pagination.isApproximateRowCount = false + case .failure(let error): + guard !DatabaseCancellationDiagnosis.isCancellation(error) else { return } + let message = DatabaseWriteRejectionDiagnosis.formatted(error) + tab.execution.errorMessage = message + tab.pagination.exactCountError = message + } + } + + /// The user asked for this count, so a failure is shown on the tab rather than dropped: an engine whose count + /// only its driver can run, such as a throttled DynamoDB scan, has no other answer to fall back on. private static func exactRowCount( scope: DatabaseScope, tableName: String, filters: [TableFilter], logicMode: FilterLogicMode, countSQL: String? - ) async -> Int? { - try? await DatabaseManager.shared.withMetadataDriver(scope: scope, workload: .bulk) { driver in - try await ExactRowCounter.count( - on: driver, table: tableName, filters: filters, logicMode: logicMode, countSQL: countSQL - ) + ) async -> Result { + do { + let count = try await DatabaseManager.shared.withMetadataDriver(scope: scope, workload: .bulk) { driver in + try await ExactRowCounter.count( + on: driver, table: tableName, filters: filters, logicMode: logicMode, countSQL: countSQL + ) + } + return .success(count) + } catch { + exactCountLog.warning("Exact row count failed: \(error.publicLogShape, privacy: .public)") + return .failure(error) } } diff --git a/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift b/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift index 61addab285..7e606166da 100644 --- a/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift +++ b/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift @@ -649,7 +649,7 @@ extension QueryExecutionCoordinator { connectionType: DatabaseType ) { let isNonSQL = PluginManager.shared.editorLanguage(for: connectionType) != .sql - let countsAutomatically = PluginManager.shared.paginationCapability(for: connectionType).allowsSeeking + let countsAutomatically = PluginManager.shared.countsRowsAutomatically(for: connectionType) let contentEpoch = parent.tabExecution.contentEpoch(for: tabId) let token = UUID() @@ -756,8 +756,8 @@ extension QueryExecutionCoordinator { } } - /// An engine that cannot skip rows has no pages for a total to bound, so it is only counted - /// when the user asks: each automatic count would be a full scan the engine may bill for. + /// `countsAutomatically` is `PluginManager.countsRowsAutomatically(for:)`: an engine that cannot skip rows, or + /// whose count is a billed scan, is only counted when the user asks. static func rowCountPlan( isNonSQL: Bool, filterState: TabFilterState, diff --git a/TablePro/Core/Database/DatabaseDriver.swift b/TablePro/Core/Database/DatabaseDriver.swift index 7504bdadf9..9bb5c852f5 100644 --- a/TablePro/Core/Database/DatabaseDriver.swift +++ b/TablePro/Core/Database/DatabaseDriver.swift @@ -252,6 +252,10 @@ protocol DatabaseDriver: AnyObject, Sendable { func createDatabase(_ request: CreateDatabaseRequest) async throws + func createTableFormSpec(schema: String?) -> PluginCreateTableFormSpec? + + func createTableStatements(for request: PluginCreateTableRequest, schema: String?) throws -> [String] + func dropDatabase(name: String) async throws func dropSchema(name: String) async throws @@ -540,6 +544,12 @@ extension DatabaseDriver { func createDatabaseFormSpec() async throws -> CreateDatabaseFormSpec? { nil } + func createTableFormSpec(schema: String?) -> PluginCreateTableFormSpec? { nil } + + func createTableStatements(for request: PluginCreateTableRequest, schema: String?) throws -> [String] { + throw PluginCreateTableFormError(message: String(localized: "This database has no Create Table form")) + } + func fetchSessionContexts() async throws -> [PluginSessionContext]? { nil } func switchSessionContext(id: String, to value: String) async throws {} diff --git a/TablePro/Core/Plugins/DatabaseType+Registry.swift b/TablePro/Core/Plugins/DatabaseType+Registry.swift index fc343ecc0a..875b268d7f 100644 --- a/TablePro/Core/Plugins/DatabaseType+Registry.swift +++ b/TablePro/Core/Plugins/DatabaseType+Registry.swift @@ -78,6 +78,10 @@ extension DatabaseType { PluginMetadataRegistry.shared.snapshot(for: self)?.capabilities.supportsConnectionPooling ?? true } + var exactRowCountIsBilledScan: Bool { + PluginMetadataRegistry.shared.snapshot(for: self)?.capabilities.exactRowCountIsBilledScan ?? false + } + var authenticationIsDatabaseScoped: Bool { PluginMetadataRegistry.shared.snapshot(for: self)? .capabilities.authenticationIsDatabaseScoped ?? false diff --git a/TablePro/Core/Plugins/PluginDriverAdapter+CreateTableForm.swift b/TablePro/Core/Plugins/PluginDriverAdapter+CreateTableForm.swift new file mode 100644 index 0000000000..9ec28777d3 --- /dev/null +++ b/TablePro/Core/Plugins/PluginDriverAdapter+CreateTableForm.swift @@ -0,0 +1,17 @@ +// +// PluginDriverAdapter+CreateTableForm.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +internal extension PluginDriverAdapter { + func createTableFormSpec(schema: String?) -> PluginCreateTableFormSpec? { + schemaPluginDriver.createTableFormSpec(schema: schema) + } + + func createTableStatements(for request: PluginCreateTableRequest, schema: String?) throws -> [String] { + try schemaPluginDriver.createTableStatements(for: request, schema: schema) + } +} diff --git a/TablePro/Core/Plugins/PluginDriverAdapter.swift b/TablePro/Core/Plugins/PluginDriverAdapter.swift index 97df2999f9..17b1eae66c 100644 --- a/TablePro/Core/Plugins/PluginDriverAdapter.swift +++ b/TablePro/Core/Plugins/PluginDriverAdapter.swift @@ -899,7 +899,13 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor // MARK: - Result Mapping private func mapQueryResult(_ pluginResult: PluginQueryResult) -> QueryResult { - let columnTypes = mapColumnTypes(rawTypeNames: pluginResult.columnTypeNames) + let columnTypes = mapColumnTypes( + rawTypeNames: pluginResult.columnTypeNames, + classificationHints: PluginResultColumnHints.hints( + from: pluginResult.columnMeta, + columnCount: pluginResult.columns.count + ) + ) var result = QueryResult( columns: pluginResult.columns, columnTypes: columnTypes, @@ -917,16 +923,22 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor return result } - private func mapColumnTypes(rawTypeNames: [String]) -> [ColumnType] { + private func mapColumnTypes(rawTypeNames: [String], classificationHints: [String?]) -> [ColumnType] { state.withLock { state in - rawTypeNames.map { rawTypeName in - if let cached = state.columnTypeCache[rawTypeName] { return cached } - let mapped = classifier.classify(rawTypeName: rawTypeName) - state.columnTypeCache[rawTypeName] = mapped - return mapped + rawTypeNames.enumerated().map { index, rawTypeName in + let hint = index < classificationHints.count ? classificationHints[index] : nil + guard let hint else { return classified(rawTypeName, cache: &state.columnTypeCache) } + return classified(hint, cache: &state.columnTypeCache).declared(as: rawTypeName) } } } + + private func classified(_ typeName: String, cache: inout [String: ColumnType]) -> ColumnType { + if let cached = cache[typeName] { return cached } + let mapped = classifier.classify(rawTypeName: typeName) + cache[typeName] = mapped + return mapped + } } private extension PluginDriverAdapter { diff --git a/TablePro/Core/Plugins/PluginManager+Registration.swift b/TablePro/Core/Plugins/PluginManager+Registration.swift index c33cb8e5ac..575e1d93a4 100644 --- a/TablePro/Core/Plugins/PluginManager+Registration.swift +++ b/TablePro/Core/Plugins/PluginManager+Registration.swift @@ -481,6 +481,16 @@ extension PluginManager { PaginationCapability.of(databaseType) } + func exactRowCountIsBilledScan(for databaseType: DatabaseType) -> Bool { + databaseType.exactRowCountIsBilledScan + } + + /// An engine that cannot skip rows has no pages for a total to bound, and one whose count is a billed scan + /// would charge for every table it opens, so neither is counted until the user asks. + func countsRowsAutomatically(for databaseType: DatabaseType) -> Bool { + paginationCapability(for: databaseType).allowsSeeking && !exactRowCountIsBilledScan(for: databaseType) + } + func isEngineReadOnly(for databaseType: DatabaseType) -> Bool { PluginMetadataRegistry.shared.snapshot(for: databaseType)? .capabilities.isEngineReadOnly ?? false diff --git a/TablePro/Core/Plugins/PluginManager.swift b/TablePro/Core/Plugins/PluginManager.swift index 25cf521337..59b21ba506 100644 --- a/TablePro/Core/Plugins/PluginManager.swift +++ b/TablePro/Core/Plugins/PluginManager.swift @@ -83,6 +83,12 @@ final class PluginManager: ObservableObject { /// the session already has open so nothing the app owns wraps a transaction the user opened. /// Both have defaults (nil and `.unknown`), so an already-built plugin keeps loading and /// answers them; the minimum stays where it is and no bulk re-release is needed. + /// + /// 33 also adds `createTableFormSpec(schema:)` and `createTableStatements(for:schema:)`, the + /// Create Table form a driver describes for tables that are not a list of typed columns. The + /// defaults answer nil and throw, so an already-built plugin keeps the column grid. It adds the + /// `modifyIndex` and `dropIndex` cases to the non-frozen `PluginSchemaOperation`, which an + /// already-built plugin answers through its `@unknown default`. nonisolated static let currentPluginKitVersion = 33 /// Still 19, so every plugin already published for the previous release keeps loading. diff --git a/TablePro/Core/Plugins/PluginMetadataRegistry+CloudDefaults.swift b/TablePro/Core/Plugins/PluginMetadataRegistry+CloudDefaults.swift index eb9e2a8947..04eebc33b4 100644 --- a/TablePro/Core/Plugins/PluginMetadataRegistry+CloudDefaults.swift +++ b/TablePro/Core/Plugins/PluginMetadataRegistry+CloudDefaults.swift @@ -12,7 +12,7 @@ extension PluginMetadataRegistry { [ ("DynamoDB", PluginMetadataSnapshot( displayName: "Amazon DynamoDB", iconName: "dynamodb-icon", defaultPort: 0, - requiresAuthentication: true, supportsForeignKeys: false, supportsSchemaEditing: false, + requiresAuthentication: true, supportsForeignKeys: false, supportsSchemaEditing: true, isDownloadable: true, primaryUrlScheme: "", parameterStyle: .questionMark, navigationModel: .standard, explainVariants: [], pathFieldRole: .database, @@ -31,7 +31,15 @@ extension PluginMetadataRegistry { supportsReadOnlyMode: true, supportsQueryProgress: false, requiresReconnectForDatabaseSwitch: false, - supportsDropDatabase: false + supportsDropDatabase: false, + supportsAddColumn: false, + supportsModifyColumn: false, + supportsDropColumn: false, + supportsRenameColumn: false, + supportsAddIndex: true, + supportsDropIndex: true, + supportsModifyPrimaryKey: false, + exactRowCountIsBilledScan: true ), schema: PluginMetadataSnapshot.SchemaInfo( defaultSchemaName: "", @@ -44,110 +52,15 @@ extension PluginMetadataRegistry { systemSchemaNames: [], fileExtensions: [], databaseGroupingStrategy: .flat, - structureColumnFields: [.name, .type] + structureColumnFields: [.name, .type, .primaryKey] ), editor: PluginMetadataSnapshot.EditorConfig( - sqlDialect: SQLDialectDescriptor( - identifierQuote: "\"", - keywords: [ - "SELECT", "FROM", "WHERE", "INSERT", "INTO", "VALUE", "SET", - "UPDATE", "DELETE", "AND", "OR", "NOT", "IN", "BETWEEN", - "EXISTS", "MISSING", "IS", "NULL", "LIMIT", - ], - functions: [ - "begins_with", "contains", "size", "attribute_type", - "attribute_exists", "attribute_not_exists", - ], - dataTypes: ["S", "N", "B", "BOOL", "NULL", "L", "M", "SS", "NS", "BS"], - caseSensitivityStyle: .driverManaged - ), - statementCompletions: [ - CompletionEntry(label: "SELECT", insertText: "SELECT"), - CompletionEntry(label: "INSERT INTO", insertText: "INSERT INTO"), - CompletionEntry(label: "UPDATE", insertText: "UPDATE"), - CompletionEntry(label: "DELETE FROM", insertText: "DELETE FROM"), - CompletionEntry(label: "VALUE", insertText: "VALUE"), - CompletionEntry(label: "SET", insertText: "SET"), - CompletionEntry(label: "WHERE", insertText: "WHERE"), - CompletionEntry(label: "begins_with", insertText: "begins_with"), - CompletionEntry(label: "contains", insertText: "contains"), - CompletionEntry(label: "size", insertText: "size"), - CompletionEntry(label: "attribute_type", insertText: "attribute_type"), - CompletionEntry(label: "attribute_exists", insertText: "attribute_exists"), - CompletionEntry(label: "attribute_not_exists", insertText: "attribute_not_exists"), - ], - columnTypesByCategory: [ - "String": ["S"], - "Number": ["N"], - "Binary": ["B"], - "Boolean": ["BOOL"], - "Null": ["NULL"], - "List": ["L"], - "Map": ["M"], - "String Set": ["SS"], - "Number Set": ["NS"], - "Binary Set": ["BS"], - ] + sqlDialect: DynamoDBCuratedDefaults.sqlDialect, + statementCompletions: DynamoDBCuratedDefaults.statementCompletions, + columnTypesByCategory: DynamoDBCuratedDefaults.columnTypesByCategory ), connection: PluginMetadataSnapshot.ConnectionConfig( - additionalConnectionFields: [ - ConnectionField( - id: "awsAuthMethod", - label: String(localized: "Auth Method"), - defaultValue: "credentials", - fieldType: .dropdown(options: [ - .init(value: "credentials", label: "Access Key + Secret Key"), - .init(value: "profile", label: "AWS Profile"), - .init(value: "sso", label: "AWS SSO"), - ]), - section: .authentication - ), - ConnectionField( - id: "awsAccessKeyId", - label: String(localized: "Access Key ID"), - placeholder: "AKIA...", - section: .authentication, - visibleWhen: FieldVisibilityRule(fieldId: "awsAuthMethod", values: ["credentials"]) - ), - ConnectionField( - id: "awsSecretAccessKey", - label: String(localized: "Secret Access Key"), - placeholder: "wJalr...", - fieldType: .secure, - section: .authentication, - hidesPassword: true, - visibleWhen: FieldVisibilityRule(fieldId: "awsAuthMethod", values: ["credentials"]) - ), - ConnectionField( - id: "awsSessionToken", - label: String(localized: "Session Token"), - placeholder: "Optional (for temporary credentials)", - fieldType: .secure, - section: .authentication, - visibleWhen: FieldVisibilityRule(fieldId: "awsAuthMethod", values: ["credentials"]) - ), - ConnectionField( - id: "awsProfileName", - label: String(localized: "Profile Name"), - placeholder: "default", - section: .authentication, - visibleWhen: FieldVisibilityRule(fieldId: "awsAuthMethod", values: ["profile", "sso"]) - ).withDynamicOptions(.awsProfiles), - ConnectionField( - id: "awsRegion", - label: String(localized: "AWS Region"), - placeholder: "us-east-1", - defaultValue: "us-east-1", - fieldType: .text, - section: .authentication - ), - ConnectionField( - id: "awsEndpointUrl", - label: String(localized: "Custom Endpoint"), - placeholder: "http://localhost:8000 (DynamoDB Local)", - section: .authentication - ), - ], + additionalConnectionFields: DynamoDBCuratedDefaults.connectionFields, category: .cloud, tagline: String(localized: "AWS managed key-value/document store"), hidesBuiltInPassword: true @@ -855,3 +768,128 @@ func snowflakeConnectionFields() -> [ConnectionField] { ) ] } + +/// The DynamoDB plugin's own declarations, copied because the app cannot import the plugin. A registry plugin may be +/// older than the app, so these are what a connection shows until one loads. +private enum DynamoDBCuratedDefaults { + static let attributeTypeNames = [ + "String", "Number", "Binary", "Boolean", "Null", "List", "Map", "String Set", "Number Set", "Binary Set" + ] + + static let sqlDialect = SQLDialectDescriptor( + identifierQuote: "\"", + keywords: [ + "SELECT", "FROM", "WHERE", "INSERT", "INTO", "VALUE", "SET", "REMOVE", "UPDATE", "DELETE", + "AND", "OR", "NOT", "IN", "BETWEEN", "EXISTS", "MISSING", "IS", "NULL", "TRUE", "FALSE", + "ORDER", "BY", "ASC", "DESC", "RETURNING", "ALL", "OLD", "NEW", "MODIFIED" + ], + functions: [ + "begins_with", "contains", "size", "attribute_type", "attribute_exists", "attribute_not_exists", "EXISTS" + ], + dataTypes: Set(attributeTypeNames), + booleanLiteralStyle: .truefalse, + autoLimitStyle: .none, + caseSensitivityStyle: .driverManaged + ) + + static let columnTypesByCategory: [String: [String]] = [ + "Key": ["String", "Number", "Binary"], + "Scalar": ["Boolean", "Null"], + "Document": ["List", "Map"], + "Set": ["String Set", "Number Set", "Binary Set"] + ] + + static var statementCompletions: [CompletionEntry] { + let partiQL = [ + "SELECT", "INSERT INTO", "UPDATE", "DELETE FROM", "VALUE", "SET", "REMOVE", "WHERE", "AND", "OR", + "BETWEEN", "IN", "IS", "NOT", "NULL", "MISSING", "EXISTS", "ORDER BY", "RETURNING ALL OLD *", + "begins_with", "contains", "size", "attribute_type" + ].map { CompletionEntry(label: $0, insertText: $0) } + let requests: [CompletionEntry] = [ + requestTemplate("Scan", #"{"TableName": ""}"#), + requestTemplate("Query", ##"{"TableName": "", "KeyConditionExpression": "#pk = :pk", "##, + ##""ExpressionAttributeNames": {"#pk": ""}, "ExpressionAttributeValues": {":pk": {"S": ""}}}"##), + requestTemplate("GetItem", #"{"TableName": "", "Key": {"": {"S": ""}}}"#), + requestTemplate("PutItem", #"{"TableName": "", "Item": {"": {"S": ""}}}"#), + requestTemplate("UpdateItem", #"{"TableName": "", "Key": {"": {"S": ""}}, "UpdateExpression": ""}"#), + requestTemplate("DeleteItem", #"{"TableName": "", "Key": {"": {"S": ""}}}"#), + requestTemplate("DescribeTable", #"{"TableName": ""}"#), + requestTemplate("CreateTable", #"{"TableName": "", "BillingMode": "PAY_PER_REQUEST", "#, + #""AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], "#, + #""KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}]}"#), + requestTemplate("UpdateTable", #"{"TableName": ""}"#), + requestTemplate("UpdateTimeToLive", #"{"TableName": "", "TimeToLiveSpecification": "#, + #"{"Enabled": true, "AttributeName": "expiresAt"}}"#) + ] + return partiQL + requests + } + + static var connectionFields: [ConnectionField] { + let accessKey = "credentials" + let profile = "profile" + let singleSignOn = "sso" + let local = "local" + return [ + ConnectionField( + id: "awsAuthMethod", + label: String(localized: "Auth Method"), + defaultValue: accessKey, + fieldType: .dropdown(options: [ + .init(value: accessKey, label: String(localized: "Access Key + Secret Key")), + .init(value: profile, label: String(localized: "AWS Profile")), + .init(value: singleSignOn, label: String(localized: "AWS SSO")), + .init(value: local, label: String(localized: "DynamoDB Local (no credentials)")) + ]), + section: .authentication + ), + ConnectionField( + id: "awsAccessKeyId", + label: String(localized: "Access Key ID"), + placeholder: "AKIA...", + section: .authentication, + visibleWhen: FieldVisibilityRule(fieldId: "awsAuthMethod", values: [accessKey]) + ), + ConnectionField( + id: "awsSecretAccessKey", + label: String(localized: "Secret Access Key"), + placeholder: "wJalr...", + fieldType: .secure, + section: .authentication, + hidesPassword: true, + visibleWhen: FieldVisibilityRule(fieldId: "awsAuthMethod", values: [accessKey]) + ), + ConnectionField( + id: "awsSessionToken", + label: String(localized: "Session Token"), + placeholder: String(localized: "Optional, for temporary credentials"), + fieldType: .secure, + section: .authentication, + visibleWhen: FieldVisibilityRule(fieldId: "awsAuthMethod", values: [accessKey]) + ), + ConnectionField( + id: "awsProfileName", + label: String(localized: "Profile Name"), + placeholder: "default", + section: .authentication, + visibleWhen: FieldVisibilityRule(fieldId: "awsAuthMethod", values: [profile, singleSignOn]) + ).withDynamicOptions(.awsProfiles), + ConnectionField( + id: "awsRegion", + label: String(localized: "AWS Region"), + placeholder: String(localized: "The profile's region, or us-east-1"), + fieldType: .text, + section: .authentication + ), + ConnectionField( + id: "awsEndpointUrl", + label: String(localized: "Custom Endpoint"), + placeholder: String(localized: "Optional, such as http://localhost:8000"), + section: .authentication + ) + ] + } + + private static func requestTemplate(_ operation: String, _ parts: String...) -> CompletionEntry { + CompletionEntry(label: "\(operation) {…}", insertText: "\(operation) " + parts.joined()) + } +} diff --git a/TablePro/Core/Plugins/PluginMetadataRegistry.swift b/TablePro/Core/Plugins/PluginMetadataRegistry.swift index 010bb4adc6..e1618d8f0f 100644 --- a/TablePro/Core/Plugins/PluginMetadataRegistry.swift +++ b/TablePro/Core/Plugins/PluginMetadataRegistry.swift @@ -88,6 +88,10 @@ struct PluginMetadataSnapshot: Sendable { /// database, such as ClickHouse's `default`, leaves this false. var browsingRequiresSelectedDatabase: Bool = false var pagination: PaginationCapability = .offset + /// Whether an exact count reads the whole table and the engine bills that read, while its query language + /// has no `COUNT(*)`. DynamoDB is the case: a count is a `Scan` of every item. Such an engine is counted + /// only when the user asks, and only by its driver. + var exactRowCountIsBilledScan: Bool = false var isEngineReadOnly: Bool = false /// Which connection field carries the path of the local database file this driver opens, @@ -708,6 +712,7 @@ final class PluginMetadataRegistry: @unchecked Sendable { browsingRequiresSelectedDatabase: existingSnapshot?.capabilities .browsingRequiresSelectedDatabase ?? false, pagination: existingSnapshot?.capabilities.pagination ?? .offset, + exactRowCountIsBilledScan: existingSnapshot?.capabilities.exactRowCountIsBilledScan ?? false, isEngineReadOnly: existingSnapshot?.capabilities.isEngineReadOnly ?? false, localFilePathField: existingSnapshot?.capabilities.localFilePathField, supportsRemoteDatabaseFile: existingSnapshot?.capabilities diff --git a/TablePro/Core/Plugins/PluginResultColumnHints.swift b/TablePro/Core/Plugins/PluginResultColumnHints.swift new file mode 100644 index 0000000000..ef52f9af71 --- /dev/null +++ b/TablePro/Core/Plugins/PluginResultColumnHints.swift @@ -0,0 +1,43 @@ +// +// PluginResultColumnHints.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +/// The name a driver asks the app to classify each result column by, beside the name it declares. +/// +/// DynamoDB declares a column `Map`, `List` or `String Set`, which no SQL classifier reads as a document, and hints +/// `JSON` so the cell opens in the JSON editor while the header still reads `Map`. +enum PluginResultColumnHints { + /// One entry per column, nil where the driver set no hint. A list that does not describe every column is + /// ignored whole, because its entries could not be matched to columns by position. + static func hints(from columnMeta: [PluginColumnInfo]?, columnCount: Int) -> [String?] { + guard let columnMeta, columnMeta.count == columnCount else { + return Array(repeating: nil, count: columnCount) + } + return columnMeta.map(\.classificationTypeName) + } +} + +extension ColumnType { + /// The same kind of column under the name the server declared for it. + func declared(as rawType: String) -> ColumnType { + switch self { + case .text: return .text(rawType: rawType) + case .integer: return .integer(rawType: rawType) + case .decimal: return .decimal(rawType: rawType) + case .date: return .date(rawType: rawType) + case .timestamp: return .timestamp(rawType: rawType) + case .datetime: return .datetime(rawType: rawType) + case .boolean: return .boolean(rawType: rawType) + case .blob: return .blob(rawType: rawType) + case .json: return .json(rawType: rawType) + case .enumType(_, let values): return .enumType(rawType: rawType, values: values) + case .set(_, let values): return .set(rawType: rawType, values: values) + case .spatial: return .spatial(rawType: rawType) + case .array(_, let element): return .array(rawType: rawType, element: element) + } + } +} diff --git a/TablePro/Core/SchemaTracking/SchemaOperationRefusal.swift b/TablePro/Core/SchemaTracking/SchemaOperationRefusal.swift index 578a8123bf..bf0860faf8 100644 --- a/TablePro/Core/SchemaTracking/SchemaOperationRefusal.swift +++ b/TablePro/Core/SchemaTracking/SchemaOperationRefusal.swift @@ -17,16 +17,20 @@ internal enum SchemaOperationRefusal { switch change { case .addColumn(let column): return driver.schemaOperationRefusal(.addColumn(column.toPlugin())) - case .addIndex(let index), .modifyIndex(_, let index): + case .addIndex(let index): return driver.schemaOperationRefusal(.addIndex(index.toPlugin())) + case .modifyIndex(let old, let new): + return driver.schemaOperationRefusal(.modifyIndex(old: old.toPlugin(), new: new.toPlugin())) + ?? driver.schemaOperationRefusal(.addIndex(new.toPlugin())) + case .deleteIndex(let index): + return driver.schemaOperationRefusal(.dropIndex(index.toPlugin())) case .modifyCheckConstraint(let old, let new): if let refusal = driver.checkConstraintRefusal { return refusal } guard old.expression == new.expression, old.name != new.name else { return nil } return driver.schemaOperationRefusal(.renameCheckConstraint(from: old.name, to: new.name)) case .addCheckConstraint, .deleteCheckConstraint: return driver.checkConstraintRefusal - case .modifyColumn, .deleteColumn, .deleteIndex, .addForeignKey, .modifyForeignKey, - .deleteForeignKey, .modifyPrimaryKey: + case .modifyColumn, .deleteColumn, .addForeignKey, .modifyForeignKey, .deleteForeignKey, .modifyPrimaryKey: return nil } } diff --git a/TablePro/Core/SchemaTracking/SchemaStatementGenerator.swift b/TablePro/Core/SchemaTracking/SchemaStatementGenerator.swift index 3e9d8191d2..b84e08cceb 100644 --- a/TablePro/Core/SchemaTracking/SchemaStatementGenerator.swift +++ b/TablePro/Core/SchemaTracking/SchemaStatementGenerator.swift @@ -44,11 +44,11 @@ struct SchemaStatementGenerator { func generate(changes: [SchemaChange]) throws -> [SchemaStatement] { var statements: [SchemaStatement] = [] - let sortedChanges = sortByDependency(changes) - let refusals = sortedChanges.lazy.compactMap { SchemaOperationRefusal.reason(for: $0, driver: pluginDriver) } + let refusals = changes.lazy.compactMap { SchemaOperationRefusal.reason(for: $0, driver: pluginDriver) } if let reason = refusals.first { throw SchemaOperationRefusedError(reason: reason) } + let sortedChanges = sortByDependency(changes) for change in sortedChanges { let stmts = try generateStatements(for: change) diff --git a/TablePro/Core/Services/Export/ForeignApp/TablePlusImporter.swift b/TablePro/Core/Services/Export/ForeignApp/TablePlusImporter.swift index ca3590987c..7c55dcac39 100644 --- a/TablePro/Core/Services/Export/ForeignApp/TablePlusImporter.swift +++ b/TablePro/Core/Services/Export/ForeignApp/TablePlusImporter.swift @@ -197,6 +197,9 @@ struct TablePlusImporter: ForeignAppImporter { default: database = entry["DatabaseName"] as? String ?? "" } + if dbType == DatabaseType.dynamodb.rawValue { + additionalFields.merge(Self.dynamoDBFields(entry)) { _, imported in imported } + } let databaseMode = TablePlusPasswordMode.resolve(entry["DatabasePasswordMode"]) if databaseMode.promptsForPassword, !Self.hidesBuiltInPassword(dbType, fields: additionalFields) { @@ -329,6 +332,26 @@ struct TablePlusImporter: ForeignAppImporter { return snapshot.connection.additionalConnectionFields.hidesPassword(forValues: fields) } + /// A DynamoDB connection keeps its AWS region where a server connection keeps its host, and its access key ID + /// where a server connection keeps its user, both signed in with an access key. + private static func dynamoDBFields(_ entry: [String: Any]) -> [String: String] { + var fields = ["awsAuthMethod": "credentials"] + if let region = trimmedValue(entry["DatabaseHost"]) { + fields["awsRegion"] = region + } + if let accessKeyId = trimmedValue(entry["DatabaseUser"]) { + fields["awsAccessKeyId"] = accessKeyId + } + return fields + } + + private static func trimmedValue(_ raw: Any?) -> String? { + guard let text = (raw as? String)?.trimmingCharacters(in: .whitespacesAndNewlines), !text.isEmpty else { + return nil + } + return text + } + private static func databaseType(forDriver driver: String) -> String { switch driver { case "MicrosoftSQLServer": return DatabaseType.mssql.rawValue diff --git a/TablePro/Core/Utilities/SQL/CatalogChangeClassifier.swift b/TablePro/Core/Utilities/SQL/CatalogChangeClassifier.swift index d0082b7978..aab21f1be6 100644 --- a/TablePro/Core/Utilities/SQL/CatalogChangeClassifier.swift +++ b/TablePro/Core/Utilities/SQL/CatalogChangeClassifier.swift @@ -46,6 +46,9 @@ enum CatalogChangeClassifier { let tier = QueryClassifier.classifyTier(trimmed, databaseType: databaseType) return tier == .safe ? .none : opaque } + if let request = dynamoDBRequestEffect(trimmed, databaseType: databaseType) { + return request + } let grammar = databaseType.lexicalGrammar if QueryClassifier.runsPLSQL(trimmed, grammar: grammar) { return opaque @@ -53,6 +56,15 @@ enum CatalogChangeClassifier { return sqlEffect(trimmed, grammar: grammar) } + /// A DynamoDB request names its action, so only the three that create, change or drop a table touch the + /// catalog. PartiQL cannot, and an action the driver does not know is sent as PartiQL, so both take the SQL path. + private static func dynamoDBRequestEffect(_ trimmed: String, databaseType: DatabaseType) -> CatalogStatementEffect? { + guard databaseType == .dynamodb, + let action = DynamoDBRequestStatement(trimmed)?.action + else { return nil } + return action.changesCatalog ? CatalogStatementEffect(kinds: .tables, endsTransaction: false) : .none + } + private static func sqlEffect(_ trimmed: String, grammar: SQLLexicalGrammar) -> CatalogStatementEffect { let tokens = leadingTokens(of: trimmed, grammar: grammar) let leading = leadingKeywordEffect(tokens: tokens, trimmed: trimmed) diff --git a/TablePro/Core/Utilities/SQL/DynamoDBRequestJSON.swift b/TablePro/Core/Utilities/SQL/DynamoDBRequestJSON.swift new file mode 100644 index 0000000000..efad6742bc --- /dev/null +++ b/TablePro/Core/Utilities/SQL/DynamoDBRequestJSON.swift @@ -0,0 +1,228 @@ +// +// DynamoDBRequestJSON.swift +// TablePro +// + +import Foundation + +/// The body of a DynamoDB request as the classifier reads it. +/// +/// It accepts every document the driver's own strict parser accepts, and a few it would refuse, so a request the +/// driver runs is never read as unparseable. An object keeps every member in order, repeated keys included, and key +/// lookups ignore case, so nothing a request spells can hide from a rule that looks for it. +enum DynamoDBRequestJSON: Sendable, Equatable { + case object([Member]) + case array([DynamoDBRequestJSON]) + case string(String) + case number(String) + case bool(Bool) + case null + + struct Member: Sendable, Equatable { + let key: String + let value: DynamoDBRequestJSON + } + + var isObject: Bool { + if case .object = self { return true } + return false + } + + var elements: [DynamoDBRequestJSON] { + guard case .array(let items) = self else { return [] } + return items + } + + var memberValues: [DynamoDBRequestJSON] { + guard case .object(let members) = self else { return [] } + return members.map(\.value) + } + + var stringValue: String? { + guard case .string(let value) = self else { return nil } + return value + } + + var boolValue: Bool? { + guard case .bool(let value) = self else { return nil } + return value + } + + func values(forKey key: String) -> [DynamoDBRequestJSON] { + guard case .object(let members) = self else { return [] } + return members.filter { $0.key.caseInsensitiveCompare(key) == .orderedSame }.map(\.value) + } + + func hasMember(_ key: String) -> Bool { + !values(forKey: key).isEmpty + } + + /// Parses the JSON value at the start of `text`, and the text after it. + static func parsePrefix(_ text: Substring) -> (value: DynamoDBRequestJSON, remainder: String)? { + var parser = Parser(scalars: Array(text.unicodeScalars)) + guard let value = parser.parseValue(depth: 0) else { return nil } + var remainder = String.UnicodeScalarView() + remainder.append(contentsOf: parser.scalars[parser.position...]) + return (value, String(remainder)) + } +} + +private extension DynamoDBRequestJSON { + struct Parser { + static let maximumDepth = 128 + static let numberScalars: Set = [ + "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "-", "+", ".", "e", "E" + ] + static let replacement: Unicode.Scalar = "\u{FFFD}" + + let scalars: [Unicode.Scalar] + var position = 0 + + mutating func parseValue(depth: Int) -> DynamoDBRequestJSON? { + guard depth <= Self.maximumDepth else { return nil } + skipWhitespace() + guard position < scalars.count else { return nil } + switch scalars[position] { + case "{": + return parseObject(depth: depth) + case "[": + return parseArray(depth: depth) + case "\"": + return parseString().map(DynamoDBRequestJSON.string) + case "t": + return parseLiteral("true", as: .bool(true)) + case "f": + return parseLiteral("false", as: .bool(false)) + case "n": + return parseLiteral("null", as: .null) + default: + return parseNumber() + } + } + + mutating func skipWhitespace() { + while position < scalars.count, scalars[position].properties.isWhitespace { + position += 1 + } + } + + private mutating func parseObject(depth: Int) -> DynamoDBRequestJSON? { + position += 1 + var members: [Member] = [] + skipWhitespace() + if position < scalars.count, scalars[position] == "}" { + position += 1 + return .object(members) + } + while true { + skipWhitespace() + guard position < scalars.count, scalars[position] == "\"", let key = parseString() else { return nil } + skipWhitespace() + guard consume(":"), let value = parseValue(depth: depth + 1) else { return nil } + members.append(Member(key: key, value: value)) + skipWhitespace() + if consume(",") { continue } + guard consume("}") else { return nil } + return .object(members) + } + } + + private mutating func parseArray(depth: Int) -> DynamoDBRequestJSON? { + position += 1 + var items: [DynamoDBRequestJSON] = [] + skipWhitespace() + if consume("]") { return .array(items) } + while true { + guard let item = parseValue(depth: depth + 1) else { return nil } + items.append(item) + skipWhitespace() + if consume(",") { continue } + guard consume("]") else { return nil } + return .array(items) + } + } + + private mutating func parseString() -> String? { + position += 1 + var result = String.UnicodeScalarView() + while position < scalars.count { + let scalar = scalars[position] + position += 1 + if scalar == "\"" { return String(result) } + guard scalar == "\\" else { + result.append(scalar) + continue + } + guard let escaped = parseEscape() else { return nil } + result.append(escaped) + } + return nil + } + + private mutating func parseEscape() -> Unicode.Scalar? { + guard position < scalars.count else { return nil } + let marker = scalars[position] + position += 1 + switch marker { + case "b": return "\u{08}" + case "f": return "\u{0C}" + case "n": return "\n" + case "r": return "\r" + case "t": return "\t" + case "u": return parseUnicodeEscape() + default: return marker + } + } + + private mutating func parseUnicodeEscape() -> Unicode.Scalar? { + guard let high = parseHexQuad() else { return nil } + guard (0xD800...0xDBFF).contains(high) else { + return Unicode.Scalar(high) ?? Self.replacement + } + guard position + 1 < scalars.count, scalars[position] == "\\", scalars[position + 1] == "u" else { + return Self.replacement + } + position += 2 + guard let low = parseHexQuad() else { return nil } + guard (0xDC00...0xDFFF).contains(low) else { return Self.replacement } + return Unicode.Scalar(0x10000 + ((high - 0xD800) << 10) + (low - 0xDC00)) ?? Self.replacement + } + + private mutating func parseHexQuad() -> UInt32? { + guard position + 4 <= scalars.count else { return nil } + var value: UInt32 = 0 + for _ in 0..<4 { + guard let digit = UInt32(String(scalars[position]), radix: 16) else { return nil } + value = value * 16 + digit + position += 1 + } + return value + } + + private mutating func parseNumber() -> DynamoDBRequestJSON? { + let start = position + while position < scalars.count, Self.numberScalars.contains(scalars[position]) { + position += 1 + } + guard position > start else { return nil } + var text = String.UnicodeScalarView() + text.append(contentsOf: scalars[start.. DynamoDBRequestJSON? { + let expected = Array(literal.unicodeScalars) + guard position + expected.count <= scalars.count, + Array(scalars[position..<(position + expected.count)]) == expected + else { return nil } + position += expected.count + return value + } + + private mutating func consume(_ scalar: Unicode.Scalar) -> Bool { + guard position < scalars.count, scalars[position] == scalar else { return false } + position += 1 + return true + } + } +} diff --git a/TablePro/Core/Utilities/SQL/DynamoDBRequestStatement.swift b/TablePro/Core/Utilities/SQL/DynamoDBRequestStatement.swift new file mode 100644 index 0000000000..a1814cde6f --- /dev/null +++ b/TablePro/Core/Utilities/SQL/DynamoDBRequestStatement.swift @@ -0,0 +1,222 @@ +// +// DynamoDBRequestStatement.swift +// TablePro +// + +import Foundation +import TableProSQLGrammar + +/// The DynamoDB API actions the driver runs from the editor as ` {request JSON}`, plus the `Browse` request a +/// table tab sends. The driver matches the name ignoring case, and so does this. +enum DynamoDBRequestAction: String, CaseIterable, Sendable { + case browse = "Browse" + case scan = "Scan" + case query = "Query" + case getItem = "GetItem" + case batchGetItem = "BatchGetItem" + case transactGetItems = "TransactGetItems" + case describeTable = "DescribeTable" + case listTables = "ListTables" + case describeTimeToLive = "DescribeTimeToLive" + case describeContinuousBackups = "DescribeContinuousBackups" + case listTagsOfResource = "ListTagsOfResource" + case describeLimits = "DescribeLimits" + case putItem = "PutItem" + case updateItem = "UpdateItem" + case deleteItem = "DeleteItem" + case batchWriteItem = "BatchWriteItem" + case transactWriteItems = "TransactWriteItems" + case executeStatement = "ExecuteStatement" + case executeTransaction = "ExecuteTransaction" + case batchExecuteStatement = "BatchExecuteStatement" + case createTable = "CreateTable" + case updateTable = "UpdateTable" + case deleteTable = "DeleteTable" + case updateTimeToLive = "UpdateTimeToLive" + case updateContinuousBackups = "UpdateContinuousBackups" + case tagResource = "TagResource" + case untagResource = "UntagResource" + + init?(named name: String) { + guard let match = Self.allCases.first(where: { $0.rawValue.caseInsensitiveCompare(name) == .orderedSame }) + else { return nil } + self = match + } + + var changesCatalog: Bool { + self == .createTable || self == .updateTable || self == .deleteTable + } + + /// The member listing the PartiQL entries a request runs, each naming its text under `Statement`. + var partiQLListKey: String? { + switch self { + case .executeTransaction: return "TransactStatements" + case .batchExecuteStatement: return "Statements" + default: return nil + } + } +} + +/// A statement in the request form, ` {request JSON}`, read the way the driver reads it: a word of letters +/// and then a JSON object, with a read window allowed after it. Anything else is PartiQL. +struct DynamoDBRequestStatement: Sendable, Equatable { + let actionName: String + let body: DynamoDBRequestJSON? + let trailingText: String + + init?(_ statement: String) { + let text = StatementBlank.trimming(QueryClassifier.strippingLeadingComments(statement)) + let verb = text.prefix { $0.isLetter } + guard !verb.isEmpty else { return nil } + let afterVerb = text[verb.endIndex...].drop { $0.isWhitespace } + guard afterVerb.first == "{" else { return nil } + actionName = String(verb) + let parsed = DynamoDBRequestJSON.parsePrefix(afterVerb) + body = parsed?.value + trailingText = parsed?.remainder ?? "" + } + + var action: DynamoDBRequestAction? { + DynamoDBRequestAction(named: actionName) + } + + /// The PartiQL a request runs, or nil when it runs none. An entry with no statement text of its own is left + /// out, so a caller has to compare against ``partiQLEntryCount`` before trusting the list is whole. + var partiQLStatements: [String]? { + guard let action, let body else { return nil } + if action == .executeStatement { + return body.values(forKey: "Statement").compactMap(\.stringValue) + } + guard let listKey = action.partiQLListKey else { return nil } + return partiQLEntries(of: body, listKey: listKey).flatMap { entry in + entry.values(forKey: "Statement").compactMap(\.stringValue) + } + } + + var partiQLEntryCount: Int { + guard let action, let body else { return 0 } + if action == .executeStatement { + return body.hasMember("Statement") ? 1 : 0 + } + guard let listKey = action.partiQLListKey else { return 0 } + return partiQLEntries(of: body, listKey: listKey).count + } + + /// Whether nothing follows the JSON but the `ORDER BY`, `LIMIT` and `OFFSET` the driver accepts after a read. + var hasOnlyReadWindowAfterBody: Bool { + DynamoDBReadWindowText.isReadWindow(trailingText) + } + + private func partiQLEntries(of body: DynamoDBRequestJSON, listKey: String) -> [DynamoDBRequestJSON] { + body.values(forKey: listKey).flatMap(\.elements) + } +} + +/// The clause the driver takes after a read request: `ORDER BY "a" [ASC|DESC], ...`, then `LIMIT n`, then +/// `OFFSET m`, each optional, with comments and a trailing semicolon allowed. +enum DynamoDBReadWindowText { + static func isReadWindow(_ text: String) -> Bool { + guard var tokens = tokens(of: text) else { return false } + if tokens.first?.isKeyword("ORDER") == true { + guard tokens.count >= 3, tokens[1].isKeyword("BY") else { return false } + tokens.removeFirst(2) + guard consumeOrderTerms(&tokens) else { return false } + } + consumeCount(after: "LIMIT", in: &tokens) + consumeCount(after: "OFFSET", in: &tokens) + return tokens.isEmpty + } + + private struct Token { + enum Kind { + case word + case quotedIdentifier + case number + case comma + } + + let kind: Kind + let text: String + + func isKeyword(_ keyword: String) -> Bool { + kind == .word && text.caseInsensitiveCompare(keyword) == .orderedSame + } + + var isIdentifier: Bool { + kind == .quotedIdentifier || kind == .word + } + } + + private static func consumeOrderTerms(_ tokens: inout [Token]) -> Bool { + while true { + guard let term = tokens.first, term.isIdentifier else { return false } + tokens.removeFirst() + if tokens.first?.isKeyword("ASC") == true || tokens.first?.isKeyword("DESC") == true { + tokens.removeFirst() + } + guard tokens.first?.kind == .comma else { return true } + tokens.removeFirst() + } + } + + private static func consumeCount(after keyword: String, in tokens: inout [Token]) { + guard tokens.count >= 2, tokens[0].isKeyword(keyword), tokens[1].kind == .number else { return } + tokens.removeFirst(2) + } + + private static func tokens(of text: String) -> [Token]? { + var tokens: [Token] = [] + var remaining = Substring(text) + while true { + remaining = remaining.drop { $0.isWhitespace || $0 == ";" } + guard let first = remaining.first else { return tokens } + if remaining.hasPrefix("--") { + remaining = remaining.drop { !$0.isNewline } + continue + } + if remaining.hasPrefix("/*") { + guard let close = remaining.range(of: "*/") else { return tokens } + remaining = remaining[close.upperBound...] + continue + } + if first == "," { + tokens.append(Token(kind: .comma, text: ",")) + remaining = remaining.dropFirst() + continue + } + if first == "\"" { + guard let quoted = quotedIdentifier(in: remaining) else { return nil } + tokens.append(Token(kind: .quotedIdentifier, text: quoted.identifier)) + remaining = quoted.rest + continue + } + if first.isNumber { + let digits = remaining.prefix { $0.isNumber } + tokens.append(Token(kind: .number, text: String(digits))) + remaining = remaining.dropFirst(digits.count) + continue + } + guard first.isLetter || first == "_" else { return nil } + let word = remaining.prefix { $0.isLetter || $0.isNumber || $0 == "_" } + tokens.append(Token(kind: .word, text: String(word))) + remaining = remaining.dropFirst(word.count) + } + } + + private static func quotedIdentifier(in text: Substring) -> (identifier: String, rest: Substring)? { + var identifier = "" + var index = text.index(after: text.startIndex) + while index < text.endIndex { + let character = text[index] + index = text.index(after: index) + guard character == "\"" else { + identifier.append(character) + continue + } + guard index < text.endIndex, text[index] == "\"" else { return (identifier, text[index...]) } + identifier.append("\"") + index = text.index(after: index) + } + return nil + } +} diff --git a/TablePro/Core/Utilities/SQL/QueryClassifier+DynamoDB.swift b/TablePro/Core/Utilities/SQL/QueryClassifier+DynamoDB.swift new file mode 100644 index 0000000000..48e5e9e8d1 --- /dev/null +++ b/TablePro/Core/Utilities/SQL/QueryClassifier+DynamoDB.swift @@ -0,0 +1,130 @@ +// +// QueryClassifier+DynamoDB.swift +// TablePro +// + +import Foundation + +/// Tiering a DynamoDB request by the action it names and what its body asks for. +/// +/// A request is one statement of its own, so it is tiered where every split statement is, under each reading. A +/// read is safe. A write is a write, and destructive where the request takes data or a safeguard away: deleting a +/// table, an index or a replica, turning deletion protection or point-in-time recovery off, enabling a TTL that +/// expires items, or deleting items in a batch or a transaction. An action it does not know is destructive, because +/// a plugin from the registry can be newer than the app and run one this list has never heard of. A body it +/// cannot read is the worst its action can do, because the text no longer shows which of those requests will run +/// and the driver may still send it. Text after the body that is not a read window is tiered as statements of its +/// own, which is how a gate reads statements joined without a semicolon. PartiQL keeps the SQL rules, including the +/// PartiQL a request carries in its body. +extension QueryClassifier { + static func dynamoDBClassification(_ statement: String, databaseType: DatabaseType) -> QueryClassification? { + guard databaseType == .dynamodb, let request = DynamoDBRequestStatement(statement) else { return nil } + return QueryClassification(tier: dynamoDBTier(of: request), reachesFilesystemOrExecutesCode: false) + } + + /// Nil for PartiQL, which the SQL rule answers. A request deletes everything only through the PartiQL it carries + /// or the statements after it. + static func dynamoDBDeletesEverything(_ statement: String, databaseType: DatabaseType) -> Bool? { + guard databaseType == .dynamodb, let request = DynamoDBRequestStatement(statement) else { return nil } + let carried = request.partiQLStatements ?? [] + let following = request.hasOnlyReadWindowAfterBody ? [] : [request.trailingText] + return (carried + following).contains { isDangerousQuery($0, databaseType: databaseType) } + } + + private static func dynamoDBTier(of request: DynamoDBRequestStatement) -> QueryTier { + guard let action = request.action else { return .destructive } + guard let body = request.body, body.isObject else { return worstTier(of: action) } + let floor: QueryTier = action == .deleteTable ? .destructive : .safe + let requestTier = QueryClassification.worse(floor, dynamoDBBodyTier(action: action, body: body, request: request)) + guard !request.hasOnlyReadWindowAfterBody else { return requestTier } + return QueryClassification.worse(requestTier, classifyTier(request.trailingText, databaseType: .dynamodb)) + } + + private static func worstTier(of action: DynamoDBRequestAction) -> QueryTier { + switch action { + case .deleteTable, .updateTable, .updateTimeToLive, .updateContinuousBackups, .batchWriteItem, + .transactWriteItems, .executeStatement, .executeTransaction, .batchExecuteStatement: + return .destructive + case .browse, .scan, .query, .getItem, .batchGetItem, .transactGetItems, .describeTable, .listTables, + .describeTimeToLive, .describeContinuousBackups, .listTagsOfResource, .describeLimits, .putItem, + .updateItem, .deleteItem, .createTable, .tagResource, .untagResource: + return .write + } + } + + private static func dynamoDBBodyTier( + action: DynamoDBRequestAction, + body: DynamoDBRequestJSON, + request: DynamoDBRequestStatement + ) -> QueryTier { + switch action { + case .browse, .scan, .query, .getItem, .batchGetItem, .transactGetItems, .describeTable, .listTables, + .describeTimeToLive, .describeContinuousBackups, .listTagsOfResource, .describeLimits: + return .safe + case .putItem, .updateItem, .deleteItem, .createTable, .tagResource, .untagResource: + return .write + case .deleteTable: + return .destructive + case .updateTable: + return updateTableTakesSomethingAway(body) ? .destructive : .write + case .updateTimeToLive: + return timeToLiveStaysOff(body) ? .write : .destructive + case .updateContinuousBackups: + return pointInTimeRecoveryStaysOn(body) ? .write : .destructive + case .batchWriteItem: + return batchDeletesItems(body) ? .destructive : .write + case .transactWriteItems: + return transactionDeletesItems(body) ? .destructive : .write + case .executeStatement, .executeTransaction, .batchExecuteStatement: + return carriedPartiQLTier(request, floor: action == .executeStatement ? .write : .safe) + } + } + + private static func updateTableTakesSomethingAway(_ body: DynamoDBRequestJSON) -> Bool { + let dropsIndex = body.values(forKey: "GlobalSecondaryIndexUpdates") + .flatMap(\.elements) + .contains { $0.hasMember("Delete") } + let dropsReplica = body.values(forKey: "ReplicaUpdates") + .flatMap(\.elements) + .contains { $0.hasMember("Delete") } + let liftsProtection = body.values(forKey: "DeletionProtectionEnabled").contains { $0.boolValue != true } + return dropsIndex || dropsReplica || liftsProtection + } + + /// Enabling a TTL starts DynamoDB deleting every item whose attribute has passed, so only a request that visibly + /// keeps TTL off is an ordinary write. + private static func timeToLiveStaysOff(_ body: DynamoDBRequestJSON) -> Bool { + let settings = body.values(forKey: "TimeToLiveSpecification").flatMap { $0.values(forKey: "Enabled") } + return !settings.isEmpty && settings.allSatisfy { $0.boolValue == false } + } + + private static func pointInTimeRecoveryStaysOn(_ body: DynamoDBRequestJSON) -> Bool { + let settings = body.values(forKey: "PointInTimeRecoverySpecification") + .flatMap { $0.values(forKey: "PointInTimeRecoveryEnabled") } + return !settings.isEmpty && settings.allSatisfy { $0.boolValue == true } + } + + private static func batchDeletesItems(_ body: DynamoDBRequestJSON) -> Bool { + body.values(forKey: "RequestItems") + .flatMap(\.memberValues) + .flatMap(\.elements) + .contains { $0.hasMember("DeleteRequest") } + } + + private static func transactionDeletesItems(_ body: DynamoDBRequestJSON) -> Bool { + body.values(forKey: "TransactItems") + .flatMap(\.elements) + .contains { $0.hasMember("Delete") } + } + + /// The worst tier of the PartiQL a request carries, and a write when any entry hides its statement. + private static func carriedPartiQLTier(_ request: DynamoDBRequestStatement, floor: QueryTier) -> QueryTier { + guard let statements = request.partiQLStatements, + !statements.isEmpty, + statements.count == request.partiQLEntryCount + else { return .write } + return statements.reduce(floor) { worst, statement in + QueryClassification.worse(worst, classifyTier(statement, databaseType: .dynamodb)) + } + } +} diff --git a/TablePro/Core/Utilities/SQL/QueryClassifier.swift b/TablePro/Core/Utilities/SQL/QueryClassifier.swift index 6332b59786..29507d26ae 100644 --- a/TablePro/Core/Utilities/SQL/QueryClassifier.swift +++ b/TablePro/Core/Utilities/SQL/QueryClassifier.swift @@ -94,7 +94,7 @@ enum QueryClassifier { } return readings.distinct(for: sql).contains { grammar in statements(of: sql, grammar: grammar).contains { statement in - statementDeletesEverything(statement, grammar: grammar) + statementDeletesEverything(statement, grammar: grammar, databaseType: databaseType) } } } @@ -128,13 +128,23 @@ enum QueryClassifier { grammar: SQLLexicalGrammar, databaseType: DatabaseType ) -> QueryClassification { + if let request = dynamoDBClassification(statement, databaseType: databaseType) { + return request + } if runsPLSQL(statement, grammar: grammar) { return plsqlBlockClassification(statement, grammar: grammar, databaseType: databaseType) } return sqlClassification(statement, grammar: grammar, databaseType: databaseType) } - private static func statementDeletesEverything(_ statement: String, grammar: SQLLexicalGrammar) -> Bool { + private static func statementDeletesEverything( + _ statement: String, + grammar: SQLLexicalGrammar, + databaseType: DatabaseType + ) -> Bool { + if let request = dynamoDBDeletesEverything(statement, databaseType: databaseType) { + return request + } if runsPLSQL(statement, grammar: grammar) { return plsqlBlockDeletesEverything(statement, grammar: grammar) } diff --git a/TablePro/Models/Query/QueryTabState.swift b/TablePro/Models/Query/QueryTabState.swift index 8f011c2818..546c6b289d 100644 --- a/TablePro/Models/Query/QueryTabState.swift +++ b/TablePro/Models/Query/QueryTabState.swift @@ -266,6 +266,9 @@ struct PaginationState: Equatable { var isLoading: Bool = false var isApproximateRowCount: Bool = false // True when totalRowCount is from fast estimate var isCountingExact: Bool = false // True while a user-requested exact count is running + /// The message a failed `Count Exactly` put on the tab, so a later count that succeeds takes + /// down its own error and no other. + var exactCountError: String? /// An automatic row count is running, so the total on screen is not the one this page settles on. /// /// Separate from `isCountingExact`, which the user asked for and which owns the spinner. This diff --git a/TablePro/Models/Schema/CreateTableFormState+Validation.swift b/TablePro/Models/Schema/CreateTableFormState+Validation.swift new file mode 100644 index 0000000000..585a877e7c --- /dev/null +++ b/TablePro/Models/Schema/CreateTableFormState+Validation.swift @@ -0,0 +1,157 @@ +// +// CreateTableFormState+Validation.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +internal extension CreateTableFormState { + struct Issue: Equatable, Sendable { + enum Kind: Equatable, Sendable { + case missing + case invalid + } + + let kind: Kind + let location: Location? + let fieldId: String? + let message: String + let qualifiedMessage: String + } + + enum Preview: Equatable, Sendable { + case statements(String) + case message(String) + } + + func issues(tableName: String) -> [Issue] { + var issues: [Issue] = [] + if tableName.trimmingCharacters(in: .whitespaces).isEmpty { + let message = String(localized: "The table needs a name.") + issues.append(Issue(kind: .missing, location: nil, fieldId: nil, message: message, qualifiedMessage: message)) + } + return issues + fieldIssues() + } + + func fieldIssues() -> [Issue] { + var issues: [Issue] = [] + for section in spec.sections where !section.isRepeating { + issues += fieldIssues(in: section, at: .topLevel, entryNumber: nil) + } + for section in spec.sections where section.isRepeating { + for (offset, entry) in entries(in: section.id).enumerated() { + let location = Location.entry(sectionId: section.id, entryId: entry.id) + issues += fieldIssues(in: section, at: location, entryNumber: offset + 1) + } + } + return issues + } + + func inlineMessage(for fieldId: String, at location: Location) -> String? { + if location == .topLevel, inlineSubmissionErrorFieldId == fieldId, let submissionError { + return submissionError.message + } + return fieldIssues() + .first { $0.kind == .invalid && $0.location == location && $0.fieldId == fieldId }? + .message + } + + var inlineSubmissionErrorFieldId: String? { + guard let fieldId = submissionError?.fieldId else { return nil } + let repeatingFieldIds = Set(spec.sections.filter(\.isRepeating).flatMap(\.fields).map(\.id)) + guard !repeatingFieldIds.contains(fieldId), + let field = topLevelFields.first(where: { $0.id == fieldId }), + isVisible(field, at: .topLevel) else { return nil } + return fieldId + } + + func preview( + tableName: String, + generate: (PluginCreateTableRequest) throws -> [String] + ) -> Preview { + if let issue = issues(tableName: tableName).first { + return .message(issue.qualifiedMessage) + } + do { + let statements = try generate(request(tableName: tableName)) + guard !statements.isEmpty else { + return .message(String(localized: "The form produced no statements to run.")) + } + return .statements(CreateTableStatements(statements: statements, issues: [], tableName: nil).preview) + } catch let formError as PluginCreateTableFormError { + return .message(formError.message) + } catch { + return .message(error.localizedDescription) + } + } + + static func entryTitle(number: Int) -> String { + String(format: String(localized: "Entry %lld"), Int64(number)) + } + + private func fieldIssues(in section: PluginFormSection, at location: Location, entryNumber: Int?) -> [Issue] { + visibleFields(in: section, at: location).compactMap { field in + guard let problem = Self.problem(with: field, value: value(of: field.id, at: location)) else { + return nil + } + return Issue( + kind: problem.kind, + location: location, + fieldId: field.id, + message: problem.message, + qualifiedMessage: Self.qualified(problem.message, section: section, entryNumber: entryNumber) + ) + } + } + + private static func qualified(_ message: String, section: PluginFormSection, entryNumber: Int?) -> String { + guard let entryNumber else { return message } + guard let title = section.title, !title.isEmpty else { + return String(format: String(localized: "Entry %1$lld: %2$@"), Int64(entryNumber), message) + } + return String(format: String(localized: "%1$@, entry %2$lld: %3$@"), title, Int64(entryNumber), message) + } + + private static func problem( + with field: PluginFormField, + value: String + ) -> (kind: Issue.Kind, message: String)? { + let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines) + switch field.kind { + case .text(_, let isRequired): + guard isRequired, trimmed.isEmpty else { return nil } + return (.missing, String(format: String(localized: "%@ is required."), field.label)) + case .integer(_, let minimum, let maximum): + guard !trimmed.isEmpty else { return nil } + guard let number = Int(trimmed) else { + return (.invalid, String(format: String(localized: "%@ must be a whole number."), field.label)) + } + guard let message = rangeProblem(number, minimum: minimum, maximum: maximum, label: field.label) else { + return nil + } + return (.invalid, message) + case .picker, .toggle: + return nil + @unknown default: + return nil + } + } + + private static func rangeProblem(_ number: Int, minimum: Int?, maximum: Int?, label: String) -> String? { + let isBelow = minimum.map { number < $0 } ?? false + let isAbove = maximum.map { number > $0 } ?? false + guard isBelow || isAbove else { return nil } + if let minimum, let maximum { + return String( + format: String(localized: "%1$@ must be between %2$lld and %3$lld."), + label, Int64(minimum), Int64(maximum) + ) + } + if let minimum { + return String(format: String(localized: "%1$@ must be at least %2$lld."), label, Int64(minimum)) + } + guard let maximum else { return nil } + return String(format: String(localized: "%1$@ must be at most %2$lld."), label, Int64(maximum)) + } +} diff --git a/TablePro/Models/Schema/CreateTableFormState.swift b/TablePro/Models/Schema/CreateTableFormState.swift new file mode 100644 index 0000000000..0f6e94c926 --- /dev/null +++ b/TablePro/Models/Schema/CreateTableFormState.swift @@ -0,0 +1,156 @@ +// +// CreateTableFormState.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +internal struct CreateTableFormState: Equatable, Sendable { + internal struct Entry: Identifiable, Equatable, Sendable { + internal let id: UUID + internal fileprivate(set) var values: [String: String] + } + + internal enum Location: Hashable, Sendable { + case topLevel + case entry(sectionId: String, entryId: UUID) + } + + internal let spec: PluginCreateTableFormSpec + internal private(set) var values: [String: String] + internal private(set) var submissionError: PluginCreateTableFormError? + private var entriesBySection: [String: [Entry]] = [:] + + internal init(spec: PluginCreateTableFormSpec) { + self.spec = spec + self.values = Self.initialValues(of: spec.sections.filter { !$0.isRepeating }.flatMap(\.fields)) + } + + internal var holdsWork: Bool { + values != Self.initialValues(of: topLevelFields) || entriesBySection.values.contains { !$0.isEmpty } + } + + internal var topLevelFields: [PluginFormField] { + spec.sections.filter { !$0.isRepeating }.flatMap(\.fields) + } + + internal func section(withId sectionId: String) -> PluginFormSection? { + spec.sections.first { $0.id == sectionId } + } + + internal func entries(in sectionId: String) -> [Entry] { + entriesBySection[sectionId] ?? [] + } + + internal func canAddEntry(to sectionId: String) -> Bool { + guard let section = section(withId: sectionId), section.isRepeating else { return false } + guard let maximumCount = section.maximumCount else { return true } + return entries(in: sectionId).count < maximumCount + } + + @discardableResult + internal mutating func addEntry(to sectionId: String) -> UUID? { + guard canAddEntry(to: sectionId), let section = section(withId: sectionId) else { return nil } + let entry = Entry(id: UUID(), values: Self.initialValues(of: section.fields)) + entriesBySection[sectionId, default: []].append(entry) + submissionError = nil + return entry.id + } + + internal mutating func removeEntry(_ entryId: UUID, from sectionId: String) { + guard entries(in: sectionId).contains(where: { $0.id == entryId }) else { return } + entriesBySection[sectionId]?.removeAll { $0.id == entryId } + submissionError = nil + } + + internal func value(of fieldId: String, at location: Location) -> String { + scopeValues(at: location)[fieldId] ?? "" + } + + internal mutating func setValue(_ value: String, of fieldId: String, at location: Location) { + guard self.value(of: fieldId, at: location) != value else { return } + switch location { + case .topLevel: + values[fieldId] = value + case .entry(let sectionId, let entryId): + guard let index = entries(in: sectionId).firstIndex(where: { $0.id == entryId }) else { return } + entriesBySection[sectionId]?[index].values[fieldId] = value + } + submissionError = nil + } + + internal mutating func recordSubmissionError(_ error: PluginCreateTableFormError) { + submissionError = error + } + + internal mutating func clearSubmissionError() { + submissionError = nil + } + + internal func isVisible(_ field: PluginFormField, at location: Location) -> Bool { + isVisible(field, among: fields(at: location), values: scopeValues(at: location), visited: []) + } + + internal func visibleFields(in section: PluginFormSection, at location: Location) -> [PluginFormField] { + section.fields.filter { isVisible($0, at: location) } + } + + internal func request(tableName: String) -> PluginCreateTableRequest { + let topLevel = topLevelFields.filter { isVisible($0, at: .topLevel) } + var repeated: [String: [[String: String]]] = [:] + for section in spec.sections where section.isRepeating { + repeated[section.id] = entries(in: section.id).map { entry in + let location = Location.entry(sectionId: section.id, entryId: entry.id) + return Self.submittedValues(of: visibleFields(in: section, at: location), from: entry.values) + } + } + return PluginCreateTableRequest( + tableName: tableName.trimmingCharacters(in: .whitespaces), + values: Self.submittedValues(of: topLevel, from: values), + repeatedValues: repeated + ) + } + + internal func fields(at location: Location) -> [PluginFormField] { + switch location { + case .topLevel: + return topLevelFields + case .entry(let sectionId, _): + return section(withId: sectionId)?.fields ?? [] + } + } + + private func scopeValues(at location: Location) -> [String: String] { + switch location { + case .topLevel: + return values + case .entry(let sectionId, let entryId): + return entries(in: sectionId).first { $0.id == entryId }?.values ?? [:] + } + } + + private func isVisible( + _ field: PluginFormField, + among fields: [PluginFormField], + values: [String: String], + visited: Set + ) -> Bool { + guard let condition = field.visibleWhen else { return true } + guard condition.isSatisfied(by: values) else { return false } + guard !visited.contains(field.id), + let controllingField = fields.first(where: { $0.id == condition.fieldId }) else { return true } + return isVisible(controllingField, among: fields, values: values, visited: visited.union([field.id])) + } + + private static func initialValues(of fields: [PluginFormField]) -> [String: String] { + Dictionary(fields.map { ($0.id, $0.initialValue) }, uniquingKeysWith: { first, _ in first }) + } + + private static func submittedValues( + of fields: [PluginFormField], + from values: [String: String] + ) -> [String: String] { + Dictionary(fields.map { ($0.id, values[$0.id] ?? "") }, uniquingKeysWith: { first, _ in first }) + } +} diff --git a/TablePro/Views/Structure/CreateTableDraft.swift b/TablePro/Views/Structure/CreateTableDraft.swift index 57ab9d6104..10feb522f6 100644 --- a/TablePro/Views/Structure/CreateTableDraft.swift +++ b/TablePro/Views/Structure/CreateTableDraft.swift @@ -5,6 +5,7 @@ import Combine import Foundation +import TableProPluginKit /// A table definition in progress, held outside the view that edits it. /// @@ -16,8 +17,22 @@ import Foundation internal final class CreateTableDraft: ObservableObject { internal let changeManager = StructureChangeManager() - @Published internal var tableName = "" + /// A new name answers whatever the driver said about the last one, so its error goes with it. + @Published internal var tableName = "" { + didSet { + guard tableName != oldValue else { return } + form?.clearSubmissionError() + } + } @Published internal var tableOptions = CreateTableOptions() + @Published internal var form: CreateTableFormState? + @Published internal private(set) var hasResolvedForm = false + + internal func resolveForm(from spec: @autoclosure () -> PluginCreateTableFormSpec?) { + guard !hasResolvedForm else { return } + hasResolvedForm = true + form = spec().map(CreateTableFormState.init(spec:)) + } /// Whether the draft holds anything worth losing. A tab that has only just opened does not: the /// editor seeds one blank column so the grid has a row to show, which registers as a pending @@ -27,6 +42,7 @@ internal final class CreateTableDraft: ObservableObject { /// holding nothing but foreign keys closed on Cmd+W with no prompt and the draft was dropped. internal var holdsWork: Bool { !tableName.isEmpty + || form?.holdsWork == true || changeManager.workingColumns.contains { !$0.name.isEmpty } || changeManager.workingIndexes.contains { !$0.name.isEmpty || !$0.columns.isEmpty } || changeManager.workingForeignKeys.contains { diff --git a/TablePro/Views/Structure/CreateTableFormEditor.swift b/TablePro/Views/Structure/CreateTableFormEditor.swift new file mode 100644 index 0000000000..604d8d0f48 --- /dev/null +++ b/TablePro/Views/Structure/CreateTableFormEditor.swift @@ -0,0 +1,85 @@ +// +// CreateTableFormEditor.swift +// TablePro +// + +import SwiftUI +import TableProPluginKit + +private enum CreateTableFormPane: CaseIterable { + case form + case preview + + var displayName: String { + switch self { + case .form: String(localized: "Form") + case .preview: String(localized: "Preview") + } + } +} + +internal struct CreateTableFormEditor: View { + @ObservedObject internal var draft: CreateTableDraft + internal let databaseType: DatabaseType + internal let isCreating: Bool + internal let generateStatements: (PluginCreateTableRequest) throws -> [String] + internal let onCreate: () -> Void + + @State private var pane: CreateTableFormPane = .form + + internal var body: some View { + if let form = draft.form { + let issues = form.issues(tableName: draft.tableName) + let messages = [form.submissionError?.message].compactMap { $0 } + issues.map(\.qualifiedMessage) + VStack(spacing: 0) { + HStack(spacing: 8) { + if let first = messages.first { + Label(first, systemImage: "exclamationmark.triangle.fill") + .font(.callout) + .foregroundStyle(.secondary) + .lineLimit(1) + .truncationMode(.tail) + .help(messages.joined(separator: "\n")) + .accessibilityIdentifier("create-table-validation") + } + + Spacer(minLength: 12) + + Picker(String(localized: "View"), selection: $pane) { + ForEach(CreateTableFormPane.allCases, id: \.self) { pane in + Text(pane.displayName).tag(pane) + } + } + .pickerStyle(.segmented) + .labelsHidden() + .fixedSize() + + Spacer(minLength: 12) + + Button( + isCreating ? String(localized: "Creating…") : String(localized: "Create Table"), + action: onCreate + ) + .buttonStyle(.borderedProminent) + .tint(.accentColor) + .disabled(!issues.isEmpty || isCreating) + .keyboardShortcut(.return, modifiers: .command) + .accessibilityIdentifier("create-table-commit") + } + .padding() + + Divider() + + switch pane { + case .form: + CreateTableFormView(draft: draft) + case .preview: + CreateTableFormPreview( + preview: form.preview(tableName: draft.tableName, generate: generateStatements), + databaseType: databaseType + ) + } + } + } + } +} diff --git a/TablePro/Views/Structure/CreateTableFormFieldRow.swift b/TablePro/Views/Structure/CreateTableFormFieldRow.swift new file mode 100644 index 0000000000..8291281d20 --- /dev/null +++ b/TablePro/Views/Structure/CreateTableFormFieldRow.swift @@ -0,0 +1,59 @@ +// +// CreateTableFormFieldRow.swift +// TablePro +// + +import SwiftUI +import TableProPluginKit + +internal struct CreateTableFormFieldRow: View { + internal let field: PluginFormField + @Binding internal var value: String + internal let message: String? + internal let identifier: String + + internal var body: some View { + control + .accessibilityIdentifier(identifier) + if let help = field.help { + Text(help) + .font(.caption) + .foregroundStyle(.secondary) + } + if let message { + Text(message) + .font(.callout) + .foregroundStyle(.red) + } + } + + @ViewBuilder + private var control: some View { + switch field.kind { + case .text(let placeholder, _): + TextField(field.label, text: $value, prompt: placeholder.map { Text($0) }) + .autocorrectionDisabled(true) + case .integer(let defaultValue, _, _): + TextField(field.label, text: $value, prompt: defaultValue.map { Text(String($0)) }) + .autocorrectionDisabled(true) + case .picker(let options, _): + Picker(field.label, selection: $value) { + ForEach(options, id: \.value) { option in + Text(option.label).tag(option.value) + } + } + .pickerStyle(.menu) + case .toggle: + Toggle(field.label, isOn: isOn) + @unknown default: + TextField(field.label, text: $value) + } + } + + private var isOn: Binding { + Binding( + get: { value == "true" }, + set: { value = $0 ? "true" : "false" } + ) + } +} diff --git a/TablePro/Views/Structure/CreateTableFormPreview.swift b/TablePro/Views/Structure/CreateTableFormPreview.swift new file mode 100644 index 0000000000..bd360e1957 --- /dev/null +++ b/TablePro/Views/Structure/CreateTableFormPreview.swift @@ -0,0 +1,34 @@ +// +// CreateTableFormPreview.swift +// TablePro +// + +import SwiftUI + +internal struct CreateTableFormPreview: View { + internal let preview: CreateTableFormState.Preview + internal let databaseType: DatabaseType + + internal var body: some View { + Group { + switch preview { + case .statements(let text): + DDLTextView(ddl: text, fontSize: .constant(13), databaseType: databaseType) + case .message(let message): + VStack(spacing: 8) { + Image(systemName: "doc.plaintext") + .font(.largeTitle) + .foregroundStyle(.secondary) + .accessibilityHidden(true) + Text(message) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + } + .padding() + .frame(maxWidth: .infinity, maxHeight: .infinity) + } + } + .accessibilityElement(children: .contain) + .accessibilityIdentifier("create-table-form-preview") + } +} diff --git a/TablePro/Views/Structure/CreateTableFormSectionView.swift b/TablePro/Views/Structure/CreateTableFormSectionView.swift new file mode 100644 index 0000000000..196cb3e673 --- /dev/null +++ b/TablePro/Views/Structure/CreateTableFormSectionView.swift @@ -0,0 +1,95 @@ +// +// CreateTableFormSectionView.swift +// TablePro +// + +import SwiftUI +import TableProPluginKit + +internal struct CreateTableFormSectionView: View { + @ObservedObject internal var draft: CreateTableDraft + internal let form: CreateTableFormState + internal let section: PluginFormSection + internal let footnote: String? + + internal var body: some View { + Section { + if section.isRepeating { + repeatingRows + } else { + ForEach(form.visibleFields(in: section, at: .topLevel), id: \.id) { field in + CreateTableFormFieldRow( + field: field, + value: binding(for: field, at: .topLevel), + message: form.inlineMessage(for: field.id, at: .topLevel), + identifier: "create-table-form-field-\(field.id)" + ) + } + } + } header: { + if let title = section.title { + Text(title) + } + } footer: { + if let footnote { + Text(footnote) + .font(.callout) + .foregroundStyle(.secondary) + } + } + } + + @ViewBuilder + private var repeatingRows: some View { + ForEach(Array(form.entries(in: section.id).enumerated()), id: \.element.id) { offset, entry in + let location = CreateTableFormState.Location.entry(sectionId: section.id, entryId: entry.id) + let title = CreateTableFormState.entryTitle(number: offset + 1) + HStack { + Text(title) + .fontWeight(.semibold) + Spacer() + Button { + removeEntry(entry.id) + } label: { + Label(String(localized: "Remove"), systemImage: "minus.circle") + } + .labelStyle(.iconOnly) + .buttonStyle(.borderless) + .help(String(localized: "Remove")) + .accessibilityLabel(String(format: String(localized: "Remove %@"), title)) + .accessibilityIdentifier("create-table-form-remove-\(section.id)-\(offset)") + } + ForEach(form.visibleFields(in: section, at: location), id: \.id) { field in + CreateTableFormFieldRow( + field: field, + value: binding(for: field, at: location), + message: form.inlineMessage(for: field.id, at: location), + identifier: "create-table-form-field-\(section.id)-\(offset)-\(field.id)" + ) + } + } + Button(action: addEntry) { + Label(section.addLabel ?? String(localized: "Add"), systemImage: "plus") + } + .disabled(!form.canAddEntry(to: section.id)) + .accessibilityIdentifier("create-table-form-add-\(section.id)") + } + + private func binding( + for field: PluginFormField, + at location: CreateTableFormState.Location + ) -> Binding { + Binding( + get: { draft.form?.value(of: field.id, at: location) ?? "" }, + set: { draft.form?.setValue($0, of: field.id, at: location) } + ) + } + + private func addEntry() { + draft.form?.addEntry(to: section.id) + } + + private func removeEntry(_ entryId: UUID) { + draft.form?.removeEntry(entryId, from: section.id) + } +} diff --git a/TablePro/Views/Structure/CreateTableFormView.swift b/TablePro/Views/Structure/CreateTableFormView.swift new file mode 100644 index 0000000000..41bb4e31e8 --- /dev/null +++ b/TablePro/Views/Structure/CreateTableFormView.swift @@ -0,0 +1,29 @@ +// +// CreateTableFormView.swift +// TablePro +// + +import SwiftUI +import TableProPluginKit + +internal struct CreateTableFormView: View { + @ObservedObject internal var draft: CreateTableDraft + + internal var body: some View { + if let form = draft.form { + Form { + ForEach(Array(form.spec.sections.enumerated()), id: \.element.id) { offset, section in + CreateTableFormSectionView( + draft: draft, + form: form, + section: section, + footnote: offset == form.spec.sections.count - 1 ? form.spec.footnote : nil + ) + } + } + .formStyle(.grouped) + .accessibilityElement(children: .contain) + .accessibilityIdentifier("create-table-form") + } + } +} diff --git a/TablePro/Views/Structure/CreateTableView.swift b/TablePro/Views/Structure/CreateTableView.swift index 145699d513..620e24f5dd 100644 --- a/TablePro/Views/Structure/CreateTableView.swift +++ b/TablePro/Views/Structure/CreateTableView.swift @@ -89,25 +89,19 @@ struct CreateTableView: View { VStack(spacing: 0) { configBar Divider() - toolbar - Divider() - tabContent + editorContent } .navigationTitle(String(localized: "Create Table")) .onAppear { selectionState.indices = [] - coordinator?.inspectorRowSource = gridDelegate - gridDelegate.onSelectedRowsChanged = { self.selectedRows = $0 } - gridDelegate.onReferenceListsChanged = { coordinator?.inspectorRowSourceRevision += 1 } - serverSupport = StructureServerSupport.forConnection(connection.id) - updateGridDelegate() - if structureChangeManager.workingColumns.isEmpty { - structureChangeManager.addNewColumn() - } + draft.resolveForm( + from: DatabaseManager.shared.driver(for: connection.id)?.createTableFormSpec(schema: scope?.schema) + ) actionHandler.createTable = { createTable() } - actionHandler.undo = { gridDelegate.dataGridUndo() } - actionHandler.redo = { gridDelegate.dataGridRedo() } coordinator?.createTableActions = actionHandler + if draft.form == nil { + attachStructureGrid() + } coordinator?.toolbarState.hasCreateTablePending = isReadyToCreate } .onDisappear { @@ -143,6 +137,38 @@ struct CreateTableView: View { } } + @ViewBuilder + private var editorContent: some View { + if draft.form != nil { + CreateTableFormEditor( + draft: draft, + databaseType: connection.type, + isCreating: isCreating, + generateStatements: { request in try formStatements(for: request) }, + onCreate: { createTable() } + ) + } else if draft.hasResolvedForm { + toolbar + Divider() + tabContent + } else { + Color.clear + } + } + + private func attachStructureGrid() { + coordinator?.inspectorRowSource = gridDelegate + gridDelegate.onSelectedRowsChanged = { self.selectedRows = $0 } + gridDelegate.onReferenceListsChanged = { coordinator?.inspectorRowSourceRevision += 1 } + serverSupport = StructureServerSupport.forConnection(connection.id) + updateGridDelegate() + if structureChangeManager.workingColumns.isEmpty { + structureChangeManager.addNewColumn() + } + actionHandler.undo = { gridDelegate.dataGridUndo() } + actionHandler.redo = { gridDelegate.dataGridRedo() } + } + // MARK: - Config Bar private var configBar: some View { @@ -431,8 +457,19 @@ struct CreateTableView: View { // MARK: - Create Table private var isReadyToCreate: Bool { + guard !isCreating else { return false } + if let form = draft.form { + return form.issues(tableName: draft.tableName).isEmpty + } let composed = currentStatements() - return !isCreating && composed.issues.isEmpty && !composed.statements.isEmpty + return composed.issues.isEmpty && !composed.statements.isEmpty + } + + private func formStatements(for request: PluginCreateTableRequest) throws -> [String] { + guard let driver = DatabaseManager.shared.driver(for: connection.id) else { + throw PluginCreateTableFormError(message: String(localized: "Not connected to database")) + } + return try driver.createTableStatements(for: request, schema: scope?.schema) } private func updateCreateTablePendingState() { @@ -447,6 +484,14 @@ struct CreateTableView: View { /// last `CREATE INDEX` takes that tab's uncommitted work with it. `schemaChangeRoute` exists to /// keep the app's own DDL off the user's connection. private func createTable() { + guard draft.form != nil else { + createTableFromStructureGrid() + return + } + createTableFromForm() + } + + private func createTableFromStructureGrid() { guard !isCreating else { return } guard currentStatements().issues.isEmpty else { errorMessage = currentStatements().issues.map(\.qualifiedMessage).joined(separator: "\n") @@ -475,15 +520,7 @@ struct CreateTableView: View { return } let createdName = composed.tableName ?? draft.tableName - try await DatabaseManager.shared.executeCreateTable( - statements: composed.statements, - databaseType: connection.type, - scope: scope - ) - coordinator?.openTableTab( - createdName, schema: scope.schema, database: scope.database.nilIfEmpty - ) - AppCommands.shared.refreshData.send(DataRefreshRequest(connectionId: connection.id)) + try await runCreateTable(statements: composed.statements, createdName: createdName, in: scope) } catch { Self.logger.error("Create table failed: \(error.publicLogShape, privacy: .public)") errorMessage = error.localizedDescription @@ -491,4 +528,58 @@ struct CreateTableView: View { } } } + + private func createTableFromForm() { + guard !isCreating, let form = draft.form else { return } + let issues = form.issues(tableName: draft.tableName) + guard issues.isEmpty else { + errorMessage = issues.map(\.qualifiedMessage).joined(separator: "\n") + showError = true + return + } + guard let scope else { + errorMessage = String(localized: "Not connected to database") + showError = true + return + } + + let request = form.request(tableName: draft.tableName) + let schema = scope.schema + isCreating = true + errorMessage = nil + draft.form?.clearSubmissionError() + updateCreateTablePendingState() + + Task { + defer { isCreating = false } + do { + let statements = try await DatabaseManager.shared.withMetadataDriver(scope: scope) { driver in + try driver.createTableStatements(for: request, schema: schema) + } + guard !statements.isEmpty else { + draft.form?.recordSubmissionError(PluginCreateTableFormError( + message: String(localized: "The form produced no statements to run.") + )) + return + } + try await runCreateTable(statements: statements, createdName: request.tableName, in: scope) + } catch let formError as PluginCreateTableFormError { + draft.form?.recordSubmissionError(formError) + } catch { + Self.logger.error("Create table failed: \(error.publicLogShape, privacy: .public)") + errorMessage = error.localizedDescription + showError = true + } + } + } + + private func runCreateTable(statements: [String], createdName: String, in scope: DatabaseScope) async throws { + try await DatabaseManager.shared.executeCreateTable( + statements: statements, + databaseType: connection.type, + scope: scope + ) + coordinator?.openTableTab(createdName, schema: scope.schema, database: scope.database.nilIfEmpty) + AppCommands.shared.refreshData.send(DataRefreshRequest(connectionId: connection.id)) + } } diff --git a/TableProTests/Core/Coordinators/ExactCountOutcomeTests.swift b/TableProTests/Core/Coordinators/ExactCountOutcomeTests.swift new file mode 100644 index 0000000000..5ef93fa6a6 --- /dev/null +++ b/TableProTests/Core/Coordinators/ExactCountOutcomeTests.swift @@ -0,0 +1,53 @@ +// +// ExactCountOutcomeTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +@MainActor +@Suite("Count Exactly outcome") +struct ExactCountOutcomeTests { + private struct Throttled: LocalizedError { + var errorDescription: String? { "Rate exceeded" } + } + + private func makeTab() -> QueryTab { + QueryTab(title: "orders", query: "", tabType: .table, tableName: "orders") + } + + @Test("A failed count shows its error, and a later count that succeeds takes it down") + func successClearsTheCountsOwnError() { + var tab = makeTab() + + PaginationCoordinator.applyExactCount(.failure(Throttled()), to: &tab) + #expect(tab.execution.errorMessage?.contains("Rate exceeded") == true) + + PaginationCoordinator.applyExactCount(.success(42), to: &tab) + #expect(tab.execution.errorMessage == nil) + #expect(tab.pagination.totalRowCount == 42) + #expect(!tab.pagination.isApproximateRowCount) + } + + @Test("A count that succeeds leaves an error something else put on the tab") + func successKeepsAnotherError() { + var tab = makeTab() + PaginationCoordinator.applyExactCount(.failure(Throttled()), to: &tab) + tab.execution.errorMessage = "Syntax error near FROM" + + PaginationCoordinator.applyExactCount(.success(3), to: &tab) + + #expect(tab.execution.errorMessage == "Syntax error near FROM") + } + + @Test("A cancelled count shows nothing") + func cancellationShowsNothing() { + var tab = makeTab() + + PaginationCoordinator.applyExactCount(.failure(CancellationError()), to: &tab) + + #expect(tab.execution.errorMessage == nil) + } +} diff --git a/TableProTests/Core/Coordinators/ExactRowCounterTests.swift b/TableProTests/Core/Coordinators/ExactRowCounterTests.swift index 6e479ee650..89e8c103ce 100644 --- a/TableProTests/Core/Coordinators/ExactRowCounterTests.swift +++ b/TableProTests/Core/Coordinators/ExactRowCounterTests.swift @@ -77,9 +77,10 @@ struct ExactRowCounterTests { private func count( _ stub: CountStubDriver, - countSQL: String? = ExactRowCounterTests.countSQL + countSQL: String? = ExactRowCounterTests.countSQL, + type: DatabaseType = .spanner ) async throws -> Int? { - let adapter = PluginDriverAdapter(connection: TestFixtures.makeConnection(type: .spanner), pluginDriver: stub) + let adapter = PluginDriverAdapter(connection: TestFixtures.makeConnection(type: type), pluginDriver: stub) return try await ExactRowCounter.count( on: adapter, table: "Orders", filters: [], logicMode: .and, countSQL: countSQL ) @@ -88,23 +89,74 @@ struct ExactRowCounterTests { @Test("A driver that builds its own queries is asked first, and the host SQL follows it") func routesQueryBuildingDriversThroughTheDriverFirst() { #expect( - ExactRowCounter.route(countSQL: Self.countSQL, driverOwnsQueryBuilding: true) - == .driverCountThenHostSQL(Self.countSQL) + ExactRowCounter.route( + countSQL: Self.countSQL, driverOwnsQueryBuilding: true, exactRowCountIsBilledScan: false + ) == .driverCountThenHostSQL(Self.countSQL) ) } @Test("Every other SQL engine keeps the host COUNT query alone") func keepsHostSQLForOtherEngines() { #expect( - ExactRowCounter.route(countSQL: Self.countSQL, driverOwnsQueryBuilding: false) - == .hostCountSQL(Self.countSQL) + ExactRowCounter.route( + countSQL: Self.countSQL, driverOwnsQueryBuilding: false, exactRowCountIsBilledScan: false + ) == .hostCountSQL(Self.countSQL) ) } @Test("Without host SQL the driver is the only source, whoever builds the queries") func withoutHostSQLTheDriverCounts() { - #expect(ExactRowCounter.route(countSQL: nil, driverOwnsQueryBuilding: true) == .driverCount) - #expect(ExactRowCounter.route(countSQL: nil, driverOwnsQueryBuilding: false) == .driverCount) + #expect( + ExactRowCounter.route(countSQL: nil, driverOwnsQueryBuilding: true, exactRowCountIsBilledScan: false) + == .driverCount + ) + #expect( + ExactRowCounter.route(countSQL: nil, driverOwnsQueryBuilding: false, exactRowCountIsBilledScan: false) + == .driverCount + ) + } + + @Test("An engine whose count is a billed scan is counted by its driver alone, whatever host SQL exists") + func billedScanEnginesCountThroughTheDriverOnly() { + #expect( + ExactRowCounter.route(countSQL: Self.countSQL, driverOwnsQueryBuilding: true, exactRowCountIsBilledScan: true) + == .driverCount + ) + #expect( + ExactRowCounter.route(countSQL: Self.countSQL, driverOwnsQueryBuilding: false, exactRowCountIsBilledScan: true) + == .driverCount + ) + } + + @Test("DynamoDB's driver count is the only count, and the host COUNT never runs") + func dynamoDBCountsThroughTheDriver() async throws { + let stub = CountStubDriver(ownsQueryBuilding: true, driverCount: .success(42)) + + let result = try await count(stub, type: .dynamodb) + + #expect(result == 42) + #expect(stub.executedQueries.isEmpty) + } + + @Test("A DynamoDB driver that has no count leaves it unknown rather than running PartiQL COUNT(*)") + func dynamoDBNilCountDoesNotFallBack() async throws { + let stub = CountStubDriver(ownsQueryBuilding: true, driverCount: .success(nil)) + + let result = try await count(stub, type: .dynamodb) + + #expect(result == nil) + #expect(stub.executedQueries.isEmpty) + } + + @Test("A failed DynamoDB count reaches the caller instead of a host COUNT") + func dynamoDBFailureIsReported() async { + let stub = CountStubDriver(ownsQueryBuilding: true, driverCount: .failure(.refused)) + + await #expect(throws: CountStubError.self) { + _ = try await count(stub, type: .dynamodb) + } + #expect(stub.driverCountCallCount == 1) + #expect(stub.executedQueries.isEmpty) } @Test("The driver's own count wins and the host COUNT never runs") diff --git a/TableProTests/Core/Coordinators/RowCountPlanTests.swift b/TableProTests/Core/Coordinators/RowCountPlanTests.swift index e8c65c069d..f9323ce43e 100644 --- a/TableProTests/Core/Coordinators/RowCountPlanTests.swift +++ b/TableProTests/Core/Coordinators/RowCountPlanTests.swift @@ -73,6 +73,31 @@ struct RowCountPlanTests { #expect(plan == .approximate) } + @Test("An engine whose count is a billed scan is never counted automatically") + func billedScanEngineIsCountedOnlyOnRequest() { + let countsAutomatically = PluginManager.shared.countsRowsAutomatically(for: .dynamodb) + + let unfiltered = QueryExecutionCoordinator.rowCountPlan( + isNonSQL: false, filterState: TabFilterState(), approximateRowCount: 100, threshold: 100_000, + countsAutomatically: countsAutomatically + ) + let filteredPlan = QueryExecutionCoordinator.rowCountPlan( + isNonSQL: false, filterState: filtered(), approximateRowCount: 100, threshold: 100_000, + countsAutomatically: countsAutomatically + ) + + #expect(!countsAutomatically) + #expect(unfiltered == .skip) + #expect(filteredPlan == .clear) + } + + @Test("An engine that can seek and counts cheaply is still counted automatically") + func ordinaryEngineIsCountedAutomatically() { + #expect(PluginManager.shared.countsRowsAutomatically(for: .postgresql)) + #expect(!PluginManager.shared.exactRowCountIsBilledScan(for: .postgresql)) + #expect(PluginManager.shared.exactRowCountIsBilledScan(for: .dynamodb)) + } + @Test("Non-SQL filtered defers to the driver filtered count") func nonSQLFiltered() { let state = filtered() diff --git a/TableProTests/Core/Plugins/PasswordHidingTests.swift b/TableProTests/Core/Plugins/PasswordHidingTests.swift index 1618c0c802..e5d3044a85 100644 --- a/TableProTests/Core/Plugins/PasswordHidingTests.swift +++ b/TableProTests/Core/Plugins/PasswordHidingTests.swift @@ -204,6 +204,7 @@ struct PluginManagerPasswordHidingTests { #expect(hides("DuckDB", ["duckdbMode": "local"])) #expect(hides("DuckDB", ["duckdbMode": "remote"])) #expect(hides("DynamoDB", ["awsAuthMethod": "profile"])) + #expect(hides("DynamoDB", ["awsAuthMethod": "local"])) #expect(hides("BigQuery", ["bqAuthMethod": "adc"])) #expect(hides("Spanner", ["spAuthMethod": "adc"])) #expect(hides("Snowflake", ["snowflakeAuthMethod": "externalBrowser"])) diff --git a/TableProTests/Core/Plugins/PluginDriverAdapterCreateTableFormTests.swift b/TableProTests/Core/Plugins/PluginDriverAdapterCreateTableFormTests.swift new file mode 100644 index 0000000000..f4bf62d252 --- /dev/null +++ b/TableProTests/Core/Plugins/PluginDriverAdapterCreateTableFormTests.swift @@ -0,0 +1,116 @@ +// +// PluginDriverAdapterCreateTableFormTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +@testable import TablePro + +private class BaseFormDriver: @unchecked Sendable { + var supportsSchemas: Bool { false } + var supportsTransactions: Bool { false } + var currentSchema: String? { nil } + var serverVersion: String? { nil } + + func connect() async throws {} + func disconnect() {} + + func execute(query: String) async throws -> PluginQueryResult { + PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + + func fetchTables(schema: String?) async throws -> [PluginTableInfo] { [] } + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { [] } + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { [] } + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { [] } + func fetchTableDDL(table: String, schema: String?) async throws -> String { "" } + func fetchViewDefinition(view: String, schema: String?) async throws -> String { "" } + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + PluginTableMetadata(tableName: table) + } + + func fetchDatabases() async throws -> [String] { [] } + func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { + PluginDatabaseMetadata(name: database) + } +} + +private final class GridOnlyDriver: BaseFormDriver, PluginDatabaseDriver, @unchecked Sendable {} + +private final class FormDriver: BaseFormDriver, PluginDatabaseDriver, @unchecked Sendable { + static let spec = PluginCreateTableFormSpec(sections: [ + PluginFormSection(id: "keys", title: "Primary Key", fields: [ + PluginFormField(id: "pk", label: "Partition key", kind: .text(placeholder: nil, isRequired: true)) + ]) + ]) + + private(set) var specSchemas: [String?] = [] + private(set) var requests: [PluginCreateTableRequest] = [] + + func createTableFormSpec(schema: String?) -> PluginCreateTableFormSpec? { + specSchemas.append(schema) + return Self.spec + } + + func createTableStatements(for request: PluginCreateTableRequest, schema: String?) throws -> [String] { + requests.append(request) + guard let key = request.values["pk"], !key.isEmpty else { + throw PluginCreateTableFormError(message: "Enter the partition key", fieldId: "pk") + } + return ["CreateTable \(request.tableName) \(key) \(schema ?? "-")"] + } +} + +@Suite("Create Table form bridge") +struct PluginDriverAdapterCreateTableFormTests { + private func makeAdapter(driver: any PluginDatabaseDriver) -> PluginDriverAdapter { + PluginDriverAdapter(connection: DatabaseConnection(name: "Test", type: .redis), pluginDriver: driver) + } + + @Test("A DatabaseDriver that does not offer a form returns nil and refuses to build statements") + func databaseDriverDefaultHasNoForm() { + let driver = MockDatabaseDriver() + let request = PluginCreateTableRequest(tableName: "orders", values: [:]) + + #expect(driver.createTableFormSpec(schema: nil) == nil) + #expect(throws: PluginCreateTableFormError.self) { + try driver.createTableStatements(for: request, schema: nil) + } + } + + @Test("A plugin that keeps the PluginKit default leaves the adapter on the column grid") + func adapterPassesThroughTheDefault() { + let adapter = makeAdapter(driver: GridOnlyDriver()) + let request = PluginCreateTableRequest(tableName: "orders", values: [:]) + + #expect(adapter.createTableFormSpec(schema: "public") == nil) + #expect(throws: PluginCreateTableFormError.self) { + try adapter.createTableStatements(for: request, schema: nil) + } + } + + @Test("The adapter hands the plugin's spec and statements through with the schema") + func adapterBridgesThePluginForm() throws { + let plugin = FormDriver() + let adapter = makeAdapter(driver: plugin) + let request = PluginCreateTableRequest(tableName: "orders", values: ["pk": "id"]) + + #expect(adapter.createTableFormSpec(schema: "app") == FormDriver.spec) + #expect(plugin.specSchemas == ["app"]) + #expect(try adapter.createTableStatements(for: request, schema: "app") == ["CreateTable orders id app"]) + #expect(plugin.requests == [request]) + } + + @Test("A plugin's form error reaches the caller unchanged") + func adapterKeepsThePluginError() { + let adapter = makeAdapter(driver: FormDriver()) + let request = PluginCreateTableRequest(tableName: "orders", values: [:]) + + #expect(throws: PluginCreateTableFormError(message: "Enter the partition key", fieldId: "pk")) { + try adapter.createTableStatements(for: request, schema: nil) + } + } +} diff --git a/TableProTests/Core/Plugins/PluginMetadataRegistryCuratedCapabilityTests.swift b/TableProTests/Core/Plugins/PluginMetadataRegistryCuratedCapabilityTests.swift index f4a7ea2b31..347cfd6010 100644 --- a/TableProTests/Core/Plugins/PluginMetadataRegistryCuratedCapabilityTests.swift +++ b/TableProTests/Core/Plugins/PluginMetadataRegistryCuratedCapabilityTests.swift @@ -77,6 +77,22 @@ private final class MockMySQLPlugin: NSObject, TableProPlugin, DriverPlugin { } } +private final class MockDynamoDBPlugin: NSObject, TableProPlugin, DriverPlugin { + static let pluginName = "Mock DynamoDB" + static let pluginVersion = "1.0.0" + static let pluginDescription = "Stands in for the registry-distributed DynamoDB plugin" + static let capabilities: [PluginCapability] = [.databaseDriver] + + static let databaseTypeId = "DynamoDB" + static let databaseDisplayName = "Amazon DynamoDB" + static let iconName = "dynamodb-icon" + static let defaultPort = 0 + + func createDriver(config: DriverConnectionConfig) -> any PluginDatabaseDriver { + fatalError("Not used in tests") + } +} + private final class MockUnknownPlugin: NSObject, TableProPlugin, DriverPlugin { static let pluginName = "Mock Unknown" static let pluginVersion = "1.0.0" @@ -128,6 +144,18 @@ struct PluginMetadataRegistryCuratedCapabilityTests { #expect(built.capabilities.authenticationIsDatabaseScoped == true) } + @Test("DynamoDB keeps its billed-scan count when its plugin registers") + func dynamoDBKeepsItsBilledScanCount() { + let registry = PluginMetadataRegistry.shared + + let built = registry.buildMetadataSnapshot(from: MockDynamoDBPlugin.self) + + #expect( + built.capabilities.exactRowCountIsBilledScan == true, + "Every automatic count would be a Scan of the whole table that AWS bills for" + ) + } + @Test("MySQL keeps browsing only inside a selected database when its plugin registers") func mySQLKeepsBrowsingRequiresSelectedDatabase() { let registry = PluginMetadataRegistry.shared @@ -187,6 +215,7 @@ struct PluginMetadataRegistryCuratedCapabilityTests { #expect(built.capabilities.supportsConnectionPooling == true) #expect(built.capabilities.authenticationIsDatabaseScoped == false) #expect(built.capabilities.browsingRequiresSelectedDatabase == false) + #expect(built.capabilities.exactRowCountIsBilledScan == false) #expect(built.schema.implicitSchemaName == nil) } } diff --git a/TableProTests/Core/Plugins/PluginResultColumnHintsTests.swift b/TableProTests/Core/Plugins/PluginResultColumnHintsTests.swift new file mode 100644 index 0000000000..bef9d5c14a --- /dev/null +++ b/TableProTests/Core/Plugins/PluginResultColumnHintsTests.swift @@ -0,0 +1,159 @@ +// +// PluginResultColumnHintsTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private final class ResultStubDriver: PluginDatabaseDriver, @unchecked Sendable { + private let result: PluginQueryResult + + init(result: PluginQueryResult) { + self.result = result + } + + func execute(query: String) async throws -> PluginQueryResult { + result + } + + func connect() async throws {} + func disconnect() {} + func fetchTables(schema: String?) async throws -> [PluginTableInfo] { [] } + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { [] } + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { [] } + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { [] } + func fetchTableDDL(table: String, schema: String?) async throws -> String { "" } + func fetchViewDefinition(view: String, schema: String?) async throws -> String { "" } + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + PluginTableMetadata(tableName: table) + } + func fetchDatabases() async throws -> [String] { [] } + func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { + PluginDatabaseMetadata(name: database) + } +} + +@Suite("Result column classification hints") +struct PluginResultColumnHintsTests { + private func column(_ name: String, declared: String, hint: String?) -> PluginColumnInfo { + PluginColumnInfo( + name: name, + dataType: declared, + generationExpression: nil, + generationKind: nil, + ddlSpelling: nil, + ddlDefault: nil, + ddlGenerationExpression: nil, + ddlCollation: nil, + classificationTypeName: hint + ) + } + + private func columnTypes( + names: [String], + typeNames: [String], + meta: [PluginColumnInfo]?, + type: DatabaseType = .dynamodb + ) async throws -> [ColumnType] { + let result = PluginQueryResult( + columns: names, + columnTypeNames: typeNames, + rows: [], + rowsAffected: 0, + executionTime: 0, + columnMeta: meta + ) + let adapter = PluginDriverAdapter( + connection: TestFixtures.makeConnection(type: type), + pluginDriver: ResultStubDriver(result: result) + ) + return try await adapter.execute(query: "SELECT 1").columnTypes + } + + @Test("A hinted column is classified by the hint and keeps the name the server declared") + func hintClassifiesWhileTheDeclaredNameStays() async throws { + let names = ["pk", "doc", "items", "tags", "total", "active", "payload"] + let declared = ["String", "Map", "List", "String Set", "Number", "Boolean", "Binary"] + let hints = ["TEXT", "JSON", "JSON", "JSON", "NUMERIC", "BOOLEAN", "BLOB"] + let meta = zip(names, zip(declared, hints)).map { column($0, declared: $1.0, hint: $1.1) } + + let types = try await columnTypes(names: names, typeNames: declared, meta: meta) + + #expect(types == [ + .text(rawType: "String"), + .json(rawType: "Map"), + .json(rawType: "List"), + .json(rawType: "String Set"), + .decimal(rawType: "Number"), + .boolean(rawType: "Boolean"), + .blob(rawType: "Binary") + ]) + } + + @Test("A column without a hint is classified exactly as before") + func unhintedColumnsKeepTheirClassification() async throws { + let names = ["id", "name", "doc"] + let declared = ["INT", "VARCHAR", "Map"] + let meta = [ + column("id", declared: "INT", hint: nil), + column("name", declared: "VARCHAR", hint: nil), + column("doc", declared: "Map", hint: "JSON") + ] + let classifier = ColumnTypeClassifier() + + let types = try await columnTypes(names: names, typeNames: declared, meta: meta, type: .mysql) + + #expect(types[0] == classifier.classify(rawTypeName: "INT")) + #expect(types[1] == classifier.classify(rawTypeName: "VARCHAR")) + #expect(types[2] == .json(rawType: "Map")) + } + + @Test("Hints that do not describe every column are ignored, never matched by position") + func partialHintsAreIgnored() async throws { + let meta = [column("tags", declared: "String Set", hint: "JSON")] + let classifier = ColumnTypeClassifier() + + let types = try await columnTypes( + names: ["pk", "tags"], typeNames: ["String", "String Set"], meta: meta + ) + + #expect(types == [ + classifier.classify(rawTypeName: "String"), + classifier.classify(rawTypeName: "String Set") + ]) + } + + @Test("A result with no column metadata is classified by its declared names alone") + func missingMetadataKeepsDeclaredClassification() async throws { + let classifier = ColumnTypeClassifier() + + let types = try await columnTypes(names: ["tags"], typeNames: ["String Set"], meta: nil) + + #expect(types == [classifier.classify(rawTypeName: "String Set")]) + } + + @Test("Hints are read one per column, nil where the driver set none") + func hintsLineUpWithColumns() { + let meta = [column("a", declared: "Map", hint: "JSON"), column("b", declared: "INT", hint: nil)] + + #expect(PluginResultColumnHints.hints(from: meta, columnCount: 2) == ["JSON", nil]) + #expect(PluginResultColumnHints.hints(from: meta, columnCount: 3) == [nil, nil, nil]) + #expect(PluginResultColumnHints.hints(from: nil, columnCount: 1) == [nil]) + } + + @Test("Declaring a name keeps the kind, an enum's values and an array's element") + func declaredKeepsTheKind() { + #expect(ColumnType.json(rawType: "JSON").declared(as: "Map") == .json(rawType: "Map")) + #expect( + ColumnType.enumType(rawType: "ENUM", values: ["a"]).declared(as: "mood") + == .enumType(rawType: "mood", values: ["a"]) + ) + #expect( + ColumnType.array(rawType: "INT[]", element: .integer(rawType: "INT")).declared(as: "Number Set") + == .array(rawType: "Number Set", element: .integer(rawType: "INT")) + ) + } +} diff --git a/TableProTests/Core/SchemaTracking/SchemaOperationRefusalTests.swift b/TableProTests/Core/SchemaTracking/SchemaOperationRefusalTests.swift index c79342ba1b..dfa9782800 100644 --- a/TableProTests/Core/SchemaTracking/SchemaOperationRefusalTests.swift +++ b/TableProTests/Core/SchemaTracking/SchemaOperationRefusalTests.swift @@ -144,6 +144,22 @@ struct SchemaOperationRefusalTests { #expect(refusal(of: .addIndex(index("ix_btree", type: .btree)), driver: driver) == nil) } + @Test("An index edited in place and a dropped index reach the driver as their own operations") + func modifyAndDropIndexReachTheDriver() { + let driver = RefusingDDLDriver() + driver.refuse = { operation in + switch operation { + case .modifyIndex(let old, let new): return "modify \(old.name) to \(new.name)" + case .dropIndex(let index): return "drop \(index.name)" + default: return nil + } + } + let modify = SchemaChange.modifyIndex(old: index("ix", type: .btree), new: index("ix", type: .hash)) + + #expect(refusal(of: modify, driver: driver) == "modify ix to ix") + #expect(refusal(of: .deleteIndex(index("ix_old", type: .btree)), driver: driver) == "drop ix_old") + } + @Test("A refusal is reported ahead of a change in the same save that the driver cannot generate") func refusalWinsOverUngeneratableChange() { let unsupportedDrop = SchemaChange.deleteIndex(index("ix_old", type: .btree)) diff --git a/TableProTests/Core/Services/ForeignApp/TablePlusImporterTests.swift b/TableProTests/Core/Services/ForeignApp/TablePlusImporterTests.swift index 072dbf697d..25f08d8419 100644 --- a/TableProTests/Core/Services/ForeignApp/TablePlusImporterTests.swift +++ b/TableProTests/Core/Services/ForeignApp/TablePlusImporterTests.swift @@ -949,6 +949,46 @@ struct TablePlusImporterTests { #expect(connection.additionalFields?["promptForPassword"] == nil) } + @Test("importConnections carries a DynamoDB connection's region and access key into its sign-in fields") + func testImportConnections_dynamoDB_mapsRegionAndAccessKey() throws { + try writeConnections([ + makeConnection( + name: "Orders", driver: "DynamoDB", host: " eu-west-1 ", port: "", user: "AKIAEXAMPLE", + database: "", id: "c1" + ) + ]) + + let connection = try importer.importConnections(includePasswords: false).envelope.connections[0] + + #expect(connection.type == "DynamoDB") + #expect(connection.additionalFields?["awsRegion"] == "eu-west-1") + #expect(connection.additionalFields?["awsAccessKeyId"] == "AKIAEXAMPLE") + #expect(connection.additionalFields?["awsAuthMethod"] == "credentials") + } + + @Test("importConnections leaves the region unset when the DynamoDB connection names none") + func testImportConnections_dynamoDBWithoutRegion_leavesRegionUnset() throws { + try writeConnections([ + makeConnection(name: "Orders", driver: "DynamoDB", host: "", port: "", user: "", database: "", id: "c1") + ]) + + let connection = try importer.importConnections(includePasswords: false).envelope.connections[0] + + #expect(connection.additionalFields?["awsRegion"] == nil) + #expect(connection.additionalFields?["awsAccessKeyId"] == nil) + #expect(connection.additionalFields?["awsAuthMethod"] == "credentials") + } + + @Test("importConnections gives no other driver AWS sign-in fields") + func testImportConnections_otherDrivers_getNoAWSFields() throws { + try writeConnections([makeConnection(name: "Shop", driver: "MySQL", host: "eu-west-1", id: "c1")]) + + let connection = try importer.importConnections(includePasswords: false).envelope.connections[0] + + #expect(connection.additionalFields?["awsRegion"] == nil) + #expect(connection.additionalFields?["awsAuthMethod"] == nil) + } + @MainActor @Test("importConnections keeps the prompt flag through analyze and into the connection") func testImportConnections_promptFlagSurvivesTheImportPipeline() throws { diff --git a/TableProTests/Core/Utilities/SQL/QueryClassifierDynamoDBTests.swift b/TableProTests/Core/Utilities/SQL/QueryClassifierDynamoDBTests.swift new file mode 100644 index 0000000000..23a0b4d087 --- /dev/null +++ b/TableProTests/Core/Utilities/SQL/QueryClassifierDynamoDBTests.swift @@ -0,0 +1,247 @@ +// +// QueryClassifierDynamoDBTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +@Suite("QueryClassifier DynamoDB requests") +struct QueryClassifierDynamoDBTests { + private func tier(_ sql: String) -> QueryTier { + QueryClassifier.classifyTier(sql, databaseType: .dynamodb) + } + + @Test("Browse and every read action are safe", arguments: [ + #"Browse {"TableName": "orders", "Filters": [], "Match": "All", "Columns": ["pk"]}"#, + #"Browse {"TableName": "orders"} ORDER BY "createdAt" DESC LIMIT 100 OFFSET 200"#, + #"Scan {"TableName": "orders"}"#, + #"scan {"TableName": "orders"}"#, + #"Scan {"TableName": "orders"} ORDER BY "a" ASC, total DESC LIMIT 5 OFFSET 10;"#, + ##"Query {"TableName": "orders", "KeyConditionExpression": "#pk = :pk"} LIMIT 20"##, + #"GetItem {"TableName": "orders", "Key": {"pk": {"S": "a"}}}"#, + #"BatchGetItem {"RequestItems": {"orders": {"Keys": [{"pk": {"S": "a"}}]}}}"#, + #"TransactGetItems {"TransactItems": [{"Get": {"TableName": "orders", "Key": {"pk": {"S": "a"}}}}]}"#, + #"DescribeTable {"TableName": "orders"}"#, + #"ListTables {}"#, + #"DescribeTimeToLive {"TableName": "orders"}"#, + #"DescribeContinuousBackups {"TableName": "orders"}"#, + #"ListTagsOfResource {"ResourceArn": "arn:aws:dynamodb:us-east-1:1:table/orders"}"#, + #"DescribeLimits {}"# + ]) + func readsAreSafe(_ sql: String) { + #expect(tier(sql) == .safe) + } + + @Test("Item and table writes that take nothing away are writes", arguments: [ + #"PutItem {"TableName": "orders", "Item": {"pk": {"S": "a"}}}"#, + #"UpdateItem {"TableName": "orders", "Key": {"pk": {"S": "a"}}, "UpdateExpression": "SET n = :n"}"#, + #"DeleteItem {"TableName": "orders", "Key": {"pk": {"S": "a"}}}"#, + #"CreateTable {"TableName": "orders", "BillingMode": "PAY_PER_REQUEST"}"#, + #"UpdateTable {"TableName": "orders", "BillingMode": "PROVISIONED"}"#, + #"UpdateTable {"TableName": "orders", "DeletionProtectionEnabled": true}"#, + #"UpdateTable {"TableName": "orders", "GlobalSecondaryIndexUpdates": [{"Create": {"IndexName": "byDate"}}]}"#, + #"UpdateTimeToLive {"TableName": "orders", "TimeToLiveSpecification": {"Enabled": false, "AttributeName": "ttl"}}"#, + #"UpdateContinuousBackups {"TableName": "orders", "PointInTimeRecoverySpecification": {"PointInTimeRecoveryEnabled": true}}"#, + #"BatchWriteItem {"RequestItems": {"orders": [{"PutRequest": {"Item": {"DeleteRequest": {"S": "a"}}}}]}}"#, + #"TransactWriteItems {"TransactItems": [{"Put": {"TableName": "orders", "Item": {"Delete": {"S": "a"}}}}]}"#, + #"TagResource {"ResourceArn": "arn", "Tags": [{"Key": "team", "Value": "data"}]}"#, + #"UntagResource {"ResourceArn": "arn", "TagKeys": ["team"]}"# + ]) + func writesAreWrites(_ sql: String) { + #expect(tier(sql) == .write) + } + + @Test("A request that drops data or a safeguard is destructive", arguments: [ + #"DeleteTable {"TableName": "orders"}"#, + #"deletetable {"TableName": "orders"}"#, + #"DeleteTable {"TableName": "orders""#, + #"UpdateTable {"TableName": "orders", "GlobalSecondaryIndexUpdates": [{"Delete": {"IndexName": "byDate"}}]}"#, + #"UpdateTable {"TableName": "orders", "ReplicaUpdates": [{"Delete": {"RegionName": "eu-west-1"}}]}"#, + #"UpdateTable {"TableName": "orders", "DeletionProtectionEnabled": false}"#, + #"UpdateTimeToLive {"TableName": "orders", "TimeToLiveSpecification": {"Enabled": true, "AttributeName": "ttl"}}"#, + #"UpdateContinuousBackups {"TableName": "orders", "PointInTimeRecoverySpecification": {"PointInTimeRecoveryEnabled": false}}"#, + #"BatchWriteItem {"RequestItems": {"orders": [{"PutRequest": {"Item": {}}}, {"DeleteRequest": {"Key": {}}}]}}"#, + #"TransactWriteItems {"TransactItems": [{"Put": {"TableName": "a"}}, {"Delete": {"TableName": "b"}}]}"#, + "-- tidy up\nDeleteTable {\"TableName\": \"orders\"}" + ]) + func removalsAreDestructive(_ sql: String) { + #expect(tier(sql) == .destructive) + } + + @Test("A key spelled with a Unicode escape is still read") + func unicodeEscapedKeyIsRead() { + let sql = #"UpdateTable {"TableName": "t", "GlobalSecondaryIndexUpdates": [{"\u0044elete": {"IndexName": "i"}}]}"# + #expect(tier(sql) == .destructive) + } + + @Test("PartiQL a request carries is tiered by the SQL rules") + func carriedPartiQLIsTiered() { + #expect(tier(#"ExecuteTransaction {"TransactStatements": [{"Statement": "SELECT * FROM t WHERE pk = ?"}]}"#) == .safe) + #expect(tier(#"BatchExecuteStatement {"Statements": [{"Statement": "SELECT * FROM t"}, {"Statement": "SELECT * FROM u"}]}"#) == .safe) + #expect(tier(#"BatchExecuteStatement {"Statements": [{"Statement": "SELECT * FROM t"}, {"Statement": "UPDATE t SET a = 1 WHERE pk = 'x'"}]}"#) == .write) + #expect(tier(#"ExecuteTransaction {"TransactStatements": [{"Statement": "INSERT INTO t VALUE {'pk': 'a'}"}]}"#) == .write) + #expect(tier(#"ExecuteStatement {"Statement": "SELECT * FROM t"}"#) == .write) + } + + @Test("A carried entry whose statement the classifier cannot read is a write", arguments: [ + #"BatchExecuteStatement {"Statements": [{"Statement": "SELECT * FROM t"}, {"Parameters": []}]}"#, + #"ExecuteTransaction {"TransactStatements": []}"#, + #"ExecuteTransaction {"TransactStatements": "SELECT * FROM t"}"#, + #"ExecuteStatement {"Statement": 42}"# + ]) + func unreadableCarriedPartiQLIsAWrite(_ sql: String) { + #expect(tier(sql) == .write) + } + + @Test("An unreadable item write or a write after the body is a write", arguments: [ + #"PutItem {"TableName": }"#, + #"Scan {"TableName": "orders""#, + #"Scan {"TableName": "orders"} DELETE FROM orders"#, + "Scan {\"TableName\": \"orders\"}\nDeleteItem {\"TableName\": \"orders\"}", + #"Scan {"TableName": "orders"} LIMIT ten"#, + #"Scan {"TableName": "orders"} ORDER BY"# + ]) + func unreadableRequestsAreWrites(_ sql: String) { + #expect(tier(sql) == .write) + } + + @Test("An action the app does not know is destructive, whatever its body says", arguments: [ + #"Frobnicate {"TableName": "orders"}"#, + #"DeleteBackup {"BackupArn": "arn:aws:dynamodb:us-east-1:1:table/orders/backup/1"}"# + ]) + func unknownActionsAreDestructive(_ sql: String) { + #expect(tier(sql) == .destructive) + } + + @Test("The app knows every action the driver runs, and no other") + func actionListsMatchTheDriver() { + let classified = Set(DynamoDBRequestAction.allCases.map(\.rawValue)).subtracting(["Browse"]) + let executed = Set(DynamoDBOperation.allCases.map(\.rawValue)) + + #expect(classified == executed) + } + + @Test("A body the classifier cannot read is the worst its action can do", arguments: [ + "UpdateTable", "UpdateTimeToLive", "UpdateContinuousBackups", "BatchWriteItem", "TransactWriteItems", + "ExecuteStatement", "ExecuteTransaction", "BatchExecuteStatement", "DeleteTable" + ]) + func unreadableBodyTakesTheWorstTier(_ action: String) { + let tooDeep = String(repeating: "[", count: 200) + String(repeating: "]", count: 200) + let padded = #"\#(action) {"TableName": "orders", "Pad": \#(tooDeep)}"# + let broken = #"\#(action) {"TableName": }"# + + #expect(tier(padded) == .destructive) + #expect(tier(broken) == .destructive) + } + + @Test("The classifier reads every body as deep as the driver sends", arguments: [1, 63, 64, 65, 127, 128, 129, 200]) + func classifierReadsWhatTheDriverReads(_ depth: Int) { + let nested = String(repeating: "[", count: depth) + String(repeating: "]", count: depth) + let body = #"{"TableName": "orders", "Pad": \#(nested)}"# + + let driverReads = (try? DynamoDBJSON.parse(body)) != nil + let classifierReads = DynamoDBRequestJSON.parsePrefix(body[...]).map { $0.remainder.isEmpty } ?? false + + #expect(classifierReads == driverReads) + } + + @Test("Requests joined without a semicolon are each tiered") + func joinedRequestsAreEachTiered() { + #expect(tier("Scan {\"TableName\": \"a\"}\nQuery {\"TableName\": \"b\"} LIMIT 5") == .safe) + #expect(tier("Scan {\"TableName\": \"a\"}\nDeleteTable {\"TableName\": \"a\"}") == .destructive) + #expect(QueryClassifier.isDangerousQuery( + "Scan {\"TableName\": \"a\"}\nDELETE FROM a", databaseType: .dynamodb + )) + } + + @Test("A script is tiered by its worst statement") + func multiStatementTakesTheWorstTier() { + #expect(tier(#"Scan {"TableName": "a"}; Query {"TableName": "b"}; SELECT * FROM "c""#) == .safe) + #expect(tier(#"Scan {"TableName": "a"}; PutItem {"TableName": "a", "Item": {}}"#) == .write) + #expect(tier(#"Scan {"TableName": "a"}; DeleteTable {"TableName": "a"}"#) == .destructive) + #expect(tier(#"SELECT * FROM "a"; UpdateTimeToLive {"TableName": "a", "TimeToLiveSpecification": {"Enabled": true}}"#) + == .destructive) + } + + @Test("An escaped quote inside a request string does not end the request") + func escapedQuoteStaysInsideTheRequest() { + #expect(tier(#"PutItem {"TableName": "t", "Item": {"a": {"S": "x\";y"}}}"#) == .write) + } + + @Test("The plain reading of an escaped quote still reaches the gate") + func plainReadingOfAnEscapedQuoteIsCounted() { + #expect(tier(#"PutItem {"TableName": "t", "Item": {"a": {"S": "x\";DROP"}}}"#) == .destructive) + } + + @Test("PartiQL keeps the SQL rules") + func partiQLKeepsSQLRules() { + #expect(tier(#"SELECT * FROM "orders" WHERE pk = 'a'"#) == .safe) + #expect(tier(#"INSERT INTO "orders" VALUE {'pk': 'a'}"#) == .write) + #expect(tier(#"DELETE FROM "orders" WHERE pk = 'a'"#) == .write) + #expect(QueryClassifier.isDangerousQuery(#"DELETE FROM "orders""#, databaseType: .dynamodb)) + #expect(!QueryClassifier.isDangerousQuery(#"DELETE FROM "orders" WHERE pk = 'a'"#, databaseType: .dynamodb)) + } + + @Test("A request is dangerous when it is destructive or carries a DELETE with no WHERE") + func dangerousRequests() { + #expect(QueryClassifier.isDangerousQuery(#"DeleteTable {"TableName": "orders"}"#, databaseType: .dynamodb)) + #expect(QueryClassifier.isDangerousQuery( + #"ExecuteStatement {"Statement": "DELETE FROM orders"}"#, databaseType: .dynamodb + )) + #expect(QueryClassifier.isDangerousQuery( + #"BatchExecuteStatement {"Statements": [{"Statement": "SELECT * FROM a"}, {"Statement": "DELETE FROM b"}]}"#, + databaseType: .dynamodb + )) + #expect(!QueryClassifier.isDangerousQuery( + #"ExecuteTransaction {"TransactStatements": [{"Statement": "DELETE FROM b WHERE pk = 'a'"}]}"#, + databaseType: .dynamodb + )) + #expect(!QueryClassifier.isDangerousQuery( + #"DeleteItem {"TableName": "orders", "Key": {"pk": {"S": "a"}}}"#, databaseType: .dynamodb + )) + #expect(!QueryClassifier.isDangerousQuery(#"Scan {"TableName": "orders"}"#, databaseType: .dynamodb)) + } + + @Test("No DynamoDB request reaches the filesystem or runs code") + func requestsNeverReachTheFilesystem() { + let sql = #"ExecuteStatement {"Statement": "SELECT LOAD_FILE('/etc/passwd') FROM t"}"# + #expect(!QueryClassifier.reachesFilesystemOrExecutesCode(sql, databaseType: .dynamodb)) + #expect(!QueryClassifier.reachesFilesystemOrExecutesCode(#"DeleteTable {"TableName": "t"}"#, databaseType: .dynamodb)) + } + + @Test("Another engine's text is never read as a DynamoDB request") + func otherEnginesIgnoreTheRequestForm() { + #expect(QueryClassifier.classifyTier(#"Scan {"TableName": "orders"}"#, databaseType: .postgresql) == .write) + } +} + +@Suite("CatalogChangeClassifier DynamoDB requests") +struct CatalogChangeClassifierDynamoDBTests { + private func kinds(_ sql: String) -> CatalogObjectKinds { + CatalogChangeClassifier.effect(of: sql, databaseType: .dynamodb).kinds + } + + @Test("Creating, changing and dropping a table refresh the tables", arguments: [ + #"CreateTable {"TableName": "orders"}"#, + #"UpdateTable {"TableName": "orders", "BillingMode": "PAY_PER_REQUEST"}"#, + #"DeleteTable {"TableName": "orders"}"#, + #"deleteTable {"TableName": "orders"}"# + ]) + func tableActionsChangeTheCatalog(_ sql: String) { + #expect(kinds(sql) == .tables) + } + + @Test("Reads, item writes and PartiQL leave the catalog alone", arguments: [ + #"Browse {"TableName": "orders"} LIMIT 100"#, + #"Scan {"TableName": "orders"}"#, + #"PutItem {"TableName": "orders", "Item": {}}"#, + #"UpdateTimeToLive {"TableName": "orders", "TimeToLiveSpecification": {"Enabled": true}}"#, + #"SELECT * FROM "orders""#, + #"DELETE FROM "orders" WHERE pk = 'a'"# + ]) + func otherStatementsLeaveTheCatalogAlone(_ sql: String) { + #expect(kinds(sql).isEmpty) + } +} diff --git a/TableProTests/Models/Schema/CreateTableFormStateTests.swift b/TableProTests/Models/Schema/CreateTableFormStateTests.swift new file mode 100644 index 0000000000..05ba8eab7a --- /dev/null +++ b/TableProTests/Models/Schema/CreateTableFormStateTests.swift @@ -0,0 +1,461 @@ +// +// CreateTableFormStateTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +@testable import TablePro + +private enum FormFixture { + static let keyTypes: PluginFormField.Kind = .picker( + options: [PluginFormOption(value: "S", label: "String"), PluginFormOption(value: "N", label: "Number")], + defaultValue: "S" + ) + + static let spec = PluginCreateTableFormSpec( + sections: [ + PluginFormSection(id: "keys", title: "Primary Key", fields: [ + PluginFormField(id: "pk", label: "Partition key", kind: .text(placeholder: "pk", isRequired: true)), + PluginFormField(id: "pkType", label: "Type", kind: keyTypes), + PluginFormField(id: "sk", label: "Sort key", kind: .text(placeholder: nil, isRequired: false)), + PluginFormField( + id: "skType", label: "Type", kind: keyTypes, + visibleWhen: PluginFormCondition(fieldId: "sk", values: nil) + ) + ]), + PluginFormSection(id: "capacity", title: "Capacity", fields: [ + PluginFormField( + id: "billing", label: "Billing", + kind: .picker(options: [ + PluginFormOption(value: "PAY_PER_REQUEST", label: "On-demand"), + PluginFormOption(value: "PROVISIONED", label: "Provisioned") + ], defaultValue: "PAY_PER_REQUEST") + ), + PluginFormField( + id: "read", label: "Read capacity", + kind: .integer(defaultValue: 5, minimum: 1, maximum: 100), + visibleWhen: PluginFormCondition(fieldId: "billing", values: ["PROVISIONED"]) + ), + PluginFormField(id: "protect", label: "Deletion protection", kind: .toggle(defaultValue: false)) + ]), + PluginFormSection( + id: "indexes", title: "Indexes", + fields: [ + PluginFormField(id: "name", label: "Index name", kind: .text(placeholder: nil, isRequired: true)), + PluginFormField( + id: "projection", label: "Attributes", + kind: .picker(options: [ + PluginFormOption(value: "ALL", label: "All"), + PluginFormOption(value: "INCLUDE", label: "Chosen") + ], defaultValue: "ALL") + ), + PluginFormField( + id: "included", label: "Chosen attributes", + kind: .text(placeholder: nil, isRequired: true), + visibleWhen: PluginFormCondition(fieldId: "projection", values: ["INCLUDE"]) + ) + ], + isRepeating: true, addLabel: "Add Index", maximumCount: 2 + ) + ], + footnote: "Only key attributes are declared." + ) + + static func field(_ id: String, in sectionId: String) -> PluginFormField? { + spec.sections.first { $0.id == sectionId }?.fields.first { $0.id == id } + } +} + +@Suite("Create Table form state") +struct CreateTableFormStateTests { + private func index(_ entryId: UUID) -> CreateTableFormState.Location { + .entry(sectionId: "indexes", entryId: entryId) + } + + private func addIndex(to state: inout CreateTableFormState) throws -> UUID { + let added = state.addEntry(to: "indexes") + return try #require(added) + } + + @Test("Top-level values are seeded from each field's initial value") + func seedsTopLevelValues() { + let state = CreateTableFormState(spec: FormFixture.spec) + + #expect(state.values == [ + "pk": "", "pkType": "S", "sk": "", "skType": "S", + "billing": "PAY_PER_REQUEST", "read": "5", "protect": "false" + ]) + #expect(state.entries(in: "indexes").isEmpty) + #expect(!state.holdsWork) + } + + @Test("A new entry is seeded with the repeating section's initial values") + func addEntrySeedsInitialValues() throws { + var state = CreateTableFormState(spec: FormFixture.spec) + + let entryId = try addIndex(to: &state) + + #expect(state.entries(in: "indexes").count == 1) + #expect(state.value(of: "projection", at: index(entryId)) == "ALL") + #expect(state.value(of: "name", at: index(entryId)) == "") + #expect(state.value(of: "included", at: index(entryId)) == "") + } + + @Test("Adding stops at the section's maximum count") + func addEntryRespectsMaximumCount() { + var state = CreateTableFormState(spec: FormFixture.spec) + + let first = state.addEntry(to: "indexes") + let second = state.addEntry(to: "indexes") + let canAddThird = state.canAddEntry(to: "indexes") + let third = state.addEntry(to: "indexes") + + #expect(first != nil) + #expect(second != nil) + #expect(!canAddThird) + #expect(third == nil) + #expect(state.entries(in: "indexes").count == 2) + } + + @Test("A section that does not repeat takes no entries") + func nonRepeatingSectionTakesNoEntries() { + var state = CreateTableFormState(spec: FormFixture.spec) + + let keys = state.addEntry(to: "keys") + let missing = state.addEntry(to: "missing") + + #expect(!state.canAddEntry(to: "keys")) + #expect(keys == nil) + #expect(missing == nil) + } + + @Test("Removing an entry keeps the others and frees a slot") + func removeEntryKeepsTheRest() throws { + var state = CreateTableFormState(spec: FormFixture.spec) + let first = try addIndex(to: &state) + let second = try addIndex(to: &state) + state.setValue("by_second", of: "name", at: index(second)) + + state.removeEntry(first, from: "indexes") + + #expect(state.entries(in: "indexes").count == 1) + #expect(state.value(of: "name", at: index(second)) == "by_second") + #expect(state.canAddEntry(to: "indexes")) + } + + @Test("A condition with no values shows the field once the other field is not blank") + func conditionWithoutValuesFollowsNonEmpty() throws { + var state = CreateTableFormState(spec: FormFixture.spec) + let sortKeyType = try #require(FormFixture.field("skType", in: "keys")) + + #expect(!state.isVisible(sortKeyType, at: .topLevel)) + + state.setValue(" ", of: "sk", at: .topLevel) + #expect(!state.isVisible(sortKeyType, at: .topLevel)) + + state.setValue("created_at", of: "sk", at: .topLevel) + #expect(state.isVisible(sortKeyType, at: .topLevel)) + } + + @Test("A condition with values shows the field only for those values") + func conditionWithValuesMatchesThem() throws { + var state = CreateTableFormState(spec: FormFixture.spec) + let read = try #require(FormFixture.field("read", in: "capacity")) + + #expect(!state.isVisible(read, at: .topLevel)) + + state.setValue("PROVISIONED", of: "billing", at: .topLevel) + #expect(state.isVisible(read, at: .topLevel)) + } + + @Test("A field in a repeating section follows its own entry, not its siblings or the top level") + func repeatingConditionIsScopedToItsEntry() throws { + var state = CreateTableFormState(spec: FormFixture.spec) + let included = try #require(FormFixture.field("included", in: "indexes")) + let chosen = try addIndex(to: &state) + let all = try addIndex(to: &state) + + state.setValue("INCLUDE", of: "projection", at: index(chosen)) + state.setValue("INCLUDE", of: "projection", at: .topLevel) + + #expect(state.isVisible(included, at: index(chosen))) + #expect(!state.isVisible(included, at: index(all))) + } + + @Test("A field whose controlling field is hidden is hidden too") + func visibilityFollowsTheControllingField() throws { + let spec = PluginCreateTableFormSpec(sections: [ + PluginFormSection(id: "main", title: nil, fields: [ + PluginFormField(id: "mode", label: "Mode", kind: .text(placeholder: nil, isRequired: false)), + PluginFormField( + id: "detail", label: "Detail", kind: .text(placeholder: nil, isRequired: false), + visibleWhen: PluginFormCondition(fieldId: "mode", values: nil) + ), + PluginFormField( + id: "note", label: "Note", kind: .text(placeholder: nil, isRequired: true), + visibleWhen: PluginFormCondition(fieldId: "detail", values: ["x"]) + ) + ]) + ]) + var state = CreateTableFormState(spec: spec) + let note = try #require(spec.sections.first?.fields.last) + + state.setValue("x", of: "detail", at: .topLevel) + #expect(!state.isVisible(note, at: .topLevel)) + #expect(state.fieldIssues().isEmpty) + + state.setValue("on", of: "mode", at: .topLevel) + #expect(state.isVisible(note, at: .topLevel)) + } + + @Test("A blank table name is an issue") + func blankTableNameIsAnIssue() { + var state = CreateTableFormState(spec: FormFixture.spec) + state.setValue("id", of: "pk", at: .topLevel) + + let issues = state.issues(tableName: " ") + + #expect(issues.count == 1) + #expect(issues.first?.location == nil) + #expect(issues.first?.kind == .missing) + #expect(state.issues(tableName: "orders").isEmpty) + } + + @Test("A required text field that is empty is an issue until it is filled") + func requiredTopLevelFieldIsAnIssue() { + var state = CreateTableFormState(spec: FormFixture.spec) + + let issues = state.issues(tableName: "orders") + #expect(issues.count == 1) + #expect(issues.first?.fieldId == "pk") + #expect(issues.first?.location == .topLevel) + #expect(issues.first?.kind == .missing) + + state.setValue("id", of: "pk", at: .topLevel) + #expect(state.issues(tableName: "orders").isEmpty) + } + + @Test("A hidden required field is never an issue") + func hiddenRequiredFieldIsIgnored() throws { + var state = CreateTableFormState(spec: FormFixture.spec) + state.setValue("id", of: "pk", at: .topLevel) + let entryId = try addIndex(to: &state) + state.setValue("by_status", of: "name", at: index(entryId)) + + #expect(state.issues(tableName: "orders").isEmpty) + + state.setValue("INCLUDE", of: "projection", at: index(entryId)) + let issues = state.issues(tableName: "orders") + #expect(issues.count == 1) + #expect(issues.first?.fieldId == "included") + #expect(issues.first?.location == index(entryId)) + #expect(issues.first?.qualifiedMessage.contains("Indexes") == true) + } + + @Test("An integer field is checked only while it is visible, and against its bounds") + func integerValidationRespectsVisibilityAndBounds() { + var state = CreateTableFormState(spec: FormFixture.spec) + state.setValue("id", of: "pk", at: .topLevel) + state.setValue("many", of: "read", at: .topLevel) + + #expect(state.issues(tableName: "orders").isEmpty) + + state.setValue("PROVISIONED", of: "billing", at: .topLevel) + let cases: [(value: String, isValid: Bool)] = [ + ("many", false), ("0", false), ("101", false), ("1", true), ("100", true), (" 50 ", true), ("", true) + ] + for testCase in cases { + state.setValue(testCase.value, of: "read", at: .topLevel) + let issues = state.issues(tableName: "orders") + #expect(issues.isEmpty == testCase.isValid, "read = \(testCase.value)") + if !testCase.isValid { + #expect(issues.first?.kind == .invalid, "read = \(testCase.value)") + #expect(state.inlineMessage(for: "read", at: .topLevel) != nil, "read = \(testCase.value)") + } + } + } + + @Test("The request carries visible values only and trims the table name") + func requestOmitsHiddenFields() throws { + var state = CreateTableFormState(spec: FormFixture.spec) + state.setValue("id", of: "pk", at: .topLevel) + state.setValue("250", of: "read", at: .topLevel) + let all = try addIndex(to: &state) + state.setValue("by_status", of: "name", at: index(all)) + state.setValue("left over", of: "included", at: index(all)) + let chosen = try addIndex(to: &state) + state.setValue("by_date", of: "name", at: index(chosen)) + state.setValue("INCLUDE", of: "projection", at: index(chosen)) + state.setValue("total, status", of: "included", at: index(chosen)) + + let request = state.request(tableName: " orders ") + + #expect(request.tableName == "orders") + #expect(request.values == [ + "pk": "id", "pkType": "S", "sk": "", "billing": "PAY_PER_REQUEST", "protect": "false" + ]) + #expect(request.repeatedValues["indexes"] == [ + ["name": "by_status", "projection": "ALL"], + ["name": "by_date", "projection": "INCLUDE", "included": "total, status"] + ]) + } + + @Test("A repeating section with no entries is sent as an empty list") + func emptyRepeatingSectionIsAnEmptyList() { + let request = CreateTableFormState(spec: FormFixture.spec).request(tableName: "orders") + + #expect(request.repeatedValues["indexes"]?.isEmpty == true) + } + + @Test("A driver error sits beside its field only when the field id is unambiguous and visible") + func submissionErrorPlacement() throws { + var state = CreateTableFormState(spec: FormFixture.spec) + state.setValue("PROVISIONED", of: "billing", at: .topLevel) + + state.recordSubmissionError(PluginCreateTableFormError(message: "Too low", fieldId: "read")) + #expect(state.inlineSubmissionErrorFieldId == "read") + #expect(state.inlineMessage(for: "read", at: .topLevel) == "Too low") + + state.recordSubmissionError(PluginCreateTableFormError(message: "Bad index", fieldId: "name")) + #expect(state.inlineSubmissionErrorFieldId == nil) + + state.recordSubmissionError(PluginCreateTableFormError(message: "Bad name", fieldId: nil)) + #expect(state.inlineSubmissionErrorFieldId == nil) + + state.setValue("PAY_PER_REQUEST", of: "billing", at: .topLevel) + #expect(state.submissionError == nil) + } + + @Test("Any edit clears the driver's last error") + func editsClearTheSubmissionError() throws { + var state = CreateTableFormState(spec: FormFixture.spec) + let error = PluginCreateTableFormError(message: "Nope") + + state.recordSubmissionError(error) + state.setValue("id", of: "pk", at: .topLevel) + #expect(state.submissionError == nil) + + state.recordSubmissionError(error) + let entryId = try addIndex(to: &state) + #expect(state.submissionError == nil) + + state.recordSubmissionError(error) + state.removeEntry(entryId, from: "indexes") + #expect(state.submissionError == nil) + + state.recordSubmissionError(error) + state.setValue("id", of: "pk", at: .topLevel) + #expect(state.submissionError == error) + } + + @Test("The form holds work once a value leaves its default or an entry exists") + func holdsWorkTracksEdits() throws { + var state = CreateTableFormState(spec: FormFixture.spec) + + state.setValue("true", of: "protect", at: .topLevel) + #expect(state.holdsWork) + + state.setValue("false", of: "protect", at: .topLevel) + #expect(!state.holdsWork) + + let entryId = try addIndex(to: &state) + #expect(state.holdsWork) + + state.removeEntry(entryId, from: "indexes") + #expect(!state.holdsWork) + } + + @Test("The preview names the first issue before it asks the driver") + func previewPrefersLocalIssues() { + let state = CreateTableFormState(spec: FormFixture.spec) + var asked = false + + let preview = state.preview(tableName: "orders") { _ in + asked = true + return ["CreateTable {}"] + } + + #expect(!asked) + guard case .message = preview else { + Issue.record("Expected a message, got \(preview)") + return + } + } + + @Test("The preview shows the driver's statements, or the driver's own error") + func previewShowsStatementsOrDriverError() { + var state = CreateTableFormState(spec: FormFixture.spec) + state.setValue("id", of: "pk", at: .topLevel) + + let statements = state.preview(tableName: "orders") { request in + ["CreateTable \(request.tableName)", "Second;"] + } + #expect(statements == .statements("CreateTable orders;\n\nSecond;")) + + let refused = state.preview(tableName: "orders") { _ in + throw PluginCreateTableFormError(message: "Name the key", fieldId: "pk") + } + #expect(refused == .message("Name the key")) + + let empty = state.preview(tableName: "orders") { _ in [] } + guard case .message = empty else { + Issue.record("An empty statement list must not preview as statements") + return + } + } +} + +@MainActor +@Suite("Create Table draft form") +struct CreateTableDraftFormTests { + @Test("A draft resolves its form once and keeps it") + func resolvesFormOnce() { + let draft = CreateTableDraft() + #expect(!draft.hasResolvedForm) + + draft.resolveForm(from: FormFixture.spec) + #expect(draft.hasResolvedForm) + #expect(draft.form?.spec == FormFixture.spec) + + draft.form?.setValue("id", of: "pk", at: .topLevel) + draft.resolveForm(from: nil) + #expect(draft.form?.value(of: "pk", at: .topLevel) == "id") + } + + @Test("Renaming the table clears what the driver said about the last attempt") + func renameClearsSubmissionError() { + let draft = CreateTableDraft() + draft.resolveForm(from: FormFixture.spec) + draft.tableName = "a" + draft.form?.recordSubmissionError(PluginCreateTableFormError(message: "A table name is too short")) + + draft.tableName = "a" + #expect(draft.form?.submissionError != nil) + + draft.tableName = "orders" + #expect(draft.form?.submissionError == nil) + } + + @Test("A driver without a form leaves the draft on the column grid") + func noSpecMeansGrid() { + let draft = CreateTableDraft() + + draft.resolveForm(from: nil) + + #expect(draft.hasResolvedForm) + #expect(draft.form == nil) + } + + @Test("An edited form counts as work worth protecting") + func editedFormHoldsWork() { + let draft = CreateTableDraft() + draft.resolveForm(from: FormFixture.spec) + #expect(!draft.holdsWork) + + draft.form?.setValue("id", of: "pk", at: .topLevel) + #expect(draft.holdsWork) + } +} diff --git a/TableProTests/Plugins/DocumentStoreCaseSensitivityTests.swift b/TableProTests/Plugins/DocumentStoreCaseSensitivityTests.swift index bca8a14b34..3e56cd939d 100644 --- a/TableProTests/Plugins/DocumentStoreCaseSensitivityTests.swift +++ b/TableProTests/Plugins/DocumentStoreCaseSensitivityTests.swift @@ -217,33 +217,3 @@ struct BigQueryCaseSensitivityTests { #expect(!sql("CONTAINS", "ali", isCaseSensitive: true).contains("ESCAPE")) } } - -@Suite("DynamoDB Case Sensitivity") -struct DynamoDBCaseSensitivityTests { - @Test("A scan tag saved before this option existed keeps each operator's old behaviour") - func testLegacySpecDefaults() { - #expect(DynamoDBFilterSpec(column: "a", op: "CONTAINS", value: "x").ignoresCase) - #expect(DynamoDBFilterSpec(column: "a", op: "STARTS WITH", value: "x").ignoresCase) - #expect(DynamoDBFilterSpec(column: "a", op: "ENDS WITH", value: "x").ignoresCase) - #expect(DynamoDBFilterSpec(column: "a", op: "=", value: "x").ignoresCase == false) - #expect(DynamoDBFilterSpec(column: "a", op: "!=", value: "x").ignoresCase == false) - } - - @Test("An explicit setting wins over the operator default") - func testExplicitSettingWins() { - #expect(DynamoDBFilterSpec(column: "a", op: "CONTAINS", value: "x", caseSensitive: true).ignoresCase == false) - #expect(DynamoDBFilterSpec(column: "a", op: "=", value: "x", caseSensitive: false).ignoresCase) - } - - @Test("The setting survives the encode and decode round trip") - func testRoundTrip() throws { - let query = DynamoDBQueryBuilder().buildFilteredQuery( - table: "Users", - filters: [PluginQueryFilter(column: "name", op: "CONTAINS", value: "al", isCaseSensitive: true)], - logicMode: "AND", sortColumns: [], columns: [], limit: 10, offset: 0, - keySchema: [] - ) - let parsed = try #require(query.flatMap { DynamoDBQueryBuilder.parseScanQuery($0) }) - #expect(parsed.filters.first?.ignoresCase == false) - } -} diff --git a/TableProTests/Plugins/DynamoDB/DynamoDBAccessPlannerTests.swift b/TableProTests/Plugins/DynamoDB/DynamoDBAccessPlannerTests.swift new file mode 100644 index 0000000000..4d42473c7f --- /dev/null +++ b/TableProTests/Plugins/DynamoDB/DynamoDBAccessPlannerTests.swift @@ -0,0 +1,722 @@ +// +// DynamoDBAccessPlannerTests.swift +// TableProTests +// + +import Foundation +import Testing + +@Suite("DynamoDB access planning") +struct DynamoDBAccessPlannerTests { + struct Scenario: Sendable, CustomTestStringConvertible { + let name: String + let filters: [DynamoDBBrowseFilter] + let matchAll: Bool + let order: [DynamoDBOrderTerm] + let columns: [String] + + init( + _ name: String, + _ filters: [DynamoDBBrowseFilter], + matchAll: Bool = true, + order: [DynamoDBOrderTerm] = [], + columns: [String] = [] + ) { + self.name = name + self.filters = filters + self.matchAll = matchAll + self.order = order + self.columns = columns + } + + var testDescription: String { name } + } + + static let ordersDescription = """ + {"Table": { + "TableName": "orders", + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"} + ], + "AttributeDefinitions": [ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "N"}, + {"AttributeName": "status", "AttributeType": "S"}, + {"AttributeName": "total", "AttributeType": "N"}, + {"AttributeName": "zip", "AttributeType": "S"}, + {"AttributeName": "created", "AttributeType": "S"}, + {"AttributeName": "email", "AttributeType": "S"}, + {"AttributeName": "phone", "AttributeType": "S"} + ], + "GlobalSecondaryIndexes": [ + { + "IndexName": "byStatus", + "KeySchema": [ + {"AttributeName": "status", "KeyType": "HASH"}, + {"AttributeName": "total", "KeyType": "RANGE"} + ], + "Projection": {"ProjectionType": "ALL"}, + "IndexStatus": "ACTIVE" + }, + { + "IndexName": "byZip", + "KeySchema": [{"AttributeName": "zip", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "KEYS_ONLY"}, + "IndexStatus": "ACTIVE" + }, + { + "IndexName": "byEmail", + "KeySchema": [{"AttributeName": "email", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL"}, + "IndexStatus": "CREATING" + }, + { + "IndexName": "byPhone", + "KeySchema": [{"AttributeName": "phone", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL"}, + "IndexStatus": "ACTIVE", + "Backfilling": true + } + ], + "LocalSecondaryIndexes": [ + { + "IndexName": "byCreated", + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "created", "KeyType": "RANGE"} + ], + "Projection": {"ProjectionType": "KEYS_ONLY"} + } + ] + }} + """ + + static let sessionsDescription = """ + {"Table": { + "TableName": "sessions", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}] + }} + """ + + static let eventsDescription = """ + {"Table": { + "TableName": "events", + "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], + "AttributeDefinitions": [ + {"AttributeName": "id", "AttributeType": "S"}, + {"AttributeName": "tenant", "AttributeType": "S"}, + {"AttributeName": "region", "AttributeType": "S"}, + {"AttributeName": "year", "AttributeType": "N"}, + {"AttributeName": "month", "AttributeType": "N"} + ], + "GlobalSecondaryIndexes": [ + { + "IndexName": "byTenantRegion", + "KeySchema": [ + {"AttributeName": "tenant", "KeyType": "HASH"}, + {"AttributeName": "region", "KeyType": "HASH"}, + {"AttributeName": "year", "KeyType": "RANGE"}, + {"AttributeName": "month", "KeyType": "RANGE"} + ], + "Projection": {"ProjectionType": "ALL"}, + "IndexStatus": "ACTIVE" + } + ] + }} + """ + + static func schema(_ description: String) throws -> DynamoDBTableSchema { + try DynamoDBTableSchema(describeTableResponse: try DynamoDBJSON.parse(description)) + } + + static func filter( + _ attribute: String, + _ op: String, + _ value: String = "", + second: String? = nil, + caseSensitive: Bool = true + ) -> DynamoDBBrowseFilter { + DynamoDBBrowseFilter( + attribute: attribute, op: op, value: value, secondValue: second, kind: nil, caseSensitive: caseSensitive + ) + } + + static func plan( + _ filters: [DynamoDBBrowseFilter], + matchAll: Bool = true, + order: [DynamoDBOrderTerm] = [], + columns: [String] = [], + on description: String = ordersDescription + ) throws -> DynamoDBReadPlan { + let tableSchema = try Self.schema(description) + let request = DynamoDBBrowseRequest( + table: tableSchema.name, filters: filters, matchAll: matchAll, columns: columns + ) + return try DynamoDBAccessPlanner(schema: tableSchema).plan(request, order: order) + } + + static func text(_ key: String, in request: [String: DynamoDBJSON]) -> String? { + request[key]?.stringValue + } + + static func names(in request: [String: DynamoDBJSON]) -> [String: String] { + (request["ExpressionAttributeNames"]?.objectValue ?? [:]).compactMapValues(\.stringValue) + } + + static func values(in request: [String: DynamoDBJSON]) throws -> [String: DynamoDBAttributeValue] { + try (request["ExpressionAttributeValues"]?.objectValue ?? [:]).mapValues(DynamoDBAttributeValue.init(wireJSON:)) + } + + static func clientPredicate(_ filter: DynamoDBBrowseFilter) -> DynamoDBClientPredicate { + DynamoDBClientPredicate( + path: DynamoDBAttributePath(attribute: filter.attribute), + op: filter.op, + value: filter.value, + secondValue: filter.secondValue, + caseSensitive: filter.caseSensitive + ) + } + + // MARK: - Placeholder check + + static let expressionKeys = [ + "KeyConditionExpression", "FilterExpression", "ProjectionExpression", "ConditionExpression", "UpdateExpression" + ] + + static func placeholders(in text: String) -> Set { + guard let regex = try? NSRegularExpression(pattern: "[#:][A-Za-z0-9_]+") else { return [] } + let range = NSRange(text.startIndex..., in: text) + return Set(regex.matches(in: text, range: range).compactMap { match in + Range(match.range, in: text).map { String(text[$0]) } + }) + } + + static func placeholderProblems(in request: [String: DynamoDBJSON]) -> [String] { + let expressions = expressionKeys.compactMap { request[$0]?.stringValue }.joined(separator: " ") + let referenced = placeholders(in: expressions) + var declared: Set = [] + var problems: [String] = [] + for (key, prefix) in [("ExpressionAttributeNames", "#"), ("ExpressionAttributeValues", ":")] { + guard let entry = request[key] else { continue } + guard let object = entry.objectValue else { + problems.append("\(key) is not an object") + continue + } + if object.isEmpty { problems.append("\(key) is empty") } + for placeholder in object.keys.sorted() { + declared.insert(placeholder) + if !placeholder.hasPrefix(prefix) { problems.append("\(placeholder) is in \(key)") } + if !referenced.contains(placeholder) { problems.append("\(placeholder) is declared but unused") } + } + } + for placeholder in referenced.subtracting(declared).sorted() { + problems.append("\(placeholder) is used but not declared") + } + return problems + } + + static func placeholderProblems(in plan: DynamoDBReadPlan) -> [String] { + plan.requests.flatMap { placeholderProblems(in: $0) } + } + + @Test("The placeholder check flags unused, undeclared and empty placeholder maps") + func placeholderCheckCanFail() { + let request: [String: DynamoDBJSON] = [ + "FilterExpression": .string("#pk = :pk2 AND #sk > :sk"), + "ExpressionAttributeNames": .object(["#pk": .string("pk")]), + "ExpressionAttributeValues": .object([ + ":pk": .object(["S": .string("a")]), + ":pk2": .object(["S": .string("b")]), + ":sk": .object(["N": .string("1")]) + ]) + ] + let empty: [String: DynamoDBJSON] = ["ExpressionAttributeNames": .object([:])] + + #expect(Self.placeholderProblems(in: request) == [":pk is declared but unused", "#sk is used but not declared"]) + #expect(Self.placeholderProblems(in: empty) == ["ExpressionAttributeNames is empty"]) + } + + // MARK: - Query on the table + + @Test("Equality on the partition key queries the table") + func partitionEqualityQueriesTable() throws { + let plan = try Self.plan([Self.filter("pk", "=", "x")]) + + #expect(plan.access == .query) + #expect(plan.indexName == nil) + #expect(plan.requests.count == 1) + let request = try #require(plan.requests.first) + #expect(request["TableName"] == .string("orders")) + #expect(Self.text("KeyConditionExpression", in: request) == "#pk = :pk") + #expect(request["IndexName"] == nil) + #expect(request["FilterExpression"] == nil) + #expect(Self.names(in: request) == ["#pk": "pk"]) + #expect(try Self.values(in: request) == [":pk": .string("x")]) + #expect(plan.clientPredicates.isEmpty) + #expect(Self.placeholderProblems(in: plan).isEmpty) + } + + @Test("A range on the sort key joins the key condition") + func sortRangeJoinsKeyCondition() throws { + let plan = try Self.plan([Self.filter("pk", "=", "x"), Self.filter("sk", "BETWEEN", "1", second: "5")]) + + let request = try #require(plan.requests.first) + #expect(plan.access == .query) + #expect(Self.text("KeyConditionExpression", in: request) == "#pk = :pk AND #sk BETWEEN :sk AND :sk2") + #expect(request["FilterExpression"] == nil) + #expect(try Self.values(in: request) == [":pk": .string("x"), ":sk": .number("1"), ":sk2": .number("5")]) + #expect(Self.placeholderProblems(in: plan).isEmpty) + } + + @Test("A reversed range on the sort key reads nothing") + func reversedSortRangeReadsNothing() throws { + let plan = try Self.plan([Self.filter("pk", "=", "x"), Self.filter("sk", "BETWEEN", "9", second: "1")]) + + #expect(plan.access == .nothing) + #expect(plan.requests.isEmpty) + } + + @Test("Descending order on the sort key reads the index backwards") + func descendingSortKeyOrder() throws { + let order = [DynamoDBOrderTerm(attribute: "sk", descending: true)] + + let plan = try Self.plan([Self.filter("pk", "=", "x")], order: order) + + let request = try #require(plan.requests.first) + #expect(request["ScanIndexForward"] == .bool(false)) + #expect(plan.unsatisfiedOrder.isEmpty) + } + + @Test("Ascending order on the sort key is the natural order") + func ascendingSortKeyOrder() throws { + let order = [DynamoDBOrderTerm(attribute: "sk", descending: false)] + + let plan = try Self.plan([Self.filter("pk", "=", "x")], order: order) + + let request = try #require(plan.requests.first) + #expect(request["ScanIndexForward"] == nil) + #expect(plan.unsatisfiedOrder.isEmpty) + } + + @Test("Order on any other attribute is left to the reader") + func otherOrderIsUnsatisfied() throws { + let order = [DynamoDBOrderTerm(attribute: "customer", descending: true)] + + let plan = try Self.plan([Self.filter("pk", "=", "x")], order: order) + + let request = try #require(plan.requests.first) + #expect(request["ScanIndexForward"] == nil) + #expect(plan.unsatisfiedOrder == order) + } + + @Test("A filter on a non-key attribute becomes the FilterExpression of the query") + func nonKeyFilterOnQuery() throws { + let plan = try Self.plan([Self.filter("pk", "=", "x"), Self.filter("customer", "=", "ann")]) + + let request = try #require(plan.requests.first) + #expect(plan.access == .query) + #expect(plan.indexName == nil) + #expect(Self.text("KeyConditionExpression", in: request) == "#pk = :pk") + #expect(Self.text("FilterExpression", in: request) == "#customer = :customer") + #expect(Self.placeholderProblems(in: plan).isEmpty) + } + + @Test("A key attribute the key condition cannot use is checked on the client, never in a FilterExpression") + func unusableKeyFilterGoesToClient() throws { + let notEqual = Self.filter("sk", "!=", "3") + + let plan = try Self.plan([Self.filter("pk", "=", "x"), notEqual]) + + let request = try #require(plan.requests.first) + #expect(plan.access == .query) + #expect(Self.text("KeyConditionExpression", in: request) == "#pk = :pk") + #expect(request["FilterExpression"] == nil) + #expect(plan.clientPredicates == [Self.clientPredicate(notEqual)]) + #expect(plan.clientMatchAll) + #expect(!plan.clientMatches(["pk": .string("x"), "sk": .number("3")])) + #expect(plan.clientMatches(["pk": .string("x"), "sk": .number("4")])) + #expect(Self.placeholderProblems(in: plan).isEmpty) + } + + // MARK: - Indexes + + @Test("Equality on a global index partition key queries that index") + func queriesGlobalIndex() throws { + let plan = try Self.plan([Self.filter("status", "=", "shipped"), Self.filter("total", ">=", "100")]) + + let request = try #require(plan.requests.first) + #expect(plan.access == .query) + #expect(plan.indexName == "byStatus") + #expect(request["IndexName"] == .string("byStatus")) + #expect(request["Select"] == nil) + #expect(Self.text("KeyConditionExpression", in: request) == "#status = :status AND #total >= :total") + #expect(try Self.values(in: request) == [":status": .string("shipped"), ":total": .number("100")]) + #expect(Self.placeholderProblems(in: plan).isEmpty) + } + + @Test("A key attribute of the chosen index is checked on the client") + func indexKeyFilterGoesToClient() throws { + let notEqual = Self.filter("total", "!=", "5") + + let plan = try Self.plan([Self.filter("status", "=", "shipped"), notEqual]) + + let request = try #require(plan.requests.first) + #expect(plan.indexName == "byStatus") + #expect(request["FilterExpression"] == nil) + #expect(plan.clientPredicates == [Self.clientPredicate(notEqual)]) + #expect(Self.placeholderProblems(in: plan).isEmpty) + } + + @Test( + "A global index is not used while an item missing its sort key could match", + arguments: [ + [DynamoDBAccessPlannerTests.filter("status", "=", "shipped")], + [DynamoDBAccessPlannerTests.filter("status", "=", "shipped"), DynamoDBAccessPlannerTests.filter("total", "IS NULL")] + ] + ) + func sparseGlobalIndexIsSkipped(_ filters: [DynamoDBBrowseFilter]) throws { + let plan = try Self.plan(filters) + + let request = try #require(plan.requests.first) + #expect(plan.access == .scan) + #expect(plan.indexName == nil) + #expect(request["IndexName"] == nil) + #expect(Self.placeholderProblems(in: plan).isEmpty) + } + + @Test("A local index is not used while an item missing its sort key could match") + func sparseLocalIndexIsSkipped() throws { + let plan = try Self.plan([Self.filter("pk", "=", "x"), Self.filter("created", "IS NULL")]) + + #expect(plan.access == .query) + #expect(plan.indexName == nil) + #expect(plan.requests.first?["IndexName"] == nil) + } + + @Test("A global index that does not project every attribute is not used") + func keysOnlyGlobalIndexIsSkipped() throws { + let plan = try Self.plan([Self.filter("zip", "=", "10001")]) + + let request = try #require(plan.requests.first) + #expect(plan.access == .scan) + #expect(plan.indexName == nil) + #expect(request["IndexName"] == nil) + #expect(request["KeyConditionExpression"] == nil) + #expect(Self.text("FilterExpression", in: request) == "#zip = :zip") + #expect(Self.placeholderProblems(in: plan).isEmpty) + } + + @Test("A local index with a sort filter is preferred and reads every attribute") + func localIndexWithSortFilter() throws { + let plan = try Self.plan([Self.filter("pk", "=", "x"), Self.filter("created", ">=", "2024")]) + + let request = try #require(plan.requests.first) + #expect(plan.access == .query) + #expect(plan.indexName == "byCreated") + #expect(request["IndexName"] == .string("byCreated")) + #expect(request["Select"] == .string("ALL_ATTRIBUTES")) + #expect(Self.text("KeyConditionExpression", in: request) == "#pk = :pk AND #created >= :created") + #expect(Self.placeholderProblems(in: plan).isEmpty) + } + + @Test("The table is preferred over a local index when no sort filter uses the index") + func tablePreferredOverLocalIndex() throws { + let plan = try Self.plan([Self.filter("pk", "=", "x")]) + + #expect(plan.indexName == nil) + #expect(plan.requests.first?["IndexName"] == nil) + } + + @Test("An index that is being built or backfilled is never chosen", arguments: [ + DynamoDBAccessPlannerTests.filter("email", "=", "a@example.com"), + DynamoDBAccessPlannerTests.filter("phone", "=", "555") + ]) + func unavailableIndexIsSkipped(_ filter: DynamoDBBrowseFilter) throws { + let plan = try Self.plan([filter]) + + #expect(plan.access == .scan) + #expect(plan.indexName == nil) + #expect(plan.requests.allSatisfy { $0["IndexName"] == nil }) + } + + // MARK: - Multi-attribute index keys + + @Test("A multi-attribute index is queried with every key attribute in the key condition") + func multiAttributeIndexFullKey() throws { + let plan = try Self.plan([ + Self.filter("tenant", "=", "t1"), + Self.filter("region", "=", "eu"), + Self.filter("year", "=", "2024"), + Self.filter("month", ">", "3") + ], on: Self.eventsDescription) + + let request = try #require(plan.requests.first) + #expect(plan.access == .query) + #expect(plan.indexName == "byTenantRegion") + #expect( + Self.text("KeyConditionExpression", in: request) + == "#tenant = :tenant AND #region = :region AND #year = :year AND #month > :month" + ) + #expect(try Self.values(in: request) == [ + ":tenant": .string("t1"), ":region": .string("eu"), ":year": .number("2024"), ":month": .number("3") + ]) + #expect(plan.clientPredicates.isEmpty) + #expect(Self.placeholderProblems(in: plan).isEmpty) + } + + @Test("A multi-attribute index needs equality on every partition attribute", arguments: [ + [DynamoDBAccessPlannerTests.filter("tenant", "=", "t1")], + [DynamoDBAccessPlannerTests.filter("tenant", "=", "t1"), DynamoDBAccessPlannerTests.filter("region", ">", "a")], + [DynamoDBAccessPlannerTests.filter("tenant", "IN", "t1, t2"), DynamoDBAccessPlannerTests.filter("region", "=", "eu")] + ]) + func multiAttributeIndexNeedsWholePartition(_ filters: [DynamoDBBrowseFilter]) throws { + let plan = try Self.plan(filters, on: Self.eventsDescription) + + #expect(plan.access == .scan) + #expect(plan.indexName == nil) + #expect(Self.placeholderProblems(in: plan).isEmpty) + } + + @Test("A range on the second sort attribute needs equality on the first") + func multiAttributeSortNeedsLeadingEquality() throws { + let hasYear = Self.filter("year", "IS NOT NULL") + let month = Self.filter("month", ">", "3") + + let plan = try Self.plan([ + Self.filter("tenant", "=", "t1"), + Self.filter("region", "=", "eu"), + hasYear, + month + ], on: Self.eventsDescription) + + let request = try #require(plan.requests.first) + #expect(plan.indexName == "byTenantRegion") + #expect(Self.text("KeyConditionExpression", in: request) == "#tenant = :tenant AND #region = :region") + #expect(request["FilterExpression"] == nil) + #expect(plan.clientPredicates == [Self.clientPredicate(hasYear), Self.clientPredicate(month)]) + #expect(Self.placeholderProblems(in: plan).isEmpty) + } + + @Test("A multi-attribute index is not used while an item missing one of its sort attributes could match") + func multiAttributeIndexNeedsEverySortAttribute() throws { + let plan = try Self.plan([ + Self.filter("tenant", "=", "t1"), + Self.filter("region", "=", "eu"), + Self.filter("month", ">", "3") + ], on: Self.eventsDescription) + + #expect(plan.access == .scan) + #expect(plan.indexName == nil) + #expect(Self.placeholderProblems(in: plan).isEmpty) + } + + @Test("A range on the first sort attribute keeps the second out of the key condition") + func multiAttributeRangeOnFirstSortAttribute() throws { + let plan = try Self.plan([ + Self.filter("tenant", "=", "t1"), + Self.filter("region", "=", "eu"), + Self.filter("year", ">", "2020"), + Self.filter("month", "=", "3") + ], on: Self.eventsDescription) + + let request = try #require(plan.requests.first) + let keyCondition = try #require(Self.text("KeyConditionExpression", in: request)) + #expect(plan.indexName == "byTenantRegion") + #expect(keyCondition.hasPrefix("#tenant = :tenant AND #region = :region")) + #expect(!keyCondition.contains("#month")) + #expect(request["FilterExpression"] == nil) + #expect(plan.clientPredicates.contains(Self.clientPredicate(Self.filter("month", "=", "3")))) + #expect(Self.placeholderProblems(in: plan).isEmpty) + } + + // MARK: - Partition lists + + @Test("An OR of partition keys on a table without a sort key reads the keys in one batch, in order") + func orOfKeysWithoutSortKeyIsBatchGet() throws { + let plan = try Self.plan( + [Self.filter("pk", "=", "b"), Self.filter("pk", "=", "a"), Self.filter("pk", "=", "b")], + matchAll: false, + on: Self.sessionsDescription + ) + + #expect(plan.access == .batchGet) + #expect(plan.requests == [[ + "RequestItems": .object(["sessions": .object([ + "Keys": .array([ + .object(["pk": .object(["S": .string("b")])]), + .object(["pk": .object(["S": .string("a")])]) + ]), + "ConsistentRead": .bool(true) + ])]) + ]]) + } + + @Test("A batch of more than 100 keys is split into requests of at most 100") + func batchGetChunks() throws { + let filters = (0..<150).map { Self.filter("pk", "=", "k\($0)") } + + let plan = try Self.plan(filters, matchAll: false, on: Self.sessionsDescription) + + let counts = plan.requests.map { request in + request["RequestItems"]?["sessions"]?["Keys"]?.arrayValue?.count ?? 0 + } + #expect(plan.access == .batchGet) + #expect(counts == [100, 50]) + } + + @Test("An OR of partition keys on a table with a sort key queries each partition") + func orOfKeysWithSortKeyIsQueries() throws { + let plan = try Self.plan([Self.filter("pk", "=", "a"), Self.filter("pk", "=", "b")], matchAll: false) + + #expect(plan.access == .query) + #expect(plan.indexName == nil) + #expect(plan.requests.map { Self.text("KeyConditionExpression", in: $0) } == ["#pk = :pk", "#pk = :pk"]) + #expect(try plan.requests.map { try Self.values(in: $0) } == [[":pk": .string("a")], [":pk": .string("b")]]) + #expect(plan.requests.allSatisfy { $0["ConsistentRead"] == .bool(true) }) + #expect(Self.placeholderProblems(in: plan).isEmpty) + } + + @Test("A page position belongs to the request it was read from, not only to the filters") + func positionKeyNamesTheAccessPath() throws { + let scan = try Self.plan([Self.filter("customer", "=", "ann")]) + let index = try Self.plan([Self.filter("status", "=", "shipped"), Self.filter("total", ">", "1")]) + let table = try Self.plan([Self.filter("pk", "=", "a")]) + + #expect(Set([scan.positionKey, index.positionKey, table.positionKey]).count == 3) + } + + @Test("In on the partition key with another filter queries each partition with that filter") + func partitionInWithFilter() throws { + let plan = try Self.plan([Self.filter("pk", "IN", "a, b"), Self.filter("customer", "=", "ann")]) + + #expect(plan.access == .query) + #expect(plan.requests.count == 2) + #expect(plan.requests.map { Self.text("KeyConditionExpression", in: $0) } == ["#pk = :pk", "#pk = :pk"]) + #expect(plan.requests.map { Self.text("FilterExpression", in: $0) } == ["#customer = :customer", "#customer = :customer"]) + #expect(try plan.requests.map { try Self.values(in: $0) } == [ + [":pk": .string("a"), ":customer": .string("ann")], + [":pk": .string("b"), ":customer": .string("ann")] + ]) + #expect(Self.placeholderProblems(in: plan).isEmpty) + } + + @Test("In on the partition key queries each distinct partition once") + func partitionInDeduplicates() throws { + let plan = try Self.plan([Self.filter("pk", "IN", "a, a, b")]) + + #expect(plan.access == .query) + #expect(try plan.requests.map { try Self.values(in: $0) } == [[":pk": .string("a")], [":pk": .string("b")]]) + } + + @Test("Order on the sort key across several partitions is left to the reader") + func partitionInOrderIsUnsatisfied() throws { + let order = [DynamoDBOrderTerm(attribute: "sk", descending: true)] + + let plan = try Self.plan([Self.filter("pk", "IN", "a, b")], order: order) + + #expect(plan.requests.count == 2) + #expect(plan.unsatisfiedOrder == order) + } + + // MARK: - Match any + + @Test("An OR with a non-key filter scans with an OR FilterExpression") + func orWithNonKeyFilterScans() throws { + let plan = try Self.plan([Self.filter("pk", "=", "a"), Self.filter("customer", "=", "ann")], matchAll: false) + + let request = try #require(plan.requests.first) + #expect(plan.access == .scan) + #expect(Self.text("FilterExpression", in: request) == "#pk = :pk OR #customer = :customer") + #expect(plan.clientPredicates.isEmpty) + #expect(Self.placeholderProblems(in: plan).isEmpty) + } + + @Test("An OR where one filter needs the client checks every filter on the client") + func orWithClientFilter() throws { + let key = Self.filter("pk", "=", "a") + let suffix = Self.filter("customer", "ENDS WITH", "nn") + + let plan = try Self.plan([key, suffix], matchAll: false) + + #expect(plan.access == .scan) + #expect(plan.requests == [["TableName": .string("orders")]]) + #expect(plan.clientPredicates == [Self.clientPredicate(key), Self.clientPredicate(suffix)]) + #expect(!plan.clientMatchAll) + #expect(plan.clientMatches(["pk": .string("a"), "customer": .string("bob")])) + #expect(plan.clientMatches(["pk": .string("b"), "customer": .string("ann")])) + #expect(!plan.clientMatches(["pk": .string("b"), "customer": .string("bob")])) + } + + @Test("An OR term that can match nothing leaves no placeholder behind") + func orDropsImpossibleTermCleanly() throws { + let plan = try Self.plan([Self.filter("customer", "=", "ann"), Self.filter("sk", "=", "abc")], matchAll: false) + + let request = try #require(plan.requests.first) + #expect(plan.access == .scan) + #expect(Self.text("FilterExpression", in: request) == "#customer = :customer") + #expect(Self.placeholderProblems(in: plan).isEmpty) + } + + // MARK: - Impossible and refused filters + + @Test("A Number key compared with text that is not a number reads nothing", arguments: [ + DynamoDBAccessPlannerTests.filter("sk", "=", "abc"), + DynamoDBAccessPlannerTests.filter("total", "=", "abc") + ]) + func numberKeyWithTextReadsNothing(_ filter: DynamoDBBrowseFilter) throws { + let plan = try Self.plan([filter]) + + #expect(plan.access == .nothing) + #expect(plan.requests.isEmpty) + } + + @Test("A raw filter is refused") + func rawFilterThrows() throws { + let raw = Self.filter(DynamoDBFilterTranslator.rawFilterColumn, "=", "a = 1") + let reason = try #require(DynamoDBFilterTranslator.unsupportedReason(for: raw)) + + #expect(throws: DynamoDBError.invalidStatement(reason)) { + try Self.plan([Self.filter("pk", "=", "x"), raw]) + } + } + + // MARK: - Placeholders + + static let scenarios: [Scenario] = [ + Scenario("table query with mixed filters", [ + filter("pk", "=", "x"), filter("sk", ">", "3"), filter("customer", "CONTAINS", "a"), + filter("total", "IS NULL"), filter("note", "IS NOT EMPTY") + ]), + Scenario("index query with filters", [ + filter("status", "=", "s"), filter("customer", "NOT IN", "a, b"), filter("flag", "=", "true") + ]), + Scenario("local index query with filters", [ + filter("pk", "=", "x"), filter("created", "BETWEEN", "a", second: "b"), filter("customer", "!=", "z") + ]), + Scenario("scan with typed readings", [ + filter("customer", "=", "5"), filter("flag", "=", "true"), filter("note", "BETWEEN", "1,9") + ]), + Scenario("per-partition queries", [filter("pk", "IN", "a, b"), filter("customer", "BETWEEN", "1", second: "9")]), + Scenario("match any scan", [ + filter("customer", "=", "5"), filter("note", "STARTS WITH", "x"), filter("total", ">", "3") + ], matchAll: false), + Scenario("nested paths", [filter("address.city", "=", "Hanoi"), filter("items[0].sku", "IN", "A1, B2")]), + Scenario( + "names that need placeholders", + [filter("order-id", "=", "1"), filter("a.b", "=", "2"), filter("name", "IS NOT NULL")], + columns: ["a.b"] + ), + Scenario("client filter beside a server filter", [filter("customer", "=", "ann"), filter("note", "ENDS WITH", "z")]), + Scenario("descending sort key", [filter("pk", "=", "x")], order: [DynamoDBOrderTerm(attribute: "sk", descending: true)]) + ] + + @Test("No request declares a placeholder its expressions do not use", arguments: scenarios) + func everyPlaceholderIsUsed(_ scenario: Scenario) throws { + let plan = try Self.plan(scenario.filters, matchAll: scenario.matchAll, order: scenario.order, columns: scenario.columns) + + #expect(plan.access != .nothing) + #expect(Self.placeholderProblems(in: plan).isEmpty) + } +} diff --git a/TableProTests/Plugins/DynamoDB/DynamoDBAttributeValueTests.swift b/TableProTests/Plugins/DynamoDB/DynamoDBAttributeValueTests.swift new file mode 100644 index 0000000000..635ab68536 --- /dev/null +++ b/TableProTests/Plugins/DynamoDB/DynamoDBAttributeValueTests.swift @@ -0,0 +1,189 @@ +import Foundation +import TableProPluginKit +import Testing + +@Suite("DynamoDB attribute value") +struct DynamoDBAttributeValueTests { + struct ValueCase: Sendable, CustomTestStringConvertible { + let name: String + let value: DynamoDBAttributeValue + let wire: String + var testDescription: String { name } + } + + struct PayloadCase: Sendable, CustomTestStringConvertible { + let name: String + let wire: String + var testDescription: String { name } + } + + static let valueCases: [ValueCase] = [ + ValueCase(name: "String", value: .string("02134"), wire: #"{"S":"02134"}"#), + ValueCase( + name: "Number", + value: .number("12345678901234567890123456789012345678"), + wire: #"{"N":"12345678901234567890123456789012345678"}"# + ), + ValueCase(name: "Number with scale", value: .number("1.50"), wire: #"{"N":"1.50"}"#), + ValueCase(name: "Binary", value: .binary(Data([1, 2, 3])), wire: #"{"B":"AQID"}"#), + ValueCase(name: "Boolean", value: .bool(false), wire: #"{"BOOL":false}"#), + ValueCase(name: "Null", value: .null, wire: #"{"NULL":true}"#), + ValueCase( + name: "List", + value: .list([.string("x"), .number("1"), .list([]), .null]), + wire: #"{"L":[{"S":"x"},{"N":"1"},{"L":[]},{"NULL":true}]}"# + ), + ValueCase( + name: "Map", + value: .map(["b": .bool(true), "a": .map(["n": .number("-0.5")])]), + wire: #"{"M":{"a":{"M":{"n":{"N":"-0.5"}}},"b":{"BOOL":true}}}"# + ), + ValueCase(name: "String Set", value: .stringSet(["b", "a"]), wire: #"{"SS":["b","a"]}"#), + ValueCase(name: "Number Set", value: .numberSet(["10", "1.5"]), wire: #"{"NS":["10","1.5"]}"#), + ValueCase(name: "Binary Set", value: .binarySet([Data([2]), Data([1])]), wire: #"{"BS":["Ag==","AQ=="]}"#) + ] + + @Test("Every type round-trips through its wire form", arguments: valueCases) + func wireRoundTrip(valueCase: ValueCase) throws { + #expect(try DynamoDBAttributeValue(wireJSON: valueCase.value.wireJSON) == valueCase.value) + } + + @Test("Every type writes the wire JSON DynamoDB expects", arguments: valueCases) + func wireText(valueCase: ValueCase) { + #expect(valueCase.value.wireJSON.serialized() == valueCase.wire) + } + + @Test("Every type decodes from the wire JSON DynamoDB sends", arguments: valueCases) + func decodesWireText(valueCase: ValueCase) throws { + #expect(try DynamoDBAttributeValue(wireJSON: DynamoDBJSON.parse(valueCase.wire)) == valueCase.value) + } + + @Test("A value reports its own type", arguments: valueCases) + func reportsType(valueCase: ValueCase) throws { + let json = try DynamoDBJSON.parse(valueCase.wire) + let tag = try #require(json.objectValue?.keys.first) + #expect(valueCase.value.type.rawValue == tag) + } + + @Test("A Number keeps its exact text on the wire, never passing through a Double") + func numberStaysText() throws { + let value = try DynamoDBAttributeValue(wireJSON: DynamoDBJSON.parse(#"{"N":"0.1000000000000000000000000000000000001"}"#)) + #expect(value == .number("0.1000000000000000000000000000000000001")) + #expect(value.wireJSON == .object(["N": .string("0.1000000000000000000000000000000000001")])) + } + + @Test( + "A payload of the wrong shape is rejected", + arguments: [ + PayloadCase(name: "String holding a number", wire: #"{"S":5}"#), + PayloadCase(name: "Number holding a bare number", wire: #"{"N":5}"#), + PayloadCase(name: "Binary that is not base64", wire: #"{"B":"not base64!"}"#), + PayloadCase(name: "Boolean holding a string", wire: #"{"BOOL":"true"}"#), + PayloadCase(name: "Null holding a string", wire: #"{"NULL":"x"}"#), + PayloadCase(name: "List holding an object", wire: #"{"L":{}}"#), + PayloadCase(name: "List element without a type", wire: #"{"L":["x"]}"#), + PayloadCase(name: "Map holding an array", wire: #"{"M":[]}"#), + PayloadCase(name: "Map entry without a type", wire: #"{"M":{"a":"x"}}"#), + PayloadCase(name: "String Set member that is a number", wire: #"{"SS":[1]}"#), + PayloadCase(name: "Number Set member that is a bare number", wire: #"{"NS":[1]}"#), + PayloadCase(name: "Binary Set member that is not base64", wire: #"{"BS":["!!"]}"#), + PayloadCase(name: "String Set that is not an array", wire: #"{"SS":"a"}"#) + ] + ) + func rejectsWrongShape(payload: PayloadCase) throws { + let json = try DynamoDBJSON.parse(payload.wire) + #expect(throws: DynamoDBError.self) { try DynamoDBAttributeValue(wireJSON: json) } + } + + @Test( + "A value that does not name exactly one known type is rejected", + arguments: [ + PayloadCase(name: "unknown tag", wire: #"{"X":"a"}"#), + PayloadCase(name: "lowercase tag", wire: #"{"s":"a"}"#), + PayloadCase(name: "two tags", wire: #"{"S":"a","N":"1"}"#), + PayloadCase(name: "no tag", wire: "{}"), + PayloadCase(name: "bare string", wire: #""a""#), + PayloadCase(name: "array", wire: #"[{"S":"a"}]"#), + PayloadCase(name: "null", wire: "null") + ] + ) + func rejectsUnknownOrAmbiguousTag(payload: PayloadCase) throws { + let json = try DynamoDBJSON.parse(payload.wire) + #expect(throws: DynamoDBError.self) { try DynamoDBAttributeValue(wireJSON: json) } + } + + @Test("The error for an unknown tag names the tag") + func unknownTagIsNamed() throws { + let json = try DynamoDBJSON.parse(#"{"XYZ":"a"}"#) + let error = #expect(throws: DynamoDBError.self) { try DynamoDBAttributeValue(wireJSON: json) } + #expect(error?.localizedDescription.contains("XYZ") == true) + } + + @Test("An item round-trips through its wire form") + func itemRoundTrip() throws { + let item: DynamoDBItem = [ + "pk": .string("user#1"), + "sk": .number("1700000000"), + "balance": .number("12345678901234567890123456789012345678"), + "avatar": .binary(Data([0xFF, 0x00])), + "active": .bool(true), + "nickname": .null, + "tags": .stringSet(["a", "b"]), + "scores": .numberSet(["1", "2.5"]), + "keys": .binarySet([Data([1])]), + "history": .list([.map(["at": .number("1"), "what": .string("login")])]), + "profile": .map(["address": .map(["zip": .string("02134")])]) + ] + #expect(try DynamoDBItem(wireItem: item.wireJSON) == item) + #expect(try DynamoDBItem(wireItem: DynamoDBJSON.parse(item.wireJSON.serialized())) == item) + } + + @Test("An item decodes from a GetItem response body") + func itemFromResponse() throws { + let response = try DynamoDBJSON.parse(#""" + {"Item":{"pk":{"S":"a"},"n":{"N":"1.50"},"doc":{"M":{"inner":{"L":[{"BOOL":true},{"NULL":true}]}}},"blob":{"B":"AQID"}}} + """#) + let item = try DynamoDBItem(wireItem: try #require(response["Item"])) + #expect(item == [ + "pk": .string("a"), + "n": .number("1.50"), + "doc": .map(["inner": .list([.bool(true), .null])]), + "blob": .binary(Data([1, 2, 3])) + ]) + } + + @Test("An empty item is an empty object on the wire") + func emptyItem() throws { + let item: DynamoDBItem = [:] + #expect(item.wireJSON == .object([:])) + #expect(try DynamoDBItem(wireItem: .object([:])).isEmpty) + } + + @Test("An item that is not an object is rejected", arguments: [#"[]"#, #""item""#, "null", "1"]) + func itemMustBeObject(text: String) throws { + let json = try DynamoDBJSON.parse(text) + #expect(throws: DynamoDBError.self) { try DynamoDBItem(wireItem: json) } + } + + @Test("An item with one malformed attribute is rejected as a whole") + func itemWithBadAttributeIsRejected() throws { + let json = try DynamoDBJSON.parse(#"{"pk":{"S":"a"},"bad":{"N":1}}"#) + #expect(throws: DynamoDBError.self) { try DynamoDBItem(wireItem: json) } + } + + @Test("A type is found by its display name or its tag, in any case") + func typeFromDisplayName() { + #expect(DynamoDBAttributeType(displayName: "Number Set") == .numberSet) + #expect(DynamoDBAttributeType(displayName: "number set") == .numberSet) + #expect(DynamoDBAttributeType(displayName: "ns") == .numberSet) + #expect(DynamoDBAttributeType(displayName: "BOOL") == .boolean) + #expect(DynamoDBAttributeType(displayName: "Boolean") == .boolean) + #expect(DynamoDBAttributeType(displayName: "Float") == nil) + } + + @Test("Only String, Number and Binary can be key types") + func keyTypes() { + let keyTypes = DynamoDBAttributeType.allCases.filter(\.isKeyType) + #expect(keyTypes == [.string, .number, .binary]) + } +} diff --git a/TableProTests/Plugins/DynamoDB/DynamoDBCatalogTests.swift b/TableProTests/Plugins/DynamoDB/DynamoDBCatalogTests.swift new file mode 100644 index 0000000000..ba88911e83 --- /dev/null +++ b/TableProTests/Plugins/DynamoDB/DynamoDBCatalogTests.swift @@ -0,0 +1,293 @@ +import Foundation +import TableProPluginKit +import Testing + +@Suite("DynamoDB catalog") +struct DynamoDBCatalogTests { + private static let fetchedAt = Date(timeIntervalSince1970: 1_700_000_000) + private static let local = DynamoDBCatalog.Scope(endpoint: "http://localhost:8000", region: "us-east-1", identity: "local") + private static let remote = DynamoDBCatalog.Scope( + endpoint: "https://dynamodb.eu-west-1.amazonaws.com/", region: "eu-west-1", identity: "key:AKIDEXAMPLE" + ) + + private static func schema(_ name: String) throws -> DynamoDBTableSchema { + let json = try DynamoDBJSON.parse( + #"{"Table":{"TableName":"\#(name)","KeySchema":[{"AttributeName":"pk","KeyType":"HASH"}],"AttributeDefinitions":[{"AttributeName":"pk","AttributeType":"S"}]}}"# + ) + return try DynamoDBTableSchema(describeTableResponse: json) + } + + private static func point(_ requestIndex: Int, key: String? = nil, skip: Int = 0) -> DynamoDBResumePoint { + DynamoDBResumePoint(requestIndex: requestIndex, startKey: key.map { ["pk": .string($0)] }, skip: skip) + } + + private static func nearest( + _ catalog: DynamoDBCatalog, + table: String = "Orders", + fingerprint: String = "scan", + offset: Int, + scope: DynamoDBCatalog.Scope = local + ) -> DynamoDBResumePoint? { + catalog.nearestResumePoint(table: table, fingerprint: fingerprint, atOrBefore: offset, in: scope)?.point + } + + @Test("A stored schema is served until it is a minute old") + func schemaLifetime() throws { + let catalog = DynamoDBCatalog() + let orders = try Self.schema("Orders") + catalog.store(orders, in: Self.local, now: Self.fetchedAt) + + #expect(catalog.schema(for: "Orders", in: Self.local, now: Self.fetchedAt) == orders) + #expect(catalog.schema(for: "Orders", in: Self.local, now: Self.fetchedAt.addingTimeInterval(59.9)) == orders) + #expect(catalog.schema(for: "Orders", in: Self.local, now: Self.fetchedAt.addingTimeInterval(60)) == nil) + #expect(catalog.schema(for: "Orders", in: Self.local, now: Self.fetchedAt.addingTimeInterval(3_600)) == nil) + } + + @Test("Storing a schema again restarts its lifetime") + func restoringRestartsLifetime() throws { + let catalog = DynamoDBCatalog() + let orders = try Self.schema("Orders") + catalog.store(orders, in: Self.local, now: Self.fetchedAt) + catalog.store(orders, in: Self.local, now: Self.fetchedAt.addingTimeInterval(50)) + + #expect(catalog.schema(for: "Orders", in: Self.local, now: Self.fetchedAt.addingTimeInterval(100)) == orders) + } + + @Test("A schema belongs to its own scope and table") + func schemaIsolation() throws { + let catalog = DynamoDBCatalog() + catalog.store(try Self.schema("Orders"), in: Self.local, now: Self.fetchedAt) + + #expect(catalog.schema(for: "Orders", in: Self.remote, now: Self.fetchedAt) == nil) + #expect(catalog.schema(for: "Customers", in: Self.local, now: Self.fetchedAt) == nil) + } + + @Test("Cached schemas are listed by name for their scope only") + func cachedSchemasByScope() throws { + let catalog = DynamoDBCatalog() + catalog.store(try Self.schema("Orders"), in: Self.local) + catalog.store(try Self.schema("Customers"), in: Self.local) + catalog.store(try Self.schema("Invoices"), in: Self.remote) + + #expect(catalog.cachedSchemas(in: Self.local).map(\.name) == ["Customers", "Orders"]) + #expect(catalog.cachedSchemas(in: Self.remote).map(\.name) == ["Invoices"]) + } + + @Test("Invalidating a table drops its schema and every resume point of it, and nothing else") + func invalidateTable() throws { + let catalog = DynamoDBCatalog() + catalog.store(try Self.schema("Orders"), in: Self.local, now: Self.fetchedAt) + catalog.store(try Self.schema("Customers"), in: Self.local, now: Self.fetchedAt) + catalog.store(try Self.schema("Orders"), in: Self.remote, now: Self.fetchedAt) + catalog.storeResumePoint(Self.point(0, key: "a"), table: "Orders", fingerprint: "scan", offset: 100, in: Self.local) + catalog.storeResumePoint(Self.point(0, key: "b"), table: "Orders", fingerprint: "query", offset: 100, in: Self.local) + catalog.storeResumePoint(Self.point(0, key: "c"), table: "Customers", fingerprint: "scan", offset: 100, in: Self.local) + catalog.storeResumePoint(Self.point(0, key: "d"), table: "Orders", fingerprint: "scan", offset: 100, in: Self.remote) + + catalog.invalidate(table: "Orders", in: Self.local) + + #expect(catalog.schema(for: "Orders", in: Self.local, now: Self.fetchedAt) == nil) + #expect(Self.nearest(catalog, fingerprint: "scan", offset: 100) == nil) + #expect(Self.nearest(catalog, fingerprint: "query", offset: 100) == nil) + #expect(catalog.schema(for: "Customers", in: Self.local, now: Self.fetchedAt)?.name == "Customers") + #expect(catalog.schema(for: "Orders", in: Self.remote, now: Self.fetchedAt)?.name == "Orders") + #expect(Self.nearest(catalog, table: "Customers", offset: 100) == Self.point(0, key: "c")) + #expect(Self.nearest(catalog, offset: 100, scope: Self.remote) == Self.point(0, key: "d")) + } + + @Test("Invalidating a table forgets the types seen in its items, and a write forgets only its read positions") + func invalidateForgetsTypesButWritesKeepThem() throws { + let catalog = DynamoDBCatalog() + catalog.store(try Self.schema("Orders"), in: Self.local, now: Self.fetchedAt) + catalog.mergeColumnTypes(["age": .number], for: "Orders", in: Self.local) + catalog.storeResumePoint(Self.point(0, key: "a"), table: "Orders", fingerprint: "scan", offset: 100, in: Self.local) + + catalog.forgetReadPositions(table: "Orders", in: Self.local) + + #expect(Self.nearest(catalog, fingerprint: "scan", offset: 100) == nil) + #expect(catalog.schema(for: "Orders", in: Self.local, now: Self.fetchedAt) != nil) + #expect(catalog.columnTypes(for: "Orders", in: Self.local) == ["age": .number]) + + catalog.invalidate(table: "Orders", in: Self.local) + + #expect(catalog.columnTypes(for: "Orders", in: Self.local).isEmpty) + } + + @Test("An invalidated table's fingerprints no longer count toward eviction") + func invalidateTableFreesEvictionSlots() { + let catalog = DynamoDBCatalog() + let kept = DynamoDBCatalog.maximumFingerprints / 2 + for index in 0.. DynamoDBAttributeValue? { + try DynamoDBCellCodec.decode(.text(text), template: template, columnType: columnType, attribute: attribute) + } + + private static func invalidValueAttribute(of error: DynamoDBError?) -> String? { + guard case .invalidValue(let attribute, _) = error else { return nil } + return attribute + } + + @Test( + "A scalar attribute becomes the cell its type reads as", + arguments: [ + CellCase(name: "String", value: .string("02134"), expected: .text("02134")), + CellCase(name: "Number", value: .number("1.50"), expected: .text("1.50")), + CellCase( + name: "38 digit Number", + value: .number("12345678901234567890123456789012345678"), + expected: .text("12345678901234567890123456789012345678") + ), + CellCase(name: "true", value: .bool(true), expected: .text("true")), + CellCase(name: "false", value: .bool(false), expected: .text("false")), + CellCase(name: "NULL", value: .null, expected: .null), + CellCase(name: "Binary", value: .binary(Data([0x00, 0xFF, 0x10])), expected: .bytes(Data([0x00, 0xFF, 0x10]))) + ] + ) + func scalarCells(cellCase: CellCase) { + #expect(DynamoDBCellCodec.cell(for: cellCase.value) == cellCase.expected) + } + + @Test("A missing attribute is a null cell") + func missingAttributeIsNull() { + #expect(DynamoDBCellCodec.cell(for: nil) == .null) + } + + @Test("A map renders as plain JSON with sorted keys and exact numbers") + func mapRendersAsPlainJSON() { + let value = DynamoDBAttributeValue.map([ + "b": .number("12345678901234567890123456789012345678"), + "a": .string("x"), + "c": .map(["z": .bool(true), "y": .null]), + "d": .number("1.50") + ]) + let expected = #"{"a":"x","b":12345678901234567890123456789012345678,"c":{"y":null,"z":true},"d":1.50}"# + #expect(DynamoDBCellCodec.cell(for: value) == .text(expected)) + } + + @Test("A list renders its elements in order, with binary as base64") + func listRendersInOrder() { + let value = DynamoDBAttributeValue.list([.string("x"), .number("1.50"), .binary(Data([0xFF])), .null]) + #expect(DynamoDBCellCodec.cell(for: value) == .text(#"["x",1.50,"/w==",null]"#)) + } + + @Test("A String Set renders its members sorted") + func stringSetRendersSorted() { + #expect(DynamoDBCellCodec.cell(for: .stringSet(["b", "a", "c"])) == .text(#"["a","b","c"]"#)) + } + + @Test("A Number Set renders its members in numeric order") + func numberSetRendersNumerically() { + let value = DynamoDBAttributeValue.numberSet(["10", "9", "-1", "1.5", "1e3"]) + #expect(DynamoDBCellCodec.cell(for: value) == .text("[-1,1.5,9,10,1e3]")) + } + + @Test("A Binary Set renders its members as sorted base64") + func binarySetRendersSortedBase64() { + let value = DynamoDBAttributeValue.binarySet([Data([2]), Data([1])]) + #expect(DynamoDBCellCodec.cell(for: value) == .text(#"["AQ==","Ag=="]"#)) + } + + @Test("Number spellings JSON does not allow are rewritten as JSON numbers inside a map") + func dynamoDBOnlyNumberFormsBecomeJSON() { + let value = DynamoDBAttributeValue.map([ + "plus": .number("+5"), + "point": .number(".5"), + "negativePoint": .number("-.5"), + "trailingPoint": .number("5.") + ]) + #expect(DynamoDBCellCodec.cell(for: value) == .text(#"{"negativePoint":-0.5,"plus":5,"point":0.5,"trailingPoint":5}"#)) + } + + @Test("Every number the codec accepts renders as a number, never as null", arguments: ["007.5", "-0012", "+.5"]) + func acceptedNumbersNeverRenderAsNull(text: String) throws { + #expect(DynamoDBNumber.isValid(text)) + guard case .text(let rendered) = DynamoDBCellCodec.cell(for: .numberSet([text])) else { + Issue.record("A Number Set must render as text") + return + } + let member = try #require(try DynamoDBJSON.parse(rendered).arrayValue?.first) + let number = try #require(member.numberText, "rendered \(rendered)") + #expect(DynamoDBNumber.areEqual(number, text)) + } + + @Test("Digits typed over a String stay a String") + func stringTemplateKeepsLeadingZeros() throws { + #expect(try Self.decodeText("02134", template: .string("x")) == .string("02134")) + } + + @Test("A template's type wins over the column's type") + func templateBeatsColumnType() throws { + #expect(try Self.decodeText("02134", template: .string("x"), columnType: .number) == .string("02134")) + } + + @Test("Digits typed over a Number become a Number with the trimmed text") + func numberTemplateKeepsTrimmedText() throws { + #expect(try Self.decodeText("02134", template: .number("5")) == .number("02134")) + #expect(try Self.decodeText(" 1.50 ", template: .number("5")) == .number("1.50")) + } + + @Test("Text that is not a number, typed over a Number, is an error naming the attribute") + func invalidNumberNamesAttribute() { + let error = #expect(throws: DynamoDBError.self) { + try Self.decodeText("12abc", template: .number("5"), attribute: "price") + } + #expect(Self.invalidValueAttribute(of: error) == "price") + #expect(error?.localizedDescription.contains("price") == true) + } + + @Test("A number outside DynamoDB's range is an error") + func outOfRangeNumberIsRejected() { + #expect(throws: DynamoDBError.self) { try Self.decodeText("1E+126", template: .number("5")) } + #expect(throws: DynamoDBError.self) { try Self.decodeText("1234567890123456789012345678901234567890", template: .number("5")) } + } + + @Test("A JSON array over a String Set stays a String Set") + func stringSetStaysStringSet() throws { + #expect(try Self.decodeText(#"["b", "a"]"#, template: .stringSet(["x"])) == .stringSet(["b", "a"])) + } + + @Test("A JSON array over a Number Set stays a Number Set, from numbers or numeric strings") + func numberSetStaysNumberSet() throws { + #expect(try Self.decodeText(#"[3, "1.5", 12345678901234567890123456789012345678]"#, template: .numberSet(["1"])) + == .numberSet(["3", "1.5", "12345678901234567890123456789012345678"])) + } + + @Test("A JSON array over a Binary Set stays a Binary Set") + func binarySetStaysBinarySet() throws { + #expect(try Self.decodeText(#"["AQ==", "AgM="]"#, template: .binarySet([Data([9])])) + == .binarySet([Data([1]), Data([2, 3])])) + } + + @Test("A JSON array over a List stays a List and each element keeps its own template") + func listKeepsElementTemplates() throws { + let template = DynamoDBAttributeValue.list([.binary(Data([9])), .stringSet(["a"]), .number("1")]) + let decoded = try Self.decodeText(#"["AQ==", ["c", "b"], 2, "extra"]"#, template: template) + #expect(decoded == .list([.binary(Data([1])), .stringSet(["c", "b"]), .number("2"), .string("extra")])) + } + + @Test("Editing a map as JSON keeps nested sets and binary from the template") + func mapKeepsNestedTypes() throws { + let template = DynamoDBAttributeValue.map([ + "tags": .numberSet(["1", "2"]), + "names": .stringSet(["a"]), + "blob": .binary(Data([1])), + "zip": .string("02134"), + "inner": .map(["hashes": .binarySet([Data([1])])]) + ]) + let text = #"{"tags":[3,1],"names":["b"],"blob":"Ag==","zip":"02134","inner":{"hashes":["AwQ="]},"new":"z"}"# + let decoded = try Self.decodeText(text, template: template) + #expect(decoded == .map([ + "tags": .numberSet(["3", "1"]), + "names": .stringSet(["b"]), + "blob": .binary(Data([2])), + "zip": .string("02134"), + "inner": .map(["hashes": .binarySet([Data([3, 4])])]), + "new": .string("z") + ])) + } + + @Test("An unedited cell decodes back to the value it was rendered from") + func renderedCellRoundTrips() throws { + let value = DynamoDBAttributeValue.map([ + "s": .string("02134"), + "n": .number("12345678901234567890123456789012345678"), + "scaled": .number("1.50"), + "b": .binary(Data([1, 2])), + "t": .bool(true), + "z": .null, + "l": .list([.string("x"), .number("1.5"), .binary(Data([7]))]), + "m": .map(["inner": .numberSet(["1", "2"])]), + "ss": .stringSet(["a", "b"]), + "ns": .numberSet(["-1", "10"]), + "bs": .binarySet([Data([1]), Data([2])]) + ]) + guard case .text(let text) = DynamoDBCellCodec.cell(for: value) else { + Issue.record("A map must render as text") + return + } + #expect(try Self.decodeText(text, template: value) == value) + } + + @Test("Map text that is not JSON is an error naming the attribute") + func invalidMapJSONNamesAttribute() { + let error = #expect(throws: DynamoDBError.self) { + try Self.decodeText(#"{"a": "#, template: .map(["a": .string("x")]), attribute: "profile") + } + #expect(Self.invalidValueAttribute(of: error) == "profile") + } + + @Test( + "An empty set is an error", + arguments: [ + SetCase(name: "String Set", template: .stringSet(["a"]), text: "[]"), + SetCase(name: "Number Set", template: .numberSet(["1"]), text: "[]"), + SetCase(name: "Binary Set", template: .binarySet([Data([1])]), text: " [ ] ") + ] + ) + func emptySetIsRejected(setCase: SetCase) { + #expect(throws: DynamoDBError.self) { try Self.decodeText(setCase.text, template: setCase.template) } + } + + @Test( + "A set holding the same member twice is an error", + arguments: [ + SetCase(name: "String Set", template: .stringSet(["a"]), text: #"["a", "b", "a"]"#), + SetCase(name: "Number Set as numbers", template: .numberSet(["1"]), text: "[1, 1.0]"), + SetCase(name: "Number Set as strings", template: .numberSet(["1"]), text: #"["1", "1.0"]"#), + SetCase(name: "Number Set in exponent form", template: .numberSet(["1"]), text: "[100, 1e2]"), + SetCase(name: "Binary Set", template: .binarySet([Data([1])]), text: #"["AQ==", "AQ=="]"#) + ] + ) + func duplicateSetMemberIsRejected(setCase: SetCase) { + #expect(throws: DynamoDBError.self) { try Self.decodeText(setCase.text, template: setCase.template) } + } + + @Test("String Set members DynamoDB stores as different bytes are not duplicates") + func canonicallyEquivalentStringsAreDistinctMembers() throws { + let composed = "caf\u{00E9}" + let decomposed = "cafe\u{0301}" + let text = DynamoDBJSON.array([.string(composed), .string(decomposed)]).serialized() + let decoded = try Self.decodeText(text, template: .stringSet(["x"])) + guard case .stringSet(let members) = decoded else { + Issue.record("Expected a String Set, got \(String(describing: decoded))") + return + } + #expect(members.map { Array($0.utf8) } == [Array(composed.utf8), Array(decomposed.utf8)]) + } + + @Test( + "Boolean text decodes over a Boolean", + arguments: [ + TextCase(text: "true", expected: .bool(true)), + TextCase(text: "1", expected: .bool(true)), + TextCase(text: " TRUE ", expected: .bool(true)), + TextCase(text: "false", expected: .bool(false)), + TextCase(text: "0", expected: .bool(false)), + TextCase(text: "False", expected: .bool(false)) + ] + ) + func booleanText(textCase: TextCase) throws { + #expect(try Self.decodeText(textCase.text, template: .bool(false)) == textCase.expected) + } + + @Test("Text that is not a boolean is an error over a Boolean", arguments: ["yes", "", "2", "truthy"]) + func invalidBooleanIsRejected(text: String) { + let error = #expect(throws: DynamoDBError.self) { + try Self.decodeText(text, template: .bool(true), attribute: "active") + } + #expect(Self.invalidValueAttribute(of: error) == "active") + } + + @Test("Empty or null text over a NULL stays NULL", arguments: ["", " ", "null", "NULL"]) + func nullTemplateKeepsNull(text: String) throws { + #expect(try Self.decodeText(text, template: .null) == .null) + } + + @Test("Other text over a NULL is read as if the attribute had no type") + func nullTemplateInfersOtherText() throws { + #expect(try Self.decodeText("02134", template: .null) == .string("02134")) + #expect(try Self.decodeText(#"{"a": 1}"#, template: .null) == .map(["a": .number("1")])) + } + + @Test("Base64 text over a Binary decodes to its bytes, and other text is an error") + func binaryTemplateDecodesBase64() throws { + #expect(try Self.decodeText(" AQID\n", template: .binary(Data([9]))) == .binary(Data([1, 2, 3]))) + #expect(throws: DynamoDBError.self) { try Self.decodeText("not base64!", template: .binary(Data([9]))) } + } + + @Test("A bytes cell is Binary whatever the template") + func bytesCellIsBinary() throws { + let decoded = try DynamoDBCellCodec.decode( + .bytes(Data([1, 2])), template: .string("x"), columnType: .string, attribute: "attr" + ) + #expect(decoded == .binary(Data([1, 2]))) + } + + @Test("A null cell removes the attribute") + func nullCellRemovesAttribute() throws { + let decoded = try DynamoDBCellCodec.decode(.null, template: .string("x"), columnType: .string, attribute: "attr") + #expect(decoded == nil) + } + + @Test( + "With no template and no column type, text is a String unless it is a JSON object or array", + arguments: [ + TextCase(text: "hello", expected: .string("hello")), + TextCase(text: "02134", expected: .string("02134")), + TextCase(text: "true", expected: .string("true")), + TextCase(text: "{oops", expected: .string("{oops")), + TextCase(text: #"{"a": 1, "b": {"c": "d"}}"#, expected: .map(["a": .number("1"), "b": .map(["c": .string("d")])])), + TextCase(text: #"[1, "x", true, null]"#, expected: .list([.number("1"), .string("x"), .bool(true), .null])), + TextCase(text: " [] ", expected: .list([])) + ] + ) + func inferredWithoutTemplate(textCase: TextCase) throws { + #expect(try Self.decodeText(textCase.text) == textCase.expected) + } + + @Test("A Number column types new text as a Number") + func numberColumnTypeDecodesNumber() throws { + #expect(try Self.decodeText("12", columnType: .number) == .number("12")) + #expect(throws: DynamoDBError.self) { try Self.decodeText("twelve", columnType: .number) } + } + + @Test("A set or map column types new JSON text as that type") + func collectionColumnTypes() throws { + #expect(try Self.decodeText(#"["a", "b"]"#, columnType: .stringSet) == .stringSet(["a", "b"])) + #expect(try Self.decodeText("[1, 2]", columnType: .numberSet) == .numberSet(["1", "2"])) + #expect(try Self.decodeText(#"{"a": [1]}"#, columnType: .map) == .map(["a": .list([.number("1")])])) + #expect(try Self.decodeText("[1]", columnType: .list) == .list([.number("1")])) + } + + @Test("Text that is not JSON is an error in a Map column") + func nonJSONInMapColumnIsRejected() { + #expect(throws: DynamoDBError.self) { try Self.decodeText("hello", columnType: .map) } + } +} diff --git a/TableProTests/Plugins/DynamoDB/DynamoDBClientTests.swift b/TableProTests/Plugins/DynamoDB/DynamoDBClientTests.swift new file mode 100644 index 0000000000..6614b30140 --- /dev/null +++ b/TableProTests/Plugins/DynamoDB/DynamoDBClientTests.swift @@ -0,0 +1,427 @@ +import Foundation +import TableProPluginKit +import Testing + +@Suite("DynamoDB client") +struct DynamoDBClientTests { + private static let exampleDate = Date(timeIntervalSince1970: 1_440_938_160) + + private static let throttled = DynamoDBClientTestTransport.Reply.http( + status: 400, + body: #"{"__type":"com.amazonaws.dynamodb.v20120810#ProvisionedThroughputExceededException","message":"Rate exceeded"}"# + ) + private static let internalError = DynamoDBClientTestTransport.Reply.http( + status: 500, body: #"{"__type":"com.amazonaws.dynamodb.v20120810#InternalServerError","message":"Internal error"}"# + ) + private static let expiredToken = DynamoDBClientTestTransport.Reply.http( + status: 400, body: #"{"__type":"com.amazonaws.dynamodb.v20120810#ExpiredTokenException","message":"expired"}"# + ) + private static let signatureExpired = DynamoDBClientTestTransport.Reply.http( + status: 400, + body: #"{"__type":"com.amazonaws.dynamodb.v20120810#InvalidSignatureException","# + + #""message":"Signature expired: 20150830T123600Z is now earlier than 20150830T124100Z"}"#, + headers: ["Date": "Sun, 30 Aug 2015 12:46:00 GMT"] + ) + private static let tableNames = DynamoDBClientTestTransport.Reply.http(status: 200, body: #"{"TableNames":["Orders"]}"#) + + private static func makeClient( + _ transport: DynamoDBClientTestTransport, + sleeps: DynamoDBClientSleepLog = DynamoDBClientSleepLog(), + sleep: (@Sendable (TimeInterval) async throws -> Void)? = nil + ) throws -> DynamoDBClient { + let endpoint = try DynamoDBEndpoint.resolve(fields: ["awsAuthMethod": "local"], profileRegion: { _ in nil }) + let credentials = DynamoDBCredentialsProvider(fields: ["awsAuthMethod": "local"], username: "", password: "") + let fixedNow = exampleDate + return DynamoDBClient( + endpoint: endpoint, + credentials: credentials, + transport: transport, + retryPolicy: DynamoDBRetryPolicy(random: { 0 }), + now: { fixedNow }, + sleep: sleep ?? { seconds in sleeps.record(seconds) } + ) + } + + private static func serviceCode(of error: (any Error)?) -> String? { + guard case .service(let service)? = error as? DynamoDBError else { return nil } + return service.code + } + + @Test("Throttling is retried until DynamoDB accepts the request") + func throttlingRetriesUntilSuccess() async throws { + let transport = DynamoDBClientTestTransport([Self.throttled, Self.throttled, Self.tableNames]) + let sleeps = DynamoDBClientSleepLog() + let client = try Self.makeClient(transport, sleeps: sleeps) + + let response = try await client.send(.listTables, [:]) + + #expect(response["TableNames"] == .array([.string("Orders")])) + #expect(transport.requests.count == 3) + #expect(sleeps.delays == [0, 0]) + } + + @Test("Throttling gives up after four attempts") + func throttlingGivesUp() async throws { + let transport = DynamoDBClientTestTransport(Array(repeating: Self.throttled, count: 6)) + let client = try Self.makeClient(transport) + + let error = await #expect(throws: DynamoDBError.self) { + try await client.send(.updateItem, [:]) + } + + #expect(Self.serviceCode(of: error) == "ProvisionedThroughputExceededException") + #expect(transport.requests.count == DynamoDBRetryPolicy.maximumAttempts) + } + + @Test("A 500 on an UpdateItem is not sent again") + func updateItemServerErrorIsNotRetried() async throws { + let transport = DynamoDBClientTestTransport([Self.internalError, Self.tableNames]) + let client = try Self.makeClient(transport) + + let error = await #expect(throws: DynamoDBError.self) { + try await client.send(.updateItem, ["TableName": .string("Orders")]) + } + + #expect(Self.serviceCode(of: error) == "InternalServerError") + #expect(transport.requests.count == 1) + } + + @Test("A 500 on a read is sent again") + func readServerErrorIsRetried() async throws { + let transport = DynamoDBClientTestTransport([Self.internalError, Self.tableNames]) + let client = try Self.makeClient(transport) + + _ = try await client.send(.getItem, ["TableName": .string("Orders")]) + + #expect(transport.requests.count == 2) + } + + @Test("A redirect is a configuration error, refused without a retry") + func redirectIsRefused() async throws { + let transport = DynamoDBClientTestTransport([ + .http(status: 307, body: "", headers: ["Location": "http://attacker.example.com/"]) + ]) + let client = try Self.makeClient(transport) + + await #expect(throws: DynamoDBError.configuration(String( + localized: "The endpoint answered with a redirect, which DynamoDB never sends. Check the Custom Endpoint." + ))) { + try await client.send(.updateItem, [:]) + } + #expect(transport.requests.count == 1) + } + + @Test("A clock skew error moves the signing clock to the server's Date") + func clockSkewShiftsSigningDate() async throws { + let transport = DynamoDBClientTestTransport([Self.signatureExpired, Self.tableNames]) + let client = try Self.makeClient(transport) + + _ = try await client.send(.listTables, [:]) + + let requests = transport.requests + #expect(requests.count == 2) + #expect(requests.first?.value(forHTTPHeaderField: "X-Amz-Date") == "20150830T123600Z") + #expect(requests.last?.value(forHTTPHeaderField: "X-Amz-Date") == "20150830T124600Z") + } + + @Test("The clock is corrected once, and a second skew error fails") + func clockSkewCorrectsOnce() async throws { + let transport = DynamoDBClientTestTransport([Self.signatureExpired, Self.signatureExpired, Self.tableNames]) + let client = try Self.makeClient(transport) + + let error = await #expect(throws: DynamoDBError.self) { + try await client.send(.listTables, [:]) + } + + #expect(Self.serviceCode(of: error) == "InvalidSignatureException") + #expect(transport.requests.count == 2) + } + + @Test("An expired token is refreshed once, and a second expiry fails") + func expiredTokenRefreshesOnce() async throws { + let transport = DynamoDBClientTestTransport([Self.expiredToken, Self.expiredToken, Self.tableNames]) + let client = try Self.makeClient(transport) + + let error = await #expect(throws: DynamoDBError.self) { + try await client.send(.updateItem, [:]) + } + + #expect(Self.serviceCode(of: error) == "ExpiredTokenException") + #expect(transport.requests.count == 2) + } + + @Test("Cancelling the task while it waits to retry ends with cancelled") + func cancellingDuringSleep() async throws { + let transport = DynamoDBClientTestTransport([Self.throttled, Self.tableNames]) + let (started, startedContinuation) = AsyncStream.makeStream() + let client = try Self.makeClient(transport, sleep: { _ in + startedContinuation.yield() + try await Task.sleep(nanoseconds: 60_000_000_000) + }) + + let task = Task { try await client.send(.listTables, [:]) } + var iterator = started.makeAsyncIterator() + _ = await iterator.next() + task.cancel() + let result = await task.result + + guard case .failure(let error) = result else { + Issue.record("The request finished after its task was cancelled") + return + } + #expect(error as? DynamoDBError == .cancelled) + #expect(transport.requests.count == 1) + } + + @Test("A task cancelled before it starts sends nothing") + func cancelledBeforeStart() async throws { + let transport = DynamoDBClientTestTransport([Self.tableNames]) + let client = try Self.makeClient(transport) + let (gate, gateContinuation) = AsyncStream.makeStream() + + let task = Task { + var iterator = gate.makeAsyncIterator() + _ = await iterator.next() + return try await client.send(.listTables, [:]) + } + task.cancel() + gateContinuation.yield() + let result = await task.result + + guard case .failure(let error) = result else { + Issue.record("A cancelled task sent its request") + return + } + #expect(error as? DynamoDBError == .cancelled) + #expect(transport.requests.isEmpty) + } + + @Test("A cancellation the transport reports is surfaced as cancelled", arguments: [true, false]) + func transportCancellation(_ asURLError: Bool) async throws { + let reply: DynamoDBClientTestTransport.Reply = asURLError ? .urlError(.cancelled) : .taskCancelled + let transport = DynamoDBClientTestTransport([reply, Self.tableNames]) + let client = try Self.makeClient(transport) + + await #expect(throws: DynamoDBError.cancelled) { + try await client.send(.listTables, [:]) + } + #expect(transport.requests.count == 1) + } + + @Test("A dropped connection is retried for a read and not for an UpdateItem") + func transportFailureRetriesIdempotentOnly() async throws { + let readTransport = DynamoDBClientTestTransport([.urlError(.networkConnectionLost), Self.tableNames]) + _ = try await Self.makeClient(readTransport).send(.scan, ["TableName": .string("Orders")]) + #expect(readTransport.requests.count == 2) + + let writeTransport = DynamoDBClientTestTransport([.urlError(.networkConnectionLost), Self.tableNames]) + let error = await #expect(throws: DynamoDBError.self) { + try await Self.makeClient(writeTransport).send(.updateItem, ["TableName": .string("Orders")]) + } + guard case .transport? = error else { + Issue.record("A dropped connection was not reported as a transport error: \(String(describing: error))") + return + } + #expect(writeTransport.requests.count == 1) + } + + @Test("An empty 200 answers an empty object") + func emptySuccessBody() async throws { + let transport = DynamoDBClientTestTransport([.http(status: 200, body: "")]) + let response = try await Self.makeClient(transport).send(.deleteItem, [:]) + #expect(response == .object([:])) + } + + @Test("A 200 that is not JSON is an invalid response") + func unparsableSuccessBody() async throws { + let transport = DynamoDBClientTestTransport([.http(status: 200, body: "")]) + let error = await #expect(throws: DynamoDBError.self) { + try await Self.makeClient(transport).send(.listTables, [:]) + } + guard case .invalidResponse? = error else { + Issue.record("Expected an invalid response, got \(String(describing: error))") + return + } + } + + @Test("Every request is a signed POST carrying the target, the content type and the body") + func requestShape() async throws { + let transport = DynamoDBClientTestTransport([Self.throttled, Self.tableNames]) + let client = try Self.makeClient(transport) + let body: [String: DynamoDBJSON] = ["Limit": .number("1")] + + _ = try await client.send(.listTables, body) + + let requests = transport.requests + #expect(requests.count == 2) + for request in requests { + #expect(request.httpMethod == "POST") + #expect(request.url?.absoluteString == "http://localhost:8000") + #expect(request.value(forHTTPHeaderField: "X-Amz-Target") == "DynamoDB_20120810.ListTables") + #expect(request.value(forHTTPHeaderField: "Content-Type") == "application/x-amz-json-1.0") + #expect(request.value(forHTTPHeaderField: "Host") == "localhost:8000") + #expect(request.httpBody == DynamoDBJSON.object(body).serializedData) + let authorization = request.value(forHTTPHeaderField: "Authorization") ?? "" + #expect(authorization.hasPrefix( + "AWS4-HMAC-SHA256 Credential=local/20150830/us-east-1/dynamodb/aws4_request, " + + "SignedHeaders=content-type;host;x-amz-date;x-amz-target, Signature=" + )) + } + } +} + +@Suite("DynamoDB credentials provider", .serialized) +struct DynamoDBCredentialsProviderTests { + @Test("Local auth signs with the fixed local key and needs no AWS files") + func localCredentials() async throws { + let provider = DynamoDBCredentialsProvider(fields: ["awsAuthMethod": "local"], username: "ignored", password: "ignored") + let credentials = try await provider.credentials() + #expect(credentials.accessKeyId == DynamoDBCredentialsProvider.localAccessKey) + #expect(credentials.secretAccessKey == DynamoDBCredentialsProvider.localAccessKey) + #expect(credentials.sessionToken == nil) + #expect(provider.identity == "local") + } + + @Test("Access key auth falls back to the username and password") + func accessKeyFromUsernameAndPassword() async throws { + let provider = DynamoDBCredentialsProvider( + fields: ["awsAuthMethod": "credentials"], username: "AKIDEXAMPLE", password: "secret" + ) + let credentials = try await provider.credentials() + #expect(credentials.accessKeyId == "AKIDEXAMPLE") + #expect(credentials.secretAccessKey == "secret") + #expect(provider.identity == "key:AKIDEXAMPLE") + } + + @Test("An SSO profile that does not exist throws the AWS error, not a DynamoDB error") + func missingSSOProfileKeepsAWSError() async throws { + let profile = "tablepro-missing-\(UUID().uuidString)" + let provider = DynamoDBCredentialsProvider( + fields: ["awsAuthMethod": "sso", "awsProfileName": profile], username: "", password: "" + ) + + let error = try await DynamoDBTestAWSFiles.withEmptyConfiguration { + await #expect(throws: (any Error).self) { + try await provider.credentials() + } + } + + let thrown = try #require(error) + #expect(!(thrown is DynamoDBError)) + #expect(thrown is AWSAuthError || thrown is AWSSSOError) + #expect(provider.identity == "profile:\(profile)") + } + + @Test("The client passes a credential error through unwrapped and sends nothing") + func clientPassesCredentialErrorThrough() async throws { + let profile = "tablepro-missing-\(UUID().uuidString)" + let transport = DynamoDBClientTestTransport([]) + let endpoint = try DynamoDBEndpoint.resolve( + fields: ["awsAuthMethod": "sso", "awsRegion": "us-east-1"], profileRegion: { _ in nil } + ) + let client = DynamoDBClient( + endpoint: endpoint, + credentials: DynamoDBCredentialsProvider( + fields: ["awsAuthMethod": "sso", "awsProfileName": profile], username: "", password: "" + ), + transport: transport, + retryPolicy: DynamoDBRetryPolicy(random: { 0 }), + sleep: { _ in } + ) + + let error = try await DynamoDBTestAWSFiles.withEmptyConfiguration { + await #expect(throws: (any Error).self) { + try await client.send(.listTables, [:]) + } + } + + let thrown = try #require(error) + #expect(!(thrown is DynamoDBError)) + #expect(transport.requests.isEmpty) + } +} + +private enum DynamoDBTestAWSFiles { + static let variables = ["AWS_CONFIG_FILE", "AWS_SHARED_CREDENTIALS_FILE"] + + static func withEmptyConfiguration(_ body: () async throws -> Value) async throws -> Value { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("tablepro-dynamodb-aws-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + let previous = variables.map { name in getenv(name).map { String(cString: $0) } } + for name in variables { + let file = directory.appendingPathComponent(name.lowercased()) + try Data().write(to: file) + setenv(name, file.path, 1) + } + defer { + for (name, value) in zip(variables, previous) { + if let value { + setenv(name, value, 1) + } else { + unsetenv(name) + } + } + try? FileManager.default.removeItem(at: directory) + } + return try await body() + } +} + +private final class DynamoDBClientTestTransport: DynamoDBTransport, @unchecked Sendable { + enum Reply: Sendable { + case http(status: Int, body: String, headers: [String: String] = [:]) + case urlError(URLError.Code) + case taskCancelled + } + + private let lock = NSLock() + private var replies: [Reply] + private var recorded: [URLRequest] = [] + + init(_ replies: [Reply]) { + self.replies = replies + } + + var requests: [URLRequest] { + lock.withLock { recorded } + } + + func send(_ request: URLRequest) async throws -> (Data, HTTPURLResponse) { + let reply = lock.withLock { () -> Reply? in + recorded.append(request) + return replies.isEmpty ? nil : replies.removeFirst() + } + guard let reply else { + throw URLError(.resourceUnavailable) + } + switch reply { + case .urlError(let code): + throw URLError(code) + case .taskCancelled: + throw CancellationError() + case .http(let status, let body, let headers): + guard let url = request.url, + let response = HTTPURLResponse(url: url, statusCode: status, httpVersion: "HTTP/1.1", headerFields: headers) + else { + throw URLError(.badServerResponse) + } + return (Data(body.utf8), response) + } + } + + func invalidate() {} +} + +private final class DynamoDBClientSleepLog: @unchecked Sendable { + private let lock = NSLock() + private var recorded: [TimeInterval] = [] + + func record(_ seconds: TimeInterval) { + lock.withLock { recorded.append(seconds) } + } + + var delays: [TimeInterval] { + lock.withLock { recorded } + } +} diff --git a/TableProTests/Plugins/DynamoDB/DynamoDBDriverTests.swift b/TableProTests/Plugins/DynamoDB/DynamoDBDriverTests.swift new file mode 100644 index 0000000000..327c2dbc2d --- /dev/null +++ b/TableProTests/Plugins/DynamoDB/DynamoDBDriverTests.swift @@ -0,0 +1,1018 @@ +// +// DynamoDBDriverTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +final class DynamoDBScriptedTransport: DynamoDBTransport, @unchecked Sendable { + struct Call: Sendable { + let action: String + let body: DynamoDBJSON + } + + enum Reply: Sendable { + case json(String) + case failure(status: Int, type: String, message: String) + } + + typealias Handler = @Sendable (_ action: String, _ body: DynamoDBJSON, _ attempt: Int) -> Reply + + private let lock = NSLock() + private let handler: Handler + private var recorded: [Call] = [] + private var attempts: [String: Int] = [:] + + init(_ handler: @escaping Handler) { + self.handler = handler + } + + var calls: [Call] { + lock.withLock { recorded } + } + + func bodies(of action: String) -> [DynamoDBJSON] { + calls.filter { $0.action == action }.map(\.body) + } + + func count(of action: String) -> Int { + lock.withLock { attempts[action] ?? 0 } + } + + func send(_ request: URLRequest) async throws -> (Data, HTTPURLResponse) { + guard let url = request.url else { throw URLError(.badURL) } + let target = request.value(forHTTPHeaderField: "X-Amz-Target") ?? "" + let action = target.split(separator: ".").last.map(String.init) ?? target + let body = try DynamoDBJSON.parse(request.httpBody ?? Data()) + let attempt = lock.withLock { () -> Int in + recorded.append(Call(action: action, body: body)) + attempts[action, default: 0] += 1 + return attempts[action] ?? 1 + } + let status: Int + let text: String + switch handler(action, body, attempt) { + case .json(let json): + status = 200 + text = json + case .failure(let code, let type, let message): + status = code + text = #"{"__type":"com.amazonaws.dynamodb.v20120810#\#(type)","message":"\#(message)"}"# + } + guard let response = HTTPURLResponse( + url: url, statusCode: status, httpVersion: "HTTP/1.1", + headerFields: ["Content-Type": "application/x-amz-json-1.0"] + ) else { + throw URLError(.badServerResponse) + } + return (Data(text.utf8), response) + } + + func invalidate() {} +} + +enum DynamoDBDriverFixture { + static func driver(_ transport: DynamoDBScriptedTransport) async throws -> DynamoDBPluginDriver { + let config = DriverConnectionConfig( + host: "", port: 0, username: "", password: "", database: "", + additionalFields: ["awsAuthMethod": "local", "awsRegion": "us-east-1"] + ) + let driver = DynamoDBPluginDriver(config: config, catalog: DynamoDBCatalog()) { endpoint, credentials in + DynamoDBClient( + endpoint: endpoint, + credentials: credentials, + transport: transport, + retryPolicy: DynamoDBRetryPolicy(random: { 0 }), + sleep: { _ in } + ) + } + try await driver.connect() + return driver + } + + static let orders = describe() + + static func describe(provisioned: Bool = false) -> String { + let billing = provisioned + ? #""ProvisionedThroughput":{"ReadCapacityUnits":5,"WriteCapacityUnits":7}"# + : #""BillingModeSummary":{"BillingMode":"PAY_PER_REQUEST"}"# + return #""" + {"Table":{"TableName":"orders","TableStatus":"ACTIVE", + "KeySchema":[{"AttributeName":"pk","KeyType":"HASH"},{"AttributeName":"sk","KeyType":"RANGE"}], + "AttributeDefinitions":[{"AttributeName":"pk","AttributeType":"S"},{"AttributeName":"sk","AttributeType":"N"}, + {"AttributeName":"status","AttributeType":"S"},{"AttributeName":"total","AttributeType":"N"}], + "GlobalSecondaryIndexes":[{"IndexName":"byStatus","IndexStatus":"ACTIVE", + "KeySchema":[{"AttributeName":"status","KeyType":"HASH"},{"AttributeName":"total","KeyType":"RANGE"}], + "Projection":{"ProjectionType":"ALL"}}], + \#(billing)}} + """# + } + + static func item(_ pk: String, _ sk: Int, _ extra: String = "") -> String { + #"{"pk":{"S":"\#(pk)"},"sk":{"N":"\#(sk)"}\#(extra)}"# + } + + static func key(_ pk: String, _ sk: Int) -> String { + item(pk, sk) + } + + static func startIndex(_ body: DynamoDBJSON) -> Int? { + body["ExclusiveStartKey"]?["sk"]?["N"]?.stringValue.flatMap(Int.init) + } + + static func text(_ cell: PluginCellValue?) -> String? { + guard case .text(let value)? = cell else { return nil } + return value + } + + static func column(_ name: String, of result: PluginQueryResult) -> [String?] { + guard let index = result.columns.firstIndex(of: name) else { return [] } + return result.rows.map { text($0[index]) } + } +} + +@Suite("DynamoDB driver over a scripted transport") +struct DynamoDBDriverTests { + private typealias Fixture = DynamoDBDriverFixture + + // MARK: - Streaming + + @Test("A stream delivers every row of every page to a consumer that starts late") + func streamDeliversEveryRow() async throws { + let transport = DynamoDBScriptedTransport { action, body, _ in + switch action { + case "DescribeTable": + return .json(Fixture.orders) + case "Scan": + let from = (Fixture.startIndex(body) ?? -1) + 1 + let isLast = from + 1_000 >= 12_000 + let items = (from..<(from + 1_000)).map { index in + Fixture.item("p", index, index == 11_999 ? #","late":{"S":"x"}"# : "") + } + let next = isLast ? "" : #","LastEvaluatedKey":\#(Fixture.key("p", from + 999))"# + return .json(#"{"Items":[\#(items.joined(separator: ","))]\#(next)}"#) + default: + return .json(#"{"TableNames":[]}"#) + } + } + let driver = try await Fixture.driver(transport) + let query = try #require(driver.defaultExportQuery(table: "orders")) + + let stream = driver.streamRows(query: query) + try await Task.sleep(nanoseconds: 200_000_000) + var columns: [String] = [] + var rowCount = 0 + for try await element in stream { + switch element { + case .header(let header): columns = header.columns + case .rows(let rows): rowCount += rows.count + } + } + + #expect(rowCount == 12_000) + #expect(columns.contains("late")) + } + + @Test("A stream applies ORDER BY and OFFSET, and OFFSET without LIMIT does not trap") + func streamAppliesWindow() async throws { + let transport = DynamoDBScriptedTransport { action, _, _ in + switch action { + case "DescribeTable": + return .json(Fixture.orders) + case "Scan": + let items = [2, 3, 1, 5].enumerated().map { index, value in + Fixture.item("p", index, #","v":{"N":"\#(value)"}"#) + } + return .json(#"{"Items":[\#(items.joined(separator: ","))]}"#) + default: + return .json(#"{"TableNames":[]}"#) + } + } + let driver = try await Fixture.driver(transport) + + let sorted = try await Self.collect(driver.streamRows(query: #"Scan {"TableName":"orders"} ORDER BY "v" DESC"#)) + let skipped = try await Self.collect(driver.streamRows(query: #"Scan {"TableName":"orders"} OFFSET 1"#)) + + #expect(sorted.column("v") == ["5", "3", "2", "1"]) + #expect(skipped.rows.count == 3) + } + + @Test("A stream finishes while another reader in the process waits on a quiet pipe") + func streamIgnoresABlockedAsyncBytesReader() async throws { + let transport = DynamoDBScriptedTransport { action, _, _ in + switch action { + case "DescribeTable": + return .json(Fixture.orders) + case "Scan": + return .json(#"{"Items":[\#(Fixture.item("p", 0)),\#(Fixture.item("p", 1))]}"#) + default: + return .json(#"{"TableNames":[]}"#) + } + } + let driver = try await Fixture.driver(transport) + let query = try #require(driver.defaultExportQuery(table: "orders")) + let pipe = Pipe() + let blocker = Task { + for try await _ in pipe.fileHandleForReading.bytes {} + } + try await Task.sleep(nanoseconds: 100_000_000) + + let finished = await withTaskGroup(of: Bool.self) { group in + group.addTask { + let result = try? await Self.collect(driver.streamRows(query: query)) + return result?.rows.count == 2 + } + group.addTask { + try? await Task.sleep(nanoseconds: 10_000_000_000) + return false + } + let first = await group.next() ?? false + try? pipe.fileHandleForWriting.close() + group.cancelAll() + return first + } + blocker.cancel() + + #expect(finished) + } + + // MARK: - Paging + + @Test("An empty filtered page that still has a LastEvaluatedKey is not the end") + func filteredPagesKeepReading() async throws { + let transport = DynamoDBScriptedTransport { action, body, _ in + switch action { + case "DescribeTable": + return .json(Fixture.orders) + case "Scan": + guard Fixture.startIndex(body) != nil else { + return .json(#"{"Items":[],"LastEvaluatedKey":\#(Fixture.key("p", 1))}"#) + } + return .json(#"{"Items":[\#(Fixture.item("p", 2, #","v":{"N":"5"}"#))]}"#) + default: + return .json(#"{"TableNames":[]}"#) + } + } + let driver = try await Fixture.driver(transport) + let query = try #require(driver.buildFilteredQuery( + table: "orders", schema: nil, + queryFilters: [PluginQueryFilter(column: "v", op: "=", value: "5")], logicMode: "AND", + sortColumns: [], columns: [], limit: 10, offset: 0, columnKinds: [:] + )) + + let result = try await driver.execute(query: query) + + let scans = transport.bodies(of: "Scan") + #expect(result.rows.count == 1) + #expect(scans.count == 2) + #expect(scans.first?["Limit"] == nil) + #expect(scans.last.flatMap(Fixture.startIndex) == 1) + } + + @Test("The next page starts from the key where the previous one stopped") + func nextPageResumes() async throws { + let transport = DynamoDBScriptedTransport { action, body, _ in + switch action { + case "DescribeTable": + return .json(Fixture.orders) + case "Scan": + let from = (Fixture.startIndex(body) ?? -1) + 1 + let limit = body["Limit"]?.intValue ?? 5 + let end = min(from + limit, 5) + let items = (from..) async throws -> PluginQueryResult { + var columns: [String] = [] + var rows: [[PluginCellValue]] = [] + for try await element in stream { + switch element { + case .header(let header): columns = header.columns + case .rows(let batch): rows += batch + } + } + return PluginQueryResult( + columns: columns, columnTypeNames: columns.map { _ in "" }, rows: rows, + rowsAffected: 0, timing: PluginQueryTiming(total: 0) + ) + } + + private static func writeTransport( + currentItem: String = #"{"Item":\#(DynamoDBDriverFixture.item("p", 1, #","v":{"N":"1"}"#))}"#, + execute: DynamoDBScriptedTransport.Reply + ) -> DynamoDBScriptedTransport { + DynamoDBScriptedTransport { action, _, _ in + switch action { + case "DescribeTable": return .json(Fixture.orders) + case "GetItem": return .json(currentItem) + case "Scan": return .json(#"{"Items":[]}"#) + case "ExecuteStatement": return execute + default: return .json(#"{"TableNames":[]}"#) + } + } + } + + private static func updateStatement( + _ driver: DynamoDBPluginDriver + ) throws -> (statement: String, parameters: [PluginCellValue]) { + let change = PluginRowChange( + rowIndex: 0, type: .update, + cellChanges: [(columnIndex: 2, columnName: "v", oldValue: .text("1"), newValue: .text("2"))], + originalRow: [.text("p"), .text("1"), .text("1")] + ) + return try #require(driver.generateStatements( + table: "orders", columns: ["pk", "sk", "v"], primaryKeyColumns: ["pk", "sk"], changes: [change], + insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: [] + )?.first) + } + + private static func partiQLTransport(items: [Int]) -> DynamoDBScriptedTransport { + DynamoDBScriptedTransport { action, _, _ in + switch action { + case "DescribeTable": + return .json(Fixture.orders) + case "ExecuteStatement": + let rows = items.map { Fixture.item("p", $0, #","status":{"S":"open"},"total":{"N":"1"}"#) } + return .json(#"{"Items":[\#(rows.joined(separator: ","))]}"#) + default: + return .json(#"{"TableNames":[]}"#) + } + } + } + + private static func endlessTransport() -> DynamoDBScriptedTransport { + DynamoDBScriptedTransport { action, body, _ in + switch action { + case "DescribeTable": + return .json(Fixture.orders) + case "Scan": + let next = (Fixture.startIndex(body) ?? 0) + 1 + return .json(#"{"Items":[],"LastEvaluatedKey":\#(Fixture.key("p", next))}"#) + default: + return .json(#"{"TableNames":[]}"#) + } + } + } + + private static func endlessQuery(_ driver: DynamoDBPluginDriver) throws -> String { + try #require(driver.buildFilteredQuery( + table: "orders", schema: nil, + queryFilters: [PluginQueryFilter(column: "v", op: "=", value: "5")], logicMode: "AND", + sortColumns: [], columns: [], limit: 10, offset: 0, columnKinds: [:] + )) + } +} + +private extension PluginQueryResult { + func column(_ name: String) -> [String?] { + DynamoDBDriverFixture.column(name, of: self) + } +} diff --git a/TableProTests/Plugins/DynamoDB/DynamoDBEndpointTests.swift b/TableProTests/Plugins/DynamoDB/DynamoDBEndpointTests.swift new file mode 100644 index 0000000000..5ab9c43a39 --- /dev/null +++ b/TableProTests/Plugins/DynamoDB/DynamoDBEndpointTests.swift @@ -0,0 +1,246 @@ +import Foundation +import TableProPluginKit +import Testing + +@Suite("DynamoDB endpoint resolution") +struct DynamoDBEndpointTests { + struct RegionCase: Sendable, CustomTestStringConvertible { + let region: String + let url: String + var testDescription: String { region } + } + + struct LoopbackCase: Sendable, CustomTestStringConvertible { + let host: String + let isLoopback: Bool + var testDescription: String { host } + } + + private static func noProfileRegion(_ name: String) -> String? { + nil + } + + private static var plainHTTPRefusal: DynamoDBError { + .configuration(String( + localized: "Plain HTTP is only allowed for an endpoint on this Mac (localhost). Use https:// for any other host." + )) + } + + private static func configurationMessage(of error: (any Error)?) -> String? { + guard case .configuration(let message)? = error as? DynamoDBError else { return nil } + return message + } + + @Test( + "The AWS host follows the region's partition", + arguments: [ + RegionCase(region: "us-east-1", url: "https://dynamodb.us-east-1.amazonaws.com/"), + RegionCase(region: "cn-north-1", url: "https://dynamodb.cn-north-1.amazonaws.com.cn/"), + RegionCase(region: "us-gov-west-1", url: "https://dynamodb.us-gov-west-1.amazonaws.com/"), + RegionCase(region: "eusc-de-east-1", url: "https://dynamodb.eusc-de-east-1.amazonaws.eu/") + ] + ) + func partitionHost(_ testCase: RegionCase) throws { + let endpoint = try DynamoDBEndpoint.resolve( + fields: ["awsAuthMethod": "credentials", "awsRegion": testCase.region], + profileRegion: Self.noProfileRegion + ) + #expect(endpoint.url.absoluteString == testCase.url) + #expect(endpoint.signingRegion == testCase.region) + #expect(!endpoint.isLocal) + } + + @Test( + "A region that is not a region name is refused before it becomes part of a host", + arguments: ["evil.example#", "us-east-1/../x", "user@evil.example", "us east 1", "-us-east-1", "us-east-1.", "a\r\nb"] + ) + func malformedRegionIsRefused(_ region: String) { + #expect(throws: DynamoDBError.self) { + try DynamoDBEndpoint.resolve(fields: ["awsRegion": region], profileRegion: Self.noProfileRegion) + } + } + + @Test("A profile's malformed region is refused too") + func malformedProfileRegionIsRefused() { + #expect(throws: DynamoDBError.self) { + try DynamoDBEndpoint.resolve(fields: ["awsAuthMethod": "profile"], profileRegion: { _ in "evil.example#" }) + } + } + + @Test("A typed region is trimmed and lowercased") + func typedRegionIsCanonical() throws { + let endpoint = try DynamoDBEndpoint.resolve( + fields: ["awsRegion": " EU-West-2 \n"], profileRegion: Self.noProfileRegion + ) + #expect(endpoint.signingRegion == "eu-west-2") + #expect(endpoint.url.absoluteString == "https://dynamodb.eu-west-2.amazonaws.com/") + } + + @Test("An empty region takes the profile's region for profile and SSO auth", arguments: ["profile", "sso"]) + func emptyRegionUsesProfileRegion(_ method: String) throws { + let asked = EndpointProfileLookupLog() + let endpoint = try DynamoDBEndpoint.resolve( + fields: ["awsAuthMethod": method, "awsRegion": "", "awsProfileName": "work"], + profileRegion: { name in + asked.append(name) + return name == "work" ? "ap-southeast-2" : nil + } + ) + #expect(endpoint.signingRegion == "ap-southeast-2") + #expect(endpoint.url.absoluteString == "https://dynamodb.ap-southeast-2.amazonaws.com/") + #expect(asked.names == ["work"]) + } + + @Test("An empty profile name reads the default profile's region") + func emptyProfileNameReadsDefault() throws { + let endpoint = try DynamoDBEndpoint.resolve( + fields: ["awsAuthMethod": "profile", "awsProfileName": ""], + profileRegion: { $0 == "default" ? "sa-east-1" : nil } + ) + #expect(endpoint.signingRegion == "sa-east-1") + } + + @Test("A profile with no region falls back to us-east-1") + func profileWithoutRegionFallsBack() throws { + let endpoint = try DynamoDBEndpoint.resolve( + fields: ["awsAuthMethod": "profile", "awsProfileName": "work"], profileRegion: Self.noProfileRegion + ) + #expect(endpoint.signingRegion == "us-east-1") + } + + @Test("Access key auth with no region uses us-east-1 and never reads a profile") + func accessKeyIgnoresProfileRegion() throws { + let asked = EndpointProfileLookupLog() + let endpoint = try DynamoDBEndpoint.resolve( + fields: ["awsAuthMethod": "credentials", "awsProfileName": "work"], + profileRegion: { name in + asked.append(name) + return "eu-central-1" + } + ) + #expect(endpoint.signingRegion == "us-east-1") + #expect(asked.names.isEmpty) + } + + @Test("The region field wins over the profile's region") + func regionFieldWinsOverProfile() throws { + let endpoint = try DynamoDBEndpoint.resolve( + fields: ["awsAuthMethod": "profile", "awsProfileName": "work", "awsRegion": "ap-northeast-1"], + profileRegion: { _ in "eu-west-1" } + ) + #expect(endpoint.signingRegion == "ap-northeast-1") + #expect(endpoint.url.absoluteString == "https://dynamodb.ap-northeast-1.amazonaws.com/") + } + + @Test("A custom HTTPS endpoint is kept with its path and signs for the typed region") + func customHTTPSEndpointKeepsPath() throws { + let endpoint = try DynamoDBEndpoint.resolve( + fields: ["awsRegion": "eu-west-1", "awsEndpointUrl": " https://ddb.example.com/prefix/ "], + profileRegion: Self.noProfileRegion + ) + #expect(endpoint.url.absoluteString == "https://ddb.example.com/prefix/") + #expect(endpoint.url.path(percentEncoded: true) == "/prefix/") + #expect(endpoint.signingRegion == "eu-west-1") + #expect(!endpoint.isLocal) + } + + @Test( + "Plain HTTP to this Mac is allowed and marked local", + arguments: ["http://localhost:8000", "http://127.0.0.1:8000", "http://[::1]:8000"] + ) + func loopbackHTTPIsAllowed(_ text: String) throws { + let endpoint = try DynamoDBEndpoint.resolve( + fields: ["awsAuthMethod": "credentials", "awsEndpointUrl": text], profileRegion: Self.noProfileRegion + ) + #expect(endpoint.url.absoluteString == text) + #expect(endpoint.isLocal) + #expect(endpoint.signingRegion == "us-east-1") + } + + @Test( + "Plain HTTP to any other host is refused", + arguments: ["http://192.168.1.5:8000", "http://localhost.evil.com", "http://127.0.0.1.nip.io:8000"] + ) + func remoteHTTPIsRefused(_ text: String) { + #expect(throws: Self.plainHTTPRefusal) { + try DynamoDBEndpoint.resolve( + fields: ["awsAuthMethod": "credentials", "awsEndpointUrl": text], profileRegion: Self.noProfileRegion + ) + } + } + + @Test("HTTPS to a remote host is allowed and not local") + func remoteHTTPSIsAllowed() throws { + let endpoint = try DynamoDBEndpoint.resolve( + fields: ["awsEndpointUrl": "https://192.168.1.5:8000"], profileRegion: Self.noProfileRegion + ) + #expect(!endpoint.isLocal) + } + + @Test("Local auth with no endpoint reaches localhost:8000", arguments: ["", " "]) + func localAuthDefaultsToLocalhost(_ endpointText: String) throws { + let endpoint = try DynamoDBEndpoint.resolve( + fields: ["awsAuthMethod": "local", "awsEndpointUrl": endpointText], profileRegion: Self.noProfileRegion + ) + #expect(endpoint.url.absoluteString == "http://localhost:8000") + #expect(endpoint.isLocal) + #expect(endpoint.signingRegion == "us-east-1") + } + + @Test("Local auth with no endpoint field at all reaches localhost:8000") + func localAuthWithoutField() throws { + let endpoint = try DynamoDBEndpoint.resolve(fields: ["awsAuthMethod": "local"], profileRegion: Self.noProfileRegion) + #expect(endpoint.url.absoluteString == DynamoDBEndpoint.localDefaultURL) + #expect(endpoint.isLocal) + } + + @Test("A malformed endpoint is a configuration error", arguments: ["not a url", "localhost:8000", "https://"]) + func malformedEndpointIsRejected(_ text: String) { + let error = #expect(throws: DynamoDBError.self) { + try DynamoDBEndpoint.resolve(fields: ["awsEndpointUrl": text], profileRegion: Self.noProfileRegion) + } + #expect(Self.configurationMessage(of: error) != nil) + } + + @Test("An endpoint with another scheme is a configuration error") + func otherSchemeIsRejected() { + #expect(throws: DynamoDBError.configuration(String(localized: "The endpoint must start with https:// or http://"))) { + try DynamoDBEndpoint.resolve(fields: ["awsEndpointUrl": "ftp://localhost:8000"], profileRegion: Self.noProfileRegion) + } + } + + @Test( + "Loopback is decided from the literal host", + arguments: [ + LoopbackCase(host: "localhost", isLoopback: true), + LoopbackCase(host: "LocalHost", isLoopback: true), + LoopbackCase(host: "127.0.0.1", isLoopback: true), + LoopbackCase(host: "127.1.2.3", isLoopback: true), + LoopbackCase(host: "::1", isLoopback: true), + LoopbackCase(host: "[::1]", isLoopback: true), + LoopbackCase(host: "127.0.0.1.nip.io", isLoopback: false), + LoopbackCase(host: "localhost.evil.com", isLoopback: false), + LoopbackCase(host: "128.0.0.1", isLoopback: false), + LoopbackCase(host: "127.0.0", isLoopback: false), + LoopbackCase(host: "127.256.0.1", isLoopback: false), + LoopbackCase(host: "0.0.0.0", isLoopback: false), + LoopbackCase(host: "", isLoopback: false) + ] + ) + func loopbackHost(_ testCase: LoopbackCase) { + #expect(DynamoDBEndpoint.isLoopbackHost(testCase.host) == testCase.isLoopback) + } +} + +private final class EndpointProfileLookupLog: @unchecked Sendable { + private let lock = NSLock() + private var stored: [String] = [] + + func append(_ name: String) { + lock.withLock { stored.append(name) } + } + + var names: [String] { + lock.withLock { stored } + } +} diff --git a/TableProTests/Plugins/DynamoDB/DynamoDBErrorTests.swift b/TableProTests/Plugins/DynamoDB/DynamoDBErrorTests.swift new file mode 100644 index 0000000000..344196c567 --- /dev/null +++ b/TableProTests/Plugins/DynamoDB/DynamoDBErrorTests.swift @@ -0,0 +1,244 @@ +import Foundation +import TableProPluginKit +import Testing + +@Suite("DynamoDB errors") +struct DynamoDBErrorTests { + struct CategoryCase: Sendable, CustomTestStringConvertible { + let code: String + let status: Int + let message: String + let category: DynamoDBServiceError.Category + var testDescription: String { "\(code) \(status)" } + + init(_ code: String, status: Int = 400, message: String = "", _ category: DynamoDBServiceError.Category) { + self.code = code + self.status = status + self.message = message + self.category = category + } + } + + private static func parse(_ body: String, status: Int = 400) -> DynamoDBServiceError { + DynamoDBServiceError.parse(body: Data(body.utf8), httpStatus: status) + } + + @Test("The code is the part of the DynamoDB __type after the hash") + func parsesDynamoDBNamespace() { + let error = Self.parse( + #"{"__type":"com.amazonaws.dynamodb.v20120810#ConditionalCheckFailedException","message":"The conditional request failed"}"# + ) + #expect(error.code == "ConditionalCheckFailedException") + #expect(error.message == "The conditional request failed") + #expect(error.httpStatus == 400) + #expect(error.cancellationReasons.isEmpty) + #expect(error.isConditionalCheckFailure) + } + + @Test("The coral validation namespace and a capitalised Message are read too") + func parsesCoralNamespace() { + let error = Self.parse( + #"{"__type":"com.amazon.coral.validate#ValidationException","Message":"One or more parameter values were invalid"}"# + ) + #expect(error.code == "ValidationException") + #expect(error.message == "One or more parameter values were invalid") + #expect(!error.isConditionalCheckFailure) + } + + @Test("A lowercase message wins over a capitalised one") + func lowercaseMessageWins() { + let error = Self.parse(#"{"__type":"x#ValidationException","message":"lower","Message":"upper"}"#) + #expect(error.message == "lower") + } + + @Test("A __type with no namespace is the code as is") + func typeWithoutNamespace() { + #expect(Self.parse(#"{"__type":"ThrottlingException","message":"slow down"}"#).code == "ThrottlingException") + } + + @Test("A body with no message names the error by its code") + func missingMessageUsesCode() { + let error = Self.parse(#"{"__type":"com.amazonaws.dynamodb.v20120810#ResourceNotFoundException"}"#) + #expect(error.message == "ResourceNotFoundException") + } + + @Test("A JSON body with no __type is named by its HTTP status") + func missingTypeUsesStatus() { + let error = Self.parse(#"{"message":"upstream failed"}"#, status: 503) + #expect(error.code == "HTTP503") + #expect(error.message == "upstream failed") + #expect(error.category == .transient) + } + + @Test("A body that is not JSON becomes an HTTP error with its size", arguments: ["Bad Gateway", ""]) + func nonJSONBody(_ body: String) { + let error = Self.parse(body, status: 502) + #expect(error.code == "HTTP502") + #expect(error.message == String(format: String(localized: "HTTP %d with a body of %d bytes"), 502, Data(body.utf8).count)) + #expect(error.httpStatus == 502) + #expect(error.category == .transient) + } + + @Test("Cancellation reasons are decoded in order, with None for a reason that has no code") + func decodesCancellationReasons() { + let error = Self.parse( + #""" + {"__type":"com.amazonaws.dynamodb.v20120810#TransactionCanceledException", + "Message":"Transaction cancelled, please refer cancellation reasons for specific reasons [None, ConditionalCheckFailed]", + "CancellationReasons":[{"Code":"None"},{"Code":"ConditionalCheckFailed","Message":"The conditional request failed"},{}]} + """# + ) + #expect(error.code == "TransactionCanceledException") + #expect(error.cancellationReasons == [ + DynamoDBCancellationReason(code: "None", message: nil), + DynamoDBCancellationReason(code: "ConditionalCheckFailed", message: "The conditional request failed"), + DynamoDBCancellationReason(code: "None", message: nil) + ]) + } + + @Test( + "Every error falls into the category that decides its retry", + arguments: [ + CategoryCase("ProvisionedThroughputExceededException", .throttling), + CategoryCase("ThrottlingException", .throttling), + CategoryCase("RequestLimitExceeded", .throttling), + CategoryCase("LimitExceededException", .throttling), + CategoryCase("TransactionInProgressException", .throttling), + CategoryCase("ReplicatedWriteConflictException", .throttling), + CategoryCase("ItemCollectionSizeLimitExceededException", .throttling), + CategoryCase("InternalServerError", status: 500, .transient), + CategoryCase("ServiceUnavailable", status: 503, .transient), + CategoryCase("HTTP500", status: 500, .transient), + CategoryCase("SomethingNew", status: 500, .transient), + CategoryCase("ExpiredTokenException", .expiredCredentials), + CategoryCase("ExpiredToken", .expiredCredentials), + CategoryCase( + "InvalidSignatureException", + message: "Signature expired: 20150830T123600Z is now earlier than 20150830T124100Z (20150830T124600Z - 5 min.)", + .clockSkew + ), + CategoryCase( + "InvalidSignatureException", + message: "Signature not yet current: 20150830T130100Z is still later than 20150830T125100Z", + .clockSkew + ), + CategoryCase("RequestTimeTooSkewed", .clockSkew), + CategoryCase("RequestExpired", .clockSkew), + CategoryCase( + "InvalidSignatureException", + message: "The request signature we calculated does not match the signature you provided.", + .authentication + ), + CategoryCase("UnrecognizedClientException", message: "The security token included in the request is invalid.", .authentication), + CategoryCase("MissingAuthenticationTokenException", .authentication), + CategoryCase("IncompleteSignatureException", .authentication), + CategoryCase("AccessDeniedException", message: "User is not authorized to perform: dynamodb:Scan", .fatal), + CategoryCase("ValidationException", .fatal), + CategoryCase("ResourceNotFoundException", .fatal), + CategoryCase("ConditionalCheckFailedException", .fatal), + CategoryCase("TransactionCanceledException", .fatal) + ] + ) + func category(_ testCase: CategoryCase) { + let error = DynamoDBServiceError(code: testCase.code, message: testCase.message, httpStatus: testCase.status) + #expect(error.category == testCase.category) + } + + @Test("An authentication failure says so") + func authenticationUserMessage() { + let error = DynamoDBServiceError( + code: "UnrecognizedClientException", message: "The security token included in the request is invalid.", httpStatus: 400 + ) + #expect(error.userMessage == String( + format: String(localized: "Authentication failed: %@"), "The security token included in the request is invalid." + )) + } + + @Test("Access denied is reported as a DynamoDB error, not as a failed sign-in") + func accessDeniedIsNotAuthentication() { + let error = DynamoDBServiceError( + code: "AccessDeniedException", message: "User is not authorized to perform: dynamodb:Scan", httpStatus: 400 + ) + #expect(error.userMessage == String( + format: String(localized: "DynamoDB error: [%1$@] %2$@"), + "AccessDeniedException", "User is not authorized to perform: dynamodb:Scan" + )) + #expect(!error.userMessage.hasPrefix(String(format: String(localized: "Authentication failed: %@"), ""))) + } + + @Test("A cancelled transaction lists each failed action by its 1-based number and skips None") + func cancellationReasonsInUserMessage() { + let error = DynamoDBServiceError( + code: "TransactionCanceledException", + message: "Transaction cancelled", + httpStatus: 400, + cancellationReasons: [ + DynamoDBCancellationReason(code: "None", message: nil), + DynamoDBCancellationReason(code: "ConditionalCheckFailed", message: "The conditional request failed"), + DynamoDBCancellationReason(code: "None", message: nil), + DynamoDBCancellationReason(code: "ValidationException", message: nil) + ] + ) + let lines = error.userMessage.components(separatedBy: "\n") + #expect(lines == [ + String(format: String(localized: "DynamoDB error: [%1$@] %2$@"), "TransactionCanceledException", "Transaction cancelled"), + String( + format: String(localized: "Action %1$d: %2$@%3$@"), 2, "ConditionalCheckFailed", " The conditional request failed" + ), + String(format: String(localized: "Action %1$d: %2$@%3$@"), 4, "ValidationException", "") + ]) + } + + @Test("A cancelled transaction with only None reasons adds no lines") + func noneReasonsAddNothing() { + let error = DynamoDBServiceError( + code: "TransactionCanceledException", + message: "Transaction cancelled", + httpStatus: 400, + cancellationReasons: [DynamoDBCancellationReason(code: "None", message: nil)] + ) + #expect(!error.userMessage.contains("\n")) + } + + @Test("A service error describes itself with its user message") + func serviceErrorDescription() { + let service = DynamoDBServiceError(code: "ValidationException", message: "bad key", httpStatus: 400) + #expect(DynamoDBError.service(service).errorDescription == service.userMessage) + } + + @Test("A partial batch reports how many writes applied, then each failure on its own line") + func partialBatchDescription() { + let error = DynamoDBError.partialBatch(applied: 3, total: 5, failures: ["pk = a: throttled", "pk = b: throttled"]) + #expect(error.errorDescription == [ + String(format: String(localized: "%1$d of %2$d were applied."), 3, 5), + "pk = a: throttled", + "pk = b: throttled" + ].joined(separator: "\n")) + } + + @Test("A changed item names its key and says to refresh") + func itemChangedDescription() { + let error = DynamoDBError.itemChanged(key: "pk = a, sk = 3") + #expect(error.errorDescription == String( + format: String(localized: "The item %@ changed after it was loaded. Refresh the table and edit it again."), + "pk = a, sk = 3" + )) + #expect(error.errorDescription?.contains("pk = a, sk = 3") == true) + } + + @Test("A timeout names the seconds it waited") + func timedOutDescription() { + let error = DynamoDBError.timedOut(seconds: 30) + #expect(error.errorDescription == String( + format: String(localized: "Stopped after %d seconds, the query timeout"), 30 + )) + #expect(error.errorDescription?.contains("30") == true) + } + + @Test("An invalid value names its attribute unless it has none") + func invalidValueDescription() { + #expect(DynamoDBError.invalidValue(attribute: "", reason: "Not a number").errorDescription == "Not a number") + #expect(DynamoDBError.invalidValue(attribute: "age", reason: "Not a number").errorDescription + == String(format: String(localized: "%@: %@"), "age", "Not a number")) + } +} diff --git a/TableProTests/Plugins/DynamoDB/DynamoDBExpressionTests.swift b/TableProTests/Plugins/DynamoDB/DynamoDBExpressionTests.swift new file mode 100644 index 0000000000..1d6b9d56c8 --- /dev/null +++ b/TableProTests/Plugins/DynamoDB/DynamoDBExpressionTests.swift @@ -0,0 +1,259 @@ +// +// DynamoDBExpressionTests.swift +// TableProTests +// + +import Foundation +import Testing + +@Suite("DynamoDB attribute paths") +struct DynamoDBAttributePathTests { + struct ParseCase: Sendable, CustomTestStringConvertible { + let text: String + let known: Set + let segments: [DynamoDBAttributePath.Segment] + + var testDescription: String { text } + } + + static let parseCases: [ParseCase] = [ + ParseCase(text: "status", known: [], segments: [.name("status")]), + ParseCase(text: "a.b", known: [], segments: [.name("a"), .name("b")]), + ParseCase(text: "address.city.name", known: [], segments: [.name("address"), .name("city"), .name("name")]), + ParseCase(text: "items[0].sku", known: [], segments: [.name("items"), .index(0), .name("sku")]), + ParseCase(text: "matrix[1][2]", known: [], segments: [.name("matrix"), .index(1), .index(2)]), + ParseCase(text: "a.b", known: ["a.b"], segments: [.name("a.b")]), + ParseCase(text: "tags[0]", known: ["tags[0]"], segments: [.name("tags[0]")]), + ParseCase(text: "items[x]", known: [], segments: [.name("items[x]")]), + ParseCase(text: "items[0", known: [], segments: [.name("items[0")]), + ParseCase(text: "items[]", known: [], segments: [.name("items[]")]), + ParseCase(text: "[0].sku", known: [], segments: [.name("[0].sku")]) + ] + + @Test("Parses a document path, keeping a known or malformed name whole", arguments: parseCases) + func parses(_ testCase: ParseCase) { + let path = DynamoDBAttributePath.parse(testCase.text, knownAttributes: testCase.known) + + #expect(path.segments == testCase.segments) + } + + @Test("A negative list index is not a document path") + func negativeIndexStaysOneName() { + let path = DynamoDBAttributePath.parse("items[-1]", knownAttributes: []) + + #expect(path.segments == [.name("items[-1]")]) + } + + @Test("The root is the first name and only a single name is top level") + func rootAndTopLevel() { + let nested = DynamoDBAttributePath.parse("items[0].sku", knownAttributes: []) + let single = DynamoDBAttributePath(attribute: "a.b") + + #expect(nested.root == "items") + #expect(nested.isTopLevel == false) + #expect(single.root == "a.b") + #expect(single.isTopLevel) + } + + private static let item: DynamoDBItem = [ + "a.b": .string("literal"), + "address": .map(["city": .string("Hanoi"), "geo": .map(["lat": .number("21.03")])]), + "items": .list([.map(["sku": .string("A1")]), .map(["sku": .string("B2")])]), + "matrix": .list([.list([.number("1"), .number("2")]), .list([.number("3")])]), + "tags": .stringSet(["x"]) + ] + + @Test("Walks maps and lists to the value a path names") + func walksMapsAndLists() { + let known: Set = [] + + #expect(DynamoDBAttributePath.parse("address.city", knownAttributes: known).value(in: Self.item) == .string("Hanoi")) + #expect(DynamoDBAttributePath.parse("address.geo.lat", knownAttributes: known).value(in: Self.item) == .number("21.03")) + #expect(DynamoDBAttributePath.parse("items[1].sku", knownAttributes: known).value(in: Self.item) == .string("B2")) + #expect(DynamoDBAttributePath.parse("matrix[0][1]", knownAttributes: known).value(in: Self.item) == .number("2")) + #expect(DynamoDBAttributePath.parse("items", knownAttributes: known).value(in: Self.item) == Self.item["items"]) + } + + @Test("A literal attribute named with a dot is read as that attribute") + func literalDottedName() { + let path = DynamoDBAttributePath.parse("a.b", knownAttributes: ["a.b"]) + + #expect(path.value(in: Self.item) == .string("literal")) + } + + @Test("A path that leaves the document or crosses the wrong type reads nothing") + func missingValues() { + let known: Set = [] + + #expect(DynamoDBAttributePath.parse("items[5].sku", knownAttributes: known).value(in: Self.item) == nil) + #expect(DynamoDBAttributePath.parse("items.sku", knownAttributes: known).value(in: Self.item) == nil) + #expect(DynamoDBAttributePath.parse("address[0]", knownAttributes: known).value(in: Self.item) == nil) + #expect(DynamoDBAttributePath.parse("address.zip", knownAttributes: known).value(in: Self.item) == nil) + #expect(DynamoDBAttributePath.parse("tags[0]", knownAttributes: known).value(in: Self.item) == nil) + #expect(DynamoDBAttributePath.parse("missing.x", knownAttributes: known).value(in: Self.item) == nil) + } +} + +@Suite("DynamoDB expression placeholders") +struct DynamoDBExpressionContextTests { + static func isValidPlaceholder(_ placeholder: String, prefix: Character) -> Bool { + guard placeholder.first == prefix else { return false } + let body = placeholder.dropFirst() + guard let first = body.first, !first.isNumber else { return false } + return body.allSatisfy { $0.isASCII && ($0.isLetter || $0.isNumber || $0 == "_") } + } + + @Test( + "Every attribute name gets a valid placeholder that maps back to it", + arguments: ["status", "name", "order-id", "a.b", "1st", "2024-total", "näme", "with space", "x:y#z"] + ) + func namePlaceholderIsValid(attribute: String) { + var context = DynamoDBExpressionContext() + + let placeholder = context.name(attribute) + + #expect(Self.isValidPlaceholder(placeholder, prefix: "#")) + #expect(context.names == [placeholder: attribute]) + } + + @Test("A reserved word is still sent through a placeholder") + func reservedWord() { + var context = DynamoDBExpressionContext() + + #expect(context.name("status") == "#status") + #expect(context.name("data") == "#data") + } + + @Test("A name used twice reuses its placeholder") + func reusesNamePlaceholder() { + var context = DynamoDBExpressionContext() + + let first = context.name("order-id") + let second = context.name("order-id") + + #expect(first == second) + #expect(context.names.count == 1) + } + + @Test("Names that sanitize to the same text get distinct placeholders") + func distinctNamesStayDistinct() { + var context = DynamoDBExpressionContext() + + let dashed = context.name("a-b") + let dotted = context.name("a.b") + let underscored = context.name("a_b") + + #expect(Set([dashed, dotted, underscored]).count == 3) + #expect(context.names[dashed] == "a-b") + #expect(context.names[dotted] == "a.b") + #expect(context.names[underscored] == "a_b") + } + + @Test("Long names that share a prefix get distinct placeholders") + func longNamesStayDistinct() { + var context = DynamoDBExpressionContext() + let stem = String(repeating: "x", count: 45) + + let first = context.name(stem + "1") + let second = context.name(stem + "2") + + #expect(first != second) + #expect(context.names[first] == stem + "1") + #expect(context.names[second] == stem + "2") + } + + @Test("A path renders each name through its placeholder and keeps list indexes") + func rendersPath() { + var context = DynamoDBExpressionContext() + + let rendered = context.path(DynamoDBAttributePath.parse("items[0].sku", knownAttributes: [])) + let repeated = context.path(DynamoDBAttributePath.parse("a.a", knownAttributes: [])) + + #expect(rendered == "#items[0].#sku") + #expect(repeated == "#a.#a") + #expect(context.names == ["#items": "items", "#sku": "sku", "#a": "a"]) + } + + @Test("The same value under the same hint reuses its placeholder") + func dedupesValues() { + var context = DynamoDBExpressionContext() + + let first = context.value(.string("x"), hint: "pk") + let second = context.value(.string("x"), hint: "pk") + + #expect(first == ":pk") + #expect(second == ":pk") + #expect(context.values == [":pk": .string("x")]) + } + + @Test("Different values under one hint get numbered placeholders") + func numbersDistinctValues() { + var context = DynamoDBExpressionContext() + + let first = context.value(.string("x"), hint: "pk") + let second = context.value(.string("y"), hint: "pk") + let third = context.value(.number("1"), hint: "pk") + let textOne = context.value(.string("1"), hint: "pk") + + #expect([first, second, third, textOne] == [":pk", ":pk2", ":pk3", ":pk4"]) + #expect(context.values == [":pk": .string("x"), ":pk2": .string("y"), ":pk3": .number("1"), ":pk4": .string("1")]) + } + + @Test( + "A value placeholder is valid whatever the hint", + arguments: ["order-id", "a.b", "1st", "näme", "", "with space"] + ) + func valuePlaceholderIsValid(hint: String) { + var context = DynamoDBExpressionContext() + + let placeholder = context.value(.string("v"), hint: hint) + + #expect(Self.isValidPlaceholder(placeholder, prefix: ":")) + } + + @Test("An empty context adds nothing to a request") + func emptyApplyAddsNothing() { + let context = DynamoDBExpressionContext() + var body: [String: DynamoDBJSON] = ["TableName": .string("orders")] + + context.apply(to: &body) + + #expect(body == ["TableName": .string("orders")]) + } + + @Test("A context holding only names writes no ExpressionAttributeValues") + func namesOnly() { + var context = DynamoDBExpressionContext() + _ = context.name("status") + var body: [String: DynamoDBJSON] = ["TableName": .string("orders")] + + context.apply(to: &body) + + #expect(body["ExpressionAttributeNames"] == .object(["#status": .string("status")])) + #expect(body["ExpressionAttributeValues"] == nil) + } + + @Test("Apply writes the names and the wire form of each value, keeping the rest of the body") + func appliesNamesAndValues() { + var context = DynamoDBExpressionContext() + let status = context.name("status") + let total = context.name("total") + let text = context.value(.string("shipped"), hint: "status") + let number = context.value(.number("12345678901234567890123456789012345678"), hint: "total") + let filter = "\(status) = \(text) AND \(total) > \(number)" + var body: [String: DynamoDBJSON] = [ + "TableName": .string("orders"), + "FilterExpression": .string(filter) + ] + + context.apply(to: &body) + + #expect(body["TableName"] == .string("orders")) + #expect(body["FilterExpression"] == .string(filter)) + #expect(body["ExpressionAttributeNames"] == .object(["#status": .string("status"), "#total": .string("total")])) + #expect(body["ExpressionAttributeValues"] == .object([ + ":status": .object(["S": .string("shipped")]), + ":total": .object(["N": .string("12345678901234567890123456789012345678")]) + ])) + } +} diff --git a/TableProTests/Plugins/DynamoDB/DynamoDBFilterTranslatorTests.swift b/TableProTests/Plugins/DynamoDB/DynamoDBFilterTranslatorTests.swift new file mode 100644 index 0000000000..ea3cfd7854 --- /dev/null +++ b/TableProTests/Plugins/DynamoDB/DynamoDBFilterTranslatorTests.swift @@ -0,0 +1,642 @@ +// +// DynamoDBFilterTranslatorTests.swift +// TableProTests +// + +import Foundation +import Testing + +@Suite("DynamoDB filter translation") +struct DynamoDBFilterTranslatorTests { + struct Translation { + let outcome: DynamoDBFilterTranslator.Outcome + let context: DynamoDBExpressionContext + } + + struct OperatorCase: Sendable, CustomTestStringConvertible { + let op: String + let value: String + + var testDescription: String { "\(op) \(value)" } + } + + static func schema() throws -> DynamoDBTableSchema { + let json = try DynamoDBJSON.parse(""" + {"Table": { + "TableName": "orders", + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"} + ], + "AttributeDefinitions": [ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "N"} + ] + }} + """) + return try DynamoDBTableSchema(describeTableResponse: json) + } + + static func filter( + _ attribute: String, + _ op: String, + _ value: String = "", + second: String? = nil, + kind: String? = nil, + caseSensitive: Bool = true + ) -> DynamoDBBrowseFilter { + DynamoDBBrowseFilter( + attribute: attribute, op: op, value: value, secondValue: second, kind: kind, caseSensitive: caseSensitive + ) + } + + static func translate(_ filter: DynamoDBBrowseFilter) throws -> Translation { + let translator = DynamoDBFilterTranslator(schema: try schema()) + var context = DynamoDBExpressionContext() + let path: DynamoDBAttributePath? = filter.attribute == DynamoDBFilterTranslator.anyAttributeColumn + ? nil + : DynamoDBAttributePath.parse(filter.attribute, knownAttributes: []) + let outcome = translator.translate(filter, path: path, context: &context) + return Translation(outcome: outcome, context: context) + } + + static func keyCondition(_ filter: DynamoDBBrowseFilter) throws -> (term: String?, context: DynamoDBExpressionContext) { + let translator = DynamoDBFilterTranslator(schema: try schema()) + var context = DynamoDBExpressionContext() + let term = translator.keyCondition(filter, attribute: filter.attribute, context: &context) + return (term, context) + } + + static func clientPredicate(_ filter: DynamoDBBrowseFilter, path: String?) -> DynamoDBClientPredicate { + DynamoDBClientPredicate( + path: path.map { DynamoDBAttributePath(attribute: $0) }, + op: filter.op, + value: filter.value, + secondValue: filter.secondValue, + caseSensitive: filter.caseSensitive + ) + } + + // MARK: - Equality and ordering + + @Test("Numeric text on a non-key attribute is compared as both a String and a Number") + func equalityOnNumericText() throws { + let result = try Self.translate(Self.filter("total", "=", "5")) + + #expect(result.outcome == .server("(#total = :total OR #total = :total2)")) + #expect(result.context.names == ["#total": "total"]) + #expect(result.context.values == [":total": .string("5"), ":total2": .number("5")]) + } + + @Test("Equality with true also matches a Boolean") + func equalityOnTrue() throws { + let result = try Self.translate(Self.filter("flag", "=", "true")) + + #expect(result.outcome == .server("(#flag = :flag OR #flag = :flag2)")) + #expect(result.context.values == [":flag": .string("true"), ":flag2": .bool(true)]) + } + + @Test("Plain text is compared as a String only") + func equalityOnText() throws { + let result = try Self.translate(Self.filter("name", "=", "Ann")) + + #expect(result.outcome == .server("#name = :name")) + #expect(result.context.values == [":name": .string("Ann")]) + } + + @Test("An ordering comparison on numeric text uses a String and a Number", arguments: [">", ">=", "<", "<="]) + func orderingOnNumericText(op: String) throws { + let result = try Self.translate(Self.filter("total", op, "5")) + + #expect(result.outcome == .server("(#total \(op) :total OR #total \(op) :total2)")) + #expect(result.context.values == [":total": .string("5"), ":total2": .number("5")]) + } + + @Test("An ordering comparison never compares a Boolean", arguments: [">", ">=", "<", "<="]) + func orderingNeverUsesBoolean(op: String) throws { + let result = try Self.translate(Self.filter("flag", op, "true")) + + #expect(result.outcome == .server("#flag \(op) :flag")) + #expect(result.context.values == [":flag": .string("true")]) + } + + @Test("Not equal requires the attribute and excludes every reading of the text", arguments: ["!=", "<>"]) + func notEqual(op: String) throws { + let result = try Self.translate(Self.filter("total", op, "5")) + + #expect(result.outcome == .server("(attribute_exists(#total) AND #total <> :total AND #total <> :total2)")) + #expect(result.context.values == [":total": .string("5"), ":total2": .number("5")]) + } + + @Test("Not equal to true excludes the text and the Boolean") + func notEqualToTrue() throws { + let result = try Self.translate(Self.filter("flag", "!=", "true")) + + #expect(result.outcome == .server("(attribute_exists(#flag) AND #flag <> :flag AND #flag <> :flag2)")) + #expect(result.context.values == [":flag": .string("true"), ":flag2": .bool(true)]) + } + + // MARK: - Text operators + + @Test("Contains becomes the contains function") + func contains() throws { + let text = try Self.translate(Self.filter("name", "CONTAINS", "ab")) + let numeric = try Self.translate(Self.filter("total", "CONTAINS", "5")) + + #expect(text.outcome == .server("contains(#name, :name)")) + #expect(text.context.values == [":name": .string("ab")]) + #expect(numeric.outcome == .server("(contains(#total, :total) OR contains(#total, :total2))")) + #expect(numeric.context.values == [":total": .string("5"), ":total2": .number("5")]) + } + + @Test("Not contains requires the attribute") + func notContains() throws { + let result = try Self.translate(Self.filter("name", "NOT CONTAINS", "ab")) + + #expect(result.outcome == .server("(attribute_exists(#name) AND NOT contains(#name, :name))")) + #expect(result.context.values == [":name": .string("ab")]) + } + + @Test("Starts with becomes begins_with on a String") + func startsWith() throws { + let text = try Self.translate(Self.filter("name", "STARTS WITH", "ab")) + let digits = try Self.translate(Self.filter("name", "STARTS WITH", "12")) + + #expect(text.outcome == .server("begins_with(#name, :name)")) + #expect(text.context.values == [":name": .string("ab")]) + #expect(digits.outcome == .server("begins_with(#name, :name)")) + #expect(digits.context.values == [":name": .string("12")]) + } + + @Test("Ends with and a regular expression are evaluated on the client", arguments: ["ENDS WITH", "REGEX"]) + func clientOnlyOperators(op: String) throws { + let filter = Self.filter("name", op, "son") + + let result = try Self.translate(filter) + + #expect(result.outcome == .client(Self.clientPredicate(filter, path: "name"))) + #expect(result.context == DynamoDBExpressionContext()) + } + + // MARK: - Null and empty + + @Test("Is null matches a missing attribute or the NULL type") + func isNull() throws { + let result = try Self.translate(Self.filter("total", "IS NULL")) + + #expect(result.outcome == .server("(attribute_not_exists(#total) OR attribute_type(#total, :null))")) + #expect(result.context.values == [":null": .string("NULL")]) + } + + @Test("Is not null requires the attribute and rejects the NULL type") + func isNotNull() throws { + let result = try Self.translate(Self.filter("total", "IS NOT NULL")) + + #expect(result.outcome == .server("(attribute_exists(#total) AND NOT attribute_type(#total, :null))")) + #expect(result.context.values == [":null": .string("NULL")]) + } + + @Test("Is empty compares with the empty String") + func isEmpty() throws { + let result = try Self.translate(Self.filter("name", "IS EMPTY")) + + #expect(result.outcome == .server("#name = :empty")) + #expect(result.context.values == [":empty": .string("")]) + } + + @Test("Is not empty requires the attribute and a value other than the empty String") + func isNotEmpty() throws { + let result = try Self.translate(Self.filter("name", "IS NOT EMPTY")) + + #expect(result.outcome == .server("(attribute_exists(#name) AND #name <> :empty)")) + #expect(result.context.values == [":empty": .string("")]) + } + + // MARK: - Membership + + @Test("In lists every typed reading of each value") + func inList() throws { + let text = try Self.translate(Self.filter("name", "IN", "a, b")) + let numeric = try Self.translate(Self.filter("total", "IN", "1, 2")) + + #expect(text.outcome == .server("#name IN (:name, :name2)")) + #expect(text.context.values == [":name": .string("a"), ":name2": .string("b")]) + #expect(numeric.outcome == .server("#total IN (:total, :total2, :total3, :total4)")) + #expect(numeric.context.values == [ + ":total": .string("1"), ":total2": .number("1"), ":total3": .string("2"), ":total4": .number("2") + ]) + } + + @Test("Not in requires the attribute") + func notInList() throws { + let result = try Self.translate(Self.filter("name", "NOT IN", "a, b")) + + #expect(result.outcome == .server("(attribute_exists(#name) AND NOT #name IN (:name, :name2))")) + } + + @Test("In with 150 values is split into IN terms of at most 100") + func inListChunks() throws { + let values = (0..<150).map { "v\($0)" }.joined(separator: ", ") + + let result = try Self.translate(Self.filter("name", "IN", values)) + + guard case .server(let expression) = result.outcome else { + Issue.record("Expected a server expression, got \(result.outcome)") + return + } + let regex = try NSRegularExpression(pattern: #"IN \(([^)]*)\)"#) + let range = NSRange(expression.startIndex..., in: expression) + let chunkSizes = regex.matches(in: expression, range: range).compactMap { match -> Int? in + Range(match.range(at: 1), in: expression).map { expression[$0].components(separatedBy: ", ").count } + } + #expect(chunkSizes == [100, 50]) + #expect(expression.hasPrefix("(#name IN (")) + #expect(expression.contains(") OR #name IN (")) + #expect(result.context.values.count == 150) + } + + // MARK: - Between + + @Test("Between with a second value compares as a String range and a Number range") + func betweenWithSecondValue() throws { + let result = try Self.translate(Self.filter("total", "BETWEEN", "1", second: "9")) + + #expect(result.outcome == .server( + "(#total BETWEEN :total AND :total2 OR #total BETWEEN :total3 AND :total4)" + )) + #expect(result.context.values == [ + ":total": .string("1"), ":total2": .string("9"), ":total3": .number("1"), ":total4": .number("9") + ]) + } + + @Test("Between reads both bounds from one comma separated value", arguments: [ + OperatorCase(op: "BETWEEN", value: "1,9"), + OperatorCase(op: "BETWEEN", value: " 1 , 9 ") + ]) + func betweenFromOneValue(_ testCase: OperatorCase) throws { + let result = try Self.translate(Self.filter("total", testCase.op, testCase.value)) + + #expect(result.outcome == .server( + "(#total BETWEEN :total AND :total2 OR #total BETWEEN :total3 AND :total4)" + )) + #expect(result.context.values[":total3"] == .number("1")) + #expect(result.context.values[":total4"] == .number("9")) + } + + @Test("Between drops the String range when its bounds are out of order as text") + func betweenDropsReversedTextRange() throws { + let result = try Self.translate(Self.filter("total", "BETWEEN", "2", second: "10")) + + #expect(result.outcome == .server("#total BETWEEN :total AND :total2")) + #expect(result.context.values == [":total": .number("2"), ":total2": .number("10")]) + } + + @Test("Between never compares Booleans") + func betweenNeverUsesBoolean() throws { + let result = try Self.translate(Self.filter("flag", "BETWEEN", "false", second: "true")) + + #expect(result.outcome == .server("#flag BETWEEN :flag AND :flag2")) + #expect(result.context.values == [":flag": .string("false"), ":flag2": .string("true")]) + } + + // MARK: - Key attributes + + @Test("A Number key compares as a Number only") + func numberKeyIsTypedNumber() throws { + let equal = try Self.translate(Self.filter("sk", "=", "5")) + let greater = try Self.translate(Self.filter("sk", ">", "5")) + + #expect(equal.outcome == .server("#sk = :sk")) + #expect(equal.context.values == [":sk": .number("5")]) + #expect(greater.outcome == .server("#sk > :sk")) + #expect(greater.context.values == [":sk": .number("5")]) + } + + @Test("A String key compares numeric text as a String only") + func stringKeyIsTypedString() throws { + let result = try Self.translate(Self.filter("pk", "=", "5")) + + #expect(result.outcome == .server("#pk = :pk")) + #expect(result.context.values == [":pk": .string("5")]) + } + + @Test("A Number key compared with text that is not a number matches nothing", arguments: [ + OperatorCase(op: "=", value: "abc"), + OperatorCase(op: ">", value: "abc"), + OperatorCase(op: ">=", value: "abc"), + OperatorCase(op: "<", value: "abc"), + OperatorCase(op: "<=", value: "abc"), + OperatorCase(op: "IN", value: "abc, def"), + OperatorCase(op: "BETWEEN", value: "a,b"), + OperatorCase(op: "BETWEEN", value: "9,1") + ]) + func numberKeyWithTextMatchesNothing(_ testCase: OperatorCase) throws { + let result = try Self.translate(Self.filter("sk", testCase.op, testCase.value)) + + #expect(result.outcome == .never) + } + + @Test("Starts with on a Number key matches nothing") + func startsWithOnNumberKey() throws { + let result = try Self.translate(Self.filter("sk", "STARTS WITH", "1")) + + #expect(result.outcome == .never) + } + + @Test("In on a Number key keeps only the values that are numbers") + func numberKeyInKeepsNumbers() throws { + let result = try Self.translate(Self.filter("sk", "IN", "1, abc, 2")) + + #expect(result.outcome == .server("#sk IN (:sk, :sk2)")) + #expect(result.context.values == [":sk": .number("1"), ":sk2": .number("2")]) + } + + @Test("Not equal or not in on a Number key with text keeps every item that has the key", arguments: [ + OperatorCase(op: "!=", value: "abc"), + OperatorCase(op: "NOT IN", value: "abc") + ]) + func numberKeyExclusionWithText(_ testCase: OperatorCase) throws { + let result = try Self.translate(Self.filter("sk", testCase.op, testCase.value)) + + #expect(result.outcome == .server("attribute_exists(#sk)")) + #expect(result.context.values.isEmpty) + } + + // MARK: - Client side filters + + @Test("A search across every attribute is evaluated on the client") + func anyAttributeColumn() throws { + let filter = Self.filter("*", "=", "x") + + let result = try Self.translate(filter) + + #expect(result.outcome == .client(Self.clientPredicate(filter, path: nil))) + #expect(DynamoDBFilterTranslator(schema: try Self.schema()).needsClient(filter)) + } + + @Test( + "A case-insensitive match on text with letters is evaluated on the client", + arguments: ["=", "!=", "<>", "CONTAINS", "NOT CONTAINS", "STARTS WITH", "IN", "NOT IN"] + ) + func caseInsensitiveNeedsClient(op: String) throws { + let filter = Self.filter("name", op, "Ab", caseSensitive: false) + + let result = try Self.translate(filter) + + #expect(result.outcome == .client(Self.clientPredicate(filter, path: "name"))) + #expect(result.context == DynamoDBExpressionContext()) + } + + @Test("A case-insensitive match on text without letters stays on the server") + func caseInsensitiveWithoutLetters() throws { + let result = try Self.translate(Self.filter("name", "CONTAINS", "12", caseSensitive: false)) + + #expect(result.outcome == .server("(contains(#name, :name) OR contains(#name, :name2))")) + } + + @Test("Only the raw filter column is refused") + func unsupportedReason() { + let raw = DynamoDBFilterTranslator.unsupportedReason(for: Self.filter("__RAW__", "=", "a = 1")) + + #expect(raw?.isEmpty == false) + #expect(DynamoDBFilterTranslator.unsupportedReason(for: Self.filter("name", "=", "a")) == nil) + #expect(DynamoDBFilterTranslator.unsupportedReason(for: Self.filter("*", "=", "a")) == nil) + } + + // MARK: - Key conditions + + @Test("Equality on a String key is a key condition") + func keyConditionEquality() throws { + let result = try Self.keyCondition(Self.filter("pk", "=", "a")) + + #expect(result.term == "#pk = :pk") + #expect(result.context.names == ["#pk": "pk"]) + #expect(result.context.values == [":pk": .string("a")]) + } + + @Test("A range on a Number key is a key condition typed as a Number") + func keyConditionRange() throws { + let result = try Self.keyCondition(Self.filter("sk", "<", "5")) + + #expect(result.term == "#sk < :sk") + #expect(result.context.values == [":sk": .number("5")]) + } + + @Test("Between on a Number key is a key condition", arguments: [ + OperatorCase(op: "BETWEEN", value: "1,9"), + OperatorCase(op: "BETWEEN", value: "1 , 9") + ]) + func keyConditionBetween(_ testCase: OperatorCase) throws { + let result = try Self.keyCondition(Self.filter("sk", testCase.op, testCase.value)) + + #expect(result.term == "#sk BETWEEN :sk AND :sk2") + #expect(result.context.values == [":sk": .number("1"), ":sk2": .number("9")]) + } + + @Test("Between with a second value is a key condition") + func keyConditionBetweenWithSecondValue() throws { + let result = try Self.keyCondition(Self.filter("sk", "BETWEEN", "1", second: "9")) + + #expect(result.term == "#sk BETWEEN :sk AND :sk2") + #expect(result.context.values == [":sk": .number("1"), ":sk2": .number("9")]) + } + + @Test("Starts with on a String key is a begins_with key condition") + func keyConditionStartsWith() throws { + let result = try Self.keyCondition(Self.filter("pk", "STARTS WITH", "ab")) + + #expect(result.term == "begins_with(#pk, :pk)") + #expect(result.context.values == [":pk": .string("ab")]) + } + + @Test("Filters a key condition cannot express are refused", arguments: [ + DynamoDBFilterTranslatorTests.filter("sk", "STARTS WITH", "1"), + DynamoDBFilterTranslatorTests.filter("sk", "=", "abc"), + DynamoDBFilterTranslatorTests.filter("sk", "BETWEEN", "a,b"), + DynamoDBFilterTranslatorTests.filter("pk", "!=", "a"), + DynamoDBFilterTranslatorTests.filter("pk", "CONTAINS", "a"), + DynamoDBFilterTranslatorTests.filter("pk", "IN", "a, b"), + DynamoDBFilterTranslatorTests.filter("pk", "ENDS WITH", "a"), + DynamoDBFilterTranslatorTests.filter("pk", "=", "Ab", caseSensitive: false), + DynamoDBFilterTranslatorTests.filter("name", "=", "a") + ]) + func keyConditionRefusals(_ filter: DynamoDBBrowseFilter) throws { + let result = try Self.keyCondition(filter) + + #expect(result.term == nil) + } +} + +@Suite("DynamoDB client-side predicates") +struct DynamoDBClientPredicateMatchingTests { + static func predicate( + _ op: String, + _ value: String = "", + second: String? = nil, + path: String? = "name", + caseSensitive: Bool = true + ) -> DynamoDBClientPredicate { + DynamoDBClientPredicate( + path: path.map { DynamoDBAttributePath.parse($0, knownAttributes: []) }, + op: op, + value: value, + secondValue: second, + caseSensitive: caseSensitive + ) + } + + static let alice: DynamoDBItem = ["name": .string("Alice"), "city": .string("Hanoi")] + + @Test("Equality compares the displayed text, folding case only when asked") + func equality() { + #expect(Self.predicate("=", "Alice").matches(Self.alice)) + #expect(!Self.predicate("=", "alice").matches(Self.alice)) + #expect(Self.predicate("=", "alice", caseSensitive: false).matches(Self.alice)) + #expect(Self.predicate("=", "true").matches(["name": .bool(true)])) + } + + @Test("A Number equals the same number written another way") + func numericEquality() { + #expect(Self.predicate("=", "5.0").matches(["name": .number("5")])) + #expect(!Self.predicate("!=", "5.0").matches(["name": .number("5")])) + } + + @Test("Not equal is the inverse of equality for an attribute that exists", arguments: ["!=", "<>"]) + func notEqual(op: String) { + #expect(Self.predicate(op, "Bob").matches(Self.alice)) + #expect(!Self.predicate(op, "Alice").matches(Self.alice)) + #expect(!Self.predicate(op, "ALICE", caseSensitive: false).matches(Self.alice)) + } + + @Test("Contains and not contains test a substring") + func contains() { + #expect(Self.predicate("CONTAINS", "lic").matches(Self.alice)) + #expect(!Self.predicate("CONTAINS", "LIC").matches(Self.alice)) + #expect(Self.predicate("CONTAINS", "LIC", caseSensitive: false).matches(Self.alice)) + #expect(Self.predicate("NOT CONTAINS", "xyz").matches(Self.alice)) + #expect(!Self.predicate("NOT CONTAINS", "lic").matches(Self.alice)) + } + + @Test("Starts with and ends with test a prefix and a suffix") + func prefixAndSuffix() { + #expect(Self.predicate("STARTS WITH", "Al").matches(Self.alice)) + #expect(!Self.predicate("STARTS WITH", "li").matches(Self.alice)) + #expect(Self.predicate("STARTS WITH", "al", caseSensitive: false).matches(Self.alice)) + #expect(Self.predicate("ENDS WITH", "ice").matches(Self.alice)) + #expect(!Self.predicate("ENDS WITH", "Ali").matches(Self.alice)) + #expect(Self.predicate("ENDS WITH", "ICE", caseSensitive: false).matches(Self.alice)) + } + + @Test("Is empty and is not empty need the attribute") + func emptiness() { + #expect(Self.predicate("IS EMPTY").matches(["name": .string("")])) + #expect(!Self.predicate("IS EMPTY").matches(Self.alice)) + #expect(!Self.predicate("IS EMPTY").matches([:])) + #expect(Self.predicate("IS NOT EMPTY").matches(Self.alice)) + #expect(!Self.predicate("IS NOT EMPTY").matches(["name": .string("")])) + #expect(!Self.predicate("IS NOT EMPTY").matches([:])) + } + + @Test("Is null matches a missing attribute and the NULL type, not the text NULL") + func nullness() { + #expect(Self.predicate("IS NULL").matches([:])) + #expect(Self.predicate("IS NULL").matches(["name": .null])) + #expect(!Self.predicate("IS NULL").matches(["name": .string("NULL")])) + #expect(!Self.predicate("IS NULL").matches(Self.alice)) + #expect(Self.predicate("IS NOT NULL").matches(Self.alice)) + #expect(!Self.predicate("IS NOT NULL").matches(["name": .null])) + #expect(!Self.predicate("IS NOT NULL").matches([:])) + } + + @Test("In and not in test membership of a comma separated list") + func membership() { + #expect(Self.predicate("IN", "Bob, Alice").matches(Self.alice)) + #expect(!Self.predicate("IN", "Bob, Carol").matches(Self.alice)) + #expect(Self.predicate("IN", "bob, alice", caseSensitive: false).matches(Self.alice)) + #expect(Self.predicate("NOT IN", "Bob, Carol").matches(Self.alice)) + #expect(!Self.predicate("NOT IN", "Alice").matches(Self.alice)) + } + + @Test("A regular expression is searched in the text, folding case only when asked") + func regularExpression() { + #expect(Self.predicate("REGEX", "^A.*e$").matches(Self.alice)) + #expect(Self.predicate("REGEX", "lic").matches(Self.alice)) + #expect(!Self.predicate("REGEX", "^a").matches(Self.alice)) + #expect(Self.predicate("REGEX", "^a", caseSensitive: false).matches(Self.alice)) + #expect(!Self.predicate("REGEX", "(").matches(Self.alice)) + } + + @Test("Numbers are ordered by value, not as text") + func numericOrdering() { + let ten: DynamoDBItem = ["name": .number("10")] + + #expect(Self.predicate(">", "9").matches(ten)) + #expect(!Self.predicate("<", "9").matches(ten)) + #expect(Self.predicate(">=", "10.0").matches(ten)) + #expect(Self.predicate("<=", "1E1").matches(ten)) + #expect(!Self.predicate(">", "10").matches(ten)) + } + + @Test("Text is ordered as text") + func textOrdering() { + let banana: DynamoDBItem = ["name": .string("banana")] + + #expect(Self.predicate(">", "apple").matches(banana)) + #expect(Self.predicate("<", "cherry").matches(banana)) + #expect(!Self.predicate("<", "apple").matches(banana)) + } + + @Test("Between is inclusive and numeric when both bounds are numbers") + func between() { + #expect(Self.predicate("BETWEEN", "2", second: "10").matches(["name": .number("5")])) + #expect(Self.predicate("BETWEEN", "2,10").matches(["name": .number("5")])) + #expect(Self.predicate("BETWEEN", "2", second: "10").matches(["name": .number("2")])) + #expect(Self.predicate("BETWEEN", "2", second: "10").matches(["name": .number("10")])) + #expect(!Self.predicate("BETWEEN", "2", second: "10").matches(["name": .number("11")])) + #expect(Self.predicate("BETWEEN", "a", second: "c").matches(["name": .string("b")])) + #expect(!Self.predicate("BETWEEN", "5").matches(["name": .number("5")])) + } + + @Test("CONTAINS and STARTS WITH on the client answer as DynamoDB's contains and begins_with would") + func clientMatchingFollowsDynamoDB() { + let list: DynamoDBItem = ["tags": .list([.string("Abc"), .number("5")])] + let number: DynamoDBItem = ["tags": .number("123")] + let numbers: DynamoDBItem = ["tags": .numberSet(["5", "7"])] + let path = DynamoDBAttributePath(attribute: "tags") + + func matches(_ op: String, _ value: String, _ item: DynamoDBItem) -> Bool { + DynamoDBClientPredicate(path: path, op: op, value: value, secondValue: nil, caseSensitive: false).matches(item) + } + + #expect(matches("CONTAINS", "abc", list)) + #expect(!matches("CONTAINS", "ab", list)) + #expect(matches("CONTAINS", "5.0", list)) + #expect(!matches("CONTAINS", "12", number)) + #expect(matches("NOT CONTAINS", "12", number)) + #expect(matches("CONTAINS", "5.0", numbers)) + #expect(!matches("STARTS WITH", "12", number)) + } + + @Test("A missing attribute matches no comparison", arguments: [ + "=", "!=", "<>", ">", "<", "CONTAINS", "NOT CONTAINS", "STARTS WITH", "ENDS WITH", "IN", "NOT IN", "REGEX" + ]) + func missingAttribute(op: String) { + #expect(!Self.predicate(op, "x").matches(["other": .string("x")])) + } + + @Test("A search across every attribute matches when any attribute does") + func anyAttribute() { + #expect(Self.predicate("=", "Hanoi", path: nil).matches(Self.alice)) + #expect(!Self.predicate("=", "Paris", path: nil).matches(Self.alice)) + #expect(Self.predicate("CONTAINS", "NOI", path: nil, caseSensitive: false).matches(Self.alice)) + #expect(Self.predicate("REGEX", "^Han", path: nil).matches(Self.alice)) + } + + @Test("A nested path reads the value inside a map") + func nestedPath() { + let item: DynamoDBItem = ["address": .map(["city": .string("Hanoi")])] + + #expect(Self.predicate("=", "Hanoi", path: "address.city").matches(item)) + #expect(!Self.predicate("=", "Hanoi", path: "address.zip").matches(item)) + } +} diff --git a/TableProTests/Plugins/DynamoDB/DynamoDBItemTableTests.swift b/TableProTests/Plugins/DynamoDB/DynamoDBItemTableTests.swift new file mode 100644 index 0000000000..8e220e3dbd --- /dev/null +++ b/TableProTests/Plugins/DynamoDB/DynamoDBItemTableTests.swift @@ -0,0 +1,316 @@ +import Foundation +import TableProPluginKit +import Testing + +@Suite("DynamoDB item table") +struct DynamoDBItemTableTests { + struct ClassificationCase: Sendable, CustomTestStringConvertible { + let value: DynamoDBAttributeValue + let displayName: String + let classification: String + var testDescription: String { displayName } + } + + private static func schema() throws -> DynamoDBTableSchema { + try DynamoDBTableSchema(describeTableResponse: DynamoDBJSON.parse(#""" + { + "Table": { + "TableName": "Orders", + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"} + ], + "AttributeDefinitions": [ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "N"}, + {"AttributeName": "status", "AttributeType": "S"} + ], + "GlobalSecondaryIndexes": [ + { + "IndexName": "byStatus", + "KeySchema": [{"AttributeName": "status", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL"} + } + ] + } + } + """#)) + } + + private static func item(_ values: [String: DynamoDBAttributeValue]) -> DynamoDBItem { + values + } + + private static func ids(_ items: [DynamoDBItem]) -> [String] { + items.map { item in + guard case .string(let id)? = item["id"] else { return "?" } + return id + } + } + + @Test("Columns run preferred first, then table keys, then the rest alphabetically") + func columnOrder() throws { + let items = [ + Self.item(["pk": .string("a"), "sk": .number("1"), "zeta": .string("z"), "beta": .string("b")]), + Self.item(["pk": .string("b"), "alpha": .number("2")]) + ] + let table = DynamoDBItemTable(items: items, schema: try Self.schema(), preferredColumns: ["zeta", "missing"]) + #expect(table.columns == ["zeta", "missing", "pk", "sk", "alpha", "beta"]) + } + + @Test("A preferred column that is also a key is listed once, where it was preferred") + func preferredKeyIsNotRepeated() throws { + let items = [Self.item(["pk": .string("a"), "sk": .number("1"), "note": .string("n")])] + let table = DynamoDBItemTable(items: items, schema: try Self.schema(), preferredColumns: ["note", "sk"]) + #expect(table.columns == ["note", "sk", "pk"]) + } + + @Test("Every table key is a column even when no item carries it") + func allKeysIncludedByDefault() throws { + let table = DynamoDBItemTable(items: [Self.item(["other": .string("x")])], schema: try Self.schema()) + #expect(table.columns == ["pk", "sk", "other"]) + #expect(table.rows == [[.null, .null, .text("x")]]) + } + + @Test("With includeAllKeys off, only the keys some item carries are columns") + func onlyPresentKeysWhenNotIncludingAll() throws { + let items = [Self.item(["pk": .string("a"), "other": .string("x")])] + let table = DynamoDBItemTable(items: items, schema: try Self.schema(), includeAllKeys: false) + #expect(table.columns == ["pk", "other"]) + } + + @Test("An index key no item carries is not a column") + func indexKeysAreNotForced() throws { + let table = DynamoDBItemTable(items: [Self.item(["pk": .string("a")])], schema: try Self.schema()) + #expect(!table.columns.contains("status")) + } + + @Test("With no schema and no items the table is empty") + func emptyTable() { + let table = DynamoDBItemTable(items: [], schema: nil) + #expect(table.columns.isEmpty) + #expect(table.rows.isEmpty) + #expect(table.types.isEmpty) + } + + @Test("Each row holds its item's cells in column order, with a missing attribute as null") + func rowsFollowColumns() { + let items = [ + Self.item(["a": .string("x"), "b": .number("1.50")]), + Self.item(["b": .bool(true), "c": .map(["k": .number("2")])]) + ] + let table = DynamoDBItemTable(items: items, schema: nil) + #expect(table.columns == ["a", "b", "c"]) + #expect(table.rows == [ + [.text("x"), .text("1.50"), .null], + [.null, .text("true"), .text(#"{"k":2}"#)] + ]) + } + + @Test("A column takes the type most of its values have, ignoring NULL") + func majorityType() { + let items = [ + Self.item(["v": .number("1")]), + Self.item(["v": .string("x")]), + Self.item(["v": .number("2")]), + Self.item(["v": .null]), + Self.item(["v": .null]), + Self.item(["v": .null]) + ] + #expect(DynamoDBItemTable.majorityType(of: "v", in: items) == .number) + #expect(DynamoDBItemTable(items: items, schema: nil).types == [.number]) + } + + @Test("A column whose values are all NULL or missing has no type") + func nullOnlyColumnHasNoType() { + let items = [Self.item(["v": .null]), Self.item(["w": .string("x")])] + #expect(DynamoDBItemTable.majorityType(of: "v", in: items) == nil) + #expect(DynamoDBItemTable.majorityType(of: "absent", in: items) == nil) + } + + @Test("A tie between types resolves the same way whatever order the items arrive in") + func majorityTieIsDeterministic() { + let tied = [ + Self.item(["v": .string("x")]), + Self.item(["v": .number("1")]), + Self.item(["v": .map([:])]), + Self.item(["v": .string("y")]), + Self.item(["v": .number("2")]), + Self.item(["v": .map([:])]) + ] + let forward = DynamoDBItemTable.majorityType(of: "v", in: tied) + let backward = DynamoDBItemTable.majorityType(of: "v", in: tied.reversed()) + let rotated = DynamoDBItemTable.majorityType(of: "v", in: Array(tied[2...] + tied[..<2])) + #expect(forward != nil) + #expect(forward == backward) + #expect(forward == rotated) + #expect([DynamoDBAttributeType.string, .number, .map].contains(forward ?? .null)) + } + + @Test("Table key columns take their type from the schema, not from the items") + func keyColumnsTypedFromSchema() throws { + let items = [ + Self.item(["pk": .number("1"), "sk": .string("not a number")]), + Self.item(["pk": .number("2"), "sk": .string("still not")]) + ] + let table = DynamoDBItemTable(items: items, schema: try Self.schema()) + #expect(table.columns == ["pk", "sk"]) + #expect(table.types == [.string, .number]) + } + + @Test("A key column no item carries is still typed from the schema") + func absentKeyTypedFromSchema() throws { + let table = DynamoDBItemTable(items: [], schema: try Self.schema()) + #expect(table.columns == ["pk", "sk"]) + #expect(table.typeNames == ["String", "Number"]) + } + + @Test("typeNames uses display names and falls back to String for an untyped column") + func typeNamesFallBackToString() { + let items = [Self.item(["n": .numberSet(["1"]), "z": .null])] + let table = DynamoDBItemTable(items: items, schema: nil, preferredColumns: ["missing"]) + #expect(table.columns == ["missing", "n", "z"]) + #expect(table.typeNames == ["String", "Number Set", "String"]) + } + + @Test("observedTypes lists only the columns that have a type") + func observedTypesSkipUntyped() { + let items = [Self.item(["n": .number("1"), "z": .null])] + let table = DynamoDBItemTable(items: items, schema: nil) + #expect(table.observedTypes == ["n": .number]) + } + + @Test( + "Column metadata carries the type the app classifies the column by", + arguments: [ + ClassificationCase(value: .map(["a": .string("x")]), displayName: "Map", classification: "JSON"), + ClassificationCase(value: .list([.string("x")]), displayName: "List", classification: "JSON"), + ClassificationCase(value: .stringSet(["x"]), displayName: "String Set", classification: "JSON"), + ClassificationCase(value: .numberSet(["1"]), displayName: "Number Set", classification: "JSON"), + ClassificationCase(value: .binarySet([Data([1])]), displayName: "Binary Set", classification: "JSON"), + ClassificationCase(value: .number("1"), displayName: "Number", classification: "NUMERIC"), + ClassificationCase(value: .binary(Data([1])), displayName: "Binary", classification: "BLOB"), + ClassificationCase(value: .bool(true), displayName: "Boolean", classification: "BOOLEAN"), + ClassificationCase(value: .string("x"), displayName: "String", classification: "TEXT") + ] + ) + func columnMetaClassification(expected: ClassificationCase) throws { + let table = DynamoDBItemTable(items: [Self.item(["v": expected.value])], schema: nil) + let meta = try #require(table.columnMeta(schema: nil).first) + #expect(meta.name == "v") + #expect(meta.dataType == expected.displayName) + #expect(meta.classificationTypeName == expected.classification) + #expect(meta.typeNameForClassification == expected.classification) + } + + @Test("An untyped column's metadata reads as a String classified as TEXT") + func untypedColumnMeta() throws { + let table = DynamoDBItemTable(items: [Self.item(["v": .null])], schema: nil) + let meta = try #require(table.columnMeta(schema: nil).first) + #expect(meta.dataType == "String") + #expect(meta.classificationTypeName == "TEXT") + } + + @Test("Key columns are primary keys and not nullable, other columns the reverse") + func columnMetaMarksKeys() throws { + let schema = try Self.schema() + let items = [Self.item(["pk": .string("a"), "sk": .number("1"), "status": .string("open")])] + let meta = DynamoDBItemTable(items: items, schema: schema).columnMeta(schema: schema) + #expect(meta.map(\.name) == ["pk", "sk", "status"]) + #expect(meta.map(\.isPrimaryKey) == [true, true, false]) + #expect(meta.map(\.isNullable) == [false, false, true]) + #expect(meta.map(\.dataType) == ["String", "Number", "String"]) + #expect(meta.map(\.classificationTypeName) == ["TEXT", "NUMERIC", "TEXT"]) + } + + @Test("Numbers sort by value, not by text") + func sortsNumbersNumerically() { + let items = [ + Self.item(["id": .string("a"), "n": .number("10")]), + Self.item(["id": .string("b"), "n": .number("9")]), + Self.item(["id": .string("c"), "n": .number("-1")]), + Self.item(["id": .string("d"), "n": .number("1e1")]), + Self.item(["id": .string("e"), "n": .number("12345678901234567890123456789012345678")]) + ] + let ascending = DynamoDBItemTable.sorted(items, by: [DynamoDBOrderTerm(attribute: "n", descending: false)]) + #expect(Self.ids(ascending) == ["c", "b", "a", "d", "e"]) + let descending = DynamoDBItemTable.sorted(items, by: [DynamoDBOrderTerm(attribute: "n", descending: true)]) + #expect(Self.ids(descending) == ["e", "a", "d", "b", "c"]) + } + + @Test("Strings sort by their text") + func sortsText() { + let items = [ + Self.item(["id": .string("1"), "s": .string("b")]), + Self.item(["id": .string("2"), "s": .string("a")]), + Self.item(["id": .string("3"), "s": .string("C")]) + ] + let sorted = DynamoDBItemTable.sorted(items, by: [DynamoDBOrderTerm(attribute: "s", descending: false)]) + #expect(Self.ids(sorted) == ["3", "2", "1"]) + } + + @Test("Items missing the attribute sort first ascending and last descending") + func missingSortsFirst() { + let items = [ + Self.item(["id": .string("two"), "n": .number("2")]), + Self.item(["id": .string("none")]), + Self.item(["id": .string("one"), "n": .number("1")]) + ] + let ascending = DynamoDBItemTable.sorted(items, by: [DynamoDBOrderTerm(attribute: "n", descending: false)]) + #expect(Self.ids(ascending) == ["none", "one", "two"]) + let descending = DynamoDBItemTable.sorted(items, by: [DynamoDBOrderTerm(attribute: "n", descending: true)]) + #expect(Self.ids(descending) == ["two", "one", "none"]) + } + + @Test("Ties keep the order DynamoDB returned, in both directions") + func tiesKeepOrder() { + let items = (0..<8).map { index in + Self.item(["id": .string("\(index)"), "group": .number(index.isMultiple(of: 2) ? "1" : "1.0")]) + } + let ascending = DynamoDBItemTable.sorted(items, by: [DynamoDBOrderTerm(attribute: "group", descending: false)]) + let descending = DynamoDBItemTable.sorted(items, by: [DynamoDBOrderTerm(attribute: "group", descending: true)]) + let original = (0..<8).map { "\($0)" } + #expect(Self.ids(ascending) == original) + #expect(Self.ids(descending) == original) + } + + @Test("Later terms break ties left by earlier ones") + func multipleTerms() { + let items = [ + Self.item(["id": .string("a"), "g": .string("x"), "n": .number("1")]), + Self.item(["id": .string("b"), "g": .string("w"), "n": .number("5")]), + Self.item(["id": .string("c"), "g": .string("x"), "n": .number("3")]), + Self.item(["id": .string("d"), "g": .string("w"), "n": .number("2")]) + ] + let order = [ + DynamoDBOrderTerm(attribute: "g", descending: false), + DynamoDBOrderTerm(attribute: "n", descending: true) + ] + #expect(Self.ids(DynamoDBItemTable.sorted(items, by: order)) == ["b", "d", "c", "a"]) + } + + @Test("No order terms leaves the items as they were") + func noTermsKeepsOrder() { + let items = (0..<5).map { Self.item(["id": .string("\($0)")]) } + #expect(Self.ids(DynamoDBItemTable.sorted(items, by: [])) == ["0", "1", "2", "3", "4"]) + } + + @Test("A column mixing Numbers and Strings sorts the same whatever order the items arrive in") + func mixedTypesSortConsistently() { + let items = [ + Self.item(["id": .string("nine"), "v": .number("9")]), + Self.item(["id": .string("ten"), "v": .number("10")]), + Self.item(["id": .string("text"), "v": .string("5a")]) + ] + let order = [DynamoDBOrderTerm(attribute: "v", descending: false)] + let permutations = [[0, 1, 2], [0, 2, 1], [1, 0, 2], [1, 2, 0], [2, 0, 1], [2, 1, 0]] + let results = permutations.map { permutation in + Self.ids(DynamoDBItemTable.sorted(permutation.map { items[$0] }, by: order)) + } + #expect(Set(results).count == 1, "orders seen: \(results)") + let first = results.first ?? [] + let nine = first.firstIndex(of: "nine") ?? -1 + let ten = first.firstIndex(of: "ten") ?? -1 + #expect(nine < ten) + } +} diff --git a/TableProTests/Plugins/DynamoDB/DynamoDBJSONTests.swift b/TableProTests/Plugins/DynamoDB/DynamoDBJSONTests.swift new file mode 100644 index 0000000000..212f388742 --- /dev/null +++ b/TableProTests/Plugins/DynamoDB/DynamoDBJSONTests.swift @@ -0,0 +1,256 @@ +import Foundation +import TableProPluginKit +import Testing + +@Suite("DynamoDB JSON") +struct DynamoDBJSONTests { + struct NumberCase: Sendable, CustomTestStringConvertible { + let literal: String + var testDescription: String { literal } + } + + struct MalformedCase: Sendable, CustomTestStringConvertible { + let name: String + let text: String + var testDescription: String { name } + } + + @Test("An object with nested arrays, literals and strings parses into the matching tree") + func parsesObject() throws { + let json = try DynamoDBJSON.parse(#"{ "a" : 1, "b" : [true, false, null, "x"], "c" : {} }"#) + #expect(json == .object([ + "a": .number("1"), + "b": .array([.bool(true), .bool(false), .null, .string("x")]), + "c": .object([:]) + ])) + } + + @Test("A top-level array and an empty array parse") + func parsesArrays() throws { + #expect(try DynamoDBJSON.parse("[]") == .array([])) + #expect(try DynamoDBJSON.parse(" [ [1], [\"a\"] ] ") == .array([.array([.number("1")]), .array([.string("a")])])) + } + + @Test("String escapes decode to the characters they name") + func decodesEscapes() throws { + let json = try DynamoDBJSON.parse(#""line\nbreak \"quoted\" back\\slash \/ tab\t café""#) + #expect(json == .string("line\nbreak \"quoted\" back\\slash / tab\t caf\u{00E9}")) + } + + @Test("A surrogate pair escape decodes to one scalar outside the basic plane") + func decodesSurrogatePair() throws { + let json = try DynamoDBJSON.parse(#""😀""#) + #expect(json == .string("\u{1F600}")) + #expect(json.stringValue?.unicodeScalars.count == 1) + } + + @Test("A lone surrogate escape is rejected", arguments: [#""\ud83d""#, #""\ud83dx""#, #""\ude00""#]) + func rejectsLoneSurrogate(text: String) { + #expect(throws: DynamoDBJSON.ParseError.self) { try DynamoDBJSON.parse(text) } + } + + @Test("An unknown escape and a raw control character are rejected", arguments: [#""\x""#, "\"a\u{01}b\""]) + func rejectsBadStringContent(text: String) { + #expect(throws: DynamoDBJSON.ParseError.self) { try DynamoDBJSON.parse(text) } + } + + @Test( + "A number keeps the exact text it was written with", + arguments: [ + NumberCase(literal: "12345678901234567890123456789012345678"), + NumberCase(literal: "1.50"), + NumberCase(literal: "-0"), + NumberCase(literal: "1e-130"), + NumberCase(literal: "9.9999999999999999999999999999999999999E+125"), + NumberCase(literal: "0.000") + ] + ) + func keepsNumberText(number: NumberCase) throws { + #expect(try DynamoDBJSON.parse(number.literal) == .number(number.literal)) + let wrapped = try DynamoDBJSON.parse("{\"n\":\(number.literal)}") + #expect(wrapped["n"]?.numberText == number.literal) + } + + @Test("Text that is not a JSON number is rejected", arguments: ["01", "+1", ".5", "1.", "1e", "-", "NaN", "0x10"]) + func rejectsNonJSONNumbers(text: String) { + #expect(throws: DynamoDBJSON.ParseError.self) { try DynamoDBJSON.parse(text) } + } + + @Test("A key that appears twice in one object is rejected") + func rejectsDuplicateKey() { + #expect(throws: DynamoDBJSON.ParseError.duplicateKey("a")) { + try DynamoDBJSON.parse(#"{"a": 1, "b": 2, "a": 3}"#) + } + } + + @Test("The same key in two different objects is not a duplicate") + func allowsKeyInSiblingObjects() throws { + let json = try DynamoDBJSON.parse(#"[{"a": 1}, {"a": 2}]"#) + #expect(json == .array([.object(["a": .number("1")]), .object(["a": .number("2")])])) + } + + @Test("Text after the document is rejected with its position") + func rejectsTrailingText() { + #expect(throws: DynamoDBJSON.ParseError.trailingText(offset: 8)) { + try DynamoDBJSON.parse(#"{"a":1} x"#) + } + #expect(throws: DynamoDBJSON.ParseError.trailingText(offset: 3)) { + try DynamoDBJSON.parse("[1][2]") + } + } + + @Test("Whitespace around the document is allowed") + func allowsSurroundingWhitespace() throws { + #expect(try DynamoDBJSON.parse("\n\t {\"a\": true} \r\n") == .object(["a": .bool(true)])) + } + + @Test( + "Malformed documents are rejected", + arguments: [ + MalformedCase(name: "empty", text: ""), + MalformedCase(name: "unclosed object", text: #"{"a": 1"#), + MalformedCase(name: "unclosed array", text: "[1, 2"), + MalformedCase(name: "unclosed string", text: #""abc"#), + MalformedCase(name: "unquoted key", text: "{a: 1}"), + MalformedCase(name: "trailing comma", text: "[1, 2,]"), + MalformedCase(name: "misspelled literal", text: "tru"), + MalformedCase(name: "missing colon", text: #"{"a" 1}"#) + ] + ) + func rejectsMalformed(malformed: MalformedCase) { + #expect(throws: DynamoDBJSON.ParseError.self) { try DynamoDBJSON.parse(malformed.text) } + } + + @Test("Data that is not UTF-8 is rejected") + func rejectsInvalidUTF8() { + #expect(throws: DynamoDBJSON.ParseError.self) { try DynamoDBJSON.parse(Data([0x22, 0xFF, 0xFE, 0x22])) } + } + + @Test("A document nested 32 levels deep parses") + func parsesDynamoDBNestingDepth() throws { + let text = String(repeating: #"{"a":"#, count: 32) + "1" + String(repeating: "}", count: 32) + var json = try DynamoDBJSON.parse(text) + for _ in 0..<32 { + json = try #require(json["a"]) + } + #expect(json == .number("1")) + } + + @Test("A document nested far past the limit is rejected as too deep") + func rejectsExcessiveNesting() { + let text = String(repeating: "[", count: 200) + String(repeating: "]", count: 200) + #expect(throws: DynamoDBJSON.ParseError.tooDeep) { try DynamoDBJSON.parse(text) } + } + + @Test("A Scan page holding an item nested as deeply as DynamoDB allows parses") + func parsesResponseAtDynamoDBNestingLimit() throws { + var attribute = #"{"S":"leaf"}"# + for _ in 0..<31 { + attribute = #"{"M":{"child":"# + attribute + "}}" + } + let response = #"{"Count":1,"Items":[{"pk":{"S":"a"},"deep":"# + attribute + "}]}" + let json = try DynamoDBJSON.parse(response) + let items = try #require(json["Items"]?.arrayValue) + let item = try DynamoDBItem(wireItem: try #require(items.first)) + var value = try #require(item["deep"]) + for _ in 0..<31 { + guard case .map(let entries) = value else { + Issue.record("Expected a map, got \(value)") + return + } + value = try #require(entries["child"]) + } + #expect(value == .string("leaf")) + } + + @Test("parsePrefix returns the value and the text after it") + func parsePrefixReturnsRemainder() throws { + let result = try DynamoDBJSON.parsePrefix(#" {"a": [1, 2]} WHERE "pk" = 'x'"#) + #expect(result.value == .object(["a": .array([.number("1"), .number("2")])])) + #expect(result.remainder == #" WHERE "pk" = 'x'"#) + } + + @Test("parsePrefix of a complete document leaves no remainder") + func parsePrefixOfWholeDocument() throws { + let result = try DynamoDBJSON.parsePrefix("[1]") + #expect(result.value == .array([.number("1")])) + #expect(result.remainder.isEmpty) + } + + @Test("parsePrefix keeps text that follows directly after the value") + func parsePrefixKeepsAdjacentText() throws { + let result = try DynamoDBJSON.parsePrefix(#""café"rest"#) + #expect(result.value == .string("caf\u{00E9}")) + #expect(result.remainder == "rest") + } + + @Test("parsePrefix rejects text that does not start with JSON") + func parsePrefixRejectsNonJSON() { + #expect(throws: DynamoDBJSON.ParseError.self) { try DynamoDBJSON.parsePrefix("WHERE x = 1") } + } + + @Test("Serialized output sorts object keys at every level") + func serializedSortsKeys() { + let json = DynamoDBJSON.object([ + "b": .number("2"), + "a": .object(["z": .bool(true), "y": .null]), + "c": .array([.string("x")]) + ]) + #expect(json.serialized() == #"{"a":{"y":null,"z":true},"b":2,"c":["x"]}"#) + } + + @Test("Serialized output writes numbers exactly as they were given") + func serializedWritesNumbersVerbatim() { + let json = DynamoDBJSON.array([ + .number("12345678901234567890123456789012345678"), + .number("1.50"), + .number("-0"), + .number("1e-130"), + .number("1E+125") + ]) + #expect(json.serialized() == "[12345678901234567890123456789012345678,1.50,-0,1e-130,1E+125]") + } + + @Test("Serialized output escapes quotes, backslashes and control characters") + func serializedEscapesStrings() { + let json = DynamoDBJSON.string("a\"b\\c\nd\u{01}") + #expect(json.serialized() == #""a\"b\\c\nd\u0001""#) + } + + @Test("Serialized output parses back to the same value") + func serializedRoundTrips() throws { + let json = Self.sample + #expect(try DynamoDBJSON.parse(json.serialized()) == json) + #expect(try DynamoDBJSON.parse(json.serializedData) == json) + } + + @Test("Pretty output spans several lines and parses back to the same value") + func prettyOutputRoundTrips() throws { + let json = Self.sample + let pretty = json.serialized(pretty: true) + #expect(pretty.contains("\n")) + #expect(try DynamoDBJSON.parse(pretty) == json) + } + + @Test("Accessors return the payload of their own case only") + func accessorsMatchTheirCase() { + #expect(DynamoDBJSON.number("42").intValue == 42) + #expect(DynamoDBJSON.number("1.5").doubleValue == 1.5) + #expect(DynamoDBJSON.string("42").intValue == nil) + #expect(DynamoDBJSON.string("x").numberText == nil) + #expect(DynamoDBJSON.bool(true).boolValue == true) + #expect(DynamoDBJSON.null.boolValue == nil) + #expect(DynamoDBJSON.array([]).objectValue == nil) + #expect(DynamoDBJSON.array([.null])["key"] == nil) + } + + private static let sample = DynamoDBJSON.object([ + "text": .string("quote \" backslash \\ newline \n caf\u{00E9} \u{1F600}"), + "big": .number("12345678901234567890123456789012345678"), + "scaled": .number("1.50"), + "tiny": .number("1e-130"), + "negative zero": .number("-0"), + "flags": .array([.bool(true), .bool(false), .null]), + "nested": .object(["empty object": .object([:]), "empty array": .array([])]) + ]) +} diff --git a/TableProTests/Plugins/DynamoDB/DynamoDBLocalIntegrationTests.swift b/TableProTests/Plugins/DynamoDB/DynamoDBLocalIntegrationTests.swift new file mode 100644 index 0000000000..089a04f841 --- /dev/null +++ b/TableProTests/Plugins/DynamoDB/DynamoDBLocalIntegrationTests.swift @@ -0,0 +1,367 @@ +import Foundation +import TableProPluginKit +import Testing + +enum DynamoDBTestLocal { + static let host = "127.0.0.1" + static let port = 18_000 + + /// A socket probe rather than an environment variable: `xcodebuild` does not pass the shell's + /// environment to the test host, so a variable would read as unset on every run. + static let isReachable: Bool = { + let descriptor = socket(AF_INET, SOCK_STREAM, 0) + guard descriptor >= 0 else { return false } + defer { close(descriptor) } + var timeout = timeval(tv_sec: 1, tv_usec: 0) + setsockopt(descriptor, SOL_SOCKET, SO_SNDTIMEO, &timeout, socklen_t(MemoryLayout.size)) + var address = sockaddr_in() + address.sin_len = UInt8(MemoryLayout.size) + address.sin_family = sa_family_t(AF_INET) + address.sin_port = UInt16(port).bigEndian + guard inet_pton(AF_INET, host, &address.sin_addr) == 1 else { return false } + let result = withUnsafePointer(to: &address) { pointer in + pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { connect(descriptor, $0, socklen_t(MemoryLayout.size)) } + } + return result == 0 + }() + + static func connectedDriver() async throws -> DynamoDBPluginDriver { + let config = DriverConnectionConfig( + host: "", port: 0, username: "", password: "", database: "", + additionalFields: [ + "awsAuthMethod": "local", + "awsEndpointUrl": "http://\(host):\(port)", + "awsRegion": "us-east-1" + ] + ) + let driver = DynamoDBPluginDriver(config: config, catalog: DynamoDBCatalog()) + try await driver.connect() + return driver + } + + static func uniqueTableName(_ prefix: String) -> String { + "tp_it_\(prefix)_\(UUID().uuidString.prefix(8))" + } +} + +/// Orders across three customers, keyed by customer and order number, with a global index on +/// status and total. Order numbers divisible by four are FAILED. +private struct OrdersFixture { + static let columns = ["customer", "orderId", "status", "total", "zip", "doc", "tags", "bin", "big"] + static let keys = ["customer", "orderId"] + + let driver: DynamoDBPluginDriver + let table: String + + static func make() async throws -> OrdersFixture { + let driver = try await DynamoDBTestLocal.connectedDriver() + let table = DynamoDBTestLocal.uniqueTableName("orders") + let form = PluginCreateTableRequest( + tableName: table, + values: [ + "partitionKeyName": "customer", "partitionKeyType": "S", + "sortKeyName": "orderId", "sortKeyType": "N", + "billingMode": "PAY_PER_REQUEST", "tableClass": "STANDARD", "deletionProtection": "false" + ], + repeatedValues: ["globalIndexes": [[ + "indexName": "byStatus", "partitionKeyName": "status", "partitionKeyType": "S", + "sortKeyName": "total", "sortKeyType": "N", "projection": "ALL" + ]]] + ) + for statement in try driver.createTableStatements(for: form, schema: nil) { + _ = try await driver.execute(query: statement) + } + let fixture = OrdersFixture(driver: driver, table: table) + try await fixture.seed() + return fixture + } + + private func seed() async throws { + for customer in ["c1", "c2", "c3"] { + for order in 0..<40 { + let row: [PluginCellValue] = [ + .text(customer), .text(String(order)), .text(order % 4 == 0 ? "FAILED" : "PAID"), + .text(String(order * 10)), .text("0213\(order % 10)"), .text(#"{"a":{"b":[1,2]}}"#), + .text(#"["x","y"]"#), .bytes(Data([0xDE, 0xAD, UInt8(order)])), + .text("12345678901234567890123456789012345678") + ] + let insert = try #require(driver.generateIdentityPreservingInsert( + table: table, schema: nil, columns: Self.columns, primaryKeyColumns: Self.keys, rows: [row] + )?.first) + _ = try await driver.executeParameterized(query: insert.statement, parameters: insert.parameters) + } + } + } + + func tearDown() async { + _ = try? await driver.execute(query: #"DeleteTable {"TableName": "\#(table)"}"#) + driver.disconnect() + } + + func browse( + _ filters: [PluginQueryFilter], + logic: String = "and", + sort: [(columnIndex: Int, ascending: Bool)] = [], + columns: [String] = [], + limit: Int = 1_000, + offset: Int = 0, + kinds: [String: PluginColumnKind] = [:] + ) async throws -> PluginQueryResult { + let statement = try #require(driver.buildFilteredQuery( + table: table, schema: nil, queryFilters: filters, logicMode: logic, sortColumns: sort, + columns: columns, limit: limit, offset: offset, columnKinds: kinds + )) + return try #require(try await driver.executeBoundedQuery(query: statement, rowCap: 10_000)) + } + + func item(_ customer: String, _ order: Int) async throws -> [String: PluginCellValue] { + let result = try await driver.execute(query: #"GetItem {"TableName": "\#(table)", "Key": {"customer": {"S": "\#(customer)"}, "orderId": {"N": "\#(order)"}}}"#) + return Dictionary(uniqueKeysWithValues: zip(result.columns, result.rows.first ?? [])) + } +} + +private func text(_ cell: PluginCellValue?) -> String? { + guard case .text(let value) = cell else { return nil } + return value +} + +@Suite("DynamoDB Local integration", .enabled(if: DynamoDBTestLocal.isReachable), .serialized) +struct DynamoDBLocalIntegrationTests { + @Test("A grid insert keeps a zip code a String and a 38-digit value exact") + func insertKeepsTypes() async throws { + let fixture = try await OrdersFixture.make() + let row = try await fixture.item("c1", 4) + await fixture.tearDown() + #expect(text(row["zip"]) == "02134") + #expect(text(row["big"]) == "12345678901234567890123456789012345678") + #expect(row["bin"] == .bytes(Data([0xDE, 0xAD, 4]))) + } + + @Test("Pages cover every item exactly once without re-reading earlier pages") + func pagesCoverEveryItem() async throws { + let fixture = try await OrdersFixture.make() + var keys = Set() + var columns: [String] = [] + for page in 0..<3 { + let result = try await fixture.browse([], columns: columns, limit: 50, offset: page * 50) + columns = result.columns + for row in result.rows { keys.insert("\(row[0])|\(row[1])") } + } + await fixture.tearDown() + #expect(keys.count == 120) + #expect(Array(columns.prefix(2)) == ["customer", "orderId"]) + } + + @Test("A partition key with a sort key range becomes a Query sorted by the sort key") + func queryOnTableKeys() async throws { + let fixture = try await OrdersFixture.make() + let result = try await fixture.browse( + [PluginQueryFilter(column: "customer", op: "=", value: "c2"), PluginQueryFilter(column: "orderId", op: ">=", value: "30")], + sort: [(columnIndex: 1, ascending: false)], + columns: ["customer", "orderId"] + ) + await fixture.tearDown() + #expect(result.rows.compactMap { text($0[1]) } == (30...39).reversed().map(String.init)) + #expect(result.statusMessage?.contains("Query on the table") == true) + } + + @Test("A filter on a global index key queries that index") + func queryOnGlobalIndex() async throws { + let fixture = try await OrdersFixture.make() + let result = try await fixture.browse([ + PluginQueryFilter(column: "status", op: "=", value: "FAILED"), + PluginQueryFilter(column: "total", op: "BETWEEN", value: "100,200", secondValue: "200", elementScope: nil) + ]) + await fixture.tearDown() + #expect(result.rows.count == 9) + #expect(result.statusMessage?.contains("byStatus") == true) + } + + @Test("An OR that names a partition key still returns matches from other partitions") + func orKeepsOtherPartitions() async throws { + let fixture = try await OrdersFixture.make() + let result = try await fixture.browse( + [PluginQueryFilter(column: "customer", op: "=", value: "c1"), PluginQueryFilter(column: "status", op: "=", value: "FAILED")], + logic: "or" + ) + await fixture.tearDown() + #expect(result.rows.count == 60) + } + + struct OperatorCase: Sendable, CustomStringConvertible { + let op: String + let value: String + let expected: Int + var description: String { op } + } + + @Test("Every filter-bar operator matches what it names", arguments: [ + OperatorCase(op: "IS NULL", value: "", expected: 0), + OperatorCase(op: "IS NOT NULL", value: "", expected: 120), + OperatorCase(op: "IN", value: "FAILED, PAID", expected: 120), + OperatorCase(op: "NOT IN", value: "FAILED", expected: 90), + OperatorCase(op: "CONTAINS", value: "AIL", expected: 30), + OperatorCase(op: "NOT CONTAINS", value: "AIL", expected: 90), + OperatorCase(op: "STARTS WITH", value: "PA", expected: 90), + OperatorCase(op: "ENDS WITH", value: "ED", expected: 30), + OperatorCase(op: "REGEX", value: "^F.*D$", expected: 30), + OperatorCase(op: "!=", value: "PAID", expected: 30) + ]) + func operators(_ testCase: OperatorCase) async throws { + let fixture = try await OrdersFixture.make() + let result = try await fixture.browse([PluginQueryFilter(column: "status", op: testCase.op, value: testCase.value)]) + await fixture.tearDown() + #expect(result.rows.count == testCase.expected) + } + + @Test("Filtered pages fill up and cover every match") + func filteredPagesFill() async throws { + let fixture = try await OrdersFixture.make() + var keys = Set() + for page in 0..<5 { + let result = try await fixture.browse( + [PluginQueryFilter(column: "status", op: "=", value: "FAILED")], limit: 7, offset: page * 7 + ) + for row in result.rows { keys.insert("\(row[0])|\(row[1])") } + } + let exact = try await fixture.driver.fetchExactRowCount( + table: fixture.table, schema: nil, + queryFilters: [PluginQueryFilter(column: "status", op: "=", value: "FAILED")], logicMode: "and" + ) + await fixture.tearDown() + #expect(keys.count == 30) + #expect(exact == 30) + } + + @Test("An edit keeps each attribute's type, removes a cleared one, and refuses a stale edit") + func editKeepsTypes() async throws { + let fixture = try await OrdersFixture.make() + _ = try await fixture.driver.execute(query: #"UpdateItem {"TableName": "\#(fixture.table)", "Key": {"customer": {"S": "c1"}, "orderId": {"N": "4"}}, "UpdateExpression": "SET tags = :ss", "ExpressionAttributeValues": {":ss": {"SS": ["x", "y"]}}}"#) + let loaded = try await fixture.browse( + [PluginQueryFilter(column: "customer", op: "=", value: "c1"), PluginQueryFilter(column: "orderId", op: "=", value: "4")] + ) + let row = try #require(loaded.rows.first) + let index = { (name: String) in loaded.columns.firstIndex(of: name) ?? 0 } + let change = PluginRowChange( + rowIndex: 0, type: .update, + cellChanges: [ + (columnIndex: index("total"), columnName: "total", oldValue: row[index("total")], newValue: .text("999")), + (columnIndex: index("tags"), columnName: "tags", oldValue: row[index("tags")], newValue: .text(#"["x","y","z"]"#)), + (columnIndex: index("zip"), columnName: "zip", oldValue: row[index("zip")], newValue: .null) + ], + originalRow: row + ) + let statements = try #require(fixture.driver.generateStatements( + table: fixture.table, columns: loaded.columns, primaryKeyColumns: OrdersFixture.keys, changes: [change], + insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: [] + )) + for statement in statements { + _ = try await fixture.driver.executeParameterized(query: statement.statement, parameters: statement.parameters) + } + let after = try await fixture.driver.execute(query: #"GetItem {"TableName": "\#(fixture.table)", "Key": {"customer": {"S": "c1"}, "orderId": {"N": "4"}}}"#) + var staleError: Error? + do { + for statement in statements { + _ = try await fixture.driver.executeParameterized(query: statement.statement, parameters: statement.parameters) + } + } catch { + staleError = error + } + await fixture.tearDown() + let types = Dictionary(uniqueKeysWithValues: zip(after.columns, after.columnTypeNames)) + #expect(types["total"] == "Number") + #expect(types["tags"] == "String Set") + #expect(!after.columns.contains("zip")) + #expect(staleError?.localizedDescription.contains("changed after it was loaded") == true) + } + + @Test("A delete reports one item, then none") + func deleteReportsCount() async throws { + let fixture = try await OrdersFixture.make() + let loaded = try await fixture.browse( + [PluginQueryFilter(column: "customer", op: "=", value: "c3"), PluginQueryFilter(column: "orderId", op: "=", value: "7")] + ) + let original = try #require(loaded.rows.first) + let change = PluginRowChange(rowIndex: 0, type: .delete, cellChanges: [], originalRow: original) + let statement = try #require(fixture.driver.generateStatements( + table: fixture.table, columns: loaded.columns, primaryKeyColumns: OrdersFixture.keys, changes: [change], + insertedRowData: [:], deletedRowIndices: [0], insertedRowIndices: [] + )?.first) + let first = try await fixture.driver.executeParameterized(query: statement.statement, parameters: statement.parameters) + let second = try await fixture.driver.executeParameterized(query: statement.statement, parameters: statement.parameters) + await fixture.tearDown() + #expect(first.rowsAffected == 1) + #expect(second.rowsAffected == 0) + } + + @Test("A stream carries an attribute that only the last item has") + func streamKeepsLateAttributes() async throws { + let fixture = try await OrdersFixture.make() + _ = try await fixture.driver.execute(query: #"UpdateItem {"TableName": "\#(fixture.table)", "Key": {"customer": {"S": "c3"}, "orderId": {"N": "39"}}, "UpdateExpression": "SET late = :v", "ExpressionAttributeValues": {":v": {"S": "only here"}}}"#) + var columns: [String] = [] + var rows = 0 + for try await element in fixture.driver.streamRows(query: try #require(fixture.driver.defaultExportQuery(table: fixture.table))) { + switch element { + case .header(let header): columns = header.columns + case .rows(let batch): rows += batch.count + } + } + await fixture.tearDown() + #expect(columns.contains("late")) + #expect(rows == 120) + } + + @Test("PartiQL reads every page, honours LIMIT, and sorts a complete result") + func partiQLReads() async throws { + let fixture = try await OrdersFixture.make() + let limited = try await fixture.driver.executeBoundedQuery( + query: #"SELECT * FROM "\#(fixture.table)" WHERE status = 'FAILED' LIMIT 5"#, rowCap: 1_000 + ) + let projected = try await fixture.driver.executeBoundedQuery( + query: #"SELECT customer FROM "\#(fixture.table)" WHERE status = 'FAILED'"#, rowCap: 1_000 + ) + let sorted = try await fixture.driver.executeBoundedQuery( + query: #"SELECT * FROM "\#(fixture.table)" WHERE customer = 'c2' ORDER BY "total" DESC"#, rowCap: 1_000 + ) + await fixture.tearDown() + #expect(limited?.rows.count == 5) + #expect(projected?.rows.count == 30) + #expect(projected?.columns == ["customer"]) + let totalIndex = try #require(sorted?.columns.firstIndex(of: "total")) + let totals = (sorted?.rows ?? []).compactMap { text($0[totalIndex]).flatMap(Int.init) } + #expect(totals.count == 40) + #expect(totals == totals.sorted(by: >)) + } + + @Test("Structure: executable DDL, indexes, nested paths, and a new global index") + func structure() async throws { + let fixture = try await OrdersFixture.make() + let ddl = try await fixture.driver.fetchTableDDL(table: fixture.table, schema: nil) + let indexes = try await fixture.driver.fetchIndexes(table: fixture.table, schema: nil) + let paths = try await fixture.driver.sampleFieldPaths(table: fixture.table, schema: nil, limit: 20) + let addIndex = try #require(fixture.driver.generateAddIndexSQL( + table: fixture.table, index: PluginIndexDefinition(name: "byZip", columns: ["zip"], isUnique: false) + )) + _ = try await fixture.driver.execute(query: addIndex) + let afterAdd = try await fixture.driver.fetchIndexes(table: fixture.table, schema: nil) + await fixture.tearDown() + #expect(ddl.hasPrefix("CreateTable {")) + #expect(indexes.map(\.name) == ["PRIMARY", "byStatus"]) + #expect(paths.contains { $0.path == "doc.a.b" }) + #expect(afterAdd.contains { $0.name == "byZip" }) + } + + @Test("A batch with one failing statement reports the failure, not success") + func batchReportsPartialFailure() async throws { + let fixture = try await OrdersFixture.make() + let statement = #"BatchExecuteStatement {"Statements": [{"Statement": "INSERT INTO \"\#(fixture.table)\" VALUE {'customer': 'c9', 'orderId': 1}"}, {"Statement": "INSERT INTO \"\#(fixture.table)\" VALUE {'customer': 'c1', 'orderId': 1}"}]}"# + var failure: Error? + do { + _ = try await fixture.driver.execute(query: statement) + } catch { + failure = error + } + await fixture.tearDown() + #expect(failure?.localizedDescription.contains("1 of 2") == true) + } +} diff --git a/TableProTests/Plugins/DynamoDB/DynamoDBNumberTests.swift b/TableProTests/Plugins/DynamoDB/DynamoDBNumberTests.swift new file mode 100644 index 0000000000..826d64fd94 --- /dev/null +++ b/TableProTests/Plugins/DynamoDB/DynamoDBNumberTests.swift @@ -0,0 +1,170 @@ +import Foundation +import TableProPluginKit +import Testing + +@Suite("DynamoDB number") +struct DynamoDBNumberTests { + struct PartsCase: Sendable, CustomTestStringConvertible { + let text: String + let isNegative: Bool + let digits: [UInt8] + let leadingExponent: Int + var testDescription: String { text } + } + + struct ComparisonCase: Sendable, CustomTestStringConvertible { + let lhs: String + let rhs: String + let expected: ComparisonResult + var testDescription: String { "\(lhs) vs \(rhs)" } + } + + private static let largestNumber = "9." + String(repeating: "9", count: 37) + "E+125" + + @Test("Thirty-eight significant digits are valid and thirty-nine are not") + func significantDigitLimit() { + let thirtyEight = "12345678901234567890123456789012345678" + let thirtyNine = thirtyEight + "9" + #expect(DynamoDBNumber.isValid(thirtyEight)) + #expect(!DynamoDBNumber.isValid(thirtyNine)) + #expect(!DynamoDBNumber.isValid("-" + thirtyNine)) + #expect(!DynamoDBNumber.isValid("0." + thirtyNine)) + } + + @Test("Leading and trailing zeros are not significant digits") + func zerosAreNotSignificant() { + let thirtyEight = "12345678901234567890123456789012345678" + #expect(DynamoDBNumber.isValid(thirtyEight + "000")) + #expect(DynamoDBNumber.isValid("000" + thirtyEight)) + #expect(DynamoDBNumber.isValid("0.000" + thirtyEight)) + #expect(DynamoDBNumber.isValid("1.2345678901234567890123456789012345678000")) + } + + @Test("The magnitude limits are 1E-130 and below 1E+126") + func magnitudeLimits() { + #expect(DynamoDBNumber.isValid(Self.largestNumber)) + #expect(DynamoDBNumber.isValid("-" + Self.largestNumber)) + #expect(!DynamoDBNumber.isValid("1E+126")) + #expect(!DynamoDBNumber.isValid("-1E+126")) + #expect(!DynamoDBNumber.isValid("10E+125")) + #expect(DynamoDBNumber.isValid("1E-130")) + #expect(DynamoDBNumber.isValid("-1E-130")) + #expect(DynamoDBNumber.isValid("1.5E-130")) + #expect(!DynamoDBNumber.isValid("1E-131")) + #expect(!DynamoDBNumber.isValid("0.1E-130")) + #expect(DynamoDBNumber.isValid("0.0001e129")) + #expect(!DynamoDBNumber.isValid("0.0001e130")) + } + + @Test( + "Forms DynamoDB accepts are valid", + arguments: ["0", "-0.000", ".5", "5.", "+5", "007.50e1", "-12.5", "1e2", "1E+2", "2e-3", " 42 "] + ) + func acceptsNumberForms(text: String) { + #expect(DynamoDBNumber.isValid(text)) + } + + @Test( + "Text that is not a number is invalid", + arguments: ["", " ", "abc", "1e", "e5", "--1", "+-1", "NaN", "Infinity", ".", "1.2.3", "1,5", "0x10", "1e+-5", "１"] + ) + func rejectsNonNumbers(text: String) { + #expect(!DynamoDBNumber.isValid(text)) + #expect(DynamoDBNumber.parts(of: text) == nil) + } + + @Test("An exponent at the edge of Int is rejected rather than overflowing") + func extremeExponentIsInvalid() { + #expect(!DynamoDBNumber.isValid("10e9223372036854775807")) + #expect(!DynamoDBNumber.isValid("0.01e-9223372036854775808")) + #expect(!DynamoDBNumber.isValid("1e99999999999999999999")) + } + + @Test( + "parts splits the sign, the significant digits and the power of ten of the first digit", + arguments: [ + PartsCase(text: "0", isNegative: false, digits: [], leadingExponent: 0), + PartsCase(text: "-0.000", isNegative: false, digits: [], leadingExponent: 0), + PartsCase(text: "-12.340", isNegative: true, digits: [1, 2, 3, 4], leadingExponent: 1), + PartsCase(text: "0.00123", isNegative: false, digits: [1, 2, 3], leadingExponent: -3), + PartsCase(text: "007.50e1", isNegative: false, digits: [7, 5], leadingExponent: 1), + PartsCase(text: ".5", isNegative: false, digits: [5], leadingExponent: -1), + PartsCase(text: "5.", isNegative: false, digits: [5], leadingExponent: 0), + PartsCase(text: "+5", isNegative: false, digits: [5], leadingExponent: 0), + PartsCase(text: "1E-130", isNegative: false, digits: [1], leadingExponent: -130), + PartsCase(text: "12000", isNegative: false, digits: [1, 2], leadingExponent: 4) + ] + ) + func splitsParts(expected: PartsCase) { + #expect( + DynamoDBNumber.parts(of: expected.text) + == DynamoDBNumber.Parts( + isNegative: expected.isNegative, + significantDigits: expected.digits, + leadingExponent: expected.leadingExponent + ) + ) + } + + @Test( + "compare orders by numeric value", + arguments: [ + ComparisonCase(lhs: "-5", rhs: "3", expected: .orderedAscending), + ComparisonCase(lhs: "3", rhs: "-5", expected: .orderedDescending), + ComparisonCase(lhs: "-1", rhs: "0", expected: .orderedAscending), + ComparisonCase(lhs: "0", rhs: "1e-130", expected: .orderedAscending), + ComparisonCase(lhs: "-1e-130", rhs: "0", expected: .orderedAscending), + ComparisonCase(lhs: "1e2", rhs: "99", expected: .orderedDescending), + ComparisonCase(lhs: "9.9e1", rhs: "1e2", expected: .orderedAscending), + ComparisonCase(lhs: "-1e2", rhs: "-99", expected: .orderedAscending), + ComparisonCase(lhs: "0.001", rhs: "0.01", expected: .orderedAscending), + ComparisonCase(lhs: "9", rhs: "10", expected: .orderedAscending), + ComparisonCase(lhs: "1.23", rhs: "1.2", expected: .orderedDescending), + ComparisonCase(lhs: "-1.23", rhs: "-1.2", expected: .orderedAscending), + ComparisonCase( + lhs: "12345678901234567890123456789012345678", + rhs: "12345678901234567890123456789012345677", + expected: .orderedDescending + ), + ComparisonCase(lhs: "1.0", rhs: "1", expected: .orderedSame), + ComparisonCase(lhs: "1e2", rhs: "100", expected: .orderedSame), + ComparisonCase(lhs: "100", rhs: "1E+2", expected: .orderedSame), + ComparisonCase(lhs: "-0", rhs: "0.000", expected: .orderedSame), + ComparisonCase(lhs: "0.5", rhs: ".5", expected: .orderedSame), + ComparisonCase(lhs: "+5", rhs: "5.", expected: .orderedSame), + ComparisonCase(lhs: "007.50e1", rhs: "75", expected: .orderedSame) + ] + ) + func comparesNumerically(comparison: ComparisonCase) { + #expect(DynamoDBNumber.compare(comparison.lhs, comparison.rhs) == comparison.expected) + } + + @Test( + "compare is antisymmetric", + arguments: [ + ComparisonCase(lhs: "-5", rhs: "3", expected: .orderedAscending), + ComparisonCase(lhs: "1e2", rhs: "99", expected: .orderedDescending), + ComparisonCase(lhs: "1.0", rhs: "1", expected: .orderedSame) + ] + ) + func compareIsAntisymmetric(comparison: ComparisonCase) { + let forward = DynamoDBNumber.compare(comparison.lhs, comparison.rhs) + let backward = DynamoDBNumber.compare(comparison.rhs, comparison.lhs) + #expect(forward.rawValue == -backward.rawValue) + } + + @Test("areEqual treats different spellings of one value as equal") + func areEqualIgnoresSpelling() { + #expect(DynamoDBNumber.areEqual("1.0", "1")) + #expect(DynamoDBNumber.areEqual("1e2", "100")) + #expect(DynamoDBNumber.areEqual("-0", "0")) + #expect(!DynamoDBNumber.areEqual("1", "1.0000000000000000000000000000000000001")) + } + + @Test("Text that is not a number compares as text") + func unparseableComparesAsText() { + #expect(DynamoDBNumber.compare("abc", "abd") == .orderedAscending) + #expect(DynamoDBNumber.compare("abc", "abc") == .orderedSame) + #expect(DynamoDBNumber.compare("b", "a") == .orderedDescending) + } +} diff --git a/TableProTests/Plugins/DynamoDB/DynamoDBParameterBinderTests.swift b/TableProTests/Plugins/DynamoDB/DynamoDBParameterBinderTests.swift new file mode 100644 index 0000000000..a30e7b0cb8 --- /dev/null +++ b/TableProTests/Plugins/DynamoDB/DynamoDBParameterBinderTests.swift @@ -0,0 +1,243 @@ +import Foundation +import TableProPluginKit +import Testing + +@Suite("DynamoDB parameter binder") +struct DynamoDBParameterBinderTests { + struct KeyCase: Sendable, CustomTestStringConvertible { + let role: DynamoDBPartiQL.ParameterRole + let cell: PluginCellValue + let expected: DynamoDBAttributeValue + + var testDescription: String { "\(role) \(cell)" } + } + + struct MissingKeyCase: Sendable, CustomTestStringConvertible { + let role: DynamoDBPartiQL.ParameterRole + let cell: PluginCellValue + + var testDescription: String { "\(role) \(cell)" } + } + + static let digest = Data([0x00, 0x01, 0x02]) + + static let keyCases: [KeyCase] = [ + KeyCase(role: .compared(DynamoDBAttributePath(attribute: "sk")), cell: "7", expected: .number("7")), + KeyCase(role: .compared(DynamoDBAttributePath(attribute: "sk")), cell: " -1.5E3 ", expected: .number("-1.5E3")), + KeyCase(role: .inserted("sk"), cell: "42", expected: .number("42")), + KeyCase(role: .compared(DynamoDBAttributePath(attribute: "pk")), cell: "7", expected: .string("7")), + KeyCase(role: .inserted("pk"), cell: "02134", expected: .string("02134")), + KeyCase(role: .compared(DynamoDBAttributePath(attribute: "rank")), cell: "3", expected: .number("3")), + KeyCase(role: .compared(DynamoDBAttributePath(attribute: "digest")), cell: .bytes(digest), expected: .binary(digest)), + KeyCase(role: .compared(DynamoDBAttributePath(attribute: "digest")), cell: "AAEC", expected: .binary(digest)), + KeyCase(role: .inserted("digest"), cell: .bytes(digest), expected: .binary(digest)) + ] + + static let missingKeyCases: [MissingKeyCase] = [ + MissingKeyCase(role: .compared(DynamoDBAttributePath(attribute: "pk")), cell: .null), + MissingKeyCase(role: .compared(DynamoDBAttributePath(attribute: "pk")), cell: ""), + MissingKeyCase(role: .compared(DynamoDBAttributePath(attribute: "pk")), cell: "__DEFAULT__"), + MissingKeyCase(role: .inserted("pk"), cell: .null), + MissingKeyCase(role: .inserted("pk"), cell: ""), + MissingKeyCase(role: .inserted("sk"), cell: "__DEFAULT__"), + MissingKeyCase(role: .inserted("rank"), cell: "") + ] + + private static func schema() throws -> DynamoDBTableSchema { + try DynamoDBTableSchema(describeTableResponse: DynamoDBJSON.parse(""" + {"Table": { + "TableName": "Orders", + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"} + ], + "AttributeDefinitions": [ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "N"}, + {"AttributeName": "digest", "AttributeType": "B"}, + {"AttributeName": "rank", "AttributeType": "N"} + ], + "GlobalSecondaryIndexes": [{ + "IndexName": "ByDigest", + "KeySchema": [ + {"AttributeName": "digest", "KeyType": "HASH"}, + {"AttributeName": "rank", "KeyType": "RANGE"} + ], + "Projection": {"ProjectionType": "ALL"} + }] + }} + """)) + } + + private func binder( + observedTypes: [String: DynamoDBAttributeType] = [:], + currentItem: DynamoDBItem? = nil + ) throws -> DynamoDBParameterBinder { + DynamoDBParameterBinder(schema: try Self.schema(), observedTypes: observedTypes, currentItem: currentItem) + } + + private func bound( + _ cell: PluginCellValue, + as role: DynamoDBPartiQL.ParameterRole, + by binder: DynamoDBParameterBinder + ) throws -> DynamoDBAttributeValue { + let values = try binder.bind([cell], roles: [role]) + #expect(values.count == 1) + return try #require(values.first) + } + + private func refusedAttribute( + _ cell: PluginCellValue, + as role: DynamoDBPartiQL.ParameterRole, + by binder: DynamoDBParameterBinder + ) -> String? { + do { + _ = try binder.bind([cell], roles: [role]) + return nil + } catch DynamoDBError.invalidValue(let attribute, _) { + return attribute + } catch { + return nil + } + } + + // MARK: - Keys + + @Test("A key attribute takes the type its schema declares", arguments: keyCases) + func keyIsTypedFromSchema(keyCase: KeyCase) throws { + let observedAsString = Dictionary(uniqueKeysWithValues: ["pk", "sk", "digest", "rank"].map { + ($0, DynamoDBAttributeType.string) + }) + let binder = try binder( + observedTypes: observedAsString, + currentItem: ["sk": .string("x"), "rank": .string("y")] + ) + #expect(try bound(keyCase.cell, as: keyCase.role, by: binder) == keyCase.expected) + } + + @Test("A Number key that is not a number is refused", arguments: ["seven", "1,5", "NaN", "1e200"]) + func invalidNumberKeyIsRefused(text: String) throws { + #expect(refusedAttribute(.text(text), as: .compared(DynamoDBAttributePath(attribute: "sk")), by: try binder()) == "sk") + } + + @Test("A key with no value is refused", arguments: missingKeyCases) + func missingKeyIsRefused(missingKeyCase: MissingKeyCase) throws { + let attribute: String + switch missingKeyCase.role { + case .compared(let path), .assigned(let path): + attribute = path.root + case .inserted(let name): + attribute = name + case .unknown: + attribute = "" + } + #expect(refusedAttribute(missingKeyCase.cell, as: missingKeyCase.role, by: try binder()) == attribute) + } + + @Test("Bytes are refused for a key that is not Binary") + func bytesForNonBinaryKeyAreRefused() throws { + let binder = try binder() + #expect(refusedAttribute(.bytes(Self.digest), as: .compared(DynamoDBAttributePath(attribute: "pk")), by: binder) == "pk") + #expect(refusedAttribute(.bytes(Self.digest), as: .inserted("sk"), by: binder) == "sk") + } + + @Test("A Binary key that is not base64 is refused") + func invalidBinaryKeyIsRefused() throws { + #expect(refusedAttribute("not base64!", as: .compared(DynamoDBAttributePath(attribute: "digest")), by: try binder()) == "digest") + } + + @Test("Assigning a table key is refused", arguments: ["pk", "sk"]) + func assigningKeyIsRefused(attribute: String) throws { + #expect(refusedAttribute("new", as: .assigned(DynamoDBAttributePath(attribute: attribute)), by: try binder()) == attribute) + } + + @Test("An assigned index key takes the type its schema declares") + func assignedIndexKeyIsTypedFromSchema() throws { + #expect(try bound("5", as: .assigned(DynamoDBAttributePath(attribute: "rank")), by: try binder()) == .number("5")) + } + + // MARK: - Other attributes + + @Test("Assigned and compared attributes take the type of the item's current value") + func currentValueDecidesType() throws { + let binder = try binder( + observedTypes: ["total": .string, "tags": .string, "active": .string], + currentItem: ["total": .number("10"), "tags": .stringSet(["a"]), "active": .bool(true)] + ) + #expect(try bound("12", as: .assigned(DynamoDBAttributePath(attribute: "total")), by: binder) == .number("12")) + #expect(try bound("10", as: .compared(DynamoDBAttributePath(attribute: "total")), by: binder) == .number("10")) + #expect(try bound("[\"b\", \"c\"]", as: .assigned(DynamoDBAttributePath(attribute: "tags")), by: binder) == .stringSet(["b", "c"])) + #expect(try bound("false", as: .compared(DynamoDBAttributePath(attribute: "active")), by: binder) == .bool(false)) + } + + @Test("Without a current value, the type observed for the column is used") + func observedTypeDecidesType() throws { + let binder = try binder( + observedTypes: ["total": .number, "flag": .boolean], + currentItem: ["other": .string("x")] + ) + #expect(try bound("12", as: .assigned(DynamoDBAttributePath(attribute: "total")), by: binder) == .number("12")) + #expect(try bound("12", as: .compared(DynamoDBAttributePath(attribute: "total")), by: binder) == .number("12")) + #expect(try bound("true", as: .inserted("flag"), by: binder) == .bool(true)) + } + + @Test("With no type anywhere, text is a String even when it looks like a number") + func untypedTextIsString() throws { + let binder = try binder() + #expect(try bound("02134", as: .assigned(DynamoDBAttributePath(attribute: "zip")), by: binder) == .string("02134")) + #expect(try bound("12", as: .compared(DynamoDBAttributePath(attribute: "count")), by: binder) == .string("12")) + #expect(try bound("true", as: .inserted("flag"), by: binder) == .string("true")) + } + + @Test("Text that does not fit the current value's type is refused") + func mismatchedTextIsRefused() throws { + let binder = try binder(currentItem: ["total": .number("10")]) + #expect(refusedAttribute("ten", as: .assigned(DynamoDBAttributePath(attribute: "total")), by: binder) == "total") + } + + @Test("A NULL cell for a plain attribute binds as NULL and bytes as Binary") + func nullAndBytesForPlainAttribute() throws { + let binder = try binder(currentItem: ["total": .number("10")]) + #expect(try bound(.null, as: .assigned(DynamoDBAttributePath(attribute: "total")), by: binder) == .null) + #expect(try bound(.bytes(Self.digest), as: .assigned(DynamoDBAttributePath(attribute: "total")), by: binder) == .binary(Self.digest)) + } + + @Test("Without a schema a key name gets no key typing") + func noSchemaMeansNoKeyTyping() throws { + let binder = DynamoDBParameterBinder(schema: nil, observedTypes: [:], currentItem: nil) + #expect(try bound("7", as: .compared(DynamoDBAttributePath(attribute: "sk")), by: binder) == .string("7")) + #expect(try bound("7", as: .assigned(DynamoDBAttributePath(attribute: "sk")), by: binder) == .string("7")) + } + + // MARK: - Unknown role + + @Test("A parameter of unknown role binds text as S, bytes as B and NULL as NULL") + func unknownRole() throws { + let binder = try binder(observedTypes: ["sk": .number], currentItem: ["sk": .number("1")]) + #expect(try bound("42", as: .unknown, by: binder) == .string("42")) + #expect(try bound("", as: .unknown, by: binder) == .string("")) + #expect(try bound(.bytes(Self.digest), as: .unknown, by: binder) == .binary(Self.digest)) + #expect(try bound(.null, as: .unknown, by: binder) == .null) + } + + @Test("Parameters beyond the roles read from the statement bind as unknown") + func extraParametersAreUnknown() throws { + let values = try binder().bind(["7", "8", .null], roles: [.compared(DynamoDBAttributePath(attribute: "sk"))]) + #expect(values == [.number("7"), .string("8"), .null]) + } + + @Test("Parameters bind in order against their own roles") + func parametersBindInOrder() throws { + let binder = try binder(currentItem: ["total": .number("10")]) + let values = try binder.bind( + ["11", "p1", "7", "10"], + roles: [ + .assigned(DynamoDBAttributePath(attribute: "total")), + .compared(DynamoDBAttributePath(attribute: "pk")), + .compared(DynamoDBAttributePath(attribute: "sk")), + .compared(DynamoDBAttributePath(attribute: "total")) + ] + ) + #expect(values == [.number("11"), .string("p1"), .number("7"), .number("10")]) + } +} diff --git a/TableProTests/Plugins/DynamoDB/DynamoDBPartiQLTests.swift b/TableProTests/Plugins/DynamoDB/DynamoDBPartiQLTests.swift new file mode 100644 index 0000000000..2ae6cffce1 --- /dev/null +++ b/TableProTests/Plugins/DynamoDB/DynamoDBPartiQLTests.swift @@ -0,0 +1,403 @@ +import Foundation +import TableProPluginKit +import Testing + +@Suite("DynamoDB PartiQL reading") +struct DynamoDBPartiQLTests { + struct KindCase: Sendable, CustomTestStringConvertible { + let statement: String + let kind: DynamoDBPartiQL.Kind + + var testDescription: String { statement } + } + + struct TargetCase: Sendable, CustomTestStringConvertible { + let statement: String + let table: String + let index: String? + + var testDescription: String { statement } + } + + struct SplitCase: Sendable, CustomTestStringConvertible { + let statement: String + let remaining: String + let window: DynamoDBReadWindow + + var testDescription: String { statement } + } + + struct RolesCase: Sendable, CustomTestStringConvertible { + let statement: String + let roles: [DynamoDBPartiQL.ParameterRole] + + var testDescription: String { statement } + } + + static let kindCases: [KindCase] = [ + KindCase(statement: "SELECT * FROM \"Orders\"", kind: .select), + KindCase(statement: " select * from Orders", kind: .select), + KindCase(statement: "-- note\nINSERT INTO \"Orders\" VALUE {'pk': 'a'}", kind: .insert), + KindCase(statement: "/* edit */ Update \"Orders\" SET \"a\" = 1 WHERE \"pk\" = 'a'", kind: .update), + KindCase(statement: "delete from \"Orders\" where \"pk\" = 'a'", kind: .delete), + KindCase(statement: "EXISTS(SELECT * FROM \"Orders\" WHERE \"pk\" = 'a')", kind: .other), + KindCase(statement: "\"SELECT\" * FROM \"Orders\"", kind: .other), + KindCase(statement: "'SELECT'", kind: .other), + KindCase(statement: "", kind: .other) + ] + + static let targetCases: [TargetCase] = [ + TargetCase(statement: "SELECT * FROM \"Orders\"", table: "Orders", index: nil), + TargetCase( + statement: "SELECT * FROM \"Orders\".\"ByCustomer\" WHERE \"customer\" = 'c1'", + table: "Orders", index: "ByCustomer" + ), + TargetCase( + statement: "SELECT \"a\" FROM \"My \"\"Quoted\"\" Table\".\"By \"\"X\"\"\"", + table: "My \"Quoted\" Table", index: "By \"X\"" + ), + TargetCase(statement: "select * from orders.bycustomer", table: "orders", index: "bycustomer"), + TargetCase(statement: "SELECT * FROM \"Orders\" WHERE \"x\" = 'FROM \"Other\"'", table: "Orders", index: nil), + TargetCase(statement: "INSERT INTO \"Or\"\"ders\" VALUE {'pk': 'a'}", table: "Or\"ders", index: nil), + TargetCase(statement: "UPDATE \"Orders\" SET \"a\" = 1 WHERE \"pk\" = 'a'", table: "Orders", index: nil), + TargetCase(statement: "UPDATE \"a\"\"b\" REMOVE \"c\" WHERE \"pk\" = 'a'", table: "a\"b", index: nil), + TargetCase(statement: "DELETE FROM \"Orders\" WHERE \"pk\" = 'a'", table: "Orders", index: nil), + TargetCase(statement: "-- which table\nDELETE FROM \"Orders\" WHERE \"pk\" = 'a'", table: "Orders", index: nil) + ] + + static let splitCases: [SplitCase] = [ + SplitCase( + statement: "SELECT * FROM \"Orders\" LIMIT 10", + remaining: "SELECT * FROM \"Orders\"", + window: DynamoDBReadWindow(limit: 10) + ), + SplitCase( + statement: "SELECT * FROM \"Orders\" LIMIT 10 OFFSET 20", + remaining: "SELECT * FROM \"Orders\"", + window: DynamoDBReadWindow(limit: 10, offset: 20) + ), + SplitCase( + statement: "SELECT * FROM \"Orders\" OFFSET 20", + remaining: "SELECT * FROM \"Orders\"", + window: DynamoDBReadWindow(offset: 20) + ), + SplitCase( + statement: "SELECT * FROM \"Orders\" WHERE \"pk\" = 'a' ORDER BY \"sk\" DESC", + remaining: "SELECT * FROM \"Orders\" WHERE \"pk\" = 'a'", + window: DynamoDBReadWindow(order: [DynamoDBOrderTerm(attribute: "sk", descending: true)]) + ), + SplitCase( + statement: "SELECT * FROM \"Orders\" ORDER BY \"a\" DESC, b, \"c\" ASC LIMIT 5 OFFSET 1", + remaining: "SELECT * FROM \"Orders\"", + window: DynamoDBReadWindow( + order: [ + DynamoDBOrderTerm(attribute: "a", descending: true), + DynamoDBOrderTerm(attribute: "b", descending: false), + DynamoDBOrderTerm(attribute: "c", descending: false) + ], + limit: 5, + offset: 1 + ) + ), + SplitCase( + statement: "select * from \"Orders\" order by \"a\"\"b\" asc limit 3", + remaining: "select * from \"Orders\"", + window: DynamoDBReadWindow(order: [DynamoDBOrderTerm(attribute: "a\"b", descending: false)], limit: 3) + ), + SplitCase( + statement: "SELECT * FROM \"Orders\" WHERE \"a\" IN [1, 2] LIMIT 7", + remaining: "SELECT * FROM \"Orders\" WHERE \"a\" IN [1, 2]", + window: DynamoDBReadWindow(limit: 7) + ), + SplitCase( + statement: "SELECT * FROM \"Orders\" WHERE \"a\" IN (SELECT \"b\" FROM \"Other\" ORDER BY \"b\")", + remaining: "SELECT * FROM \"Orders\" WHERE \"a\" IN (SELECT \"b\" FROM \"Other\" ORDER BY \"b\")", + window: DynamoDBReadWindow() + ), + SplitCase( + statement: "SELECT * FROM \"Orders\" WHERE \"a\" IN (SELECT \"b\" FROM \"Other\" LIMIT 5)", + remaining: "SELECT * FROM \"Orders\" WHERE \"a\" IN (SELECT \"b\" FROM \"Other\" LIMIT 5)", + window: DynamoDBReadWindow() + ), + SplitCase( + statement: "SELECT * FROM \"Orders\" ORDER BY \"doc\".\"n\"", + remaining: "SELECT * FROM \"Orders\" ORDER BY \"doc\".\"n\"", + window: DynamoDBReadWindow() + ), + SplitCase( + statement: "SELECT * FROM \"Orders\" LIMIT ?", + remaining: "SELECT * FROM \"Orders\" LIMIT ?", + window: DynamoDBReadWindow() + ), + SplitCase( + statement: "SELECT * FROM \"Orders\" WHERE \"a\" = 'x LIMIT 5'", + remaining: "SELECT * FROM \"Orders\" WHERE \"a\" = 'x LIMIT 5'", + window: DynamoDBReadWindow() + ), + SplitCase( + statement: "SELECT * FROM \"Orders\" WHERE \"a\" = 'ORDER BY \"b\" DESC'", + remaining: "SELECT * FROM \"Orders\" WHERE \"a\" = 'ORDER BY \"b\" DESC'", + window: DynamoDBReadWindow() + ), + SplitCase( + statement: "SELECT * FROM \"Orders\" LIMIT 5 -- first page", + remaining: "SELECT * FROM \"Orders\"", + window: DynamoDBReadWindow(limit: 5) + ) + ] + + static let untouchedStatements: [String] = [ + "DELETE FROM \"Orders\" WHERE \"pk\" = 'a' LIMIT 5", + "UPDATE \"Orders\" SET \"a\" = 1 WHERE \"pk\" = 'a' ORDER BY \"a\" DESC", + "INSERT INTO \"Orders\" VALUE {'pk': 'a'} OFFSET 3 ", + "EXISTS(SELECT * FROM \"Orders\" LIMIT 1)" + ] + + static let rolesCases: [RolesCase] = [ + RolesCase( + statement: """ + UPDATE "Orders" SET "name" = ?, "total" = ? REMOVE "note" \ + WHERE "pk" = ? AND "sk" = ? AND "name" = ? AND "total" = ? + """, + roles: [ + .assigned(DynamoDBAttributePath(attribute: "name")), .assigned(DynamoDBAttributePath(attribute: "total")), + .compared(DynamoDBAttributePath(attribute: "pk")), + .compared(DynamoDBAttributePath(attribute: "sk")), + .compared(DynamoDBAttributePath(attribute: "name")), + .compared(DynamoDBAttributePath(attribute: "total")) + ] + ), + RolesCase( + statement: "INSERT INTO \"Orders\" VALUE {'a': ?, 'b''c': ?}", + roles: [.inserted("a"), .inserted("b'c")] + ), + RolesCase( + statement: "SELECT * FROM \"Orders\" WHERE \"a\" IN [?, ?]", + roles: [.compared(DynamoDBAttributePath(attribute: "a")), .compared(DynamoDBAttributePath(attribute: "a"))] + ), + RolesCase( + statement: "SELECT * FROM \"Orders\" WHERE contains(\"name\", ?)", + roles: [.compared(DynamoDBAttributePath(attribute: "name"))] + ), + RolesCase( + statement: "SELECT * FROM \"Orders\" WHERE \"n\" BETWEEN ? AND ?", + roles: [.compared(DynamoDBAttributePath(attribute: "n")), .compared(DynamoDBAttributePath(attribute: "n"))] + ), + RolesCase( + statement: "SELECT * FROM \"Orders\" WHERE \"pk\" = ? AND begins_with(\"sk\", ?) OR \"x\" <> ?", + roles: [ + .compared(DynamoDBAttributePath(attribute: "pk")), + .compared(DynamoDBAttributePath(attribute: "sk")), + .compared(DynamoDBAttributePath(attribute: "x")) + ] + ), + RolesCase( + statement: "UPDATE \"Orders\" SET \"a\"\"b\" = ? WHERE \"pk\" = ?", + roles: [.assigned(DynamoDBAttributePath(attribute: "a\"b")), .compared(DynamoDBAttributePath(attribute: "pk"))] + ), + RolesCase( + statement: "UPDATE \"Orders\" SET \"WHERE\" = ?, \"in\" = ? WHERE \"pk\" = ? AND \"is\" = ?", + roles: [ + .assigned(DynamoDBAttributePath(attribute: "WHERE")), + .assigned(DynamoDBAttributePath(attribute: "in")), + .compared(DynamoDBAttributePath(attribute: "pk")), + .compared(DynamoDBAttributePath(attribute: "is")) + ] + ), + RolesCase( + statement: "UPDATE \"Orders\" SET \"doc\".\"n\"[0] = ? WHERE \"pk\" = ?", + roles: [ + .assigned(DynamoDBAttributePath(segments: [.name("doc"), .name("n"), .index(0)])), + .compared(DynamoDBAttributePath(attribute: "pk")) + ] + ), + RolesCase( + statement: "UPDATE \"Orders\" SET \"items\" = list_append(\"items\", ?) WHERE \"pk\" = ?", + roles: [.assigned(DynamoDBAttributePath(attribute: "items")), .compared(DynamoDBAttributePath(attribute: "pk"))] + ), + RolesCase( + statement: "SELECT * FROM \"Orders\" WHERE \"a\" = '?' AND \"b\" = ?", + roles: [.compared(DynamoDBAttributePath(attribute: "b"))] + ), + RolesCase( + statement: "UPDATE \"Orders\" SET \"a\" = 1 WHERE \"pk\" = 'x'", + roles: [] + ) + ] + + // MARK: - Kind + + @Test("The first word decides the statement kind", arguments: kindCases) + func kind(kindCase: KindCase) { + #expect(DynamoDBPartiQL.kind(of: kindCase.statement) == kindCase.kind) + } + + // MARK: - Target + + @Test("The target table and index are read from the statement", arguments: targetCases) + func target(targetCase: TargetCase) throws { + let target = try #require(DynamoDBPartiQL.target(of: targetCase.statement)) + #expect(target.table == targetCase.table) + #expect(target.index == targetCase.index) + } + + @Test( + "A statement that names no table has no target", + arguments: ["", "SELECT 1", "SELECT * FROM", "UPDATE", "INSERT \"Orders\"", "EXISTS(SELECT * FROM \"Orders\")"] + ) + func noTarget(statement: String) { + #expect(DynamoDBPartiQL.target(of: statement) == nil) + } + + // MARK: - RETURNING + + @Test( + "A RETURNING clause is found in any case", + arguments: [ + "DELETE FROM \"Orders\" WHERE \"pk\" = ? RETURNING ALL OLD *", + "UPDATE \"Orders\" SET \"a\" = 1 WHERE \"pk\" = 'a' returning all new *" + ] + ) + func hasReturning(statement: String) { + #expect(DynamoDBPartiQL.hasReturning(statement)) + } + + @Test( + "RETURNING inside a string, a quoted name or a comment is not a clause", + arguments: [ + "DELETE FROM \"Orders\" WHERE \"pk\" = 'RETURNING ALL OLD *'", + "UPDATE \"Orders\" SET \"RETURNING\" = 1 WHERE \"pk\" = 'a'", + "DELETE FROM \"Orders\" WHERE \"pk\" = 'a' -- RETURNING ALL OLD *", + "DELETE FROM \"Orders\" WHERE \"pk\" = 'a' /* RETURNING ALL OLD * */", + "DELETE FROM \"Orders\" WHERE \"pk\" = 'a'" + ] + ) + func hasNoReturning(statement: String) { + #expect(!DynamoDBPartiQL.hasReturning(statement)) + } + + // MARK: - WHERE + + @Test( + "An equality or IN on the attribute fixes it", + arguments: [ + "SELECT * FROM \"Orders\" WHERE \"pk\" = 'a'", + "SELECT * FROM \"Orders\" WHERE \"pk\" IN ['a', 'b']", + "SELECT * FROM \"Orders\" WHERE \"sk\" > 3 AND \"pk\" = ?", + "select * from Orders where pk = 'a'", + "SELECT * FROM \"Orders\" WHERE (\"pk\" = 'a' AND \"sk\" > 1)", + "SELECT * FROM \"Orders\" WHERE \"pk\" = 'a' AND (\"sk\" = 1 OR \"sk\" = 2)", + "SELECT * FROM \"Orders\" WHERE ((\"pk\" = 'a')) AND \"sk\" > 1", + "SELECT * FROM \"Orders\" WHERE \"sk\" > 1 AND (\"status\" = 'open' AND \"pk\" = 'a')" + ] + ) + func whereFixes(statement: String) { + #expect(DynamoDBPartiQL.whereFixes("pk", in: statement)) + } + + @Test( + "A range, a function, a string or a missing WHERE does not fix the attribute", + arguments: [ + "SELECT * FROM \"Orders\"", + "SELECT \"pk\" FROM \"Orders\" WHERE \"sk\" = 1", + "SELECT * FROM \"Orders\" WHERE \"pk\" > 'a'", + "SELECT * FROM \"Orders\" WHERE \"pk\" <> 'a'", + "SELECT * FROM \"Orders\" WHERE begins_with(\"pk\", 'a')", + "SELECT * FROM \"Orders\" WHERE \"other\" = 'pk'", + "SELECT * FROM \"Orders\" WHERE \"PK\" = 'a'", + "SELECT * FROM \"Orders\" WHERE NOT \"pk\" = 'a'", + "SELECT * FROM \"Orders\" WHERE NOT (\"pk\" = 'a')", + "SELECT * FROM \"Orders\" WHERE (\"pk\" = 'a' OR \"sk\" = 1)", + "SELECT * FROM \"Orders\" WHERE \"sk\" = 1 AND (\"status\" = 'open' OR \"pk\" = 'a')", + "SELECT * FROM \"Orders\" WHERE size(\"pk\") = 1" + ] + ) + func whereDoesNotFix(statement: String) { + #expect(!DynamoDBPartiQL.whereFixes("pk", in: statement)) + } + + @Test("An equality joined to the rest of the WHERE clause by a top-level OR does not fix the attribute") + func whereWithTopLevelOrDoesNotFix() { + #expect(!DynamoDBPartiQL.whereFixes("pk", in: "SELECT * FROM \"Orders\" WHERE \"pk\" = 'a' OR \"status\" = 'open'")) + #expect(!DynamoDBPartiQL.whereFixes("pk", in: "SELECT * FROM \"Orders\" WHERE \"status\" = 'open' OR \"pk\" = 'a'")) + } + + @Test("An equality on a nested attribute of the same name does not fix the top-level attribute") + func whereOnNestedPathDoesNotFix() { + #expect(!DynamoDBPartiQL.whereFixes("pk", in: "SELECT * FROM \"Orders\" WHERE \"doc\".\"pk\" = 'a'")) + } + + // MARK: - Trailing window + + @Test("A SELECT's trailing ORDER BY, LIMIT and OFFSET are split off", arguments: splitCases) + func splitTrailingWindow(splitCase: SplitCase) { + let split = DynamoDBPartiQL.splitTrailingWindow(splitCase.statement) + #expect(split.statement == splitCase.remaining) + #expect(split.window == splitCase.window) + } + + @Test("A statement that is not a SELECT is returned untouched", arguments: untouchedStatements) + func nonSelectIsUntouched(statement: String) { + let split = DynamoDBPartiQL.splitTrailingWindow(statement) + #expect(split.statement == statement) + #expect(split.window.isEmpty) + } + + @Test("A negative LIMIT or OFFSET never becomes a window", arguments: [ + "SELECT * FROM \"Orders\" LIMIT -1", + "SELECT * FROM \"Orders\" LIMIT 5 OFFSET -2", + "SELECT * FROM \"Orders\" OFFSET -2" + ]) + func negativeWindowIsNotTaken(statement: String) { + let window = DynamoDBPartiQL.splitTrailingWindow(statement).window + #expect((window.limit ?? 0) >= 0) + #expect(window.offset >= 0) + } + + @Test("A comment written straight after a number still ends the statement") + func commentAfterNumber() { + let split = DynamoDBPartiQL.splitTrailingWindow("SELECT * FROM \"Orders\" LIMIT 5-- first page") + #expect(split.statement == "SELECT * FROM \"Orders\"") + #expect(split.window == DynamoDBReadWindow(limit: 5)) + } + + // MARK: - Parameter roles + + @Test("Each parameter's role is read from the words around it", arguments: rolesCases) + func parameterRoles(rolesCase: RolesCase) { + #expect(DynamoDBPartiQL.parameterRoles(in: rolesCase.statement) == rolesCase.roles) + } + + @Test("Parameters inside comments are not counted") + func parameterRolesIgnoreComments() { + let statement = """ + UPDATE "Orders" SET "a" = ? -- , "b" = ? it's "odd" + /* , "c" = ? + WHERE "d" = ? */ WHERE "pk" = ? /* AND "e" = ? */ + """ + #expect(DynamoDBPartiQL.parameterRoles(in: statement) == [ + .assigned(DynamoDBAttributePath(attribute: "a")), + .compared(DynamoDBAttributePath(attribute: "pk")) + ]) + } + + // MARK: - Tokenizer + + @Test("Quoted names and strings are unescaped and comments dropped") + func tokenizer() { + let tokens = DynamoDBPartiQL.tokens(of: "SELECT \"a\"\"b\" -- c\nFROM /* d */ 'e''f' [?] -1.5e3 <>") + #expect(tokens.map(\.kind) == [ + .word, .quotedIdentifier, .word, .string, .symbol, .parameter, .symbol, .number, .symbol + ]) + #expect(tokens.map(\.text) == ["SELECT", "a\"b", "FROM", "e'f", "[", "?", "]", "-1.5e3", "<>"]) + #expect(tokens.map(\.depth) == [0, 0, 0, 0, 0, 1, 0, 0, 0]) + } + + @Test("An unterminated comment or string runs to the end of the text") + func unterminatedRegions() { + #expect(DynamoDBPartiQL.tokens(of: "SELECT /* never closed ?").map(\.text) == ["SELECT"]) + let tokens = DynamoDBPartiQL.tokens(of: "SELECT 'never closed ?") + #expect(tokens.map(\.kind) == [.word, .string]) + #expect(tokens.last?.text == "never closed ?") + } +} diff --git a/TableProTests/Plugins/DynamoDB/DynamoDBRetryPolicyTests.swift b/TableProTests/Plugins/DynamoDB/DynamoDBRetryPolicyTests.swift new file mode 100644 index 0000000000..9277e3a955 --- /dev/null +++ b/TableProTests/Plugins/DynamoDB/DynamoDBRetryPolicyTests.swift @@ -0,0 +1,264 @@ +import Foundation +import TableProPluginKit +import Testing + +@Suite("DynamoDB retry policy") +struct DynamoDBRetryPolicyTests { + struct DelayCase: Sendable, CustomTestStringConvertible { + let attempt: Int + let seconds: TimeInterval + var testDescription: String { "attempt \(attempt)" } + } + + struct IdempotencyCase: Sendable, CustomTestStringConvertible { + let label: String + let operation: DynamoDBOperation + let body: String + let isIdempotent: Bool + var testDescription: String { label } + } + + private static let fullJitter = DynamoDBRetryPolicy(random: { 1.0 }) + private static let noJitter = DynamoDBRetryPolicy(random: { 0.0 }) + + private static func service(_ code: String, status: Int = 400, message: String = "") -> DynamoDBError { + .service(DynamoDBServiceError(code: code, message: message, httpStatus: status)) + } + + private static func decide( + _ policy: DynamoDBRetryPolicy, + _ error: DynamoDBError, + attempt: Int = 1, + operation: DynamoDBOperation = .getItem, + body: DynamoDBJSON = .object([:]), + refreshed: Bool = false, + corrected: Bool = false + ) -> DynamoDBRetryPolicy.Decision { + policy.decision( + for: error, + attempt: attempt, + operation: operation, + body: body, + alreadyRefreshedCredentials: refreshed, + alreadyCorrectedClock: corrected + ) + } + + @Test( + "Throttling waits one second doubled per attempt", + arguments: [ + DelayCase(attempt: 1, seconds: 1), + DelayCase(attempt: 2, seconds: 2), + DelayCase(attempt: 3, seconds: 4) + ] + ) + func throttlingDelay(_ testCase: DelayCase) { + let decision = Self.decide( + Self.fullJitter, Self.service("ProvisionedThroughputExceededException"), + attempt: testCase.attempt, operation: .updateItem + ) + #expect(decision == .retry(after: testCase.seconds)) + } + + @Test("Throttling delays are scaled by the jitter") + func throttlingDelayJitter() { + let decision = Self.decide(Self.noJitter, Self.service("ThrottlingException"), attempt: 3, operation: .putItem) + #expect(decision == .retry(after: 0)) + let half = DynamoDBRetryPolicy(random: { 0.5 }) + #expect(Self.decide(half, Self.service("ThrottlingException"), attempt: 3) == .retry(after: 2)) + } + + @Test("The delay ceiling stops at twenty seconds") + func delayCeiling() { + #expect(Self.fullJitter.delay(base: DynamoDBRetryPolicy.throttlingBase, attempt: 5) == 16) + #expect(Self.fullJitter.delay(base: DynamoDBRetryPolicy.throttlingBase, attempt: 6) == 20) + #expect(Self.fullJitter.delay(base: DynamoDBRetryPolicy.throttlingBase, attempt: 30) == 20) + #expect(Self.noJitter.delay(base: DynamoDBRetryPolicy.throttlingBase, attempt: 6) == 0) + } + + @Test("Throttling retries a request that is not idempotent") + func throttlingRetriesAnyRequest() { + let decision = Self.decide(Self.fullJitter, Self.service("RequestLimitExceeded"), operation: .updateItem) + #expect(decision == .retry(after: 1)) + } + + @Test( + "A transient failure of an idempotent request waits 25 ms doubled per attempt", + arguments: [ + DelayCase(attempt: 1, seconds: 0.025), + DelayCase(attempt: 2, seconds: 0.05), + DelayCase(attempt: 3, seconds: 0.1) + ] + ) + func transientDelay(_ testCase: DelayCase) { + let decision = Self.decide( + Self.fullJitter, Self.service("InternalServerError", status: 500), attempt: testCase.attempt + ) + #expect(decision == .retry(after: testCase.seconds)) + } + + @Test("A transient failure of a request that may have applied is not retried") + func transientNotIdempotentFails() { + #expect(Self.decide(Self.fullJitter, Self.service("InternalServerError", status: 500), operation: .updateItem) == .fail) + #expect(Self.decide(Self.fullJitter, Self.service("SomethingNew", status: 503), operation: .updateItem) == .fail) + } + + @Test("A transient 5xx of an idempotent request is retried whatever its code") + func transientAnyCodeRetried() { + #expect(Self.decide(Self.noJitter, Self.service("SomethingNew", status: 503), operation: .scan) == .retry(after: 0)) + } + + @Test("The fourth attempt is the last", arguments: [4, 5]) + func attemptLimit(_ attempt: Int) { + #expect(Self.decide(Self.fullJitter, Self.service("ThrottlingException"), attempt: attempt) == .fail) + #expect(Self.decide(Self.fullJitter, Self.service("InternalServerError", status: 500), attempt: attempt) == .fail) + #expect(Self.decide(Self.fullJitter, Self.service("ExpiredTokenException"), attempt: attempt) == .fail) + #expect(Self.decide(Self.fullJitter, .transport("reset"), attempt: attempt) == .fail) + } + + @Test("An expired token is refreshed once and then fails") + func expiredTokenRefreshesOnce() { + let error = Self.service("ExpiredTokenException") + #expect(Self.decide(Self.fullJitter, error, operation: .updateItem) == .refreshCredentialsAndRetry) + #expect(Self.decide(Self.fullJitter, error, attempt: 2, operation: .updateItem, refreshed: true) == .fail) + } + + @Test("A skewed clock is corrected once and then fails") + func clockSkewCorrectsOnce() { + let error = Self.service("InvalidSignatureException", message: "Signature expired: 20150830T123600Z is now earlier than 20150830T124100Z") + #expect(Self.decide(Self.fullJitter, error, operation: .updateItem) == .correctClockAndRetry) + #expect(Self.decide(Self.fullJitter, error, attempt: 2, operation: .updateItem, corrected: true) == .fail) + #expect(Self.decide(Self.fullJitter, Self.service("RequestTimeTooSkewed")) == .correctClockAndRetry) + } + + @Test( + "Authentication and fatal errors fail at once", + arguments: [ + "UnrecognizedClientException", "InvalidSignatureException", "AccessDeniedException", + "ValidationException", "ResourceNotFoundException", "ConditionalCheckFailedException" + ] + ) + func authenticationAndFatalFail(_ code: String) { + #expect(Self.decide(Self.fullJitter, Self.service(code), operation: .scan) == .fail) + } + + @Test("A transport error is retried only for an idempotent request") + func transportErrorRetriesIdempotentOnly() { + #expect(Self.decide(Self.fullJitter, .transport("The network connection was lost."), operation: .query) + == .retry(after: 0.025)) + #expect(Self.decide(Self.fullJitter, .transport("The network connection was lost."), operation: .updateItem) + == .fail) + } + + @Test( + "Errors the driver raised itself are never retried", + arguments: [ + DynamoDBError.cancelled, .notConnected, .timedOut(seconds: 30), .invalidResponse("bad"), + .configuration("bad"), .invalidStatement("bad") + ] + ) + func localErrorsFail(_ error: DynamoDBError) { + #expect(Self.decide(Self.fullJitter, error, operation: .scan) == .fail) + } + + @Test( + "Every read is idempotent", + arguments: DynamoDBOperation.allCases.filter(\.isRead) + ) + func readsAreIdempotent(_ operation: DynamoDBOperation) { + #expect(DynamoDBRetryPolicy.isIdempotent(operation, body: .object([:]))) + } + + @Test( + "Whether a write may be sent twice", + arguments: [ + IdempotencyCase(label: "PutItem", operation: .putItem, body: #"{"TableName":"t"}"#, isIdempotent: true), + IdempotencyCase( + label: "PutItem with a condition", operation: .putItem, + body: #"{"TableName":"t","ConditionExpression":"attribute_not_exists(pk)"}"#, isIdempotent: false + ), + IdempotencyCase( + label: "PutItem with legacy Expected", operation: .putItem, + body: #"{"TableName":"t","Expected":{"pk":{"Exists":false}}}"#, isIdempotent: false + ), + IdempotencyCase(label: "DeleteItem", operation: .deleteItem, body: #"{"TableName":"t"}"#, isIdempotent: true), + IdempotencyCase( + label: "DeleteItem with a condition", operation: .deleteItem, + body: #"{"TableName":"t","ConditionExpression":"attribute_exists(pk)"}"#, isIdempotent: false + ), + IdempotencyCase(label: "UpdateItem", operation: .updateItem, body: #"{"TableName":"t"}"#, isIdempotent: false), + IdempotencyCase( + label: "UpdateItem with a condition", operation: .updateItem, + body: #"{"TableName":"t","ConditionExpression":"attribute_exists(pk)"}"#, isIdempotent: false + ), + IdempotencyCase(label: "BatchWriteItem", operation: .batchWriteItem, body: #"{"RequestItems":{}}"#, isIdempotent: true), + IdempotencyCase( + label: "TransactWriteItems with a token", operation: .transactWriteItems, + body: #"{"TransactItems":[],"ClientRequestToken":"f3c1"}"#, isIdempotent: true + ), + IdempotencyCase( + label: "TransactWriteItems without a token", operation: .transactWriteItems, + body: #"{"TransactItems":[]}"#, isIdempotent: false + ), + IdempotencyCase( + label: "TransactWriteItems with an empty token", operation: .transactWriteItems, + body: #"{"TransactItems":[],"ClientRequestToken":""}"#, isIdempotent: false + ), + IdempotencyCase( + label: "ExecuteTransaction with a token", operation: .executeTransaction, + body: #"{"TransactStatements":[],"ClientRequestToken":"f3c1"}"#, isIdempotent: true + ), + IdempotencyCase( + label: "ExecuteTransaction without a token", operation: .executeTransaction, + body: #"{"TransactStatements":[]}"#, isIdempotent: false + ), + IdempotencyCase( + label: "ExecuteStatement SELECT", operation: .executeStatement, + body: #"{"Statement":"SELECT * FROM \"t\" WHERE pk = 'a'"}"#, isIdempotent: true + ), + IdempotencyCase( + label: "ExecuteStatement lowercase select", operation: .executeStatement, + body: #"{"Statement":" select * from t"}"#, isIdempotent: true + ), + IdempotencyCase( + label: "ExecuteStatement UPDATE", operation: .executeStatement, + body: #"{"Statement":"UPDATE \"t\" SET n = n + 1 WHERE pk = 'a'"}"#, isIdempotent: false + ), + IdempotencyCase( + label: "ExecuteStatement INSERT", operation: .executeStatement, + body: #"{"Statement":"INSERT INTO \"t\" VALUE {'pk': 'a'}"}"#, isIdempotent: false + ), + IdempotencyCase( + label: "ExecuteStatement with no statement", operation: .executeStatement, + body: #"{}"#, isIdempotent: false + ), + IdempotencyCase( + label: "BatchExecuteStatement all SELECT", operation: .batchExecuteStatement, + body: #"{"Statements":[{"Statement":"SELECT * FROM t WHERE pk = 'a'"},{"Statement":"SELECT * FROM t WHERE pk = 'b'"}]}"#, + isIdempotent: true + ), + IdempotencyCase( + label: "BatchExecuteStatement mixed", operation: .batchExecuteStatement, + body: #"{"Statements":[{"Statement":"SELECT * FROM t WHERE pk = 'a'"},{"Statement":"DELETE FROM t WHERE pk = 'b'"}]}"#, + isIdempotent: false + ), + IdempotencyCase( + label: "BatchExecuteStatement empty", operation: .batchExecuteStatement, + body: #"{"Statements":[]}"#, isIdempotent: false + ), + IdempotencyCase( + label: "CreateTable", operation: .createTable, body: #"{"TableName":"t"}"#, isIdempotent: false + ), + IdempotencyCase( + label: "DeleteTable", operation: .deleteTable, body: #"{"TableName":"t"}"#, isIdempotent: false + ), + IdempotencyCase( + label: "UpdateTimeToLive", operation: .updateTimeToLive, body: #"{"TableName":"t"}"#, isIdempotent: false + ) + ] + ) + func writeIdempotency(_ testCase: IdempotencyCase) throws { + let body = try DynamoDBJSON.parse(testCase.body) + #expect(DynamoDBRetryPolicy.isIdempotent(testCase.operation, body: body) == testCase.isIdempotent) + } +} diff --git a/TableProTests/Plugins/DynamoDB/DynamoDBSignerTests.swift b/TableProTests/Plugins/DynamoDB/DynamoDBSignerTests.swift new file mode 100644 index 0000000000..7446c5a56e --- /dev/null +++ b/TableProTests/Plugins/DynamoDB/DynamoDBSignerTests.swift @@ -0,0 +1,189 @@ +import Foundation +import TableProPluginKit +import Testing + +@Suite("DynamoDB request signing") +struct DynamoDBSignerTests { + private static let exampleDate = Date(timeIntervalSince1970: 1_440_938_160) + private static let exampleCredentials = AWSCredentials( + accessKeyId: "AKIDEXAMPLE", + secretAccessKey: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + sessionToken: nil + ) + private static let exampleToken = + "AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGd" + + struct HostCase: Sendable, CustomTestStringConvertible { + let url: String + let expected: String + var testDescription: String { url } + } + + struct PathCase: Sendable, CustomTestStringConvertible { + let url: String + let expected: String + var testDescription: String { url } + } + + private static func listTablesRequest(_ urlText: String) throws -> URLRequest { + var request = URLRequest(url: try #require(URL(string: urlText))) + request.httpMethod = "POST" + request.setValue("application/x-amz-json-1.0", forHTTPHeaderField: "Content-Type") + request.setValue("DynamoDB_20120810.ListTables", forHTTPHeaderField: "X-Amz-Target") + return request + } + + @Test("Timestamps are formatted in UTC whatever the local time zone") + func timestampsAreUTC() { + let stamps = DynamoDBSigner.timestamps(for: Date(timeIntervalSince1970: 1_709_251_199)) + #expect(stamps == DynamoDBSigner.Timestamps(amzDate: "20240229T235959Z", dateStamp: "20240229")) + } + + @Test("Timestamps pad every field to its fixed width") + func timestampsArePadded() { + #expect(DynamoDBSigner.timestamps(for: Date(timeIntervalSince1970: 1_767_582_245)).amzDate == "20260105T030405Z") + #expect(DynamoDBSigner.timestamps(for: Date(timeIntervalSince1970: 0)).amzDate == "19700101T000000Z") + #expect(DynamoDBSigner.timestamps(for: Self.exampleDate).dateStamp == "20150830") + } + + @Test( + "The Host header drops only the scheme's default port", + arguments: [ + HostCase(url: "https://dynamodb.us-east-1.amazonaws.com:443/", expected: "dynamodb.us-east-1.amazonaws.com"), + HostCase(url: "https://dynamodb.us-east-1.amazonaws.com/", expected: "dynamodb.us-east-1.amazonaws.com"), + HostCase(url: "http://localhost:80/", expected: "localhost"), + HostCase(url: "http://localhost:8000", expected: "localhost:8000"), + HostCase(url: "https://ddb.example.com:8443/", expected: "ddb.example.com:8443"), + HostCase(url: "http://ddb.example.com:443/", expected: "ddb.example.com:443"), + HostCase(url: "https://ddb.example.com:80/", expected: "ddb.example.com:80") + ] + ) + func hostHeader(_ testCase: HostCase) throws { + let url = try #require(URL(string: testCase.url)) + #expect(DynamoDBSigner.hostHeader(for: url) == testCase.expected) + } + + @Test( + "The Host header keeps the brackets around an IPv6 literal", + arguments: [ + HostCase(url: "http://[::1]:8000", expected: "[::1]:8000"), + HostCase(url: "https://[2001:db8::1]/", expected: "[2001:db8::1]") + ] + ) + func hostHeaderForIPv6(_ testCase: HostCase) throws { + let url = try #require(URL(string: testCase.url)) + #expect(DynamoDBSigner.hostHeader(for: url) == testCase.expected) + } + + @Test( + "The canonical URI is the path as sent", + arguments: [ + PathCase(url: "https://h/dynamodb/", expected: "/dynamodb/"), + PathCase(url: "https://h/a%20b/c", expected: "/a%20b/c"), + PathCase(url: "https://h/", expected: "/"), + PathCase(url: "https://h", expected: "/"), + PathCase(url: "http://localhost:8000", expected: "/") + ] + ) + func canonicalURI(_ testCase: PathCase) throws { + let url = try #require(URL(string: testCase.url)) + #expect(DynamoDBSigner.canonicalURI(for: url) == testCase.expected) + } + + @Test("Signing sets the date, the host and an Authorization header with the credential scope") + func signSetsHeaders() throws { + var request = try Self.listTablesRequest("https://dynamodb.us-east-1.amazonaws.com/") + DynamoDBSigner.sign( + &request, body: Data("{}".utf8), credentials: Self.exampleCredentials, + region: "us-east-1", date: Self.exampleDate + ) + #expect(request.value(forHTTPHeaderField: "X-Amz-Date") == "20150830T123600Z") + #expect(request.value(forHTTPHeaderField: "Host") == "dynamodb.us-east-1.amazonaws.com") + #expect(request.value(forHTTPHeaderField: "X-Amz-Security-Token") == nil) + let authorization = try #require(request.value(forHTTPHeaderField: "Authorization")) + #expect(authorization.hasPrefix( + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/dynamodb/aws4_request, " + )) + #expect(authorization.contains("SignedHeaders=content-type;host;x-amz-date;x-amz-target, ")) + } + + @Test("The signature matches an independent SigV4 computation") + func signatureMatchesReference() throws { + var request = try Self.listTablesRequest("https://dynamodb.us-east-1.amazonaws.com/") + DynamoDBSigner.sign( + &request, body: Data("{}".utf8), credentials: Self.exampleCredentials, + region: "us-east-1", date: Self.exampleDate + ) + #expect( + request.value(forHTTPHeaderField: "Authorization") + == "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/dynamodb/aws4_request, " + + "SignedHeaders=content-type;host;x-amz-date;x-amz-target, " + + "Signature=75214f17608dbd636679e18f6f89744844ae96fcd228b8147167152488d817de" + ) + } + + @Test("A session token is sent and signed") + func sessionTokenIsSigned() throws { + var request = try Self.listTablesRequest("https://dynamodb.us-east-1.amazonaws.com/") + let credentials = AWSCredentials( + accessKeyId: "AKIDEXAMPLE", + secretAccessKey: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + sessionToken: Self.exampleToken + ) + DynamoDBSigner.sign( + &request, body: Data("{}".utf8), credentials: credentials, region: "us-east-1", date: Self.exampleDate + ) + #expect(request.value(forHTTPHeaderField: "X-Amz-Security-Token") == Self.exampleToken) + #expect( + request.value(forHTTPHeaderField: "Authorization") + == "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/dynamodb/aws4_request, " + + "SignedHeaders=content-type;host;x-amz-date;x-amz-security-token;x-amz-target, " + + "Signature=6fa03fc9356b96e0d654a933310ae874b0752797246b39df2a03cc67d4302dcd" + ) + } + + @Test("An empty session token is neither sent nor signed") + func emptySessionTokenIsIgnored() throws { + var request = try Self.listTablesRequest("https://dynamodb.us-east-1.amazonaws.com/") + let credentials = AWSCredentials( + accessKeyId: "AKIDEXAMPLE", + secretAccessKey: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + sessionToken: "" + ) + DynamoDBSigner.sign( + &request, body: Data("{}".utf8), credentials: credentials, region: "us-east-1", date: Self.exampleDate + ) + #expect(request.value(forHTTPHeaderField: "X-Amz-Security-Token") == nil) + let authorization = try #require(request.value(forHTTPHeaderField: "Authorization")) + #expect(authorization.hasSuffix( + "Signature=75214f17608dbd636679e18f6f89744844ae96fcd228b8147167152488d817de" + )) + } + + @Test("A custom endpoint signs its port and its percent-encoded path") + func customEndpointSignature() throws { + var request = try Self.listTablesRequest("http://localhost:8000/a%20b/c") + DynamoDBSigner.sign( + &request, body: Data("{}".utf8), credentials: Self.exampleCredentials, + region: "us-east-1", date: Self.exampleDate + ) + #expect(request.value(forHTTPHeaderField: "Host") == "localhost:8000") + let authorization = try #require(request.value(forHTTPHeaderField: "Authorization")) + #expect(authorization.hasSuffix( + "Signature=54093f8b1a85d2095f252df1192f952b4d2b2205be51b10a3f313d0940dd0f52" + )) + } + + @Test("The signing region and date change the credential scope") + func scopeFollowsRegionAndDate() throws { + var request = try Self.listTablesRequest("https://dynamodb.cn-north-1.amazonaws.com.cn/") + DynamoDBSigner.sign( + &request, body: Data("{}".utf8), credentials: Self.exampleCredentials, + region: "cn-north-1", date: Date(timeIntervalSince1970: 1_709_251_199) + ) + let authorization = try #require(request.value(forHTTPHeaderField: "Authorization")) + #expect(authorization.hasPrefix( + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20240229/cn-north-1/dynamodb/aws4_request, " + )) + } +} diff --git a/TableProTests/Plugins/DynamoDB/DynamoDBStatementTests.swift b/TableProTests/Plugins/DynamoDB/DynamoDBStatementTests.swift new file mode 100644 index 0000000000..fbb66cb0ce --- /dev/null +++ b/TableProTests/Plugins/DynamoDB/DynamoDBStatementTests.swift @@ -0,0 +1,423 @@ +import Foundation +import TableProPluginKit +import Testing + +@Suite("DynamoDB statement parsing") +struct DynamoDBStatementTests { + struct WindowCase: Sendable, CustomTestStringConvertible { + let clause: String + let window: DynamoDBReadWindow + + var testDescription: String { clause } + } + + static let ordersBody: DynamoDBJSON = .object(["TableName": .string("Orders")]) + + static let windowCases: [WindowCase] = [ + WindowCase( + clause: "ORDER BY \"a\"\"b\" DESC", + window: DynamoDBReadWindow(order: [DynamoDBOrderTerm(attribute: "a\"b", descending: true)]) + ), + WindowCase( + clause: "ORDER BY \"a\" DESC, b, \"c\" ASC", + window: DynamoDBReadWindow(order: [ + DynamoDBOrderTerm(attribute: "a", descending: true), + DynamoDBOrderTerm(attribute: "b", descending: false), + DynamoDBOrderTerm(attribute: "c", descending: false) + ]) + ), + WindowCase(clause: "LIMIT 25", window: DynamoDBReadWindow(limit: 25)), + WindowCase(clause: "LIMIT 0", window: DynamoDBReadWindow(limit: 0)), + WindowCase(clause: "OFFSET 40", window: DynamoDBReadWindow(offset: 40)), + WindowCase(clause: "LIMIT 25 OFFSET 50", window: DynamoDBReadWindow(limit: 25, offset: 50)), + WindowCase( + clause: "order by \"sk\" desc limit 5 offset 10", + window: DynamoDBReadWindow( + order: [DynamoDBOrderTerm(attribute: "sk", descending: true)], + limit: 5, + offset: 10 + ) + ), + WindowCase( + clause: "ORDER BY \"sk\" ASC LIMIT 5 -- first page", + window: DynamoDBReadWindow(order: [DynamoDBOrderTerm(attribute: "sk", descending: false)], limit: 5) + ) + ] + + static let malformedWindows: [String] = [ + "WHERE \"a\" = 1", + "ORDER BY", + "ORDER \"a\"", + "ORDER BY 1", + "ORDER BY \"a\",", + "ORDER BY \"a\" DESC DESC", + "LIMIT", + "LIMIT ten", + "LIMIT -1", + "LIMIT 1.5", + "OFFSET -2", + "LIMIT 5 ORDER BY \"a\"", + "OFFSET 2 LIMIT 5", + "LIMIT 5 LIMIT 6", + "; Scan {\"TableName\":\"Orders\"}" + ] + + static let roundTripStatements: [DynamoDBStatement] = [ + .partiQL(text: "SELECT * FROM \"Orders\"", window: DynamoDBReadWindow()), + .partiQL( + text: "SELECT * FROM \"Orders\" WHERE \"pk\" = 'a'", + window: DynamoDBReadWindow( + order: [ + DynamoDBOrderTerm(attribute: "sk", descending: true), + DynamoDBOrderTerm(attribute: "a\"b", descending: false) + ], + limit: 5, + offset: 10 + ) + ), + .partiQL(text: "UPDATE \"Orders\" SET \"a\" = 1 WHERE \"pk\" = 'a'", window: DynamoDBReadWindow()), + .apiCall(DynamoDBAPICall(operation: .scan, body: ordersBody), window: DynamoDBReadWindow()), + .apiCall( + DynamoDBAPICall( + operation: .query, + body: .object([ + "TableName": .string("Orders"), + "Limit": .number("12345678901234567890123456789012345678"), + "ConsistentRead": .bool(true), + "ExpressionAttributeValues": .object([":p": .object(["S": .string("a/b \"c\"")])]) + ]) + ), + window: DynamoDBReadWindow( + order: [DynamoDBOrderTerm(attribute: "a\"b", descending: true)], + limit: 5, + offset: 10 + ) + ), + .apiCall( + DynamoDBAPICall(operation: .putItem, body: .object([ + "TableName": .string("Orders"), + "Item": .object(["pk": .object(["S": .string("a")])]) + ])), + window: DynamoDBReadWindow() + ), + .browse( + DynamoDBBrowseRequest(table: "Orders", filters: [], matchAll: true, columns: []), + window: DynamoDBReadWindow() + ), + .browse( + DynamoDBBrowseRequest( + table: "Orders", + filters: [ + DynamoDBBrowseFilter( + attribute: "status", op: "=", value: "open", + secondValue: nil, kind: "text", caseSensitive: false + ) + ], + matchAll: false, + columns: ["pk", "status"] + ), + window: DynamoDBReadWindow(order: [DynamoDBOrderTerm(attribute: "pk", descending: false)], limit: 300) + ) + ] + + private func failureMessage(_ text: String) -> String? { + do { + _ = try DynamoDBStatement.parse(text) + return nil + } catch DynamoDBError.invalidStatement(let message) { + return message + } catch { + return nil + } + } + + // MARK: - API calls + + @Test("An action verb is matched without regard to case", arguments: ["Scan", "scan", "SCAN", "sCaN"]) + func verbIsCaseInsensitive(verb: String) throws { + let statement = try DynamoDBStatement.parse("\(verb) {\"TableName\": \"Orders\"}") + #expect(statement == .apiCall( + DynamoDBAPICall(operation: .scan, body: Self.ordersBody), + window: DynamoDBReadWindow() + )) + } + + @Test("Every action the editor runs parses under its own name", arguments: DynamoDBOperation.allCases) + func everyOperationParses(operation: DynamoDBOperation) throws { + let statement = try DynamoDBStatement.parse("\(operation.rawValue) {}") + #expect(statement == .apiCall( + DynamoDBAPICall(operation: operation, body: .object([:])), + window: DynamoDBReadWindow() + )) + } + + @Test("Trailing semicolons and whitespace are stripped") + func trailingSemicolonsAreStripped() throws { + let call = try DynamoDBStatement.parse(" Scan {\"TableName\":\"Orders\"} ;; \n") + #expect(call == .apiCall(DynamoDBAPICall(operation: .scan, body: Self.ordersBody), window: DynamoDBReadWindow())) + + let partiQL = try DynamoDBStatement.parse("SELECT * FROM \"Orders\";\n") + #expect(partiQL == .partiQL(text: "SELECT * FROM \"Orders\"", window: DynamoDBReadWindow())) + } + + @Test("The body may follow the verb directly or after a line break") + func whitespaceBeforeTheBodyIsOptional() throws { + let expected = DynamoDBStatement.apiCall( + DynamoDBAPICall(operation: .getItem, body: Self.ordersBody), + window: DynamoDBReadWindow() + ) + #expect(try DynamoDBStatement.parse("GetItem{\"TableName\":\"Orders\"}") == expected) + #expect(try DynamoDBStatement.parse("GetItem\n {\"TableName\":\"Orders\"}") == expected) + } + + @Test("A request that is not valid JSON names the action and where the JSON broke") + func invalidJSONIsReported() throws { + let message = try #require(failureMessage("Scan {\"TableName\": }")) + #expect(message.contains("Scan")) + #expect(message.contains(DynamoDBJSON.ParseError.unexpected(offset: 14).localizedDescription)) + } + + @Test("A request with a duplicate key is reported as invalid JSON") + func duplicateKeyIsReported() throws { + let message = try #require(failureMessage("PutItem {\"TableName\": \"a\", \"TableName\": \"b\"}")) + #expect(message.contains("PutItem")) + #expect(message.contains(DynamoDBJSON.ParseError.duplicateKey("TableName").localizedDescription)) + } + + @Test("An unterminated request is refused") + func unterminatedRequestIsRefused() { + #expect(failureMessage("Query {\"TableName\": \"Orders\"") != nil) + } + + @Test( + "An action this editor does not run is refused when a JSON body follows it", + arguments: ["Frobnicate {\"TableName\": \"Orders\"}", "ListBackups {}", "Scna {\"TableName\": \"Orders\"}"] + ) + func unknownActionWithBodyIsRefused(text: String) throws { + let message = try #require(failureMessage(text)) + let verb = String(text.prefix { $0.isLetter }) + #expect(message.contains(verb)) + } + + @Test( + "A verb not followed by a JSON object is read as PartiQL", + arguments: [ + "Scan", + "Scan [{\"TableName\": \"Orders\"}]", + "DeleteItem \"Orders\"", + "select * from \"Orders\"", + "EXISTS(SELECT * FROM \"Orders\" WHERE \"pk\" = 'a')" + ] + ) + func verbWithoutObjectFallsBackToPartiQL(text: String) throws { + #expect(try DynamoDBStatement.parse(text) == .partiQL(text: text, window: DynamoDBReadWindow())) + } + + // MARK: - Browse + + @Test("Browse reads its table, filters, match mode and columns") + func browseParses() throws { + let text = """ + Browse {"TableName": "Orders", "Filters": [\ + {"Attribute": "status", "Operator": "begins_with", "Value": "op", "Kind": "text", "CaseSensitive": false}, \ + {"Attribute": "total", "Operator": "BETWEEN", "Value": "1", "SecondValue": "9"}], \ + "Match": "any", "Columns": ["pk", "status"]} + """ + let statement = try DynamoDBStatement.parse(text) + let expected = DynamoDBBrowseRequest( + table: "Orders", + filters: [ + DynamoDBBrowseFilter( + attribute: "status", op: "BEGINS_WITH", value: "op", + secondValue: nil, kind: "text", caseSensitive: false + ), + DynamoDBBrowseFilter( + attribute: "total", op: "BETWEEN", value: "1", + secondValue: "9", kind: nil, caseSensitive: true + ) + ], + matchAll: false, + columns: ["pk", "status"] + ) + #expect(statement == .browse(expected, window: DynamoDBReadWindow())) + } + + @Test("The Browse verb is matched without regard to case and takes a window") + func browseVerbIsCaseInsensitive() throws { + let statement = try DynamoDBStatement.parse("bRoWsE {\"TableName\": \"Orders\"} LIMIT 300 OFFSET 600") + #expect(statement == .browse( + DynamoDBBrowseRequest(table: "Orders", filters: [], matchAll: true, columns: []), + window: DynamoDBReadWindow(limit: 300, offset: 600) + )) + } + + @Test( + "Browse without a table or with an incomplete filter is refused", + arguments: [ + "Browse {}", + "Browse {\"TableName\": \"\"}", + "Browse {\"TableName\": 5}", + "Browse {\"TableName\": \"Orders\", \"Filters\": [{\"Attribute\": \"a\"}]}", + "Browse {\"TableName\": \"Orders\", \"Filters\": [{\"Operator\": \"=\"}]}" + ] + ) + func incompleteBrowseIsRefused(text: String) { + #expect(failureMessage(text) != nil) + } + + @Test("A Browse filter value written as a JSON number keeps its digits") + func browseNumericValueIsKept() throws { + let json = try DynamoDBJSON.parse(""" + {"TableName": "Orders", "Filters": [{"Attribute": "total", "Operator": "=", "Value": 30}]} + """) + let request = try DynamoDBBrowseRequest(json: json) + #expect(request.filters.first?.value == "30") + } + + @Test("A Browse request survives its JSON form", arguments: [ + DynamoDBBrowseRequest(table: "Orders", filters: [], matchAll: true, columns: []), + DynamoDBBrowseRequest(table: "Or\"ders", filters: [], matchAll: true, columns: ["pk", "a\"b"]), + DynamoDBBrowseRequest( + table: "Orders", + filters: [ + DynamoDBBrowseFilter( + attribute: "status", op: "=", value: "open", + secondValue: nil, kind: "text", caseSensitive: false + ), + DynamoDBBrowseFilter( + attribute: "total", op: "BETWEEN", value: "1", + secondValue: "9", kind: "decimal", caseSensitive: true + ) + ], + matchAll: false, + columns: [] + ), + DynamoDBBrowseRequest( + table: "Orders", + filters: [ + DynamoDBBrowseFilter( + attribute: "note", op: "IS NULL", value: "", + secondValue: nil, kind: nil, caseSensitive: true + ) + ], + matchAll: true, + columns: ["note"] + ) + ]) + func browseJSONRoundTrip(request: DynamoDBBrowseRequest) throws { + #expect(try DynamoDBBrowseRequest(json: request.json) == request) + #expect(try DynamoDBBrowseRequest(json: DynamoDBJSON.parse(request.json.serialized())) == request) + } + + @Test("A Browse request with no filters or columns writes only its table") + func browseJSONOmitsEmptyParts() { + let request = DynamoDBBrowseRequest(table: "Orders", filters: [], matchAll: true, columns: []) + #expect(request.json == Self.ordersBody) + } + + // MARK: - Window + + @Test("A read window after a Scan is parsed", arguments: windowCases) + func windowAfterScan(windowCase: WindowCase) throws { + let statement = try DynamoDBStatement.parse("Scan {\"TableName\": \"Orders\"} \(windowCase.clause)") + #expect(statement == .apiCall( + DynamoDBAPICall(operation: .scan, body: Self.ordersBody), + window: windowCase.window + )) + } + + @Test("A read window after a Query is parsed") + func windowAfterQuery() throws { + let statement = try DynamoDBStatement.parse("query {\"TableName\": \"Orders\"} LIMIT 3;") + #expect(statement == .apiCall( + DynamoDBAPICall(operation: .query, body: Self.ordersBody), + window: DynamoDBReadWindow(limit: 3) + )) + } + + @Test("Malformed or misordered text after a request is refused", arguments: malformedWindows) + func malformedWindowIsRefused(clause: String) { + #expect(failureMessage("Scan {\"TableName\": \"Orders\"} \(clause)") != nil) + } + + @Test("The refusal names the action and repeats the unexpected text") + func malformedWindowMessage() throws { + let message = try #require(failureMessage("Scan {\"TableName\": \"Orders\"} WHERE x = 1 ")) + #expect(message.contains("Scan")) + #expect(message.contains("WHERE x = 1")) + } + + @Test( + "An action that is not a Scan or a Query takes no window", + arguments: [DynamoDBOperation.putItem, .getItem, .deleteItem, .describeTable, .batchWriteItem, .executeStatement] + ) + func windowOnOtherActionIsRefused(operation: DynamoDBOperation) throws { + for clause in ["LIMIT 1", "OFFSET 1", "ORDER BY \"a\""] { + let text = "\(operation.rawValue.lowercased()) {\"TableName\": \"Orders\"} \(clause)" + let message = try #require(failureMessage(text)) + #expect(message.contains(operation.rawValue)) + } + } + + // MARK: - Text + + @Test("Every statement parses back from its own text", arguments: roundTripStatements) + func textRoundTrip(statement: DynamoDBStatement) throws { + #expect(try DynamoDBStatement.parse(statement.text) == statement) + } + + @Test("A statement's text is its verb, compact sorted JSON and its window, a PartiQL window on its own line") + func textSpelling() { + let scan = DynamoDBStatement.apiCall( + DynamoDBAPICall(operation: .scan, body: .object(["TableName": .string("Orders"), "Limit": .number("10")])), + window: DynamoDBReadWindow( + order: [DynamoDBOrderTerm(attribute: "a\"b", descending: true)], + limit: 5, + offset: 10 + ) + ) + #expect(scan.text == "Scan {\"Limit\":10,\"TableName\":\"Orders\"} ORDER BY \"a\"\"b\" DESC LIMIT 5 OFFSET 10") + + let browse = DynamoDBStatement.browse( + DynamoDBBrowseRequest(table: "Orders", filters: [], matchAll: true, columns: []), + window: DynamoDBReadWindow() + ) + #expect(browse.text == "Browse {\"TableName\":\"Orders\"}") + + let partiQL = DynamoDBStatement.partiQL( + text: "SELECT * FROM \"Orders\"", + window: DynamoDBReadWindow(order: [DynamoDBOrderTerm(attribute: "sk", descending: false)], offset: 3) + ) + #expect(partiQL.text == "SELECT * FROM \"Orders\"\nORDER BY \"sk\" ASC OFFSET 3") + } + + @Test("An empty window writes nothing") + func emptyWindowText() { + let window = DynamoDBReadWindow() + #expect(window.isEmpty) + #expect(window.text.isEmpty) + #expect(!DynamoDBReadWindow(offset: 1).isEmpty) + #expect(!DynamoDBReadWindow(limit: 0).isEmpty) + } + + // MARK: - Quoting + + @Test("An identifier is wrapped in double quotes with inner quotes doubled") + func quoteDoublesInnerQuotes() { + #expect(DynamoDBStatement.quote("Orders") == "\"Orders\"") + #expect(DynamoDBStatement.quote("a\"b") == "\"a\"\"b\"") + #expect(DynamoDBStatement.quote("\"") == "\"\"\"\"") + #expect(DynamoDBStatement.quote("") == "\"\"") + } + + @Test( + "A quoted identifier reads back as the same name", + arguments: ["Orders", "a\"b", "\"\"", "it's", "with space", "-- not a comment", "/* nor this */"] + ) + func quoteRoundTripsThroughTheTokenizer(name: String) { + let tokens = DynamoDBPartiQL.tokens(of: DynamoDBStatement.quote(name)) + #expect(tokens.count == 1) + #expect(tokens.first?.kind == .quotedIdentifier) + #expect(tokens.first?.identifierValue == name) + } +} diff --git a/TableProTests/Plugins/DynamoDB/DynamoDBTableManagementTests.swift b/TableProTests/Plugins/DynamoDB/DynamoDBTableManagementTests.swift new file mode 100644 index 0000000000..f4ac8c479c --- /dev/null +++ b/TableProTests/Plugins/DynamoDB/DynamoDBTableManagementTests.swift @@ -0,0 +1,348 @@ +// +// DynamoDBTableManagementTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +@Suite("DynamoDB table management statements") +struct DynamoDBTableManagementTests { + private typealias Field = DynamoDBTableDefinition.Field + + private static let driver = DynamoDBPluginDriver( + config: DriverConnectionConfig(host: "", port: 0, username: "", password: "", database: ""), + catalog: DynamoDBCatalog() + ) + + private static func request(_ text: String) throws -> DynamoDBJSON { + let statement = try DynamoDBStatement.parse(text) + guard case .apiCall(let call, _) = statement else { + Issue.record("Not an API call: \(text)") + return .null + } + return call.body + } + + private static func formError(_ body: () throws -> Void) -> PluginCreateTableFormError? { + do { + try body() + return nil + } catch let error as PluginCreateTableFormError { + return error + } catch { + return nil + } + } + + // MARK: - Create Table form + + @Test("A table with only a partition key is on-demand and declares that key alone") + func minimalTable() throws { + let body = try DynamoDBTableDefinition.createTableRequest(from: PluginCreateTableRequest( + tableName: " Orders ", values: [Field.partitionKeyName: "pk", Field.partitionKeyType: "S"] + )) + + #expect(body["TableName"] == .string("Orders")) + #expect(body["BillingMode"] == .string("PAY_PER_REQUEST")) + #expect(body["KeySchema"] == .array([.object(["AttributeName": .string("pk"), "KeyType": .string("HASH")])])) + #expect(body["AttributeDefinitions"] == .array([ + .object(["AttributeName": .string("pk"), "AttributeType": .string("S")]) + ])) + #expect(body["ProvisionedThroughput"] == nil) + #expect(body["TableClass"] == nil) + #expect(body["DeletionProtectionEnabled"] == nil) + } + + @Test("A provisioned table gives its global indexes the same capacity and declares each key once") + func provisionedTableWithIndexes() throws { + let body = try DynamoDBTableDefinition.createTableRequest(from: PluginCreateTableRequest( + tableName: "Orders", + values: [ + Field.partitionKeyName: "pk", Field.partitionKeyType: "S", + Field.sortKeyName: "sk", Field.sortKeyType: "N", + Field.billingMode: "PROVISIONED", Field.readCapacity: "5", Field.writeCapacity: "7", + Field.tableClass: "STANDARD_INFREQUENT_ACCESS", Field.deletionProtection: "true" + ], + repeatedValues: [ + Field.globalIndexes: [[ + Field.indexName: "byStatus", Field.partitionKeyName: "status", Field.partitionKeyType: "S", + Field.sortKeyName: "sk", Field.sortKeyType: "N", Field.projection: "KEYS_ONLY" + ]], + Field.localIndexes: [[ + Field.indexName: "byTotal", Field.sortKeyName: "total", Field.sortKeyType: "N", + Field.projection: "INCLUDE", Field.includedAttributes: "a, b" + ]] + ] + )) + + let capacity: DynamoDBJSON = .object(["ReadCapacityUnits": .number("5"), "WriteCapacityUnits": .number("7")]) + let global = body["GlobalSecondaryIndexes"]?.arrayValue?.first + let local = body["LocalSecondaryIndexes"]?.arrayValue?.first + let declared = body["AttributeDefinitions"]?.arrayValue?.compactMap { $0["AttributeName"]?.stringValue } + #expect(body["ProvisionedThroughput"] == capacity) + #expect(global?["ProvisionedThroughput"] == capacity) + #expect(global?["Projection"] == .object(["ProjectionType": .string("KEYS_ONLY")])) + #expect(local?["ProvisionedThroughput"] == nil) + #expect(local?["KeySchema"]?.arrayValue?.first?["AttributeName"] == .string("pk")) + #expect(local?["Projection"]?["NonKeyAttributes"] == .array([.string("a"), .string("b")])) + #expect(declared == ["pk", "sk", "status", "total"]) + #expect(body["TableClass"] == .string("STANDARD_INFREQUENT_ACCESS")) + #expect(body["DeletionProtectionEnabled"] == .bool(true)) + } + + @Test("A local index on a table without a sort key is refused at the sort key field") + func localIndexNeedsTableSortKey() { + let error = Self.formError { + _ = try DynamoDBTableDefinition.createTableRequest(from: PluginCreateTableRequest( + tableName: "Orders", + values: [Field.partitionKeyName: "pk"], + repeatedValues: [Field.localIndexes: [[Field.indexName: "byTotal", Field.sortKeyName: "total"]]] + )) + } + + #expect(error?.fieldId == Field.sortKeyName) + } + + @Test("An attribute declared as two types is refused") + func conflictingTypesAreRefused() { + let error = Self.formError { + _ = try DynamoDBTableDefinition.createTableRequest(from: PluginCreateTableRequest( + tableName: "Orders", + values: [Field.partitionKeyName: "pk", Field.partitionKeyType: "S"], + repeatedValues: [Field.globalIndexes: [[ + Field.indexName: "byPk", Field.partitionKeyName: "pk", Field.partitionKeyType: "N" + ]]] + )) + } + + #expect(error?.fieldId == Field.partitionKeyType) + } + + @Test("An incomplete form names the field to fix") + func incompleteFormNamesField() { + let cases: [(values: [String: String], field: String)] = [ + ([Field.partitionKeyName: "pk", Field.billingMode: "PROVISIONED", Field.writeCapacity: "1"], Field.readCapacity), + ( + [Field.partitionKeyName: "pk", Field.billingMode: "PROVISIONED", Field.readCapacity: "1", Field.writeCapacity: "0"], + Field.writeCapacity + ), + ([Field.partitionKeyName: " "], Field.partitionKeyName) + ] + for testCase in cases { + let error = Self.formError { + _ = try DynamoDBTableDefinition.createTableRequest( + from: PluginCreateTableRequest(tableName: "Orders", values: testCase.values) + ) + } + #expect(error?.fieldId == testCase.field, "\(testCase.values)") + } + } + + @Test("An included projection with no attributes is refused") + func includeNeedsAttributes() { + let error = Self.formError { + _ = try DynamoDBTableDefinition.createTableRequest(from: PluginCreateTableRequest( + tableName: "Orders", + values: [Field.partitionKeyName: "pk"], + repeatedValues: [Field.globalIndexes: [[ + Field.indexName: "byStatus", Field.partitionKeyName: "status", Field.projection: "INCLUDE" + ]]] + )) + } + + #expect(error?.fieldId == Field.includedAttributes) + } + + @Test("A table name DynamoDB would reject is refused", arguments: ["ab", "has space", String(repeating: "a", count: 256)]) + func invalidTableNameIsRefused(name: String) { + let error = Self.formError { + _ = try DynamoDBTableDefinition.createTableRequest(from: PluginCreateTableRequest( + tableName: name, values: [Field.partitionKeyName: "pk"] + )) + } + + #expect(error != nil) + } + + @Test("The form's statement is a CreateTable request the editor parses back") + func formStatementRoundTrips() throws { + let statements = try Self.driver.createTableStatements( + for: PluginCreateTableRequest(tableName: "Orders", values: [Field.partitionKeyName: "pk"]), schema: nil + ) + + let body = try Self.request(try #require(statements.first)) + #expect(statements.count == 1) + #expect(body["TableName"] == .string("Orders")) + } + + // MARK: - Maintenance + + @Test("A stream turned off sends no view type") + func streamOff() throws { + let statement = try #require(Self.driver.maintenanceStatements( + operation: DynamoDBPluginDriver.MaintenanceName.stream, table: "Orders", schema: nil, + options: ["choice": String(localized: "Off")] + )?.first) + + let body = try Self.request(statement) + #expect(body["StreamSpecification"] == .object(["StreamEnabled": .bool(false)])) + } + + @Test("A stream turned on sends the chosen view type") + func streamOn() throws { + let statement = try #require(Self.driver.maintenanceStatements( + operation: DynamoDBPluginDriver.MaintenanceName.stream, table: "Orders", schema: nil, + options: ["choice": String(localized: "Keys only")] + )?.first) + + let body = try Self.request(statement) + #expect(body["StreamSpecification"] == .object([ + "StreamEnabled": .bool(true), "StreamViewType": .string("KEYS_ONLY") + ])) + } + + @Test("Each setting becomes the request that changes it") + func maintenanceRequests() throws { + let name = DynamoDBPluginDriver.MaintenanceName.self + let recovery = try Self.request(try #require(Self.driver.maintenanceStatements( + operation: name.pointInTimeRecovery, table: "Orders", schema: nil, options: ["enabled": "false"] + )?.first)) + let protection = try Self.request(try #require(Self.driver.maintenanceStatements( + operation: name.deletionProtection, table: "Orders", schema: nil, options: [:] + )?.first)) + let tableClass = try Self.request(try #require(Self.driver.maintenanceStatements( + operation: name.tableClass, table: "Orders", schema: nil, + options: ["choice": String(localized: "Standard-Infrequent Access")] + )?.first)) + let onDemand = try Self.request(try #require(Self.driver.maintenanceStatements( + operation: name.onDemand, table: "Orders", schema: nil, options: [:] + )?.first)) + + #expect(recovery["PointInTimeRecoverySpecification"] == .object(["PointInTimeRecoveryEnabled": .bool(false)])) + #expect(protection["DeletionProtectionEnabled"] == .bool(true)) + #expect(tableClass["TableClass"] == .string("STANDARD_INFREQUENT_ACCESS")) + #expect(onDemand["BillingMode"] == .string("PAY_PER_REQUEST")) + } + + @Test("An unknown operation, or one with no table, builds nothing") + func unknownMaintenance() { + #expect(Self.driver.maintenanceStatements(operation: "VACUUM", table: "Orders", schema: nil, options: [:]) == nil) + #expect(Self.driver.maintenanceStatements( + operation: DynamoDBPluginDriver.MaintenanceName.onDemand, table: nil, schema: nil, options: [:] + ) == nil) + } + + // MARK: - Indexes and drop + + @Test("An index edited in place is refused, and so is dropping the primary key or a local index") + func indexEditsAndDrops() { + let global = PluginIndexDefinition(name: "byStatus", columns: ["status"], indexType: "GLOBAL All attributes") + let local = PluginIndexDefinition(name: "byTotal", columns: ["pk", "total"], indexType: "LOCAL Keys only") + let primary = PluginIndexDefinition(name: "PRIMARY", columns: ["pk"], indexType: "PRIMARY KEY") + + #expect(Self.driver.schemaOperationRefusal(.modifyIndex(old: global, new: global)) != nil) + #expect(Self.driver.schemaOperationRefusal(.dropIndex(global)) == nil) + #expect(Self.driver.schemaOperationRefusal(.dropIndex(local)) != nil) + #expect(Self.driver.schemaOperationRefusal(.dropIndex(primary)) != nil) + } + + @Test("A global index from the Structure tab takes its keys from the columns and its projection from the included ones") + func addIndex() throws { + let index = PluginIndexDefinition( + name: "byStatus", columns: ["status", "total"], expressions: nil, includedColumns: ["a"], + ddlMethodAndKeys: nil, ddlWhereClause: nil + ) + let statement = try #require(Self.driver.generateAddIndexSQL(table: "Orders", index: index)) + + let create = try Self.request(statement)["GlobalSecondaryIndexUpdates"]?.arrayValue?.first?["Create"] + #expect(create?["KeySchema"] == .array([ + .object(["AttributeName": .string("status"), "KeyType": .string("HASH")]), + .object(["AttributeName": .string("total"), "KeyType": .string("RANGE")]) + ])) + #expect(create?["Projection"] == .object([ + "ProjectionType": .string("INCLUDE"), "NonKeyAttributes": .array([.string("a")]) + ])) + } + + @Test("A unique index, or one with more than two columns, is refused with a reason") + func refusedIndexes() { + let unique = PluginIndexDefinition(name: "u", columns: ["a"], isUnique: true) + let wide = PluginIndexDefinition(name: "w", columns: ["a", "b", "c"]) + + #expect(Self.driver.generateAddIndexSQL(table: "Orders", index: unique) == nil) + #expect(Self.driver.generateAddIndexSQL(table: "Orders", index: wide) == nil) + #expect(Self.driver.schemaOperationRefusal(.addIndex(unique)) != nil) + #expect(Self.driver.schemaOperationRefusal(.addIndex(wide)) != nil) + } + + @Test("Dropping an index deletes that global index, and the primary key is never dropped") + func dropIndex() throws { + let statement = try #require(Self.driver.generateDropIndexSQL(table: "Orders", indexName: "byStatus")) + + let update = try Self.request(statement)["GlobalSecondaryIndexUpdates"]?.arrayValue?.first + #expect(update == .object(["Delete": .object(["IndexName": .string("byStatus")])])) + #expect(Self.driver.generateDropIndexSQL(table: "Orders", indexName: "PRIMARY") == nil) + } + + @Test("Dropping a table is a DeleteTable request, and only for a valid table name") + func dropTable() throws { + let statement = try #require(Self.driver.dropObjectStatement( + name: "Orders", objectType: "TABLE", schema: nil, cascade: false + )) + + #expect(try Self.request(statement) == .object(["TableName": .string("Orders")])) + #expect(Self.driver.dropObjectStatement(name: "Or\"ders", objectType: "TABLE", schema: nil, cascade: false) == nil) + #expect(Self.driver.dropObjectStatement(name: "Orders", objectType: "VIEW", schema: nil, cascade: false) == nil) + } + + // MARK: - Field paths + + @Test("Field paths reach into maps up to four levels, and never through a list") + func fieldPaths() { + let item: DynamoDBItem = [ + "address": .map(["city": .string("Paris"), "geo": .map(["lat": .map(["deg": .map(["x": .number("1")])])])]), + "items": .list([.map(["sku": .string("A1")])]) + ] + + let paths = Dictionary(uniqueKeysWithValues: DynamoDBFieldPaths.collect(from: [item]).map { ($0.path, $0) }) + + #expect(paths["address.city"]?.typeName == "String") + #expect(paths["address.geo.lat.deg"]?.depth == 4) + #expect(paths["address.geo.lat.deg.x"] == nil) + #expect(paths["items"]?.typeName == "List") + #expect(paths["items.sku"] == nil) + } + + // MARK: - Row limits and ORDER BY + + @Test("An injected row limit keeps a smaller one and survives a trailing comment") + func injectRowLimit() { + let driver = Self.driver + + let kept = driver.injectRowLimit("SELECT * FROM \"t\" LIMIT 3", limit: 10) + let commented = driver.injectRowLimit("SELECT * FROM \"t\" -- note", limit: 5) + + #expect(kept?.hasSuffix("\nLIMIT 3") == true) + #expect(kept?.contains("LIMIT 10") == false) + #expect(commented?.hasSuffix("-- note\nLIMIT 5") == true) + #expect(driver.injectRowLimit(#"Scan {"TableName":"t"}"#, limit: 5) == #"Scan {"TableName":"t"} LIMIT 5"#) + #expect(driver.injectRowLimit("DELETE FROM \"t\" WHERE \"pk\" = 'a'", limit: 5) == nil) + #expect(driver.injectRowLimit(#"PutItem {"TableName":"t"}"#, limit: 5) == nil) + } + + @Test("The read limits never go negative and never overflow") + func readLimits() { + func limits(_ window: DynamoDBReadWindow, _ cap: Int) -> [Int] { + let result = DynamoDBPluginDriver.readLimits(window: window, rowCap: cap) + return [result.wanted, result.read] + } + + #expect(limits(DynamoDBReadWindow(), 100) == [100, 101]) + #expect(limits(DynamoDBReadWindow(), Int.max) == [Int.max, Int.max]) + #expect(limits(DynamoDBReadWindow(limit: 5), 100) == [5, 5]) + #expect(limits(DynamoDBReadWindow(limit: 500), 100) == [100, 101]) + #expect(limits(DynamoDBReadWindow(limit: -1), 100) == [0, 0]) + } +} diff --git a/TableProTests/Plugins/DynamoDB/DynamoDBWriteStatementsTests.swift b/TableProTests/Plugins/DynamoDB/DynamoDBWriteStatementsTests.swift new file mode 100644 index 0000000000..bc57b89c01 --- /dev/null +++ b/TableProTests/Plugins/DynamoDB/DynamoDBWriteStatementsTests.swift @@ -0,0 +1,279 @@ +import Foundation +import TableProPluginKit +import Testing + +@Suite("DynamoDB write statements") +struct DynamoDBWriteStatementsTests { + typealias CellChange = (columnIndex: Int, columnName: String, oldValue: PluginCellValue, newValue: PluginCellValue) + + private let writer = DynamoDBWriteStatements( + table: "Orders", + columns: ["pk", "sk", "name", "total", "o'clock"], + keyColumns: ["pk", "sk"] + ) + + private let originalRow: [PluginCellValue] = ["p1", "7", "Ann", "10", .null] + + private func cell( + _ index: Int, + _ name: String, + _ old: PluginCellValue, + _ new: PluginCellValue + ) -> CellChange { + (columnIndex: index, columnName: name, oldValue: old, newValue: new) + } + + private func update(_ cells: [CellChange], row: Int = 0) -> PluginRowChange { + PluginRowChange(rowIndex: row, type: .update, cellChanges: cells, originalRow: originalRow) + } + + private func generate( + _ changes: [PluginRowChange], + insertedRowData: [Int: [PluginCellValue]] = [:], + deleted: Set = [], + inserted: Set = [] + ) -> [(statement: String, parameters: [PluginCellValue])] { + writer.statements( + for: changes, + insertedRowData: insertedRowData, + deletedRowIndices: deleted, + insertedRowIndices: inserted + ) + } + + private static func schema() throws -> DynamoDBTableSchema { + try DynamoDBTableSchema(describeTableResponse: DynamoDBJSON.parse(""" + {"Table": { + "TableName": "Orders", + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"} + ], + "AttributeDefinitions": [ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "N"} + ] + }} + """)) + } + + // MARK: - UPDATE + + @Test("A changed value is set, guarded by its loaded value and keyed by the original row") + func updateChangedValue() throws { + let result = generate([update([cell(2, "name", "Ann", "Bea")])]) + let statement = try #require(result.first) + #expect(result.count == 1) + #expect(statement.statement == "UPDATE \"Orders\" SET \"name\" = ? WHERE \"pk\" = ? AND \"sk\" = ? AND \"name\" = ?") + #expect(statement.parameters == ["Bea", "p1", "7", "Ann"]) + } + + @Test("A value set to NULL is removed, still guarded by its loaded value") + func updateToNullRemoves() throws { + let statement = try #require(generate([update([cell(3, "total", "10", .null)])]).first) + #expect(statement.statement == "UPDATE \"Orders\" REMOVE \"total\" WHERE \"pk\" = ? AND \"sk\" = ? AND \"total\" = ?") + #expect(statement.parameters == ["p1", "7", "10"]) + } + + @Test("SET comes before REMOVE, and parameters run SET values, key, then guards") + func updateParameterOrder() throws { + let statement = try #require(generate([update([ + cell(2, "name", "Ann", "Bea"), + cell(3, "total", "10", .null), + cell(4, "o'clock", .null, "noon") + ])]).first) + #expect(statement.statement == """ + UPDATE "Orders" SET "name" = ?, "o'clock" = ? REMOVE "total" \ + WHERE "pk" = ? AND "sk" = ? AND "name" = ? AND "total" = ? + """) + #expect(statement.parameters == ["Bea", "noon", "p1", "7", "Ann", "10"]) + } + + @Test("A value that was NULL is set without a guard") + func updateFromNullHasNoGuard() throws { + let statement = try #require(generate([update([cell(4, "o'clock", .null, "noon")])]).first) + #expect(statement.statement == "UPDATE \"Orders\" SET \"o'clock\" = ? WHERE \"pk\" = ? AND \"sk\" = ?") + #expect(statement.parameters == ["noon", "p1", "7"]) + } + + @Test("A binary value is guarded and set as bytes") + func updateBinaryValue() throws { + let old = Data([0x01, 0x02]) + let new = Data([0x03]) + let statement = try #require(generate([update([cell(3, "total", .bytes(old), .bytes(new))])]).first) + #expect(statement.parameters == [.bytes(new), "p1", "7", .bytes(old)]) + } + + @Test("A changed key is still written, keyed by the original values and never guarded") + func updateChangedKey() throws { + let statement = try #require(generate([update([cell(0, "pk", "p1", "p2")])]).first) + #expect(statement.statement == "UPDATE \"Orders\" SET \"pk\" = ? WHERE \"pk\" = ? AND \"sk\" = ?") + #expect(statement.parameters == ["p2", "p1", "7"]) + } + + @Test("The binder refuses the key change the generator writes") + func changedKeyIsRefusedWhenBound() throws { + let statement = try #require(generate([update([cell(1, "sk", "7", "8")])]).first) + let roles = DynamoDBPartiQL.parameterRoles(in: statement.statement) + #expect(roles == [ + .assigned(DynamoDBAttributePath(attribute: "sk")), + .compared(DynamoDBAttributePath(attribute: "pk")), + .compared(DynamoDBAttributePath(attribute: "sk")) + ]) + + let binder = DynamoDBParameterBinder(schema: try Self.schema(), observedTypes: [:], currentItem: nil) + #expect(throws: DynamoDBError.self) { + try binder.bind(statement.parameters, roles: roles) + } + } + + @Test("The roles read from a generated UPDATE line up with its parameters") + func updateRolesLineUp() throws { + let statement = try #require(generate([update([ + cell(2, "name", "Ann", "Bea"), + cell(3, "total", "10", .null), + cell(4, "o'clock", .null, "noon") + ])]).first) + let roles = DynamoDBPartiQL.parameterRoles(in: statement.statement) + #expect(roles == [ + .assigned(DynamoDBAttributePath(attribute: "name")), .assigned(DynamoDBAttributePath(attribute: "o'clock")), + .compared(DynamoDBAttributePath(attribute: "pk")), + .compared(DynamoDBAttributePath(attribute: "sk")), + .compared(DynamoDBAttributePath(attribute: "name")), + .compared(DynamoDBAttributePath(attribute: "total")) + ]) + #expect(roles.count == statement.parameters.count) + } + + @Test("An update with no original row or no changed cells writes nothing") + func updateWithoutOriginalOrCells() { + let noOriginal = PluginRowChange( + rowIndex: 0, type: .update, cellChanges: [cell(2, "name", "Ann", "Bea")], originalRow: nil + ) + let noCells = PluginRowChange(rowIndex: 1, type: .update, cellChanges: [], originalRow: originalRow) + #expect(generate([noOriginal, noCells]).isEmpty) + } + + @Test("Table and attribute names with double quotes are escaped") + func updateQuotesNames() throws { + let quoted = DynamoDBWriteStatements(table: "Or\"ders", columns: ["p\"k", "a\"b"], keyColumns: ["p\"k"]) + let change = PluginRowChange( + rowIndex: 0, + type: .update, + cellChanges: [cell(1, "a\"b", "x", "y")], + originalRow: ["k", "x"] + ) + let result = quoted.statements( + for: [change], insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: [] + ) + let statement = try #require(result.first) + #expect(statement.statement == "UPDATE \"Or\"\"ders\" SET \"a\"\"b\" = ? WHERE \"p\"\"k\" = ? AND \"a\"\"b\" = ?") + #expect(DynamoDBPartiQL.target(of: statement.statement)?.table == "Or\"ders") + #expect(DynamoDBPartiQL.parameterRoles(in: statement.statement) == [ + .assigned(DynamoDBAttributePath(attribute: "a\"b")), + .compared(DynamoDBAttributePath(attribute: "p\"k")), + .compared(DynamoDBAttributePath(attribute: "a\"b")) + ]) + } + + // MARK: - INSERT + + @Test("An insert leaves out NULL cells and the default sentinel") + func insertSkipsNullAndDefault() { + let data = Data([0xFF]) + let result = writer.insert(row: ["p1", "7", .null, "__DEFAULT__", .bytes(data)]) + #expect(result.statement == "INSERT INTO \"Orders\" VALUE {'pk': ?, 'sk': ?, 'o''clock': ?}") + #expect(result.parameters == ["p1", "7", .bytes(data)]) + } + + @Test("An empty string is inserted, only the exact sentinel is skipped") + func insertKeepsEmptyText() { + let result = writer.insert(row: ["p1", "7", "", "__default__", " __DEFAULT__"]) + #expect(result.statement == "INSERT INTO \"Orders\" VALUE {'pk': ?, 'sk': ?, 'name': ?, 'total': ?, 'o''clock': ?}") + #expect(result.parameters == ["p1", "7", "", "__default__", " __DEFAULT__"]) + } + + @Test("The roles read from a generated INSERT name each attribute") + func insertRolesLineUp() { + let result = writer.insert(row: ["p1", "7", .null, "10", "noon"]) + #expect(DynamoDBPartiQL.parameterRoles(in: result.statement) == [ + .inserted("pk"), .inserted("sk"), .inserted("total"), .inserted("o'clock") + ]) + } + + @Test("Single quotes in attribute names are doubled") + func literalDoublesQuotes() { + #expect(DynamoDBWriteStatements.literal("name") == "'name'") + #expect(DynamoDBWriteStatements.literal("it's") == "'it''s'") + #expect(DynamoDBWriteStatements.literal("''") == "''''''") + } + + @Test("An inserted row with no row data is built from its changed cells") + func insertFromCellChanges() throws { + let change = PluginRowChange( + rowIndex: 5, + type: .insert, + cellChanges: [cell(0, "pk", .null, "p9"), cell(2, "name", .null, "Cy"), cell(9, "ghost", .null, "z")], + originalRow: nil + ) + let statement = try #require(generate([change], inserted: [5]).first) + #expect(statement.statement == "INSERT INTO \"Orders\" VALUE {'pk': ?, 'name': ?}") + #expect(statement.parameters == ["p9", "Cy"]) + } + + @Test("Row data takes precedence over the changed cells of an inserted row") + func insertPrefersRowData() throws { + let change = PluginRowChange( + rowIndex: 5, type: .insert, cellChanges: [cell(0, "pk", .null, "stale")], originalRow: nil + ) + let statement = try #require(generate([change], insertedRowData: [5: ["p5", "1"]], inserted: [5]).first) + #expect(statement.statement == "INSERT INTO \"Orders\" VALUE {'pk': ?, 'sk': ?}") + #expect(statement.parameters == ["p5", "1"]) + } + + // MARK: - DELETE + + @Test("A delete is keyed by the original row and returns the old item") + func deleteStatement() throws { + let change = PluginRowChange(rowIndex: 2, type: .delete, cellChanges: [], originalRow: originalRow) + let statement = try #require(generate([change], deleted: [2]).first) + #expect(statement.statement == "DELETE FROM \"Orders\" WHERE \"pk\" = ? AND \"sk\" = ? RETURNING ALL OLD *") + #expect(statement.parameters == ["p1", "7"]) + #expect(DynamoDBPartiQL.hasReturning(statement.statement)) + #expect(DynamoDBPartiQL.parameterRoles(in: statement.statement) == [ + .compared(DynamoDBAttributePath(attribute: "pk")), + .compared(DynamoDBAttributePath(attribute: "sk")) + ]) + } + + @Test("A delete with no original row writes nothing") + func deleteWithoutOriginal() { + let change = PluginRowChange(rowIndex: 2, type: .delete, cellChanges: [], originalRow: nil) + #expect(generate([change], deleted: [2]).isEmpty) + } + + // MARK: - Batches + + @Test("Only rows still marked inserted or deleted are written, in change order") + func statementsRespectIndices() { + let changes = [ + PluginRowChange(rowIndex: 3, type: .insert, cellChanges: [], originalRow: nil), + PluginRowChange(rowIndex: 4, type: .insert, cellChanges: [], originalRow: nil), + PluginRowChange(rowIndex: 1, type: .delete, cellChanges: [], originalRow: ["d1", "1", .null, .null, .null]), + PluginRowChange(rowIndex: 2, type: .delete, cellChanges: [], originalRow: ["d2", "2", .null, .null, .null]), + update([cell(2, "name", "Ann", "Bea")]) + ] + let result = generate( + changes, + insertedRowData: [3: ["n3", "3"], 4: ["n4", "4"]], + deleted: [2], + inserted: [3] + ) + #expect(result.map(\.statement) == [ + "INSERT INTO \"Orders\" VALUE {'pk': ?, 'sk': ?}", + "DELETE FROM \"Orders\" WHERE \"pk\" = ? AND \"sk\" = ? RETURNING ALL OLD *", + "UPDATE \"Orders\" SET \"name\" = ? WHERE \"pk\" = ? AND \"sk\" = ? AND \"name\" = ?" + ]) + #expect(result.map(\.parameters) == [["n3", "3"], ["d2", "2"], ["Bea", "p1", "7", "Ann"]]) + } +} diff --git a/TableProTests/Plugins/DynamoDBMetadataParityTests.swift b/TableProTests/Plugins/DynamoDBMetadataParityTests.swift new file mode 100644 index 0000000000..6025dc2e46 --- /dev/null +++ b/TableProTests/Plugins/DynamoDBMetadataParityTests.swift @@ -0,0 +1,114 @@ +// +// DynamoDBMetadataParityTests.swift +// TableProTests +// +// The app describes DynamoDB from a curated copy of the plugin's statics until the registry plugin +// loads. `DynamoDBPlugin.swift` is compiled into this target, so the two copies are compared here +// instead of drifting apart until the plugin silently replaces one with the other. +// + +import Foundation +import TableProPluginKit +import Testing + +@testable import TablePro + +@Suite("DynamoDB curated metadata parity") +struct DynamoDBMetadataParityTests { + private func curated() throws -> PluginMetadataSnapshot { + try #require(PluginMetadataRegistry.shared.builtInDefaults().first { $0.typeId == "DynamoDB" }?.snapshot) + } + + @Test("Identity and structure editing match the plugin") + func identityAndStructureEditing() throws { + let snapshot = try curated() + + #expect(snapshot.displayName == DynamoDBPlugin.databaseDisplayName) + #expect(snapshot.iconName == DynamoDBPlugin.iconName) + #expect(snapshot.brandColorHex == DynamoDBPlugin.brandColorHex) + #expect(snapshot.queryLanguageName == DynamoDBPlugin.queryLanguageName) + #expect(snapshot.editorLanguage == DynamoDBPlugin.editorLanguage) + #expect(snapshot.parameterStyle == DynamoDBPlugin.parameterStyle) + #expect(snapshot.connectionMode == DynamoDBPlugin.connectionMode) + #expect(snapshot.supportsSchemaEditing == DynamoDBPlugin.supportsSchemaEditing) + #expect(snapshot.supportsForeignKeys == DynamoDBPlugin.supportsForeignKeys) + #expect(snapshot.supportsDatabaseSwitching == DynamoDBPlugin.supportsDatabaseSwitching) + #expect(snapshot.schema.structureColumnFields == DynamoDBPlugin.structureColumnFields) + #expect(snapshot.schema.databaseGroupingStrategy == DynamoDBPlugin.databaseGroupingStrategy) + #expect(snapshot.schema.defaultGroupName == DynamoDBPlugin.defaultGroupName) + #expect(snapshot.schema.tableEntityName == DynamoDBPlugin.tableEntityName) + } + + @Test("Structure capabilities match the plugin") + func structureCapabilities() throws { + let capabilities = try curated().capabilities + + #expect(capabilities.supportsAddColumn == DynamoDBPlugin.supportsAddColumn) + #expect(capabilities.supportsModifyColumn == DynamoDBPlugin.supportsModifyColumn) + #expect(capabilities.supportsDropColumn == DynamoDBPlugin.supportsDropColumn) + #expect(capabilities.supportsRenameColumn == DynamoDBPlugin.supportsRenameColumn) + #expect(capabilities.supportsModifyPrimaryKey == DynamoDBPlugin.supportsModifyPrimaryKey) + #expect(capabilities.supportsAddIndex == DynamoDBPlugin.supportsAddIndex) + #expect(capabilities.supportsDropIndex == DynamoDBPlugin.supportsDropIndex) + #expect(capabilities.supportsImport == DynamoDBPlugin.supportsImport) + #expect(capabilities.supportsExport == DynamoDBPlugin.supportsExport) + #expect(capabilities.supportsSSH == DynamoDBPlugin.supportsSSH) + #expect(capabilities.supportsSSL == DynamoDBPlugin.supportsSSL) + #expect(capabilities.supportsReadOnlyMode == DynamoDBPlugin.supportsReadOnlyMode) + #expect(capabilities.exactRowCountIsBilledScan) + } + + @Test("Editor metadata matches the plugin") + func editor() throws { + let snapshot = try curated() + let dialect = try #require(snapshot.editor.sqlDialect) + let shipped = try #require(DynamoDBPlugin.sqlDialect) + + #expect(dialect.identifierQuote == shipped.identifierQuote) + #expect(dialect.keywords == shipped.keywords) + #expect(dialect.functions == shipped.functions) + #expect(dialect.dataTypes == shipped.dataTypes) + #expect(dialect.booleanLiteralStyle == shipped.booleanLiteralStyle) + #expect(dialect.autoLimitStyle == shipped.autoLimitStyle) + #expect(dialect.caseSensitivityStyle == shipped.caseSensitivityStyle) + #expect(dialect.caseSensitivityStyle == .driverManaged) + #expect(DynamoDBPlugin.caseSensitivityStyle == .driverManaged) + #expect(snapshot.editor.columnTypesByCategory == DynamoDBPlugin.columnTypesByCategory) + #expect(snapshot.editor.statementCompletions.map { [$0.label, $0.insertText] } + == DynamoDBPlugin.statementCompletions.map { [$0.label, $0.insertText] }) + } + + @Test("Connection fields match the plugin") + func connectionFields() throws { + let fields = try curated().connection.additionalConnectionFields + let shipped = DynamoDBConnectionFields.all + + #expect(fields.map(\.id) == shipped.map(\.id)) + #expect(fields.map(\.label) == shipped.map(\.label)) + #expect(fields.map(\.placeholder) == shipped.map(\.placeholder)) + #expect(fields.map(\.defaultValue) == shipped.map(\.defaultValue)) + #expect(fields.map(\.fieldType) == shipped.map(\.fieldType)) + #expect(fields.map(\.section) == shipped.map(\.section)) + #expect(fields.map(\.hidesPassword) == shipped.map(\.hidesPassword)) + #expect(fields.map(\.visibleWhen) == shipped.map(\.visibleWhen)) + #expect(fields.map(\.dynamicOptions) == shipped.map(\.dynamicOptions)) + } + + @Test("The region names no default, so a profile's own region is used") + func regionHasNoDefault() throws { + let region = try #require(try curated().connection.additionalConnectionFields.first { $0.id == "awsRegion" }) + + #expect(region.defaultValue == nil) + } + + @Test("DynamoDB Local is offered as a sign-in method") + func localSignInIsOffered() throws { + let method = try #require(try curated().connection.additionalConnectionFields.first { $0.id == "awsAuthMethod" }) + guard case .dropdown(let options) = method.fieldType else { + Issue.record("The auth method is not a dropdown") + return + } + + #expect(options.map(\.value) == ["credentials", "profile", "sso", "local"]) + } +} diff --git a/TableProTests/Plugins/DynamoDBOperationsTests.swift b/TableProTests/Plugins/DynamoDBOperationsTests.swift deleted file mode 100644 index 2e67d2f36d..0000000000 --- a/TableProTests/Plugins/DynamoDBOperationsTests.swift +++ /dev/null @@ -1,57 +0,0 @@ -// -// DynamoDBOperationsTests.swift -// TableProTests -// - -import Foundation -@testable import TablePro -import Testing - -@Suite("DynamoDB table operations") -struct DynamoDBOperationsTests { - @Test("Dropping a table produces a statement the driver recognises") - func dropIsRecognised() throws { - let statement = try #require(DynamoDBOperations.dropTable(named: "orders", objectType: "TABLE")) - #expect(statement == "DROP TABLE \"orders\"") - #expect(DynamoDBOperations.droppedTableName(in: statement) == "orders") - } - - /// The statement is shown in the confirmation and then run, so a statement the driver's own - /// dispatch does not route would be #2884 in a third engine. - @Test("Every generated drop round-trips back to its table name", arguments: [ - "orders", "my-table", "my_table", "my.table", "Orders2024", - ]) - func dropRoundTrips(name: String) throws { - let statement = try #require(DynamoDBOperations.dropTable(named: name, objectType: "TABLE")) - #expect(DynamoDBOperations.droppedTableName(in: statement) == name) - } - - /// A DynamoDB table name is 3 to 255 characters of letters, digits, underscore, hyphen and dot. - /// A name holding anything else did not come from the table listing. - @Test("A name DynamoDB could not have is refused", arguments: [ - "", "ab", "has space", "has\"quote", "semi;colon", "star*", "slash/y", - ]) - func invalidNamesRefused(name: String) { - #expect(DynamoDBOperations.dropTable(named: name, objectType: "TABLE") == nil) - } - - @Test("Only a table is droppable", arguments: ["VIEW", "MATERIALIZED VIEW", "FOREIGN TABLE"]) - func onlyTablesAreDroppable(objectType: String) { - #expect(DynamoDBOperations.dropTable(named: "orders", objectType: objectType) == nil) - } - - @Test("Text that is not a drop statement routes nowhere", arguments: [ - "SELECT * FROM \"orders\"", "DROP TABLE orders", "DROP TABLE \"\"", "DELETE FROM \"orders\"", "", - ]) - func nonDropStatementsAreNotRouted(statement: String) { - #expect(DynamoDBOperations.droppedTableName(in: statement) == nil) - } - - /// PartiQL has no DDL, so this text is the driver's own vocabulary. It is spelled `DROP TABLE` - /// partly so the generic SQL classifier tiers it destructive without a DynamoDB arm. - @Test("A drop reads as destructive") - func dropClassifiesDestructive() throws { - let statement = try #require(DynamoDBOperations.dropTable(named: "orders", objectType: "TABLE")) - #expect(QueryClassifier.classify(statement, databaseType: .dynamodb).tier == .destructive) - } -} diff --git a/TableProTests/Plugins/DynamoDBQueryBuilderTests.swift b/TableProTests/Plugins/DynamoDBQueryBuilderTests.swift deleted file mode 100644 index 6d248f1f25..0000000000 --- a/TableProTests/Plugins/DynamoDBQueryBuilderTests.swift +++ /dev/null @@ -1,336 +0,0 @@ -// -// DynamoDBQueryBuilderTests.swift -// TableProTests -// -// Tests for DynamoDBQueryBuilder (compiled via symlink from DynamoDBDriverPlugin). -// - -import Foundation -import TableProPluginKit -import Testing - -@Suite("DynamoDBQueryBuilder - Browse Query") -struct DynamoDBQueryBuilderBrowseTests { - private let builder = DynamoDBQueryBuilder() - - @Test("Browse query returns scan-tagged string") - func browseReturnsScanTag() { - let query = builder.buildBrowseQuery(table: "Users", sortColumns: [], limit: 100, offset: 0) - #expect(query.hasPrefix(DynamoDBQueryBuilder.scanTag)) - } - - @Test("Browse query round-trips through parseScanQuery") - func browseRoundTrip() { - let query = builder.buildBrowseQuery(table: "Users", sortColumns: [], limit: 50, offset: 10) - let parsed = DynamoDBQueryBuilder.parseScanQuery(query) - #expect(parsed != nil) - #expect(parsed?.tableName == "Users") - #expect(parsed?.limit == 50) - #expect(parsed?.offset == 10) - #expect(parsed?.filters.isEmpty == true) - } -} - -@Suite("DynamoDBQueryBuilder - Filtered Query") -struct DynamoDBQueryBuilderFilteredTests { - private let builder = DynamoDBQueryBuilder() - - @Test("Without PK filter returns scan-tagged") - func nonPkFilterReturnsScan() { - let query = builder.buildFilteredQuery( - table: "Users", - filters: [PluginQueryFilter(column: "name", op: "=", value: "Alice")], - logicMode: "AND", - sortColumns: [], - columns: ["id", "name"], - limit: 100, - offset: 0, - keySchema: [("id", "HASH")] - ) - - #expect(query != nil) - #expect(query!.hasPrefix(DynamoDBQueryBuilder.scanTag)) - } - - @Test("With PK equals filter returns query-tagged") - func pkFilterReturnsQuery() { - let query = builder.buildFilteredQuery( - table: "Users", - filters: [PluginQueryFilter(column: "id", op: "=", value: "pk1")], - logicMode: "AND", - sortColumns: [], - columns: ["id", "name"], - limit: 100, - offset: 0, - keySchema: [("id", "HASH")] - ) - - #expect(query != nil) - #expect(query!.hasPrefix(DynamoDBQueryBuilder.queryTag)) - } - - @Test("PK filter with additional filters returns query-tagged with remaining filters") - func pkPlusAdditionalFilters() { - let query = builder.buildFilteredQuery( - table: "Users", - filters: [ - PluginQueryFilter(column: "id", op: "=", value: "pk1"), - PluginQueryFilter(column: "name", op: "CONTAINS", value: "Al") - ], - logicMode: "AND", - sortColumns: [], - columns: ["id", "name"], - limit: 100, - offset: 0, - keySchema: [("id", "HASH")] - ) - - #expect(query != nil) - #expect(query!.hasPrefix(DynamoDBQueryBuilder.queryTag)) - let parsed = DynamoDBQueryBuilder.parseQueryQuery(query!) - #expect(parsed != nil) - #expect(parsed?.partitionKeyValue == "pk1") - #expect(parsed?.filters.count == 1) - #expect(parsed?.filters.first?.column == "name") - } - - @Test("Multiple non-key filters returns scan-tagged with all filters") - func multipleNonKeyFilters() { - let query = builder.buildFilteredQuery( - table: "Users", - filters: [ - PluginQueryFilter(column: "name", op: "=", value: "Alice"), - PluginQueryFilter(column: "age", op: ">", value: "25") - ], - logicMode: "AND", - sortColumns: [], - columns: ["id", "name", "age"], - limit: 100, - offset: 0, - keySchema: [("id", "HASH")] - ) - - #expect(query != nil) - #expect(query!.hasPrefix(DynamoDBQueryBuilder.scanTag)) - let parsed = DynamoDBQueryBuilder.parseScanQuery(query!) - #expect(parsed?.filters.count == 2) - } -} - -// TODO: Re-enable when buildCombinedQuery API is restored or tests are updated -#if false -@Suite("DynamoDBQueryBuilder - Combined Query") -struct DynamoDBQueryBuilderCombinedTests { - private let builder = DynamoDBQueryBuilder() - - @Test("Filters only produces filtered query") - func filtersOnly() { - let query = builder.buildCombinedQuery( - table: "Users", - filters: [PluginQueryFilter(column: "name", op: "=", value: "Alice")], - logicMode: "AND", - searchText: "", - sortColumns: [], - limit: 100, - offset: 0, - keySchema: [("id", "HASH")] - ) - - #expect(query != nil) - #expect(query!.hasPrefix(DynamoDBQueryBuilder.scanTag)) - let parsed = DynamoDBQueryBuilder.parseScanQuery(query!) - #expect(parsed?.filters.count == 1) - } - - @Test("Search only produces scan with CONTAINS") - func searchOnly() { - let query = builder.buildCombinedQuery( - table: "Users", - filters: [], - logicMode: "AND", - searchText: "test", - sortColumns: [], - limit: 100, - offset: 0, - keySchema: [("id", "HASH")] - ) - - #expect(query != nil) - let parsed = DynamoDBQueryBuilder.parseScanQuery(query!) - #expect(parsed?.filters.count == 1) - #expect(parsed?.filters.first?.column == "*") - #expect(parsed?.filters.first?.op == "CONTAINS") - } - - @Test("Both filters and search are merged") - func filtersAndSearch() { - let query = builder.buildCombinedQuery( - table: "Users", - filters: [PluginQueryFilter(column: "name", op: "=", value: "Alice")], - logicMode: "AND", - searchText: "test", - sortColumns: [], - limit: 100, - offset: 0, - keySchema: [("id", "HASH")] - ) - - #expect(query != nil) - let parsed = DynamoDBQueryBuilder.parseScanQuery(query!) - #expect(parsed?.filters.count == 2) - } - - @Test("Empty filters and empty search produces plain scan") - func emptyBoth() { - let query = builder.buildCombinedQuery( - table: "Users", - filters: [], - logicMode: "AND", - searchText: "", - sortColumns: [], - limit: 100, - offset: 0, - keySchema: [("id", "HASH")] - ) - - #expect(query != nil) - let parsed = DynamoDBQueryBuilder.parseScanQuery(query!) - #expect(parsed?.filters.isEmpty == true) - } -} -#endif - -@Suite("DynamoDBQueryBuilder - Parse Scan Query") -struct DynamoDBQueryBuilderParseScanTests { - @Test("Valid scan string parses correctly") - func validScanParse() { - let builder = DynamoDBQueryBuilder() - let query = builder.buildBrowseQuery(table: "MyTable", sortColumns: [], limit: 200, offset: 50) - let parsed = DynamoDBQueryBuilder.parseScanQuery(query) - #expect(parsed != nil) - #expect(parsed?.tableName == "MyTable") - #expect(parsed?.limit == 200) - #expect(parsed?.offset == 50) - #expect(parsed?.logicMode == "AND") - } - - @Test("Invalid prefix returns nil") - func invalidPrefix() { - let parsed = DynamoDBQueryBuilder.parseScanQuery("SELECT * FROM users") - #expect(parsed == nil) - } - - @Test("Too few parts returns nil") - func tooFewParts() { - let parsed = DynamoDBQueryBuilder.parseScanQuery("DYNAMODB_SCAN:abc:123") - #expect(parsed == nil) - } -} - -@Suite("DynamoDBQueryBuilder - Parse Query Query") -struct DynamoDBQueryBuilderParseQueryTests { - @Test("Valid query string parses correctly") - func validQueryParse() { - let builder = DynamoDBQueryBuilder() - let query = builder.buildFilteredQuery( - table: "Users", - filters: [PluginQueryFilter(column: "id", op: "=", value: "pk1")], - logicMode: "AND", - sortColumns: [], - columns: ["id", "name"], - limit: 100, - offset: 0, - keySchema: [("id", "HASH")] - ) - - #expect(query != nil) - let parsed = DynamoDBQueryBuilder.parseQueryQuery(query!) - #expect(parsed != nil) - #expect(parsed?.tableName == "Users") - #expect(parsed?.partitionKeyName == "id") - #expect(parsed?.partitionKeyValue == "pk1") - #expect(parsed?.partitionKeyType == "S") - #expect(parsed?.limit == 100) - #expect(parsed?.offset == 0) - } - - @Test("Too few parts returns nil") - func tooFewParts() { - let parsed = DynamoDBQueryBuilder.parseQueryQuery("DYNAMODB_QUERY:abc:123:456") - #expect(parsed == nil) - } -} - -@Suite("DynamoDBQueryBuilder - Parse Count Query") -struct DynamoDBQueryBuilderParseCountTests { - @Test("Basic count query parses correctly") - func basicCount() { - let query = DynamoDBQueryBuilder.encodeCountQuery(tableName: "Users") - let parsed = DynamoDBQueryBuilder.parseCountQuery(query) - #expect(parsed != nil) - #expect(parsed?.tableName == "Users") - #expect(parsed?.filterColumn == nil) - #expect(parsed?.filterOp == nil) - #expect(parsed?.filterValue == nil) - } - - @Test("Count with filter parses correctly") - func countWithFilter() { - let query = DynamoDBQueryBuilder.encodeCountQuery( - tableName: "Users", - filterColumn: "status", - filterOp: "=", - filterValue: "active" - ) - let parsed = DynamoDBQueryBuilder.parseCountQuery(query) - #expect(parsed != nil) - #expect(parsed?.tableName == "Users") - #expect(parsed?.filterColumn == "status") - #expect(parsed?.filterOp == "=") - #expect(parsed?.filterValue == "active") - } - - @Test("Wrong prefix returns nil") - func wrongPrefix() { - let parsed = DynamoDBQueryBuilder.parseCountQuery("DYNAMODB_SCAN:abc:100:0:W10=:QU5E") - #expect(parsed == nil) - } -} - -@Suite("DynamoDBQueryBuilder - isTaggedQuery") -struct DynamoDBQueryBuilderIsTaggedTests { - @Test("Scan-tagged string returns true") - func scanTagged() { - let builder = DynamoDBQueryBuilder() - let query = builder.buildBrowseQuery(table: "T", sortColumns: [], limit: 10, offset: 0) - #expect(DynamoDBQueryBuilder.isTaggedQuery(query)) - } - - @Test("Query-tagged string returns true") - func queryTagged() { - let builder = DynamoDBQueryBuilder() - let query = builder.buildFilteredQuery( - table: "T", - filters: [PluginQueryFilter(column: "id", op: "=", value: "x")], - logicMode: "AND", - sortColumns: [], - columns: ["id"], - limit: 10, - offset: 0, - keySchema: [("id", "HASH")] - ) - #expect(query != nil) - #expect(DynamoDBQueryBuilder.isTaggedQuery(query!)) - } - - @Test("Count-tagged string returns true") - func countTagged() { - let query = DynamoDBQueryBuilder.encodeCountQuery(tableName: "T") - #expect(DynamoDBQueryBuilder.isTaggedQuery(query)) - } - - @Test("Regular SQL returns false") - func regularSql() { - #expect(!DynamoDBQueryBuilder.isTaggedQuery("SELECT * FROM users")) - } -} diff --git a/docs/databases/dynamodb.mdx b/docs/databases/dynamodb.mdx index c8b90a2ac7..30a68628d8 100644 --- a/docs/databases/dynamodb.mdx +++ b/docs/databases/dynamodb.mdx @@ -1,41 +1,44 @@ --- title: Amazon DynamoDB -description: Connect to Amazon DynamoDB with PartiQL queries, GSI/LSI browsing, and DynamoDB Local support +description: Browse DynamoDB tables through their indexes, edit items with their types intact, and run PartiQL or DynamoDB API requests --- import RegistryPlugin from "/snippets/registry-plugin.mdx"; -Opening a table runs a Scan. A filter becomes a Query only when it pins the partition key to a single value with `=`; every other filter is applied to the items after they come back, so it narrows what you see and not what the table read. +What a table read costs depends on the filter bar. Pin the partition key with `=` and the grid runs a Query that reads one partition; leave it out and the grid runs a Scan, which reads the whole table and is billed for every item it reads, including the ones your filters then drop. ## Quick setup -Click **New Connection…**, select **DynamoDB**, choose an **Auth Method**, enter credentials and region, then click **Save & Connect**. +Click **New Connection…**, select **DynamoDB**, choose an **Auth Method**, fill in its fields and the **AWS Region**, then click **Save & Connect**. - - DynamoDB connection form - DynamoDB connection form + + DynamoDB connection form with Auth Method, Access Key ID, Secret Access Key, Session Token, AWS Region and Custom Endpoint fields + DynamoDB connection form with Auth Method, Access Key ID, Secret Access Key, Session Token, AWS Region and Custom Endpoint fields ## Connection settings -The form asks for no host, no port and no database, and offers no SSL/TLS section. Every request is an HTTPS call to the AWS endpoint signed with SigV4, and one connection sees one region's tables. +There is no host, port, database or connection URL. A connection is one AWS account in one region, and the sidebar lists that region's tables. Every region works, as does DynamoDB Local. | Field | Description | |-------|-------------| -| **AWS Region** | Region the tables live in. Defaults to `us-east-1` | -| **Custom Endpoint** | Overrides the endpoint URL. Leave it empty unless you run DynamoDB Local | +| **AWS Region** | Region the tables live in. Left empty, the region comes from the AWS profile, then `us-east-1` | +| **Custom Endpoint** | Replaces the regional endpoint. Leave it empty unless you run DynamoDB Local or a VPC endpoint | + +The endpoint follows the region's partition, so `cn-north-1` reaches `dynamodb.cn-north-1.amazonaws.com.cn` and the European Sovereign Cloud reaches its own domain without a custom endpoint. ## Authentication | Auth Method | Fields | Credentials come from | |-------------|--------|----------------------| -| **Access Key + Secret Key** | **Access Key ID**, **Secret Access Key**, and **Session Token** for STS | What you type | +| **Access Key + Secret Key** | **Access Key ID**, **Secret Access Key**, **Session Token** for temporary credentials | What you type | | **AWS Profile** | **Profile Name** | That profile in `~/.aws/config` and `~/.aws/credentials`, read the way the AWS CLI reads it | -| **AWS SSO** | **Profile Name** | The IAM Identity Center token cached in `~/.aws/sso/cache`. Run `aws sso login --profile ` first | +| **AWS SSO** | **Profile Name** | The IAM Identity Center session for that profile. An expired session opens the sign-in prompt | +| **DynamoDB Local (no credentials)** | None | Fixed placeholder keys that DynamoDB Local accepts | -Pick **AWS Profile** if the AWS CLI already works on this Mac. Static keys, `credential_process` helpers and `role_arn` chains all resolve; the rules are on [AWS IAM Authentication](/connections/aws-iam#profiles). +Pick **AWS Profile** if the AWS CLI already works on this Mac. Static keys, `credential_process` helpers and `role_arn` chains all resolve; the rules are on [AWS IAM Authentication](/connections/aws-iam#profiles). Expired temporary credentials are fetched again and the request retried once. ## DynamoDB Local @@ -43,32 +46,106 @@ Pick **AWS Profile** if the AWS CLI already works on this Mac. Static keys, `cre docker run -p 8000:8000 amazon/dynamodb-local ``` -Set **Auth Method** to Access Key + Secret Key, put any non-empty string in **Access Key ID** and **Secret Access Key**, and set **Custom Endpoint** to `http://localhost:8000`. +Set **Auth Method** to **DynamoDB Local (no credentials)**. With **Custom Endpoint** empty it connects to `http://localhost:8000`; fill it in for any other port. + +## How a table is read + +Opening a table, filtering it and paging it all go through one planner. It picks the cheapest read your filters allow and names it at the start of the status bar, followed by how many items came back, how many DynamoDB read to find them, and the read capacity units it charged: `Query on index byStatus · 50 returned · 212 read · 26.5 RCU`. + +| Filters | Read | +|---------|------| +| Partition key `=` one value | **Query on the table** | +| Partition key `IN` a list, or several partition keys joined by **Match Any** | One Query per partition, or **Get by key** when the table has no sort key | +| A global index's partition key `=`, with a filter on each of its sort keys | **Query on index** | +| The partition key `=` and a key-condition filter on a local index's sort key | **Query on index** for that local index | +| Anything else | **Scan with filters** | + +A filter on the sort key of the chosen table or index joins the key condition when it is `=`, `<`, `<=`, `>`, `>=`, **BETWEEN** or **STARTS WITH**. Other filters become a FilterExpression, which DynamoDB applies after reading, so they narrow the rows without lowering the bill. **ENDS WITH**, regular expressions, matches that ignore case and a search across every column are checked on the returned items instead. **CONTAINS** means what DynamoDB's `contains` means on either side: text inside a String, or a member of a set or a list. **STARTS WITH** ignores case until you select **Match Case** in its operator menu, and only then can it run as a key condition. + +A global index is used only when it projects every attribute, and only when every one of its sort keys carries a filter, because an item missing a key attribute is not in the index at all. + +Filter on a nested attribute with a path such as `address.city` or `items[0].sku`. Autocomplete offers the map paths found in the table's first 100 items. + +### Sorting and paging + +Clicking the sort key's header on a Query that reads one partition makes DynamoDB return the items in that order, forwards or backwards. Any other sort needs every matching item. When the read holds them all, the grid sorts them. When it does not, the rows stay in DynamoDB's order and the status bar says so: `In DynamoDB order: sorting by total needs the whole result, or a Query on that sort key`. + +The next page starts from where the previous one stopped rather than reading the table again from the start. A page can take several requests: DynamoDB counts the items it examined against the request's limit before a FilterExpression drops any, so the reader keeps going until the page is full or the table has no more. -## Columns over schemaless items +### Row counts -A table declares only its key attributes, so the grid builds columns from the items it fetched: the union of their attribute names, partition key first, sort key next, the rest alphabetically. An attribute none of those items carries gets no column. Each column's type is a majority vote over the same items; the Structure tab votes on a sample of up to 100. +The total under the grid is DynamoDB's own item count, which it refreshes about every six hours. No count runs on its own. **Count Exactly** reads every item your filters match with `Select: COUNT`, so it is billed as a full read of those items. The query timeout does not stop it; `Cmd+.` does. + +## Items in the grid + +A table declares only its key attributes, so the columns come from the items themselves: every attribute in the table's first 100 items and in the page on screen, plus the key attributes of each index. The table's keys come first. An attribute that appears in none of those items has no column until a page returns it. Column types use the AWS console's names, taken from the most common type in the column. | Attribute type | In the cell | |----------------|-------------| -| `S`, `N`, `BOOL`, `NULL` | The value | -| `B` | Base64 | -| `L`, `M` | DynamoDB-typed JSON, `[{"S":"a"}]` and `{"k":{"S":"a"}}` | -| `SS`, `NS`, `BS` | A plain JSON array | +| String, Number, Boolean | The value | +| Null | `NULL` | +| Binary | Hex, `0x…` | +| Map, List | Plain JSON, `{"city":"Paris"}` and `["a",1]` | +| String Set, Number Set, Binary Set | A JSON array | + +Maps, lists and sets open in the JSON editor. Numbers keep all 38 digits; nothing in the grid rounds them. + +### Saving edits + +Each changed row becomes one PartiQL statement, sent when you save, and each value keeps the type it had: an edited Number stays a Number, a String that looks like `02134` stays a String, and a String Set edited as JSON goes back as a String Set, nested Binary values included. A new attribute takes the type its column shows. **Set NULL** removes the attribute from the item. -An edit is parsed back through the same shape, so keep the typed envelopes when you change a list or a map. +An update also checks each attribute it changes against the value the grid loaded, so it fails rather than overwrite a value someone else changed since, or recreate an item someone deleted. **Duplicate Row** copies every attribute but the keys; fill in the new keys before saving. -In [Table Structure](/features/table-structure), **Indexes** lists the primary key with every GSI and LSI, and **DDL** prints the key schema, billing mode, capacity, item count, table size and each index's projection. +## PartiQL and the DynamoDB API -## PartiQL +The editor runs two kinds of statement. PartiQL takes double quotes around names and single quotes around strings, and Amazon's [PartiQL reference](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.html) has the grammar: -The editor runs PartiQL. Table names take double quotes, string values single quotes: `SELECT * FROM "Users" WHERE userId = 'user123'`. Amazon's [PartiQL reference](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.html) has the grammar. +```sql +SELECT * FROM "Orders" WHERE "pk" = 'customer#42' AND "sk" > 100 +``` + +A DynamoDB action name followed by its request JSON runs that action, exactly as the [API reference](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Operations_Amazon_DynamoDB.html) describes the request: + +```json +Query {"TableName": "Orders", "IndexName": "byStatus", + "KeyConditionExpression": "#s = :s", + "ExpressionAttributeNames": {"#s": "status"}, + "ExpressionAttributeValues": {":s": {"S": "shipped"}}} +``` -Grid edits become PartiQL as well, one `INSERT`, `UPDATE` or `DELETE` per row. A key attribute is never part of a `SET`: to change a key, delete the item and insert it again. +| Kind | Actions | +|------|---------| +| Read | `Scan`, `Query`, `GetItem`, `BatchGetItem`, `TransactGetItems` | +| Write | `PutItem`, `UpdateItem`, `DeleteItem`, `BatchWriteItem`, `TransactWriteItems` | +| PartiQL | `ExecuteStatement`, `ExecuteTransaction`, `BatchExecuteStatement` | +| Describe | `ListTables`, `DescribeTable`, `DescribeTimeToLive`, `DescribeContinuousBackups`, `ListTagsOfResource`, `DescribeLimits` | +| Manage | `CreateTable`, `UpdateTable`, `DeleteTable`, `UpdateTimeToLive`, `UpdateContinuousBackups`, `TagResource`, `UntagResource` | + +Any other action name is refused. Autocomplete inserts a template for the common ones. Items come back in the grid; any other response comes back as JSON. + +- A `Scan` or `Query` request takes a trailing `ORDER BY`, `LIMIT` and `OFFSET`, which count the items returned. With `"Select": "COUNT"` it pages to the end and returns one row of `Count` and `ScannedCount`. +- A PartiQL `SELECT` takes them too. An `ORDER BY` on the sort key goes to DynamoDB when the WHERE clause fixes the partition key with `=`; any other is applied to the result. A `SELECT` that does not fix the partition key reads the whole table, and the status bar says so. +- A `BatchWriteItem` sends its unprocessed items again until they are written, and reports the ones still left after 10 attempts rather than calling the batch a success. +- A throttled request retries with backoff, and so does a failed one that is safe to send twice: a read, or a write whose repeat changes nothing. Four attempts in all. `Cmd+.` stops a running statement. + +## Creating and changing tables + +**New Table…** opens a form rather than a column grid: the partition and sort key with their types, on-demand or provisioned capacity, the table class, deletion protection, up to 20 global and 5 local secondary indexes. **Preview** shows the `CreateTable` request it will send. A local secondary index can only be defined here, when the table is created. The new table opens empty until DynamoDB finishes creating it. + + + Create Table form with Primary Key, Capacity, Settings and secondary index sections + Create Table form with Primary Key, Capacity, Settings and secondary index sections + + +In [Table Structure](/features/table-structure), **Indexes** lists the key and every index. Add a global secondary index there with its partition key as the first column and an optional sort key as the second, or drop one; either is an `UpdateTable` request, and DynamoDB builds or removes the index in the background. The key's type comes from the table's items; when no item read so far holds that attribute, the index is refused, so create it from the editor with an `UpdateTable` request that declares `AttributeDefinitions`. An index cannot be edited in place: delete it, save, and add the replacement once the old one is gone. The primary key and a local secondary index go only with the table. **DDL** shows the `CreateTable` request that recreates the table, plus the Time to Live and point-in-time recovery settings it carries. + +Right-click a table and choose **Maintenance** for point-in-time recovery, deletion protection, the stream, the table class, switching to on-demand capacity and turning Time to Live off. [Table Operations](/features/table-operations#maintenance) covers the sheet. + +Dropping a table from the sidebar sends `DeleteTable`. It fails while deletion protection is on. ## IAM permissions -The driver calls `ListTables`, `DescribeTable`, `Scan`, `Query` and `ExecuteStatement`, and nothing else. That is this policy: +Browsing, filtering, exporting and the Structure tab call these actions: ```json { @@ -79,12 +156,13 @@ The driver calls `ListTables`, `DescribeTable`, `Scan`, `Query` and `ExecuteStat "Action": [ "dynamodb:ListTables", "dynamodb:DescribeTable", + "dynamodb:DescribeTimeToLive", + "dynamodb:DescribeContinuousBackups", "dynamodb:Scan", "dynamodb:Query", - "dynamodb:PartiQLSelect", - "dynamodb:PartiQLInsert", - "dynamodb:PartiQLUpdate", - "dynamodb:PartiQLDelete" + "dynamodb:GetItem", + "dynamodb:BatchGetItem", + "dynamodb:PartiQLSelect" ], "Resource": "*" } @@ -92,23 +170,36 @@ The driver calls `ListTables`, `DescribeTable`, `Scan`, `Query` and `ExecuteStat } ``` -For read-only access, drop `PartiQLInsert`, `PartiQLUpdate` and `PartiQLDelete`. +Saving grid edits adds `dynamodb:PartiQLInsert`, `dynamodb:PartiQLUpdate` and `dynamodb:PartiQLDelete`. Creating, changing and dropping tables adds `dynamodb:CreateTable`, `dynamodb:UpdateTable`, `dynamodb:DeleteTable`, `dynamodb:UpdateTimeToLive` and `dynamodb:UpdateContinuousBackups`. A request typed in the editor needs the action it names. + +## SSL/TLS + +Every request goes to the AWS endpoint over HTTPS, signed with Signature Version 4, and there is no SSL/TLS section to set. A custom endpoint may use plain HTTP only when it is on this Mac: `localhost`, `127.0.0.1` or `::1`. ## Limitations -- Consumed capacity is never reported. Nothing in the app shows what a browse cost; read it in CloudWatch. -- Paging is cursor-based, so page 40 re-scans everything before it. -- A list or map cell is cut at 10,000 characters and ends in `...`. Saving an edit to a cut cell stores the fragment; change long nested values with PartiQL instead. -- Table structure is fixed at creation: no structure editing, no transactions, no import. -- Truncate is not offered. DynamoDB empties a table by scanning it and deleting every item, which is a long billed job rather than a statement. Deleting the table is available, from the sidebar or with `DROP TABLE ""`, which issues `DeleteTable`. -- Item counts come from DynamoDB and refresh roughly every six hours, so they lag. -- DAX endpoints are not supported. Leave **Custom Endpoint** empty or point it at a standard endpoint. +- A key attribute cannot be edited in place, and saving the change fails. Duplicate the row, give the copy its new key, save, then delete the original. +- The Structure tab cannot add, change or drop an attribute. Write the attribute to an item and it appears as a column. +- A global index that projects only its keys or chosen attributes is never used for browsing, so a filter on its key scans the table. Query that index from the editor with a `Query` request naming its `IndexName`. +- Import is not available. Write items from the editor with `BatchWriteItem` or PartiQL `INSERT`. +- Truncate is not offered, because DynamoDB can only empty a table by deleting every item. Drop the table and create it again from its DDL. +- Saving writes each changed row on its own, so there is no transaction across rows. Group writes that must succeed together in an `ExecuteTransaction` or `TransactWriteItems` request. +- A read through a global secondary index is eventually consistent, so an edit saved a moment ago can be missing from it. Refresh, or filter on the table's own partition key, which reads the table. +- DAX endpoints are not supported. Leave **Custom Endpoint** empty or point it at a DynamoDB endpoint. ## Troubleshooting ### Authentication failed: … -The credentials were rejected: an unrecognized key, a bad signature, or a policy that denies the call. Check the key and secret, the profile name, or the SSO session with `aws sso login --profile `. An aged-out STS session token needs replacing. +DynamoDB rejected the credentials: an unknown key, a bad signature, or a session token that aged out. Check the key and secret, the profile name, or sign in again with `aws sso login --profile `. + +### "…" is not an AWS region + +**AWS Region**, or the region in the AWS profile, holds something other than a region name. Region names are lowercase letters, digits and hyphens. Set it to one such as `us-east-1`. + +### DynamoDB error: [AccessDeniedException] … + +The credentials work but their policy does not allow the action the message names. Add it to the policy, using the list in [IAM permissions](#iam-permissions). ### DynamoDB error: [ResourceNotFoundException] … @@ -116,9 +207,19 @@ The table is not in this region. Tables are regional; set **AWS Region** to the ### DynamoDB error: [ProvisionedThroughputExceededException] … -The table's provisioned read capacity is used up. Retry, move the table to on-demand billing, or filter on the partition key so the browse runs as a Query. +The table used up its provisioned read or write capacity and four attempts did not get through. Filter on the partition key so the read runs as a Query, raise the capacity, or switch the table to on-demand from **Maintenance**. + +### The item … changed after it was loaded. Refresh the table and edit it again. + +Someone changed the item between loading it and saving it, and the save was refused so their change survives. Refresh, then make the edit again. + +### Plain HTTP is only allowed for an endpoint on this Mac (localhost). Use https:// for any other host. + +**Custom Endpoint** starts with `http://` and names another machine. Use `https://`, or run DynamoDB Local on this Mac. ## Related - [AWS IAM Authentication](/connections/aws-iam) +- [Filtering](/features/filtering) +- [Table Operations](/features/table-operations) - [Import & Export](/features/import-export) diff --git a/docs/development/plugin-development.mdx b/docs/development/plugin-development.mdx index 1bc515fd3d..a1c5faee26 100644 --- a/docs/development/plugin-development.mdx +++ b/docs/development/plugin-development.mdx @@ -66,6 +66,7 @@ Five defaults are worth a second look before you accept them: - `fetchAllColumns(schema:)` and `fetchAllForeignKeys(schema:)` loop one round-trip per table. Any SQL driver should replace them with a single catalog query. - `quoteIdentifier`, `escapeStringLiteral`, `executeParameterized`, and `streamRows` assume generic SQL. - A non-SQL database implements `buildBrowseQuery`, `buildFilteredQuery`, and `generateStatements` instead, which is what makes browsing and editing work without SQL. Implement the `schema:`-aware overloads if your database has schemas; the schema-less defaults throw the schema away. +- A database whose tables are not a list of typed columns returns a `PluginCreateTableFormSpec` from `createTableFormSpec(schema:)` and turns the filled-in form into statements in `createTableStatements(for:schema:)`. **New Table…** then shows that form in place of the column grid. The default returns `nil`, which keeps the grid. [Testing a Custom Plugin](/development/testing-plugins) covers getting the built bundle into a running app and reading the failure if it does not load. diff --git a/docs/features/table-operations.mdx b/docs/features/table-operations.mdx index f55bec5d79..b8b3764e80 100644 --- a/docs/features/table-operations.mdx +++ b/docs/features/table-operations.mdx @@ -7,7 +7,7 @@ Dropping a table cannot be undone, and **Cascade** in the confirmation dialog wi ## Create a table or view -**Database > New Table…** opens the visual structure editor (see [Table Structure](/features/table-structure)). **Database > New View…** opens a query tab holding a `CREATE VIEW` template for the engine. Both are also on the sidebar's right-click menu over empty space, where [safe mode](/features/safe-mode) removes them rather than dimming them. +**Database > New Table…** opens the visual structure editor (see [Table Structure](/features/table-structure)); on [DynamoDB](/databases/dynamodb#creating-and-changing-tables) it opens a form for the keys, capacity and indexes instead. **Database > New View…** opens a query tab holding a `CREATE VIEW` template for the engine. Both are also on the sidebar's right-click menu over empty space, where [safe mode](/features/safe-mode) removes them rather than dimming them. ## Drop and truncate @@ -77,6 +77,7 @@ Each operation lists the object kinds it applies to, so a view offers only the o | PostgreSQL | VACUUM, ANALYZE, REINDEX, CLUSTER | | MySQL / MariaDB | OPTIMIZE TABLE, ANALYZE TABLE, CHECK TABLE, REPAIR TABLE | | SQLite | VACUUM, ANALYZE, REINDEX, Integrity Check | +| DynamoDB | Point-in-Time Recovery, Deletion Protection, Stream, Table Class, Switch to On-Demand Capacity, Turn Off Time to Live | PostgreSQL VACUUM carries FULL (rewrites the table and blocks access), ANALYZE, and VERBOSE toggles. MySQL CHECK TABLE offers QUICK, FAST, MEDIUM (the default), EXTENDED, and CHANGED. The options come from the driver, so an engine that adds one shows it without an app change. No other engine reports maintenance operations, so the submenu is hidden there, and in read-only safe mode. diff --git a/docs/images/dynamodb-create-table-dark.png b/docs/images/dynamodb-create-table-dark.png new file mode 100644 index 0000000000000000000000000000000000000000..6e4ba3007510b7eefaa62d2ebd20f93c1366f216 GIT binary patch literal 6565 zcmeAS@N?(olHy`uVBq!ia0y~yV3S~AU_QXX1QhX5ePYbOAerOo;uunK>&=aWKrx;p z8xGI3e<;E=LE+0yS)c#|%VZGQaL5CsMZs_qi09Fx0-}VJIe}!ukx{A9&=^e{quF7! za2PEQfhFN+?J!y)j#h}H72;@xI9ef&R*0h&;%E&=aWKrx;p z8$PGY9ueW1pz!6UEKq=fWip6tIOGA+qF^`)#PjG;0Z~HAoItYS$f(q4XpE+f(d;l< zIE;vZb{MS?M=Qk93URbT9IX&XE5y+XakPgx+9Dcl5skKpMq5OqEuzsD(P)c^ z9xb9rw>L5~f{I+nf7i5m0wh3e9VKwsyQ+XV8#=&4*kTh|fSdzL;9>0u7a<^rp&MLT lB?x+gIHQ_I1A=-fLiKXg9PK-;p_QPd<>~6@vd$@?2>{=3g|Pqt literal 0 HcmV?d00001 diff --git a/docs/scripts/check-docs-against-source.py b/docs/scripts/check-docs-against-source.py index ea38a76d2d..9c92a080ce 100755 --- a/docs/scripts/check-docs-against-source.py +++ b/docs/scripts/check-docs-against-source.py @@ -264,7 +264,8 @@ def check_heading_case(root: Path, docs: Path) -> list[str]: ui = set() sources = [root / "TablePro", root / "Plugins", root / "Packages" / "TableProCore" / "Sources"] for swift in (path for source in sources for path in source.rglob("*.swift")): - ui |= set(re.findall(r'String\(localized:\s*"((?:[^"\\]|\\.)*)"', swift.read_text())) + ui |= set(re.findall(r'String\(\s*localized:\s*"((?:[^"\\]|\\.)*)"', swift.read_text())) + ui |= {re.sub(r"%(?:\d+\$)?(?:@|l{0,2}d)", "…", s) for s in ui} acronym = re.compile(r"^[A-Z0-9]{2,}$") inner = re.compile(r"^[A-Za-z][a-z0-9.]*[A-Z]") diff --git a/project.yml b/project.yml index b7422842dd..e76955a8ad 100644 --- a/project.yml +++ b/project.yml @@ -441,9 +441,34 @@ targets: - Plugins/DuckDBDriverPlugin/DuckDBTypeRendering.swift - Plugins/DuckDBDriverPlugin/DuckDBViewDefinition.swift - Plugins/DuckDBDriverPlugin/QuackConnectBuilder.swift - - Plugins/DynamoDBDriverPlugin/DynamoDBOperations.swift - - Plugins/DynamoDBDriverPlugin/DynamoDBQueryBuilder.swift - - Plugins/DynamoDBDriverPlugin/DynamoDBStatementGenerator.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBAccessPlanner.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBAttributeValue.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBBrowseRequest.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBCatalog.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBCellCodec.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBClient.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBEndpoint.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBError.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBExpression.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBFilterTranslator.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBItemTable.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBJSON.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBNumber.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBPartiQL.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBPlugin.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+API.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Execution.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Reading.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Schema.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+TableManagement.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Writes.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBRetryPolicy.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBSigner.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBStatement.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBTableDefinition.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBTableSchema.swift + - Plugins/DynamoDBDriverPlugin/DynamoDBWriteStatements.swift - Plugins/ElasticsearchDriverPlugin/ElasticsearchConsoleParser.swift - Plugins/ElasticsearchDriverPlugin/ElasticsearchMappingFlattener.swift - Plugins/ElasticsearchDriverPlugin/ElasticsearchOperations.swift diff --git a/scripts/dynamodb-test-local.sh b/scripts/dynamodb-test-local.sh new file mode 100755 index 0000000000..a4007ca37d --- /dev/null +++ b/scripts/dynamodb-test-local.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# +# dynamodb-test-local.sh: start or stop the DynamoDB Local that DynamoDBLocalIntegrationTests runs +# against. +# +# The DynamoDB unit suites are pure logic and never open a socket. The integration suite drives the +# real driver end to end, so it needs a DynamoDB, and it skips itself unless one answers on +# 127.0.0.1:18000. It creates its own uniquely named tables and deletes them afterwards. +# +# Usage: +# scripts/dynamodb-test-local.sh up # start it (in memory, shared database) +# scripts/dynamodb-test-local.sh down # remove it +# +# Then: +# .claude/skills/fix-issue/scripts/verify.sh test DynamoDBLocalIntegrationTests +# +# DynamoDB Local is not the service. It updates ItemCount live where AWS refreshes it about every +# six hours, it has no point-in-time recovery or tags, and it accepts any credentials, so a green +# run here says nothing about authentication, throttling or those settings. + +set -euo pipefail + +CONTAINER="${TABLEPRO_DYNAMODB_TEST_CONTAINER:-tp-dynamodb-it}" +PORT="${TABLEPRO_DYNAMODB_TEST_PORT:-18000}" +IMAGE="${TABLEPRO_DYNAMODB_TEST_IMAGE:-amazon/dynamodb-local:latest}" +ACTION="${1:-up}" + +case "$ACTION" in +up) + if docker ps --format '{{.Names}}' | grep -qx "$CONTAINER"; then + echo "$CONTAINER is already running on port $PORT." + exit 0 + fi + docker rm -f "$CONTAINER" > /dev/null 2>&1 || true + docker run -d --name "$CONTAINER" -p "127.0.0.1:${PORT}:8000" "$IMAGE" \ + -jar DynamoDBLocal.jar -inMemory -sharedDb > /dev/null + for _ in $(seq 1 30); do + if curl -s -o /dev/null "http://127.0.0.1:${PORT}"; then + echo "DynamoDB Local is answering on 127.0.0.1:${PORT}." + exit 0 + fi + sleep 1 + done + echo "DynamoDB Local did not answer on 127.0.0.1:${PORT} within 30 seconds." >&2 + exit 1 + ;; +down) + docker rm -f "$CONTAINER" > /dev/null 2>&1 || true + echo "Removed $CONTAINER." + ;; +*) + echo "Usage: $0 up|down" >&2 + exit 2 + ;; +esac From 9f362c5b1f39f0aa26da2c75b4d6faa2e94628f4 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Wed, 23 Sep 2026 18:36:02 +0700 Subject: [PATCH 2/2] fix(plugin-dynamodb): add the rewrite's new strings to the string catalog --- TablePro/Resources/Localizable.xcstrings | 432 +++++++++++++++++++++++ 1 file changed, 432 insertions(+) diff --git a/TablePro/Resources/Localizable.xcstrings b/TablePro/Resources/Localizable.xcstrings index 07c23c5c64..ca36e1898b 100644 --- a/TablePro/Resources/Localizable.xcstrings +++ b/TablePro/Resources/Localizable.xcstrings @@ -183057,6 +183057,438 @@ }, "This connection loads SQLite extensions, and their functions can read and write files. A statement from outside TablePro can call only the functions built into SQLite. Run this one in TablePro instead." : { + }, + "Query on the table" : { + + }, + "Query on index %@" : { + + }, + "Scan with filters" : { + + }, + "Scan" : { + + }, + "Get by key" : { + + }, + "No read needed" : { + + }, + "An attribute value must name exactly one type" : { + + }, + "Unknown attribute type \"%@\"" : { + + }, + "A %@ attribute has a payload of the wrong shape" : { + + }, + "A set member must be a string" : { + + }, + "A binary value is not valid base64" : { + + }, + "An item must be a JSON object" : { + + }, + "Browse needs a TableName" : { + + }, + "Each Browse filter needs an Attribute and an Operator" : { + + }, + "\"%@\" is not a DynamoDB number" : { + + }, + "\"%@\" is not true or false" : { + + }, + "A Binary value must be base64" : { + + }, + "A %@ value must be JSON: %@" : { + + }, + "A set can't be empty. Clear the cell to remove it." : { + + }, + "A set can't contain the same value twice" : { + + }, + "The endpoint did not answer over HTTP" : { + + }, + "The endpoint answered with a redirect, which DynamoDB never sends. Check the Custom Endpoint." : { + + }, + "\"%@\" is not an AWS region" : { + + }, + "\"%@\" is not a URL. Enter an endpoint such as http://localhost:8000." : { + + }, + "The endpoint must start with https:// or http://" : { + + }, + "Plain HTTP is only allowed for an endpoint on this Mac (localhost). Use https:// for any other host." : { + + }, + "Stopped after %d seconds, the query timeout" : { + + }, + "The item %@ changed after it was loaded. Refresh the table and edit it again." : { + + }, + "The item %@ no longer exists." : { + + }, + "%1$d of %2$d were applied." : { + + }, + "HTTP %d with a body of %d bytes" : { + + }, + "DynamoDB error: [%1$@] %2$@" : { + + }, + "Action %1$d: %2$@%3$@" : { + + }, + "Raw filters aren't available for DynamoDB. Write the condition in PartiQL in the editor." : { + + }, + "Not valid JSON at character %d" : { + + }, + "The key \"%@\" appears twice in one JSON object" : { + + }, + "The JSON is nested more deeply than DynamoDB allows" : { + + }, + "Unexpected text after the JSON at character %d" : { + + }, + "Access Key + Secret Key" : { + + }, + "AWS SSO" : { + + }, + "DynamoDB Local (no credentials)" : { + + }, + "The profile's region, or us-east-1" : { + + }, + "Optional, such as http://localhost:8000" : { + + }, + "The request must be a JSON object" : { + + }, + "The transaction was applied" : { + + }, + "More tables follow. Pass LastEvaluatedTableName as ExclusiveStartTableName." : { + + }, + "%@ succeeded" : { + + }, + "%1$@ accepted. Table status: %2$@" : { + + }, + "%d requests were not processed after 10 attempts" : { + + }, + "%d requests applied" : { + + }, + "Statement %1$d: %2$@ %3$@" : { + + }, + "%d statements applied" : { + + }, + "No item read so far holds \"%@\" as a key type. Create this index from the editor with its AttributeDefinitions." : { + + }, + "Time to Live is not on for this table" : { + + }, + "Parameters apply only to PartiQL statements" : { + + }, + "DynamoDB is still creating this table. Refresh in a moment." : { + + }, + "In DynamoDB order: sorting by %@ needs the whole result, or a Query on that sort key" : { + + }, + "The request needs a TableName" : { + + }, + "This SELECT reads the whole table" : { + + }, + "Could not create a temporary file for the export" : { + + }, + "DynamoDB left some keys unread after 10 attempts" : { + + }, + "%@ returned" : { + + }, + "%@ read" : { + + }, + "%@ RCU" : { + + }, + "Partition key" : { + + }, + "Sort key" : { + + }, + "Key of %@" : { + + }, + "DynamoDB indexes are never unique" : { + + }, + "A global secondary index takes a partition key and an optional sort key" : { + + }, + "A DynamoDB table declares only its keys. Add the attribute by writing it to an item." : { + + }, + "A DynamoDB index can't be changed. Delete it and save, then add the new one once the old one is gone." : { + + }, + "The primary key is part of the table and can't be dropped." : { + + }, + "A local secondary index is part of its table and is removed only with the table." : { + + }, + "Point-in-Time Recovery" : { + + }, + "Deletion Protection" : { + + }, + "Stream" : { + + }, + "Table Class" : { + + }, + "Switch to On-Demand Capacity" : { + + }, + "Turn Off Time to Live" : { + + }, + "Keys only" : { + + }, + "New image" : { + + }, + "Old image" : { + + }, + "New and old images" : { + + }, + "Standard" : { + + }, + "Standard-Infrequent Access" : { + + }, + "On" : { + + }, + "Class" : { + + }, + "An item with this key already exists" : { + + }, + "DELETE ran. DynamoDB reports whether an item was deleted only with RETURNING ALL OLD *." : { + + }, + "%d item(s) affected" : { + + }, + "A key attribute needs a value" : { + + }, + "DynamoDB has no views" : { + + }, + "The %1$@ request is not valid JSON: %2$@" : { + + }, + "The %@ request must be a JSON object" : { + + }, + "\"%@\" is not a DynamoDB action this editor runs" : { + + }, + "%@ takes no ORDER BY, LIMIT or OFFSET" : { + + }, + "Unexpected text after the %1$@ request: %2$@" : { + + }, + "On-demand capacity" : { + + }, + "Provisioned: %1$lld read, %2$lld write" : { + + }, + "Deletion protection on" : { + + }, + "Stream: %@" : { + + }, + "Status: %@" : { + + }, + "Item count is approximate, updated about every six hours" : { + + }, + "Capacity" : { + + }, + "Billing" : { + + }, + "On-demand" : { + + }, + "Provisioned" : { + + }, + "Read capacity units" : { + + }, + "Write capacity units" : { + + }, + "Table class" : { + + }, + "Deletion protection" : { + + }, + "Global Secondary Indexes" : { + + }, + "Add Global Index" : { + + }, + "Local Secondary Indexes" : { + + }, + "Add Local Index" : { + + }, + "A DynamoDB table declares only its key attributes. The items you write add every other attribute." : { + + }, + "All attributes" : { + + }, + "Keys and chosen attributes" : { + + }, + "Chosen attributes" : { + + }, + "Comma-separated names" : { + + }, + "A table name is 3 to 255 characters of letters, digits, underscore, hyphen and period." : { + + }, + "\"%@\" is declared as two different types" : { + + }, + "Enter the partition key's name" : { + + }, + "A local secondary index needs a table with a sort key" : { + + }, + "An index name is 3 to 255 characters of letters, digits, underscore, hyphen and period." : { + + }, + "Enter the partition key of index %@" : { + + }, + "Enter the sort key of index %@" : { + + }, + "Name the attributes index %@ includes" : { + + }, + "Read capacity must be a whole number of at least 1" : { + + }, + "Write capacity must be a whole number of at least 1" : { + + }, + "DescribeTable returned no table" : { + + }, + "DynamoDB can't change a key. Duplicate the row with the new key, then delete the old one." : { + + }, + "This key is a %@, not binary data" : { + + }, + "This database has no Create Table form" : { + + }, + "Form" : { + + }, + "%1$@ must be at least %2$lld." : { + + }, + "%1$@ must be at most %2$lld." : { + + }, + "%1$@ must be between %2$lld and %3$lld." : { + + }, + "%1$@, entry %2$lld: %3$@" : { + + }, + "%@ is required." : { + + }, + "%@ must be a whole number." : { + + }, + "Entry %1$lld: %2$@" : { + + }, + "Entry %lld" : { + + }, + "The form produced no statements to run." : { + } }, "version" : "1.1"