From 8619d818d451e794922216345c38118c65038bb1 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sat, 26 Sep 2026 12:11:06 +0700 Subject: [PATCH] fix(plugin-mongodb): rename and remove MongoDB fields from the Structure tab, carrying the validator along and refusing what would break an index, a view or a document --- CHANGELOG.md | 8 + .../MongoCollectionCatalog.swift | 278 +++++++++++ .../MongoDBCollectionDDL.swift | 2 +- .../MongoDBConnection+FieldChangeReads.swift | 258 ++++++++++ .../MongoDBConnection.swift | 22 +- .../MongoDBDriverPlugin/MongoDBPlugin.swift | 7 +- .../MongoDBPluginDriver+Structure.swift | 68 +++ .../MongoDBPluginDriver.swift | 18 +- .../MongoDBStructureEditing.swift | 22 + .../MongoFieldChange.swift | 164 ++++++ .../MongoFieldChangeAssessment.swift | 385 ++++++++++++++ .../MongoFieldChangeCheck.swift | 324 ++++++++++++ .../MongoFieldDataProbe.swift | 268 ++++++++++ .../MongoFieldDependent.swift | 114 +++++ .../MongoFieldReferences.swift | 470 ++++++++++++++++++ .../MongoScriptCommandBuilder.swift | 9 +- .../MongoDBDriverPlugin/MongoScriptHost.swift | 6 +- .../MongoWriteConcern.swift | 53 ++ .../MongoWriteFailure.swift | 23 +- .../PluginDatabaseDriver.swift | 85 ++++ .../PluginSchemaChangeReview.swift | 25 + .../PluginSchemaOperation.swift | 4 + .../Compare/CompareSyncEngineFamily.swift | 12 +- TablePro/Core/Database/DatabaseDriver.swift | 52 ++ .../Database/DatabaseManager+Schema.swift | 101 +++- .../DatabaseManager+SchemaComposition.swift | 32 +- TablePro/Core/Events/AppCommands.swift | 3 + .../Core/Plugins/DatabaseType+Registry.swift | 4 + ...uginDriverAdapter+SchemaChangeChecks.swift | 49 ++ TablePro/Core/Plugins/PluginManager.swift | 10 + ...ginMetadataRegistry+RegistryDefaults.swift | 13 +- .../Core/Plugins/PluginMetadataRegistry.swift | 5 + .../SchemaTracking/SchemaChangeScript.swift | 17 + .../SchemaOperationRefusal.swift | 32 +- .../SchemaStatementGenerator.swift | 6 + .../StructureChangeManager.swift | 55 +- .../Services/Query/SchemaColumnStore.swift | 5 + TablePro/Resources/Localizable.xcstrings | 120 +++++ ...ntentCoordinator+DatabaseObjectTools.swift | 18 + .../MainContentCoordinator+Refresh.swift | 40 ++ .../StructureEditingSession+Apply.swift | 84 +++- .../Structure/StructureEditingSession.swift | 11 +- .../Structure/StructureFooterPolicy.swift | 15 + .../StructureGridDelegate+Inspector.swift | 2 +- .../Views/Structure/StructureSavePlan.swift | 6 +- .../TableStructureView+ColumnReorder.swift | 2 +- .../TableStructureView+EditGate.swift | 2 + .../Views/Structure/TableStructureView.swift | 19 +- .../Autocomplete/SQLSchemaProviderTests.swift | 28 ++ .../CompareSyncSampledColumnsTests.swift | 53 ++ ...abaseManagerSchemaChangeRoutingTests.swift | 349 ++++++++++++- .../Database/SchemaOperationKindTests.swift | 40 ++ .../MongoDB/MongoScriptCommandTests.swift | 26 +- ...tadataRegistryCuratedCapabilityTests.swift | 15 + .../SchemaOperationRefusalTests.swift | 49 ++ .../StructureChangeManagerSaveHoldTests.swift | 141 ++++++ .../MongoDBStructureEditingParityTests.swift | 37 ++ .../MongoFieldChangeAssessmentTests.swift | 403 +++++++++++++++ .../Plugins/MongoFieldChangeTests.swift | 159 ++++++ .../Plugins/MongoFieldDataProbeTests.swift | 282 +++++++++++ .../Plugins/MongoFieldReferencesTests.swift | 364 ++++++++++++++ .../Plugins/MongoSearchIndexTests.swift | 135 +++++ .../Plugins/MongoWriteFailureTests.swift | 12 + .../Views/Main/CatalogChangeWindowTests.swift | 141 ++++++ .../Structure/StructureEditGateTests.swift | 27 +- .../StructureEditingSessionTests.swift | 138 +++++ .../StructureFooterPolicyTests.swift | 20 +- .../StructureGridDelegateInspectorTests.swift | 15 + docs/databases/mongodb.mdx | 49 +- docs/features/safe-mode.mdx | 2 +- docs/features/table-structure.mdx | 10 +- project.yml | 8 + 72 files changed, 5738 insertions(+), 93 deletions(-) create mode 100644 Plugins/MongoDBDriverPlugin/MongoCollectionCatalog.swift create mode 100644 Plugins/MongoDBDriverPlugin/MongoDBConnection+FieldChangeReads.swift create mode 100644 Plugins/MongoDBDriverPlugin/MongoDBPluginDriver+Structure.swift create mode 100644 Plugins/MongoDBDriverPlugin/MongoDBStructureEditing.swift create mode 100644 Plugins/MongoDBDriverPlugin/MongoFieldChange.swift create mode 100644 Plugins/MongoDBDriverPlugin/MongoFieldChangeAssessment.swift create mode 100644 Plugins/MongoDBDriverPlugin/MongoFieldChangeCheck.swift create mode 100644 Plugins/MongoDBDriverPlugin/MongoFieldDataProbe.swift create mode 100644 Plugins/MongoDBDriverPlugin/MongoFieldDependent.swift create mode 100644 Plugins/MongoDBDriverPlugin/MongoFieldReferences.swift create mode 100644 Plugins/MongoDBDriverPlugin/MongoWriteConcern.swift create mode 100644 Plugins/TableProPluginKit/PluginSchemaChangeReview.swift create mode 100644 TablePro/Core/Plugins/PluginDriverAdapter+SchemaChangeChecks.swift create mode 100644 TablePro/Core/SchemaTracking/SchemaChangeScript.swift create mode 100644 TableProTests/Core/Compare/CompareSyncSampledColumnsTests.swift create mode 100644 TableProTests/Core/Database/SchemaOperationKindTests.swift create mode 100644 TableProTests/Core/SchemaTracking/StructureChangeManagerSaveHoldTests.swift create mode 100644 TableProTests/Plugins/MongoDBStructureEditingParityTests.swift create mode 100644 TableProTests/Plugins/MongoFieldChangeAssessmentTests.swift create mode 100644 TableProTests/Plugins/MongoFieldChangeTests.swift create mode 100644 TableProTests/Plugins/MongoFieldDataProbeTests.swift create mode 100644 TableProTests/Plugins/MongoFieldReferencesTests.swift create mode 100644 TableProTests/Plugins/MongoSearchIndexTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index fd43cf0c69..8dfd91eec3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -117,6 +117,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - ⌘F finds and ⇧⌘F toggles filters in data file windows, as in the table grid. - Large data files opened, filtered, sorted and searched in the background, with progress and Cancel. - `.json` and `.ndjson` files opened in the Data Files window rather than as a DuckDB connection. +- Column type changes from the Structure tab confirmed under Safe Mode, like a dropped column. +- No structure sync script for MongoDB, whose fields are read from a sample of documents. ### Removed @@ -531,6 +533,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - MongoDB edits that stored dates and ObjectIds as text, rounded integers past 2^53, or missed a string `_id` that looks numeric. - **New Table…** offered on databases that cannot create a table, such as Redis and Kafka. - Executing indicator and Stop button carried over for a moment onto the query tab switched to. +- MongoDB fields that could not be renamed or removed from the Structure tab. (#3132) +- Structure tab edits made during a save, cleared without being saved. +- Rows not reloaded after a Structure save that failed partway through its statements. +- Stale rows after a Structure save in the table's other tabs and in its Data view behind Structure. +- A Structure save reloading, or asking to discard edits in, tabs on other tables of the same database. +- `writeConcern` option of `updateOne` and `updateMany` ignored by the MongoDB shell. ### Security diff --git a/Plugins/MongoDBDriverPlugin/MongoCollectionCatalog.swift b/Plugins/MongoDBDriverPlugin/MongoCollectionCatalog.swift new file mode 100644 index 0000000000..feeced296e --- /dev/null +++ b/Plugins/MongoDBDriverPlugin/MongoCollectionCatalog.swift @@ -0,0 +1,278 @@ +// +// MongoCollectionCatalog.swift +// MongoDBDriverPlugin +// + +import Foundation + +/// What `listCollections` says about one collection, read from its canonical Extended JSON. +/// +/// The validator is kept as the exact text the server sent, because it is written back verbatim: in +/// a `collMod` when a rename has to carry it along, and in the aggregation that tries the change +/// against it before anything is written. +struct MongoCollectionInfo: Equatable, Sendable { + enum Kind: Equatable, Sendable { + case missing + case collection + case view + case timeseries + case other(String) + } + + let name: String + let kind: Kind + let validatorJson: String? + let validationLevel: String + let validationAction: String + let encryptedFieldPaths: [String] + let isCapped: Bool + + /// Filters `listCollections` to one name, exactly, which `db.getCollectionInfos` in the shell + /// does not do. + static func filterJson(for collection: String) -> String { + "{\"name\": \(MongoScriptJson.jsonString(collection))}" + } + + static let viewFilterJson = "{\"type\": \"view\"}" + + init(collection: String, infoJson: String?) { + self.name = collection + guard let infoJson else { + self.kind = .missing + self.validatorJson = nil + self.validationLevel = "strict" + self.validationAction = "error" + self.encryptedFieldPaths = [] + self.isCapped = false + return + } + let options = MongoScriptJson.member(of: infoJson, key: "options") + let type = MongoScriptJson.member(of: infoJson, key: "type").flatMap(MongoJsonValue.parse) as? String + self.kind = Self.kind(of: type ?? "collection") + self.validatorJson = options.flatMap { Self.nonEmptyDocument(MongoScriptJson.member(of: $0, key: "validator")) } + self.validationLevel = options.flatMap { Self.string(MongoScriptJson.member(of: $0, key: "validationLevel")) } + ?? "strict" + self.validationAction = options.flatMap { Self.string(MongoScriptJson.member(of: $0, key: "validationAction")) } + ?? "error" + self.encryptedFieldPaths = options.map(Self.encryptedFieldPaths(in:)) ?? [] + self.isCapped = options.flatMap { Self.bool(MongoScriptJson.member(of: $0, key: "capped")) } ?? false + } + + /// Whether the server checks a changed document against the validator and refuses the write + /// when it fails. `warn` only logs, and `off` checks nothing. + var enforcesValidator: Bool { + validatorJson != nil && validationLevel != "off" && validationAction != "warn" + } + + var validatesOnlyValidDocuments: Bool { validationLevel == "moderate" } + + private static func kind(of type: String) -> Kind { + switch type { + case "collection": return .collection + case "view": return .view + case "timeseries": return .timeseries + default: return .other(type) + } + } + + private static func string(_ json: String?) -> String? { + json.flatMap(MongoJsonValue.parse) as? String + } + + private static func bool(_ json: String?) -> Bool? { + json.flatMap(MongoJsonValue.parse) as? Bool + } + + private static func nonEmptyDocument(_ json: String?) -> String? { + guard let json, let object = MongoJsonValue.parse(json) as? [String: Any], !object.isEmpty else { return nil } + return json + } + + private static func encryptedFieldPaths(in optionsJson: String) -> [String] { + guard let options = MongoJsonValue.parse(optionsJson) as? [String: Any], + let encrypted = options["encryptedFields"] as? [String: Any], + let fields = encrypted["fields"] as? [[String: Any]] else { return [] } + return fields.compactMap { $0["path"] as? String } + } +} + +struct MongoIndexSpec { + let name: String + let spec: [String: Any] + + init?(json: String) { + guard let spec = MongoJsonValue.parse(json) as? [String: Any] else { return nil } + self.name = spec["name"] as? String ?? "" + self.spec = spec + } + + func reaches(_ field: String) -> Bool { + MongoIndexFieldReferences.reaches(spec, field: field) + } +} + +/// An Atlas Search or Vector Search index as `$listSearchIndexes` lists it. `listIndexes` never +/// returns one, because `mongot` holds them, so a rename that checked `listIndexes` alone left a +/// search index pointing at a path no document has. +struct MongoSearchIndex { + static let listingPipelineJson = "[{\"$listSearchIndexes\": {}}]" + + /// Asked of a server that does not know `$listSearchIndexes`, to learn whether it runs search at + /// all. An answer, empty or not, means it does. + static let searchProbePipelineJson = "[{\"$search\": {\"exists\": {\"path\": \"_id\"}}}, {\"$limit\": 1}]" + + /// What a server with no search answers every search stage with, `$listSearchIndexes` and + /// `$search` alike, so it has no search index to break. Measured: 6047401, "stage is only allowed + /// on MongoDB Atlas", on 6.0.28 and 7.0.43 community, and 31082 SearchNotEnabled, "requires + /// additional configuration", on 8.2.12 community with no `mongot` configured. + static let noSearchErrorCodes: Set = [6_047_401, 31_082] + + /// "Unrecognized pipeline stage name", what a server older than `$listSearchIndexes` answers. + /// That alone says nothing about search: an Atlas cluster that predates the stage runs `$search` + /// and holds search indexes it cannot list. Measured on 5.0.33 community, which answers it for + /// `$search` too. + static let unknownStageErrorCode: UInt32 = 40_324 + + enum ListingFailure: Equatable { + /// The server has no search, so the collection has no search index. + case serverWithoutSearch + /// The server predates the listing stage. Whether it has search is asked of `$search`. + case listingStageUnknown + /// The indexes are unknown, and the save stops. + case unknown + } + + static func listingFailure(code: UInt32) -> ListingFailure { + if noSearchErrorCodes.contains(code) { return .serverWithoutSearch } + return code == unknownStageErrorCode ? .listingStageUnknown : .unknown + } + + /// Whether the server's answer to `searchProbePipelineJson` shows it has no search: it does not + /// know the stage either, or refuses it the way a server without search does. Nil is an answer. + static func searchIsUnavailable(probeErrorCode: UInt32?) -> Bool { + guard let probeErrorCode else { return false } + return listingFailure(code: probeErrorCode) != .unknown + } + + /// Why a save stops on a server that runs search and cannot list its search indexes. + static var unlistableIndexesReason: String { + String( + localized: """ + This server runs Atlas Search but cannot list its search indexes, so one that uses the field cannot be \ + ruled out. Check the collection's search indexes in Atlas, then change the field from a query tab. + """ + ) + } + + let name: String + let definitions: [Any] + + init?(json: String) { + guard let listing = MongoJsonValue.parse(json) as? [String: Any] else { return nil } + self.name = listing["name"] as? String ?? "" + self.definitions = Self.definitions(in: listing) + } + + /// Whether any definition of the index names the field, the one being built included. + func mentions(_ field: String) -> Bool { + definitions.contains { MongoSearchDefinitionFieldReferences.reaches($0, field: field) } + } + + /// `latestDefinition` is the newest, and every `mongot` reports the definition it serves from + /// and the one it is building under `statusDetail`, which can still be an older one. + private static func definitions(in value: Any) -> [Any] { + if let list = value as? [Any] { + return list.flatMap(definitions(in:)) + } + guard let object = value as? [String: Any] else { return [] } + return object.flatMap { key, member -> [Any] in + key == "latestDefinition" || key == "definition" ? [member] : definitions(in: member) + } + } +} + +/// A view as `listCollections` describes it: the collection or view it reads, and its pipeline. +struct MongoViewDefinition { + let name: String + let viewOn: String + let pipeline: [Any] + + init?(json: String) { + guard let info = MongoJsonValue.parse(json) as? [String: Any], + let name = info["name"] as? String, + let options = info["options"] as? [String: Any], + let viewOn = options["viewOn"] as? String else { return nil } + self.name = name + self.viewOn = viewOn + self.pipeline = options["pipeline"] as? [Any] ?? [] + } + + init(name: String, viewOn: String, pipeline: [Any]) { + self.name = name + self.viewOn = viewOn + self.pipeline = pipeline + } + + /// Every view whose output depends on the collection: those defined on it, those defined on + /// such a view, and those that join or union it from any stage, at any depth. Taken to a fixed + /// point, so the order `listCollections` lists views in does not matter. + static func dependents(of collection: String, among views: [MongoViewDefinition]) -> [MongoViewDefinition] { + var reached: Set = [collection] + var dependents: [MongoViewDefinition] = [] + var grew = true + while grew { + grew = false + for view in views where !reached.contains(view.name) && view.reads(anyOf: reached) { + reached.insert(view.name) + dependents.append(view) + grew = true + } + } + return dependents + } + + func reads(anyOf sources: Set) -> Bool { + sources.contains(viewOn) || Self.joins(pipeline, anyOf: sources) + } + + func readsField(_ field: String) -> Bool { + MongoPipelineFieldReferences.pipelineReads(pipeline, field: field) + } + + private static func joins(_ value: Any, anyOf sources: Set) -> Bool { + if let list = value as? [Any] { + return list.contains { joins($0, anyOf: sources) } + } + guard let object = value as? [String: Any] else { return false } + for (key, member) in object { + if key == "$lookup" || key == "$graphLookup", + let spec = member as? [String: Any], namesSource(spec["from"], in: sources) { + return true + } + if key == "$unionWith", namesSource(member, in: sources) || namesSource((member as? [String: Any])?["coll"], in: sources) { + return true + } + if joins(member, anyOf: sources) { return true } + } + return false + } + + private static func namesSource(_ value: Any?, in sources: Set) -> Bool { + if let name = value as? String { return sources.contains(name) } + if let spec = value as? [String: Any], let name = spec["coll"] as? String { return sources.contains(name) } + return false + } +} + +/// How the driver keys what it learned about a collection's documents: by database and collection, +/// because two databases can hold a collection of the same name with different field types. Neither +/// name can hold a NUL, so the one in the key separates them. +enum MongoCollectionCacheKey { + static func key(database: String, collection: String) -> String { + "\(database)\u{0}\(collection)" + } + + static func names(_ key: String, collection: String) -> Bool { + key.hasSuffix("\u{0}\(collection)") + } +} diff --git a/Plugins/MongoDBDriverPlugin/MongoDBCollectionDDL.swift b/Plugins/MongoDBDriverPlugin/MongoDBCollectionDDL.swift index 21d80c4627..bd32f95dfa 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDBCollectionDDL.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDBCollectionDDL.swift @@ -59,7 +59,7 @@ enum MongoDBCollectionDDL { return columnRefusal(column) case .addIndex(let index), .modifyIndex(_, let index): return indexRefusal(index) - case .renameCheckConstraint, .dropIndex: + case .renameCheckConstraint, .dropIndex, .modifyColumn, .dropColumn: return nil @unknown default: return nil diff --git a/Plugins/MongoDBDriverPlugin/MongoDBConnection+FieldChangeReads.swift b/Plugins/MongoDBDriverPlugin/MongoDBConnection+FieldChangeReads.swift new file mode 100644 index 0000000000..658cdf85f2 --- /dev/null +++ b/Plugins/MongoDBDriverPlugin/MongoDBConnection+FieldChangeReads.swift @@ -0,0 +1,258 @@ +// +// MongoDBConnection+FieldChangeReads.swift +// MongoDBDriverPlugin +// + +#if canImport(CLibMongoc) +import CLibMongoc +#endif +import Foundation +import TableProPluginKit + +#if canImport(CLibMongoc) +/// The reads behind a field rename or removal, each through a cursor libmongoc advances to the end. +/// +/// The shell's `db.getCollectionInfos()` answers with `cursor.firstBatch` and ignores its filter, so +/// a database with more views than one batch holds would hide the one that reads the field. +/// +/// Every read goes to the primary, whatever the connection's read preference, because the writes +/// do. libmongoc 1.28 sends `listCollections` and `listIndexes` there itself: both build their +/// cursor with no read preference, which it resolves to primary, as the enumeration specs require. +/// An aggregation would take the connection's instead, so each one here names the primary. +extension MongoDBConnection { + func collectionInfoJson(database: String, collection: String) async throws -> String? { + let filter = MongoCollectionInfo.filterJson(for: collection) + return try await readCatalog { connection, client in + try connection.collectionInfosJsonSync(client: client, database: database, filter: filter).first + } + } + + func viewInfosJson(database: String) async throws -> [String] { + try await readCatalog { connection, client in + try connection.collectionInfosJsonSync(client: client, database: database, filter: MongoCollectionInfo.viewFilterJson) + } + } + + func indexSpecsJson(database: String, collection: String) async throws -> [String] { + try await readCatalog { connection, client in + try connection.listIndexesJsonSync(client: client, database: database, collection: collection) + } + } + + /// The collection's Atlas Search and Vector Search indexes. A server with no search says so with + /// an error, which here is an empty list. One too old to list them is asked whether it runs + /// `$search`, and one that does stops the save, as does any other failure, because either leaves + /// the indexes unknown. + func searchIndexesJson(database: String, collection: String) async throws -> [String] { + do { + return try await aggregatedDocumentsJson( + database: database, collection: collection, pipeline: MongoSearchIndex.listingPipelineJson + ) + } catch let error as MongoDBError { + switch MongoSearchIndex.listingFailure(code: error.code) { + case .serverWithoutSearch: + return [] + case .unknown: + throw error + case .listingStageUnknown: + guard try await searchIsUnavailable(database: database, collection: collection) else { + throw MongoDBError(code: error.code, message: MongoSearchIndex.unlistableIndexesReason) + } + return [] + } + } + } + + private func searchIsUnavailable(database: String, collection: String) async throws -> Bool { + do { + _ = try await aggregatedDocumentsJson( + database: database, collection: collection, pipeline: MongoSearchIndex.searchProbePipelineJson + ) + return false + } catch let error as MongoDBError where MongoSearchIndex.searchIsUnavailable(probeErrorCode: error.code) { + return true + } + } + + private func aggregatedDocumentsJson(database: String, collection: String, pipeline: String) async throws -> [String] { + try await readCatalog { connection, client in + try connection.aggregatedDocumentsJsonSync( + client: client, database: database, collection: collection, pipeline: pipeline + ) + } + } + + /// The first document an aggregation returns, or nil. Bounded by `maxTimeMS` and bound to a + /// server session, so a cancelled task ends the scan on the server rather than only here. + func firstAggregatedDocumentJson( + database: String, + collection: String, + pipeline: String, + maxTimeMS: Int32 + ) async throws -> String? { + try Task.checkCancellation() + beginScriptRun() + return try await withTaskCancellationHandler { + try await withClient { [self] client in + try firstAggregatedDocumentJsonSync( + client: client, database: database, collection: collection, + pipeline: pipeline, maxTimeMS: maxTimeMS + ) + } + } onCancel: { [self] in + cancelCurrentQuery() + } + } + + private func readCatalog( + _ body: @escaping @Sendable (MongoDBConnection, OpaquePointer) throws -> T + ) async throws -> T { + try Task.checkCancellation() + beginScriptRun() + return try await withTaskCancellationHandler { + try await withClient { [self] client in try body(self, client) } + } onCancel: { [self] in + cancelCurrentQuery() + } + } + + private func collectionInfosJsonSync(client: OpaquePointer, database: String, filter: String) throws -> [String] { + try checkCancelled() + guard let options = jsonToBson("{\"filter\": \(filter)}") else { + throw MongoDBError(code: 0, message: MongoScriptText.invalidFilter(filter)) + } + defer { bson_destroy(options) } + guard let handle = database.withCString({ mongoc_client_get_database(client, $0) }) else { + throw MongoDBError(code: 0, message: MongoScriptText.cursorFailed) + } + defer { mongoc_database_destroy(handle) } + guard let cursor = mongoc_database_find_collections_with_opts(handle, options) else { + throw MongoDBError(code: 0, message: MongoScriptText.cursorFailed) + } + defer { mongoc_cursor_destroy(cursor) } + return try drain(cursor) + } + + private func aggregatedDocumentsJsonSync( + client: OpaquePointer, + database: String, + collection: String, + pipeline: String + ) throws -> [String] { + try checkCancelled() + guard let pipelineBson = jsonToBson(pipeline) else { + throw MongoDBError(code: 0, message: MongoScriptText.invalidPipeline(pipeline)) + } + defer { bson_destroy(pipelineBson) } + guard let primary = mongoc_read_prefs_new(MONGOC_READ_PRIMARY) else { + throw MongoDBError(code: 0, message: MongoScriptText.cursorFailed) + } + defer { mongoc_read_prefs_destroy(primary) } + let handle = try getCollection(client, database: database, collection: collection) + defer { mongoc_collection_destroy(handle) } + guard let cursor = mongoc_collection_aggregate(handle, MONGOC_QUERY_NONE, pipelineBson, nil, primary) else { + throw MongoDBError(code: 0, message: MongoScriptText.cursorFailed) + } + defer { mongoc_cursor_destroy(cursor) } + return try drain(cursor) + } + + private func firstAggregatedDocumentJsonSync( + client: OpaquePointer, + database: String, + collection: String, + pipeline: String, + maxTimeMS: Int32 + ) throws -> String? { + try checkCancelled() + guard let pipelineBson = jsonToBson(pipeline) else { + throw MongoDBError(code: 0, message: MongoScriptText.invalidPipeline(pipeline)) + } + defer { bson_destroy(pipelineBson) } + guard let options = jsonToBson(MongoFieldDataProbe.aggregateOptionsJson(maxTimeMS: maxTimeMS)) else { + throw MongoDBError(code: 0, message: MongoScriptText.cursorFailed) + } + defer { bson_destroy(options) } + guard let primary = mongoc_read_prefs_new(MONGOC_READ_PRIMARY) else { + throw MongoDBError(code: 0, message: MongoScriptText.cursorFailed) + } + defer { mongoc_read_prefs_destroy(primary) } + + let session = attachCancellableSession(client: client, opts: options) + defer { + if let session { + releaseSessionLsid() + mongoc_client_session_destroy(session) + } + } + + let handle = try getCollection(client, database: database, collection: collection) + defer { mongoc_collection_destroy(handle) } + try checkCancelled() + + guard let cursor = mongoc_collection_aggregate(handle, MONGOC_QUERY_NONE, pipelineBson, options, primary) else { + throw MongoDBError(code: 0, message: MongoScriptText.cursorFailed) + } + defer { mongoc_cursor_destroy(cursor) } + + var pointer: OpaquePointer? + let found = mongoc_cursor_next(cursor, &pointer) ? pointer.flatMap { bsonToJson($0) } : nil + var error = bson_error_t() + if mongoc_cursor_error(cursor, &error) { + throw makeError(error) + } + try checkCancelled() + return found + } + + /// Every document to the end of the cursor. A catalog read cut short would answer for part of + /// the database, so running past the ceiling is an error rather than a shorter answer. + private func drain(_ cursor: OpaquePointer) throws -> [String] { + var documents: [String] = [] + var pointer: OpaquePointer? + while mongoc_cursor_next(cursor, &pointer) { + try checkCancelled() + if let document = pointer, let json = bsonToJson(document) { + documents.append(json) + } + guard documents.count <= PluginRowLimits.emergencyMax else { + throw MongoDBError(code: 0, message: MongoScriptText.cursorFailed) + } + } + var error = bson_error_t() + if mongoc_cursor_error(cursor, &error) { + throw makeError(error) + } + return documents + } +} + +extension MongoWriteConcern { + init(client: OpaquePointer) { + guard let concern = mongoc_client_get_write_concern(client) else { + self = .serverDefault + return + } + let timeout = mongoc_write_concern_get_wtimeout_int64(concern) + self.init( + acknowledgement: Self.acknowledgement(of: concern), + journal: mongoc_write_concern_journal_is_set(concern) ? mongoc_write_concern_get_journal(concern) : nil, + timeoutMS: timeout > 0 ? timeout : nil + ) + } + + private static func acknowledgement(of concern: OpaquePointer) -> Acknowledgement? { + let w = mongoc_write_concern_get_w(concern) + switch w { + case MONGOC_WRITE_CONCERN_W_DEFAULT: + return nil + case MONGOC_WRITE_CONCERN_W_MAJORITY: + return .majority + case MONGOC_WRITE_CONCERN_W_TAG: + return mongoc_write_concern_get_wtag(concern).map { .tag(String(cString: $0)) } + default: + return .members(max(0, w)) + } + } +} +#endif diff --git a/Plugins/MongoDBDriverPlugin/MongoDBConnection.swift b/Plugins/MongoDBDriverPlugin/MongoDBConnection.swift index 310cee6cd3..8392fe6de0 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDBConnection.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDBConnection.swift @@ -82,6 +82,7 @@ final class MongoDBConnection: @unchecked Sendable { private var _isConnected: Bool = false private var _isShuttingDown: Bool = false private var _cachedServerVersion: String? + private var _writeConcern = MongoWriteConcern.serverDefault private var _isCancelled: Bool = false private var _queryTimeoutMS: Int32 = 0 #if canImport(CLibMongoc) @@ -113,6 +114,13 @@ final class MongoDBConnection: @unchecked Sendable { return _queryTimeoutMS } + /// The write concern the connection's URI sets, read from the client once it connects. + var configuredWriteConcern: MongoWriteConcern { + stateLock.lock() + defer { stateLock.unlock() } + return _writeConcern + } + func setQueryTimeout(_ seconds: Int) { stateLock.lock() _queryTimeoutMS = Int32(seconds * 1_000) @@ -185,6 +193,14 @@ final class MongoDBConnection: @unchecked Sendable { return try body(client) } } + + /// Runs a libmongoc call on the connection's own queue without blocking the caller. + func withClient(_ body: @escaping @Sendable (OpaquePointer) throws -> T) async throws -> T { + try await pluginDispatchAsync(on: queue) { [self] in + guard !isShuttingDown, let client else { throw MongoDBError.notConnected } + return try body(client) + } + } #endif /// Clears a stale cancellation flag so a new script run starts clean. @@ -282,8 +298,8 @@ final class MongoDBConnection: @unchecked Sendable { "tls", "tlsAllowInvalidCertificates", "tlsAllowInvalidHostnames", "tlsCAFile", "tlsCertificateKeyFile" ] - if readPreference != nil, !readPreference!.isEmpty { explicitKeys.insert("readPreference") } - if writeConcern != nil, !writeConcern!.isEmpty { explicitKeys.insert("w") } + if let readPreference, !readPreference.isEmpty { explicitKeys.insert("readPreference") } + if let writeConcern, !writeConcern.isEmpty { explicitKeys.insert("w") } for (key, value) in extraUriParams where !explicitKeys.contains(key) { let encodedValue = value.addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed) ?? value params.append("\(key)=\(encodedValue)") @@ -364,9 +380,11 @@ final class MongoDBConnection: @unchecked Sendable { } self.client = newClient + let configuredWriteConcern = MongoWriteConcern(client: newClient) self.stateLock.lock() self._isConnected = true + self._writeConcern = configuredWriteConcern self.stateLock.unlock() logger.info("Connected to MongoDB at \(self.host):\(self.port)") diff --git a/Plugins/MongoDBDriverPlugin/MongoDBPlugin.swift b/Plugins/MongoDBDriverPlugin/MongoDBPlugin.swift index 10bd65fe46..df50b2f03f 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDBPlugin.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDBPlugin.swift @@ -96,7 +96,12 @@ final class MongoDBPlugin: NSObject, TableProPlugin, DriverPlugin { static let queryLanguageName = "MQL" static let editorLanguage: EditorLanguage = .javascript static let supportsForeignKeys = false - static let supportsSchemaEditing = false + static let supportsSchemaEditing = MongoDBStructureEditing.supportsSchemaEditing + static let supportsAddColumn = MongoDBStructureEditing.supportsAddColumn + static let supportsModifyColumn = MongoDBStructureEditing.supportsModifyColumn + static let supportsDropColumn = MongoDBStructureEditing.supportsDropColumn + static let supportsAddIndex = MongoDBStructureEditing.supportsAddIndex + static let supportsDropIndex = MongoDBStructureEditing.supportsDropIndex static let systemDatabaseNames: [String] = ["admin", "local", "config"] static let tableEntityName = "Collections" static let supportsForeignKeyDisable = false diff --git a/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver+Structure.swift b/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver+Structure.swift new file mode 100644 index 0000000000..0beddafd90 --- /dev/null +++ b/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver+Structure.swift @@ -0,0 +1,68 @@ +// +// MongoDBPluginDriver+Structure.swift +// MongoDBDriverPlugin +// + +import Foundation +import TableProPluginKit + +extension MongoDBPluginDriver { + func generateModifyColumnSQL( + table: String, + oldColumn: PluginColumnDefinition, + newColumn: PluginColumnDefinition + ) -> String? { + let operation = PluginSchemaOperation.modifyColumn(old: oldColumn, new: newColumn) + guard MongoFieldChange.refusal(for: operation) == nil, let change = MongoFieldChange(operation) else { + return nil + } + return change.statement(collection: table, writeConcern: writeConcern) + } + + func generateDropColumnSQL(table: String, columnName: String) -> String? { + guard columnName != MongoDBCollectionDDL.idField, MongoFieldName.addressingRefusal(columnName) == nil else { + return nil + } + return MongoFieldChange.remove(columnName).statement(collection: table, writeConcern: writeConcern) + } + + func reviewSchemaChange( + table: String, + schema: String?, + operations: [PluginSchemaOperation] + ) async throws -> PluginSchemaChangeReview { + guard !MongoFieldChangePlan(operations: operations).isEmpty else { return PluginSchemaChangeReview() } + return try await fieldChangeCheck(collection: table).review(operations: operations) + } + + func schemaChangeRefusalBeforeWriting( + table: String, + schema: String?, + operations: [PluginSchemaOperation], + review: PluginSchemaChangeReview + ) async throws -> String? { + guard !MongoFieldChangePlan(operations: operations).isEmpty else { return nil } + return try await fieldChangeCheck(collection: table).refusalBeforeWriting(operations: operations, review: review) + } + + func schemaChangeShortfallAfterWriting( + table: String, + schema: String?, + operations: [PluginSchemaOperation], + review: PluginSchemaChangeReview + ) async throws -> String? { + guard !MongoFieldChangePlan(operations: operations).isEmpty else { return nil } + return try await fieldChangeCheck(collection: table).shortfallAfterWriting(operations: operations, review: review) + } + + /// The statements carry the write concern the connection's URI sets. A driver that has not + /// connected composes nothing to run, since the review before it throws. + private var writeConcern: MongoWriteConcern { + mongoConnection?.configuredWriteConcern ?? .serverDefault + } + + private func fieldChangeCheck(collection: String) throws -> MongoFieldChangeCheck { + guard let connection = mongoConnection else { throw MongoDBPluginError.notConnected } + return MongoFieldChangeCheck(connection: connection, database: currentDb, collection: collection) + } +} diff --git a/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift b/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift index 5c6da17ebe..0a1d618c5c 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift @@ -714,7 +714,7 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func schemaOperationRefusal(_ operation: PluginSchemaOperation) -> String? { - MongoDBCollectionDDL.refusal(for: operation) + MongoFieldChange.refusal(for: operation) ?? MongoDBCollectionDDL.refusal(for: operation) } var unsupportedIndexTypes: Set { MongoDBCollectionDDL.unsupportedIndexTypes } @@ -1043,7 +1043,21 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { /// Two databases can hold a collection of the same name with different field types. private func columnKindKey(_ collection: String) -> String { - "\(currentDb)\u{0}\(collection)" + MongoCollectionCacheKey.key(database: currentDb, collection: collection) + } + + /// A Structure save renamed or removed fields on another connection, so the fields and types + /// this driver learned from the collection's documents and its validator no longer hold: a + /// later page would reuse the declared schema, and a write would type a renamed field by its old + /// name. Dropped in every database, since the save's database need not be this driver's. + func tableDefinitionDidChange(table: String, schema: String?) { + let isStale = { (key: String) in MongoCollectionCacheKey.names(key, collection: table) } + columnKindLock.withLock { + columnKindsByCollection = columnKindsByCollection.filter { !isStale($0.key) } + fieldPathKindsByCollection = fieldPathKindsByCollection.filter { !isStale($0.key) } + declaredSchemasByCollection = declaredSchemasByCollection.filter { !isStale($0.key) } + identityKindsByCollection = identityKindsByCollection.filter { !isStale($0.key) } + } } } diff --git a/Plugins/MongoDBDriverPlugin/MongoDBStructureEditing.swift b/Plugins/MongoDBDriverPlugin/MongoDBStructureEditing.swift new file mode 100644 index 0000000000..a81602b9d6 --- /dev/null +++ b/Plugins/MongoDBDriverPlugin/MongoDBStructureEditing.swift @@ -0,0 +1,22 @@ +// +// MongoDBStructureEditing.swift +// MongoDBDriverPlugin +// + +import Foundation + +/// What the Structure tab can change on a collection: a field's name, carried out on every document +/// that holds it, and the field's removal. A field is added by writing it to a document, and index +/// changes stay in the shell, because the index list the tab shows does not carry the key order or +/// the index type a recreated index would need. +/// +/// Kept apart from `MongoDBPlugin` so the app's curated copy of these flags can be compared with +/// them in tests, which cannot compile the plugin class and the driver it creates. +enum MongoDBStructureEditing { + static let supportsSchemaEditing = true + static let supportsAddColumn = false + static let supportsModifyColumn = true + static let supportsDropColumn = true + static let supportsAddIndex = false + static let supportsDropIndex = false +} diff --git a/Plugins/MongoDBDriverPlugin/MongoFieldChange.swift b/Plugins/MongoDBDriverPlugin/MongoFieldChange.swift new file mode 100644 index 0000000000..efecf4b606 --- /dev/null +++ b/Plugins/MongoDBDriverPlugin/MongoFieldChange.swift @@ -0,0 +1,164 @@ +// +// MongoFieldChange.swift +// MongoDBDriverPlugin +// + +import Foundation +import TableProPluginKit + +/// A Structure tab column edit, carried out on the documents of a collection. +/// +/// A collection declares no columns: the Structure tab lists the fields its documents hold. A rename +/// is therefore `$rename` in every document that has the field, and a removal is `$unset` in every +/// document that has it. Each statement is one `updateMany` whose filter keeps every document atomic +/// and every run idempotent: a rename skips a document that already holds the new name, so nothing +/// is ever overwritten, and running the same statement again after a stop touches only the documents +/// it had not reached. +enum MongoFieldChange: Equatable, Sendable { + case rename(from: String, to: String) + case remove(String) + + init?(_ operation: PluginSchemaOperation) { + switch operation { + case .modifyColumn(let old, let new): + guard old.name != new.name else { return nil } + self = .rename(from: old.name, to: new.name) + case .dropColumn(let column): + self = .remove(column.name) + default: + return nil + } + } + + var source: String { + switch self { + case .rename(let from, _): return from + case .remove(let name): return name + } + } + + var target: String? { + switch self { + case .rename(_, let to): return to + case .remove: return nil + } + } + + var names: [String] { + [source] + (target.map { [$0] } ?? []) + } + + /// The filter of the statement, which is also the set of documents the server validates when + /// the statement runs. + var filterJson: String { + let source = MongoScriptJson.jsonString(source) + guard let target else { return "{\(source): {\"$exists\": true}}" } + return "{\(source): {\"$exists\": true}, \(MongoScriptJson.jsonString(target)): {\"$exists\": false}}" + } + + /// The `updateMany`, carrying the connection's write concern as its option when the connection + /// sets one, so the save is acknowledged the way the user asked for. + func statement(collection: String, writeConcern: MongoWriteConcern) -> String { + let accessor = MongoCollectionAccessor.expression(for: collection) + let source = MongoScriptJson.jsonString(source) + let update = target.map { "{\"$rename\": {\(source): \(MongoScriptJson.jsonString($0))}}" } + ?? "{\"$unset\": {\(source): \"\"}}" + guard let concern = writeConcern.schemaChangeJson else { + return "\(accessor).updateMany(\(filterJson), \(update))" + } + return "\(accessor).updateMany(\(filterJson), \(update), {\"writeConcern\": \(concern)})" + } + + /// Why a column edit cannot be carried out on the documents, or nil when it can. Nil too for + /// every operation that is not a column edit, so the collection's own refusals still answer. + static func refusal(for operation: PluginSchemaOperation) -> String? { + switch operation { + case .modifyColumn(let old, let new): + guard old.name != new.name, !changesMoreThanTheName(old, new) else { + return String(localized: "A MongoDB field has no type, default or nullability to change. Only its name can change.") + } + return keyRefusal(old.name) ?? MongoFieldName.addressingRefusal(old.name) + ?? keyRefusal(new.name) ?? MongoFieldName.addressingRefusal(new.name) + case .dropColumn(let column): + return keyRefusal(column.name) ?? MongoFieldName.addressingRefusal(column.name) + default: + return nil + } + } + + private static func changesMoreThanTheName(_ old: PluginColumnDefinition, _ new: PluginColumnDefinition) -> Bool { + old.dataType != new.dataType || old.isNullable != new.isNullable + || old.defaultValue != new.defaultValue || old.comment != new.comment + } + + private static func keyRefusal(_ name: String) -> String? { + guard name == MongoDBCollectionDDL.idField else { return nil } + return String(localized: "_id cannot be renamed, removed or used as a new name. MongoDB keys every document by it.") + } +} + +enum MongoFieldName { + /// Why an update cannot name this field, or nil when it can. + /// + /// `$rename` and `$unset` read a dot as a path into an embedded document and refuse a leading + /// `$`, so a literal key spelled either way cannot be reached by name. A NUL cannot be carried in + /// a BSON key, and `__proto__` vanishes from the statement's object literal in the shell, which + /// turns `$unset` into a write that matches and changes nothing. + static func addressingRefusal(_ name: String) -> String? { + if name.isEmpty { + return String(localized: "A MongoDB field needs a name.") + } + if name.hasPrefix("$") || name.contains(".") { + return String( + format: String(localized: "MongoDB cannot address a field named %@. A field name cannot start with $ or contain a dot."), + name + ) + } + if name.unicodeScalars.contains("\u{0}") { + return String(localized: "A MongoDB field name cannot contain a NUL character.") + } + if name == "__proto__" { + return String( + format: String(localized: "The shell would reorder or drop a field named %@. Choose another name."), + name + ) + } + return nil + } +} + +/// The field edits of one save, in the order their statements run. +struct MongoFieldChangePlan: Equatable, Sendable { + let changes: [MongoFieldChange] + let refusal: String? + + /// Each statement runs on its own, so a field named by two edits of one save would be read by + /// the second after the first had already moved it: `a` to `b` then `b` to `c` carries `a`'s + /// values on to `c`, and a swap overwrites nothing and completes nothing. Such a save is refused + /// before anything is read. + init(operations: [PluginSchemaOperation]) { + let changes = operations.compactMap(MongoFieldChange.init) + self.changes = changes + self.refusal = Self.sharedNameRefusal(changes) + } + + var isEmpty: Bool { changes.isEmpty } + + var renames: [(from: String, to: String)] { + changes.compactMap { change in + guard let target = change.target else { return nil } + return (change.source, target) + } + } + + private static func sharedNameRefusal(_ changes: [MongoFieldChange]) -> String? { + var seen = Set() + for name in changes.flatMap(\.names) where !seen.insert(name).inserted { + return String( + format: String(localized: "%@ is changed twice in this save. Save one change to it at a time."), + name + ) + } + return nil + } +} diff --git a/Plugins/MongoDBDriverPlugin/MongoFieldChangeAssessment.swift b/Plugins/MongoDBDriverPlugin/MongoFieldChangeAssessment.swift new file mode 100644 index 0000000000..ac74b43bb4 --- /dev/null +++ b/Plugins/MongoDBDriverPlugin/MongoFieldChangeAssessment.swift @@ -0,0 +1,385 @@ +// +// MongoFieldChangeAssessment.swift +// MongoDBDriverPlugin +// + +import Foundation +import TableProPluginKit + +/// Whether a save's field edits can run against the collection as the catalog describes it, and the +/// validator the collection has to carry for them to. +/// +/// Everything here is read from the catalog and nothing from the documents, so it is cheap enough +/// for SQL Preview. What only the documents can show is `MongoFieldDataProbe`'s, on Save. +struct MongoFieldChangeAssessment: Equatable, Sendable { + let refusal: String? + /// The validator after the edits, when they change it. It runs first, as a `collMod`. + let rewrittenValidatorJson: String? + /// The validator the statements are checked against: the rewritten one when there is one. + let effectiveValidatorJson: String? + + /// Every check the catalog can answer, in the order the reads come in: the collection's kind + /// decides whether it has indexes to list at all, since `listIndexes` on a view fails with + /// CommandNotSupportedOnView and on a missing collection with NamespaceNotFound. + static func kindRefusal(_ info: MongoCollectionInfo) -> String? { + if info.name.hasPrefix("system.") { + return String( + format: String(localized: "%@ is a system collection. Its fields cannot be renamed or removed."), + info.name + ) + } + switch info.kind { + case .collection: + return nil + case .missing: + return String(format: String(localized: "Collection %@ no longer exists."), info.name) + case .view: + return String( + format: String(localized: "%@ is a view. Rename or remove the field in the collection it reads."), + info.name + ) + case .timeseries: + return String( + format: String(localized: "%@ is a time series collection. MongoDB cannot rename or remove its fields."), + info.name + ) + case .other(let type): + return String( + format: String(localized: "%1$@ is a collection of type %2$@. Its fields cannot be renamed or removed."), + info.name, type + ) + } + } + + static func assess( + _ changes: [MongoFieldChange], + info: MongoCollectionInfo, + indexes: [MongoIndexSpec], + searchIndexes: [MongoSearchIndex], + views: [MongoViewDefinition] + ) -> MongoFieldChangeAssessment { + if let refusal = kindRefusal(info) + ?? cappedRefusal(changes, info: info) + ?? MongoFieldDependent.index(of: changes, among: indexes)?.refusal(collection: info.name) + ?? MongoFieldDependent.searchIndex(of: changes, among: searchIndexes)?.refusal(collection: info.name) + ?? encryptionRefusal(changes, paths: info.encryptedFieldPaths) { + return refused(refusal) + } + let validator = MongoValidatorRewrite.rewrite(info.validatorJson, applying: changes) + if let refusal = validator.refusal + ?? MongoFieldDependent.view(of: changes, among: views, collection: info.name)?.refusal(collection: info.name) { + return refused(refusal) + } + return MongoFieldChangeAssessment( + refusal: nil, + rewrittenValidatorJson: validator.rewrittenJson, + effectiveValidatorJson: validator.rewrittenJson ?? info.validatorJson + ) + } + + /// The statement that puts the rewritten validator in place, ahead of the updates. Level and + /// action are left out, so the collection keeps its own. `mongoc_client_command_simple` sends + /// no write concern of its own, so the connection's goes in the command. + static func validatorStatement( + collection: String, + validatorJson: String, + writeConcern: MongoWriteConcern + ) -> String { + var members = [ + "\"collMod\": \(MongoScriptJson.jsonString(collection))", + "\"validator\": \(validatorJson)" + ] + if let concern = writeConcern.schemaChangeJson { + members.append("\"writeConcern\": \(concern)") + } + return "db.runCommand({\(members.joined(separator: ", "))})" + } + + /// What runs ahead of the updates: the `collMod` that carries the validator along, when the + /// edits change it. + func leadingStatements(collection: String, writeConcern: MongoWriteConcern) -> [String] { + guard let rewrittenValidatorJson else { return [] } + return [ + Self.validatorStatement(collection: collection, validatorJson: rewrittenValidatorJson, writeConcern: writeConcern) + ] + } + + /// Why the statements the save was composed with must not run, now that the catalog has been + /// read again: the collection's `listCollections` entry is no longer the one the save was + /// composed from. The `collMod` holds the whole validator as it was read then, so running it + /// after someone else changed the validator would put the old one back over theirs, and the + /// checks after writing read the level, the action and the validator from that entry. + static func changedSinceComposedRefusal(composedFrom basis: String?, current infoJson: String?, collection: String) -> String? { + guard basis != infoJson else { return nil } + return String( + format: String(localized: "%@ changed after this save was prepared, so nothing was changed. Review the save and save again."), + collection + ) + } + + /// Why the save must not write after all, now that the catalog has been read once more, as the + /// last thing before the first statement. Another client's `collMod` during a scan would + /// otherwise be overwritten by the one the save composed, and a validation level or action + /// changed then would leave the scans answering for rules the collection no longer has. Any + /// change to the collection's `listCollections` entry refuses, and so does any index, search + /// index or view that now refuses the change. + static func changedDuringChecksRefusal( + checked: MongoCatalogRead, + current: MongoCatalogRead, + collection: String + ) -> String? { + if let refusal = current.assessment.refusal { + return refusal + } + guard current.infoJson == checked.infoJson else { + return String( + format: String(localized: "%@ changed while its documents were being checked, so nothing was changed. Save again."), + collection + ) + } + return nil + } + + private static func refused(_ reason: String) -> MongoFieldChangeAssessment { + MongoFieldChangeAssessment(refusal: reason, rewrittenValidatorJson: nil, effectiveValidatorJson: nil) + } + + /// A capped collection holds a fixed number of bytes. Measured on 7.0.43: a rename to a longer + /// name succeeds and leaves the collection over its size, and the next insert then deletes the + /// oldest documents until it fits (one insert into a full 4,096-byte collection removed 47 of + /// 97). A rename that keeps or shortens the name, and a removal, never make a document larger. + private static func cappedRefusal(_ changes: [MongoFieldChange], info: MongoCollectionInfo) -> String? { + guard info.isCapped else { return nil } + for change in changes { + guard let target = change.target, target.utf8.count > change.source.utf8.count else { continue } + return String( + format: String(localized: "%1$@ is a capped collection, and a longer field name makes MongoDB delete its oldest documents. Choose a name no longer than %2$@."), + info.name, change.source + ) + } + return nil + } + + private static func encryptionRefusal(_ changes: [MongoFieldChange], paths: [String]) -> String? { + for change in changes { + for name in change.names where paths.contains(where: { MongoFieldPath.reaches($0, field: name) }) { + return String(format: String(localized: "%@ is an encrypted field. MongoDB cannot rename or remove it."), name) + } + } + return nil + } +} + +/// Carries a rename or a removal into a `$jsonSchema` validator that declares the field. +/// +/// A collection created from New Table declares every field in `$jsonSchema.properties` and lists +/// every field that is not nullable in `required`. Renaming such a field on the documents alone +/// leaves the validator requiring the old name, so every renamed document would fail it, and a +/// field no document holds yet is listed from the validator and could never be removed at all. So +/// the rename or removal is applied to `properties`, `required` and `dependencies` at the schema's +/// top level, where it is a plain change of key. A validator that names the field anywhere else +/// (a query operator, `$expr`, a pattern that matches it, a nested schema) cannot be rewritten +/// faithfully and refuses the save. +enum MongoValidatorRewrite { + struct Outcome: Equatable { + let rewrittenJson: String? + let refusal: String? + } + + static func rewrite(_ validatorJson: String?, applying changes: [MongoFieldChange]) -> Outcome { + guard let validatorJson else { return Outcome(rewrittenJson: nil, refusal: nil) } + var schemaJson = soleJsonSchema(of: validatorJson) + var changed = false + + if let original = schemaJson { + var members = MongoScriptJson.members(of: original) + for change in changes { + guard let result = apply(change, to: members) else { + return Outcome(rewrittenJson: nil, refusal: targetDeclaredRefusal(change)) + } + changed = changed || result.changed + members = result.members + } + schemaJson = object(members) + } + + let rewritten = changed ? schemaJson.map { "{\"$jsonSchema\": \($0)}" } : nil + let effective = rewritten ?? validatorJson + guard let parsed = MongoJsonValue.parse(effective) else { + return Outcome(rewrittenJson: nil, refusal: unreadableRefusal) + } + for change in changes where MongoQueryFieldReferences.reaches(parsed, field: change.source) { + return Outcome(rewrittenJson: nil, refusal: namedElsewhereRefusal(change.source)) + } + if let name = nameRuleConflict(changes, original: MongoJsonValue.parse(validatorJson), rewritten: parsed) { + return Outcome(rewrittenJson: nil, refusal: namedElsewhereRefusal(name)) + } + if rewritten != nil, let key = keyTheShellWouldMove(in: parsed) { + return Outcome( + rewrittenJson: nil, + refusal: String( + format: String(localized: "The validator has a key named %@ that the shell would reorder or drop. Change the validator from a query tab instead."), + key + ) + ) + } + return Outcome(rewrittenJson: rewritten, refusal: nil) + } + + /// A name a `patternProperties` or `additionalProperties` rule applies to: the old name as the + /// validator stands, where the rule is what checked the field, and the new name as it will + /// stand, where the rule is what would check it. A name `properties` declares is outside + /// `additionalProperties`, so a declared field that the rewrite carries along is not caught. + private static func nameRuleConflict(_ changes: [MongoFieldChange], original: Any?, rewritten: Any) -> String? { + for change in changes { + if let original, MongoQueryFieldReferences.appliesByName(original, to: change.source) { return change.source } + if let target = change.target, MongoQueryFieldReferences.appliesByName(rewritten, to: target) { return target } + } + return nil + } + + private static func soleJsonSchema(of validatorJson: String) -> String? { + let members = MongoScriptJson.members(of: validatorJson) + guard members.count == 1, members[0].key == "$jsonSchema" else { return nil } + return members[0].value + } + + private typealias Members = [(key: String, value: String)] + + /// Nil when a rename would declare the new name twice. + private static func apply(_ change: MongoFieldChange, to members: Members) -> (members: Members, changed: Bool)? { + var changed = false + var result: Members = [] + for member in members { + switch member.key { + case "properties": + guard let rewritten = renamedKeys(member.value, change: change) else { return nil } + changed = changed || rewritten.changed + result.append((member.key, rewritten.json)) + case "required": + let rewritten = renamedNames(member.value, change: change) + changed = changed || rewritten.changed + if let json = rewritten.json { result.append((member.key, json)) } + case "dependencies": + guard let rewritten = renamedDependencies(member.value, change: change) else { return nil } + changed = changed || rewritten.changed + if let json = rewritten.json { result.append((member.key, json)) } + default: + result.append(member) + } + } + return (result, changed) + } + + private static func renamedKeys(_ objectJson: String, change: MongoFieldChange) -> (json: String, changed: Bool)? { + let members = MongoScriptJson.members(of: objectJson) + guard members.contains(where: { $0.key == change.source }) else { return (objectJson, false) } + if let target = change.target, members.contains(where: { $0.key == target }) { return nil } + let rewritten: Members = members.compactMap { member in + guard member.key == change.source else { return member } + return change.target.map { ($0, member.value) } + } + return (object(rewritten), true) + } + + /// Nil `json` when the list is left empty, which the server refuses: measured on 7.0.43, + /// `required: []` fails with "$jsonSchema keyword 'required' cannot be an empty array". + private static func renamedNames(_ arrayJson: String, change: MongoFieldChange) -> (json: String?, changed: Bool) { + let elements = MongoScriptJson.topLevelElements(arrayJson) + let names = elements.map { MongoJsonValue.parse($0) as? String } + guard names.contains(change.source) else { return (arrayJson, false) } + var rewritten: [String] = [] + for (element, name) in zip(elements, names) { + guard name == change.source else { + if !rewritten.contains(element) { rewritten.append(element) } + continue + } + guard let target = change.target else { continue } + let quoted = MongoScriptJson.jsonString(target) + if !rewritten.contains(quoted), !names.contains(target) { rewritten.append(quoted) } + } + guard !rewritten.isEmpty else { return (nil, true) } + return ("[\(rewritten.joined(separator: ", "))]", true) + } + + /// A property dependency (`a: ["b"]`) is a list of names and is rewritten like `required`. A + /// schema dependency is left alone; the check that follows refuses it when it names the field. + /// Measured on 7.0.43: a dependency's list cannot be empty either, so an emptied one is dropped. + private static func renamedDependencies(_ objectJson: String, change: MongoFieldChange) -> (json: String?, changed: Bool)? { + let members = MongoScriptJson.members(of: objectJson) + if let target = change.target, + members.contains(where: { $0.key == change.source }), members.contains(where: { $0.key == target }) { + return nil + } + var changed = false + var rewritten: Members = [] + for member in members { + var key = member.key + if key == change.source { + changed = true + guard let target = change.target else { continue } + key = target + } + guard member.value.hasPrefix("[") else { + rewritten.append((key, member.value)) + continue + } + let names = renamedNames(member.value, change: change) + changed = changed || names.changed + if let json = names.json { rewritten.append((key, json)) } + } + guard changed else { return (objectJson, false) } + return (rewritten.isEmpty ? nil : object(rewritten), true) + } + + private static func object(_ members: Members) -> String { + "{" + members.map { "\(MongoScriptJson.jsonString($0.key)): \($0.value)" }.joined(separator: ", ") + "}" + } + + /// The `collMod` is a JavaScript object literal, which lists integer-like keys first and reads + /// `__proto__` as the prototype, so either kind of key would not reach the server as written. + private static func keyTheShellWouldMove(in value: Any) -> String? { + if let list = value as? [Any] { + return list.lazy.compactMap(keyTheShellWouldMove(in:)).first + } + guard let object = value as? [String: Any] else { return nil } + for (key, member) in object { + if key == "__proto__" || isIntegerLike(key) { return key } + if let nested = keyTheShellWouldMove(in: member) { return nested } + } + return nil + } + + private static func isIntegerLike(_ key: String) -> Bool { + guard !key.isEmpty, key.utf8.allSatisfy({ (UInt8(ascii: "0") ... UInt8(ascii: "9")).contains($0) }) else { + return false + } + return key == "0" || !key.hasPrefix("0") + } + + private static func targetDeclaredRefusal(_ change: MongoFieldChange) -> String { + String( + format: String(localized: "The validator already declares %@. Remove that declaration first, then save again."), + change.target ?? change.source + ) + } + + private static func namedElsewhereRefusal(_ field: String) -> String { + String( + format: String(localized: "The validator uses %@ in a way that cannot be updated with the field. Change the validator first, then save again."), + field + ) + } + + private static var unreadableRefusal: String { + String(localized: "The validator could not be read, so the change was not checked against it.") + } +} + +/// One read of the catalog for a save, and what it says about the save. +struct MongoCatalogRead { + /// The collection's `listCollections` entry exactly as the server sent it. + let infoJson: String? + let info: MongoCollectionInfo + let assessment: MongoFieldChangeAssessment +} diff --git a/Plugins/MongoDBDriverPlugin/MongoFieldChangeCheck.swift b/Plugins/MongoDBDriverPlugin/MongoFieldChangeCheck.swift new file mode 100644 index 0000000000..4b9c81a781 --- /dev/null +++ b/Plugins/MongoDBDriverPlugin/MongoFieldChangeCheck.swift @@ -0,0 +1,324 @@ +// +// MongoFieldChangeCheck.swift +// MongoDBDriverPlugin +// + +import Foundation +import TableProPluginKit + +/// Reads the server for a save's field renames and removals and answers for the save as a whole. +/// +/// `review` reads the catalog alone and is what SQL Preview pays for, and keeps the collection's +/// `listCollections` entry as the review's basis. `refusalBeforeWriting` reads the catalog again, +/// because the prompts between the two can take any length of time, and refuses when the entry is +/// no longer the one the user confirmed a save for. Then it reads the documents, bounded by the +/// query timeout and ended on the server if the task is cancelled, and reads the catalog a last +/// time, because each of those scans can run for minutes. Every read goes to the primary, where the +/// writes go. A read the user's role does not allow stops the save rather than letting it run +/// unchecked. `shortfallAfterWriting` checks what the statements left behind. +struct MongoFieldChangeCheck { + let connection: MongoDBConnection + let database: String + let collection: String + + func review(operations: [PluginSchemaOperation]) async throws -> PluginSchemaChangeReview { + let plan = MongoFieldChangePlan(operations: operations) + guard !plan.isEmpty else { return PluginSchemaChangeReview() } + if let refusal = plan.refusal { return PluginSchemaChangeReview(refusal: refusal) } + + let read = try await assess(plan) + if let refusal = read.assessment.refusal { return PluginSchemaChangeReview(refusal: refusal) } + return PluginSchemaChangeReview( + leadingStatements: read.assessment.leadingStatements( + collection: collection, writeConcern: connection.configuredWriteConcern + ), + basis: read.infoJson + ) + } + + func refusalBeforeWriting( + operations: [PluginSchemaOperation], + review: PluginSchemaChangeReview + ) async throws -> String? { + let plan = MongoFieldChangePlan(operations: operations) + guard !plan.isEmpty else { return nil } + if let refusal = plan.refusal { return refusal } + + let checked = try await assess(plan) + if let refusal = checked.assessment.refusal + ?? MongoFieldChangeAssessment.changedSinceComposedRefusal( + composedFrom: review.basis, current: checked.infoJson, collection: collection + ) { + return refusal + } + if let refusal = try await documentRefusal(plan, assessment: checked.assessment, info: checked.info) { + return refusal + } + return MongoFieldChangeAssessment.changedDuringChecksRefusal( + checked: checked, current: try await assess(plan), collection: collection + ) + } + + /// What the statements left behind, checked once all of them have run. + /// + /// First the documents that still hold an old name: one another client wrote it into while an + /// `updateMany` ran, and one it gave the new name to before the rename reached it, which the + /// rename's filter then skips. Then, when the save rewrote the validator, a document the new + /// validator rejects: the `collMod` checks no document, so one another client gave only the new + /// name after the last check before writing was accepted by the old validator, which did not + /// read that name, and is left failing the rule it now falls under. `review` is what the save + /// was composed with, and the check before writing refused unless its basis was still the + /// collection's entry, so the validator, level and action read from it are the ones the + /// statements ran against. + func shortfallAfterWriting( + operations: [PluginSchemaOperation], + review: PluginSchemaChangeReview + ) async throws -> String? { + let plan = MongoFieldChangePlan(operations: operations) + guard !plan.isEmpty, plan.refusal == nil else { return nil } + var remaining: [(change: MongoFieldChange, documents: Int64)] = [] + for change in plan.changes { + let found = try await count(MongoFieldDataProbe.remainderPipeline(change)) + remaining.append((change, found)) + } + if let shortfall = MongoFieldDataProbe.shortfall(remaining) { + return shortfall + } + if let violation = try await violationAfterWriting(plan, composedFrom: review.basis) { + return violation + } + return try await dependencyAfterWriting(plan) + } + + /// An index, a search index or a view another client made on either name while the statements + /// ran, after the last read before writing, which nothing earlier could see. + private func dependencyAfterWriting(_ plan: MongoFieldChangePlan) async throws -> String? { + let indexes = try await afterWriting { try await connection.indexSpecsJson(database: database, collection: collection) } + let searchIndexes = try await afterWriting { + try await connection.searchIndexesJson(database: database, collection: collection) + } + let views = try await afterWriting { try await connection.viewInfosJson(database: database) } + return MongoFieldDependent.dependent( + of: plan.changes, + indexes: indexes.compactMap(MongoIndexSpec.init(json:)), + searchIndexes: searchIndexes.compactMap(MongoSearchIndex.init(json:)), + views: views.compactMap(MongoViewDefinition.init(json:)), + collection: collection + )?.appearedDuringSave(collection: collection) + } + + private func afterWriting(_ body: () async throws -> T) async throws -> T { + do { + return try await body() + } catch { + if error is CancellationError || Task.isCancelled { throw CancellationError() } + throw couldNotConfirm(error.localizedDescription) + } + } + + private func violationAfterWriting(_ plan: MongoFieldChangePlan, composedFrom basis: String?) async throws -> String? { + let composed = MongoCollectionInfo(collection: collection, infoJson: basis) + guard composed.enforcesValidator, let original = composed.validatorJson, + let rewritten = MongoValidatorRewrite.rewrite(original, applying: plan.changes).rewrittenJson else { + return nil + } + let pipeline = MongoFieldDataProbe.violationAfterWritingPipeline( + plan.changes, + originalValidatorJson: original, + rewrittenValidatorJson: rewritten, + onlyValidDocuments: composed.validatesOnlyValidDocuments + ) + guard let found = try await readAfterWriting(pipeline) else { return nil } + return MongoFieldDataProbe.violationAfterWriting(identifier: MongoFieldDataProbe.identifier(in: found) ?? "?", collection: collection) + } + + private func documentRefusal( + _ plan: MongoFieldChangePlan, + assessment: MongoFieldChangeAssessment, + info: MongoCollectionInfo + ) async throws -> String? { + if let rename = try await renameHoldingBothNames(plan) { + return String( + format: String(localized: "Some documents hold both %1$@ and %2$@. Remove one of the two from those documents first."), + rename.from, rename.to + ) + } + if let refusal = try await oversizeRefusal(plan) { + return refusal + } + guard info.enforcesValidator, let validator = assessment.effectiveValidatorJson else { return nil } + if let refusal = try await validatorRefusal( + plan, validatorJson: validator, onlyValidDocuments: info.validatesOnlyValidDocuments + ) { + return refusal + } + guard let original = info.validatorJson, let rewritten = assessment.rewrittenValidatorJson else { return nil } + return try await rewrittenRuleRefusal( + plan, originalValidatorJson: original, rewrittenValidatorJson: rewritten, + onlyValidDocuments: info.validatesOnlyValidDocuments + ) + } + + private func assess(_ plan: MongoFieldChangePlan) async throws -> MongoCatalogRead { + let infoJson = try await read { try await connection.collectionInfoJson(database: database, collection: collection) } + let info = MongoCollectionInfo(collection: collection, infoJson: infoJson) + if let refusal = MongoFieldChangeAssessment.kindRefusal(info) { + return MongoCatalogRead( + infoJson: infoJson, + info: info, + assessment: MongoFieldChangeAssessment(refusal: refusal, rewrittenValidatorJson: nil, effectiveValidatorJson: nil) + ) + } + let indexes = try await read { try await connection.indexSpecsJson(database: database, collection: collection) } + let searchIndexes = try await read { + try await connection.searchIndexesJson(database: database, collection: collection) + } + let views = try await read { try await connection.viewInfosJson(database: database) } + let assessment = MongoFieldChangeAssessment.assess( + plan.changes, + info: info, + indexes: indexes.compactMap(MongoIndexSpec.init(json:)), + searchIndexes: searchIndexes.compactMap(MongoSearchIndex.init(json:)), + views: views.compactMap(MongoViewDefinition.init(json:)) + ) + return MongoCatalogRead(infoJson: infoJson, info: info, assessment: assessment) + } + + private func renameHoldingBothNames(_ plan: MongoFieldChangePlan) async throws -> (from: String, to: String)? { + let renames = plan.renames + guard let pipeline = MongoFieldDataProbe.bothNamesPipeline(renames) else { return nil } + let found = try await scan(pipeline) + return MongoFieldDataProbe.renameHoldingBothNames(in: found, renames: renames) + } + + /// A server before 4.4 cannot measure a document, so there the save goes ahead unmeasured, as + /// it did before this check. + private func oversizeRefusal(_ plan: MongoFieldChangePlan) async throws -> String? { + guard let pipeline = MongoFieldDataProbe.oversizePipeline(plan.renames), + let found = try await scan(pipeline, unmeasurableCodes: [MongoFieldDataProbe.unknownExpressionCode]) else { + return nil + } + return String( + format: String(localized: "Renaming would take the document with _id %@ past MongoDB's 16 MB limit. Choose shorter names, or shrink that document first."), + MongoFieldDataProbe.identifier(in: found) ?? "?" + ) + } + + private func validatorRefusal( + _ plan: MongoFieldChangePlan, + validatorJson: String, + onlyValidDocuments: Bool + ) async throws -> String? { + let pipelines = MongoFieldDataProbe.validatorPipelines( + plan.changes, validatorJson: validatorJson, onlyValidDocuments: onlyValidDocuments + ) + for (change, pipeline) in zip(plan.changes, pipelines) { + guard let found = try await scan(pipeline) else { continue } + let identifier = MongoFieldDataProbe.identifier(in: found) ?? "?" + guard let target = change.target else { + return String( + format: String(localized: "The validator would reject the document with _id %1$@ once %2$@ is removed. Fix the document or the validator first."), + identifier, change.source + ) + } + return String( + format: String(localized: "The validator would reject the document with _id %1$@ once %2$@ is renamed to %3$@. Fix the document or the validator first."), + identifier, change.source, target + ) + } + return nil + } + + private func rewrittenRuleRefusal( + _ plan: MongoFieldChangePlan, + originalValidatorJson: String, + rewrittenValidatorJson: String, + onlyValidDocuments: Bool + ) async throws -> String? { + let pipeline = MongoFieldDataProbe.rewrittenRulePipeline( + plan.changes, + originalValidatorJson: originalValidatorJson, + rewrittenValidatorJson: rewrittenValidatorJson, + onlyValidDocuments: onlyValidDocuments + ) + guard let found = try await scan(pipeline) else { return nil } + return String( + format: String(localized: "Once this save updates the validator, it would reject the document with _id %@. Fix the document or the validator first."), + MongoFieldDataProbe.identifier(in: found) ?? "?" + ) + } + + private func scan(_ pipeline: String, unmeasurableCodes: Set = []) async throws -> String? { + let maxTimeMS = MongoFieldDataProbe.maxTimeMS(queryTimeoutMS: connection.queryTimeoutMS) + do { + return try await connection.firstAggregatedDocumentJson( + database: database, collection: collection, pipeline: pipeline, maxTimeMS: maxTimeMS + ) + } catch let error as MongoDBError where unmeasurableCodes.contains(error.code) { + return nil + } catch let error as MongoDBError where MongoDBTimeoutPolicy.isTimeoutCode(error.code) { + throw MongoDBError( + code: 0, + message: String( + format: String(localized: "Checking the documents of %1$@ took over %2$d seconds, so nothing was changed. Raise the query timeout in Settings and save again."), + collection, max(1, Int(maxTimeMS / 1_000)) + ) + ) + } catch { + throw try couldNotCheck(error) + } + } + + /// A count read after the statements ran. + private func count(_ pipeline: String) async throws -> Int64 { + let found = try await readAfterWriting(pipeline) + guard let remaining = MongoFieldDataProbe.remainderCount(in: found) else { + throw couldNotConfirm(found ?? "") + } + return remaining + } + + /// A read after the statements ran. Its failure cannot say nothing was changed, because the + /// statements already did change it. + private func readAfterWriting(_ pipeline: String) async throws -> String? { + let maxTimeMS = MongoFieldDataProbe.maxTimeMS(queryTimeoutMS: connection.queryTimeoutMS) + do { + return try await connection.firstAggregatedDocumentJson( + database: database, collection: collection, pipeline: pipeline, maxTimeMS: maxTimeMS + ) + } catch { + if error is CancellationError || Task.isCancelled { throw CancellationError() } + throw couldNotConfirm(error.localizedDescription) + } + } + + private func couldNotConfirm(_ reason: String) -> MongoDBError { + MongoDBError( + code: 0, + message: String( + format: String(localized: "The save ran, but checking %1$@ afterwards failed: %2$@. Save again to make sure every document was changed."), + collection, reason + ) + ) + } + + private func read(_ body: () async throws -> T) async throws -> T { + do { + return try await body() + } catch { + throw try couldNotCheck(error) + } + } + + /// A cancelled task stays a cancellation. Anything else becomes the reason the save stopped. + private func couldNotCheck(_ error: Error) throws -> Error { + if error is CancellationError || Task.isCancelled { throw CancellationError() } + return MongoDBError( + code: 0, + message: String( + format: String(localized: "Couldn't check %1$@ before changing its documents: %2$@"), + collection, error.localizedDescription + ) + ) + } +} diff --git a/Plugins/MongoDBDriverPlugin/MongoFieldDataProbe.swift b/Plugins/MongoDBDriverPlugin/MongoFieldDataProbe.swift new file mode 100644 index 0000000000..6ed001f1ae --- /dev/null +++ b/Plugins/MongoDBDriverPlugin/MongoFieldDataProbe.swift @@ -0,0 +1,268 @@ +// +// MongoFieldDataProbe.swift +// MongoDBDriverPlugin +// + +import Foundation + +/// The aggregations that read the documents a save is about to change, run on Save alone. +/// +/// Each is a full pass over the documents that hold a changed field, which the catalog cannot +/// replace: a document holding both names of a rename is invisible to it, and so is a document the +/// validator would reject once the field moves. Both would stop the write partway, and a stopped +/// `updateMany` keeps what it already changed. +/// +/// The stages are ones MongoDB 4.0 has, the oldest server libmongoc 1.28 connects to: `$addFields` +/// with `$$REMOVE` rather than the `$set` and `$unset` stages, which arrived in 4.2. +enum MongoFieldDataProbe { + /// The ceiling for each pass when the query timeout is set to No limit. A scan the user cannot + /// stop from the Structure tab must still end. + static let unlimitedTimeoutCeilingMS: Int32 = 600_000 + + static func maxTimeMS(queryTimeoutMS: Int32) -> Int32 { + queryTimeoutMS > 0 ? queryTimeoutMS : unlimitedTimeoutCeilingMS + } + + /// The options of every pass. Each is sent to the primary as well, as its read preference, and + /// reads at `local`: the write it guards goes to the primary and changes what is there now, + /// while a lagging secondary or a `majority` read concern from the connection string can miss a + /// document the write then reaches. + static func aggregateOptionsJson(maxTimeMS: Int32) -> String { + "{\"maxTimeMS\": \(maxTimeMS), \"readConcern\": {\"level\": \"local\"}}" + } + + /// Finds one document that holds both names of any rename, and says which rename. Such a + /// document is skipped by the rename's filter, so it would keep the old name after the save. + static func bothNamesPipeline(_ renames: [(from: String, to: String)]) -> String? { + guard !renames.isEmpty else { return nil } + let holdsBoth = renames.map { "{\(exists($0.from)), \(exists($0.to))}" } + let flags = renames.enumerated().map { index, rename in + "\"p\(index)\": {\"$and\": [\(present(rename.from)), \(present(rename.to))]}" + } + return "[{\"$match\": {\"$or\": [\(holdsBoth.joined(separator: ", "))]}}, {\"$limit\": 1}, " + + "{\"$project\": {\"_id\": 0, \(flags.joined(separator: ", "))}}]" + } + + /// The most bytes MongoDB stores in one document. + static let largestDocumentBytes = 16 * 1_024 * 1_024 + + /// Finds one document the renames would take past the most bytes MongoDB stores. A rename to a + /// longer name adds the difference to every document holding the old one, and `updateMany` is + /// not atomic, so such a document stops the write partway, after the ones before it changed, + /// and every retry stops at it again. `$bsonSize` needs MongoDB 4.4. + static func oversizePipeline(_ renames: [(from: String, to: String)]) -> String? { + let growing = renames.filter { $0.to.utf8.count > $0.from.utf8.count } + guard !growing.isEmpty else { return nil } + let holdsOld = growing.map { "{\(exists($0.from))}" } + let growth = growing.map { rename in + "{\"$cond\": [\(present(rename.from)), \(rename.to.utf8.count - rename.from.utf8.count), 0]}" + } + let size = "{\"$add\": [{\"$bsonSize\": \"$$ROOT\"}, \(growth.joined(separator: ", "))]}" + return "[{\"$match\": {\"$or\": [\(holdsOld.joined(separator: ", "))]}}, " + + "{\"$match\": {\"$expr\": {\"$gt\": [\(size), \(largestDocumentBytes)]}}}, " + + "{\"$limit\": 1}, {\"$project\": {\"_id\": 1}}]" + } + + /// InvalidPipelineOperator: a server before 4.4, which has no `$bsonSize` to measure with. + static let unknownExpressionCode: UInt32 = 168 + + static func renameHoldingBothNames( + in documentJson: String?, + renames: [(from: String, to: String)] + ) -> (from: String, to: String)? { + guard let documentJson, let flags = MongoJsonValue.parse(documentJson) as? [String: Any] else { return nil } + return renames.indices.first { flags["p\($0)"] as? Bool == true }.map { renames[$0] } + } + + /// One pipeline per change, in the order the statements run, each finding a document the + /// server would refuse at that statement. + /// + /// The statements run one at a time and the server validates every document each one changes, + /// so every step is checked against the state the steps before it leave, never against the + /// final state alone: renaming `p` to `x` and `q` to `y` under `dependencies: {x: ["y"]}` ends + /// valid and still fails at the first statement on a document holding `p` and `q`. The earlier + /// steps are replayed with the same guard their filters apply, the step itself is applied to + /// the documents its filter selects, and a `moderate` validator first sets aside the documents + /// it already rejects, because the server does not validate an update to one of those. + static func validatorPipelines( + _ changes: [MongoFieldChange], + validatorJson: String, + onlyValidDocuments: Bool + ) -> [String] { + changes.indices.map { step in + var stages = changes[.. String { + rejectedByRewrittenValidator( + changes, + originalValidatorJson: originalValidatorJson, + rewrittenValidatorJson: rewrittenValidatorJson, + onlyValidDocuments: onlyValidDocuments, + replaying: changes.map(replay) + ) + } + + /// The same question once the statements have run, so nothing is replayed: one document the + /// rewritten validator rejects among every document that holds a name the save changed. The + /// server validated each document a statement changed, so under `strict` a document found here + /// is one no statement touched, which another client wrote under the old validator after the + /// last check before writing. Under `moderate` only a document the old validator accepts as it + /// now stands counts, as before writing. + static func violationAfterWritingPipeline( + _ changes: [MongoFieldChange], + originalValidatorJson: String, + rewrittenValidatorJson: String, + onlyValidDocuments: Bool + ) -> String { + rejectedByRewrittenValidator( + changes, + originalValidatorJson: originalValidatorJson, + rewrittenValidatorJson: rewrittenValidatorJson, + onlyValidDocuments: onlyValidDocuments, + replaying: [] + ) + } + + /// Why a save whose statements all ran did not finish: a document the validator it put in place + /// rejects. Saving again cannot fix that, so the message says what does. + static func violationAfterWriting(identifier: String, collection: String) -> String { + String( + format: String( + localized: """ + The save did not finish: the updated validator of %1$@ rejects the document with _id %2$@, \ + most likely written by another client while the save ran. Fix that document, then save again. + """ + ), + collection, identifier + ) + } + + private static func rejectedByRewrittenValidator( + _ changes: [MongoFieldChange], + originalValidatorJson: String, + rewrittenValidatorJson: String, + onlyValidDocuments: Bool, + replaying replayStages: [String] + ) -> String { + var names: [String] = [] + for name in changes.flatMap(\.names) where !names.contains(name) { + names.append(name) + } + var stages = ["{\"$match\": {\"$or\": [\(names.map { "{\(exists($0))}" }.joined(separator: ", "))]}}"] + if onlyValidDocuments { + stages.append("{\"$match\": \(originalValidatorJson)}") + } + stages.append(contentsOf: replayStages) + stages.append("{\"$match\": {\"$nor\": [\(rewrittenValidatorJson)]}}") + stages.append("{\"$limit\": 1}") + stages.append("{\"$project\": {\"_id\": 1}}") + return "[\(stages.joined(separator: ", "))]" + } + + /// Counts the documents that still hold a change's old name, read once every statement has run. + /// `$count` answers with no document at all when nothing matches. + static func remainderPipeline(_ change: MongoFieldChange) -> String { + "[{\"$match\": {\(exists(change.source))}}, {\"$count\": \"n\"}]" + } + + /// The count `remainderPipeline` returned, or nil when the answer is not a count. + static func remainderCount(in documentJson: String?) -> Int64? { + guard let documentJson else { return 0 } + guard let document = MongoJsonValue.parse(documentJson) as? [String: Any] else { return nil } + if let wrapper = document["n"] as? [String: Any], wrapper.count == 1, + let text = (wrapper["$numberInt"] ?? wrapper["$numberLong"]) as? String { + return Int64(text) + } + return (document["n"] as? NSNumber)?.int64Value + } + + /// The first change whose old name some documents still hold, as the reason the save did not + /// finish. Each statement skips what it has already done, so saving again finishes it, or + /// refuses and says why when a document now holds both names of a rename. + static func shortfall(_ remaining: [(change: MongoFieldChange, documents: Int64)]) -> String? { + guard let first = remaining.first(where: { $0.documents > 0 }) else { return nil } + guard first.documents > 1 else { + return String( + format: String(localized: "The save did not finish: one document still holds %@, most likely written by another client while the save ran. Save again to finish."), + first.change.source + ) + } + return String( + format: String(localized: "The save did not finish: %1$lld documents still hold %2$@, most likely written by another client while the save ran. Save again to finish."), + first.documents, first.change.source + ) + } + + /// The `_id` of the document a pass found, as a person would type it. + static func identifier(in documentJson: String?) -> String? { + guard let documentJson, let idJson = MongoScriptJson.member(of: documentJson, key: "_id") else { return nil } + let value = MongoJsonValue.parse(idJson) + if let text = value as? String { return MongoScriptJson.jsonString(text) } + guard let wrapper = value as? [String: Any], wrapper.count == 1, let key = wrapper.keys.first else { return idJson } + switch key { + case "$oid": return (wrapper[key] as? String).map { "ObjectId(\"\($0)\")" } ?? idJson + case "$numberInt", "$numberLong", "$numberDouble": return wrapper[key] as? String ?? idJson + default: return idJson + } + } + + /// Every expression in one `$addFields` reads the document as it came in, so a rename sets the + /// new name from the old one and removes the old one in a single stage. + private static func apply(_ change: MongoFieldChange) -> String { + let source = MongoScriptJson.jsonString(change.source) + guard let target = change.target else { return "{\"$addFields\": {\(source): \"$$REMOVE\"}}" } + return "{\"$addFields\": {\(MongoScriptJson.jsonString(target)): \(path(change.source)), \(source): \"$$REMOVE\"}}" + } + + /// The step as its filter applies it to every document: a rename moves the value only where the + /// old name is present and the new one absent. Measured on 7.0.43: adding a field from a missing + /// path removes it, so the untouched branch sets each name back to itself. + private static func replay(_ change: MongoFieldChange) -> String { + let source = change.source + guard let target = change.target else { return apply(change) } + let moves = "{\"$and\": [\(present(source)), {\"$eq\": [{\"$type\": \(path(target))}, \"missing\"]}]}" + return "{\"$addFields\": {" + + "\(MongoScriptJson.jsonString(target)): {\"$cond\": [\(moves), \(path(source)), \(path(target))]}, " + + "\(MongoScriptJson.jsonString(source)): {\"$cond\": [\(moves), \"$$REMOVE\", \(path(source))]}" + + "}}" + } + + private static func path(_ field: String) -> String { + MongoScriptJson.jsonString("$" + field) + } + + private static func exists(_ field: String) -> String { + "\(MongoScriptJson.jsonString(field)): {\"$exists\": true}" + } + + private static func present(_ field: String) -> String { + "{\"$ne\": [{\"$type\": \(path(field))}, \"missing\"]}" + } +} diff --git a/Plugins/MongoDBDriverPlugin/MongoFieldDependent.swift b/Plugins/MongoDBDriverPlugin/MongoFieldDependent.swift new file mode 100644 index 0000000000..f02fee1afe --- /dev/null +++ b/Plugins/MongoDBDriverPlugin/MongoFieldDependent.swift @@ -0,0 +1,114 @@ +// +// MongoFieldDependent.swift +// MongoDBDriverPlugin +// + +import Foundation +import TableProPluginKit + +/// Something that reads a field by name, and would be left pointing at a path no document has, or +/// reading values meant for another field, once the field is renamed or removed. +enum MongoFieldDependent: Equatable { + case index(name: String, field: String) + case searchIndex(name: String, field: String) + case view(name: String, field: String) + + /// Both names are checked. The old name stops matching the index the moment the first document + /// moves, and the new name can meet a unique key or a text index's `language_override` that + /// stops the write partway. + static func index(of changes: [MongoFieldChange], among indexes: [MongoIndexSpec]) -> MongoFieldDependent? { + for change in changes { + for name in change.names { + guard let index = indexes.first(where: { $0.reaches(name) }) else { continue } + return .index(name: index.name, field: name) + } + } + return nil + } + + /// Both names, as for any other index. Search indexes build in the background, so nothing stops + /// the write: an old name leaves the definition pointing at a path no document has, and a new + /// name puts values under a mapping written for a different field. + static func searchIndex( + of changes: [MongoFieldChange], + among searchIndexes: [MongoSearchIndex] + ) -> MongoFieldDependent? { + for change in changes { + for name in change.names { + guard let index = searchIndexes.first(where: { $0.mentions(name) }) else { continue } + return .searchIndex(name: index.name, field: name) + } + } + return nil + } + + /// Only the old name counts. A view that reads the new name already expects the field there. + static func view( + of changes: [MongoFieldChange], + among views: [MongoViewDefinition], + collection: String + ) -> MongoFieldDependent? { + let dependents = MongoViewDefinition.dependents(of: collection, among: views) + for change in changes { + guard let view = dependents.first(where: { $0.readsField(change.source) }) else { continue } + return .view(name: view.name, field: change.source) + } + return nil + } + + /// The first of each kind, in the order the save checks them before writing. + static func dependent( + of changes: [MongoFieldChange], + indexes: [MongoIndexSpec], + searchIndexes: [MongoSearchIndex], + views: [MongoViewDefinition], + collection: String + ) -> MongoFieldDependent? { + index(of: changes, among: indexes) + ?? searchIndex(of: changes, among: searchIndexes) + ?? view(of: changes, among: views, collection: collection) + } + + /// Why the save stops before writing, and what to do first. + func refusal(collection: String) -> String { + switch self { + case .index(let name, let field): + let drop = "\(MongoCollectionAccessor.expression(for: collection)).dropIndex(\(MongoScriptJson.jsonString(name)))" + return String( + format: String(localized: "Index %1$@ uses %2$@. Drop it first by running %3$@ in a query tab, then save again."), + name, field, drop + ) + case .searchIndex(let name, let field): + return String( + format: String(localized: "Search index %1$@ uses %2$@. Change or drop that search index first, then save again."), + name, field + ) + case .view(let name, let field): + return String( + format: String(localized: "View %1$@ reads %2$@. Change the view first, then save again."), + name, field + ) + } + } + + /// Said once the statements ran, for one another client made while they did. + func appearedDuringSave(collection: String) -> String { + switch self { + case .index(let name, let field): + return String( + format: String(localized: "The save ran, but index %1$@ on %2$@ was created while it did and uses %3$@. Check that it indexes the field you meant."), + name, collection, field + ) + case .searchIndex(let name, let field): + return String( + format: String(localized: "The save ran, but search index %1$@ on %2$@ was created while it did and uses %3$@. Check that it maps the field you meant."), + name, collection, field + ) + case .view(let name, let field): + return String( + format: String(localized: "The save ran, but view %1$@ was created while it did and reads %2$@. Change it to read the field where it is now."), + name, field + ) + } + } +} diff --git a/Plugins/MongoDBDriverPlugin/MongoFieldReferences.swift b/Plugins/MongoDBDriverPlugin/MongoFieldReferences.swift new file mode 100644 index 0000000000..27a35841c9 --- /dev/null +++ b/Plugins/MongoDBDriverPlugin/MongoFieldReferences.swift @@ -0,0 +1,470 @@ +// +// MongoFieldReferences.swift +// MongoDBDriverPlugin +// + +import Foundation + +/// Whether a server-side definition reads a top-level field, walked from the definition's canonical +/// Extended JSON. +/// +/// A validator's query is read by its grammar: it names a field by the key it matches on and never +/// by a value, so `{status: {$in: ["active"]}}` names `status` and not `active`, and an expression +/// names one with a `$`-prefixed string. A view's pipeline and a search index definition are read +/// the other way round, because their grammar keeps growing: `$densify` and `$fill` take a list of +/// plain field names, Atlas Search takes a `path` that can be a list, and a stage added next year +/// can name a field any way it likes. Any string in either that could be the field counts. Where a +/// construct cannot be read statically (`$where`, `$function`, a variable bound to a document, a +/// field name `$getField` computes), every walker answers that it does read the field, because the +/// cost of a wrong yes is a refused save and the cost of a wrong no is a definition left pointing +/// at a field that is gone. An expression handed the whole document, as `$$ROOT` and `$$CURRENT` +/// hand it to `$objectToArray`, can reach any field by a name it builds at run time, so it reads +/// every field too; `MongoWholeDocumentReads` decides that for validators and views alike. +enum MongoFieldPath { + /// Whether a dotted path is the field or a path under it. + static func reaches(_ path: String, field: String) -> Bool { + path == field || path.hasPrefix(field + ".") + } + + /// Whether a string could name the field: bare or behind `$` or `$$`, and at any depth of a + /// dotted path, because an earlier stage can put the whole document under another name, as + /// `$lookup` does with `as` and `$push: "$$ROOT"` does, and `joined.status` then reads `status`. + static func mayBeNamed(by text: String, field: String) -> Bool { + text.drop { $0 == "$" }.split(separator: ".", omittingEmptySubsequences: false).contains { $0 == field } + } +} + +enum MongoJsonValue { + static func parse(_ text: String) -> Any? { + guard let data = text.data(using: .utf8) else { return nil } + return try? JSONSerialization.jsonObject(with: data, options: [.fragmentsAllowed]) + } + + /// The keys canonical Extended JSON wraps one BSON value in. Such an object is a value, never + /// a document with fields. + static let wrapperKeys: Set = [ + "$oid", "$numberInt", "$numberLong", "$numberDouble", "$numberDecimal", "$date", + "$regularExpression", "$binary", "$timestamp", "$minKey", "$maxKey", "$code", "$symbol", + "$dbPointer", "$undefined", "$uuid" + ] + + static func isWrapper(_ object: [String: Any]) -> Bool { + guard let first = object.keys.first else { return false } + if object.count == 1 { return wrapperKeys.contains(first) } + return object.count == 2 && object["$code"] != nil && object["$scope"] != nil + } + + static func string(_ value: Any?) -> String? { + if let text = value as? String { return text } + guard let object = value as? [String: Any], object.count == 1 else { return nil } + return object["$literal"] as? String + } +} + +enum MongoExpressionFieldReferences { + private static let opaqueOperators: Set = ["$function", "$accumulator", "$where"] + private static let fieldOperators: Set = ["$getField", "$setField", "$unsetField"] + + static func reaches(_ expression: Any, field: String) -> Bool { + MongoWholeDocumentReads.inExpression(expression) || names(expression, field: field) + } + + private static func names(_ expression: Any, field: String) -> Bool { + if let text = expression as? String { + return stringReaches(text, field: field) + } + if let list = expression as? [Any] { + return list.contains { names($0, field: field) } + } + guard let object = expression as? [String: Any], !MongoJsonValue.isWrapper(object) else { return false } + for (key, value) in object { + if key == "$literal" { continue } + if opaqueOperators.contains(key) { return true } + if fieldOperators.contains(key), fieldOperatorReaches(value, field: field) { return true } + if names(value, field: field) { return true } + } + return false + } + + /// `"$a.b"` reads `a`. `"$$ROOT.a"` and `"$$CURRENT.a"` read it too, and so may any other + /// variable followed by a path, because `$let` and `$lookup` can bind a variable to the whole + /// document. A bare variable names no field: `"$$ROOT"` and `"$$CURRENT"` alone are the whole + /// document, which `MongoWholeDocumentReads` answers for, and wherever another variable is bound + /// to the document, the binding names `$$ROOT` itself. + static func stringReaches(_ text: String, field: String) -> Bool { + guard text.hasPrefix("$") else { return false } + guard text.hasPrefix("$$") else { + return MongoFieldPath.reaches(String(text.dropFirst()), field: field) + } + let variablePath = text.dropFirst(2) + guard let dot = variablePath.firstIndex(of: ".") else { return false } + return MongoFieldPath.reaches(String(variablePath[variablePath.index(after: dot)...]), field: field) + } + + /// `$getField: "a"`, and `$getField: {field: "a", input: ...}`, name `a` as a literal. From + /// MongoDB 7.2 the field argument can be any expression, and one that is not a literal, such as + /// `{$concat: ["sta", "tus"]}` or `"$name"`, names whatever it evaluates to, so it reaches every + /// field. + static func fieldOperatorReaches(_ value: Any, field: String) -> Bool { + guard let name = literalFieldName(of: value) else { return true } + return name == field + } + + /// The field an operator's literal field argument names, or nil when it computes one. + static func literalFieldName(of value: Any) -> String? { + literalFieldName(fieldArgument(of: value)) + } + + /// The operator's own value in its short form, and its `field` member in the long one. + private static func fieldArgument(of value: Any) -> Any? { + guard let spec = value as? [String: Any], spec["$literal"] == nil else { return value } + return spec["field"] + } + + /// A string starting with `$` is a field path or a variable, which evaluates to a name. + private static func literalFieldName(_ argument: Any?) -> String? { + if let text = argument as? String { return text.hasPrefix("$") ? nil : text } + guard let object = argument as? [String: Any], object.count == 1 else { return nil } + return object["$literal"] as? String + } +} + +/// Whether a validator or a view hands the whole document, `$$ROOT` or `$$CURRENT`, to something +/// that reads its field names, which reads every field. One rule for both, so a validator and a view +/// holding the same expression answer the same. +/// +/// `{$objectToArray: "$$ROOT"}` turns every name into a value a literal can match, a comparison of +/// the whole document depends on every name in it, and a document placed under a name, in an array +/// or in a variable can be taken apart by a later stage under that name. The document is only passed +/// on where it stays the document: `$replaceRoot` and `$replaceWith` of it, directly or through +/// `$mergeObjects`, a `$setField` or `$unsetField` with a literal field name, or a branch of `$cond`, +/// `$switch`, `$ifNull` or `$let`. There a later stage can reach it only as `$$ROOT` again, which this +/// rule reads where it stands. A `$getField` with a literal field name reads that one field, which the +/// walkers name as they name any other. +enum MongoWholeDocumentReads { + private enum Use { + case readNames + case passedOn + } + + private static let wholeDocument: Set = ["$$ROOT", "$$CURRENT"] + private static let passingOperators: Set = ["$mergeObjects", "$ifNull"] + private static let documentFieldOperators: Set = ["$setField", "$unsetField"] + + /// A validator's `$expr`, whose value the server reads as a whole. + static func inExpression(_ expression: Any) -> Bool { + reads(expression, as: .readNames) + } + + /// A view's pipeline and every pipeline its stages nest. + static func inPipeline(_ pipeline: Any) -> Bool { + guard let stages = pipeline as? [Any] else { return reads(pipeline, as: .readNames) } + return stages.contains(where: stageReads) + } + + private static func stageReads(_ stage: Any) -> Bool { + guard let object = stage as? [String: Any] else { return reads(stage, as: .readNames) } + return object.contains { name, spec in + switch name { + case "$replaceRoot": + guard let newRoot = (spec as? [String: Any])?["newRoot"] else { return reads(spec, as: .readNames) } + return reads(newRoot, as: .passedOn) + case "$replaceWith": + return reads(spec, as: .passedOn) + case "$facet": + guard let facets = spec as? [String: Any] else { return reads(spec, as: .readNames) } + return facets.values.contains(where: inPipeline) + case "$lookup", "$unionWith": + guard let options = spec as? [String: Any] else { return reads(spec, as: .readNames) } + return options.contains { key, member in + key == "pipeline" ? inPipeline(member) : reads(member, as: .readNames) + } + default: + return reads(spec, as: .readNames) + } + } + } + + private static func reads(_ value: Any, as use: Use) -> Bool { + if let text = value as? String { + return use == .readNames && wholeDocument.contains(text) + } + if let list = value as? [Any] { + return list.contains { reads($0, as: .readNames) } + } + guard let object = value as? [String: Any], !MongoJsonValue.isWrapper(object) else { return false } + guard object.count == 1, let (key, member) = object.first, key.hasPrefix("$") else { + return object.values.contains { reads($0, as: .readNames) } + } + return operatorReads(key, member, as: use) + } + + private static func operatorReads(_ name: String, _ member: Any, as use: Use) -> Bool { + if name == "$literal" { return false } + if passingOperators.contains(name) { + return arguments(of: member).contains { reads($0, as: use) } + } + if name == "$getField", let spec = member as? [String: Any], + MongoExpressionFieldReferences.literalFieldName(of: member) != nil { + return spec["input"].map { reads($0, as: .passedOn) } ?? false + } + if documentFieldOperators.contains(name), let spec = member as? [String: Any], + MongoExpressionFieldReferences.literalFieldName(of: member) != nil { + return spec.contains { key, argument in reads(argument, as: key == "input" ? use : .readNames) } + } + switch name { + case "$cond": + return branchesRead(member, conditions: ["if"], results: ["then", "else"], positional: [0], as: use) + case "$let": + return branchesRead(member, conditions: ["vars"], results: ["in"], positional: [], as: use) + case "$switch": + guard let spec = member as? [String: Any] else { return reads(member, as: .readNames) } + let branches = (spec["branches"] as? [Any] ?? []).contains { branch in + branchesRead(branch, conditions: ["case"], results: ["then"], positional: [], as: use) + } + return branches || spec["default"].map { reads($0, as: use) } ?? false + default: + return reads(member, as: .readNames) + } + } + + /// A conditional reads its conditions and passes on whichever result it picks. Written as an + /// array, `$cond` holds its condition first. + private static func branchesRead( + _ member: Any, + conditions: Set, + results: Set, + positional: Set, + as use: Use + ) -> Bool { + if let list = member as? [Any] { + return list.enumerated().contains { index, argument in + reads(argument, as: positional.contains(index) ? .readNames : use) + } + } + guard let spec = member as? [String: Any] else { return reads(member, as: .readNames) } + return spec.contains { key, argument in + reads(argument, as: results.contains(key) && !conditions.contains(key) ? use : .readNames) + } + } + + private static func arguments(of member: Any) -> [Any] { + member as? [Any] ?? [member] + } +} + +enum MongoQueryFieldReferences { + private static let logicalOperators: Set = ["$and", "$or", "$nor"] + private static let fieldlessOperators: Set = [ + "$comment", "$text", "$alwaysTrue", "$alwaysFalse", "$sampleRate" + ] + + /// Whether a `$jsonSchema` anywhere in the query applies a rule to this name without declaring + /// it: a `patternProperties` pattern, or `additionalProperties`. See + /// `MongoJsonSchemaFieldReferences.appliesByName(_:to:)`. + static func appliesByName(_ query: Any, to name: String) -> Bool { + guard let object = query as? [String: Any] else { return false } + for (key, value) in object { + if logicalOperators.contains(key), let list = value as? [Any], + list.contains(where: { appliesByName($0, to: name) }) { + return true + } + if key == "$jsonSchema", MongoJsonSchemaFieldReferences.appliesByName(value, to: name) { return true } + } + return false + } + + /// A query names a field by the key it matches on. What sits under that key is relative to the + /// field (`$elemMatch` reads the field's own elements), so it is never read for another name. + static func reaches(_ query: Any, field: String) -> Bool { + guard let object = query as? [String: Any] else { return false } + for (key, value) in object { + if logicalOperators.contains(key) { + if let list = value as? [Any], list.contains(where: { reaches($0, field: field) }) { return true } + continue + } + switch key { + case "$expr": + if MongoExpressionFieldReferences.reaches(value, field: field) { return true } + case "$jsonSchema": + if MongoJsonSchemaFieldReferences.reaches(value, field: field) { return true } + default: + if fieldlessOperators.contains(key) { continue } + if key.hasPrefix("$") || MongoFieldPath.reaches(key, field: field) { return true } + } + } + return false + } +} + +enum MongoJsonSchemaFieldReferences { + private static let combinators = ["allOf", "anyOf", "oneOf"] + + /// A `$jsonSchema` names a document's fields at its own level: `properties`, `required`, + /// `dependencies`, a `patternProperties` pattern that matches the field, a document listed in + /// `enum`, and every schema in `allOf`, `anyOf`, `oneOf` and `not`, which apply to the same + /// document. What a property's own schema says is about that field's value, and `description`, + /// `title` and a scalar `enum` entry are values. + static func reaches(_ schema: Any, field: String) -> Bool { + guard let object = schema as? [String: Any] else { return false } + if let properties = object["properties"] as? [String: Any], properties[field] != nil { return true } + if enumListsDocumentsNaming(object, field) { return true } + if let required = object["required"] as? [Any], required.contains(where: { ($0 as? String) == field }) { + return true + } + if let dependencies = object["dependencies"] as? [String: Any], dependenciesReach(dependencies, field: field) { + return true + } + if let patterns = object["patternProperties"] as? [String: Any], + patterns.keys.contains(where: { pattern($0, matches: field) }) { + return true + } + for key in combinators { + if let list = object[key] as? [Any], list.contains(where: { reaches($0, field: field) }) { return true } + } + if let negated = object["not"], reaches(negated, field: field) { return true } + return false + } + + /// Whether the schema applies a rule to this name by pattern, or through `additionalProperties` + /// because it neither declares the name nor matches it by pattern, at the document's level or in + /// a combinator. Such a rule follows the name rather than the field, so a rename moves the field + /// out from under it or under one it was never checked against, and neither can be carried + /// along. `additionalProperties: true` and an empty schema check nothing. + static func appliesByName(_ schema: Any, to name: String) -> Bool { + guard let object = schema as? [String: Any] else { return false } + if let patterns = object["patternProperties"] as? [String: Any], + patterns.keys.contains(where: { pattern($0, matches: name) }) { + return true + } + let declared = (object["properties"] as? [String: Any])?[name] != nil + if !declared, let additional = object["additionalProperties"], constrains(additional) { return true } + if enumListsDocumentsNaming(object, name) { return true } + for key in combinators { + if let list = object[key] as? [Any], list.contains(where: { appliesByName($0, to: name) }) { return true } + } + if let negated = object["not"], appliesByName(negated, to: name) { return true } + return false + } + + /// An `enum` at the document's own level lists whole documents, and a document matches one only + /// while it holds exactly that entry's names, so an entry naming the field ties the rule to the + /// name. The rewrite cannot carry it along: a `$rename` moves the field to the end of the + /// document, and a document is equal to an entry only with its fields in the entry's order. + private static func enumListsDocumentsNaming(_ schema: [String: Any], _ name: String) -> Bool { + guard let entries = schema["enum"] as? [Any] else { return false } + return entries.contains { entry in + guard let document = entry as? [String: Any], !MongoJsonValue.isWrapper(document) else { return false } + return document[name] != nil + } + } + + private static func constrains(_ additionalProperties: Any) -> Bool { + if let allowed = additionalProperties as? Bool { return !allowed } + guard let schema = additionalProperties as? [String: Any] else { return true } + return !schema.isEmpty + } + + private static func dependenciesReach(_ dependencies: [String: Any], field: String) -> Bool { + if dependencies[field] != nil { return true } + return dependencies.values.contains { value in + if let names = value as? [Any] { return names.contains { ($0 as? String) == field } } + return reaches(value, field: field) + } + } + + /// A pattern that does not compile counts as a match, because nothing can say it does not. + static func pattern(_ pattern: String, matches field: String) -> Bool { + guard let expression = try? NSRegularExpression(pattern: pattern) else { return true } + let range = NSRange(field.startIndex..., in: field) + return expression.firstMatch(in: field, range: range) != nil + } +} + +enum MongoIndexFieldReferences { + private static let projectionKeys = ["wildcardProjection", "columnstoreProjection"] + + /// An index reads a field through its key, a text index's `weights` and `language_override`, + /// its `partialFilterExpression`, and a wildcard or columnstore projection. A text index reports + /// `language_override: "language"` even when it was never set, and a document that gains a + /// `language` field with a value the text index does not know stops the write that gave it one. + static func reaches(_ index: [String: Any], field: String) -> Bool { + if let key = index["key"] as? [String: Any], key.keys.contains(where: { MongoFieldPath.reaches($0, field: field) }) { + return true + } + if let weights = index["weights"] as? [String: Any], + weights.keys.contains(where: { MongoFieldPath.reaches($0, field: field) }) { + return true + } + if let override = index["language_override"] as? String, override == field { return true } + if let filter = index["partialFilterExpression"], MongoQueryFieldReferences.reaches(filter, field: field) { + return true + } + return projectionKeys.contains { key in + guard let projection = index[key] as? [String: Any] else { return false } + return projection.keys.contains { MongoFieldPath.reaches($0, field: field) } + } + } +} + +enum MongoPipelineFieldReferences { + private static let opaqueOperators: Set = ["$function", "$accumulator", "$where"] + private static let fieldOperators: Set = ["$getField", "$setField", "$unsetField"] + + /// A view's pipeline reads a field wherever a string in it could name the field, in any stage, + /// any position and any list, `$literal` included, and wherever a key that is not an operator + /// does. Output names count too: telling them from inputs means knowing every stage's grammar, + /// and a stale output name is as wrong as a stale input. The operator-aware rules only add to + /// that: a code body, and a field name computed at run time. A value BSON keeps as a number, a + /// date or an id is not a string and names nothing. + /// Whether a view's pipeline reads the field: by a name that could be the field, or by handing + /// the whole document to something that reads every name in it. + static func pipelineReads(_ pipeline: [Any], field: String) -> Bool { + MongoWholeDocumentReads.inPipeline(pipeline) || reaches(pipeline, field: field) + } + + static func reaches(_ value: Any, field: String) -> Bool { + if let text = value as? String { + return MongoFieldPath.mayBeNamed(by: text, field: field) + } + if let list = value as? [Any] { + return list.contains { reaches($0, field: field) } + } + guard let object = value as? [String: Any] else { return false } + if MongoJsonValue.isWrapper(object) { return wrapperReaches(object, field: field) } + for (key, member) in object { + if !key.hasPrefix("$"), MongoFieldPath.mayBeNamed(by: key, field: field) { return true } + if opaqueOperators.contains(key) { return true } + if fieldOperators.contains(key), MongoExpressionFieldReferences.fieldOperatorReaches(member, field: field) { + return true + } + if reaches(member, field: field) { return true } + } + return false + } + + /// A symbol is a string under another type, and code can read any field. + private static func wrapperReaches(_ wrapper: [String: Any], field: String) -> Bool { + if wrapper["$code"] != nil { return true } + guard let symbol = wrapper["$symbol"] as? String else { return false } + return MongoFieldPath.mayBeNamed(by: symbol, field: field) + } +} + +/// Whether an Atlas Search or Vector Search index definition names a field. A definition names one +/// as a key under `mappings.fields`, as a `path` string or list, and in `storedSource`, and every +/// key and every string in it counts, for the same reason a view's do. +enum MongoSearchDefinitionFieldReferences { + static func reaches(_ value: Any, field: String) -> Bool { + if let text = value as? String { + return MongoFieldPath.mayBeNamed(by: text, field: field) + } + if let list = value as? [Any] { + return list.contains { reaches($0, field: field) } + } + guard let object = value as? [String: Any], !MongoJsonValue.isWrapper(object) else { return false } + return object.contains { key, member in + MongoFieldPath.mayBeNamed(by: key, field: field) || reaches(member, field: field) + } + } +} diff --git a/Plugins/MongoDBDriverPlugin/MongoScriptCommandBuilder.swift b/Plugins/MongoDBDriverPlugin/MongoScriptCommandBuilder.swift index ab29d13618..9c21650c3b 100644 --- a/Plugins/MongoDBDriverPlugin/MongoScriptCommandBuilder.swift +++ b/Plugins/MongoDBDriverPlugin/MongoScriptCommandBuilder.swift @@ -31,9 +31,12 @@ enum MongoScriptCommandBuilder { "\"upsert\": \(options["upsert"] as? Bool ?? false)" ] appendPassThrough(&fields, options: options, keys: ["arrayFilters", "hint", "collation"]) - return """ - {"update": \(MongoScriptJson.jsonString(collection)), "updates": [{\(fields.joined(separator: ", "))}]} - """ + var command = [ + "\"update\": \(MongoScriptJson.jsonString(collection))", + "\"updates\": [{\(fields.joined(separator: ", "))}]" + ] + appendPassThrough(&command, options: options, keys: ["writeConcern"]) + return "{\(command.joined(separator: ", "))}" } static func delete(collection: String, filter: String, multi: Bool, options: [String: Any]) -> String { diff --git a/Plugins/MongoDBDriverPlugin/MongoScriptHost.swift b/Plugins/MongoDBDriverPlugin/MongoScriptHost.swift index 637f5bc8c1..070f91015f 100644 --- a/Plugins/MongoDBDriverPlugin/MongoScriptHost.swift +++ b/Plugins/MongoDBDriverPlugin/MongoScriptHost.swift @@ -355,7 +355,11 @@ final class MongoScriptHost { } private func runCommand(_ request: [String: Any]) throws -> String { - try command(MongoScriptJson.rawJson(request["command"]) ?? "{}", request) + let reply = try command(MongoScriptJson.rawJson(request["command"]) ?? "{}", request) + if let failure = MongoWriteFailure.concernFailure(fromReply: reply) { + throw MongoDBError(code: failure.code, message: failure.message) + } + return reply } private func command(_ document: String, _ request: [String: Any]) throws -> String { diff --git a/Plugins/MongoDBDriverPlugin/MongoWriteConcern.swift b/Plugins/MongoDBDriverPlugin/MongoWriteConcern.swift new file mode 100644 index 0000000000..26cfe316b2 --- /dev/null +++ b/Plugins/MongoDBDriverPlugin/MongoWriteConcern.swift @@ -0,0 +1,53 @@ +// +// MongoWriteConcern.swift +// MongoDBDriverPlugin +// + +import Foundation + +/// The write concern a connection's URI sets: the Write Concern setting as `w`, and `journal` and +/// `wtimeoutMS` from an imported connection string. +/// +/// A command sent through `mongoc_client_command_simple` takes no write concern from the client, so +/// the statements a Structure save writes name it themselves, where SQL Preview shows it. +struct MongoWriteConcern: Equatable, Sendable { + enum Acknowledgement: Equatable, Sendable { + case members(Int32) + case majority + case tag(String) + } + + let acknowledgement: Acknowledgement? + let journal: Bool? + let timeoutMS: Int64? + + static let serverDefault = MongoWriteConcern(acknowledgement: nil, journal: nil, timeoutMS: nil) + + /// The `writeConcern` document, or nil when the URI sets none and the server's own default + /// applies. + /// + /// A save has to hear the server's answer to know whether it finished, and with `w: 0` and no + /// `j: true` the server answers `n: 0` whatever the write changed and reports no error, measured + /// on MongoDB 7.0.43. Such a concern is raised to `w: 1`, keeping the rest of it. + var schemaChangeJson: String? { + var members: [String] = [] + switch acknowledgement { + case .members(let count): + let acknowledged = count > 0 || journal == true + members.append("\"w\": \(acknowledged ? count : 1)") + case .majority: + members.append("\"w\": \"majority\"") + case .tag(let tag): + members.append("\"w\": \(MongoScriptJson.jsonString(tag))") + case nil: + break + } + if let journal { + members.append("\"j\": \(journal)") + } + if let timeoutMS, timeoutMS > 0 { + members.append("\"wtimeout\": \(timeoutMS)") + } + return members.isEmpty ? nil : "{\(members.joined(separator: ", "))}" + } +} diff --git a/Plugins/MongoDBDriverPlugin/MongoWriteFailure.swift b/Plugins/MongoDBDriverPlugin/MongoWriteFailure.swift index 45f3f3c8ef..fa86e844e9 100644 --- a/Plugins/MongoDBDriverPlugin/MongoWriteFailure.swift +++ b/Plugins/MongoDBDriverPlugin/MongoWriteFailure.swift @@ -22,14 +22,23 @@ struct MongoWriteFailure: Equatable, Sendable { if let writeErrors = reply["writeErrors"] as? [[String: Any]], let first = writeErrors.first { return entry(first) } - if let concernError = reply["writeConcernError"] as? [String: Any] { - let failure = entry(concernError) - return MongoWriteFailure( - code: failure.code, - message: MongoScriptText.writeNotAcknowledged(reason: failure.message) - ) + return concernFailure(in: reply) + } + + /// A reply that did what it was asked without the write concern it was given. mongosh's driver + /// throws on one from any command, `collMod` included, so the shell's `db.runCommand` does too. + static func concernFailure(fromReply replyJson: String) -> MongoWriteFailure? { + guard let data = replyJson.data(using: .utf8), + let reply = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { + return nil } - return nil + return concernFailure(in: reply) + } + + private static func concernFailure(in reply: [String: Any]) -> MongoWriteFailure? { + guard let concernError = reply["writeConcernError"] as? [String: Any] else { return nil } + let failure = entry(concernError) + return MongoWriteFailure(code: failure.code, message: MongoScriptText.writeNotAcknowledged(reason: failure.message)) } private static func entry(_ entry: [String: Any]) -> MongoWriteFailure { diff --git a/Plugins/TableProPluginKit/PluginDatabaseDriver.swift b/Plugins/TableProPluginKit/PluginDatabaseDriver.swift index 109867ce7f..6e0050ad54 100644 --- a/Plugins/TableProPluginKit/PluginDatabaseDriver.swift +++ b/Plugins/TableProPluginKit/PluginDatabaseDriver.swift @@ -383,6 +383,62 @@ public protocol PluginDatabaseDriver: AnyObject, Sendable { var unsupportedIndexTypes: Set { get } func schemaOperationRefusal(_ operation: PluginSchemaOperation) -> String? + /// Answers for a Structure save as a whole once the driver has read what it depends on: a + /// refusal, or statements that have to run ahead of the save's own. + /// + /// Asked by the Structure tab when it composes a save, for SQL Preview and for Save alike, after + /// every operation has passed `schemaOperationRefusal(_:)` and on the connection the save is + /// composed on. A table rebuild and a Compare & Sync script do not ask it. `operations` holds + /// every change of the save that a `PluginSchemaOperation` can express, in the order their + /// statements run; foreign key, primary key and check constraint changes other than a rename + /// have no case and are left out. Throws when the driver cannot read what it has to check, + /// which stops the save. + func reviewSchemaChange( + table: String, + schema: String?, + operations: [PluginSchemaOperation] + ) async throws -> PluginSchemaChangeReview + + /// The last question before a Structure save writes anything: why it must not run, or nil. + /// + /// Asked on Save alone, never for SQL Preview, after every confirmation and on the connection + /// that then runs the statements, just before the first of them. `operations` is the list + /// `reviewSchemaChange(table:schema:operations:)` was given, and `review` is what it answered + /// then: its leading statements are about to run as the user confirmed them. A driver that + /// composed them from server state reads that state again and refuses when they would now + /// differ, because running them would undo whatever changed it. This is where a driver reads + /// the data itself, which can cost a scan of the table, so it bounds its own reads and stops + /// them when the task is cancelled. Throws when the driver cannot read what it has to check. + func schemaChangeRefusalBeforeWriting( + table: String, + schema: String?, + operations: [PluginSchemaOperation], + review: PluginSchemaChangeReview + ) async throws -> String? + + /// Why a Structure save whose statements all succeeded did not finish, worded for the user, or + /// nil when it did. + /// + /// Asked on Save once the last statement has run, on the same connection. A statement that + /// changes many rows can succeed and still miss one another client wrote while it ran, so a + /// driver whose statements can leave such a row reads for it here. `operations` and `review` + /// are what the save was composed with. The app reports the save as failed with its statements + /// already run, keeps the edits staged so Save can run them again, and reloads the table's rows. + /// Throws when the driver cannot read what it checks, which the app reports the same way. + func schemaChangeShortfallAfterWriting( + table: String, + schema: String?, + operations: [PluginSchemaOperation], + review: PluginSchemaChangeReview + ) async throws -> String? + + /// Told on the session's own connection that the app changed a table's definition on another + /// one: a Structure save, a table rebuild or a column reorder, finished or stopped partway. A + /// driver that keeps what it learned about the table's columns and their types drops it here, + /// for the table in every database it holds it for, so the next read learns them again. Called + /// on the main actor while the driver may be running a query, so it only clears what it keeps. + func tableDefinitionDidChange(table: String, schema: String?) + /// Why the connected server has no check constraints to list or edit, or nil when it has. /// /// The engine's capability flags describe its newest release; this describes the server in @@ -986,6 +1042,35 @@ public extension PluginDatabaseDriver { var unsupportedStructureColumnFields: Set { [] } var unsupportedIndexTypes: Set { [] } func schemaOperationRefusal(_ operation: PluginSchemaOperation) -> String? { nil } + + func reviewSchemaChange( + table: String, + schema: String?, + operations: [PluginSchemaOperation] + ) async throws -> PluginSchemaChangeReview { + PluginSchemaChangeReview() + } + + func schemaChangeRefusalBeforeWriting( + table: String, + schema: String?, + operations: [PluginSchemaOperation], + review: PluginSchemaChangeReview + ) async throws -> String? { + nil + } + + func schemaChangeShortfallAfterWriting( + table: String, + schema: String?, + operations: [PluginSchemaOperation], + review: PluginSchemaChangeReview + ) async throws -> String? { + nil + } + + func tableDefinitionDidChange(table: String, schema: String?) {} + var checkConstraintRefusal: String? { nil } func generateCreateTableSQL(definition: PluginCreateTableDefinition) -> String? { nil } diff --git a/Plugins/TableProPluginKit/PluginSchemaChangeReview.swift b/Plugins/TableProPluginKit/PluginSchemaChangeReview.swift new file mode 100644 index 0000000000..e5c6d09eaf --- /dev/null +++ b/Plugins/TableProPluginKit/PluginSchemaChangeReview.swift @@ -0,0 +1,25 @@ +// +// PluginSchemaChangeReview.swift +// TableProPluginKit +// + +import Foundation + +/// A driver's answer to a whole Structure save, given once it has read what the save depends on. +public struct PluginSchemaChangeReview: Sendable, Equatable { + /// Why the save cannot run as it stands, worded for the user, or nil when it can. + public var refusal: String? + /// Statements that run before the save's own, in order. Shown in SQL Preview and run on Save. + public var leadingStatements: [String] + /// What the driver read to compose this answer, in a form only the driver reads. The app keeps + /// it with the save and hands it back before and after writing, so the driver can refuse a save + /// the server has moved on from and check what the statements did against what they were + /// composed from. + public var basis: String? + + public init(refusal: String? = nil, leadingStatements: [String] = [], basis: String? = nil) { + self.refusal = refusal + self.leadingStatements = leadingStatements + self.basis = basis + } +} diff --git a/Plugins/TableProPluginKit/PluginSchemaOperation.swift b/Plugins/TableProPluginKit/PluginSchemaOperation.swift index 1183e0efc7..b1fce603e4 100644 --- a/Plugins/TableProPluginKit/PluginSchemaOperation.swift +++ b/Plugins/TableProPluginKit/PluginSchemaOperation.swift @@ -12,4 +12,8 @@ public enum PluginSchemaOperation: Sendable { /// An index edited in place, which the app otherwise saves as a drop followed by an add. case modifyIndex(old: PluginIndexDefinition, new: PluginIndexDefinition) case dropIndex(PluginIndexDefinition) + /// A column changed in place. For a document store, a rename carried out on every document + /// that holds the field. + case modifyColumn(old: PluginColumnDefinition, new: PluginColumnDefinition) + case dropColumn(PluginColumnDefinition) } diff --git a/TablePro/Core/Compare/CompareSyncEngineFamily.swift b/TablePro/Core/Compare/CompareSyncEngineFamily.swift index 64208392bc..0cf57bd72f 100644 --- a/TablePro/Core/Compare/CompareSyncEngineFamily.swift +++ b/TablePro/Core/Compare/CompareSyncEngineFamily.swift @@ -11,7 +11,11 @@ import Foundation internal enum CompareSyncEngineFamily { + /// A sampled column list is not a schema. A field the source's sample missed reads as one the + /// target holds and the source does not, and a script would remove it from every document of + /// the target. internal static func canGenerateStructureScript(from source: DatabaseType, to target: DatabaseType) -> Bool { + guard !source.columnsAreSampled, !target.columnsAreSampled else { return false } guard source != target else { return true } return sameFamily(source, target) } @@ -30,7 +34,13 @@ internal enum CompareSyncEngineFamily { }() internal static func structureScriptRefusal(from source: DatabaseType, to target: DatabaseType) -> String { - String( + if let sampled = [source, target].first(where: \.columnsAreSampled) { + return String( + format: String(localized: "%@ lists a collection's fields from a sample of its documents, so structures can be compared but no script is generated."), + sampled.rawValue + ) + } + return String( format: String(localized: "Structure sync needs matching database types. %@ and %@ can be compared, but no script is generated."), source.rawValue, target.rawValue ) diff --git a/TablePro/Core/Database/DatabaseDriver.swift b/TablePro/Core/Database/DatabaseDriver.swift index 7b24ba7927..4d47f5f078 100644 --- a/TablePro/Core/Database/DatabaseDriver.swift +++ b/TablePro/Core/Database/DatabaseDriver.swift @@ -168,6 +168,30 @@ protocol DatabaseDriver: AnyObject, Sendable { /// Why the connected server has no check constraints to list or edit, or nil when it has. var checkConstraintRefusal: String? { get } + /// The save-level questions of a Structure save. See `PluginDatabaseDriver` for each. The + /// defaults approve every save, find every save finished and keep nothing to forget. + func reviewSchemaChange( + table: String, + schema: String?, + operations: [PluginSchemaOperation] + ) async throws -> PluginSchemaChangeReview + + func schemaChangeRefusalBeforeWriting( + table: String, + schema: String?, + operations: [PluginSchemaOperation], + review: PluginSchemaChangeReview + ) async throws -> String? + + func schemaChangeShortfallAfterWriting( + table: String, + schema: String?, + operations: [PluginSchemaOperation], + review: PluginSchemaChangeReview + ) async throws -> String? + + func tableDefinitionDidChange(table: String, schema: String?) + /// Fetch foreign keys for all tables in the current database/schema in bulk. /// Default implementation falls back to per-table fetchForeignKeys. func fetchAllForeignKeys() async throws -> [String: [ForeignKeyInfo]] @@ -515,6 +539,34 @@ extension DatabaseDriver { var unsupportedIndexTypes: Set { [] } var checkConstraintRefusal: String? { nil } + func reviewSchemaChange( + table: String, + schema: String?, + operations: [PluginSchemaOperation] + ) async throws -> PluginSchemaChangeReview { + PluginSchemaChangeReview() + } + + func schemaChangeRefusalBeforeWriting( + table: String, + schema: String?, + operations: [PluginSchemaOperation], + review: PluginSchemaChangeReview + ) async throws -> String? { + nil + } + + func schemaChangeShortfallAfterWriting( + table: String, + schema: String?, + operations: [PluginSchemaOperation], + review: PluginSchemaChangeReview + ) async throws -> String? { + nil + } + + func tableDefinitionDidChange(table: String, schema: String?) {} + func ping() async throws { _ = try await execute(query: "SELECT 1") } diff --git a/TablePro/Core/Database/DatabaseManager+Schema.swift b/TablePro/Core/Database/DatabaseManager+Schema.swift index 9c50ad1177..af261a4bac 100644 --- a/TablePro/Core/Database/DatabaseManager+Schema.swift +++ b/TablePro/Core/Database/DatabaseManager+Schema.swift @@ -21,17 +21,27 @@ extension DatabaseManager { /// Authorization sits outside the scoped block: it awaits a confirmation sheet and Touch ID, /// and holding the connection's driver gate across a human prompt would freeze every other /// tab on that connection. + /// + /// The driver is asked `schemaChangeRefusalBeforeWriting` on the same connection, after the + /// user has confirmed and before the first statement, which is the only point a check that + /// reads the data belongs: SQL Preview composes the same script and must not pay for it. It is + /// asked `schemaChangeShortfallAfterWriting` after the last one, because a statement that + /// succeeds over many rows can still miss one another client wrote while it ran. + /// + /// A save that fails once its first statement has started reports the table changed, as a + /// successful one does. MongoDB keeps every document an `updateMany` changed before it stopped, + /// and MySQL, MariaDB and Oracle commit each DDL statement as it runs, so the rows on screen + /// can describe a table that has moved on. func executeSchemaChanges( - _ statements: [SchemaStatement], + _ script: SchemaChangeScript, databaseType: DatabaseType, scope: DatabaseScope ) async throws { let route = schemaChangeRoute(for: scope) + let statements = script.statements let combinedSQL = statements.map(\.sql).joined(separator: "\n") - let schemaKind: OperationKind = - QueryClassifier.classifyTier(combinedSQL, databaseType: databaseType) == .destructive - ? .destructiveQuery : .schemaMutation + let schemaKind = Self.schemaOperationKind(for: statements, combinedSQL: combinedSQL, databaseType: databaseType) let authorization = await ExecutionGateProvider.shared.authorize( OperationRequest( connectionId: scope.connectionId, @@ -56,12 +66,13 @@ extension DatabaseManager { route: route, cancellation: .protectedWrite ) { driver in + try await Self.refuseBeforeWriting(script, scope: scope, on: driver) let useTransaction = driver.supportsTransactions if useTransaction { try await driver.beginTransaction(mode: schemaKind.declaresWrite ? .readWrite : .serverDefault) } + var measured: [TimeInterval] = [] do { - var measured: [TimeInterval] = [] for stmt in statements { let startedAt = Date() _ = try await driver.execute(query: stmt.sql) @@ -70,7 +81,6 @@ extension DatabaseManager { if useTransaction { try await driver.commitTransaction() } - return measured } catch { if useTransaction { do { @@ -79,9 +89,17 @@ extension DatabaseManager { Self.logger.error("Rollback failed after schema change error: \(error.localizedDescription)") } } - throw DatabaseError.queryFailed("Schema change failed: \(error.localizedDescription)") + throw SchemaChangeFailedAfterWriting(message: "Schema change failed: \(error.localizedDescription)") } + try await Self.confirmFinished(script, scope: scope, on: driver) + return measured } + } catch let refusal as SchemaOperationRefusedError { + throw refusal + } catch let failure as SchemaChangeFailedAfterWriting { + Self.reportCatalogChangeAfterFailure(in: scope) + reportTableDefinitionChange(table: script.tableName, in: scope) + throw DatabaseError.queryFailed(failure.message) } catch { Self.reportCatalogChangeAfterFailure(in: scope) throw error @@ -104,12 +122,73 @@ extension DatabaseManager { ) } - AppCommands.shared.refreshData.send(DataRefreshRequest(connectionId: scope.connectionId, scope: scope)) + reportTableDefinitionChange(table: script.tableName, in: scope) CatalogChangeService.post( .changed(CatalogChange(connectionId: scope.connectionId, database: scope.database, kinds: .tables)) ) } + /// Tells everything that remembers the table's definition that it changed: the session's driver, + /// which may keep what it learned about the columns, and every window, whose tabs on the table + /// reload or are marked to reload their rows and structure. Addressed by the table, so a tab on + /// another table in the same database is left alone. + func reportTableDefinitionChange(table: String, in scope: DatabaseScope) { + activeSessions[scope.connectionId]?.driver?.tableDefinitionDidChange(table: table, schema: scope.schema) + AppCommands.shared.objectChanged.send( + DatabaseObjectChange(connectionId: scope.connectionId, scope: scope, name: table, kind: .structure) + ) + } + + /// Destructive when the statements' text says so or when the change a statement came from does. + /// A removed MongoDB field is an `updateMany` with `$unset`, which the text classifier tiers as a + /// plain write, so only the statement's own flag puts it behind the Safe Mode level that confirms + /// a dropped column. + nonisolated static func schemaOperationKind( + for statements: [SchemaStatement], + combinedSQL: String, + databaseType: DatabaseType + ) -> OperationKind { + let destructiveText = QueryClassifier.classifyTier(combinedSQL, databaseType: databaseType) == .destructive + return destructiveText || statements.contains(where: \.isDestructive) ? .destructiveQuery : .schemaMutation + } + + nonisolated private static func refuseBeforeWriting( + _ script: SchemaChangeScript, + scope: DatabaseScope, + on driver: DatabaseDriver + ) async throws { + let refusal = try await driver.schemaChangeRefusalBeforeWriting( + table: script.tableName, + schema: scope.schema, + operations: script.operations, + review: script.review + ) + if let refusal { + throw SchemaOperationRefusedError(reason: refusal) + } + } + + nonisolated private static func confirmFinished( + _ script: SchemaChangeScript, + scope: DatabaseScope, + on driver: DatabaseDriver + ) async throws { + let shortfall: String? + do { + shortfall = try await driver.schemaChangeShortfallAfterWriting( + table: script.tableName, + schema: scope.schema, + operations: script.operations, + review: script.review + ) + } catch { + throw SchemaChangeFailedAfterWriting(message: error.localizedDescription) + } + if let shortfall { + throw SchemaChangeFailedAfterWriting(message: shortfall) + } + } + /// Run a Create Table draft's statements, on the same isolated route and in the same shape as /// `executeSchemaChanges`. /// @@ -209,3 +288,9 @@ extension DatabaseManager { ) } } + +/// A schema save that stopped once its statements had started to run, so the table may have +/// changed even though the save did not finish. +private struct SchemaChangeFailedAfterWriting: Error { + let message: String +} diff --git a/TablePro/Core/Database/DatabaseManager+SchemaComposition.swift b/TablePro/Core/Database/DatabaseManager+SchemaComposition.swift index 3d21a28109..a3bd5f7a29 100644 --- a/TablePro/Core/Database/DatabaseManager+SchemaComposition.swift +++ b/TablePro/Core/Database/DatabaseManager+SchemaComposition.swift @@ -21,22 +21,48 @@ extension DatabaseManager { } } + /// Composes a Structure save for SQL Preview and for Save. The per-operation refusals run first + /// and cost nothing; the driver's save-level review follows, reading the server where the + /// engine needs it, and can refuse the save or put statements ahead of it. func schemaChangeStatements( tableName: String, changes: [SchemaChange], scope: DatabaseScope - ) async throws -> [SchemaStatement] { + ) async throws -> SchemaChangeScript { try await withSchemaComposer(scope: scope, route: schemaChangeRoute(for: scope)) { driver, pluginDriver in let constraintName = await PrimaryKeyConstraintLookup.constraintName( tableName: tableName, changes: changes, driver: driver ) - return try SchemaStatementGenerator( + let generator = SchemaStatementGenerator( tableName: tableName, primaryKeyConstraintName: constraintName, pluginDriver: pluginDriver - ).generate(changes: changes) + ) + let statements = try generator.generate(changes: changes) + let operations = generator.orderedOperations(for: changes) + let review = try await driver.reviewSchemaChange( + table: tableName, + schema: scope.schema, + operations: operations + ) + if let refusal = review.refusal { + throw SchemaOperationRefusedError(reason: refusal) + } + let leading = review.leadingStatements.map { sql in + SchemaStatement( + sql: sql.hasSuffix(";") ? sql : sql + ";", + description: "Prepare '\(tableName)'", + isDestructive: false + ) + } + return SchemaChangeScript( + tableName: tableName, + statements: leading + statements, + operations: operations, + review: review + ) } } diff --git a/TablePro/Core/Events/AppCommands.swift b/TablePro/Core/Events/AppCommands.swift index 2c8877c805..47bee7ac72 100644 --- a/TablePro/Core/Events/AppCommands.swift +++ b/TablePro/Core/Events/AppCommands.swift @@ -38,6 +38,9 @@ struct DatabaseObjectChange: Sendable, Equatable { enum Kind: Sendable, Equatable { /// The object's rows were recomputed, as a materialized view refresh does. case rows + /// The object's columns, keys or indexes changed, and with them possibly its rows: a + /// Structure save, a table rebuild or a column reorder, finished or stopped partway. + case structure /// The object's comment changed. case comment /// The object no longer exists. diff --git a/TablePro/Core/Plugins/DatabaseType+Registry.swift b/TablePro/Core/Plugins/DatabaseType+Registry.swift index 875b268d7f..c296e32b2e 100644 --- a/TablePro/Core/Plugins/DatabaseType+Registry.swift +++ b/TablePro/Core/Plugins/DatabaseType+Registry.swift @@ -82,6 +82,10 @@ extension DatabaseType { PluginMetadataRegistry.shared.snapshot(for: self)?.capabilities.exactRowCountIsBilledScan ?? false } + var columnsAreSampled: Bool { + PluginMetadataRegistry.shared.snapshot(for: self)?.capabilities.columnsAreSampled ?? false + } + var authenticationIsDatabaseScoped: Bool { PluginMetadataRegistry.shared.snapshot(for: self)? .capabilities.authenticationIsDatabaseScoped ?? false diff --git a/TablePro/Core/Plugins/PluginDriverAdapter+SchemaChangeChecks.swift b/TablePro/Core/Plugins/PluginDriverAdapter+SchemaChangeChecks.swift new file mode 100644 index 0000000000..8f28aa10e1 --- /dev/null +++ b/TablePro/Core/Plugins/PluginDriverAdapter+SchemaChangeChecks.swift @@ -0,0 +1,49 @@ +// +// PluginDriverAdapter+SchemaChangeChecks.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +internal extension PluginDriverAdapter { + func reviewSchemaChange( + table: String, + schema: String?, + operations: [PluginSchemaOperation] + ) async throws -> PluginSchemaChangeReview { + try await schemaPluginDriver.reviewSchemaChange(table: table, schema: schema, operations: operations) + } + + func schemaChangeRefusalBeforeWriting( + table: String, + schema: String?, + operations: [PluginSchemaOperation], + review: PluginSchemaChangeReview + ) async throws -> String? { + try await schemaPluginDriver.schemaChangeRefusalBeforeWriting( + table: table, + schema: schema, + operations: operations, + review: review + ) + } + + func schemaChangeShortfallAfterWriting( + table: String, + schema: String?, + operations: [PluginSchemaOperation], + review: PluginSchemaChangeReview + ) async throws -> String? { + try await schemaPluginDriver.schemaChangeShortfallAfterWriting( + table: table, + schema: schema, + operations: operations, + review: review + ) + } + + func tableDefinitionDidChange(table: String, schema: String?) { + schemaPluginDriver.tableDefinitionDidChange(table: table, schema: schema) + } +} diff --git a/TablePro/Core/Plugins/PluginManager.swift b/TablePro/Core/Plugins/PluginManager.swift index 4a0cc62834..086fe6ebdd 100644 --- a/TablePro/Core/Plugins/PluginManager.swift +++ b/TablePro/Core/Plugins/PluginManager.swift @@ -99,6 +99,16 @@ final class PluginManager: ObservableObject { /// defaults answer nil and throw, so an already-built plugin keeps the column grid. It adds the /// `modifyIndex` and `dropIndex` cases to the non-frozen `PluginSchemaOperation`, which an /// already-built plugin answers through its `@unknown default`. + /// + /// 33 also adds the `modifyColumn` and `dropColumn` cases to `PluginSchemaOperation`, the + /// `PluginSchemaChangeReview` value, and `reviewSchemaChange(table:schema:operations:)`, + /// `schemaChangeRefusalBeforeWriting(table:schema:operations:review:)` and + /// `schemaChangeShortfallAfterWriting(table:schema:operations:review:)`, the save-level + /// questions a document store needs the server to answer, and + /// `tableDefinitionDidChange(table:schema:)`, which tells the session's driver to drop what it + /// learned about a table another connection changed. The defaults approve every save, find + /// every save finished and keep nothing, so an already-built plugin keeps loading and saves as + /// before. nonisolated static let currentPluginKitVersion = 33 /// Still 19, so every plugin already published for the previous release keeps loading. diff --git a/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift b/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift index 75083b5ee1..9b2504202d 100644 --- a/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift +++ b/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift @@ -20,13 +20,18 @@ extension PluginMetadataRegistry { return [ ("MongoDB", PluginMetadataSnapshot( displayName: "MongoDB", iconName: "mongodb-icon", defaultPort: 27_017, - requiresAuthentication: false, supportsForeignKeys: false, supportsSchemaEditing: false, + requiresAuthentication: false, supportsForeignKeys: false, supportsSchemaEditing: true, isDownloadable: true, primaryUrlScheme: "mongodb", parameterStyle: .questionMark, navigationModel: .standard, explainVariants: [], pathFieldRole: .database, supportsHealthMonitor: true, urlSchemes: ["mongodb", "mongodb+srv"], postConnectActions: [], brandColorHex: "#00ED63", queryLanguageName: "MQL", editorLanguage: .javascript, connectionMode: .network, supportsDatabaseSwitching: true, + structureEditing: SchemaEditingSupport( + structureEdits: StructureObjectEditMatrix([ + .table: [.addColumn, .renameColumn, .dropColumn, .addIndex, .dropIndex] + ]) + ), capabilities: PluginMetadataSnapshot.CapabilityFlags( supportsSchemaSwitching: false, supportsImport: true, @@ -40,8 +45,12 @@ extension PluginMetadataRegistry { requiresReconnectForDatabaseSwitch: false, supportsDropDatabase: true, supportsDocumentEditing: true, + supportsAddColumn: false, + supportsAddIndex: false, + supportsDropIndex: false, supportsOpportunisticTLS: false, - authenticationIsDatabaseScoped: true + authenticationIsDatabaseScoped: true, + columnsAreSampled: true ), schema: PluginMetadataSnapshot.SchemaInfo( defaultSchemaName: "public", diff --git a/TablePro/Core/Plugins/PluginMetadataRegistry.swift b/TablePro/Core/Plugins/PluginMetadataRegistry.swift index e9057ad406..c89a0f2263 100644 --- a/TablePro/Core/Plugins/PluginMetadataRegistry.swift +++ b/TablePro/Core/Plugins/PluginMetadataRegistry.swift @@ -93,6 +93,10 @@ struct PluginMetadataSnapshot: Sendable { /// has no `COUNT(*)`. DynamoDB is the case: a count is a `Scan` of every item. Such an engine is counted /// only when the user asks, and only by its driver. var exactRowCountIsBilledScan: Bool = false + /// Whether a table's columns are a sample of its rows rather than a declared schema. A + /// MongoDB collection lists the fields found in its first documents, so a field missing from + /// one side's list says nothing about whether that side holds it. + var columnsAreSampled: Bool = false var isEngineReadOnly: Bool = false /// Which connection field carries the path of the local database file this driver opens, @@ -715,6 +719,7 @@ final class PluginMetadataRegistry: @unchecked Sendable { .browsingRequiresSelectedDatabase ?? false, pagination: existingSnapshot?.capabilities.pagination ?? .offset, exactRowCountIsBilledScan: existingSnapshot?.capabilities.exactRowCountIsBilledScan ?? false, + columnsAreSampled: existingSnapshot?.capabilities.columnsAreSampled ?? false, isEngineReadOnly: existingSnapshot?.capabilities.isEngineReadOnly ?? false, localFilePathField: existingSnapshot?.capabilities.localFilePathField, supportsRemoteDatabaseFile: existingSnapshot?.capabilities diff --git a/TablePro/Core/SchemaTracking/SchemaChangeScript.swift b/TablePro/Core/SchemaTracking/SchemaChangeScript.swift new file mode 100644 index 0000000000..da74fa8392 --- /dev/null +++ b/TablePro/Core/SchemaTracking/SchemaChangeScript.swift @@ -0,0 +1,17 @@ +// +// SchemaChangeScript.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +/// A Structure save composed for one table: the statements that run, in order, the operations +/// they carry out, and the driver's review they were composed with. The driver is asked about the +/// operations and that review once more just before the first statement runs. +struct SchemaChangeScript: Sendable { + let tableName: String + let statements: [SchemaStatement] + let operations: [PluginSchemaOperation] + let review: PluginSchemaChangeReview +} diff --git a/TablePro/Core/SchemaTracking/SchemaOperationRefusal.swift b/TablePro/Core/SchemaTracking/SchemaOperationRefusal.swift index bf0860faf8..506cf2d286 100644 --- a/TablePro/Core/SchemaTracking/SchemaOperationRefusal.swift +++ b/TablePro/Core/SchemaTracking/SchemaOperationRefusal.swift @@ -30,11 +30,41 @@ internal enum SchemaOperationRefusal { return driver.schemaOperationRefusal(.renameCheckConstraint(from: old.name, to: new.name)) case .addCheckConstraint, .deleteCheckConstraint: return driver.checkConstraintRefusal - case .modifyColumn, .deleteColumn, .addForeignKey, .modifyForeignKey, .deleteForeignKey, .modifyPrimaryKey: + case .modifyColumn(let old, let new): + return driver.schemaOperationRefusal(.modifyColumn(old: old.toPlugin(), new: new.toPlugin())) + case .deleteColumn(let column): + return driver.schemaOperationRefusal(.dropColumn(column.toPlugin())) + case .addForeignKey, .modifyForeignKey, .deleteForeignKey, .modifyPrimaryKey: return nil } } + /// The operations a change carries out, as the driver's save-level questions receive them. A + /// check constraint is an operation only when it is renamed, and foreign key and primary key + /// changes have no case at all. + static func operations(for change: SchemaChange) -> [PluginSchemaOperation] { + switch change { + case .addColumn(let column): + return [.addColumn(column.toPlugin())] + case .modifyColumn(let old, let new): + return [.modifyColumn(old: old.toPlugin(), new: new.toPlugin())] + case .deleteColumn(let column): + return [.dropColumn(column.toPlugin())] + case .addIndex(let index): + return [.addIndex(index.toPlugin())] + case .modifyIndex(let old, let new): + return [.modifyIndex(old: old.toPlugin(), new: new.toPlugin())] + case .deleteIndex(let index): + return [.dropIndex(index.toPlugin())] + case .modifyCheckConstraint(let old, let new): + guard old.expression == new.expression, old.name != new.name else { return [] } + return [.renameCheckConstraint(from: old.name, to: new.name)] + case .addCheckConstraint, .deleteCheckConstraint, .addForeignKey, .modifyForeignKey, .deleteForeignKey, + .modifyPrimaryKey: + return [] + } + } + static func reason(for definition: PluginCreateTableDefinition, driver: any PluginDatabaseDriver) -> String? { let operations = definition.columns.map(PluginSchemaOperation.addColumn) + definition.indexes.map(PluginSchemaOperation.addIndex) diff --git a/TablePro/Core/SchemaTracking/SchemaStatementGenerator.swift b/TablePro/Core/SchemaTracking/SchemaStatementGenerator.swift index 891862c155..8a3970c5b6 100644 --- a/TablePro/Core/SchemaTracking/SchemaStatementGenerator.swift +++ b/TablePro/Core/SchemaTracking/SchemaStatementGenerator.swift @@ -68,6 +68,12 @@ struct SchemaStatementGenerator { return statements } + /// Every operation the changes carry out, in the order `generate(changes:)` runs their + /// statements, which is the order a driver's save-level questions are asked in. + func orderedOperations(for changes: [SchemaChange]) -> [PluginSchemaOperation] { + sortByDependency(changes).flatMap(SchemaOperationRefusal.operations(for:)) + } + // MARK: - Dependency Ordering private func sortByDependency(_ changes: [SchemaChange]) -> [SchemaChange] { diff --git a/TablePro/Core/SchemaTracking/StructureChangeManager.swift b/TablePro/Core/SchemaTracking/StructureChangeManager.swift index e108305759..70fdec676a 100644 --- a/TablePro/Core/SchemaTracking/StructureChangeManager.swift +++ b/TablePro/Core/SchemaTracking/StructureChangeManager.swift @@ -35,6 +35,17 @@ final class StructureChangeManager: ObservableObject, ChangeManaging { @Published var tableName: String? + /// The edits a save in flight is writing, from the press until the save ends. + /// + /// While it is set nothing stages, undoes, discards or reloads. The save writes what it read at + /// the press and clears it when it lands, so an edit accepted in between is missing from the + /// script it runs and would then be cleared with the edits it did run. On MongoDB the time in + /// between includes a read of every document the save changes, which can run for as long as + /// the query timeout. + @Published private(set) var heldSave: StructureSaveSnapshot? + + var isHeldForSave: Bool { heldSave != nil } + // MARK: - Undo/Redo Support /// Private `NSUndoManager` owned by this change manager. Each @@ -61,8 +72,8 @@ final class StructureChangeManager: ObservableObject, ChangeManaging { return manager }() - var canUndo: Bool { undoManager.canUndo } - var canRedo: Bool { undoManager.canRedo } + var canUndo: Bool { !isHeldForSave && undoManager.canUndo } + var canRedo: Bool { !isHeldForSave && undoManager.canRedo } /// Mirrors `DataChangeManager.registerUndo`. The `groupingLevel` check is what lets /// `performAsOneUndoStep` nest: inside one, a group is already open and this adds to it rather @@ -79,6 +90,7 @@ final class StructureChangeManager: ObservableObject, ChangeManaging { /// selection is the case that needs it: the grid calls `deleteColumn` once per row, and one /// Cmd+Z should bring the whole selection back. func performAsOneUndoStep(_ body: () -> Void) { + guard !isHeldForSave else { return } undoManager.beginUndoGrouping() defer { undoManager.endUndoGrouping() } body() @@ -94,6 +106,7 @@ final class StructureChangeManager: ObservableObject, ChangeManaging { checkConstraints: [CheckConstraintInfo] = [], primaryKey: [String] ) { + guard !isHeldForSave else { return } self.tableName = tableName self.currentColumns = columns.map { EditableColumnDefinition.from($0) } @@ -232,6 +245,7 @@ final class StructureChangeManager: ObservableObject, ChangeManaging { _ entity: Entity, using operations: SchemaEntityOperations ) { + guard !isHeldForSave else { return } self[keyPath: operations.working].append(entity) let key = operations.identifier(entity.id) pendingChanges[key] = operations.addition(entity) @@ -247,6 +261,7 @@ final class StructureChangeManager: ObservableObject, ChangeManaging { with newEntity: Entity, using operations: SchemaEntityOperations ) { + guard !isHeldForSave else { return } if let workingIndex = self[keyPath: operations.working].firstIndex(where: { $0.id == id }) { let oldWorking = self[keyPath: operations.working][workingIndex] if oldWorking != newEntity { @@ -279,6 +294,7 @@ final class StructureChangeManager: ObservableObject, ChangeManaging { } private func stageDeletion(id: UUID, using operations: SchemaEntityOperations) { + guard !isHeldForSave else { return } let key = operations.identifier(id) if let entity = self[keyPath: operations.current].first(where: { $0.id == id }) { registerUndo(operations.deleteActionName) { target in @@ -373,6 +389,7 @@ final class StructureChangeManager: ObservableObject, ChangeManaging { /// row-specific affordance and the global undo stack are independent /// affordances. The data tab uses the same separation. func undoDelete(for tab: StructureTab, at row: Int) { + guard !isHeldForSave else { return } let key: SchemaChangeIdentifier switch tab { case .columns: @@ -575,6 +592,7 @@ final class StructureChangeManager: ObservableObject, ChangeManaging { } func discardChanges() { + guard !isHeldForSave else { return } pendingChanges.removeAll() changeOrder.removeAll() validationErrors.removeAll() @@ -587,15 +605,38 @@ final class StructureChangeManager: ObservableObject, ChangeManaging { changeOrder.compactMap { pendingChanges[$0] } } + // MARK: - Save Hold + + /// Takes the staged edits for a save, or nil when there are none or a save already holds them. + /// Taken before the save's first suspension, which is what refuses a second press. + func holdForSave() -> StructureSaveSnapshot? { + guard heldSave == nil, hasChanges else { return nil } + let snapshot = StructureSaveSnapshot(changes: getChangesArray()) + heldSave = snapshot + return snapshot + } + + /// Ends the hold a save took. A save that wrote clears the staged edits only while they are + /// still exactly the ones it read, so nothing it did not write is cleared, and a hold that has + /// already ended cannot clear what was staged after it. Returns whether the edits were cleared. + @discardableResult + func releaseHold(_ snapshot: StructureSaveSnapshot, written: Bool) -> Bool { + guard heldSave?.id == snapshot.id else { return false } + heldSave = nil + guard written, getChangesArray() == snapshot.changes else { return false } + discardChanges() + return true + } + // MARK: - Undo/Redo Operations func undo() { - guard undoManager.canUndo else { return } + guard !isHeldForSave, undoManager.canUndo else { return } undoManager.undo() } func redo() { - guard undoManager.canRedo else { return } + guard !isHeldForSave, undoManager.canRedo else { return } undoManager.redo() } @@ -787,3 +828,9 @@ enum SchemaUndoAction { case checkConstraintDelete(constraint: EditableCheckConstraintDefinition, at: Int?) case primaryKeyChange(old: [String], new: [String]) } + +/// The staged edits a save read when it was pressed, and so the only edits it may clear. +struct StructureSaveSnapshot: Equatable { + let id = UUID() + let changes: [SchemaChange] +} diff --git a/TablePro/Core/Services/Query/SchemaColumnStore.swift b/TablePro/Core/Services/Query/SchemaColumnStore.swift index 733cfda3a3..f6f38a209b 100644 --- a/TablePro/Core/Services/Query/SchemaColumnStore.swift +++ b/TablePro/Core/Services/Query/SchemaColumnStore.swift @@ -68,6 +68,11 @@ final class SchemaColumnStore { entries.removeAll() } + func remove(_ key: String) { + loads.removeValue(forKey: key)?.task.cancel() + entries.removeValue(forKey: key) + } + /// A cancelled load is never joined. Between the last waiter leaving and its `load` clearing /// the entry there is a window where the task is already cancelled, and a caller that adopted /// it would wait for a fetch that is never going to produce anything. Clicking back to a table diff --git a/TablePro/Resources/Localizable.xcstrings b/TablePro/Resources/Localizable.xcstrings index 55d3752f85..f746766c76 100644 --- a/TablePro/Resources/Localizable.xcstrings +++ b/TablePro/Resources/Localizable.xcstrings @@ -184363,6 +184363,126 @@ }, "This database does not store whole documents" : { + }, + "A MongoDB field has no type, default or nullability to change. Only its name can change." : { + + }, + "_id cannot be renamed, removed or used as a new name. MongoDB keys every document by it." : { + + }, + "A MongoDB field needs a name." : { + + }, + "A MongoDB field name cannot contain a NUL character." : { + + }, + "%@ is changed twice in this save. Save one change to it at a time." : { + + }, + "%@ is a system collection. Its fields cannot be renamed or removed." : { + + }, + "Collection %@ no longer exists." : { + + }, + "%@ is a view. Rename or remove the field in the collection it reads." : { + + }, + "%@ is a time series collection. MongoDB cannot rename or remove its fields." : { + + }, + "%1$@ is a collection of type %2$@. Its fields cannot be renamed or removed." : { + + }, + "Index %1$@ uses %2$@. Drop it first by running %3$@ in a query tab, then save again." : { + + }, + "%@ is an encrypted field. MongoDB cannot rename or remove it." : { + + }, + "View %1$@ reads %2$@. Change the view first, then save again." : { + + }, + "The validator has a key named %@ that the shell would reorder or drop. Change the validator from a query tab instead." : { + + }, + "The validator already declares %@. Remove that declaration first, then save again." : { + + }, + "The validator uses %@ in a way that cannot be updated with the field. Change the validator first, then save again." : { + + }, + "The validator could not be read, so the change was not checked against it." : { + + }, + "Some documents hold both %1$@ and %2$@. Remove one of the two from those documents first." : { + + }, + "The validator would reject the document with _id %1$@ once %2$@ is removed. Fix the document or the validator first." : { + + }, + "The validator would reject the document with _id %1$@ once %2$@ is renamed to %3$@. Fix the document or the validator first." : { + + }, + "Checking the documents of %1$@ took over %2$d seconds, so nothing was changed. Raise the query timeout in Settings and save again." : { + + }, + "Couldn't check %1$@ before changing its documents: %2$@" : { + + }, + "%@ lists a collection's fields from a sample of its documents, so structures can be compared but no script is generated." : { + + }, + "%1$@ is a capped collection, and a longer field name makes MongoDB delete its oldest documents. Choose a name no longer than %2$@." : { + + }, + "Search index %1$@ uses %2$@. Change or drop that search index first, then save again." : { + + }, + "Once this save updates the validator, it would reject the document with _id %@. Fix the document or the validator first." : { + + }, + "Saving Changes…" : { + + }, + "The staged changes are being saved." : { + + }, + "The staged changes were edited after this script was prepared. Save again to review the new script." : { + + }, + "%@ changed while its documents were being checked, so nothing was changed. Save again." : { + + }, + "The save ran, but checking %1$@ afterwards failed: %2$@. Save again to make sure every document was changed." : { + + }, + "The save did not finish: one document still holds %@, most likely written by another client while the save ran. Save again to finish." : { + + }, + "The save did not finish: %1$lld documents still hold %2$@, most likely written by another client while the save ran. Save again to finish." : { + + }, + "This server runs Atlas Search but cannot list its search indexes, so one that uses the field cannot be ruled out. Check the collection's search indexes in Atlas, then change the field from a query tab." : { + + }, + "%@ changed after this save was prepared, so nothing was changed. Review the save and save again." : { + + }, + "The save did not finish: the updated validator of %1$@ rejects the document with _id %2$@, most likely written by another client while the save ran. Fix that document, then save again." : { + + }, + "Renaming would take the document with _id %@ past MongoDB's 16 MB limit. Choose shorter names, or shrink that document first." : { + + }, + "The save ran, but index %1$@ on %2$@ was created while it did and uses %3$@. Check that it indexes the field you meant." : { + + }, + "The save ran, but search index %1$@ on %2$@ was created while it did and uses %3$@. Check that it maps the field you meant." : { + + }, + "The save ran, but view %1$@ was created while it did and reads %2$@. Change it to read the field where it is now." : { + } }, "version" : "1.1" diff --git a/TablePro/Views/Main/Extensions/MainContentCoordinator+DatabaseObjectTools.swift b/TablePro/Views/Main/Extensions/MainContentCoordinator+DatabaseObjectTools.swift index d74ef19556..6600ff5fd2 100644 --- a/TablePro/Views/Main/Extensions/MainContentCoordinator+DatabaseObjectTools.swift +++ b/TablePro/Views/Main/Extensions/MainContentCoordinator+DatabaseObjectTools.swift @@ -119,6 +119,12 @@ extension MainContentCoordinator { /// the data reload: that one refuses to run while the Structure pane is in front and refreshes /// the structure instead, and a tab excluded from the eviction loop for being selected would /// otherwise keep its rows with nothing left to reload them. + /// + /// A structure change reaches both halves of every tab on the object. The selected tab's rows + /// reload even while its Structure pane is in front, because Data is one click away and shows no + /// sign of being stale, and a structure the user has staged edits against keeps them, whichever + /// tab it is on: the tab whose save made the change is one of them, and a save that stopped + /// partway keeps its edits for the retry. func applyObjectChange( _ change: DatabaseObjectChange, hasPendingTableOps: Bool, @@ -142,6 +148,18 @@ extension MainContentCoordinator { if selected != nil { handleRefresh(hasPendingTableOps: hasPendingTableOps, onDiscard: onDiscard) } + case .structure: + forgetSchemaColumns(of: change, tabs: showing) + for tab in showing where tab.id != selected?.id { + evictReloadableTableRows(for: tab.id) + } + refreshStructure(ofTabs: showing) + guard let selected else { return } + if selected.display.resultsViewMode == .structure { + reloadRowsBehindStructure(hasPendingTableOps: hasPendingTableOps) + } else { + reloadActiveTableData(hasPendingTableOps: hasPendingTableOps, onDiscard: onDiscard) + } case .comment: for tab in showing { tableMetadataCache.removeValue(forKey: tab.id) diff --git a/TablePro/Views/Main/Extensions/MainContentCoordinator+Refresh.swift b/TablePro/Views/Main/Extensions/MainContentCoordinator+Refresh.swift index a968b11ac8..7545c57c6e 100644 --- a/TablePro/Views/Main/Extensions/MainContentCoordinator+Refresh.swift +++ b/TablePro/Views/Main/Extensions/MainContentCoordinator+Refresh.swift @@ -79,6 +79,46 @@ extension MainContentCoordinator { } } + /// Reloads the selected tab's rows while its Structure pane is in front, so Data shows the table + /// as it now is. Rows holding the user's edits are kept, and a tab that never loaded its rows has + /// none to reload. + func reloadRowsBehindStructure(hasPendingTableOps: Bool) { + guard let (tab, tabIndex) = tabManager.selectedTabAndIndex, + tab.tabType == .table, + tab.display.resultsViewMode == .structure, + tab.execution.lastExecutedAt != nil, + !changeManager.hasChanges, + !tab.pendingChanges.hasChanges, + !hasPendingTableOps + else { return } + reloadTableTab(at: tabIndex) + } + + /// Fetches the structure again where no one is editing it: now for the one on screen, on its next + /// mount for the rest. A structure holding staged edits keeps them and the baseline they were + /// staged against, because a fetch adopts a new baseline and clears them without asking. + func refreshStructure(ofTabs tabs: [QueryTab]) { + let selectedId = tabManager.selectedTabId + for tab in tabs { + guard let session = structureSessions[tab.id], !session.changeManager.hasChanges else { continue } + if tab.id == selectedId, tab.display.resultsViewMode == .structure, let refresh = structureActions?.refresh { + refresh() + } else { + session.markStructureStale() + } + } + } + + /// The columns a column-scoped query is built from describe the table before the change, and a + /// reload builds its select list from them before it fetches anything, so a dropped column would + /// stay in it. The reload's own fetch stores the new set. + func forgetSchemaColumns(of change: DatabaseObjectChange, tabs: [QueryTab]) { + schemaColumns.remove(schemaColumnsKey(change.name, scope: change.scope)) + for tab in tabs { + schemaColumns.remove(schemaColumnsKey(change.name, scope: scope(for: tab))) + } + } + private func reloadTableTab(at tabIndex: Int) { stopExecution(for: tabManager.tabs[tabIndex].id) /// A refresh asks for the table as it is now, so the exact count the user requested earlier diff --git a/TablePro/Views/Structure/StructureEditingSession+Apply.swift b/TablePro/Views/Structure/StructureEditingSession+Apply.swift index 9a32c9d5b2..bd97a46631 100644 --- a/TablePro/Views/Structure/StructureEditingSession+Apply.swift +++ b/TablePro/Views/Structure/StructureEditingSession+Apply.swift @@ -5,6 +5,7 @@ import Combine import Foundation +import os import TableProPluginKit /// What happened when a tab was asked to apply its staged structure edits. @@ -44,6 +45,11 @@ internal extension StructureEditingSession { let changes = changeManager.getChangesArray() guard !changes.isEmpty else { return .nothingToApply } + /// A save that is already running holds the tab until it ends. On MongoDB that includes the + /// check that reads the documents, which can take as long as the query timeout, and a second + /// press would otherwise start the same save again behind it. + guard !isApplying, !changeManager.isHeldForSave else { return .refused } + /// Asked before Safe Mode and before the destructive prompt, because an incomplete row is /// not a change the user meant to make. Without this a foreign key added and never filled /// in reached DDL generation as `ADD CONSTRAINT "" FOREIGN KEY () REFERENCES "" ()`, which @@ -75,6 +81,31 @@ internal extension StructureEditingSession { return .refused } + /// Taken before the first suspension and held until the save ends, so a second press is + /// refused before it can start the same save again, and no edit can land between the + /// statements being composed and the staged edits being cleared. The edits a save clears + /// are the ones it held, and only while they are still exactly those. + guard let hold = changeManager.holdForSave() else { return .refused } + isApplying = true + let outcome = await saveHeldChanges(hold.changes, coordinator: coordinator) + let cleared = changeManager.releaseHold(hold, written: outcome == .applied) + isApplying = false + guard outcome == .applied else { return outcome } + guard cleared else { + Self.logger.fault("A structure save landed over staged edits it did not write; they stay staged") + return .refused + } + tabData.markAllStale() + hasLoaded = false + lastAppliedAt = Date() + markApplied() + return outcome + } + + private func saveHeldChanges( + _ changes: [SchemaChange], + coordinator: MainContentCoordinator? + ) async -> StructureSaveOutcome { let planStart = ContinuousClock.Instant.now let plan: StructureSavePlan do { @@ -99,14 +130,14 @@ internal extension StructureEditingSession { /// that confirmation and shows the exact script rather than a list of descriptions, so /// asking first would be two dialogs for one decision. The HIG's rule is one alert at a /// time. - return presentRebuildReview(prepared, startedAt: planStart, coordinator: coordinator) - case .alter(let statements): - return await applyAlterStatements(statements, changes: changes, coordinator: coordinator) + return presentRebuildReview(prepared, preparedFrom: changes, startedAt: planStart, coordinator: coordinator) + case .alter(let script): + return await applyAlterStatements(script, changes: changes, coordinator: coordinator) } } private func applyAlterStatements( - _ statements: [SchemaStatement], + _ script: SchemaChangeScript, changes: [SchemaChange], coordinator: MainContentCoordinator? ) async -> StructureSaveOutcome { @@ -130,24 +161,16 @@ internal extension StructureEditingSession { /// this and the user can take as long as they like over it, so a clock started earlier /// measures their reading time and reports an instant ALTER as having taken a minute. let operationStart = ContinuousClock.Instant.now - isApplying = true do { try await DatabaseManager.shared.executeSchemaChanges( - statements, + script, databaseType: connection.type, scope: scope ) - changeManager.discardChanges() - tabData.markAllStale() - hasLoaded = false - lastAppliedAt = Date() - isApplying = false - markApplied() report(.succeeded(OperationSummary()), startedAt: operationStart, coordinator: coordinator) return .applied } catch { - isApplying = false report(.failed(reason: error.localizedDescription), startedAt: operationStart, coordinator: coordinator) AlertHelper.showErrorSheet( title: String(localized: "Error Applying Changes"), @@ -165,6 +188,7 @@ internal extension StructureEditingSession { /// action if the user confirms it there. private func presentRebuildReview( _ prepared: StructureRebuildPlanRunner.Prepared, + preparedFrom changes: [SchemaChange], startedAt operationStart: ContinuousClock.Instant, coordinator: MainContentCoordinator? ) -> StructureSaveOutcome { @@ -176,7 +200,9 @@ internal extension StructureEditingSession { action: TableRebuildReviewRequest.Action( title: String(localized: "Apply and Rebuild"), perform: { [weak coordinator] in - await self.runRebuild(prepared, startedAt: operationStart, coordinator: coordinator) + await self.runRebuild( + prepared, preparedFrom: changes, startedAt: operationStart, coordinator: coordinator + ) } ) ) @@ -189,11 +215,24 @@ internal extension StructureEditingSession { /// The table was dropped and recreated, so the grid's rows, the query history and the saved /// column layout all describe a table that no longer exists in that form. The ordinary save /// path does not record history or clear a layout because an `ALTER` leaves both valid. + /// + /// The script was built from the edits staged when Save was pressed, and the sheet can stay up + /// for as long as the user likes, so it runs only while those are still the staged edits, and + /// holds them the way a save does until it ends. private func runRebuild( _ prepared: StructureRebuildPlanRunner.Prepared, + preparedFrom changes: [SchemaChange], startedAt: ContinuousClock.Instant, coordinator: MainContentCoordinator? ) async { + guard !isApplying, changeManager.getChangesArray() == changes, let hold = changeManager.holdForSave() else { + AlertHelper.showErrorSheet( + title: String(localized: "Error Applying Changes"), + message: String(localized: "The staged changes were edited after this script was prepared. Save again to review the new script."), + window: coordinator?.contentWindow + ) + return + } isApplying = true do { try await StructureRebuildPlanRunner.execute( @@ -202,6 +241,7 @@ internal extension StructureEditingSession { operationDescription: String(localized: "Apply Schema Changes") ) } catch { + changeManager.releaseHold(hold, written: false) isApplying = false CatalogChangeService.post( .changed(CatalogChange(connectionId: connection.id, database: prepared.scope.database, kinds: .tables)) @@ -230,16 +270,20 @@ internal extension StructureEditingSession { ) ) - changeManager.discardChanges() - tabData.markAllStale() - hasLoaded = false - lastAppliedAt = Date() + let cleared = changeManager.releaseHold(hold, written: true) isApplying = false - markApplied() + if cleared { + tabData.markAllStale() + hasLoaded = false + lastAppliedAt = Date() + markApplied() + } else { + Self.logger.fault("A table rebuild landed over staged edits it did not write; they stay staged") + } if let clearTarget = coordinator?.selectedColumnLayoutClearTarget() { coordinator?.clearColumnLayout(clearTarget) } - AppCommands.shared.refreshData.send(DataRefreshRequest(connectionId: connection.id)) + DatabaseManager.shared.reportTableDefinitionChange(table: prepared.tableName, in: prepared.scope) CatalogChangeService.post( .changed(CatalogChange(connectionId: connection.id, database: prepared.scope.database, kinds: .tables)) ) diff --git a/TablePro/Views/Structure/StructureEditingSession.swift b/TablePro/Views/Structure/StructureEditingSession.swift index f5f921104e..f665c6e8df 100644 --- a/TablePro/Views/Structure/StructureEditingSession.swift +++ b/TablePro/Views/Structure/StructureEditingSession.swift @@ -34,7 +34,7 @@ import TableProPluginKit /// fetch is the only version of this that keeps the edits. @MainActor internal final class StructureEditingSession: ObservableObject { - private static let logger = Logger(subsystem: "com.TablePro", category: "StructureEditingSession") + static let logger = Logger(subsystem: "com.TablePro", category: "StructureEditingSession") /// The scope and table this session was opened against. A tab retargeted to another table gets /// a new session rather than inheriting edits staged against the old one. @@ -153,6 +153,15 @@ internal final class StructureEditingSession: ObservableObject { appliedVersion += 1 } + /// The object changed outside this editor, so the next mount fetches it again. A session holding + /// staged edits is left alone: the fetch re-baselines the change manager, which would discard + /// them without asking. + internal func markStructureStale() { + guard !changeManager.hasChanges else { return } + tabData.markAllStale() + hasLoaded = false + } + internal func reloadConcurrentRefreshAvailability( provider: any ScopedMetadataProviding = DatabaseManager.shared ) async { diff --git a/TablePro/Views/Structure/StructureFooterPolicy.swift b/TablePro/Views/Structure/StructureFooterPolicy.swift index 3cf115738b..3155d98e00 100644 --- a/TablePro/Views/Structure/StructureFooterPolicy.swift +++ b/TablePro/Views/Structure/StructureFooterPolicy.swift @@ -38,6 +38,11 @@ enum StructureFooterPolicy { } } + /// Why the grid and the pair under it take no edits while a save runs. + static var savingReason: String { + String(localized: "The staged changes are being saved.") + } + static func labels(for tab: StructureTab) -> (add: String, remove: String)? { switch tab { case .columns: @@ -63,6 +68,7 @@ enum StructureFooterPolicy { tab: StructureTab, canEditSchema: Bool, hasSelection: Bool, + isSaving: Bool, resolve: (StructureEditOperation) -> StructureEditAvailability ) -> StructureFooterCapability { guard canEditSchema, @@ -73,6 +79,15 @@ enum StructureFooterPolicy { return StructureFooterCapability() } + /// A save holds what is staged until it ends, so nothing can be added or removed meanwhile. + guard !isSaving else { + return StructureFooterCapability( + addLabel: labels.add, + removeLabel: labels.remove, + unavailableReason: savingReason + ) + } + let addAvailability = resolve(adding) let removeAvailability = resolve(removing) diff --git a/TablePro/Views/Structure/StructureGridDelegate+Inspector.swift b/TablePro/Views/Structure/StructureGridDelegate+Inspector.swift index 97b565962a..97768b3333 100644 --- a/TablePro/Views/Structure/StructureGridDelegate+Inspector.swift +++ b/TablePro/Views/Structure/StructureGridDelegate+Inspector.swift @@ -15,7 +15,7 @@ extension StructureGridDelegate: InspectorRowSource { atDisplayRow: displayRow, tab: selectedTab, provider: provider, - canEditSchema: editGate.allowsAnyEdit, + canEditSchema: editGate.allowsAnyEdit && !structureChangeManager.isHeldForSave, lockedFieldIndices: lockedFieldIndices, rowOptions: { dataGridMenuOptions(forRow: displayRow, columnIndex: $0) } ) diff --git a/TablePro/Views/Structure/StructureSavePlan.swift b/TablePro/Views/Structure/StructureSavePlan.swift index 004f829264..6b995b9460 100644 --- a/TablePro/Views/Structure/StructureSavePlan.swift +++ b/TablePro/Views/Structure/StructureSavePlan.swift @@ -7,13 +7,13 @@ import Foundation import TableProPluginKit internal enum StructureSavePlan { - case alter([SchemaStatement]) + case alter(SchemaChangeScript) case rebuild(StructureRebuildPlanRunner.Prepared) internal var displayStatements: [String] { switch self { - case .alter(let statements): - statements.map(\.sql) + case .alter(let script): + script.statements.map(\.sql) case .rebuild(let prepared): prepared.plan.scriptStatements } diff --git a/TablePro/Views/Structure/TableStructureView+ColumnReorder.swift b/TablePro/Views/Structure/TableStructureView+ColumnReorder.swift index 5d6a354371..8c65e71a4a 100644 --- a/TablePro/Views/Structure/TableStructureView+ColumnReorder.swift +++ b/TablePro/Views/Structure/TableStructureView+ColumnReorder.swift @@ -123,7 +123,7 @@ extension TableStructureView { if let clearTarget { coordinator?.clearColumnLayout(clearTarget) } - AppCommands.shared.refreshData.send(DataRefreshRequest(connectionId: connection.id)) + DatabaseManager.shared.reportTableDefinitionChange(table: tableName, in: prepared.scope) CatalogChangeService.post( .changed(CatalogChange(connectionId: connection.id, database: prepared.scope.database, kinds: .tables)) ) diff --git a/TablePro/Views/Structure/TableStructureView+EditGate.swift b/TablePro/Views/Structure/TableStructureView+EditGate.swift index ef5f4e0ccb..5e5942cd36 100644 --- a/TablePro/Views/Structure/TableStructureView+EditGate.swift +++ b/TablePro/Views/Structure/TableStructureView+EditGate.swift @@ -37,6 +37,7 @@ extension TableStructureView { tab: selectedTab, canEditSchema: connection.type.supportsSchemaEditing, hasSelection: !selectedRows.isEmpty, + isSaving: structureChangeManager.isHeldForSave, resolve: { gate.resolve($0) } ) } @@ -56,6 +57,7 @@ extension TableStructureView { /// Why the Columns grid refuses every keystroke, when it does. A grid that will not take an edit /// and says nothing reads as broken, so the pointer carries this as the grid's tooltip. var structureEditRefusal: String? { + if structureChangeManager.isHeldForSave { return StructureFooterPolicy.savingReason } guard !editGate.allowsAnyEdit else { return nil } return editGate.resolve(.renameColumn).unavailableReason } diff --git a/TablePro/Views/Structure/TableStructureView.swift b/TablePro/Views/Structure/TableStructureView.swift index 90f4cebdd8..6e58bb73fd 100644 --- a/TablePro/Views/Structure/TableStructureView.swift +++ b/TablePro/Views/Structure/TableStructureView.swift @@ -268,6 +268,10 @@ struct TableStructureView: View { coordinator?.toolbarState.hasStructureChanges = newValue updateGridDelegate() } + .onChange(of: structureChangeManager.isHeldForSave) { _ in + publishFooterCapability() + updateGridDelegate() + } .onChange(of: session.appliedVersion) { _ in Task { await refreshAfterApply() } } @@ -322,6 +326,19 @@ struct TableStructureView: View { Spacer() } .padding() + .overlay(alignment: .trailing) { + if structureChangeManager.isHeldForSave { + HStack(spacing: 6) { + ProgressView().controlSize(.small) + Text("Saving Changes…") + .font(.callout) + .foregroundStyle(.secondary) + } + .padding(.trailing) + .accessibilityElement(children: .combine) + .accessibilityIdentifier("structure-save-progress") + } + } } // MARK: - Tab Label with Count Badge @@ -488,7 +505,7 @@ struct TableStructureView: View { private var structureGrid: some View { let provider = makeCurrentProvider() - let canEdit = editGate.allowsAnyEdit + let canEdit = editGate.allowsAnyEdit && !structureChangeManager.isHeldForSave let customOptions = provider.customDropdownOptions let allDropdownColumns = provider.dropdownColumns /// Resolved once. It reads the engine's curated capabilities and the object's own kind, and diff --git a/TableProTests/Core/Autocomplete/SQLSchemaProviderTests.swift b/TableProTests/Core/Autocomplete/SQLSchemaProviderTests.swift index d2f02dc3e9..a803aa9818 100644 --- a/TableProTests/Core/Autocomplete/SQLSchemaProviderTests.swift +++ b/TableProTests/Core/Autocomplete/SQLSchemaProviderTests.swift @@ -103,12 +103,40 @@ final class MockDatabaseDriver: DatabaseDriver, SchemaSwitchable, @unchecked Sen } func execute(query: String) async throws -> QueryResult { + executedQueries.append(query) if executeDelaySeconds > 0 { try await Task.sleep(nanoseconds: UInt64(executeDelaySeconds * 1_000_000_000)) } return QueryResult(columns: [], columnTypes: [], rows: [], rowsAffected: 0, executionTime: 0, error: nil) } + private(set) var executedQueries: [String] = [] + var schemaChangeRefusalToReturn: String? + var schemaChangeShortfallToReturn: String? + private(set) var changedTableDefinitions: [String] = [] + + func schemaChangeRefusalBeforeWriting( + table: String, + schema: String?, + operations: [PluginSchemaOperation], + review: PluginSchemaChangeReview + ) async throws -> String? { + schemaChangeRefusalToReturn + } + + func schemaChangeShortfallAfterWriting( + table: String, + schema: String?, + operations: [PluginSchemaOperation], + review: PluginSchemaChangeReview + ) async throws -> String? { + schemaChangeShortfallToReturn + } + + func tableDefinitionDidChange(table: String, schema: String?) { + changedTableDefinitions.append(table) + } + func executeParameterized(query: String, parameters: [Any?]) async throws -> QueryResult { QueryResult(columns: [], columnTypes: [], rows: [], rowsAffected: 0, executionTime: 0, error: nil) } diff --git a/TableProTests/Core/Compare/CompareSyncSampledColumnsTests.swift b/TableProTests/Core/Compare/CompareSyncSampledColumnsTests.swift new file mode 100644 index 0000000000..eb5b71870e --- /dev/null +++ b/TableProTests/Core/Compare/CompareSyncSampledColumnsTests.swift @@ -0,0 +1,53 @@ +// +// CompareSyncSampledColumnsTests.swift +// TableProTests +// +// A MongoDB collection's columns are the fields found in a sample of its documents, so a field the +// source's sample missed reads as one the target holds and the source does not. A structure script +// written from that would be an `$unset` across every document of the target. +// + +import Foundation +@testable import TablePro +import Testing + +@MainActor +struct CompareSyncSampledColumnsTests { + private func endpoint(_ name: String, _ type: DatabaseType) -> DatabaseEndpoint { + DatabaseEndpoint( + scope: DatabaseScope(connectionId: UUID(), database: "shop", schema: nil), + connectionName: name, + databaseType: type, + safeModeLevel: .silent, + color: .blue + ) + } + + @Test("An engine whose columns are sampled never generates a structure script, on either side") + func sampledColumnsGenerateNoScript() { + #expect(!CompareSyncEngineFamily.canGenerateStructureScript(from: .mongodb, to: .mongodb)) + #expect(!CompareSyncEngineFamily.canGenerateStructureScript(from: .postgresql, to: .mongodb)) + #expect(!CompareSyncEngineFamily.canGenerateStructureScript(from: .mongodb, to: .postgresql)) + #expect(CompareSyncEngineFamily.canGenerateStructureScript(from: .postgresql, to: .postgresql)) + } + + @Test("The refusal names the sampled engine rather than a type mismatch") + func refusalNamesTheSample() { + #expect(CompareSyncEngineFamily.structureScriptRefusal(from: .mongodb, to: .mongodb) == String( + format: String(localized: "%@ lists a collection's fields from a sample of its documents, so structures can be compared but no script is generated."), + "MongoDB" + )) + } + + @Test("A structure compare between two MongoDB databases offers no script to build") + func compareSessionOffersNoScript() { + let session = CompareSyncSession(connectionsProvider: { [] }) + session.mode = .structure + session.source = endpoint("staging", .mongodb) + session.target = endpoint("production", .mongodb) + + #expect(!session.canGenerateStructureScript) + #expect(session.crossEngineNotice == CompareSyncEngineFamily.structureScriptRefusal(from: .mongodb, to: .mongodb)) + #expect(!session.canBuildScript) + } +} diff --git a/TableProTests/Core/Database/DatabaseManagerSchemaChangeRoutingTests.swift b/TableProTests/Core/Database/DatabaseManagerSchemaChangeRoutingTests.swift index 8dfa1c0450..65cfc66d99 100644 --- a/TableProTests/Core/Database/DatabaseManagerSchemaChangeRoutingTests.swift +++ b/TableProTests/Core/Database/DatabaseManagerSchemaChangeRoutingTests.swift @@ -57,9 +57,59 @@ private final class SchemaRoutingDriver: SchemaRoutingBaseDriver, PluginDatabase func execute(query: String) async throws -> PluginQueryResult { executedQueries.append(query) + if let failingStatement, query.contains(failingStatement) { + throw DatabaseError.queryFailed("statement timed out") + } return PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) } + var failingStatement: String? + var shortfallAfterWriting: String? + private(set) var statementsRunBeforeShortfallCheck: [Int] = [] + var review = PluginSchemaChangeReview() + var refusalBeforeWriting: String? + private(set) var reviewedOperationCounts: [Int] = [] + private(set) var statementsRunBeforeEachCheck: [Int] = [] + private(set) var reviewsCheckedBeforeWriting: [PluginSchemaChangeReview] = [] + + func reviewSchemaChange( + table: String, + schema: String?, + operations: [PluginSchemaOperation] + ) async throws -> PluginSchemaChangeReview { + reviewedOperationCounts.append(operations.count) + return review + } + + func schemaChangeRefusalBeforeWriting( + table: String, + schema: String?, + operations: [PluginSchemaOperation], + review: PluginSchemaChangeReview + ) async throws -> String? { + statementsRunBeforeEachCheck.append(executedQueries.count) + reviewsCheckedBeforeWriting.append(review) + return refusalBeforeWriting + } + + private(set) var reviewsCheckedAfterWriting: [PluginSchemaChangeReview] = [] + private(set) var changedTableDefinitions: [String] = [] + + func schemaChangeShortfallAfterWriting( + table: String, + schema: String?, + operations: [PluginSchemaOperation], + review: PluginSchemaChangeReview + ) async throws -> String? { + statementsRunBeforeShortfallCheck.append(executedQueries.count) + reviewsCheckedAfterWriting.append(review) + return shortfallAfterWriting + } + + func tableDefinitionDidChange(table: String, schema: String?) { + changedTableDefinitions.append(table) + } + func switchDatabase(to database: String) async throws { if let switchDatabaseError { throw switchDatabaseError @@ -161,12 +211,12 @@ struct DatabaseManagerSchemaChangeRoutingTests { databaseType: DatabaseType, scope: DatabaseScope ) async throws { - let statements = try await DatabaseManager.shared.schemaChangeStatements( + let script = try await DatabaseManager.shared.schemaChangeStatements( tableName: "orders", changes: changes, scope: scope ) - try await DatabaseManager.shared.executeSchemaChanges(statements, databaseType: databaseType, scope: scope) + try await DatabaseManager.shared.executeSchemaChanges(script, databaseType: databaseType, scope: scope) } private static func tearDown(_ connections: DatabaseConnection...) { @@ -199,6 +249,85 @@ struct DatabaseManagerSchemaChangeRoutingTests { #expect(DatabaseManager.shared.schemaChangeRoute(for: serverScope) == .sessionDriver) } + @Test("A review's leading statements show in the script, reach the check before writing as composed, and run first") + func reviewLeadingStatementsRunFirst() async throws { + let (connection, _) = Self.makeSession(savedDatabase: "orders") + defer { Self.tearDown(connection) } + + let scope = try #require(Self.makeScope(connection, database: "orders")) + let pooled = try await Self.seedPooledDriver(connection, scope: scope) + pooled.review = PluginSchemaChangeReview(leadingStatements: ["PREPARE orders"]) + + let script = try await DatabaseManager.shared.schemaChangeStatements( + tableName: "orders", changes: [Self.makeAddColumnChange()], scope: scope + ) + #expect(script.statements.first?.sql == "PREPARE orders;") + #expect(script.statements.first?.isDestructive == false) + #expect(script.operations.count == 1) + #expect(script.review == PluginSchemaChangeReview(leadingStatements: ["PREPARE orders"])) + #expect(pooled.reviewedOperationCounts == [1]) + + pooled.review = PluginSchemaChangeReview(leadingStatements: ["PREPARE orders AGAIN"]) + try await DatabaseManager.shared.executeSchemaChanges(script, databaseType: .mysql, scope: scope) + #expect(pooled.reviewsCheckedBeforeWriting == [PluginSchemaChangeReview(leadingStatements: ["PREPARE orders"])]) + #expect(pooled.executedQueries.count == 2) + #expect(pooled.executedQueries.first == "PREPARE orders;") + #expect(pooled.executedQueries.last?.contains("ADD COLUMN") == true) + } + + @Test("A review refusal stops SQL Preview and Save with the driver's reason before anything runs") + func reviewRefusalStopsTheSave() async throws { + let (connection, _) = Self.makeSession(savedDatabase: "orders") + defer { Self.tearDown(connection) } + + let scope = try #require(Self.makeScope(connection, database: "orders")) + let pooled = try await Self.seedPooledDriver(connection, scope: scope) + pooled.review = PluginSchemaChangeReview(refusal: "Index email_1 uses email.") + + await #expect(throws: SchemaOperationRefusedError(reason: "Index email_1 uses email.")) { + _ = try await DatabaseManager.shared.schemaChangeStatements( + tableName: "orders", changes: [Self.makeAddColumnChange()], scope: scope + ) + } + #expect(pooled.executedQueries.isEmpty) + #expect(pooled.statementsRunBeforeEachCheck.isEmpty) + } + + /// SQL Preview composes the same script as Save, so a check that reads the data belongs only on + /// the path that goes on to write. + @Test("The check that reads the data runs on Save alone, on the writing connection, before any statement") + func checkBeforeWritingRunsOnSaveOnly() async throws { + let (connection, driver) = Self.makeSession(savedDatabase: "orders") + defer { Self.tearDown(connection) } + + let scope = try #require(Self.makeScope(connection, database: "orders")) + let pooled = try await Self.seedPooledDriver(connection, scope: scope) + + let script = try await DatabaseManager.shared.schemaChangeStatements( + tableName: "orders", changes: [Self.makeAddColumnChange()], scope: scope + ) + #expect(pooled.statementsRunBeforeEachCheck.isEmpty) + + try await DatabaseManager.shared.executeSchemaChanges(script, databaseType: .mysql, scope: scope) + #expect(pooled.statementsRunBeforeEachCheck == [0]) + #expect(driver.statementsRunBeforeEachCheck.isEmpty) + } + + @Test("A refusal before writing stops the save before its first statement") + func refusalBeforeWritingStopsTheSave() async throws { + let (connection, _) = Self.makeSession(savedDatabase: "orders") + defer { Self.tearDown(connection) } + + let scope = try #require(Self.makeScope(connection, database: "orders")) + let pooled = try await Self.seedPooledDriver(connection, scope: scope) + pooled.refusalBeforeWriting = "Some documents hold both a and b." + + await #expect(throws: SchemaOperationRefusedError(reason: "Some documents hold both a and b.")) { + try await Self.composeAndSave(changes: [Self.makeAddColumnChange()], databaseType: .mysql, scope: scope) + } + #expect(pooled.executedQueries.isEmpty) + } + @Test("Schema changes run on the requested connection, not the last activated one") func schemaChangeUsesRequestedConnection() async throws { let (connectionA, driverA) = Self.makeSession(savedDatabase: "alpha") @@ -334,32 +463,207 @@ struct DatabaseManagerSchemaChangeRoutingTests { #expect(driver.executedQueries.isEmpty) } - @Test("A save broadcasts a refresh scoped to the edited tab, not to the browse cursor") - func schemaChangeBroadcastsTheEditedScope() async throws { + /// A scope-wide refresh reloaded whichever tab each window had selected in the database, and + /// asked it to discard its edits, whatever table it showed, while a background tab on the saved + /// table and the saving tab's own Data view kept their rows. + @Test("A save reports its own table in the edited tab's scope, and nothing scope-wide") + func schemaChangeReportsItsTable() async throws { let (connection, _) = Self.makeSession( savedDatabase: "analytics", browseDatabase: "inventory" ) defer { Self.tearDown(connection) } - let recorder = RefreshRequestRecorder() - let cancellable = AppCommands.shared.refreshData.sink { request in - recorder.record(request) + let scope = try #require(Self.makeScope(connection, database: "orders")) + _ = try await Self.seedPooledDriver(connection, scope: scope) + let outcome = await Self.recordChanges { + try await Self.composeAndSave(changes: [Self.makeAddColumnChange()], databaseType: .mysql, scope: scope) } - defer { cancellable.cancel() } + + #expect(outcome.error == nil) + let changes = outcome.changes.filter { $0.connectionId == connection.id } + #expect(changes == [DatabaseObjectChange(connectionId: connection.id, scope: scope, name: "orders", kind: .structure)]) + #expect(!outcome.refreshes.contains { $0.connectionId == connection.id }) + } + + /// The session driver is not the one the save ran on, and it keeps what it learned about the + /// table's columns: a MongoDB driver typed a renamed field's writes and later pages by the old + /// name until a first page read them again. + @Test("The session's own driver is told the table changed after a save that wrote, and never after a refusal") + func sessionDriverForgetsTheTableDefinition() async throws { + let (connection, driver) = Self.makeSession(savedDatabase: "orders") + defer { Self.tearDown(connection) } let scope = try #require(Self.makeScope(connection, database: "orders")) - _ = try await Self.seedPooledDriver(connection, scope: scope) - try await Self.composeAndSave( - changes: [Self.makeAddColumnChange()], - databaseType: .mysql, - scope: scope + let pooled = try await Self.seedPooledDriver(connection, scope: scope) + try await Self.composeAndSave(changes: [Self.makeAddColumnChange()], databaseType: .mysql, scope: scope) + #expect(driver.changedTableDefinitions == ["orders"]) + #expect(pooled.changedTableDefinitions.isEmpty) + + pooled.failingStatement = "ADD COLUMN" + _ = await Self.recordChanges { + try await Self.composeAndSave(changes: [Self.makeAddColumnChange()], databaseType: .mysql, scope: scope) + } + #expect(driver.changedTableDefinitions == ["orders", "orders"]) + + pooled.failingStatement = nil + pooled.refusalBeforeWriting = "Some documents hold both a and b." + _ = await Self.recordChanges { + try await Self.composeAndSave(changes: [Self.makeAddColumnChange()], databaseType: .mysql, scope: scope) + } + #expect(driver.changedTableDefinitions == ["orders", "orders"]) + } + + @Test("The check after writing is handed the review the save was composed with") + func checkAfterWritingGetsTheComposedReview() async throws { + let (connection, _) = Self.makeSession(savedDatabase: "orders") + defer { Self.tearDown(connection) } + + let scope = try #require(Self.makeScope(connection, database: "orders")) + let pooled = try await Self.seedPooledDriver(connection, scope: scope) + let composed = PluginSchemaChangeReview(leadingStatements: ["PREPARE orders"], basis: "entry 1") + pooled.review = composed + let script = try await DatabaseManager.shared.schemaChangeStatements( + tableName: "orders", changes: [Self.makeAddColumnChange()], scope: scope + ) + pooled.review = PluginSchemaChangeReview(basis: "entry 2") + try await DatabaseManager.shared.executeSchemaChanges(script, databaseType: .mysql, scope: scope) + + #expect(pooled.reviewsCheckedBeforeWriting == [composed]) + #expect(pooled.reviewsCheckedAfterWriting == [composed]) + } + + /// The save's questions went to the driver through a cast to the plugin adapter, so a + /// `DatabaseDriver` of any other kind was never asked and its save ran unchecked. + @Test("A DatabaseDriver that is not a plugin adapter is asked before and after writing") + func databaseDriverAnswersThroughTheProtocol() async throws { + let connection = TestFixtures.makeConnection(database: "orders", type: Self.singleConnectionType) + let mock = MockDatabaseDriver(connection: connection) + var session = ConnectionSession(connection: connection, driver: mock) + session.browseDatabase = "orders" + DatabaseManager.shared.injectSession(session, for: connection.id) + defer { Self.tearDown(connection) } + + let scope = try #require(Self.makeScope(connection, database: "orders")) + let script = SchemaChangeScript( + tableName: "orders", + statements: [SchemaStatement(sql: "ALTER orders", description: "Alter", isDestructive: false)], + operations: [], + review: PluginSchemaChangeReview() ) + mock.schemaChangeRefusalToReturn = "Refused by the driver." + await #expect(throws: SchemaOperationRefusedError(reason: "Refused by the driver.")) { + try await DatabaseManager.shared.executeSchemaChanges(script, databaseType: Self.singleConnectionType, scope: scope) + } + #expect(!mock.executedQueries.contains("ALTER orders")) + + mock.schemaChangeRefusalToReturn = nil + mock.schemaChangeShortfallToReturn = "Not finished." + let outcome = await Self.recordChanges { + try await DatabaseManager.shared.executeSchemaChanges(script, databaseType: Self.singleConnectionType, scope: scope) + } + #expect(outcome.error?.localizedDescription == "Not finished.") + #expect(mock.executedQueries.contains("ALTER orders")) + #expect(mock.changedTableDefinitions == ["orders"]) + } + + private static func recordChanges( + during save: () async throws -> Void + ) async -> (changes: [DatabaseObjectChange], refreshes: [DataRefreshRequest], error: Error?) { + let recorder = ChangeRecorder() + let changes = AppCommands.shared.objectChanged.sink { recorder.record($0) } + let refreshes = AppCommands.shared.refreshData.sink { recorder.record($0) } + defer { + changes.cancel() + refreshes.cancel() + } + do { + try await save() + return (recorder.changes, recorder.refreshes, nil) + } catch { + return (recorder.changes, recorder.refreshes, error) + } + } + + private static func structureChange(_ connection: DatabaseConnection, _ scope: DatabaseScope) -> DatabaseObjectChange { + DatabaseObjectChange(connectionId: connection.id, scope: scope, name: "orders", kind: .structure) + } + + /// MongoDB keeps every document an `updateMany` changed before it stopped, and MySQL commits + /// each DDL statement as it runs, so the rows on screen can describe a table that moved on. + @Test("A statement that fails once the save has started writing reloads the rows") + func failureAfterWritingReloadsRows() async throws { + let (connection, _) = Self.makeSession(savedDatabase: "orders") + defer { Self.tearDown(connection) } + + let scope = try #require(Self.makeScope(connection, database: "orders")) + let pooled = try await Self.seedPooledDriver(connection, scope: scope) + pooled.review = PluginSchemaChangeReview(leadingStatements: ["PREPARE orders"]) + pooled.failingStatement = "ADD COLUMN" - let broadcast = recorder.requests.filter { $0.connectionId == connection.id } - #expect(broadcast.count == 1) - #expect(broadcast.first?.scope == scope) - #expect(broadcast.first?.scope?.database == "orders") + let outcome = await Self.recordChanges { + try await Self.composeAndSave(changes: [Self.makeAddColumnChange()], databaseType: .mysql, scope: scope) + } + let error = try #require(outcome.error) + #expect(error is DatabaseError) + #expect(error.localizedDescription.contains("statement timed out")) + #expect(pooled.executedQueries.count == 2) + #expect(pooled.statementsRunBeforeShortfallCheck.isEmpty) + #expect(outcome.changes.filter { $0.connectionId == connection.id } == [Self.structureChange(connection, scope)]) + } + + @Test("A shortfall found after the last statement fails the save with the driver's reason and reloads the rows") + func shortfallAfterWritingFailsTheSave() async throws { + let (connection, _) = Self.makeSession(savedDatabase: "orders") + defer { Self.tearDown(connection) } + + let scope = try #require(Self.makeScope(connection, database: "orders")) + let pooled = try await Self.seedPooledDriver(connection, scope: scope) + let reason = "The save did not finish: one document still holds old." + pooled.shortfallAfterWriting = reason + + let outcome = await Self.recordChanges { + try await Self.composeAndSave(changes: [Self.makeAddColumnChange()], databaseType: .mysql, scope: scope) + } + let error = try #require(outcome.error) + #expect(error is DatabaseError) + #expect(error.localizedDescription == reason) + #expect(pooled.statementsRunBeforeShortfallCheck == [1]) + #expect(outcome.changes.filter { $0.connectionId == connection.id } == [Self.structureChange(connection, scope)]) + } + + @Test("The check after writing runs once every statement has run, and a finished save reloads the rows once") + func finishedSaveChecksAfterItsLastStatement() async throws { + let (connection, _) = Self.makeSession(savedDatabase: "orders") + defer { Self.tearDown(connection) } + + let scope = try #require(Self.makeScope(connection, database: "orders")) + let pooled = try await Self.seedPooledDriver(connection, scope: scope) + pooled.review = PluginSchemaChangeReview(leadingStatements: ["PREPARE orders"]) + + let outcome = await Self.recordChanges { + try await Self.composeAndSave(changes: [Self.makeAddColumnChange()], databaseType: .mysql, scope: scope) + } + #expect(outcome.error == nil) + #expect(pooled.statementsRunBeforeShortfallCheck == [2]) + #expect(outcome.changes.filter { $0.connectionId == connection.id }.count == 1) + } + + @Test("A save refused before writing reloads nothing") + func refusalBeforeWritingReloadsNothing() async throws { + let (connection, _) = Self.makeSession(savedDatabase: "orders") + defer { Self.tearDown(connection) } + + let scope = try #require(Self.makeScope(connection, database: "orders")) + let pooled = try await Self.seedPooledDriver(connection, scope: scope) + pooled.refusalBeforeWriting = "Some documents hold both a and b." + + let outcome = await Self.recordChanges { + try await Self.composeAndSave(changes: [Self.makeAddColumnChange()], databaseType: .mysql, scope: scope) + } + #expect(outcome.error is SchemaOperationRefusedError) + #expect(pooled.statementsRunBeforeShortfallCheck.isEmpty) + #expect(!outcome.changes.contains { $0.connectionId == connection.id }) } private static func invoicesDefinition() -> PluginCreateTableDefinition { @@ -446,10 +750,15 @@ struct DatabaseManagerSchemaChangeRoutingTests { } @MainActor -private final class RefreshRequestRecorder { - private(set) var requests: [DataRefreshRequest] = [] +private final class ChangeRecorder { + private(set) var changes: [DatabaseObjectChange] = [] + private(set) var refreshes: [DataRefreshRequest] = [] + + func record(_ change: DatabaseObjectChange) { + changes.append(change) + } func record(_ request: DataRefreshRequest) { - requests.append(request) + refreshes.append(request) } } diff --git a/TableProTests/Core/Database/SchemaOperationKindTests.swift b/TableProTests/Core/Database/SchemaOperationKindTests.swift new file mode 100644 index 0000000000..bd1f02082a --- /dev/null +++ b/TableProTests/Core/Database/SchemaOperationKindTests.swift @@ -0,0 +1,40 @@ +// +// SchemaOperationKindTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +struct SchemaOperationKindTests { + private func kind(_ sql: String, destructive: Bool, _ type: DatabaseType) -> OperationKind { + let statement = SchemaStatement(sql: sql, description: "change", isDestructive: destructive) + return DatabaseManager.schemaOperationKind(for: [statement], combinedSQL: sql, databaseType: type) + } + + @Test("A removed MongoDB field is destructive, though its statement is an update") + func removedFieldIsDestructive() { + let sql = #"db.users.updateMany({"legacy": {"$exists": true}}, {"$unset": {"legacy": ""}});"# + #expect(kind(sql, destructive: true, .mongodb) == .destructiveQuery) + } + + @Test("A renamed MongoDB field and the collMod ahead of it stay a schema change") + func renamedFieldIsNotDestructive() { + let rename = #"db.users.updateMany({"a": {"$exists": true}, "b": {"$exists": false}}, {"$rename": {"a": "b"}});"# + let validator = #"db.runCommand({"collMod": "users", "validator": {"$jsonSchema": {}}});"# + #expect(kind(rename, destructive: false, .mongodb) == .schemaMutation) + #expect(kind(validator, destructive: false, .mongodb) == .schemaMutation) + } + + @Test("A SQL DROP COLUMN is destructive from its text, as before") + func sqlDropColumn() { + #expect(kind("ALTER TABLE t DROP COLUMN c;", destructive: false, .postgresql) == .destructiveQuery) + #expect(kind("ALTER TABLE t ADD COLUMN c int;", destructive: false, .postgresql) == .schemaMutation) + } + + @Test("A SQL column type change is destructive, as the Structure tab already marks it") + func sqlTypeChange() { + #expect(kind("ALTER TABLE t ALTER COLUMN c TYPE bigint;", destructive: true, .postgresql) == .destructiveQuery) + } +} diff --git a/TableProTests/Core/MongoDB/MongoScriptCommandTests.swift b/TableProTests/Core/MongoDB/MongoScriptCommandTests.swift index 6fcd19fb9d..32daa8af40 100644 --- a/TableProTests/Core/MongoDB/MongoScriptCommandTests.swift +++ b/TableProTests/Core/MongoDB/MongoScriptCommandTests.swift @@ -187,6 +187,28 @@ struct MongoScriptCommandBuilderTests { #expect(command.contains("\"arrayFilters\":")) } + /// mongosh sends `updateMany(filter, update, {writeConcern})` with that write concern. The + /// shell dropped the option, and `mongoc_client_command_simple` adds none of its own, so the + /// update went out with the server's default. + @Test("An update's writeConcern option goes on the command, not inside the update statement") + func updateWriteConcern() throws { + let command = MongoScriptCommandBuilder.update( + collection: "orders", filter: "{}", update: "{\"$set\":{\"b\":2}}", multi: true, + options: ["writeConcern": ["w": "majority", "wtimeout": 5_000]] + ) + let parsed = try #require( + try JSONSerialization.jsonObject(with: Data(command.utf8)) as? [String: Any] + ) + let concern = try #require(parsed["writeConcern"] as? [String: Any]) + #expect(concern["w"] as? String == "majority") + #expect((concern["wtimeout"] as? NSNumber)?.intValue == 5_000) + let statement = try #require((parsed["updates"] as? [[String: Any]])?.first) + #expect(statement["writeConcern"] == nil) + #expect(!MongoScriptCommandBuilder.update( + collection: "orders", filter: "{}", update: "{}", multi: true, options: [:] + ).contains("writeConcern")) + } + @Test("deleteOne limits to one document and deleteMany to none") func deleteLimits() { #expect( @@ -294,7 +316,9 @@ struct MongoScriptObjectIdTests { @Test("Two generated ids differ") func uniqueness() { - #expect(MongoScriptObjectId.generate() != MongoScriptObjectId.generate()) + let first = MongoScriptObjectId.generate() + let second = MongoScriptObjectId.generate() + #expect(first != second) } @Test("The leading four bytes are the current time") diff --git a/TableProTests/Core/Plugins/PluginMetadataRegistryCuratedCapabilityTests.swift b/TableProTests/Core/Plugins/PluginMetadataRegistryCuratedCapabilityTests.swift index 347cfd6010..cd3c8bcf99 100644 --- a/TableProTests/Core/Plugins/PluginMetadataRegistryCuratedCapabilityTests.swift +++ b/TableProTests/Core/Plugins/PluginMetadataRegistryCuratedCapabilityTests.swift @@ -144,6 +144,20 @@ struct PluginMetadataRegistryCuratedCapabilityTests { #expect(built.capabilities.authenticationIsDatabaseScoped == true) } + @Test("MongoDB keeps its sampled columns and its structure matrix when its plugin registers") + func mongoDBKeepsSampledColumnsAndStructureMatrix() { + let registry = PluginMetadataRegistry.shared + + let built = registry.buildMetadataSnapshot(from: MockMongoDBPlugin.self) + + #expect( + built.capabilities.columnsAreSampled == true, + "A structure sync would read a field missing from one sample as a field to remove" + ) + #expect(StructureEditEligibility.allows(.renameColumn, on: .table, matrix: built.structureEditing.structureEdits)) + #expect(StructureEditEligibility.allows(.dropColumn, on: .table, matrix: built.structureEditing.structureEdits)) + } + @Test("DynamoDB keeps its billed-scan count when its plugin registers") func dynamoDBKeepsItsBilledScanCount() { let registry = PluginMetadataRegistry.shared @@ -216,6 +230,7 @@ struct PluginMetadataRegistryCuratedCapabilityTests { #expect(built.capabilities.authenticationIsDatabaseScoped == false) #expect(built.capabilities.browsingRequiresSelectedDatabase == false) #expect(built.capabilities.exactRowCountIsBilledScan == false) + #expect(built.capabilities.columnsAreSampled == false) #expect(built.schema.implicitSchemaName == nil) } } diff --git a/TableProTests/Core/SchemaTracking/SchemaOperationRefusalTests.swift b/TableProTests/Core/SchemaTracking/SchemaOperationRefusalTests.swift index 8b8c2ec94f..88d6daf079 100644 --- a/TableProTests/Core/SchemaTracking/SchemaOperationRefusalTests.swift +++ b/TableProTests/Core/SchemaTracking/SchemaOperationRefusalTests.swift @@ -160,6 +160,55 @@ struct SchemaOperationRefusalTests { #expect(refusal(of: .deleteIndex(index("ix_old", type: .btree)), driver: driver) == "drop ix_old") } + @Test("A changed and a removed column reach the driver as their own operations") + func modifyAndDropColumnReachTheDriver() { + let driver = RefusingDDLDriver() + driver.refuse = { operation in + switch operation { + case .modifyColumn(let old, let new): return "rename \(old.name) to \(new.name)" + case .dropColumn(let column): return "drop \(column.name)" + default: return nil + } + } + let rename = SchemaChange.modifyColumn(old: column("qty", generated: false), new: column("quantity", generated: false)) + + #expect(refusal(of: rename, driver: driver) == "rename qty to quantity") + #expect(refusal(of: .deleteColumn(column("total", generated: false)), driver: driver) == "drop total") + } + + @Test("A save's operations reach the driver in the order its statements run") + func operationsFollowStatementOrder() { + let generator = SchemaStatementGenerator(tableName: "orders", pluginDriver: RefusingDDLDriver()) + let operations = generator.orderedOperations(for: [ + .addIndex(index("ix_new", type: .btree)), + .modifyColumn(old: column("a", generated: false), new: column("b", generated: false)), + .addColumn(column("c", generated: false)), + .deleteColumn(column("d", generated: false)), + .deleteIndex(index("ix_old", type: .btree)) + ]) + let names: [String] = operations.map { operation in + switch operation { + case .dropIndex(let index): return "dropIndex \(index.name)" + case .dropColumn(let column): return "dropColumn \(column.name)" + case .modifyColumn(let old, let new): return "modifyColumn \(old.name) \(new.name)" + case .addColumn(let column): return "addColumn \(column.name)" + case .addIndex(let index): return "addIndex \(index.name)" + default: return "other" + } + } + #expect(names == ["dropIndex ix_old", "dropColumn d", "modifyColumn a b", "addColumn c", "addIndex ix_new"]) + } + + @Test("Only a renamed check constraint is an operation, and key changes are none") + func constraintAndKeyChangesAreNotOperations() { + let renamed = SchemaChange.modifyCheckConstraint(old: constraint("ck_a", "qty > 0"), new: constraint("ck_b", "qty > 0")) + let rewritten = SchemaChange.modifyCheckConstraint(old: constraint("ck_a", "qty > 0"), new: constraint("ck_a", "qty > 1")) + #expect(SchemaOperationRefusal.operations(for: renamed).count == 1) + #expect(SchemaOperationRefusal.operations(for: rewritten).isEmpty) + #expect(SchemaOperationRefusal.operations(for: .addCheckConstraint(constraint("ck", "qty > 0"))).isEmpty) + #expect(SchemaOperationRefusal.operations(for: .modifyPrimaryKey(old: ["a"], new: ["b"])).isEmpty) + } + @Test("A refusal is reported ahead of a change in the same save that the driver cannot generate") func refusalWinsOverUngeneratableChange() { let unsupportedDrop = SchemaChange.deleteIndex(index("ix_old", type: .btree)) diff --git a/TableProTests/Core/SchemaTracking/StructureChangeManagerSaveHoldTests.swift b/TableProTests/Core/SchemaTracking/StructureChangeManagerSaveHoldTests.swift new file mode 100644 index 0000000000..d05335aa0a --- /dev/null +++ b/TableProTests/Core/SchemaTracking/StructureChangeManagerSaveHoldTests.swift @@ -0,0 +1,141 @@ +// +// StructureChangeManagerSaveHoldTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +/// A save writes the edits staged when it was pressed and clears them when it lands. Staging stayed +/// open in between, so on MongoDB, where the save reads every document it changes before writing, +/// an edit made during that read was left out of the script and then cleared with the edits that +/// did run. +@MainActor +struct StructureChangeManagerSaveHoldTests { + private func makeManager() -> StructureChangeManager { + let manager = StructureChangeManager() + manager.loadSchema( + tableName: "users", + columns: [ + ColumnInfo(name: "id", dataType: "INT", isNullable: false, isPrimaryKey: true, + defaultValue: nil, extra: nil, charset: nil, collation: nil, comment: nil), + ColumnInfo(name: "name", dataType: "TEXT", isNullable: true, isPrimaryKey: false, + defaultValue: nil, extra: nil, charset: nil, collation: nil, comment: nil), + ColumnInfo(name: "email", dataType: "TEXT", isNullable: true, isPrimaryKey: false, + defaultValue: nil, extra: nil, charset: nil, collation: nil, comment: nil) + ], + indexes: [], + foreignKeys: [], + primaryKey: ["id"] + ) + return manager + } + + private func rename(_ manager: StructureChangeManager, at row: Int, to name: String) { + var column = manager.workingColumns[row] + column.name = name + manager.updateColumn(id: column.id, with: column) + } + + @Test("A hold takes exactly what is staged, and a second one is refused") + func holdTakesTheStagedEdits() throws { + let manager = makeManager() + rename(manager, at: 1, to: "full_name") + let staged = manager.getChangesArray() + + let hold = try #require(manager.holdForSave()) + + #expect(hold.changes == staged) + #expect(manager.isHeldForSave) + #expect(manager.holdForSave() == nil) + } + + @Test("Nothing staged is nothing to hold") + func nothingStagedIsNotHeld() { + let manager = makeManager() + #expect(manager.holdForSave() == nil) + #expect(!manager.isHeldForSave) + } + + @Test("Every way of staging, undoing, discarding or reloading is refused while a save holds the edits") + func everyStagingPathIsRefusedWhileHeld() throws { + let manager = makeManager() + rename(manager, at: 1, to: "full_name") + let staged = manager.getChangesArray() + let working = manager.workingColumns + _ = try #require(manager.holdForSave()) + + manager.addNewColumn() + manager.addNewIndex() + manager.addNewForeignKey() + manager.addNewCheckConstraint() + manager.addColumn(EditableColumnDefinition.placeholder()) + rename(manager, at: 2, to: "mail") + manager.deleteColumn(id: manager.workingColumns[2].id) + manager.performAsOneUndoStep { manager.deleteColumn(id: manager.workingColumns[0].id) } + manager.undoDelete(for: .columns, at: 2) + manager.undo() + manager.redo() + manager.discardChanges() + manager.loadSchema(tableName: "users", columns: [], indexes: [], foreignKeys: [], primaryKey: []) + + #expect(manager.getChangesArray() == staged) + #expect(manager.workingColumns == working) + #expect(manager.tableName == "users") + #expect(!manager.canUndo) + #expect(!manager.canRedo) + } + + @Test("A save that wrote clears the edits it held and opens staging again") + func writtenSaveClearsWhatItHeld() throws { + let manager = makeManager() + rename(manager, at: 1, to: "full_name") + let hold = try #require(manager.holdForSave()) + + #expect(manager.releaseHold(hold, written: true)) + #expect(!manager.hasChanges) + #expect(!manager.isHeldForSave) + + rename(manager, at: 2, to: "mail") + #expect(manager.hasChanges) + } + + @Test("A save that did not write leaves the edits staged and editable") + func unwrittenSaveKeepsTheEdits() throws { + let manager = makeManager() + rename(manager, at: 1, to: "full_name") + let staged = manager.getChangesArray() + let hold = try #require(manager.holdForSave()) + + #expect(!manager.releaseHold(hold, written: false)) + #expect(manager.getChangesArray() == staged) + #expect(!manager.isHeldForSave) + #expect(manager.canUndo) + + rename(manager, at: 2, to: "mail") + #expect(manager.getChangesArray().count == 2) + } + + /// A completion that arrives for a hold that has already ended must not clear what was staged + /// after it. Only the hold a save still owns can clear, and only the edits it took. + @Test("An ended hold cannot clear edits staged after it") + func endedHoldClearsNothing() throws { + let manager = makeManager() + rename(manager, at: 1, to: "full_name") + let hold = try #require(manager.holdForSave()) + manager.releaseHold(hold, written: false) + rename(manager, at: 2, to: "mail") + let staged = manager.getChangesArray() + + #expect(!manager.releaseHold(hold, written: true)) + #expect(manager.getChangesArray() == staged) + + let second = try #require(manager.holdForSave()) + #expect(!manager.releaseHold(hold, written: true)) + #expect(manager.isHeldForSave) + #expect(manager.releaseHold(second, written: true)) + #expect(!manager.hasChanges) + } +} diff --git a/TableProTests/Plugins/MongoDBStructureEditingParityTests.swift b/TableProTests/Plugins/MongoDBStructureEditingParityTests.swift new file mode 100644 index 0000000000..83414ef1bc --- /dev/null +++ b/TableProTests/Plugins/MongoDBStructureEditingParityTests.swift @@ -0,0 +1,37 @@ +// +// MongoDBStructureEditingParityTests.swift +// TableProTests +// +// The app describes MongoDB from a curated copy of the plugin's statics until the registry plugin +// loads, and plugins never load under XCTest, so every Structure tab test reads the copy. The +// plugin reads its flags from `MongoDBStructureEditing`, which is compiled into this target, so the +// two are compared here instead of drifting until the plugin silently replaces one with the other. +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +struct MongoDBStructureEditingParityTests { + private func curated() throws -> PluginMetadataSnapshot { + try #require(PluginMetadataRegistry.shared.builtInDefaults().first { $0.typeId == "MongoDB" }?.snapshot) + } + + @Test("Structure capabilities match the plugin") + func structureCapabilities() throws { + let snapshot = try curated() + + #expect(snapshot.supportsSchemaEditing == MongoDBStructureEditing.supportsSchemaEditing) + #expect(snapshot.capabilities.supportsAddColumn == MongoDBStructureEditing.supportsAddColumn) + #expect(snapshot.capabilities.supportsModifyColumn == MongoDBStructureEditing.supportsModifyColumn) + #expect(snapshot.capabilities.supportsDropColumn == MongoDBStructureEditing.supportsDropColumn) + #expect(snapshot.capabilities.supportsAddIndex == MongoDBStructureEditing.supportsAddIndex) + #expect(snapshot.capabilities.supportsDropIndex == MongoDBStructureEditing.supportsDropIndex) + } + + @Test("The curated copy says a collection's columns are a sample") + func columnsAreSampled() throws { + #expect(try curated().capabilities.columnsAreSampled) + } +} diff --git a/TableProTests/Plugins/MongoFieldChangeAssessmentTests.swift b/TableProTests/Plugins/MongoFieldChangeAssessmentTests.swift new file mode 100644 index 0000000000..755e82cd60 --- /dev/null +++ b/TableProTests/Plugins/MongoFieldChangeAssessmentTests.swift @@ -0,0 +1,403 @@ +// +// MongoFieldChangeAssessmentTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +/// Every `listCollections` and `listIndexes` fixture is the canonical Extended JSON MongoDB 7.0.43 +/// returned, byte for byte. `made` is a collection New Table created. +struct MongoFieldChangeAssessmentTests { + private enum Fixture { + static let made = #"{ "name" : "made", "type" : "collection", "# + + #""options" : { "validator" : { "$jsonSchema" : { "bsonType" : "object", "required" : [ "title" ], "# + + #""properties" : { "_id" : { "bsonType" : "objectId" }, "title" : { "bsonType" : "string" }, "# + + #""qty" : { "bsonType" : [ "int", "null" ] }, "note" : { "bsonType" : [ "string", "null" ] } } } } }, "# + + #""info" : { "readOnly" : false, "uuid" : { "$binary" : { "base64" : "uO9RAKO/Q++gkdBY5B0Y3w==", "# + + #""subType" : "04" } } }, "idIndex" : { "v" : { "$numberInt" : "2" }, "# + + #""key" : { "_id" : { "$numberInt" : "1" } }, "name" : "_id_" } }"# + static let plain = #"{ "name" : "fx", "type" : "collection", "options" : { }, "info" : { "readOnly" : false, "# + + #""uuid" : { "$binary" : { "base64" : "NmEwJ/xDRUGIfXfL+Gqw/A==", "subType" : "04" } } }, "# + + #""idIndex" : { "v" : { "$numberInt" : "2" }, "key" : { "_id" : { "$numberInt" : "1" } }, "# + + #""name" : "_id_" } }"# + static let moderateWarn = #"{ "name" : "fxv", "type" : "collection", "# + + #""options" : { "validator" : { "$jsonSchema" : { "bsonType" : "object", "required" : [ "name" ], "# + + #""properties" : { "name" : { "bsonType" : "string" }, "status" : { "enum" : [ "active", "gone" ] }, "# + + #""active" : { "bsonType" : "bool" } } } }, "validationLevel" : "moderate", "# + + #""validationAction" : "warn" }, "info" : { "readOnly" : false, "# + + #""uuid" : { "$binary" : { "base64" : "k3km2BjsQWqHkYR1Le6t9g==", "subType" : "04" } } }, "# + + #""idIndex" : { "v" : { "$numberInt" : "2" }, "key" : { "_id" : { "$numberInt" : "1" } }, "# + + #""name" : "_id_" } }"# + static let timeseries = #"{ "name" : "fxts", "type" : "timeseries", "options" : { "timeseries" : { "timeField" : "t", "# + + #""metaField" : "meta", "granularity" : "seconds", "# + + #""bucketMaxSpanSeconds" : { "$numberInt" : "3600" } } }, "info" : { "readOnly" : false } }"# + static let view = #"{ "name" : "activeOrders", "type" : "view", "options" : { "viewOn" : "fxo", "# + + #""pipeline" : [ { "$match" : { "status" : "active" } }, "# + + #"{ "$project" : { "total" : { "$numberInt" : "1" } } } ] }, "info" : { "readOnly" : true } }"# + static let byTotal = #"{ "name" : "byTotal", "type" : "view", "options" : { "viewOn" : "activeOrders", "# + + #""pipeline" : [ { "$sort" : { "total" : { "$numberInt" : "-1" } } } ] }, "# + + #""info" : { "readOnly" : true } }"# + static let joined = #"{ "name" : "joined", "type" : "view", "options" : { "viewOn" : "products", "# + + #""pipeline" : [ { "$lookup" : { "from" : "fxo", "localField" : "sku", "foreignField" : "sku", "# + + #""as" : "o" } } ] }, "info" : { "readOnly" : true } }"# + static let emailIndex = #"{ "v" : { "$numberInt" : "2" }, "key" : { "email" : { "$numberInt" : "1" } }, "name" : "email_1", "unique" : true }"# + static let encrypted = #"{ "name" : "sec", "type" : "collection", "# + + #""options" : { "encryptedFields" : { "fields" : [ { "path" : "ssn", "bsonType" : "string" } ] } } }"# + static let capped = #"{ "name" : "fxcap", "type" : "collection", "# + + #""options" : { "capped" : true, "size" : { "$numberInt" : "4096" } }, "info" : { "readOnly" : false, "# + + #""uuid" : { "$binary" : { "base64" : "atlITu49RhecdL/EpQ54nw==", "subType" : "04" } } }, "# + + #""idIndex" : { "v" : { "$numberInt" : "2" }, "key" : { "_id" : { "$numberInt" : "1" } }, "# + + #""name" : "_id_" } }"# + static let madeWithExtra = made.replacingOccurrences( + of: #""note" : "#, + with: #""extra" : { "bsonType" : "string" }, "note" : "# + ) + } + + private func info(_ json: String?, _ name: String = "c") -> MongoCollectionInfo { + MongoCollectionInfo(collection: name, infoJson: json) + } + + private func assess( + _ changes: [MongoFieldChange], + info: MongoCollectionInfo, + indexes: [String] = [], + searchIndexes: [String] = [], + views: [String] = [] + ) -> MongoFieldChangeAssessment { + MongoFieldChangeAssessment.assess( + changes, + info: info, + indexes: indexes.compactMap(MongoIndexSpec.init(json:)), + searchIndexes: searchIndexes.compactMap(MongoSearchIndex.init(json:)), + views: views.compactMap(MongoViewDefinition.init(json:)) + ) + } + + // MARK: - Catalog + + @Test("A collection's kind, validator, level and action are read from listCollections") + func readsCollectionInfo() { + let validated = info(Fixture.moderateWarn, "fxv") + #expect(validated.kind == .collection) + #expect(validated.validationLevel == "moderate") + #expect(validated.validationAction == "warn") + #expect(validated.validatorJson?.hasPrefix(#"{ "$jsonSchema" : { "bsonType" : "object""#) == true) + #expect(!validated.enforcesValidator) + + let plain = info(Fixture.plain, "fx") + #expect(plain.validatorJson == nil) + #expect(plain.validationLevel == "strict") + #expect(plain.validationAction == "error") + + #expect(info(Fixture.made, "made").enforcesValidator) + #expect(info(Fixture.timeseries, "fxts").kind == .timeseries) + #expect(info(Fixture.view, "activeOrders").kind == .view) + #expect(info(nil, "gone").kind == .missing) + #expect(info(Fixture.encrypted, "sec").encryptedFieldPaths == ["ssn"]) + #expect(info(Fixture.capped, "fxcap").isCapped) + #expect(!plain.isCapped) + #expect(!info(nil, "gone").isCapped) + } + + @Test("Views that read a collection are found through viewOn chains and joins, in any listing order") + func viewDependents() throws { + let views = [Fixture.byTotal, Fixture.view, Fixture.joined].compactMap(MongoViewDefinition.init(json:)) + let dependents = MongoViewDefinition.dependents(of: "fxo", among: views).map(\.name) + #expect(Set(dependents) == ["activeOrders", "byTotal", "joined"]) + #expect(MongoViewDefinition.dependents(of: "unrelated", among: views).isEmpty) + } + + @Test("A union and a graph lookup make a view depend on the collection they name") + func unionAndGraphLookup() throws { + let union = MongoViewDefinition(name: "u", viewOn: "a", pipeline: [["$unionWith": "c"]]) + let unionSpec = MongoViewDefinition(name: "us", viewOn: "a", pipeline: [["$unionWith": ["coll": "c", "pipeline": []]]]) + let graph = MongoViewDefinition(name: "g", viewOn: "a", pipeline: [["$facet": ["x": [["$graphLookup": ["from": "c"]]]]]]) + let dependents = MongoViewDefinition.dependents(of: "c", among: [union, unionSpec, graph]).map(\.name) + #expect(Set(dependents) == ["u", "us", "g"]) + } + + // MARK: - Kinds + + @Test("Only a plain collection has fields to rename") + func kindsAreRefused() { + let change = [MongoFieldChange.rename(from: "v", to: "w")] + #expect(assess(change, info: info(Fixture.timeseries, "fxts")).refusal != nil) + #expect(assess(change, info: info(Fixture.view, "activeOrders")).refusal != nil) + #expect(assess(change, info: info(nil, "gone")).refusal == String(format: String(localized: "Collection %@ no longer exists."), "gone")) + #expect(assess(change, info: info(Fixture.plain, "system.buckets.ts")).refusal != nil) + #expect(assess(change, info: info(Fixture.plain, "fx")).refusal == nil) + } + + // MARK: - Capped collections + + @Test("A capped collection refuses a rename that makes the name longer in bytes, and nothing else") + func cappedCollectionsRefuseGrowth() throws { + let capped = info(Fixture.capped, "fxcap") + let grown = try #require(assess([.rename(from: "abc", to: "abcd")], info: capped).refusal) + #expect(grown == String( + format: String(localized: "%1$@ is a capped collection, and a longer field name makes MongoDB delete its oldest documents. Choose a name no longer than %2$@."), + "fxcap", "abc" + )) + #expect(assess([.rename(from: "ab", to: "\u{1EC5}")], info: capped).refusal != nil) + #expect(assess([.rename(from: "\u{1EC5}", to: "abc")], info: capped).refusal == nil) + #expect(assess([.rename(from: "abc", to: "xyz")], info: capped).refusal == nil) + #expect(assess([.rename(from: "longname", to: "ln")], info: capped).refusal == nil) + #expect(assess([.remove("abc")], info: capped).refusal == nil) + #expect(assess([.remove("r"), .rename(from: "a", to: "abc")], info: capped).refusal != nil) + #expect(assess([.rename(from: "abc", to: "abcd")], info: info(Fixture.plain, "fx")).refusal == nil) + } + + // MARK: - Indexes and encryption + + @Test("An index refusal names the index, the field and the statement that drops it, for either name") + func indexRefusal() throws { + let source = try #require(assess([.rename(from: "email", to: "mail")], info: info(Fixture.plain, "fx"), indexes: [Fixture.emailIndex]).refusal) + #expect(source.contains("email_1")) + #expect(source.contains(#"db.fx.dropIndex("email_1")"#)) + let target = try #require(assess([.rename(from: "x", to: "email")], info: info(Fixture.plain, "fx"), indexes: [Fixture.emailIndex]).refusal) + #expect(target.contains("email_1")) + #expect(assess([.remove("email")], info: info(Fixture.plain, "fx"), indexes: [Fixture.emailIndex]).refusal != nil) + #expect(assess([.rename(from: "x", to: "y")], info: info(Fixture.plain, "fx"), indexes: [Fixture.emailIndex]).refusal == nil) + } + + /// `listIndexes` never lists a search index, so a rename that read it alone left an Atlas Search + /// or Vector Search definition pointing at a path no document has. + @Test("A search index that mentions either name refuses the change and is named in the refusal") + func searchIndexRefusal() throws { + let searchIndexes = [MongoSearchIndexTests.Fixture.synonymMappings, MongoSearchIndexTests.Fixture.vectorSearch] + let source = try #require( + assess([.rename(from: "fullplot", to: "plot")], info: info(Fixture.plain, "movies"), searchIndexes: searchIndexes).refusal + ) + #expect(source.contains("synonym_mappings")) + #expect(source.contains("fullplot")) + let target = try #require( + assess([.rename(from: "tags", to: "genres")], info: info(Fixture.plain, "movies"), searchIndexes: searchIndexes).refusal + ) + #expect(target.contains("vector_index")) + #expect(assess([.remove("plot_embedding")], info: info(Fixture.plain, "movies"), searchIndexes: searchIndexes).refusal != nil) + #expect(assess([.rename(from: "title", to: "heading")], info: info(Fixture.plain, "movies"), searchIndexes: searchIndexes).refusal == nil) + #expect(assess([.rename(from: "fullplot", to: "plot")], info: info(Fixture.plain, "movies")).refusal == nil) + } + + @Test("An encrypted field is refused under either name") + func encryptedFields() { + #expect(assess([.rename(from: "ssn", to: "id2")], info: info(Fixture.encrypted, "sec")).refusal != nil) + #expect(assess([.rename(from: "a", to: "ssn")], info: info(Fixture.encrypted, "sec")).refusal != nil) + #expect(assess([.remove("a")], info: info(Fixture.encrypted, "sec")).refusal == nil) + } + + // MARK: - Views + + @Test("A view that reads the old name refuses, and one that reads only the new name does not") + func viewsReadTheOldNameOnly() throws { + let views = [Fixture.view, Fixture.byTotal, Fixture.joined] + let status = try #require(assess([.rename(from: "status", to: "state")], info: info(Fixture.plain, "fxo"), views: views).refusal) + #expect(status.contains("activeOrders")) + #expect(assess([.rename(from: "sku", to: "code")], info: info(Fixture.plain, "fxo"), views: views).refusal?.contains("joined") == true) + #expect(assess([.rename(from: "free", to: "status")], info: info(Fixture.plain, "fxo"), views: views).refusal == nil) + #expect(assess([.rename(from: "free", to: "gratis")], info: info(Fixture.plain, "fxo"), views: views).refusal == nil) + } + + // MARK: - Validator rewrite + + @Test("Renaming a field New Table declared renames it in properties and required, in place") + func renameRewritesNewTableValidator() throws { + let assessment = assess([.rename(from: "title", to: "name")], info: info(Fixture.made, "made")) + #expect(assessment.refusal == nil) + let rewritten = try #require(assessment.rewrittenValidatorJson) + #expect(rewritten == #"{"$jsonSchema": {"bsonType": "object", "required": ["name"], "# + + #""properties": {"_id": { "bsonType" : "objectId" }, "name": { "bsonType" : "string" }, "# + + #""qty": { "bsonType" : [ "int", "null" ] }, "note": { "bsonType" : [ "string", "null" ] }}}}"#) + #expect(assessment.effectiveValidatorJson == rewritten) + let schema = MongoDBCollectionSchema.parse(jsonSchema: try #require(MongoScriptJson.member(of: rewritten, key: "$jsonSchema"))) + #expect(schema.fields.map(\.name) == ["_id", "name", "qty", "note"]) + #expect(schema.field(named: "name")?.isRequired == true) + } + + @Test("Removing a declared field takes it out of properties, and an emptied required list goes too") + func removalRewritesNewTableValidator() throws { + let note = try #require(assess([.remove("note")], info: info(Fixture.made, "made")).rewrittenValidatorJson) + #expect(!note.contains("note")) + #expect(note.contains(#""required": [ "title" ]"#)) + let title = try #require(assess([.remove("title")], info: info(Fixture.made, "made")).rewrittenValidatorJson) + #expect(!title.contains("required")) + #expect(!title.contains("title")) + } + + @Test("The validator statement is a collMod that leaves level and action alone") + func validatorStatement() { + #expect(MongoFieldChangeAssessment.validatorStatement( + collection: "made", validatorJson: #"{"$jsonSchema": {}}"#, writeConcern: .serverDefault + ) == #"db.runCommand({"collMod": "made", "validator": {"$jsonSchema": {}}})"#) + } + + /// `db.runCommand` passes its document on as written, and `mongoc_client_command_simple` adds no + /// write concern, so a `majority` connection's `collMod` went out with the server's default. + @Test("The validator statement carries the connection's write concern") + func validatorStatementCarriesWriteConcern() { + let majority = MongoWriteConcern(acknowledgement: .majority, journal: nil, timeoutMS: 2_000) + #expect(MongoFieldChangeAssessment.validatorStatement( + collection: "made", validatorJson: #"{"$jsonSchema": {}}"#, writeConcern: majority + ) == #"db.runCommand({"collMod": "made", "validator": {"$jsonSchema": {}}, "writeConcern": {"w": "majority", "wtimeout": 2000}})"#) + let rename = [MongoFieldChange.rename(from: "title", to: "name")] + let composed = assess(rename, info: info(Fixture.made, "made")).leadingStatements(collection: "made", writeConcern: majority) + #expect(composed.count == 1) + #expect(composed.first?.hasSuffix(#""writeConcern": {"w": "majority", "wtimeout": 2000}})"#) == true) + } + + /// The composed `collMod` and the checks after writing both come from the entry the save was + /// composed from, so any change to it since, the validation level included, refuses the save. + @Test("A save composed from one catalog entry is refused before writing once the entry differs") + func catalogChangedSinceComposed() { + let changed = String( + format: String(localized: "%@ changed after this save was prepared, so nothing was changed. Review the save and save again."), + "made" + ) + func refusal(_ current: String?) -> String? { + MongoFieldChangeAssessment.changedSinceComposedRefusal(composedFrom: Fixture.made, current: current, collection: "made") + } + #expect(refusal(Fixture.made) == nil) + #expect(refusal(Fixture.madeWithExtra) == changed) + #expect(refusal(Fixture.plain) == changed) + #expect(refusal(nil) == changed) + let moderate = Fixture.made.replacingOccurrences( + of: #""options" : { "validator""#, with: #""options" : { "validationLevel" : "moderate", "validator""# + ) + #expect(refusal(moderate) == changed) + #expect(MongoFieldChangeAssessment.changedSinceComposedRefusal(composedFrom: nil, current: Fixture.made, collection: "made") == changed) + } + + private func catalogRead( + _ json: String?, + _ name: String, + changes: [MongoFieldChange], + indexes: [String] = [] + ) -> MongoCatalogRead { + MongoCatalogRead(infoJson: json, info: info(json, name), assessment: assess(changes, info: info(json, name), indexes: indexes)) + } + + /// The scans before writing can each take up to the query timeout, so the catalog is read a + /// last time once they end, and the save writes only if it still matches what they checked. + @Test("A catalog that changed while the documents were checked stops the save before its first write") + func catalogChangedDuringChecks() { + let rename = [MongoFieldChange.rename(from: "title", to: "name")] + let checked = catalogRead(Fixture.made, "made", changes: rename) + func refusal(_ current: MongoCatalogRead) -> String? { + MongoFieldChangeAssessment.changedDuringChecksRefusal(checked: checked, current: current, collection: "made") + } + + #expect(refusal(catalogRead(Fixture.made, "made", changes: rename)) == nil) + #expect(refusal(catalogRead(Fixture.madeWithExtra, "made", changes: rename)) == String( + format: String(localized: "%@ changed while its documents were being checked, so nothing was changed. Save again."), + "made" + )) + let moderate = Fixture.made.replacingOccurrences( + of: #""options" : { "validator""#, with: #""options" : { "validationLevel" : "moderate", "validator""# + ) + #expect(moderate != Fixture.made) + #expect(refusal(catalogRead(moderate, "made", changes: rename)) == String( + format: String(localized: "%@ changed while its documents were being checked, so nothing was changed. Save again."), + "made" + )) + let titleIndex = #"{ "v" : { "$numberInt" : "2" }, "key" : { "title" : { "$numberInt" : "1" } }, "name" : "title_1" }"# + #expect(refusal(catalogRead(Fixture.made, "made", changes: rename, indexes: [titleIndex]))?.contains("title_1") == true) + #expect(refusal(catalogRead(nil, "made", changes: rename)) == String(format: String(localized: "Collection %@ no longer exists."), "made")) + } + + /// Measured on 7.0.43 before this check: under `validationAction: "warn"` the rename of `status` + /// applied and left this validator matching a key no document has. + @Test("A validator that reads the whole document refuses a change to any field") + func wholeDocumentValidator() { + let validator = #"{ "$expr" : { "$in" : [ "status", { "$map" : { "input" : { "$objectToArray" : "$$ROOT" }, "# + + #""in" : "$$this.k" } } ] } }"# + let keyed = info(#"{ "name" : "k", "type" : "collection", "options" : { "validator" : "# + validator + " } }", "k") + #expect(assess([.rename(from: "status", to: "state")], info: keyed).refusal != nil) + #expect(assess([.rename(from: "qty", to: "quantity")], info: keyed).refusal != nil) + #expect(assess([.remove("note")], info: keyed).refusal != nil) + } + + @Test("A name patternProperties or additionalProperties applies to refuses, and a declared name does not") + func rulesByNameRefuse() throws { + let closed = info(#"{ "name" : "c", "type" : "collection", "options" : { "validator" : { "$jsonSchema" : { "# + + #""required" : [ "a" ], "properties" : { "_id" : { "bsonType" : "objectId" }, "a" : { "bsonType" : "int" } }, "# + + #""additionalProperties" : false } } } }"#, "c") + let carried = assess([.rename(from: "a", to: "b")], info: closed) + #expect(carried.refusal == nil) + #expect(try #require(carried.rewrittenValidatorJson).contains(#""b": { "bsonType" : "int" }"#)) + #expect(assess([.remove("a")], info: closed).refusal == nil) + #expect(assess([.rename(from: "x", to: "y")], info: closed).refusal != nil) + #expect(assess([.remove("x")], info: closed).refusal != nil) + + let patterned = info(#"{ "name" : "p", "type" : "collection", "options" : { "validator" : { "$jsonSchema" : { "# + + #""properties" : { "a" : { "bsonType" : "int" } }, "patternProperties" : { "^tmp_" : { "bsonType" : "string" } } } } } }"#, "p") + #expect(assess([.rename(from: "a", to: "tmp_a")], info: patterned).refusal != nil) + #expect(assess([.rename(from: "tmp_a", to: "b")], info: patterned).refusal != nil) + #expect(assess([.rename(from: "a", to: "b")], info: patterned).refusal == nil) + } + + @Test("A field the validator does not name needs no collMod") + func undeclaredFieldKeepsTheValidator() { + let assessment = assess([.rename(from: "extra", to: "more")], info: info(Fixture.made, "made")) + #expect(assessment.refusal == nil) + #expect(assessment.rewrittenValidatorJson == nil) + #expect(assessment.effectiveValidatorJson == info(Fixture.made, "made").validatorJson) + } + + @Test("A rename onto a name the validator already declares is refused") + func targetAlreadyDeclared() { + #expect(assess([.rename(from: "qty", to: "note")], info: info(Fixture.made, "made")).refusal != nil) + } + + @Test("Property dependencies are renamed, and an emptied dependency is dropped") + func dependenciesAreRewritten() throws { + let validator = #"{ "$jsonSchema" : { "dependencies" : { "card" : [ "billing" ], "billing" : [ "card", "zip" ] } } }"# + let json = #"{ "name" : "d", "type" : "collection", "options" : { "validator" : "# + validator + " } }" + let renamed = try #require(assess([.rename(from: "billing", to: "address")], info: info(json, "d")).rewrittenValidatorJson) + #expect(renamed == #"{"$jsonSchema": {"dependencies": {"card": ["address"], "address": [ "card", "zip" ]}}}"#) + let removed = try #require(assess([.remove("card")], info: info(json, "d")).rewrittenValidatorJson) + #expect(removed == #"{"$jsonSchema": {"dependencies": {"billing": ["zip"]}}}"#) + } + + @Test("A validator that names the field outside what can be rewritten refuses the save") + func unrewritableValidators() { + func collection(_ validator: String) -> MongoCollectionInfo { + info(#"{ "name" : "v", "type" : "collection", "options" : { "validator" : "# + validator + " } }", "v") + } + let query = collection(#"{ "status" : { "$in" : [ "active" ] } }"#) + #expect(assess([.rename(from: "status", to: "state")], info: query).refusal != nil) + #expect(assess([.rename(from: "active", to: "live")], info: query).refusal == nil) + let expression = collection(#"{ "$expr" : { "$gt" : [ "$qty", { "$numberInt" : "0" } ] } }"#) + #expect(assess([.rename(from: "qty", to: "quantity")], info: expression).refusal != nil) + let pattern = collection(#"{ "$jsonSchema" : { "patternProperties" : { "^tmp_" : { "bsonType" : "int" } } } }"#) + #expect(assess([.remove("tmp_a")], info: pattern).refusal != nil) + #expect(assess([.remove("a")], info: pattern).refusal == nil) + let mixed = collection(#"{ "$jsonSchema" : { "required" : [ "a" ] }, "b" : { "$exists" : true } }"#) + #expect(assess([.rename(from: "a", to: "c")], info: mixed).refusal != nil) + } + + @Test("A rewrite that would carry a key the shell reorders or drops is refused") + func shellUnsafeKeys() { + let json = #"{ "name" : "k", "type" : "collection", "# + + #""options" : { "validator" : { "$jsonSchema" : { "required" : [ "a" ], "# + + #""properties" : { "a" : { "bsonType" : "int" }, "7" : { "bsonType" : "int" } } } } } }"# + #expect(assess([.rename(from: "a", to: "b")], info: info(json, "k")).refusal != nil) + #expect(assess([.rename(from: "x", to: "y")], info: info(json, "k")).refusal == nil) + } + + @Test("A dependent found after the save says the save ran and names what reads the field, without asking to save again") + func dependentAfterTheSave() throws { + let index = try #require(MongoIndexSpec(json: Fixture.emailIndex)) + let changes: [MongoFieldChange] = [.rename(from: "email", to: "mail")] + let found = try #require(MongoFieldDependent.dependent(of: changes, indexes: [index], searchIndexes: [], views: [], collection: "people")) + + #expect(found == .index(name: "email_1", field: "email")) + #expect(found.refusal(collection: "people").hasSuffix("then save again.")) + let after = found.appearedDuringSave(collection: "people") + #expect(after.hasPrefix("The save ran, but index email_1 on people was created while it did and uses email.")) + #expect(!after.contains("save again")) + #expect(MongoFieldDependent.dependent(of: [.rename(from: "name", to: "title")], indexes: [index], searchIndexes: [], views: [], collection: "people") == nil) + } +} diff --git a/TableProTests/Plugins/MongoFieldChangeTests.swift b/TableProTests/Plugins/MongoFieldChangeTests.swift new file mode 100644 index 0000000000..6b670ddbf8 --- /dev/null +++ b/TableProTests/Plugins/MongoFieldChangeTests.swift @@ -0,0 +1,159 @@ +// +// MongoFieldChangeTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +struct MongoFieldChangeTests { + private func column(_ name: String, _ type: String = "string", nullable: Bool = true) -> PluginColumnDefinition { + PluginColumnDefinition(name: name, dataType: type, isNullable: nullable) + } + + private func rename(_ from: String, _ to: String) -> PluginSchemaOperation { + .modifyColumn(old: column(from), new: column(to)) + } + + private func remove(_ name: String) -> PluginSchemaOperation { + .dropColumn(column(name)) + } + + @Test("A rename moves the field only where it exists and the new name does not") + func renameStatement() throws { + let change = try #require(MongoFieldChange(rename("status", "state"))) + #expect(change.statement(collection: "people", writeConcern: .serverDefault) == """ + db.people.updateMany({"status": {"$exists": true}, "state": {"$exists": false}}, {"$rename": {"status": "state"}}) + """) + } + + @Test("A removal unsets the field only where it exists") + func removalStatement() throws { + let change = try #require(MongoFieldChange(remove("tmp"))) + #expect(change.statement(collection: "people", writeConcern: .serverDefault) == """ + db.people.updateMany({"tmp": {"$exists": true}}, {"$unset": {"tmp": ""}}) + """) + } + + @Test("A collection a method would shadow is reached through getCollection") + func shadowedCollectionName() { + #expect(MongoFieldChange.remove("a").statement(collection: "stats", writeConcern: .serverDefault).hasPrefix("db.getCollection(\"stats\").updateMany(")) + #expect(MongoFieldChange.remove("a").statement(collection: "my orders", writeConcern: .serverDefault).hasPrefix("db.getCollection(\"my orders\")")) + } + + @Test("Quotes, backslashes, newlines and line separators in a name stay inside their string") + func namesAreEscaped() { + let statement = MongoFieldChange.rename(from: "q\"uote", to: "back\\slash\nline\u{2028}").statement(collection: "c", writeConcern: .serverDefault) + #expect(statement == #"db.c.updateMany({"q\"uote": {"$exists": true}, "back\\slash\nline"# + "\u{2028}" + + #"": {"$exists": false}}, {"$rename": {"q\"uote": "back\\slash\nline"# + "\u{2028}" + #""}})"#) + } + + /// `mongoc_client_command_simple` sends a command with no write concern of its own, so a + /// connection set to `majority` saved with the server's default until the statement named it. + @Test("The statement carries the connection's write concern, and none when the connection sets none") + func statementCarriesWriteConcern() { + let majority = MongoWriteConcern(acknowledgement: .majority, journal: true, timeoutMS: 5_000) + #expect(MongoFieldChange.rename(from: "a", to: "b").statement(collection: "people", writeConcern: majority) == """ + db.people.updateMany({"a": {"$exists": true}, "b": {"$exists": false}}, {"$rename": {"a": "b"}}, \ + {"writeConcern": {"w": "majority", "j": true, "wtimeout": 5000}}) + """) + let two = MongoWriteConcern(acknowledgement: .members(2), journal: nil, timeoutMS: nil) + #expect(MongoFieldChange.remove("tmp").statement(collection: "people", writeConcern: two) == """ + db.people.updateMany({"tmp": {"$exists": true}}, {"$unset": {"tmp": ""}}, {"writeConcern": {"w": 2}}) + """) + #expect(!MongoFieldChange.remove("tmp").statement(collection: "people", writeConcern: .serverDefault).contains("writeConcern")) + } + + /// Measured on 7.0.43: an update sent with `w: 0` is answered `n: 0` with no error whatever it + /// changed, so a save could not tell a finished rename from one that failed. + @Test("A write concern that asks for no answer is raised to one acknowledgement, and the rest is kept") + func unacknowledgedConcernIsRaised() { + #expect(MongoWriteConcern(acknowledgement: .members(0), journal: nil, timeoutMS: 100).schemaChangeJson + == #"{"w": 1, "wtimeout": 100}"#) + #expect(MongoWriteConcern(acknowledgement: .members(0), journal: true, timeoutMS: nil).schemaChangeJson + == #"{"w": 0, "j": true}"#) + #expect(MongoWriteConcern(acknowledgement: .tag("dc"), journal: false, timeoutMS: 0).schemaChangeJson + == #"{"w": "dc", "j": false}"#) + #expect(MongoWriteConcern(acknowledgement: nil, journal: true, timeoutMS: nil).schemaChangeJson == #"{"j": true}"#) + #expect(MongoWriteConcern.serverDefault.schemaChangeJson == nil) + } + + /// The session driver keeps a collection's inferred and declared field types by database and + /// collection, and a Structure save runs on another connection, so the save names the + /// collection and the driver drops it wherever it keeps it. + @Test("A collection's cache keys are found in every database, and no other collection's") + func collectionCacheKeys() { + let key = MongoCollectionCacheKey.key(database: "shop", collection: "orders") + #expect(MongoCollectionCacheKey.names(key, collection: "orders")) + #expect(MongoCollectionCacheKey.names(MongoCollectionCacheKey.key(database: "archive", collection: "orders"), collection: "orders")) + #expect(!MongoCollectionCacheKey.names(key, collection: "rders")) + #expect(!MongoCollectionCacheKey.names(MongoCollectionCacheKey.key(database: "shop", collection: "old_orders"), collection: "orders")) + #expect(!MongoCollectionCacheKey.names(MongoCollectionCacheKey.key(database: "orders", collection: "items"), collection: "orders")) + } + + @Test("_id cannot be renamed, removed or taken as a new name") + func identifierIsRefused() { + #expect(MongoFieldChange.refusal(for: rename("_id", "id")) != nil) + #expect(MongoFieldChange.refusal(for: remove("_id")) != nil) + #expect(MongoFieldChange.refusal(for: rename("id", "_id")) != nil) + } + + @Test("A name an update cannot address is refused on either side of a rename and for a removal") + func unaddressableNamesAreRefused() { + for name in ["", "$x", "a.b", "a\u{0}b", "__proto__"] { + #expect(MongoFieldChange.refusal(for: rename("a", name)) != nil, "rename to \(name.debugDescription)") + #expect(MongoFieldChange.refusal(for: remove(name)) != nil, "remove \(name.debugDescription)") + } + #expect(MongoFieldChange.refusal(for: rename("price.usd", "price")) != nil) + #expect(MongoFieldChange.refusal(for: rename("a", "1")) == nil) + #expect(MongoFieldChange.refusal(for: rename("a", "tên mới")) == nil) + } + + @Test("The $ and dot refusal reads the same as the one New Table gives") + func sharedAddressingWording() { + #expect(MongoFieldName.addressingRefusal("a.b") == String( + format: String(localized: "MongoDB cannot address a field named %@. A field name cannot start with $ or contain a dot."), + "a.b" + )) + } + + @Test("A change to anything but the name is refused, with or without a rename") + func attributeChangesAreRefused() { + #expect(MongoFieldChange.refusal(for: .modifyColumn(old: column("a", "string"), new: column("a", "int"))) != nil) + #expect(MongoFieldChange.refusal(for: .modifyColumn(old: column("a"), new: column("b", "int"))) != nil) + #expect(MongoFieldChange.refusal(for: .modifyColumn(old: column("a"), new: column("b", nullable: false))) != nil) + #expect(MongoFieldChange.refusal(for: rename("a", "b")) == nil) + } + + @Test("An operation that is not a field edit is left to the collection's own refusal") + func otherOperationsPassThrough() { + let index = PluginIndexDefinition(name: "ix", columns: ["a"], isUnique: false) + #expect(MongoFieldChange.refusal(for: .addIndex(index)) == nil) + #expect(MongoFieldChange.refusal(for: .addColumn(column("_id"))) == nil) + #expect(MongoFieldChange(.addColumn(column("a"))) == nil) + } + + @Test("A name used by two edits of one save is refused before anything runs") + func sharedNamesAreRefused() { + #expect(MongoFieldChangePlan(operations: [rename("a", "b"), rename("b", "c")]).refusal != nil) + #expect(MongoFieldChangePlan(operations: [rename("a", "b"), rename("b", "a")]).refusal != nil) + #expect(MongoFieldChangePlan(operations: [remove("b"), rename("a", "b")]).refusal != nil) + #expect(MongoFieldChangePlan(operations: [rename("a", "c"), rename("b", "c")]).refusal != nil) + } + + @Test("Edits that share no name keep both steps in the order their statements run") + func disjointEditsAreKept() { + let plan = MongoFieldChangePlan(operations: [remove("c"), rename("a", "b")]) + #expect(plan.refusal == nil) + #expect(plan.changes == [.remove("c"), .rename(from: "a", to: "b")]) + #expect(plan.renames.map(\.from) == ["a"]) + } + + @Test("A save with no field edit has nothing to plan") + func noFieldEdits() { + let index = PluginIndexDefinition(name: "ix", columns: ["a"], isUnique: false) + #expect(MongoFieldChangePlan(operations: [.addIndex(index)]).isEmpty) + #expect(MongoFieldChangePlan(operations: [.modifyColumn(old: column("a"), new: column("a", "int"))]).isEmpty) + } +} diff --git a/TableProTests/Plugins/MongoFieldDataProbeTests.swift b/TableProTests/Plugins/MongoFieldDataProbeTests.swift new file mode 100644 index 0000000000..ae92cb3403 --- /dev/null +++ b/TableProTests/Plugins/MongoFieldDataProbeTests.swift @@ -0,0 +1,282 @@ +// +// MongoFieldDataProbeTests.swift +// TableProTests +// + +import Foundation +import Testing + +/// The pipelines here were run against MongoDB 7.0.43 in the pull request's live checks; these tests +/// pin their text and their shape so a refactor cannot quietly change what the server is asked. +struct MongoFieldDataProbeTests { + private func stages(_ pipeline: String) throws -> [[String: Any]] { + try #require(MongoJsonValue.parse(pipeline) as? [[String: Any]]) + } + + @Test("The both-names pass matches documents holding both names of one rename, never one of each") + func bothNamesMatchesPairs() throws { + let pipeline = try #require(MongoFieldDataProbe.bothNamesPipeline([("a", "b"), ("c", "d")])) + #expect(pipeline == #"[{"$match": {"$or": [{"a": {"$exists": true}, "b": {"$exists": true}}, {"c": {"$exists": true}, "# + + #""d": {"$exists": true}}]}}, {"$limit": 1}, {"$project": {"_id": 0, "# + + #""p0": {"$and": [{"$ne": [{"$type": "$a"}, "missing"]}, {"$ne": [{"$type": "$b"}, "missing"]}]}, "# + + #""p1": {"$and": [{"$ne": [{"$type": "$c"}, "missing"]}, {"$ne": [{"$type": "$d"}, "missing"]}]}}}]"#) + let match = try #require(try stages(pipeline).first?["$match"] as? [String: Any]) + let alternatives = try #require(match["$or"] as? [[String: Any]]) + #expect(alternatives.map { Set($0.keys) } == [["a", "b"], ["c", "d"]]) + #expect(MongoFieldDataProbe.bothNamesPipeline([]) == nil) + } + + @Test("The size pass adds each longer name's growth to documents holding the old name, and skips when nothing grows") + func oversizePassCountsGrowth() throws { + let pipeline = try #require(MongoFieldDataProbe.oversizePipeline([("a", "abc"), ("long", "l"), ("x", "xyz9")])) + #expect(pipeline == #"[{"$match": {"$or": [{"a": {"$exists": true}}, {"x": {"$exists": true}}]}}, "# + + #"{"$match": {"$expr": {"$gt": [{"$add": [{"$bsonSize": "$$ROOT"}, "# + + #"{"$cond": [{"$ne": [{"$type": "$a"}, "missing"]}, 2, 0]}, "# + + #"{"$cond": [{"$ne": [{"$type": "$x"}, "missing"]}, 3, 0]}]}, 16777216]}}}, "# + + #"{"$limit": 1}, {"$project": {"_id": 1}}]"#) + #expect(try stages(pipeline).count == 4) + #expect(MongoFieldDataProbe.oversizePipeline([("long", "l"), ("same", "SAME")]) == nil) + #expect(MongoFieldDataProbe.oversizePipeline([("é", "ee")]) == nil) + } + + @Test("The pass says which rename a document holds both names of") + func namesThePair() throws { + let renames = [(from: "a", to: "b"), (from: "c", to: "d")] + let found = try #require(MongoFieldDataProbe.renameHoldingBothNames(in: #"{ "p0" : false, "p1" : true }"#, renames: renames)) + #expect(found.from == "c") + #expect(found.to == "d") + #expect(MongoFieldDataProbe.renameHoldingBothNames(in: nil, renames: renames) == nil) + } + + @Test("One validator pass per change, each checking the documents its statement changes") + func oneStepPipeline() throws { + let pipelines = MongoFieldDataProbe.validatorPipelines( + [.rename(from: "p", to: "x")], validatorJson: #"{"$jsonSchema": {"required": ["y"]}}"#, onlyValidDocuments: false + ) + #expect(pipelines == [#"[{"$match": {"p": {"$exists": true}, "x": {"$exists": false}}}, "# + + #"{"$addFields": {"x": "$p", "p": "$$REMOVE"}}, {"$match": {"$nor": [{"$jsonSchema": {"required": ["y"]}}]}}, "# + + #"{"$limit": 1}, {"$project": {"_id": 1}}]"#]) + } + + @Test("A later step replays the earlier ones with their guard, and a removal replays as a removed field") + func laterStepsReplayEarlierOnes() throws { + let pipelines = MongoFieldDataProbe.validatorPipelines( + [.remove("r"), .rename(from: "p", to: "x"), .rename(from: "q", to: "y")], + validatorJson: "{}", + onlyValidDocuments: false + ) + #expect(pipelines.count == 3) + let third = try stages(pipelines[2]) + let removal = try #require(third[0]["$addFields"] as? [String: Any]) + #expect(removal.count == 1) + #expect(removal["r"] as? String == "$$REMOVE") + let replay = try #require(third[1]["$addFields"] as? [String: Any]) + #expect(Set(replay.keys) == ["x", "p"]) + #expect(pipelines[2].contains(#""x": {"$cond": [{"$and": [{"$ne": [{"$type": "$p"}, "missing"]}, {"$eq": [{"$type": "$x"}, "# + + #""missing"]}]}, "$p", "$x"]}"#)) + #expect(pipelines[2].contains(#""p": {"$cond": [{"$and": [{"$ne": [{"$type": "$p"}, "missing"]}, {"$eq": [{"$type": "$x"}, "# + + #""missing"]}]}, "$$REMOVE", "$p"]}"#)) + let match = try #require(third[2]["$match"] as? [String: Any]) + #expect(Set(match.keys) == ["q", "y"]) + } + + @Test("A moderate validator sets aside documents it already rejects before the step") + func moderateSetsAsideInvalidDocuments() throws { + let validator = #"{"$jsonSchema": {"required": ["k"]}}"# + let strict = try stages(MongoFieldDataProbe.validatorPipelines([.remove("p")], validatorJson: validator, onlyValidDocuments: false)[0]) + let moderate = try stages(MongoFieldDataProbe.validatorPipelines([.remove("p")], validatorJson: validator, onlyValidDocuments: true)[0]) + #expect(strict.count == 5) + #expect(moderate.count == 6) + let preFilter = try #require(moderate[1]["$match"] as? [String: Any]) + #expect(preFilter["$jsonSchema"] != nil) + } + + /// The reviewer's case, measured on 7.0.43. Validator `{old: string}`, documents `{_id: 1, old: + /// "x"}` and `{_id: 2, new: 42}`, rename `old` to `new`. The step pass checks only `_id 1`, the + /// one document the rename changes, so the committed code ran the `collMod` and the rename and + /// reported success, and the next `updateOne` on `_id 2` failed with 121. This pass, run against + /// the same collection, returned `_id 2` and the save was refused with nothing changed. + @Test("The rewritten-rule pass checks documents holding only the new name against the rewritten validator") + func rewrittenRuleReachesTargetOnlyDocuments() throws { + let original = #"{"$jsonSchema": {"properties": {"old": {"bsonType": "string"}}}}"# + let rewritten = #"{"$jsonSchema": {"properties": {"new": {"bsonType": "string"}}}}"# + let pipeline = MongoFieldDataProbe.rewrittenRulePipeline( + [.rename(from: "old", to: "new")], + originalValidatorJson: original, + rewrittenValidatorJson: rewritten, + onlyValidDocuments: false + ) + #expect(pipeline == #"[{"$match": {"$or": [{"old": {"$exists": true}}, {"new": {"$exists": true}}]}}, "# + + #"{"$addFields": {"new": {"$cond": [{"$and": [{"$ne": [{"$type": "$old"}, "missing"]}, "# + + #"{"$eq": [{"$type": "$new"}, "missing"]}]}, "$old", "$new"]}, "# + + #""old": {"$cond": [{"$and": [{"$ne": [{"$type": "$old"}, "missing"]}, "# + + #"{"$eq": [{"$type": "$new"}, "missing"]}]}, "$$REMOVE", "$old"]}}}, "# + + #"{"$match": {"$nor": [{"$jsonSchema": {"properties": {"new": {"bsonType": "string"}}}}]}}, "# + + #"{"$limit": 1}, {"$project": {"_id": 1}}]"#) + + let stepPass = try stages( + MongoFieldDataProbe.validatorPipelines([.rename(from: "old", to: "new")], validatorJson: rewritten, onlyValidDocuments: false)[0] + ) + let stepFilter = try #require(stepPass[0]["$match"] as? [String: Any]) + #expect(stepFilter["new"] as? [String: Bool] == ["$exists": false]) + } + + /// Under `moderate` the server stops checking a document that fails, so the question is whether + /// the save takes a document the old validator accepted and leaves it failing the new one. + /// Measured on 7.0.43: `{new: 42}` accepted before is refused, and `{new: 42, k: "bad"}`, which + /// the old validator already rejected, is set aside and the save applies. + @Test("Under moderate the rewritten-rule pass counts only documents the old validator accepted") + func rewrittenRuleUnderModerate() throws { + let original = #"{"$jsonSchema": {"properties": {"old": {"bsonType": "string"}}}}"# + let rewritten = #"{"$jsonSchema": {"properties": {"new": {"bsonType": "string"}}}}"# + let moderate = try stages(MongoFieldDataProbe.rewrittenRulePipeline( + [.rename(from: "old", to: "new")], + originalValidatorJson: original, + rewrittenValidatorJson: rewritten, + onlyValidDocuments: true + )) + let accepted = try #require(moderate[1]["$match"] as? [String: Any]) + let schema = try #require(accepted["$jsonSchema"] as? [String: Any]) + let properties = try #require(schema["properties"] as? [String: Any]) + #expect(Set(properties.keys) == ["old"]) + #expect(moderate[2]["$addFields"] != nil) + } + + @Test("The rewritten-rule pass covers every name the save changes once, and replays every step in order") + func rewrittenRuleCoversEveryName() throws { + let changes: [MongoFieldChange] = [.remove("r"), .rename(from: "p", to: "x"), .rename(from: "q", to: "y")] + let pipeline = try stages(MongoFieldDataProbe.rewrittenRulePipeline( + changes, originalValidatorJson: "{}", rewrittenValidatorJson: #"{"a": 1}"#, onlyValidDocuments: false + )) + let names = try #require((pipeline[0]["$match"] as? [String: Any])?["$or"] as? [[String: Any]]) + #expect(names.flatMap(\.keys) == ["r", "p", "x", "q", "y"]) + #expect(pipeline.count == 7) + let removal = try #require(pipeline[1]["$addFields"] as? [String: Any]) + #expect(removal["r"] as? String == "$$REMOVE") + #expect(Set(try #require(pipeline[2]["$addFields"] as? [String: Any]).keys) == ["x", "p"]) + #expect(Set(try #require(pipeline[3]["$addFields"] as? [String: Any]).keys) == ["y", "q"]) + let rejected = try #require(pipeline[4]["$match"] as? [String: Any]) + #expect(rejected["$nor"] != nil) + } + + /// The target-only race: a document another client gives only the new name after the last check + /// before writing is accepted by the old validator, which does not read that name, and the + /// `collMod` checks no document. Every statement then succeeds and skips it. + @Test("The pass after writing checks every document holding a changed name against the rewritten validator, replaying nothing") + func violationAfterWriting() throws { + let original = #"{"$jsonSchema": {"properties": {"old": {"bsonType": "string"}}}}"# + let rewritten = #"{"$jsonSchema": {"properties": {"new": {"bsonType": "string"}}}}"# + let pipeline = MongoFieldDataProbe.violationAfterWritingPipeline( + [.rename(from: "old", to: "new")], + originalValidatorJson: original, + rewrittenValidatorJson: rewritten, + onlyValidDocuments: false + ) + #expect(pipeline == #"[{"$match": {"$or": [{"old": {"$exists": true}}, {"new": {"$exists": true}}]}}, "# + + #"{"$match": {"$nor": [{"$jsonSchema": {"properties": {"new": {"bsonType": "string"}}}}]}}, "# + + #"{"$limit": 1}, {"$project": {"_id": 1}}]"#) + let moderate = try stages(MongoFieldDataProbe.violationAfterWritingPipeline( + [.rename(from: "old", to: "new")], + originalValidatorJson: original, + rewrittenValidatorJson: rewritten, + onlyValidDocuments: true + )) + #expect(moderate.count == 5) + let accepted = try #require(moderate[1]["$match"] as? [String: Any]) + #expect(accepted["$jsonSchema"] != nil) + #expect(!moderate.contains { $0["$addFields"] != nil }) + + let message = MongoFieldDataProbe.violationAfterWriting(identifier: "7", collection: "people") + #expect(message.hasPrefix("The save did not finish: the updated validator of people rejects the document with _id 7, ")) + #expect(message.hasSuffix(" Fix that document, then save again.")) + } + + @Test("Every stage of every pass is one MongoDB 4.0 has, so none is $set or $unset") + func stagesExistOnMongoDB40() throws { + let stagesOn40: Set = ["$match", "$addFields", "$limit", "$project"] + let changes: [MongoFieldChange] = [.remove("r"), .rename(from: "p", to: "x"), .rename(from: "q", to: "y")] + let validator = #"{"$jsonSchema": {"required": ["y"]}}"# + let bothNames = try #require(MongoFieldDataProbe.bothNamesPipeline([("p", "x"), ("q", "y")])) + let rewrittenRule = [true, false].map { onlyValid in + MongoFieldDataProbe.rewrittenRulePipeline( + changes, originalValidatorJson: validator, rewrittenValidatorJson: validator, onlyValidDocuments: onlyValid + ) + } + let pipelines = MongoFieldDataProbe.validatorPipelines(changes, validatorJson: validator, onlyValidDocuments: true) + + MongoFieldDataProbe.validatorPipelines(changes, validatorJson: validator, onlyValidDocuments: false) + + [bothNames] + rewrittenRule + + [MongoFieldDataProbe.violationAfterWritingPipeline( + changes, originalValidatorJson: validator, rewrittenValidatorJson: validator, onlyValidDocuments: true + )] + for pipeline in pipelines { + for stage in try stages(pipeline) { + #expect(stage.count == 1) + #expect(Set(stage.keys).isSubset(of: stagesOn40), "\(stage.keys) in \(pipeline)") + } + } + } + + @Test("Each pass reads at local, whatever read concern the connection string sets, within its time bound") + func passOptions() throws { + let options = try #require(MongoJsonValue.parse(MongoFieldDataProbe.aggregateOptionsJson(maxTimeMS: 5_000)) as? [String: Any]) + #expect(Set(options.keys) == ["maxTimeMS", "readConcern"]) + #expect(options["maxTimeMS"] as? Int == 5_000) + #expect(options["readConcern"] as? [String: String] == ["level": "local"]) + } + + @Test("Each pass is bounded, by the query timeout or by a ceiling when there is none") + func timeBound() { + #expect(MongoFieldDataProbe.maxTimeMS(queryTimeoutMS: 60_000) == 60_000) + #expect(MongoFieldDataProbe.maxTimeMS(queryTimeoutMS: 0) == MongoFieldDataProbe.unlimitedTimeoutCeilingMS) + #expect(MongoFieldDataProbe.unlimitedTimeoutCeilingMS > 0) + } + + @Test("The count after writing matches every document still holding the old name, for a rename and a removal") + func remainderPipelines() throws { + #expect(MongoFieldDataProbe.remainderPipeline(.rename(from: "old", to: "new")) + == #"[{"$match": {"old": {"$exists": true}}}, {"$count": "n"}]"#) + #expect(MongoFieldDataProbe.remainderPipeline(.remove("tmp")) + == #"[{"$match": {"tmp": {"$exists": true}}}, {"$count": "n"}]"#) + let stageNames = try stages(MongoFieldDataProbe.remainderPipeline(.remove("tmp"))).flatMap(\.keys) + #expect(stageNames == ["$match", "$count"]) + } + + @Test("No document back from $count is zero, and an answer that is not a count is unknown") + func remainderCounts() { + #expect(MongoFieldDataProbe.remainderCount(in: nil) == 0) + #expect(MongoFieldDataProbe.remainderCount(in: #"{ "n" : { "$numberInt" : "3" } }"#) == 3) + #expect(MongoFieldDataProbe.remainderCount(in: #"{ "n" : { "$numberLong" : "5000000000" } }"#) == 5_000_000_000) + #expect(MongoFieldDataProbe.remainderCount(in: #"{ "n" : 7 }"#) == 7) + #expect(MongoFieldDataProbe.remainderCount(in: #"{ "x" : 1 }"#) == nil) + #expect(MongoFieldDataProbe.remainderCount(in: "not json") == nil) + } + + /// Measured on 7.0.43 with a second connection inserting `{old: 1, new: 2}` between the check + /// before writing and the rename: every statement succeeded, the rename skipped that document, + /// and this count read 1. + @Test("A save that left the old name in some documents is not finished, and says how many hold it") + func shortfalls() { + let rename = MongoFieldChange.rename(from: "old", to: "new") + let removal = MongoFieldChange.remove("tmp") + #expect(MongoFieldDataProbe.shortfall([]) == nil) + #expect(MongoFieldDataProbe.shortfall([(rename, 0), (removal, 0)]) == nil) + #expect(MongoFieldDataProbe.shortfall([(rename, 1), (removal, 0)]) == String( + format: String(localized: "The save did not finish: one document still holds %@, most likely written by another client while the save ran. Save again to finish."), + "old" + )) + #expect(MongoFieldDataProbe.shortfall([(rename, 0), (removal, 12)]) == String( + format: String(localized: "The save did not finish: %1$lld documents still hold %2$@, most likely written by another client while the save ran. Save again to finish."), + Int64(12), "tmp" + )) + #expect(MongoFieldDataProbe.shortfall([(rename, 2), (removal, 12)])?.contains("old") == true) + } + + @Test("A found document's _id is written the way a person types it") + func identifiers() { + #expect(MongoFieldDataProbe.identifier(in: #"{ "_id" : { "$oid" : "6ab746850efe77e864860657" } }"#) + == #"ObjectId("6ab746850efe77e864860657")"#) + #expect(MongoFieldDataProbe.identifier(in: #"{ "_id" : { "$numberInt" : "2" } }"#) == "2") + #expect(MongoFieldDataProbe.identifier(in: #"{ "_id" : "abc" }"#) == #""abc""#) + #expect(MongoFieldDataProbe.identifier(in: nil) == nil) + } +} diff --git a/TableProTests/Plugins/MongoFieldReferencesTests.swift b/TableProTests/Plugins/MongoFieldReferencesTests.swift new file mode 100644 index 0000000000..79b165ea00 --- /dev/null +++ b/TableProTests/Plugins/MongoFieldReferencesTests.swift @@ -0,0 +1,364 @@ +// +// MongoFieldReferencesTests.swift +// TableProTests +// + +import Foundation +import Testing + +/// The index and view fixtures are the canonical Extended JSON `listIndexes` and `listCollections` +/// returned on MongoDB 7.0.43, byte for byte. +struct MongoFieldReferencesTests { + private enum Fixture { + static let idIndex = #"{ "v" : { "$numberInt" : "2" }, "key" : { "_id" : { "$numberInt" : "1" } }, "name" : "_id_" }"# + static let textIndex = #"{ "v" : { "$numberInt" : "2" }, "key" : { "_fts" : "text", "_ftsx" : { "$numberInt" : "1" } }, "# + + #""name" : "t_text_sub.x_text", "weights" : { "sub.x" : { "$numberInt" : "1" }, "# + + #""t" : { "$numberInt" : "5" } }, "default_language" : "english", "language_override" : "language", "# + + #""textIndexVersion" : { "$numberInt" : "3" } }"# + static let wildcardIndex = #"{"v":{"$numberInt":"2"},"key":{"$**":{"$numberInt":"1"}},"name":"$**_1","wildcardProjection":{"a":{"$numberInt":"1"}}}"# + static let partialIndex = #"{ "v" : { "$numberInt" : "2" }, "key" : { "b" : { "$numberInt" : "1" } }, "name" : "b_1", "# + + #""unique" : true, "partialFilterExpression" : { "a" : { "$gt" : { "$numberInt" : "0" } } } }"# + static let subtreeIndex = #"{ "v" : { "$numberInt" : "2" }, "key" : { "d.$**" : { "$numberInt" : "1" } }, "name" : "d.$**_1" }"# + } + + private func index(_ json: String) throws -> MongoIndexSpec { + try #require(MongoIndexSpec(json: json)) + } + + private func query(_ json: String, reaches field: String) throws -> Bool { + MongoQueryFieldReferences.reaches(try #require(MongoJsonValue.parse(json)), field: field) + } + + private func pipeline(_ json: String, reaches field: String) throws -> Bool { + MongoPipelineFieldReferences.reaches(try #require(MongoJsonValue.parse(json)), field: field) + } + + @Test("A path reaches a field when it is the field or a path under it") + func pathReach() { + #expect(MongoFieldPath.reaches("a", field: "a")) + #expect(MongoFieldPath.reaches("a.b", field: "a")) + #expect(!MongoFieldPath.reaches("ab", field: "a")) + #expect(!MongoFieldPath.reaches("b.a", field: "a")) + } + + @Test("The _id index names _id and nothing else") + func idIndex() throws { + let spec = try index(Fixture.idIndex) + #expect(spec.name == "_id_") + #expect(spec.reaches("_id")) + #expect(!spec.reaches("id")) + } + + @Test("A text index names its weighted fields and its language override, not what sits under them") + func textIndex() throws { + let spec = try index(Fixture.textIndex) + #expect(spec.reaches("t")) + #expect(spec.reaches("sub")) + #expect(spec.reaches("language")) + #expect(!spec.reaches("x")) + #expect(!spec.reaches("english")) + } + + @Test("A wildcard index names what its projection names, and a subtree index names its root") + func wildcardIndexes() throws { + #expect(try index(Fixture.wildcardIndex).reaches("a")) + #expect(try index(Fixture.wildcardIndex).reaches("b") == false) + #expect(try index(Fixture.subtreeIndex).reaches("d")) + #expect(try index(Fixture.subtreeIndex).reaches("e") == false) + } + + @Test("A partial index names both its key and the fields of its filter, and b does not name bb") + func partialIndex() throws { + let spec = try index(Fixture.partialIndex) + #expect(spec.reaches("a")) + #expect(spec.reaches("b")) + #expect(!spec.reaches("bb")) + } + + @Test("A query names the keys it matches on and never the values it matches") + func queryKeysNotValues() throws { + #expect(try query(#"{"status": {"$in": ["active", "gone"]}}"#, reaches: "status")) + #expect(try query(#"{"status": {"$in": ["active", "gone"]}}"#, reaches: "active") == false) + #expect(try query(#"{"$or": [{"a": 1}, {"$and": [{"b.c": 2}]}]}"#, reaches: "b")) + #expect(try query(#"{"$nor": [{"a": 1}]}"#, reaches: "a")) + #expect(try query(#"{"items": {"$elemMatch": {"qty": {"$gt": 1}}}}"#, reaches: "qty") == false) + } + + @Test("$expr names a field by a $-prefixed string or a document variable, never by a literal") + func expressions() throws { + #expect(try query(#"{"$expr": {"$gt": ["$qty", {"$numberInt": "0"}]}}"#, reaches: "qty")) + #expect(try query(#"{"$expr": {"$eq": ["$status", "active"]}}"#, reaches: "active") == false) + #expect(try query(#"{"$expr": {"$eq": [{"$literal": "$qty"}, "x"]}}"#, reaches: "qty") == false) + #expect(try query(#"{"$expr": {"$gt": ["$$ROOT.qty", 1]}}"#, reaches: "qty")) + #expect(try query(#"{"$expr": {"$gt": ["$$CURRENT.qty.n", 1]}}"#, reaches: "qty")) + #expect(try query(#"{"$expr": {"$gt": [{"$getField": "qty"}, 1]}}"#, reaches: "qty")) + #expect(try query(#"{"$expr": {"$gt": [{"$getField": {"field": {"$literal": "qty"}, "input": "$$ROOT"}}, 1]}}"#, reaches: "qty")) + #expect(try query(#"{"$expr": {"$lt": ["$$NOW", "$at"]}}"#, reaches: "qty") == false) + #expect(try query(#"{"$expr": {"$lt": ["$$NOW", "$at"]}}"#, reaches: "at")) + #expect(try query(#"{"$expr": {"$function": {"body": "function() {}", "args": [], "lang": "js"}}}"#, reaches: "qty")) + } + + /// The reviewer's case: every key of the document becomes a value, and a literal matches one. + /// Nothing in the validator names `status` the way a field is named, and a rename of `status` + /// would leave it checking a key no document has. + @Test("An expression handed the whole document reads every field") + func wholeDocumentReadsEveryField() throws { + let keys = #"{"$expr": {"$in": ["status", {"$map": {"input": {"$objectToArray": "$$ROOT"}, "in": "$$this.k"}}]}}"# + #expect(try query(keys, reaches: "status")) + #expect(try query(keys, reaches: "qty")) + #expect(try query(#"{"$expr": {"$gt": [{"$size": {"$objectToArray": "$$CURRENT"}}, 3]}}"#, reaches: "qty")) + #expect(try query(#"{"$expr": {"$eq": ["$$ROOT", "x"]}}"#, reaches: "qty")) + let merged = #"{"$expr": {"$eq": [{"$mergeObjects": ["$$ROOT", {"a": 1}]}, {"a": 1}]}}"# + #expect(try query(merged, reaches: "qty")) + let unset = #"{"$expr": {"$eq": [{"$unsetField": {"field": "a", "input": "$$CURRENT"}}, {}]}}"# + #expect(try query(unset, reaches: "qty")) + let set = #"{"$expr": {"$eq": [{"$setField": {"field": "a", "input": "$$ROOT", "value": 1}}, {"a": 1}]}}"# + #expect(try query(set, reaches: "qty")) + #expect(try query(#"{"$where": "this.a > 1"}"#, reaches: "qty")) + let elements = #"{"$expr": {"$anyElementTrue": {"$map": {"input": "$items", "in": {"$eq": ["$$this", 1]}}}}}"# + #expect(try query(elements, reaches: "items")) + #expect(try query(elements, reaches: "qty") == false) + } + + @Test("patternProperties and additionalProperties apply to a name the schema does not declare") + func rulesByName() throws { + func applies(_ json: String, to name: String) throws -> Bool { + MongoQueryFieldReferences.appliesByName(try #require(MongoJsonValue.parse(json)), to: name) + } + let closed = #"{"$jsonSchema": {"properties": {"a": {"bsonType": "int"}}, "additionalProperties": false}}"# + #expect(try applies(closed, to: "a") == false) + #expect(try applies(closed, to: "b")) + #expect(try applies(#"{"$jsonSchema": {"additionalProperties": {"bsonType": "string"}}}"#, to: "b")) + #expect(try applies(#"{"$jsonSchema": {"additionalProperties": true}}"#, to: "b") == false) + #expect(try applies(#"{"$jsonSchema": {"additionalProperties": {}}}"#, to: "b") == false) + let patterned = #"{"$jsonSchema": {"patternProperties": {"^tmp_": {"bsonType": "int"}}}}"# + #expect(try applies(patterned, to: "tmp_x")) + #expect(try applies(patterned, to: "x") == false) + #expect(try applies(#"{"$and": [{"$jsonSchema": {"allOf": [{"additionalProperties": false}]}}]}"#, to: "x")) + #expect(try applies(#"{"$jsonSchema": {"not": {"patternProperties": {"^x$": {}}}}}"#, to: "x")) + #expect(try applies(#"{"a": {"$exists": true}}"#, to: "a") == false) + } + + @Test("A field name $getField, $setField or $unsetField computes at run time reaches every field") + func computedFieldNames() throws { + let concat = #"{"$getField": {"field": {"$concat": ["sta", "tus"]}, "input": "$$ROOT"}}"# + #expect(try query(#"{"$expr": {"$eq": ["# + concat + #", 1]}}"#, reaches: "status")) + #expect(try query(#"{"$expr": {"$eq": ["# + concat + #", 1]}}"#, reaches: "qty")) + #expect(try pipeline(#"[{"$project": {"s": "# + concat + #"}}]"#, reaches: "status")) + #expect(try pipeline(#"[{"$project": {"s": {"$getField": "$name"}}}]"#, reaches: "qty")) + #expect(try pipeline(#"[{"$project": {"s": {"$getField": {"field": "$$key", "input": "$$ROOT"}}}}]"#, reaches: "qty")) + let unset = #"[{"$replaceWith": {"$unsetField": {"field": {"$toString": "$k"}, "input": "$$ROOT"}}}]"# + #expect(try pipeline(unset, reaches: "qty")) + + #expect(try pipeline(#"[{"$replaceWith": {"$setField": {"field": "a", "input": "$$ROOT", "value": 1}}}]"#, reaches: "a")) + #expect(try pipeline(#"[{"$replaceWith": {"$setField": {"field": "a", "input": "$$ROOT", "value": 1}}}]"#, reaches: "b") == false) + #expect(try query(#"{"$expr": {"$eq": [{"$getField": {"field": {"$literal": "qty"}, "input": "$sub"}}, 1]}}"#, reaches: "note") == false) + #expect(try query(#"{"$expr": {"$eq": [{"$getField": {"field": {"$literal": "qty"}, "input": "$$ROOT"}}, 1]}}"#, reaches: "note") == false) + #expect(try query(#"{"$expr": {"$eq": [{"$getField": {"field": {"$literal": "qty"}, "input": "$$ROOT"}}, 1]}}"#, reaches: "qty")) + } + + private func view(_ json: String, reads field: String) throws -> Bool { + let stages = try #require(MongoJsonValue.parse(json) as? [Any]) + return MongoViewDefinition(name: "v", viewOn: "c", pipeline: stages).readsField(field) + } + + /// The reviewer's case in a view: `{$objectToArray: "$$ROOT"}` turns every key into a value, so + /// a rename changes what the view emits while nothing in it names the field. + @Test("A view that hands the whole document to something reading its names reads every field") + func viewReadsEveryFieldThroughTheWholeDocument() throws { + #expect(try view(#"[{"$project": {"kv": {"$objectToArray": "$$ROOT"}}}]"#, reads: "status")) + #expect(try view(#"[{"$addFields": {"n": {"$size": {"$objectToArray": "$$CURRENT"}}}}]"#, reads: "qty")) + #expect(try view(#"[{"$match": {"$expr": {"$eq": ["$$ROOT", {"a": 1}]}}}]"#, reads: "qty")) + #expect(try view(#"[{"$group": {"_id": null, "docs": {"$push": "$$ROOT"}}}]"#, reads: "qty")) + #expect(try view(#"[{"$project": {"copy": "$$ROOT"}}]"#, reads: "qty")) + #expect(try view(#"[{"$lookup": {"from": "o", "let": {"d": "$$ROOT"}, "pipeline": [], "as": "x"}}]"#, reads: "qty")) + #expect(try view(#"[{"$facet": {"f": [{"$project": {"kv": {"$objectToArray": "$$CURRENT"}}}]}}]"#, reads: "qty")) + #expect(try view(#"[{"$unionWith": {"coll": "o", "pipeline": [{"$replaceWith": {"$objectToArray": "$$ROOT"}}]}}]"#, reads: "qty")) + #expect(try view(#"[{"$replaceWith": {"$cond": [{"$eq": ["$$ROOT", {}]}, "$$ROOT", {}]}}]"#, reads: "qty")) + #expect(try view(#"[{"$replaceWith": {"$mergeObjects": [{"$objectToArray": "$$ROOT"}]}}]"#, reads: "qty")) + } + + /// A view that passes the document on as the document is the view `[{$match: ...}]` already is: + /// it reads the fields it names, and a later stage can take the document apart only as `$$ROOT`, + /// which is read where it stands. + @Test("A view that keeps the whole document as the document reads only the fields it names") + func viewPassingTheDocumentOn() throws { + #expect(try view(#"[{"$replaceRoot": {"newRoot": "$$ROOT"}}]"#, reads: "qty") == false) + #expect(try view(#"[{"$replaceWith": {"$mergeObjects": [{"note": ""}, "$$ROOT"]}}]"#, reads: "qty") == false) + #expect(try view(#"[{"$replaceWith": {"$mergeObjects": [{"note": ""}, "$$ROOT"]}}]"#, reads: "note")) + #expect(try view(#"[{"$replaceWith": {"$setField": {"field": "a", "input": "$$ROOT", "value": 1}}}]"#, reads: "b") == false) + #expect(try view(#"[{"$replaceWith": {"$unsetField": {"field": "a", "input": "$$CURRENT"}}}]"#, reads: "b") == false) + let branch = #"[{"$replaceWith": {"$cond": {"if": {"$eq": ["$a", 1]}, "then": "$$ROOT", "else": {"$ifNull": ["$$ROOT", {}]}}}}]"# + #expect(try view(branch, reads: "qty") == false) + let switched = #"[{"$replaceWith": {"$switch": {"branches": [{"case": "$flag", "then": "$$ROOT"}], "default": "$$CURRENT"}}}]"# + #expect(try view(switched, reads: "qty") == false) + #expect(try view(#"[{"$replaceWith": {"$let": {"vars": {}, "in": "$$ROOT"}}}]"#, reads: "qty") == false) + #expect(try view(#"[{"$project": {"v": {"$getField": {"field": "a", "input": "$$ROOT"}}}}]"#, reads: "qty") == false) + #expect(try view(#"[{"$project": {"v": {"$getField": {"field": "a", "input": "$$ROOT"}}}}]"#, reads: "a")) + #expect(try view(#"[{"$replaceWith": {"$setField": {"field": "a", "input": "$$ROOT", "value": "$$ROOT"}}}]"#, reads: "b")) + } + + /// One rule, so the same expression reads the same fields whether a validator's `$expr` holds it + /// or a view computes it. + @Test("A validator and a view holding the same expression answer the same for a field neither names") + func validatorAndViewAgree() throws { + let expressions = [ + #"{"$objectToArray": "$$ROOT"}"#, + #"{"$size": {"$objectToArray": "$$CURRENT"}}"#, + #"{"$eq": ["$$ROOT", {"a": 1}]}"#, + #"{"$mergeObjects": ["$$ROOT", {"a": 1}]}"#, + #"{"$setField": {"field": "a", "input": "$$ROOT", "value": 1}}"#, + #"{"$unsetField": {"field": "a", "input": "$$CURRENT"}}"#, + #"{"$getField": {"field": "a", "input": "$$ROOT"}}"#, + #"{"$getField": {"field": "a", "input": {"$mergeObjects": ["$$ROOT", {}]}}}"#, + #"{"$let": {"vars": {"d": "$$ROOT"}, "in": "$$d.a"}}"#, + #"{"$cond": [true, "$$ROOT", {}]}"#, + #"{"$gt": ["$a", 1]}"# + ] + for text in expressions { + let expression = try #require(MongoJsonValue.parse(text)) + let validator = MongoQueryFieldReferences.reaches(["$expr": expression], field: "unnamed") + let pipeline = try #require(MongoJsonValue.parse(#"[{"$project": {"x": "# + text + "}}]") as? [Any]) + let viewReads = MongoViewDefinition(name: "v", viewOn: "c", pipeline: pipeline).readsField("unnamed") + #expect(validator == viewReads, "\(text)") + } + #expect(MongoQueryFieldReferences.reaches(["$expr": ["$getField": ["field": "a", "input": "$$ROOT"]]], field: "unnamed") == false) + #expect(MongoQueryFieldReferences.reaches(["$expr": ["$objectToArray": "$$ROOT"]], field: "unnamed")) + } + + /// A document-level `enum` lists whole documents, and a document matches an entry only with + /// exactly its names. A rename carried only through `properties` left this validator matching + /// no renamed document. + @Test("A document listed in a schema's enum names the fields it holds, at the document's level only") + func documentEnumEntriesNameFields() throws { + let listed = #"{"$jsonSchema": {"enum": [{"_id": 1, "old": "a"}, {"_id": 2, "old": "b"}]}}"# + #expect(try query(listed, reaches: "old")) + #expect(try query(listed, reaches: "new") == false) + #expect(try query(#"{"$jsonSchema": {"anyOf": [{"enum": [{"old": 1}]}]}}"#, reaches: "old")) + #expect(try query(#"{"$jsonSchema": {"properties": {"tags": {"enum": [{"old": 1}]}}}}"#, reaches: "old") == false) + #expect(try query(#"{"$jsonSchema": {"enum": ["old", {"$numberInt": "1"}]}}"#, reaches: "old") == false) + func applies(_ json: String, to name: String) throws -> Bool { + MongoQueryFieldReferences.appliesByName(try #require(MongoJsonValue.parse(json)), to: name) + } + #expect(try applies(listed, to: "old")) + #expect(try applies(#"{"$jsonSchema": {"not": {"enum": [{"new": 1}]}}}"#, to: "new")) + #expect(try applies(listed, to: "other") == false) + } + + @Test("A variable bound to the document with $let is read as the document") + func letBoundDocument() throws { + #expect(try query(#"{"$expr": {"$let": {"vars": {"doc": "$$ROOT"}, "in": {"$gt": ["$$doc.qty", 0]}}}}"#, reaches: "qty")) + } + + @Test("$jsonSchema names its required, properties and dependencies at the document's level only") + func jsonSchemaLevels() throws { + let schema = #"{"$jsonSchema": {"bsonType": "object", "required": ["name"], "# + + #""properties": {"name": {"bsonType": "string"}, "status": {"enum": ["active", "gone"]}, "# + + #""active": {"bsonType": "bool"}, "address": {"bsonType": "object", "# + + #""properties": {"zip": {"bsonType": "string"}}}}, "description": "city"}}"# + #expect(try query(schema, reaches: "name")) + #expect(try query(schema, reaches: "status")) + #expect(try query(schema, reaches: "active")) + #expect(try query(schema, reaches: "zip") == false) + #expect(try query(schema, reaches: "gone") == false) + #expect(try query(schema, reaches: "city") == false) + #expect(try query(#"{"$jsonSchema": {"dependencies": {"a": ["b"]}}}"#, reaches: "a")) + #expect(try query(#"{"$jsonSchema": {"dependencies": {"a": ["b"]}}}"#, reaches: "b")) + #expect(try query(#"{"$jsonSchema": {"allOf": [{"required": ["x"]}]}}"#, reaches: "x")) + #expect(try query(#"{"$jsonSchema": {"not": {"required": ["x"]}}}"#, reaches: "x")) + } + + @Test("A pattern names the fields it matches, and a pattern that does not compile names every field") + func patternProperties() throws { + let schema = #"{"$jsonSchema": {"patternProperties": {"^tmp_": {"bsonType": "int"}}}}"# + #expect(try query(schema, reaches: "tmp_x")) + #expect(try query(schema, reaches: "x") == false) + #expect(try query(#"{"$jsonSchema": {"patternProperties": {"([": {}}}}"#, reaches: "x")) + } + + @Test("A view's pipeline names match and projection keys, $-strings and literal field parameters") + func pipelines() throws { + let activeOrders = #"[ { "$match" : { "status" : "active" } }, { "$project" : { "total" : { "$numberInt" : "1" } } } ]"# + #expect(try pipeline(activeOrders, reaches: "status")) + #expect(try pipeline(activeOrders, reaches: "total")) + #expect(try pipeline(activeOrders, reaches: "stat") == false) + let joined = #"[ { "$lookup" : { "from" : "orders", "localField" : "sku", "foreignField" : "code", "as" : "o" } } ]"# + #expect(try pipeline(joined, reaches: "sku")) + #expect(try pipeline(joined, reaches: "code")) + #expect(try pipeline(#"[{"$unset": ["a", "b"]}]"#, reaches: "b")) + #expect(try pipeline(#"[{"$unset": "a"}]"#, reaches: "a")) + #expect(try pipeline(#"[{"$group": {"_id": "$$CURRENT.note"}}]"#, reaches: "note")) + #expect(try pipeline(#"[{"$facet": {"x": [{"$sortByCount": "$tag"}]}}]"#, reaches: "tag")) + #expect(try pipeline(#"[{"$match": {"at": {"$date": {"$numberLong": "0"}}}}]"#, reaches: "numberLong") == false) + #expect(try pipeline(#"[{"$addFields": {"k": {"$getField": "tag"}}}]"#, reaches: "tag")) + #expect(try pipeline(#"[{"$addFields": {"k": {"$function": {"body": "", "args": [], "lang": "js"}}}}]"#, reaches: "x")) + } + + /// A view's grammar keeps growing, so any string in it that could be the field counts, values + /// and `$literal` included. A value that only looks like the field is a refused save; a stage + /// that names the field in a way a list of parameters does not know is a view left reading a + /// field that is gone. + @Test("Any string in a view that could be the field counts, values and $literal included") + func pipelineStringsCountWherever() throws { + let activeOrders = #"[ { "$match" : { "status" : "active" } } ]"# + #expect(try pipeline(activeOrders, reaches: "active")) + #expect(try pipeline(#"[{"$lookup": {"from": "orders", "localField": "a", "foreignField": "b", "as": "o"}}]"#, reaches: "orders")) + #expect(try pipeline(#"[{"$project": {"k": {"$literal": "$tag"}}}]"#, reaches: "tag")) + #expect(try pipeline(#"[{"$project": {"p": {"$getField": {"$literal": "$price"}}}}]"#, reaches: "price")) + #expect(try pipeline(#"[{"$project": {"k": {"$literal": "tag.x"}}}]"#, reaches: "tag")) + } + + @Test("$densify and $fill name the fields of their partitionByFields list") + func partitionByFieldsLists() throws { + let densify = #"[ { "$densify" : { "field" : "ts", "partitionByFields" : [ "tenant" ], "# + + #""range" : { "step" : { "$numberInt" : "1" }, "unit" : "hour", "bounds" : "partition" } } } ]"# + #expect(try pipeline(densify, reaches: "tenant")) + #expect(try pipeline(densify, reaches: "ts")) + #expect(try pipeline(densify, reaches: "ten") == false) + let fill = #"[ { "$fill" : { "partitionByFields" : [ "region", "tenant" ], "sortBy" : { "ts" : { "$numberInt" : "1" } }, "# + + #""output" : { "qty" : { "method" : "linear" } } } } ]"# + #expect(try pipeline(fill, reaches: "tenant")) + #expect(try pipeline(fill, reaches: "region")) + #expect(try pipeline(fill, reaches: "qty")) + #expect(try pipeline(fill, reaches: "price") == false) + } + + @Test("A path given as a list names every field in it, and a path under the field names the field") + func arrayValuedPath() throws { + let search = #"[{"$search": {"text": {"query": "x", "path": ["title", "tenant.name"]}}}]"# + #expect(try pipeline(search, reaches: "tenant")) + #expect(try pipeline(search, reaches: "title")) + #expect(try pipeline(search, reaches: "x")) + #expect(try pipeline(search, reaches: "titles") == false) + #expect(try pipeline(#"[{"$search": {"text": {"query": "x", "path": {"wildcard": "tenant.*"}}}}]"#, reaches: "tenant")) + } + + @Test("A stage this build has never heard of still names the field wherever a string does") + func unknownStage() throws { + #expect(try pipeline(#"[{"$futureStage": {"by": "tenant"}}]"#, reaches: "tenant")) + #expect(try pipeline(#"[{"$futureStage": ["region", "tenant"]}]"#, reaches: "tenant")) + #expect(try pipeline(#"[{"$futureStage": {"keys": [{"on": "$tenant.id"}]}}]"#, reaches: "tenant")) + #expect(try pipeline(#"[{"$futureStage": {"tenant": true}}]"#, reaches: "tenant")) + #expect(try pipeline(#"[{"$futureStage": {"by": "tenants"}}]"#, reaches: "tenant") == false) + } + + /// Measured on 7.0.43: this view reads `[{"_id": 1, "s": [{"tenant": "a"}]}]`, and after the + /// collection's `tenant` was renamed it read `[{"_id": 1, "s": [{}]}]` with no error. + @Test("A field read under a name an earlier stage gave the document counts") + func fieldUnderAnotherName() throws { + let joined = #"[{"$lookup": {"from": "src", "localField": "sid", "foreignField": "sid", "as": "s"}}, "# + + #"{"$project": {"s.tenant": {"$numberInt": "1"}}}]"# + #expect(try pipeline(joined, reaches: "tenant")) + #expect(try pipeline(#"[{"$group": {"_id": null, "docs": {"$push": "$$ROOT"}}}, {"$match": {"docs.tenant": "a"}}]"#, reaches: "tenant")) + #expect(try pipeline(#"[{"$project": {"t": "$doc.tenant"}}]"#, reaches: "tenant")) + } + + @Test("A number, a date or an id names no field, a symbol names one like a string, and code reads every field") + func wrappedValues() throws { + #expect(try pipeline(#"[{"$limit": {"$numberInt": "5"}}]"#, reaches: "5") == false) + #expect(try pipeline(#"[{"$match": {"_id": {"$oid": "6ab746850efe77e864860657"}}}]"#, reaches: "6ab746850efe77e864860657") == false) + #expect(try pipeline(#"[{"$project": {"s": {"$literal": {"$symbol": "tenant"}}}}]"#, reaches: "tenant")) + #expect(try pipeline(#"[{"$match": {"$where": {"$code": "this.x"}}}]"#, reaches: "tenant")) + #expect(try pipeline(#"[{"$project": {"c": {"$literal": {"$code": "1"}}}}]"#, reaches: "tenant")) + } +} diff --git a/TableProTests/Plugins/MongoSearchIndexTests.swift b/TableProTests/Plugins/MongoSearchIndexTests.swift new file mode 100644 index 0000000000..ab9d43c435 --- /dev/null +++ b/TableProTests/Plugins/MongoSearchIndexTests.swift @@ -0,0 +1,135 @@ +// +// MongoSearchIndexTests.swift +// TableProTests +// + +import Foundation +import Testing + +/// The listings are the `$listSearchIndexes` output the MongoDB manual documents for Atlas, written +/// in the canonical Extended JSON libmongoc hands back, one host kept from each `statusDetail`. No +/// Atlas cluster was reachable from the live checks, and MongoDB 7.0.43 refuses the stage. +struct MongoSearchIndexTests { + enum Fixture { + static let synonymMappings = #"{ "id" : "65240be420da840844a4d077", "name" : "synonym_mappings", "status" : "READY", "# + + #""queryable" : true, "latestDefinitionVersion" : { "version" : { "$numberInt" : "0" }, "# + + #""createdAt" : { "$date" : { "$numberLong" : "1696861156305" } } }, "# + + #""latestDefinition" : { "mappings" : { "dynamic" : true, "fields" : { "fullplot" : { "type" : "string" } } }, "# + + #""synonyms" : [ { "name" : "synonym_mapping", "analyzer" : "lucene.english", "# + + #""source" : { "collection" : "synonyms" } } ] }, "synonymMappingStatus" : "READY", "# + + #""synonymMappingStatusDetail" : [ { "synonym_mapping" : { "status" : "READY", "queryable" : true } } ], "# + + #""statusDetail" : [ { "hostname" : "atlas-n1cm1j-shard-00-02", "status" : "READY", "queryable" : true, "# + + #""mainIndex" : { "status" : "READY", "queryable" : true, "definitionVersion" : { "# + + #""version" : { "$numberInt" : "0" }, "createdAt" : { "$date" : { "$numberLong" : "1696861156000" } } }, "# + + #""definition" : { "mappings" : { "dynamic" : true, "fields" : { "fullplot" : { "type" : "string", "# + + #""indexOptions" : "offsets", "store" : true, "norms" : "include" } } }, "# + + #""synonyms" : [ { "name" : "synonym_mapping", "analyzer" : "lucene.english", "# + + #""source" : { "collection" : "synonyms" } } ] } } } ] }"# + + static let storedSource = #"{ "id" : "6524096020da840844a4c4a7", "name" : "default", "status" : "BUILDING", "# + + #""queryable" : true, "latestDefinitionVersion" : { "version" : { "$numberInt" : "2" }, "# + + #""createdAt" : { "$date" : { "$numberLong" : "1696863117355" } } }, "# + + #""latestDefinition" : { "mappings" : { "dynamic" : true }, "storedSource" : { "include" : [ "awards.text" ] } }, "# + + #""statusDetail" : [ { "hostname" : "atlas-n1cm1j-shard-00-02", "status" : "BUILDING", "queryable" : true, "# + + #""mainIndex" : { "status" : "READY", "queryable" : true, "definitionVersion" : { "# + + #""version" : { "$numberInt" : "0" }, "createdAt" : { "$date" : { "$numberLong" : "1696860512000" } } }, "# + + #""definition" : { "mappings" : { "dynamic" : true, "fields" : { } } } }, "# + + #""stagedIndex" : { "status" : "PENDING", "queryable" : false, "definitionVersion" : { "# + + #""version" : { "$numberInt" : "1" }, "createdAt" : { "$date" : { "$numberLong" : "1696863089000" } } }, "# + + #""definition" : { "mappings" : { "dynamic" : true, "fields" : { } }, "storedSource" : true } } } ] }"# + + static let vectorSearch = #"{ "id" : "6524096020da840844a4c4b1", "name" : "vector_index", "type" : "vectorSearch", "# + + #""status" : "READY", "queryable" : true, "latestDefinitionVersion" : { "version" : { "$numberInt" : "0" }, "# + + #""createdAt" : { "$date" : { "$numberLong" : "1696861156305" } } }, "# + + #""latestDefinition" : { "fields" : [ { "type" : "vector", "numDimensions" : { "$numberInt" : "1536" }, "# + + #""path" : "plot_embedding", "similarity" : "dotProduct" }, { "type" : "filter", "path" : "genres" } ] } }"# + + /// An update in progress: the definition the host still serves names `tenant`, the new one + /// does not yet. + static let updating = #"{ "id" : "1", "name" : "catalog", "status" : "BUILDING", "queryable" : true, "# + + #""latestDefinition" : { "mappings" : { "dynamic" : false, "fields" : { "title" : { "type" : "string" } } } }, "# + + #""statusDetail" : [ { "hostname" : "h", "mainIndex" : { "definition" : { "mappings" : { "dynamic" : false, "# + + #""fields" : { "title" : { "type" : "string" }, "tenant" : { "type" : "token" } } } } } } ] }"# + } + + private func index(_ json: String) throws -> MongoSearchIndex { + try #require(MongoSearchIndex(json: json)) + } + + @Test("A mapping names its field as a key, in the latest definition and in each host's") + func mappingKeys() throws { + let synonyms = try index(Fixture.synonymMappings) + #expect(synonyms.name == "synonym_mappings") + #expect(synonyms.mentions("fullplot")) + #expect(!synonyms.mentions("plot")) + #expect(!synonyms.mentions("full")) + } + + @Test("A vector index names its fields by path, and storedSource by a list of paths") + func paths() throws { + let vector = try index(Fixture.vectorSearch) + #expect(vector.mentions("plot_embedding")) + #expect(vector.mentions("genres")) + #expect(!vector.mentions("plot")) + let stored = try index(Fixture.storedSource) + #expect(stored.mentions("awards")) + #expect(stored.mentions("text")) + #expect(!stored.mentions("award")) + } + + @Test("The definition a host still serves counts while a new one is being built") + func servedDefinitionCounts() throws { + let updating = try index(Fixture.updating) + #expect(updating.mentions("tenant")) + #expect(updating.mentions("title")) + } + + /// The listing around a definition is status, not definition: `status`, `queryable` and + /// `hostname` are the server's words about the index. Inside a definition every key and every + /// string counts, grammar included, because a false refusal costs one save and a missed one + /// leaves an index on a field that is gone. + @Test("Only definitions are read, and every key and string inside one counts") + func onlyDefinitionsCount() throws { + let synonyms = try index(Fixture.synonymMappings) + #expect(!synonyms.mentions("status")) + #expect(!synonyms.mentions("queryable")) + #expect(!synonyms.mentions("hostname")) + #expect(!synonyms.mentions("READY")) + #expect(synonyms.mentions("mappings")) + #expect(synonyms.mentions("synonyms")) + } + + /// Measured: `$listSearchIndexes` and `$search` both fail with 6047401 on MongoDB 6.0.28 and + /// 7.0.43 community, and both with 31082 on 8.2.12 community with no `mongot`. MongoDB 5.0.33 + /// community fails both with 40324, the code for a stage the server does not know. + @Test("A server with no search says so for every search stage, and has no search index") + func serverWithoutSearch() { + #expect(MongoSearchIndex.listingFailure(code: 6_047_401) == .serverWithoutSearch) + #expect(MongoSearchIndex.listingFailure(code: 31_082) == .serverWithoutSearch) + for code: UInt32 in [0, 6, 13, 50, 59, 89, 91, 11_600, 13_435] { + #expect(MongoSearchIndex.listingFailure(code: code) == .unknown, "\(code)") + } + #expect(MongoSearchIndex.listingPipelineJson == #"[{"$listSearchIndexes": {}}]"#) + } + + /// An Atlas cluster older than `$listSearchIndexes` answers it with 40324 and still runs + /// `$search` over search indexes it cannot list. Reading 40324 as "no indexes" let a rename + /// leave such an index mapping a field no document has. + @Test("A server that does not know the listing stage has search indexes unless it does not know $search either") + func listingStageUnknown() { + #expect(MongoSearchIndex.listingFailure(code: 40_324) == .listingStageUnknown) + #expect(MongoSearchIndex.searchIsUnavailable(probeErrorCode: nil) == false) + #expect(MongoSearchIndex.searchIsUnavailable(probeErrorCode: 40_324)) + #expect(MongoSearchIndex.searchIsUnavailable(probeErrorCode: 6_047_401)) + #expect(MongoSearchIndex.searchIsUnavailable(probeErrorCode: 31_082)) + #expect(MongoSearchIndex.searchIsUnavailable(probeErrorCode: 8) == false) + #expect(MongoSearchIndex.searchIsUnavailable(probeErrorCode: 13) == false) + #expect(MongoSearchIndex.searchProbePipelineJson.hasPrefix(#"[{"$search": "#)) + } + + @Test("A listing that is not a document is no index") + func unreadableListing() { + #expect(MongoSearchIndex(json: "[]") == nil) + #expect(MongoSearchIndex(json: "not json") == nil) + } +} diff --git a/TableProTests/Plugins/MongoWriteFailureTests.swift b/TableProTests/Plugins/MongoWriteFailureTests.swift index 24b87c9533..dd413e7995 100644 --- a/TableProTests/Plugins/MongoWriteFailureTests.swift +++ b/TableProTests/Plugins/MongoWriteFailureTests.swift @@ -79,6 +79,18 @@ struct MongoWriteFailureTests { #expect(failure?.message == MongoScriptText.writeRefused(code: 2)) } + @Test("A command reply carrying a write concern error is a failure, and its write errors are not") + func commandConcernFailure() { + let collMod = #"{"ok":1,"writeConcernError":{"code":64,"errmsg":"waiting for replication timed out"}}"# + let failure = MongoWriteFailure.concernFailure(fromReply: collMod) + #expect(failure?.code == 64) + #expect(failure?.message == MongoScriptText.writeNotAcknowledged(reason: "waiting for replication timed out")) + #expect(MongoWriteFailure.concernFailure(fromReply: #"{"ok":1}"#) == nil) + #expect(MongoWriteFailure.concernFailure(fromReply: """ + {"writeErrors":[{"index":0,"code":121,"errmsg":"Document failed validation"}],"ok":1} + """) == nil) + } + @Test("Text that is not a reply reads as no failure") func unreadableReply() { #expect(MongoWriteFailure.read(fromReply: "") == nil) diff --git a/TableProTests/Views/Main/CatalogChangeWindowTests.swift b/TableProTests/Views/Main/CatalogChangeWindowTests.swift index 2e884bc2dd..cef585276f 100644 --- a/TableProTests/Views/Main/CatalogChangeWindowTests.swift +++ b/TableProTests/Views/Main/CatalogChangeWindowTests.swift @@ -130,6 +130,147 @@ struct CatalogChangeWindowTests { #expect(tabManager.tabs[1].tableContext.schemaName == "public") } + private static func loadRows(_ tab: QueryTab, into coordinator: MainContentCoordinator, _ tabManager: QueryTabManager) { + guard let index = tabManager.tabs.firstIndex(where: { $0.id == tab.id }) else { return } + let tableRows = TestFixtures.makeTableRows(rowCount: 3) + coordinator.setActiveTableRows(tableRows, for: tab.id) + let resultSet = ResultSet(label: tab.title, tableRows: tableRows) + tabManager.mutate(at: index) { tab in + tab.display.resultSets = [resultSet] + tab.display.activeResultSetId = resultSet.id + tab.execution.lastExecutedAt = Date() + } + } + + private static func structureSession( + for tab: QueryTab, + connection: DatabaseConnection, + into coordinator: MainContentCoordinator + ) -> StructureEditingSession { + let session = StructureEditingSession( + identity: tab.id.uuidString, + connection: connection, + databaseName: tab.tableContext.databaseName, + schemaName: nil, + tableName: tab.tableContext.tableName ?? "" + ) + session.hasLoaded = true + coordinator.structureSessions[tab.id] = session + return session + } + + /// A Structure save reloaded the selected tab of each window on the database whatever table it + /// showed, skipped the saving tab's Data view because Structure was in front, and left every + /// background tab on the table with the rows from before the save. + @Test("a structure change reloads the table's rows behind Structure and in the background, and no other table's") + func structureChangeReachesEveryTabOnTheTable() { + let connection = TestFixtures.makeConnection(database: "shop") + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let (coordinator, tabManager) = Self.makeCoordinator(connection: connection) + defer { coordinator.cancelAllQueryTasks() } + var saving = Self.tableTab("orders", database: "shop", schema: nil) + saving.display.resultsViewMode = .structure + let background = Self.tableTab("orders", database: "shop", schema: nil) + let unrelated = Self.tableTab("users", database: "shop", schema: nil) + tabManager.tabs = [background, unrelated, saving] + for tab in tabManager.tabs { + Self.loadRows(tab, into: coordinator, tabManager) + } + tabManager.selectedTabId = saving.id + + coordinator.applyObjectChange( + Self.change(connection, name: "orders", database: "shop", schema: nil, kind: .structure), + hasPendingTableOps: false, + onDiscard: {} + ) + + #expect(coordinator.queryTasks.hasTask(for: saving.id)) + #expect(coordinator.tabSessionRegistry.isEvicted(background.id)) + #expect(!coordinator.tabSessionRegistry.isEvicted(unrelated.id)) + #expect(!coordinator.queryTasks.hasTask(for: unrelated.id)) + #expect(coordinator.tabSessionRegistry.tableRows(for: unrelated.id).rows.count == 3) + } + + @Test("a structure change leaves the rows of a tab holding data edits, and never prompts") + func structureChangeKeepsEditedRows() { + let connection = TestFixtures.makeConnection(database: "shop") + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let (coordinator, tabManager) = Self.makeCoordinator(connection: connection) + defer { coordinator.cancelAllQueryTasks() } + var saving = Self.tableTab("orders", database: "shop", schema: nil) + saving.display.resultsViewMode = .structure + saving.pendingChanges.deletedRowIDs = [.existing(0)] + tabManager.tabs = [saving] + Self.loadRows(saving, into: coordinator, tabManager) + tabManager.selectedTabId = saving.id + + coordinator.applyObjectChange( + Self.change(connection, name: "orders", database: "shop", schema: nil, kind: .structure), + hasPendingTableOps: false, + onDiscard: {} + ) + + #expect(!coordinator.queryTasks.hasTask(for: saving.id)) + #expect(coordinator.tabSessionRegistry.tableRows(for: saving.id).rows.count == 3) + } + + /// The saving tab still holds its edits while its save runs, and keeps them when the save + /// stops partway, so a refetch, which adopts a new baseline, would throw away what the retry + /// needs. + @Test("a structure change marks unedited structure stale and leaves staged edits and their baseline alone") + func structureChangeSparesStagedEdits() { + let connection = TestFixtures.makeConnection(database: "shop") + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let (coordinator, tabManager) = Self.makeCoordinator(connection: connection) + defer { coordinator.cancelAllQueryTasks() } + let clean = Self.tableTab("orders", database: "shop", schema: nil) + let edited = Self.tableTab("orders", database: "shop", schema: nil) + let other = Self.tableTab("users", database: "shop", schema: nil) + tabManager.tabs = [clean, edited, other] + let cleanSession = Self.structureSession(for: clean, connection: connection, into: coordinator) + let editedSession = Self.structureSession(for: edited, connection: connection, into: coordinator) + editedSession.changeManager.addNewColumn() + let otherSession = Self.structureSession(for: other, connection: connection, into: coordinator) + tabManager.selectedTabId = other.id + + coordinator.applyObjectChange( + Self.change(connection, name: "orders", database: "shop", schema: nil, kind: .structure), + hasPendingTableOps: false, + onDiscard: {} + ) + + #expect(cleanSession.hasLoaded == false) + #expect(editedSession.hasLoaded) + #expect(editedSession.changeManager.hasChanges) + #expect(otherSession.hasLoaded) + } + + /// A reload of a tab with hidden columns builds its select list from these before it fetches + /// anything, so a column the save dropped would still be named in it. + @Test("a structure change forgets the table's cached columns and keeps every other table's") + func structureChangeForgetsCachedColumns() { + let connection = TestFixtures.makeConnection(database: "shop") + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let (coordinator, tabManager) = Self.makeCoordinator(connection: connection) + let orders = Self.tableTab("orders", database: "shop", schema: nil) + tabManager.tabs = [orders] + let scope = DatabaseScope(connectionId: connection.id, database: "shop", schema: nil) + let entry = SchemaColumnStore.Entry(columns: ["id", "old"], primaryKeys: ["id"], columnTypes: [:]) + let ordersKey = coordinator.schemaColumnsKey("orders", scope: scope) + let usersKey = coordinator.schemaColumnsKey("users", scope: scope) + coordinator.schemaColumns.store(entry, for: ordersKey) + coordinator.schemaColumns.store(entry, for: usersKey) + + coordinator.applyObjectChange( + Self.change(connection, name: "orders", database: "shop", schema: nil, kind: .structure), + hasPendingTableOps: false, + onDiscard: {} + ) + + #expect(coordinator.schemaColumns.cached(ordersKey) == nil) + #expect(coordinator.schemaColumns.cached(usersKey) == entry) + } + @Test("a change for another connection is ignored") func otherConnectionIsIgnored() { let connection = TestFixtures.makeConnection(database: "shop") diff --git a/TableProTests/Views/Structure/StructureEditGateTests.swift b/TableProTests/Views/Structure/StructureEditGateTests.swift index 0afc9a7b2f..cb73ba33ea 100644 --- a/TableProTests/Views/Structure/StructureEditGateTests.swift +++ b/TableProTests/Views/Structure/StructureEditGateTests.swift @@ -4,9 +4,9 @@ // import Foundation +@testable import TablePro import TableProPluginKit import Testing -@testable import TablePro /// The gate is the one impure half of the decision: it reads the engine's curated matrix and its /// capability flags, and every call site in the Structure tab asks it rather than reading a flag of @@ -27,6 +27,31 @@ struct StructureEditGateTests { } } + /// A collection declares no columns, so the two column edits it takes are the two that can be + /// carried out on its documents. This is the gate that kept every MongoDB collection read-only. + @Test("A MongoDB collection renames and removes fields and refuses every other structure edit") + func mongoDBCollectionEdits() { + let collection = gate(.table, .mongodb) + #expect(collection.allowsAnyEdit) + #expect(collection.allows(.renameColumn)) + #expect(collection.allows(.dropColumn)) + for operation in StructureEditOperation.allCases where operation != .renameColumn && operation != .dropColumn { + #expect(!collection.allows(operation), "collection allowed \(operation)") + } + #expect(collection.editableColumnFields == [.name]) + } + + @Test("MongoDB's refusals say what MongoDB support cannot generate, not that a collection has no indexes") + func mongoDBRefusalWording() { + let collection = gate(.table, .mongodb) + #expect(collection.resolve(.addColumn).unavailableReason == String( + format: String(localized: "%@ cannot make this change to a table's columns."), "MongoDB" + )) + #expect(collection.resolve(.addIndex).unavailableReason == String( + format: String(localized: "%@ cannot add or remove a table's indexes."), "MongoDB" + )) + } + @Test("A PostgreSQL view refuses a column and an index but keeps a rename and a default") func viewSplitsByOperation() { let view = gate(.view) diff --git a/TableProTests/Views/Structure/StructureEditingSessionTests.swift b/TableProTests/Views/Structure/StructureEditingSessionTests.swift index db992ea301..f24ff14d7f 100644 --- a/TableProTests/Views/Structure/StructureEditingSessionTests.swift +++ b/TableProTests/Views/Structure/StructureEditingSessionTests.swift @@ -37,8 +37,59 @@ private class StructureSessionBaseDriver { } } +/// Holds a save inside its composition, the first place it suspends, until the test lets it go. +private final class SaveGate: @unchecked Sendable { + private let lock = NSLock() + private var hasArrived = false + private var isOpen = false + private var arrival: CheckedContinuation? + private var departures: [CheckedContinuation] = [] + + func pass() async { + await withCheckedContinuation { continuation in + let (waiting, proceed) = lock.withLock { () -> (CheckedContinuation?, Bool) in + hasArrived = true + defer { arrival = nil } + if !isOpen { departures.append(continuation) } + return (arrival, isOpen) + } + waiting?.resume() + if proceed { continuation.resume() } + } + } + + func arrived() async { + await withCheckedContinuation { continuation in + let already = lock.withLock { () -> Bool in + if !hasArrived { arrival = continuation } + return hasArrived + } + if already { continuation.resume() } + } + } + + func open() { + let waiting = lock.withLock { () -> [CheckedContinuation] in + isOpen = true + defer { departures = [] } + return departures + } + waiting.forEach { $0.resume() } + } +} + private final class StructureSessionDriver: StructureSessionBaseDriver, PluginDatabaseDriver, @unchecked Sendable { private(set) var executedQueries: [String] = [] + var compositionGate: SaveGate? + + func reviewSchemaChange( + table: String, + schema: String?, + operations: [PluginSchemaOperation] + ) async throws -> PluginSchemaChangeReview { + await compositionGate?.pass() + return PluginSchemaChangeReview() + } func execute(query: String) async throws -> PluginQueryResult { executedQueries.append(query) @@ -193,6 +244,93 @@ struct StructureEditingSessionTests { #expect(!session.hasLoaded) } + @Test("A save pressed while one is writing does nothing and keeps the edits staged") + func saveWhileApplyingIsRefused() async throws { + let connection = TestFixtures.makeConnection(database: "testdb") + let sessionDriver = StructureSessionDriver() + var connectionSession = ConnectionSession( + connection: connection, + driver: PluginDriverAdapter(connection: connection, pluginDriver: sessionDriver) + ) + connectionSession.browseDatabase = "testdb" + DatabaseManager.shared.injectSession(connectionSession, for: connection.id) + + let session = Self.makeSession(connection: connection) + let pooledDriver = try await Self.seedPooledDriver(connection, scope: session.scope) + defer { + MetadataConnectionPool.shared.closeAll(connectionId: connection.id) + DatabaseManager.shared.removeSession(for: connection.id) + } + + Self.stageAColumn(on: session) + session.isApplying = true + + #expect(await session.applyStagedChanges(coordinator: nil) == .refused) + #expect(pooledDriver.executedQueries.isEmpty) + #expect(session.changeManager.hasChanges) + } + + /// The save composes its statements before it writes, and on MongoDB composing reads the catalog, + /// so the gate holds it there. Before the hold, `isApplying` was raised only after composition, + /// so a second press passed the guard, and an edit staged meanwhile missed the script and was + /// then cleared with the edits that ran. + @Test("A save holds its edits from the press: a second Save and a mid-save edit are refused, and it clears only what it wrote") + func saveHoldsItsSnapshot() async throws { + let connection = TestFixtures.makeConnection(database: "testdb") + let sessionDriver = StructureSessionDriver() + var connectionSession = ConnectionSession( + connection: connection, + driver: PluginDriverAdapter(connection: connection, pluginDriver: sessionDriver) + ) + connectionSession.browseDatabase = "testdb" + DatabaseManager.shared.injectSession(connectionSession, for: connection.id) + + let session = Self.makeSession(connection: connection) + let pooledDriver = try await Self.seedPooledDriver(connection, scope: session.scope) + let gate = SaveGate() + pooledDriver.compositionGate = gate + defer { + MetadataConnectionPool.shared.closeAll(connectionId: connection.id) + DatabaseManager.shared.removeSession(for: connection.id) + } + + Self.stageAColumn(on: session) + let manager = session.changeManager + let pressed = manager.getChangesArray() + + /// A failed `#require` leaves this save parked at the gate for good. Opening the gate after + /// the pool is gone would send it to an error alert with no window to hang on, which is a + /// modal run loop the test host never leaves. + let first = Task { await session.applyStagedChanges(coordinator: nil) } + await gate.arrived() + + try #require(session.isApplying) + try #require(manager.isHeldForSave) + #expect(await session.applyStagedChanges(coordinator: nil) == .refused) + + manager.addNewColumn() + if var column = manager.workingColumns.last { + column.name = "later" + manager.updateColumn(id: column.id, with: column) + } + manager.undo() + manager.discardChanges() + #expect(manager.getChangesArray() == pressed) + + gate.open() + #expect(await first.value == .applied) + + #expect(pooledDriver.executedQueries.filter { $0.contains("ADD COLUMN") }.count == 1) + #expect(pooledDriver.executedQueries.allSatisfy { !$0.contains("later") }) + #expect(!manager.hasChanges) + #expect(!manager.isHeldForSave) + #expect(!session.isApplying) + #expect(session.appliedVersion == 1) + + manager.addNewColumn() + #expect(manager.hasChanges) + } + /// Stands in for the connection the pool would open on the scope. private static func seedPooledDriver( _ connection: DatabaseConnection, diff --git a/TableProTests/Views/Structure/StructureFooterPolicyTests.swift b/TableProTests/Views/Structure/StructureFooterPolicyTests.swift index 56f214f3f1..0fbb1918d4 100644 --- a/TableProTests/Views/Structure/StructureFooterPolicyTests.swift +++ b/TableProTests/Views/Structure/StructureFooterPolicyTests.swift @@ -4,8 +4,8 @@ // import Foundation -import Testing @testable import TablePro +import Testing /// The "+" and "-" under a structure list used to take their label, their enabled state and their /// tooltip from three different switches, and only the Foreign Keys one asked about the object at @@ -17,12 +17,14 @@ struct StructureFooterPolicyTests { kind: TableInfo.TableType, matrix: StructureObjectEditMatrix = .postgreSQL, canEditSchema: Bool = true, - hasSelection: Bool = true + hasSelection: Bool = true, + isSaving: Bool = false ) -> StructureFooterCapability { StructureFooterPolicy.resolve( tab: tab, canEditSchema: canEditSchema, hasSelection: hasSelection, + isSaving: isSaving, resolve: { operation in StructureEditEligibility.resolve( operation, @@ -46,6 +48,20 @@ struct StructureFooterPolicyTests { } } + /// A save holds what is staged from the press until it ends, so the pair cannot stage more + /// meanwhile, and says why rather than going grey in silence. + @Test("A save in progress dims both buttons on every editable tab and says why") + func savingDimsThePair() { + for tab in [StructureTab.columns, .indexes, .foreignKeys, .checkConstraints] { + let capability = resolve(tab: tab, kind: .table, isSaving: true) + #expect(!capability.canAdd, "\(tab.rawValue)") + #expect(!capability.canRemove, "\(tab.rawValue)") + #expect(capability.isActive, "\(tab.rawValue)") + #expect(capability.unavailableReason == StructureFooterPolicy.savingReason, "\(tab.rawValue)") + } + #expect(!resolve(tab: .ddl, kind: .table, isSaving: true).isActive) + } + /// Shown and dimmed, not hidden. The pair disappearing would read as "this tab has no columns", /// and the tooltip is the only place the refusal can be stated. @Test("A view keeps the Columns pair on screen, dimmed, with a reason") diff --git a/TableProTests/Views/Structure/StructureGridDelegateInspectorTests.swift b/TableProTests/Views/Structure/StructureGridDelegateInspectorTests.swift index 0caab48847..a9e9737a3f 100644 --- a/TableProTests/Views/Structure/StructureGridDelegateInspectorTests.swift +++ b/TableProTests/Views/Structure/StructureGridDelegateInspectorTests.swift @@ -66,6 +66,21 @@ struct StructureGridDelegateInspectorTests { try #require(delegate.orderedFields.firstIndex(of: field)) } + @Test("A row is read-only while a save holds the edits, and editable again once it lets go") + func heldSaveMakesTheRowReadOnly() throws { + let manager = loadedManager() + let delegate = makeDelegate(manager: manager) + var email = try #require(manager.workingColumns.first { $0.name == "email" }) + email.comment = "Contact" + manager.updateColumn(id: email.id, with: email) + + let snapshot = try #require(manager.holdForSave()) + #expect(delegate.inspectorRow(atDisplayRow: 1)?.isEditable == false) + + manager.releaseHold(snapshot, written: false) + #expect(delegate.inspectorRow(atDisplayRow: 1)?.isEditable == true) + } + @Test("The published row describes the structure grid, not the data grid") func publishedRowDescribesStructure() throws { let delegate = makeDelegate(manager: loadedManager()) diff --git a/docs/databases/mongodb.mdx b/docs/databases/mongodb.mdx index 8145037bd8..dd76cb2504 100644 --- a/docs/databases/mongodb.mdx +++ b/docs/databases/mongodb.mdx @@ -35,14 +35,14 @@ Naming a **Database** skips listing every database on the server, which is worth **Auth Database** is a separate question: it says where your account is defined, not what you browse. Left empty, it follows the **Database** field. An account defined in `admin` needs **Auth Database** set to `admin` whenever **Database** names something else, or authentication fails. SRV connections authenticate against `admin` regardless, unless told otherwise. Switching databases in the app never changes it, so browsing a database your user has no account in is fine. -Also in Advanced: **Read Preference**, **Write Concern**, **Use SRV Record**, **Replica Set** name, and **Legacy UUID Encoding**. There is no minimum server version; the driver adapts what it asks for to what the server answers. +Also in Advanced: **Read Preference**, **Write Concern**, **Use SRV Record**, **Replica Set** name, and **Legacy UUID Encoding**. The server has to be MongoDB 4.0 or later; the driver refuses to connect to anything older. MongoDB connection form with the multi-host Hosts editor MongoDB connection form with the multi-host Hosts editor -On MongoDB 4.0 and later the database list is requested as authorized databases only, so an account without the `listDatabases` privilege still sees what it can read. On an older server that list comes back empty: name a **Database** on the connection instead. +The database list is requested as authorized databases only, so an account without the `listDatabases` privilege still sees what it can read. ## Connection URL @@ -79,7 +79,7 @@ An edited cell keeps its field's type. A date stays a date, whether typed as `20 The filter bar's column picker lists paths inside nested objects and arrays of objects, so `customer.country` and `items.sku` filter directly; a row on an array field chooses **any element** or **same element**, which makes one array entry satisfy every row set to it. See [Filtering](/features/filtering#nested-fields). A field name containing a literal dot is left out of the picker, since MongoDB reads a dot as a path separator; reach it with `$getField` inside `$expr`. -The Structure tab lists a collection's indexes; drop one from a query tab with `db.users.dropIndex("email_1")`. **New Database** asks for a database name and a first collection, both required. **New View** opens a query tab holding a `db.createView("view_name", "source_collection", [pipeline])` template, and editing a view pre-fills `db.runCommand({"collMod": …})`. +The Structure tab lists a collection's indexes, and [renames or removes a field](#renaming-and-removing-fields) across the whole collection. Drop an index from a query tab with `db.users.dropIndex("email_1")`. **New Database** asks for a database name and a first collection, both required. **New View** opens a query tab holding a `db.createView("view_name", "source_collection", [pipeline])` template, and editing a view pre-fills `db.runCommand({"collMod": …})`. ### Binary UUIDs @@ -118,6 +118,45 @@ A field exists only in the documents that hold it, so a collection with no docum The text is Extended JSON: quote every field name, and write an ObjectId as `{"$oid": "…"}`, a date as `{"$date": "2024-05-01T10:00:00Z"}` and a decimal as `{"$numberDecimal": "1.10"}`. A whole number is stored as a 32-bit integer, or as a 64-bit one when it does not fit; `{"$numberLong": "5"}` stores a small 64-bit integer. A number with a decimal point is a double. Fields are stored in the order written. Leave out `_id` and the server generates one. +## Renaming and removing fields + +In the Structure tab, edit a field's **Name** to rename it, or remove its row to remove the field, then choose **Save Changes**. Each change is one `updateMany` over the collection, and **Preview SQL** shows it: + +```javascript +db.people.updateMany({"status": {"$exists": true}, "state": {"$exists": false}}, {"$rename": {"status": "state"}}) +db.people.updateMany({"tmp": {"$exists": true}}, {"$unset": {"tmp": ""}}) +``` + +A document that already holds the new name keeps its value, and the rename skips it. When the collection's `$jsonSchema` validator declares the field in `properties`, `required` or `dependencies`, the save starts with a `db.runCommand({"collMod": …})` that renames or removes the field there as well, so a collection made with **New Table…** keeps validating. A field only the validator declares, with no document holding it, is removed from the validator. + +With **Write Concern** set on the connection, or `w`, `journal` and `wtimeoutMS` in its connection string, every statement carries it as `{"writeConcern": {"w": "majority"}}`, and **Preview SQL** shows it. `w=0` without `journal=true` goes out as `w: 1`. + +The save is refused before anything is written when: + +| Refused when | What to do | +|---|---| +| An index uses the old or the new name | Drop the index from a query tab. The message gives the `dropIndex` command | +| An Atlas Search or Vector Search index mentions the old or the new name anywhere in its definition, or the server runs `$search` but is too old to list search indexes | Change or drop the search index in Atlas first, then rename the field from a query tab | +| A view's pipeline holds the old name anywhere, even as a plain value such as `{$match: {status: "active"}}` when renaming `active` | Change the view's pipeline first | +| A view or the validator hands the whole document, `$$ROOT` or `$$CURRENT`, to anything but `$replaceWith`, `$replaceRoot` or a `$getField` of one named field, as `{$objectToArray: "$$ROOT"}` or `{$push: "$$ROOT"}` do | Change the view's pipeline, or the validator with `collMod`, first | +| The validator uses the field outside `properties`, `required` and `dependencies`, as in `$expr`, or lists whole documents naming it in `enum` | Change the validator with `collMod` first | +| A `patternProperties` pattern matches the old or the new name, or `additionalProperties` applies to one because `properties` does not declare it | Change the validator with `collMod` first. A field `properties` declares renames as usual | +| A document holds both the old and the new name | Remove one of the two from those documents | +| The validator would reject a document once it changes, or once the save moves a rule onto a name the document already holds | Fix that document or the validator. The message gives its `_id` | +| One save changes the same name twice, as a swap does | Save one change at a time | +| The field is `_id` | Leave it: MongoDB keys every document by `_id` | +| The name starts with `$`, contains a dot, or is `__proto__` | Rewrite those documents from a query tab | +| The collection is a view, a time series collection or a `system.` collection | Change the fields in the collection underneath, if there is one | +| The collection is capped and the new name takes more bytes than the old one | Choose a name no longer than the old one. A longer one makes MongoDB delete the oldest documents to stay within the cap | +| A longer name would take a document past MongoDB's 16 MB limit | Choose a shorter name, or shrink that document. The message gives its `_id` | +| The collection's validator or options changed after the save was prepared, or while its documents were checked | Check the new `collMod` in **Preview SQL**, then choose **Save Changes** again | + +The checks for a document holding both names and for a document the validator would reject run on **Save Changes** only, never in **Preview SQL**. Under a `moderate` validator, a document the validator already rejected before the save is left alone. Each reads every document that holds a changed field and stops at the query timeout in **Settings > General**, or after 10 minutes when the timeout is **No limit**. A check that times out changes nothing. Every check reads the primary, whatever the connection's **Read Preference**. Once they pass, the save reads the collection's options, indexes and views again, and stops if the options changed or an index, search index or view now uses the field. + +An `updateMany` that stops partway, at the query timeout or on a duplicate key, keeps the documents it already changed. Another client can also write the old name while the save runs, so after the last statement the save counts the documents that still hold it, and reports how many when any do. It also reads the indexes, search indexes and views again, and reports one another client made on either name while the save ran. A `collMod` whose write concern was not met stops the save with the server's reason, after the validator was already changed. Either way the change stays queued in the Structure tab: choose **Save Changes** again and it finishes the rest, because each statement skips the documents already done. A document left holding both names stops that second save instead, with the message for that case. Removing a field asks for confirmation, as a dropped column does. + +When the save rewrote the validator, it then looks for a document holding a changed name that the new validator rejects. One another client wrote with only the new name, after the checks and before the `collMod`, passed the old validator and fails the new one, and its next update fails with `Document failed validation`. The save reports its `_id`: fix that document, then choose **Save Changes** again. + ## Writing queries Queries run through JavaScriptCore, so a statement is JavaScript and the whole language is @@ -219,6 +258,10 @@ New connections default to **Disabled**, and the driver has no TLS fallback: **P - GridFS buckets are not browsable, and change streams are unsupported. - A script that loops without touching the database cannot be stopped: JavaScriptCore has no public way to interrupt one. `Cmd+.` stops anything that reads, writes or prints, which covers every query. A script silent for 120 seconds is abandoned and the shell restarts. - Field names that look like integers (`"0"`, `"12"`) sort ahead of the rest in a document literal, which is what JavaScript does with them. +- A field rename checks indexes, search indexes, validators and views in the same database only. Other databases and application code keep the old name: update them after the rename. +- A field rename does not lock the collection. A validator another client sets in the milliseconds between the last check and the save's `collMod` is replaced by it. +- MongoDB before 4.4 cannot measure a document, so a rename to a longer name is not checked against the 16 MB limit there. Rename to a name no longer than the old one on those servers when documents are close to the limit. +- **Compare & Sync** compares MongoDB structures but writes no script for them, because a collection's fields come from a sample of its documents. ## Troubleshooting diff --git a/docs/features/safe-mode.mdx b/docs/features/safe-mode.mdx index 91efdc92d6..2db1d49d53 100644 --- a/docs/features/safe-mode.mdx +++ b/docs/features/safe-mode.mdx @@ -24,7 +24,7 @@ Four things the table cannot carry. The confirmation dialog shows the whole stat Confirmation dialog over a query tab, showing a fourteen line UPDATE with Cancel and Execute Confirmation dialog over a query tab, showing a fourteen line UPDATE with Cancel and Execute - **Silent** is not a free pass: `DROP`, `TRUNCATE`, and a `DELETE` with no `WHERE` still raise the built-in dangerous query warning even there. And **Read-Only** goes past queries to the interface itself, disabling inline cell editing, adding, deleting and duplicating rows, table truncate and drop, and import. + **Silent** is not a free pass: `DROP`, `TRUNCATE`, and a `DELETE` with no `WHERE` still raise the built-in dangerous query warning even there, and so does a Structure tab save that drops a column, changes a column's type or removes a MongoDB field. And **Read-Only** goes past queries to the interface itself, disabling inline cell editing, adding, deleting and duplicating rows, table truncate and drop, and import. ## Connections that are always read-only diff --git a/docs/features/table-structure.mdx b/docs/features/table-structure.mdx index d89ae349ca..4f37e03d70 100644 --- a/docs/features/table-structure.mdx +++ b/docs/features/table-structure.mdx @@ -32,7 +32,7 @@ PostgreSQL and PGlite answer this per operation, so the tab does too: A view and a materialized view differ in both directions: `ALTER TABLE … SET DEFAULT` runs on a view and is refused on a materialized view, and `CREATE INDEX` is the other way round. -Every other engine edits a table and a partitioned table only. A view, a materialized view, a foreign table and a system table are read-only there. +Every other engine edits a table and a partitioned table only. A view, a materialized view, a foreign table and a system table are read-only there. A MongoDB collection takes two edits, a field rename and a field removal; see [MongoDB collections](#mongodb-collections). ## Columns tab @@ -205,6 +205,8 @@ SQLite is the one engine where listing and editing part company. The tab appears - **Save Changes** (`Cmd+S` or the toolbar checkmark) applies the queue. Changes that can lose data, dropping a column, changing a type, adding NOT NULL, changing the primary key, first show a confirmation listing each one. - **Preview SQL** (`Cmd+Shift+P`) shows the statements **Save Changes** would run, on the tab's own schema, and runs none of them. A save that recreates the table previews as its rebuild script, with what the rebuild cannot carry over. +From **Save Changes** until the save ends, the tab shows **Saving Changes…**, takes no edits, and ignores a second **Save Changes**. A rebuild script runs only while the queue is still the one it was built from; edit the queue with the script on screen and the rebuild stops, so save again to review the new one. + The queue outlives everything short of an explicit discard: closing the tab, closing the window, quitting, and **Refresh** all ask first. A save that never reaches the server leaves the tab open with its queue intact. @@ -214,6 +216,8 @@ The queue outlives everything short of an explicit discard: closing the tab, clo A save runs on a connection of its own, so it never joins a transaction you have left open in a query tab. +Once a save has run, even partway, every tab on the table loads its rows again, in any window: the tab on screen at once, its Data view included while Structure is in front, and the rest when you next switch to them. A tab holding unsaved row edits keeps its rows, and asks first if it is the one on screen in Data. Tabs on other tables are left alone. + ### When one statement fails A save is often several statements, run in order. Engines with transactional DDL roll the whole set back; MySQL, MariaDB, and Oracle commit each one as it runs, so everything before the failure has landed while the queue still holds all of it. An **Error Applying Changes** sheet reports what the server said: refresh before saving again, or the second save replays work the server already did. @@ -254,7 +258,9 @@ Visual table creation is supported for MySQL, MariaDB, PostgreSQL, PGlite, SQLit ## MongoDB collections -MongoDB structure is read-only, and inferred from the collection's first 200 documents: top-level field names are unioned across the sample, and each field takes its most common BSON type. Fields the collection's `$jsonSchema` validator declares follow, including ones no sampled document holds, and a field in the validator's `required` list is not nullable. `_id` comes first, marked as the primary key. The DDL tab shows indexes as `createIndex()` commands for `mongosh`, plus the validator and capped-collection options where present. +A collection's fields are inferred from its first 200 documents: top-level field names are unioned across the sample, and each field takes its most common BSON type. Fields the collection's `$jsonSchema` validator declares follow, including ones no sampled document holds, and a field in the validator's `required` list is not nullable. `_id` comes first, marked as the primary key. The DDL tab shows indexes as `createIndex()` commands for `mongosh`, plus the validator and capped-collection options where present. + +Edit a field's **Name** to rename it in every document, or remove its row to remove it from every document. **Type**, **Nullable**, adding a field and the **Indexes** tab stay read-only. [Renaming and removing fields](/databases/mongodb#renaming-and-removing-fields) shows the statements a save runs and what refuses one. ## Limitations diff --git a/project.yml b/project.yml index ecd82d75d7..0d6602bb9c 100644 --- a/project.yml +++ b/project.yml @@ -521,7 +521,15 @@ targets: - Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift - Plugins/MongoDBDriverPlugin/MongoDocumentText.swift - Plugins/MongoDBDriverPlugin/MongoDocumentWritePlan.swift + - Plugins/MongoDBDriverPlugin/MongoDBStructureEditing.swift - Plugins/MongoDBDriverPlugin/MongoDBTimeoutPolicy.swift + - Plugins/MongoDBDriverPlugin/MongoCollectionCatalog.swift + - Plugins/MongoDBDriverPlugin/MongoFieldChange.swift + - Plugins/MongoDBDriverPlugin/MongoFieldChangeAssessment.swift + - Plugins/MongoDBDriverPlugin/MongoFieldDataProbe.swift + - Plugins/MongoDBDriverPlugin/MongoFieldDependent.swift + - Plugins/MongoDBDriverPlugin/MongoFieldReferences.swift + - Plugins/MongoDBDriverPlugin/MongoWriteConcern.swift - Plugins/MongoDBDriverPlugin/MongoScriptCommandBuilder.swift - Plugins/MongoDBDriverPlugin/MongoShellCommandLine.swift - Plugins/MongoDBDriverPlugin/MongoScriptContext.swift