From c0e17f7c2d051d4d26af6b04c3049c889a3fd32a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 21 Sep 2026 15:46:02 +0200 Subject: [PATCH 1/7] acceptance: add field_removal invariant test Removing a non-required field must be applied as an update that converges, not dropped from the request (#6343). Tag a removable field `# CAN_REMOVE`; the test strips those lines, asserts the plan is an update, and checks there is no drift after deploying it. Configs without the marker only run the baseline no-drift check. `# CANNOT_REMOVE` documents fields whose removal still misbehaves (perpetual drift, or a 400 when the field is dropped from a masked update), with the reason inline. Co-authored-by: Isaac --- acceptance/bin/verify_update.py | 49 ++++++++++++++ acceptance/bundle/invariant/README.md | 8 +++ .../bundle/invariant/configs/app.yml.tmpl | 1 + .../bundle/invariant/configs/catalog.yml.tmpl | 2 +- .../configs/catalog_optional_fields.yml.tmpl | 2 + .../invariant/configs/cluster_policy.yml.tmpl | 1 + .../configs/external_location.yml.tmpl | 2 +- .../invariant/configs/genie_space.yml.tmpl | 3 + .../invariant/configs/instance_pool.yml.tmpl | 1 + .../bundle/invariant/configs/job.yml.tmpl | 1 + .../bundle/invariant/configs/model.yml.tmpl | 1 + .../invariant/configs/pipeline.yml.tmpl | 1 + .../configs/postgres_project.yml.tmpl | 3 + .../configs/registered_model.yml.tmpl | 1 + .../bundle/invariant/configs/schema.yml.tmpl | 1 + .../bundle/invariant/configs/volume.yml.tmpl | 1 + .../invariant/field_removal/out.test.toml | 66 +++++++++++++++++++ .../bundle/invariant/field_removal/output.txt | 1 + .../bundle/invariant/field_removal/script | 32 +++++++++ .../bundle/invariant/field_removal/test.toml | 7 ++ 20 files changed, 182 insertions(+), 2 deletions(-) create mode 100644 acceptance/bin/verify_update.py create mode 100644 acceptance/bundle/invariant/field_removal/out.test.toml create mode 100644 acceptance/bundle/invariant/field_removal/output.txt create mode 100644 acceptance/bundle/invariant/field_removal/script create mode 100644 acceptance/bundle/invariant/field_removal/test.toml diff --git a/acceptance/bin/verify_update.py b/acceptance/bin/verify_update.py new file mode 100644 index 00000000000..c1363a151fc --- /dev/null +++ b/acceptance/bin/verify_update.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +""" +Check that a plan applies at least one "update" and nothing more drastic. + +Used by the field_removal invariant test after dropping an optional field: the +removal must be observable as an in-place update, never ignored (all "skip") and +never a create/delete/recreate. +""" + +import json +import sys + + +def check_plan(path): + with open(path) as fobj: + raw = fobj.read() + + updates = 0 + unexpected = 0 + + try: + data = json.loads(raw) + for key, value in data["plan"].items(): + action = value.get("action") + if action == "update": + updates += 1 + elif action != "skip": + print(f"Unexpected {action=} for {key}") + unexpected += 1 + except Exception: + print(raw, flush=True) + raise + + if not updates: + print("Expected at least one update action, found none") + unexpected += 1 + + if unexpected: + print(raw, flush=True) + sys.exit(10) + + +def main(): + for path in sys.argv[1:]: + check_plan(path) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bundle/invariant/README.md b/acceptance/bundle/invariant/README.md index 184d3f541c4..60a446622f0 100644 --- a/acceptance/bundle/invariant/README.md +++ b/acceptance/bundle/invariant/README.md @@ -4,3 +4,11 @@ no_drift test checks that there are no actions planned after successful deploy. test will dump full JSON plan to the output. In order to add a new test, add a config to configs/ and include it in test.toml. + +The field_removal test additionally exercises removing non-required fields. Tag a field with a +`# CAN_REMOVE` comment on each of its lines; after the baseline no-drift check the test strips those +lines (`grep -v CAN_REMOVE`), then asserts the resulting plan is an update and that deploying it leaves +no drift. This guards the class of bug where a field the config stops declaring is dropped from the +update request instead of being cleared (see #6343). Configs without `CAN_REMOVE` only run the baseline. +If a field's removal does not converge, tag it `# CANNOT_REMOVE` (which is not stripped) with a comment +explaining the error or drift, rather than leaving a failing `CAN_REMOVE`. diff --git a/acceptance/bundle/invariant/configs/app.yml.tmpl b/acceptance/bundle/invariant/configs/app.yml.tmpl index 153b04e9b56..bfd0b9bec6a 100644 --- a/acceptance/bundle/invariant/configs/app.yml.tmpl +++ b/acceptance/bundle/invariant/configs/app.yml.tmpl @@ -6,6 +6,7 @@ resources: foo: name: app-$UNIQUE_NAME source_code_path: ./app + description: This is a test app # CAN_REMOVE permissions: - level: CAN_USE group_name: users diff --git a/acceptance/bundle/invariant/configs/catalog.yml.tmpl b/acceptance/bundle/invariant/configs/catalog.yml.tmpl index c1bdbd3f9ec..75901f6061a 100644 --- a/acceptance/bundle/invariant/configs/catalog.yml.tmpl +++ b/acceptance/bundle/invariant/configs/catalog.yml.tmpl @@ -5,7 +5,7 @@ resources: catalogs: foo: name: test-catalog-$UNIQUE_NAME - comment: This is a test catalog + comment: This is a test catalog # CAN_REMOVE grants: - principal: account users privileges: diff --git a/acceptance/bundle/invariant/configs/catalog_optional_fields.yml.tmpl b/acceptance/bundle/invariant/configs/catalog_optional_fields.yml.tmpl index 9950f35211c..b31d0d22e1b 100644 --- a/acceptance/bundle/invariant/configs/catalog_optional_fields.yml.tmpl +++ b/acceptance/bundle/invariant/configs/catalog_optional_fields.yml.tmpl @@ -6,6 +6,8 @@ resources: foo: name: test-catalog-$UNIQUE_NAME comment: This is a test catalog + # CANNOT_REMOVE: not force-sent on update (terraform omits it too, see #6343), so + # clearing it drops the field from the PATCH and the old value drifts back every plan. custom_max_retention_hours: 48 managed_encryption_settings: customer_managed_key_id: 00000000-0000-0000-0000-000000000000 diff --git a/acceptance/bundle/invariant/configs/cluster_policy.yml.tmpl b/acceptance/bundle/invariant/configs/cluster_policy.yml.tmpl index 35c1dd4a02f..2bd9d90e538 100644 --- a/acceptance/bundle/invariant/configs/cluster_policy.yml.tmpl +++ b/acceptance/bundle/invariant/configs/cluster_policy.yml.tmpl @@ -6,6 +6,7 @@ resources: foo: name: test-cluster-policy-$UNIQUE_NAME definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' + description: This is a test cluster policy # CAN_REMOVE permissions: - level: CAN_USE group_name: users diff --git a/acceptance/bundle/invariant/configs/external_location.yml.tmpl b/acceptance/bundle/invariant/configs/external_location.yml.tmpl index 28d36924a83..3fe403dbf71 100644 --- a/acceptance/bundle/invariant/configs/external_location.yml.tmpl +++ b/acceptance/bundle/invariant/configs/external_location.yml.tmpl @@ -7,7 +7,7 @@ resources: name: test_location_$UNIQUE_NAME url: s3://test-bucket/path credential_name: test_storage_credential - comment: "Test external location from DABs" + comment: "Test external location from DABs" # CAN_REMOVE grants: - principal: account users privileges: diff --git a/acceptance/bundle/invariant/configs/genie_space.yml.tmpl b/acceptance/bundle/invariant/configs/genie_space.yml.tmpl index 2e8c534df39..0b604cb8a9a 100644 --- a/acceptance/bundle/invariant/configs/genie_space.yml.tmpl +++ b/acceptance/bundle/invariant/configs/genie_space.yml.tmpl @@ -6,6 +6,9 @@ resources: foo: warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID title: test-genie-space-$UNIQUE_NAME + # CANNOT_REMOVE: description is not force-sent on update, so clearing it drops the + # field from the request and the old value drifts back on every subsequent plan (#6343 class). + description: This is a test genie space # Structured (inline) serialized_space is marshalled to a JSON string by # ConfigureGenieSpaceSerializedSpace; this config doubles as a regression # guard that the normalization produces a drift-free deploy. Kept minimal diff --git a/acceptance/bundle/invariant/configs/instance_pool.yml.tmpl b/acceptance/bundle/invariant/configs/instance_pool.yml.tmpl index 4a8d6bc5afd..52d8492958d 100644 --- a/acceptance/bundle/invariant/configs/instance_pool.yml.tmpl +++ b/acceptance/bundle/invariant/configs/instance_pool.yml.tmpl @@ -6,6 +6,7 @@ resources: foo: instance_pool_name: test-instance-pool-$UNIQUE_NAME node_type_id: $NODE_TYPE_ID + idle_instance_autotermination_minutes: 60 # CAN_REMOVE permissions: - level: CAN_ATTACH_TO group_name: users diff --git a/acceptance/bundle/invariant/configs/job.yml.tmpl b/acceptance/bundle/invariant/configs/job.yml.tmpl index 696483648b5..7d54aec4f83 100644 --- a/acceptance/bundle/invariant/configs/job.yml.tmpl +++ b/acceptance/bundle/invariant/configs/job.yml.tmpl @@ -5,6 +5,7 @@ resources: jobs: foo: name: test-job-$UNIQUE_NAME + description: This is a test job # CAN_REMOVE permissions: - level: CAN_VIEW group_name: users diff --git a/acceptance/bundle/invariant/configs/model.yml.tmpl b/acceptance/bundle/invariant/configs/model.yml.tmpl index e105a731a33..e153b4c4081 100644 --- a/acceptance/bundle/invariant/configs/model.yml.tmpl +++ b/acceptance/bundle/invariant/configs/model.yml.tmpl @@ -5,3 +5,4 @@ resources: models: foo: name: test-model-$UNIQUE_NAME + description: This is a test model # CAN_REMOVE diff --git a/acceptance/bundle/invariant/configs/pipeline.yml.tmpl b/acceptance/bundle/invariant/configs/pipeline.yml.tmpl index 9cb1b4c7c21..331fa7124d2 100644 --- a/acceptance/bundle/invariant/configs/pipeline.yml.tmpl +++ b/acceptance/bundle/invariant/configs/pipeline.yml.tmpl @@ -5,6 +5,7 @@ resources: pipelines: foo: name: test-pipeline-$UNIQUE_NAME + development: true # CAN_REMOVE libraries: - file: path: pipeline.py diff --git a/acceptance/bundle/invariant/configs/postgres_project.yml.tmpl b/acceptance/bundle/invariant/configs/postgres_project.yml.tmpl index 7fb119b7060..315ddecd32a 100644 --- a/acceptance/bundle/invariant/configs/postgres_project.yml.tmpl +++ b/acceptance/bundle/invariant/configs/postgres_project.yml.tmpl @@ -5,6 +5,9 @@ resources: postgres_projects: foo: project_id: test-pg-project-$UNIQUE_NAME + # CANNOT_REMOVE: clearing display_name adds spec.display_name to the update_mask but drops + # the field from the request body (omitempty), which the backend rejects with 400 + # "Field 'spec.display_name' is in update_mask but not provided in request" (#6343 class). display_name: Test Postgres Project enable_pg_native_login: false permissions: diff --git a/acceptance/bundle/invariant/configs/registered_model.yml.tmpl b/acceptance/bundle/invariant/configs/registered_model.yml.tmpl index 8a146c70dd9..8b1fa846e6a 100644 --- a/acceptance/bundle/invariant/configs/registered_model.yml.tmpl +++ b/acceptance/bundle/invariant/configs/registered_model.yml.tmpl @@ -7,6 +7,7 @@ resources: name: test-model-$UNIQUE_NAME catalog_name: main schema_name: default + comment: This is a test registered model # CAN_REMOVE grants: - principal: account users privileges: diff --git a/acceptance/bundle/invariant/configs/schema.yml.tmpl b/acceptance/bundle/invariant/configs/schema.yml.tmpl index d9aebda0cef..82b859ee592 100644 --- a/acceptance/bundle/invariant/configs/schema.yml.tmpl +++ b/acceptance/bundle/invariant/configs/schema.yml.tmpl @@ -6,6 +6,7 @@ resources: foo: catalog_name: main name: test-schema-$UNIQUE_NAME + comment: This is a test schema # CAN_REMOVE grants: - principal: account users privileges: diff --git a/acceptance/bundle/invariant/configs/volume.yml.tmpl b/acceptance/bundle/invariant/configs/volume.yml.tmpl index 7868893619b..fc61b3311e7 100644 --- a/acceptance/bundle/invariant/configs/volume.yml.tmpl +++ b/acceptance/bundle/invariant/configs/volume.yml.tmpl @@ -7,6 +7,7 @@ resources: name: test-volume-$UNIQUE_NAME catalog_name: main schema_name: default + comment: This is a test volume # CAN_REMOVE grants: - principal: account users privileges: diff --git a/acceptance/bundle/invariant/field_removal/out.test.toml b/acceptance/bundle/invariant/field_removal/out.test.toml new file mode 100644 index 00000000000..8585e6f2dd6 --- /dev/null +++ b/acceptance/bundle/invariant/field_removal/out.test.toml @@ -0,0 +1,66 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] +EnvMatrix.INPUT_CONFIG = [ + "alert.yml.tmpl", + "app.yml.tmpl", + "catalog.yml.tmpl", + "catalog_optional_fields.yml.tmpl", + "cluster.yml.tmpl", + "cluster_apply_policy_default_values.yml.tmpl", + "cluster_policy.yml.tmpl", + "dashboard.yml.tmpl", + "job_apply_policy_default_values_job_cluster.yml.tmpl", + "job_apply_policy_default_values_task_cluster.yml.tmpl", + "job_apply_policy_default_values_for_each_task.yml.tmpl", + "database_catalog.yml.tmpl", + "database_instance.yml.tmpl", + "experiment.yml.tmpl", + "external_location.yml.tmpl", + "genie_space.yml.tmpl", + "instance_pool.yml.tmpl", + "job.yml.tmpl", + "job_pydabs_10_tasks.yml.tmpl", + "job_pydabs_1000_tasks.yml.tmpl", + "job_cross_resource_ref.yml.tmpl", + "job_escaped_refs.yml.tmpl", + "job_permission_ref.yml.tmpl", + "job_run.yml.tmpl", + "job_run_job_ref.yml.tmpl", + "job_table_update_trigger.yml.tmpl", + "job_with_depends_on.yml.tmpl", + "job_with_task.yml.tmpl", + "model.yml.tmpl", + "model_with_permissions.yml.tmpl", + "model_serving_endpoint.yml.tmpl", + "model_serving_endpoint_telemetry.yml.tmpl", + "pipeline.yml.tmpl", + "pipeline_allow_duplicate_names.yml.tmpl", + "pipeline_apply_policy_default_values.yml.tmpl", + "pipeline_config_dots.yml.tmpl", + "postgres_branch.yml.tmpl", + "postgres_catalog.yml.tmpl", + "postgres_database.yml.tmpl", + "postgres_endpoint.yml.tmpl", + "postgres_project.yml.tmpl", + "postgres_role.yml.tmpl", + "postgres_synced_table.yml.tmpl", + "registered_model.yml.tmpl", + "schema.yml.tmpl", + "schema_empty_grants.yml.tmpl", + "schema_grant_ref.yml.tmpl", + "schema_uppercase_name.yml.tmpl", + "secret.yml.tmpl", + "secret_scope.yml.tmpl", + "secret_scope_default_backend_type.yml.tmpl", + "sql_warehouse.yml.tmpl", + "synced_database_table.yml.tmpl", + "uc_trailing_slash.yml.tmpl", + "vector_search_endpoint.yml.tmpl", + "vector_search_index.yml.tmpl", + "volume.yml.tmpl", + "volume_external.yml.tmpl", + "volume_path_job_ref.yml.tmpl", + "volume_uppercase_name.yml.tmpl", + "model_service.yml.tmpl" +] diff --git a/acceptance/bundle/invariant/field_removal/output.txt b/acceptance/bundle/invariant/field_removal/output.txt new file mode 100644 index 00000000000..7a28cb73a58 --- /dev/null +++ b/acceptance/bundle/invariant/field_removal/output.txt @@ -0,0 +1 @@ +INPUT_CONFIG_OK diff --git a/acceptance/bundle/invariant/field_removal/script b/acceptance/bundle/invariant/field_removal/script new file mode 100644 index 00000000000..2e6ec07bb93 --- /dev/null +++ b/acceptance/bundle/invariant/field_removal/script @@ -0,0 +1,32 @@ +# Invariant to test: removing a non-required field is applied as an update and converges. +# Fields tagged `# CAN_REMOVE` in the config are dropped, and the resulting plan must show an +# update; after deploying it a fresh plan must show no drift. This guards the omitempty-drop +# class of bug (#6343): a field the config stops declaring must be sent as its zero value, not +# silently dropped from the PATCH (which either fails with "Nothing to update" or never converges). +# Additional checks: no internal errors / panics in validate/plan/deploy. + +invariant_setup + +invariant_deploy LOG.deploy $CLI bundle deploy + +invariant_verify_no_drift + +# Only configs that tag fields with `# CAN_REMOVE` exercise the removal cycle; the rest just +# assert the baseline no-drift above. `CANNOT_REMOVE` documents a field whose removal is broken +# and is intentionally not stripped (it does not contain the substring `CAN_REMOVE`). +if grep -q CAN_REMOVE databricks.yml; then + grep -v CAN_REMOVE databricks.yml > databricks.yml.new + mv databricks.yml.new databricks.yml + cp databricks.yml LOG.config.removed + + $CLI bundle plan -o json > LOG.plan_removed.json 2>LOG.plan_removed.err + cat LOG.plan_removed.err | contains.py '!panic:' '!internal error' > /dev/null + verify_update.py LOG.plan_removed.json + + # Not invariant_deploy: a second INPUT_CONFIG_OK would make output.txt differ between the + # configs that have removable fields and those that don't. + trace $CLI bundle deploy &> LOG.deploy_removed + cat LOG.deploy_removed | contains.py '!panic:' '!internal error' > /dev/null + + invariant_verify_no_drift +fi diff --git a/acceptance/bundle/invariant/field_removal/test.toml b/acceptance/bundle/invariant/field_removal/test.toml new file mode 100644 index 00000000000..c657e14500c --- /dev/null +++ b/acceptance/bundle/invariant/field_removal/test.toml @@ -0,0 +1,7 @@ +# A 1000-task job serializes to ~110 KB, over the 64 KB per-operation state limit the +# deployment metadata service accepts, so recording it fails the deploy (same as no_drift). +EnvMatrixExclude.dms_state_too_large = ["DMS=true", "INPUT_CONFIG=job_pydabs_1000_tasks.yml.tmpl"] + +# Emptying a grants node records a delete with no state, which the deployment metadata service +# rejects on a succeeded UPDATE action (known bug bundle/dms/empty-grants); skip under recording. +EnvMatrixExclude.dms_empty_grants_bug = ["DMS=true", "INPUT_CONFIG=schema_empty_grants.yml.tmpl"] From fcc9a93536634bfe8872bcd4d9928dee682674e4 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 21 Sep 2026 15:55:42 +0200 Subject: [PATCH 2/7] acceptance: mark verify_update.py executable Co-authored-by: Isaac --- acceptance/bin/verify_update.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 acceptance/bin/verify_update.py diff --git a/acceptance/bin/verify_update.py b/acceptance/bin/verify_update.py old mode 100644 new mode 100755 From 56fb70d428190bcc7b534cd7ed5834c64e567716 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 21 Sep 2026 16:10:02 +0200 Subject: [PATCH 3/7] acceptance: extend field_removal coverage to more fields Adds removal coverage for properties (single-key), enable_elastic_disk, auto_stop_mins, max_num_clusters and target_qps (all converge, CAN_REMOVE), and documents more fields whose removal misbehaves as CANNOT_REMOVE: - schemas.properties (whole map): maps ignore ForceSendFields, so clearing drifts - secrets.comment: not force-sent, drifts - dashboards.display_name: required, removal fails validation - postgres_branches.no_expiry, postgres_endpoints.autoscaling/disabled, postgres_projects.default_endpoint_settings: added to update_mask but dropped from the body, backend 400s Co-authored-by: Isaac --- .../invariant/configs/catalog_optional_fields.yml.tmpl | 3 +++ acceptance/bundle/invariant/configs/dashboard.yml.tmpl | 1 + acceptance/bundle/invariant/configs/instance_pool.yml.tmpl | 1 + .../bundle/invariant/configs/postgres_branch.yml.tmpl | 4 ++++ .../bundle/invariant/configs/postgres_endpoint.yml.tmpl | 6 ++++++ .../bundle/invariant/configs/postgres_project.yml.tmpl | 4 ++++ acceptance/bundle/invariant/configs/schema.yml.tmpl | 4 ++++ acceptance/bundle/invariant/configs/secret.yml.tmpl | 3 +++ acceptance/bundle/invariant/configs/sql_warehouse.yml.tmpl | 4 ++-- .../invariant/configs/vector_search_endpoint.yml.tmpl | 1 + 10 files changed, 29 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/invariant/configs/catalog_optional_fields.yml.tmpl b/acceptance/bundle/invariant/configs/catalog_optional_fields.yml.tmpl index b31d0d22e1b..fecebe4050b 100644 --- a/acceptance/bundle/invariant/configs/catalog_optional_fields.yml.tmpl +++ b/acceptance/bundle/invariant/configs/catalog_optional_fields.yml.tmpl @@ -11,3 +11,6 @@ resources: custom_max_retention_hours: 48 managed_encryption_settings: customer_managed_key_id: 00000000-0000-0000-0000-000000000000 + properties: + tier: bronze + team: data # CAN_REMOVE diff --git a/acceptance/bundle/invariant/configs/dashboard.yml.tmpl b/acceptance/bundle/invariant/configs/dashboard.yml.tmpl index 857e0a2543c..c739ec72b52 100644 --- a/acceptance/bundle/invariant/configs/dashboard.yml.tmpl +++ b/acceptance/bundle/invariant/configs/dashboard.yml.tmpl @@ -5,6 +5,7 @@ resources: dashboards: foo: warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID + # CANNOT_REMOVE: display_name is required; removing it fails validation ("dashboard display_name is required"). display_name: test-dashboard-$UNIQUE_NAME file_path: ./dashboard.lvdash.json permissions: diff --git a/acceptance/bundle/invariant/configs/instance_pool.yml.tmpl b/acceptance/bundle/invariant/configs/instance_pool.yml.tmpl index 52d8492958d..7e6b231f487 100644 --- a/acceptance/bundle/invariant/configs/instance_pool.yml.tmpl +++ b/acceptance/bundle/invariant/configs/instance_pool.yml.tmpl @@ -7,6 +7,7 @@ resources: instance_pool_name: test-instance-pool-$UNIQUE_NAME node_type_id: $NODE_TYPE_ID idle_instance_autotermination_minutes: 60 # CAN_REMOVE + enable_elastic_disk: true # CAN_REMOVE permissions: - level: CAN_ATTACH_TO group_name: users diff --git a/acceptance/bundle/invariant/configs/postgres_branch.yml.tmpl b/acceptance/bundle/invariant/configs/postgres_branch.yml.tmpl index e17859b0b90..9c0ec661637 100644 --- a/acceptance/bundle/invariant/configs/postgres_branch.yml.tmpl +++ b/acceptance/bundle/invariant/configs/postgres_branch.yml.tmpl @@ -11,4 +11,8 @@ resources: foo: parent: ${resources.postgres_projects.project.name} branch_id: test-branch-$UNIQUE_NAME + # CANNOT_REMOVE: clearing no_expiry adds spec.expiration to the update_mask but drops the field + # from the request body (omitempty), which the backend rejects with 400 "in update_mask but not + # provided in request" (#6343 class). (is_protected behaves the same but is omitted here: a + # protected branch cannot be deleted, which wedges the test's destroy-based cleanup.) no_expiry: true diff --git a/acceptance/bundle/invariant/configs/postgres_endpoint.yml.tmpl b/acceptance/bundle/invariant/configs/postgres_endpoint.yml.tmpl index fbffd428ff6..d70f2141495 100644 --- a/acceptance/bundle/invariant/configs/postgres_endpoint.yml.tmpl +++ b/acceptance/bundle/invariant/configs/postgres_endpoint.yml.tmpl @@ -18,3 +18,9 @@ resources: parent: ${resources.postgres_branches.branch.name} endpoint_id: test-endpoint-$UNIQUE_NAME endpoint_type: ENDPOINT_TYPE_READ_WRITE + # CANNOT_REMOVE: clearing these adds them to the update_mask but drops the fields from the + # request body (omitempty), which the backend rejects with 400 "Field 'spec.autoscaling_limit_max_cu' + # is in update_mask but not provided in request" (#6343 class). + autoscaling_limit_min_cu: 1 + autoscaling_limit_max_cu: 4 + disabled: true diff --git a/acceptance/bundle/invariant/configs/postgres_project.yml.tmpl b/acceptance/bundle/invariant/configs/postgres_project.yml.tmpl index 315ddecd32a..a7855f3d864 100644 --- a/acceptance/bundle/invariant/configs/postgres_project.yml.tmpl +++ b/acceptance/bundle/invariant/configs/postgres_project.yml.tmpl @@ -10,6 +10,10 @@ resources: # "Field 'spec.display_name' is in update_mask but not provided in request" (#6343 class). display_name: Test Postgres Project enable_pg_native_login: false + # CANNOT_REMOVE: same update_mask 400 as display_name above — clearing it adds + # spec.default_endpoint_settings to the update_mask but drops it from the request body. + default_endpoint_settings: + autoscaling_limit_max_cu: 4 permissions: - level: CAN_USE group_name: users diff --git a/acceptance/bundle/invariant/configs/schema.yml.tmpl b/acceptance/bundle/invariant/configs/schema.yml.tmpl index 82b859ee592..01256022c3e 100644 --- a/acceptance/bundle/invariant/configs/schema.yml.tmpl +++ b/acceptance/bundle/invariant/configs/schema.yml.tmpl @@ -7,6 +7,10 @@ resources: catalog_name: main name: test-schema-$UNIQUE_NAME comment: This is a test schema # CAN_REMOVE + # CANNOT_REMOVE: properties is a map, and ForceSendFields is inert for maps (#6343), so + # clearing the whole map drops it from the request and the old value drifts back every plan. + properties: + tier: bronze grants: - principal: account users privileges: diff --git a/acceptance/bundle/invariant/configs/secret.yml.tmpl b/acceptance/bundle/invariant/configs/secret.yml.tmpl index 2218b827032..c9c32bf9ed7 100644 --- a/acceptance/bundle/invariant/configs/secret.yml.tmpl +++ b/acceptance/bundle/invariant/configs/secret.yml.tmpl @@ -12,6 +12,9 @@ resources: schema_name: default name: test-secret-$UNIQUE_NAME value: ${var.secret_value} + # CANNOT_REMOVE: comment is not force-sent on update, so clearing it drops the field + # from the request and the old value drifts back on every subsequent plan (#6343 class). + comment: This is a test secret grants: - principal: account users privileges: diff --git a/acceptance/bundle/invariant/configs/sql_warehouse.yml.tmpl b/acceptance/bundle/invariant/configs/sql_warehouse.yml.tmpl index 56901354c6e..0110ebdbb2d 100644 --- a/acceptance/bundle/invariant/configs/sql_warehouse.yml.tmpl +++ b/acceptance/bundle/invariant/configs/sql_warehouse.yml.tmpl @@ -6,8 +6,8 @@ resources: foo: name: test-warehouse-$UNIQUE_NAME cluster_size: 2X-Small - auto_stop_mins: 10 - max_num_clusters: 1 + auto_stop_mins: 10 # CAN_REMOVE + max_num_clusters: 1 # CAN_REMOVE min_num_clusters: 1 warehouse_type: CLASSIC permissions: diff --git a/acceptance/bundle/invariant/configs/vector_search_endpoint.yml.tmpl b/acceptance/bundle/invariant/configs/vector_search_endpoint.yml.tmpl index cea1a4d026c..5b3714d3701 100644 --- a/acceptance/bundle/invariant/configs/vector_search_endpoint.yml.tmpl +++ b/acceptance/bundle/invariant/configs/vector_search_endpoint.yml.tmpl @@ -6,6 +6,7 @@ resources: foo: name: test-endpoint-$UNIQUE_NAME endpoint_type: STANDARD + target_qps: 2 # CAN_REMOVE bar: # Endpoint names must be < 50 chars, so keep this prefix short. name: test-vse-perm-$UNIQUE_NAME From 56facb916d5a7667c72cce0fb86ce4b5b293bba7 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 21 Sep 2026 16:18:31 +0200 Subject: [PATCH 4/7] acceptance: update selection selftest golden for field_removal target Adding the field_removal invariant target means editing a config now also selects its field_removal variants; regenerate selftest/selection/output.txt. Co-authored-by: Isaac --- acceptance/selftest/selection/output.txt | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/acceptance/selftest/selection/output.txt b/acceptance/selftest/selection/output.txt index 505dd11316c..f8916682d51 100644 --- a/acceptance/selftest/selection/output.txt +++ b/acceptance/selftest/selection/output.txt @@ -28,10 +28,12 @@ Selected 2 changed tests (limit=50, 0 not selected) === Touching an invariant config runs every invariant test, for that config only >>> selection M:acceptance/bundle/invariant/configs/job.yml.tmpl -Selected 5 changed tests (limit=50, 0 not selected) +Selected 6 changed tests (limit=50, 0 not selected) 5 ^bundle$/^invariant$/^continue_293$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^delete_idempotent$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^destroy_idempotent$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ + 5 ^bundle$/^invariant$/^field_removal$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ + 5 ^bundle$/^invariant$/^field_removal$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=true$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^migrate$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^no_drift$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^no_drift$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=true$/^INPUT_CONFIG=job\.yml\.tmpl$ @@ -39,16 +41,20 @@ Selected 5 changed tests (limit=50, 0 not selected) === Adding one invariant config and touching another: only the new variant is new >>> selection A:acceptance/bundle/invariant/configs/pipeline.yml.tmpl M:acceptance/bundle/invariant/configs/job.yml.tmpl -Selected 10 changed tests (limit=50, 0 not selected) +Selected 12 changed tests (limit=50, 0 not selected) 10 ^bundle$/^invariant$/^continue_293$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=pipeline\.yml\.tmpl$ 10 ^bundle$/^invariant$/^delete_idempotent$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=pipeline\.yml\.tmpl$ 10 ^bundle$/^invariant$/^destroy_idempotent$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=pipeline\.yml\.tmpl$ + 10 ^bundle$/^invariant$/^field_removal$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=pipeline\.yml\.tmpl$ + 10 ^bundle$/^invariant$/^field_removal$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=true$/^INPUT_CONFIG=pipeline\.yml\.tmpl$ 10 ^bundle$/^invariant$/^migrate$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=pipeline\.yml\.tmpl$ 10 ^bundle$/^invariant$/^no_drift$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=pipeline\.yml\.tmpl$ 10 ^bundle$/^invariant$/^no_drift$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=true$/^INPUT_CONFIG=pipeline\.yml\.tmpl$ 5 ^bundle$/^invariant$/^continue_293$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^delete_idempotent$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^destroy_idempotent$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ + 5 ^bundle$/^invariant$/^field_removal$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ + 5 ^bundle$/^invariant$/^field_removal$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=true$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^migrate$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^no_drift$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^no_drift$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=true$/^INPUT_CONFIG=job\.yml\.tmpl$ @@ -56,9 +62,11 @@ Selected 10 changed tests (limit=50, 0 not selected) === A config an invariant test excludes does not select that test >>> selection M:acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl -Selected 3 changed tests (limit=50, 0 not selected) +Selected 4 changed tests (limit=50, 0 not selected) 5 ^bundle$/^invariant$/^delete_idempotent$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=vector_search_index\.yml\.tmpl$ 5 ^bundle$/^invariant$/^destroy_idempotent$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=vector_search_index\.yml\.tmpl$ + 5 ^bundle$/^invariant$/^field_removal$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=vector_search_index\.yml\.tmpl$ + 5 ^bundle$/^invariant$/^field_removal$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=true$/^INPUT_CONFIG=vector_search_index\.yml\.tmpl$ 5 ^bundle$/^invariant$/^no_drift$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=vector_search_index\.yml\.tmpl$ 5 ^bundle$/^invariant$/^no_drift$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=true$/^INPUT_CONFIG=vector_search_index\.yml\.tmpl$ @@ -71,7 +79,7 @@ Selected 1 changed tests (limit=50, 0 not selected) === The limit keeps the highest scoring tests >>> selection -limit 3 A:acceptance/selftest/basic/script M:acceptance/selftest/diff/script M:acceptance/bundle/invariant/configs/job.yml.tmpl -Selected 3 changed tests (limit=3, 4 not selected) +Selected 3 changed tests (limit=3, 5 not selected) 10 ^selftest$/^basic$ 5 ^bundle$/^invariant$/^continue_293$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^delete_idempotent$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ From 8ebf80a4a95be217a7a6c2a5c90ddeb6d2465760 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 21 Sep 2026 16:30:43 +0200 Subject: [PATCH 5/7] acceptance: match `# CAN_REMOVE` with its prefix; inline kept maps Match the marker as `# CAN_REMOVE` (with the `# ` prefix) so that neither `# CANNOT_REMOVE` nor a comment that merely mentions the marker in prose can trigger the removal cycle or be stripped. Keep the map-valued CANNOT_REMOVE fields (schemas.properties, postgres_projects.default_endpoint_settings) on a single line so a future fix is a one-line flip to `# CAN_REMOVE`. Co-authored-by: Isaac --- acceptance/bundle/invariant/README.md | 11 ++++++----- .../invariant/configs/postgres_project.yml.tmpl | 4 ++-- acceptance/bundle/invariant/configs/schema.yml.tmpl | 4 ++-- acceptance/bundle/invariant/field_removal/script | 9 +++++---- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/acceptance/bundle/invariant/README.md b/acceptance/bundle/invariant/README.md index 60a446622f0..eecdbfc6c3f 100644 --- a/acceptance/bundle/invariant/README.md +++ b/acceptance/bundle/invariant/README.md @@ -7,8 +7,9 @@ In order to add a new test, add a config to configs/ and include it in test.toml The field_removal test additionally exercises removing non-required fields. Tag a field with a `# CAN_REMOVE` comment on each of its lines; after the baseline no-drift check the test strips those -lines (`grep -v CAN_REMOVE`), then asserts the resulting plan is an update and that deploying it leaves -no drift. This guards the class of bug where a field the config stops declaring is dropped from the -update request instead of being cleared (see #6343). Configs without `CAN_REMOVE` only run the baseline. -If a field's removal does not converge, tag it `# CANNOT_REMOVE` (which is not stripped) with a comment -explaining the error or drift, rather than leaving a failing `CAN_REMOVE`. +lines (`grep -v '# CAN_REMOVE'`), then asserts the resulting plan is an update and that deploying it +leaves no drift. This guards the class of bug where a field the config stops declaring is dropped from +the update request instead of being cleared (see #6343). Configs without `# CAN_REMOVE` only run the +baseline. If a field's removal does not converge, tag it `# CANNOT_REMOVE` with a comment explaining the +error or drift, rather than leaving a failing `# CAN_REMOVE`. The marker is matched with its `# ` prefix, +so `# CANNOT_REMOVE` and prose that mentions the marker are neither stripped nor treated as a tag. diff --git a/acceptance/bundle/invariant/configs/postgres_project.yml.tmpl b/acceptance/bundle/invariant/configs/postgres_project.yml.tmpl index a7855f3d864..63652868c57 100644 --- a/acceptance/bundle/invariant/configs/postgres_project.yml.tmpl +++ b/acceptance/bundle/invariant/configs/postgres_project.yml.tmpl @@ -12,8 +12,8 @@ resources: enable_pg_native_login: false # CANNOT_REMOVE: same update_mask 400 as display_name above — clearing it adds # spec.default_endpoint_settings to the update_mask but drops it from the request body. - default_endpoint_settings: - autoscaling_limit_max_cu: 4 + # Kept on one line so a future fix can flip it to CAN_REMOVE and have grep -v strip it cleanly. + default_endpoint_settings: {"autoscaling_limit_max_cu": 4} permissions: - level: CAN_USE group_name: users diff --git a/acceptance/bundle/invariant/configs/schema.yml.tmpl b/acceptance/bundle/invariant/configs/schema.yml.tmpl index 01256022c3e..bd2af70a77b 100644 --- a/acceptance/bundle/invariant/configs/schema.yml.tmpl +++ b/acceptance/bundle/invariant/configs/schema.yml.tmpl @@ -9,8 +9,8 @@ resources: comment: This is a test schema # CAN_REMOVE # CANNOT_REMOVE: properties is a map, and ForceSendFields is inert for maps (#6343), so # clearing the whole map drops it from the request and the old value drifts back every plan. - properties: - tier: bronze + # Kept on one line so a future fix can flip it to CAN_REMOVE and have grep -v strip it cleanly. + properties: {"tier": "bronze"} grants: - principal: account users privileges: diff --git a/acceptance/bundle/invariant/field_removal/script b/acceptance/bundle/invariant/field_removal/script index 2e6ec07bb93..38b4d119c75 100644 --- a/acceptance/bundle/invariant/field_removal/script +++ b/acceptance/bundle/invariant/field_removal/script @@ -12,10 +12,11 @@ invariant_deploy LOG.deploy $CLI bundle deploy invariant_verify_no_drift # Only configs that tag fields with `# CAN_REMOVE` exercise the removal cycle; the rest just -# assert the baseline no-drift above. `CANNOT_REMOVE` documents a field whose removal is broken -# and is intentionally not stripped (it does not contain the substring `CAN_REMOVE`). -if grep -q CAN_REMOVE databricks.yml; then - grep -v CAN_REMOVE databricks.yml > databricks.yml.new +# assert the baseline no-drift above. The marker is matched with its `# ` prefix so that neither +# `# CANNOT_REMOVE` (a documented, deliberately-kept field) nor prose mentioning the marker in a +# comment triggers the cycle or gets stripped. +if grep -q '# CAN_REMOVE' databricks.yml; then + grep -v '# CAN_REMOVE' databricks.yml > databricks.yml.new mv databricks.yml.new databricks.yml cp databricks.yml LOG.config.removed From 2e6283413c623be1ff894943fc7a0b40271a1c03 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 21 Sep 2026 16:50:07 +0200 Subject: [PATCH 6/7] acceptance: mark model.description and target_qps CANNOT_REMOVE (cloud-only) A cloud run of field_removal surfaced two removals that converge against the local testserver but fail on a real workspace: - models.description: the MLflow model registry rejects an empty description on update with 400 "Description cannot be empty" - vector_search_endpoints.target_qps: not re-sent on update, so it drifts back Both are kept as CANNOT_REMOVE with the cloud-only caveat noted inline. Rest of the cloud-eligible field_removal matrix passes (43/43). Co-authored-by: Isaac --- acceptance/bundle/invariant/configs/model.yml.tmpl | 5 ++++- .../bundle/invariant/configs/vector_search_endpoint.yml.tmpl | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/invariant/configs/model.yml.tmpl b/acceptance/bundle/invariant/configs/model.yml.tmpl index e153b4c4081..8c8deed8aa2 100644 --- a/acceptance/bundle/invariant/configs/model.yml.tmpl +++ b/acceptance/bundle/invariant/configs/model.yml.tmpl @@ -5,4 +5,7 @@ resources: models: foo: name: test-model-$UNIQUE_NAME - description: This is a test model # CAN_REMOVE + # CANNOT_REMOVE: the MLflow model registry rejects an empty description on update with + # 400 "Description cannot be empty" (only reproducible on cloud; the local testserver + # accepts the cleared value). + description: This is a test model diff --git a/acceptance/bundle/invariant/configs/vector_search_endpoint.yml.tmpl b/acceptance/bundle/invariant/configs/vector_search_endpoint.yml.tmpl index 5b3714d3701..1a8ff4c4ce5 100644 --- a/acceptance/bundle/invariant/configs/vector_search_endpoint.yml.tmpl +++ b/acceptance/bundle/invariant/configs/vector_search_endpoint.yml.tmpl @@ -6,7 +6,10 @@ resources: foo: name: test-endpoint-$UNIQUE_NAME endpoint_type: STANDARD - target_qps: 2 # CAN_REMOVE + # CANNOT_REMOVE: target_qps is not re-sent on update, so clearing it leaves the old value + # and the plan re-proposes the change on every deploy (only reproducible on cloud; the + # local testserver converges). + target_qps: 2 bar: # Endpoint names must be < 50 chars, so keep this prefix short. name: test-vse-perm-$UNIQUE_NAME From cc5a9d6c90d6c14601f3d249a8dbf9a6aca9ec47 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 21 Sep 2026 17:01:43 +0200 Subject: [PATCH 7/7] acceptance: fold field_removal into no_drift; drop #6343 references The field_removal target re-ran no_drift's deploy+plan as its baseline for every config, to add a removal check only ~14 use. Move the removal cycle into no_drift (guarded to the non-READPLAN variant, and only when a `# CAN_REMOVE` field is present) so it reuses the existing baseline instead of duplicating it, and delete the separate target. Also drop the #6343 references from the CANNOT_REMOVE comments: #6343 fixed comment-clearing and is not the source of these failures, so each comment now just states its own mechanism. Co-authored-by: Isaac --- acceptance/bundle/invariant/README.md | 10 +-- .../configs/catalog_optional_fields.yml.tmpl | 2 +- .../invariant/configs/genie_space.yml.tmpl | 2 +- .../configs/postgres_branch.yml.tmpl | 2 +- .../configs/postgres_endpoint.yml.tmpl | 2 +- .../configs/postgres_project.yml.tmpl | 2 +- .../bundle/invariant/configs/schema.yml.tmpl | 2 +- .../bundle/invariant/configs/secret.yml.tmpl | 2 +- .../invariant/field_removal/out.test.toml | 66 ------------------- .../bundle/invariant/field_removal/output.txt | 1 - .../bundle/invariant/field_removal/script | 33 ---------- .../bundle/invariant/field_removal/test.toml | 7 -- acceptance/bundle/invariant/no_drift/script | 25 ++++++- acceptance/selftest/selection/output.txt | 16 ++--- 14 files changed, 40 insertions(+), 132 deletions(-) delete mode 100644 acceptance/bundle/invariant/field_removal/out.test.toml delete mode 100644 acceptance/bundle/invariant/field_removal/output.txt delete mode 100644 acceptance/bundle/invariant/field_removal/script delete mode 100644 acceptance/bundle/invariant/field_removal/test.toml diff --git a/acceptance/bundle/invariant/README.md b/acceptance/bundle/invariant/README.md index eecdbfc6c3f..3dc1bf04fab 100644 --- a/acceptance/bundle/invariant/README.md +++ b/acceptance/bundle/invariant/README.md @@ -5,11 +5,11 @@ test will dump full JSON plan to the output. In order to add a new test, add a config to configs/ and include it in test.toml. -The field_removal test additionally exercises removing non-required fields. Tag a field with a +The no_drift test additionally exercises removing non-required fields. Tag a field with a `# CAN_REMOVE` comment on each of its lines; after the baseline no-drift check the test strips those lines (`grep -v '# CAN_REMOVE'`), then asserts the resulting plan is an update and that deploying it leaves no drift. This guards the class of bug where a field the config stops declaring is dropped from -the update request instead of being cleared (see #6343). Configs without `# CAN_REMOVE` only run the -baseline. If a field's removal does not converge, tag it `# CANNOT_REMOVE` with a comment explaining the -error or drift, rather than leaving a failing `# CAN_REMOVE`. The marker is matched with its `# ` prefix, -so `# CANNOT_REMOVE` and prose that mentions the marker are neither stripped nor treated as a tag. +the update request instead of being cleared. Configs without `# CAN_REMOVE` only run the baseline. If a +field's removal does not converge, tag it `# CANNOT_REMOVE` with a comment explaining the error or drift, +rather than leaving a failing `# CAN_REMOVE`. The marker is matched with its `# ` prefix, so +`# CANNOT_REMOVE` and prose that mentions the marker are neither stripped nor treated as a tag. diff --git a/acceptance/bundle/invariant/configs/catalog_optional_fields.yml.tmpl b/acceptance/bundle/invariant/configs/catalog_optional_fields.yml.tmpl index fecebe4050b..e13b9d45289 100644 --- a/acceptance/bundle/invariant/configs/catalog_optional_fields.yml.tmpl +++ b/acceptance/bundle/invariant/configs/catalog_optional_fields.yml.tmpl @@ -6,7 +6,7 @@ resources: foo: name: test-catalog-$UNIQUE_NAME comment: This is a test catalog - # CANNOT_REMOVE: not force-sent on update (terraform omits it too, see #6343), so + # CANNOT_REMOVE: not force-sent on update (terraform omits it too), so # clearing it drops the field from the PATCH and the old value drifts back every plan. custom_max_retention_hours: 48 managed_encryption_settings: diff --git a/acceptance/bundle/invariant/configs/genie_space.yml.tmpl b/acceptance/bundle/invariant/configs/genie_space.yml.tmpl index 0b604cb8a9a..8b5756d50ab 100644 --- a/acceptance/bundle/invariant/configs/genie_space.yml.tmpl +++ b/acceptance/bundle/invariant/configs/genie_space.yml.tmpl @@ -7,7 +7,7 @@ resources: warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID title: test-genie-space-$UNIQUE_NAME # CANNOT_REMOVE: description is not force-sent on update, so clearing it drops the - # field from the request and the old value drifts back on every subsequent plan (#6343 class). + # field from the request and the old value drifts back on every subsequent plan. description: This is a test genie space # Structured (inline) serialized_space is marshalled to a JSON string by # ConfigureGenieSpaceSerializedSpace; this config doubles as a regression diff --git a/acceptance/bundle/invariant/configs/postgres_branch.yml.tmpl b/acceptance/bundle/invariant/configs/postgres_branch.yml.tmpl index 9c0ec661637..2a7fe6d2dc7 100644 --- a/acceptance/bundle/invariant/configs/postgres_branch.yml.tmpl +++ b/acceptance/bundle/invariant/configs/postgres_branch.yml.tmpl @@ -13,6 +13,6 @@ resources: branch_id: test-branch-$UNIQUE_NAME # CANNOT_REMOVE: clearing no_expiry adds spec.expiration to the update_mask but drops the field # from the request body (omitempty), which the backend rejects with 400 "in update_mask but not - # provided in request" (#6343 class). (is_protected behaves the same but is omitted here: a + # provided in request". (is_protected behaves the same but is omitted here: a # protected branch cannot be deleted, which wedges the test's destroy-based cleanup.) no_expiry: true diff --git a/acceptance/bundle/invariant/configs/postgres_endpoint.yml.tmpl b/acceptance/bundle/invariant/configs/postgres_endpoint.yml.tmpl index d70f2141495..2e678e70ad2 100644 --- a/acceptance/bundle/invariant/configs/postgres_endpoint.yml.tmpl +++ b/acceptance/bundle/invariant/configs/postgres_endpoint.yml.tmpl @@ -20,7 +20,7 @@ resources: endpoint_type: ENDPOINT_TYPE_READ_WRITE # CANNOT_REMOVE: clearing these adds them to the update_mask but drops the fields from the # request body (omitempty), which the backend rejects with 400 "Field 'spec.autoscaling_limit_max_cu' - # is in update_mask but not provided in request" (#6343 class). + # is in update_mask but not provided in request". autoscaling_limit_min_cu: 1 autoscaling_limit_max_cu: 4 disabled: true diff --git a/acceptance/bundle/invariant/configs/postgres_project.yml.tmpl b/acceptance/bundle/invariant/configs/postgres_project.yml.tmpl index 63652868c57..c8f5dbd13ea 100644 --- a/acceptance/bundle/invariant/configs/postgres_project.yml.tmpl +++ b/acceptance/bundle/invariant/configs/postgres_project.yml.tmpl @@ -7,7 +7,7 @@ resources: project_id: test-pg-project-$UNIQUE_NAME # CANNOT_REMOVE: clearing display_name adds spec.display_name to the update_mask but drops # the field from the request body (omitempty), which the backend rejects with 400 - # "Field 'spec.display_name' is in update_mask but not provided in request" (#6343 class). + # "Field 'spec.display_name' is in update_mask but not provided in request". display_name: Test Postgres Project enable_pg_native_login: false # CANNOT_REMOVE: same update_mask 400 as display_name above — clearing it adds diff --git a/acceptance/bundle/invariant/configs/schema.yml.tmpl b/acceptance/bundle/invariant/configs/schema.yml.tmpl index bd2af70a77b..f64f43c511f 100644 --- a/acceptance/bundle/invariant/configs/schema.yml.tmpl +++ b/acceptance/bundle/invariant/configs/schema.yml.tmpl @@ -7,7 +7,7 @@ resources: catalog_name: main name: test-schema-$UNIQUE_NAME comment: This is a test schema # CAN_REMOVE - # CANNOT_REMOVE: properties is a map, and ForceSendFields is inert for maps (#6343), so + # CANNOT_REMOVE: properties is a map, and ForceSendFields is inert for maps, so # clearing the whole map drops it from the request and the old value drifts back every plan. # Kept on one line so a future fix can flip it to CAN_REMOVE and have grep -v strip it cleanly. properties: {"tier": "bronze"} diff --git a/acceptance/bundle/invariant/configs/secret.yml.tmpl b/acceptance/bundle/invariant/configs/secret.yml.tmpl index c9c32bf9ed7..ed8ba82eb32 100644 --- a/acceptance/bundle/invariant/configs/secret.yml.tmpl +++ b/acceptance/bundle/invariant/configs/secret.yml.tmpl @@ -13,7 +13,7 @@ resources: name: test-secret-$UNIQUE_NAME value: ${var.secret_value} # CANNOT_REMOVE: comment is not force-sent on update, so clearing it drops the field - # from the request and the old value drifts back on every subsequent plan (#6343 class). + # from the request and the old value drifts back on every subsequent plan. comment: This is a test secret grants: - principal: account users diff --git a/acceptance/bundle/invariant/field_removal/out.test.toml b/acceptance/bundle/invariant/field_removal/out.test.toml deleted file mode 100644 index 8585e6f2dd6..00000000000 --- a/acceptance/bundle/invariant/field_removal/out.test.toml +++ /dev/null @@ -1,66 +0,0 @@ -Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] -EnvMatrix.INPUT_CONFIG = [ - "alert.yml.tmpl", - "app.yml.tmpl", - "catalog.yml.tmpl", - "catalog_optional_fields.yml.tmpl", - "cluster.yml.tmpl", - "cluster_apply_policy_default_values.yml.tmpl", - "cluster_policy.yml.tmpl", - "dashboard.yml.tmpl", - "job_apply_policy_default_values_job_cluster.yml.tmpl", - "job_apply_policy_default_values_task_cluster.yml.tmpl", - "job_apply_policy_default_values_for_each_task.yml.tmpl", - "database_catalog.yml.tmpl", - "database_instance.yml.tmpl", - "experiment.yml.tmpl", - "external_location.yml.tmpl", - "genie_space.yml.tmpl", - "instance_pool.yml.tmpl", - "job.yml.tmpl", - "job_pydabs_10_tasks.yml.tmpl", - "job_pydabs_1000_tasks.yml.tmpl", - "job_cross_resource_ref.yml.tmpl", - "job_escaped_refs.yml.tmpl", - "job_permission_ref.yml.tmpl", - "job_run.yml.tmpl", - "job_run_job_ref.yml.tmpl", - "job_table_update_trigger.yml.tmpl", - "job_with_depends_on.yml.tmpl", - "job_with_task.yml.tmpl", - "model.yml.tmpl", - "model_with_permissions.yml.tmpl", - "model_serving_endpoint.yml.tmpl", - "model_serving_endpoint_telemetry.yml.tmpl", - "pipeline.yml.tmpl", - "pipeline_allow_duplicate_names.yml.tmpl", - "pipeline_apply_policy_default_values.yml.tmpl", - "pipeline_config_dots.yml.tmpl", - "postgres_branch.yml.tmpl", - "postgres_catalog.yml.tmpl", - "postgres_database.yml.tmpl", - "postgres_endpoint.yml.tmpl", - "postgres_project.yml.tmpl", - "postgres_role.yml.tmpl", - "postgres_synced_table.yml.tmpl", - "registered_model.yml.tmpl", - "schema.yml.tmpl", - "schema_empty_grants.yml.tmpl", - "schema_grant_ref.yml.tmpl", - "schema_uppercase_name.yml.tmpl", - "secret.yml.tmpl", - "secret_scope.yml.tmpl", - "secret_scope_default_backend_type.yml.tmpl", - "sql_warehouse.yml.tmpl", - "synced_database_table.yml.tmpl", - "uc_trailing_slash.yml.tmpl", - "vector_search_endpoint.yml.tmpl", - "vector_search_index.yml.tmpl", - "volume.yml.tmpl", - "volume_external.yml.tmpl", - "volume_path_job_ref.yml.tmpl", - "volume_uppercase_name.yml.tmpl", - "model_service.yml.tmpl" -] diff --git a/acceptance/bundle/invariant/field_removal/output.txt b/acceptance/bundle/invariant/field_removal/output.txt deleted file mode 100644 index 7a28cb73a58..00000000000 --- a/acceptance/bundle/invariant/field_removal/output.txt +++ /dev/null @@ -1 +0,0 @@ -INPUT_CONFIG_OK diff --git a/acceptance/bundle/invariant/field_removal/script b/acceptance/bundle/invariant/field_removal/script deleted file mode 100644 index 38b4d119c75..00000000000 --- a/acceptance/bundle/invariant/field_removal/script +++ /dev/null @@ -1,33 +0,0 @@ -# Invariant to test: removing a non-required field is applied as an update and converges. -# Fields tagged `# CAN_REMOVE` in the config are dropped, and the resulting plan must show an -# update; after deploying it a fresh plan must show no drift. This guards the omitempty-drop -# class of bug (#6343): a field the config stops declaring must be sent as its zero value, not -# silently dropped from the PATCH (which either fails with "Nothing to update" or never converges). -# Additional checks: no internal errors / panics in validate/plan/deploy. - -invariant_setup - -invariant_deploy LOG.deploy $CLI bundle deploy - -invariant_verify_no_drift - -# Only configs that tag fields with `# CAN_REMOVE` exercise the removal cycle; the rest just -# assert the baseline no-drift above. The marker is matched with its `# ` prefix so that neither -# `# CANNOT_REMOVE` (a documented, deliberately-kept field) nor prose mentioning the marker in a -# comment triggers the cycle or gets stripped. -if grep -q '# CAN_REMOVE' databricks.yml; then - grep -v '# CAN_REMOVE' databricks.yml > databricks.yml.new - mv databricks.yml.new databricks.yml - cp databricks.yml LOG.config.removed - - $CLI bundle plan -o json > LOG.plan_removed.json 2>LOG.plan_removed.err - cat LOG.plan_removed.err | contains.py '!panic:' '!internal error' > /dev/null - verify_update.py LOG.plan_removed.json - - # Not invariant_deploy: a second INPUT_CONFIG_OK would make output.txt differ between the - # configs that have removable fields and those that don't. - trace $CLI bundle deploy &> LOG.deploy_removed - cat LOG.deploy_removed | contains.py '!panic:' '!internal error' > /dev/null - - invariant_verify_no_drift -fi diff --git a/acceptance/bundle/invariant/field_removal/test.toml b/acceptance/bundle/invariant/field_removal/test.toml deleted file mode 100644 index c657e14500c..00000000000 --- a/acceptance/bundle/invariant/field_removal/test.toml +++ /dev/null @@ -1,7 +0,0 @@ -# A 1000-task job serializes to ~110 KB, over the 64 KB per-operation state limit the -# deployment metadata service accepts, so recording it fails the deploy (same as no_drift). -EnvMatrixExclude.dms_state_too_large = ["DMS=true", "INPUT_CONFIG=job_pydabs_1000_tasks.yml.tmpl"] - -# Emptying a grants node records a delete with no state, which the deployment metadata service -# rejects on a succeeded UPDATE action (known bug bundle/dms/empty-grants); skip under recording. -EnvMatrixExclude.dms_empty_grants_bug = ["DMS=true", "INPUT_CONFIG=schema_empty_grants.yml.tmpl"] diff --git a/acceptance/bundle/invariant/no_drift/script b/acceptance/bundle/invariant/no_drift/script index 9f031c0f613..0961e8edac8 100644 --- a/acceptance/bundle/invariant/no_drift/script +++ b/acceptance/bundle/invariant/no_drift/script @@ -1,4 +1,4 @@ -# Invariant to test: no drift after deploy +# Invariant to test: no drift after deploy, and that removing a non-required field converges. # Additional checks: no internal errors / panics in validate/plan/deploy invariant_setup @@ -14,3 +14,26 @@ fi invariant_deploy LOG.deploy $CLI bundle deploy $(readplanarg plan.json) invariant_verify_no_drift + +# Field removal: dropping a non-required field (tagged `# CAN_REMOVE` on each of its lines) must be +# applied as an update that converges -- a field the config stops declaring must be sent as its zero +# value, not silently dropped from the request (which either fails or drifts forever). Runs only on the +# non-READPLAN variant (the saved-plan path is orthogonal) and only for configs that tag a field. +# `# CANNOT_REMOVE` documents a field whose removal is broken and is deliberately kept; the marker is +# matched with its `# ` prefix so neither it nor prose mentioning the marker is stripped. +if [[ -z "$READPLAN" ]] && grep -q '# CAN_REMOVE' databricks.yml; then + grep -v '# CAN_REMOVE' databricks.yml > databricks.yml.new + mv databricks.yml.new databricks.yml + cp databricks.yml LOG.config.removed + + $CLI bundle plan -o json > LOG.plan_removed.json 2>LOG.plan_removed.err + cat LOG.plan_removed.err | contains.py '!panic:' '!internal error' > /dev/null + verify_update.py LOG.plan_removed.json + + # Plain deploy (not invariant_deploy): a second INPUT_CONFIG_OK would make output.txt differ + # from the variants that skip this block. + trace $CLI bundle deploy &> LOG.deploy_removed + cat LOG.deploy_removed | contains.py '!panic:' '!internal error' > /dev/null + + invariant_verify_no_drift +fi diff --git a/acceptance/selftest/selection/output.txt b/acceptance/selftest/selection/output.txt index f8916682d51..505dd11316c 100644 --- a/acceptance/selftest/selection/output.txt +++ b/acceptance/selftest/selection/output.txt @@ -28,12 +28,10 @@ Selected 2 changed tests (limit=50, 0 not selected) === Touching an invariant config runs every invariant test, for that config only >>> selection M:acceptance/bundle/invariant/configs/job.yml.tmpl -Selected 6 changed tests (limit=50, 0 not selected) +Selected 5 changed tests (limit=50, 0 not selected) 5 ^bundle$/^invariant$/^continue_293$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^delete_idempotent$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^destroy_idempotent$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ - 5 ^bundle$/^invariant$/^field_removal$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ - 5 ^bundle$/^invariant$/^field_removal$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=true$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^migrate$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^no_drift$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^no_drift$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=true$/^INPUT_CONFIG=job\.yml\.tmpl$ @@ -41,20 +39,16 @@ Selected 6 changed tests (limit=50, 0 not selected) === Adding one invariant config and touching another: only the new variant is new >>> selection A:acceptance/bundle/invariant/configs/pipeline.yml.tmpl M:acceptance/bundle/invariant/configs/job.yml.tmpl -Selected 12 changed tests (limit=50, 0 not selected) +Selected 10 changed tests (limit=50, 0 not selected) 10 ^bundle$/^invariant$/^continue_293$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=pipeline\.yml\.tmpl$ 10 ^bundle$/^invariant$/^delete_idempotent$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=pipeline\.yml\.tmpl$ 10 ^bundle$/^invariant$/^destroy_idempotent$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=pipeline\.yml\.tmpl$ - 10 ^bundle$/^invariant$/^field_removal$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=pipeline\.yml\.tmpl$ - 10 ^bundle$/^invariant$/^field_removal$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=true$/^INPUT_CONFIG=pipeline\.yml\.tmpl$ 10 ^bundle$/^invariant$/^migrate$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=pipeline\.yml\.tmpl$ 10 ^bundle$/^invariant$/^no_drift$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=pipeline\.yml\.tmpl$ 10 ^bundle$/^invariant$/^no_drift$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=true$/^INPUT_CONFIG=pipeline\.yml\.tmpl$ 5 ^bundle$/^invariant$/^continue_293$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^delete_idempotent$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^destroy_idempotent$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ - 5 ^bundle$/^invariant$/^field_removal$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ - 5 ^bundle$/^invariant$/^field_removal$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=true$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^migrate$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^no_drift$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^no_drift$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=true$/^INPUT_CONFIG=job\.yml\.tmpl$ @@ -62,11 +56,9 @@ Selected 12 changed tests (limit=50, 0 not selected) === A config an invariant test excludes does not select that test >>> selection M:acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl -Selected 4 changed tests (limit=50, 0 not selected) +Selected 3 changed tests (limit=50, 0 not selected) 5 ^bundle$/^invariant$/^delete_idempotent$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=vector_search_index\.yml\.tmpl$ 5 ^bundle$/^invariant$/^destroy_idempotent$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=vector_search_index\.yml\.tmpl$ - 5 ^bundle$/^invariant$/^field_removal$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=vector_search_index\.yml\.tmpl$ - 5 ^bundle$/^invariant$/^field_removal$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=true$/^INPUT_CONFIG=vector_search_index\.yml\.tmpl$ 5 ^bundle$/^invariant$/^no_drift$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=vector_search_index\.yml\.tmpl$ 5 ^bundle$/^invariant$/^no_drift$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=true$/^INPUT_CONFIG=vector_search_index\.yml\.tmpl$ @@ -79,7 +71,7 @@ Selected 1 changed tests (limit=50, 0 not selected) === The limit keeps the highest scoring tests >>> selection -limit 3 A:acceptance/selftest/basic/script M:acceptance/selftest/diff/script M:acceptance/bundle/invariant/configs/job.yml.tmpl -Selected 3 changed tests (limit=3, 5 not selected) +Selected 3 changed tests (limit=3, 4 not selected) 10 ^selftest$/^basic$ 5 ^bundle$/^invariant$/^continue_293$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$ 5 ^bundle$/^invariant$/^delete_idempotent$/^DATABRICKS_BUNDLE_ENGINE=direct$/^DMS=$/^INPUT_CONFIG=job\.yml\.tmpl$