diff --git a/.nextchanges/bundles/migrate-before-deploy.md b/.nextchanges/bundles/migrate-before-deploy.md new file mode 100644 index 00000000000..a7f00fc7bb1 --- /dev/null +++ b/.nextchanges/bundles/migrate-before-deploy.md @@ -0,0 +1 @@ +* Migrate existing Terraform deployment state to the direct engine before deploying (previously done after a Terraform deploy), so the deploy runs on the direct engine. ([#6749](https://github.com/databricks/cli/pull/6749)) diff --git a/acceptance/bundle/deploy/snapshot-comparison/output.txt b/acceptance/bundle/deploy/snapshot-comparison/output.txt index 35c1b7ad43d..a6fa2b86346 100644 --- a/acceptance/bundle/deploy/snapshot-comparison/output.txt +++ b/acceptance/bundle/deploy/snapshot-comparison/output.txt @@ -54,6 +54,8 @@ Destroy: 2 deleted === Cleanup bundle 2 >>> [CLI] bundle destroy --auto-approve +Notice: automatically migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). +Migrated 2 resources to direct deployment engine. The following resources will be deleted: delete resources.jobs.test_job delete resources.pipelines.test_pipeline diff --git a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/script b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/script deleted file mode 100644 index e72b1cadab4..00000000000 --- a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/script +++ /dev/null @@ -1,31 +0,0 @@ -# A terraform.tfstate with no databricks_* resources has no state to -# migrate. When the user has opted in to direct, sweep the empty terraform -# state aside (locally + remotely) so the next deploy picks the direct -# engine by default instead of getting stuck on a permanent mismatch warning. - -mkdir -p .databricks/bundle/default/terraform -cat > .databricks/bundle/default/terraform/terraform.tfstate <<'EOF' -{"version": 4, "serial": 1, "lineage": "test-lineage", "resources": []} -EOF - -title "Deploy with bundle.engine: direct and an empty terraform state — sweep, don't migrate" -trace $CLI bundle deploy 2>&1 | contains.py 'Removing empty terraform state' '!Migrating state to direct' - -title "Terraform state is renamed to .backup; no resources.json (empty state, nothing to persist)\n" -trace find .databricks/bundle -name "resources.json" -type f -trace find .databricks/bundle -name "terraform.tfstate*" -type f - -title "Sweep is recorded in telemetry as via-config\n" -trace print_migration_telemetry - -title "Sweep also affected the workspace: .backup uploaded, terraform.tfstate deleted\n" -trace print_requests.py //workspace-files/import-file //workspace/delete --sort | jq '.method + " " + .path' | contains.py 'terraform.tfstate.backup' 'workspace/delete' - -title "debug states confirms no terraform.tfstate anywhere; only .backup remains\n" -trace $CLI bundle debug states --force-pull - -title "Next deploy uses direct engine by default (no more mismatch warning)" -trace $CLI bundle deploy 2>&1 | contains.py '!does not match the existing state' -trace print_migration_telemetry - -rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/migrate/auto-migrate-plan-failure/output.txt b/acceptance/bundle/migrate/auto-migrate-plan-failure/output.txt deleted file mode 100644 index 410ec2b580e..00000000000 --- a/acceptance/bundle/migrate/auto-migrate-plan-failure/output.txt +++ /dev/null @@ -1,55 +0,0 @@ - -=== Initial deploy on terraform ->>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Created jobs.test_job -Files: 2 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> print_migration_telemetry -direct_drymigrate_success true -direct_drymigrate_warnings false - -=== Inject GET failure so the plan check blocks the migration ->>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy -Warn: Direct engine selected via DATABRICKS_BUNDLE_ENGINE environment variable but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy. -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Files: 0 uploaded, 0 deleted -Resources: 0 created, 0 changed, 0 deleted, 1 unchanged -Warn: post-deploy dry-run migration to direct: cannot plan resources.jobs.test_job: reading id="[NUMID]": Fault injected by test. (403 INJECTED): Endpoint: GET [DATABRICKS_URL]/api/2.2/jobs/get?job_id=[NUMID] -HTTP Status: 403 Forbidden -API error_code: INJECTED -API message: Fault injected by test. -Warn: post-deploy dry-run migration to direct: planning failed -Warn: The warnings above are from a dry-run migration to the direct deployment engine (https://docs.databricks.com/aws/en/dev-tools/bundles/direct). -Your deployment is not affected and works normally, but you may experience these issues when migrating to the direct deployment engine. -Please forward these warnings to dabs-feedback@databricks.com -Warn: Direct engine was selected but the migration reported issues; automatic migration to the direct deployment engine is stopped. Address the issues above or run "databricks bundle deployment migrate" manually. - ->>> print_migration_telemetry -direct_migrate_plan_error true - -=== Local state was NOT rewritten (still terraform) ->>> find .databricks/bundle -name resources.json -type f - ->>> find .databricks/bundle -name terraform.tfstate* -type f -.databricks/bundle/default/terraform/terraform.tfstate - -=== Retry: plan check passes this time, migration succeeds ->>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy -Warn: Direct engine selected via DATABRICKS_BUNDLE_ENGINE environment variable but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy. -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Files: 0 uploaded, 0 deleted -Resources: 0 created, 0 changed, 0 deleted, 1 unchanged -Migrating state to direct deployment engine (selected via DATABRICKS_BUNDLE_ENGINE environment variable)... -Migrated 1 resource to direct deployment engine. - ->>> print_migration_telemetry -direct_migrated_via_env true - -=== Local state is now direct ->>> find .databricks/bundle -name resources.json -type f -.databricks/bundle/default/resources.json - ->>> find .databricks/bundle -name terraform.tfstate* -type f -.databricks/bundle/default/terraform/terraform.tfstate.backup diff --git a/acceptance/bundle/migrate/auto/artifacts/databricks.yml b/acceptance/bundle/migrate/auto/artifacts/databricks.yml new file mode 100644 index 00000000000..bf98a6ebc3b --- /dev/null +++ b/acceptance/bundle/migrate/auto/artifacts/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: test-bundle + +resources: + jobs: + test_job: + name: "Test Auto-Migrate Artifacts Job" + tasks: + - task_key: TestTask + existing_cluster_id: "0717-132531-5opeqon1" + python_wheel_task: + package_name: "my_test_code" + entry_point: "run" + libraries: + - whl: ./dist/*.whl diff --git a/acceptance/bundle/migrate/auto/artifacts/dist/my_test_code-0.0.1-py3-none-any.whl b/acceptance/bundle/migrate/auto/artifacts/dist/my_test_code-0.0.1-py3-none-any.whl new file mode 100644 index 00000000000..4bb80477caf Binary files /dev/null and b/acceptance/bundle/migrate/auto/artifacts/dist/my_test_code-0.0.1-py3-none-any.whl differ diff --git a/acceptance/bundle/migrate/added-direct-only/out.test.toml b/acceptance/bundle/migrate/auto/artifacts/out.test.toml similarity index 100% rename from acceptance/bundle/migrate/added-direct-only/out.test.toml rename to acceptance/bundle/migrate/auto/artifacts/out.test.toml diff --git a/acceptance/bundle/migrate/auto/artifacts/output.txt b/acceptance/bundle/migrate/auto/artifacts/output.txt new file mode 100644 index 00000000000..4b18ca60194 --- /dev/null +++ b/acceptance/bundle/migrate/auto/artifacts/output.txt @@ -0,0 +1,43 @@ + +=== Initial deploy on terraform (expands the whl glob and uploads the wheel) +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy +Warn: unable to get spark version for cluster 0717-132531-5opeqon1, err: Not Found +Uploading dist/my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created jobs.test_job +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Redeploy with direct default: auto-migration runs after the build, so library refs are resolved +>>> [CLI] bundle deploy +Warn: unable to get spark version for cluster 0717-132531-5opeqon1, err: Not Found +Notice: automatically migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). +Migrated 1 resource to direct deployment engine. +Uploading dist/my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +=== Local state is now direct + +>>> find .databricks/bundle -name resources.json -type f +.databricks/bundle/default/resources.json + +>>> find .databricks/bundle -name terraform.tfstate* -type f +.databricks/bundle/default/terraform/terraform.tfstate.backup + +=== Migrated state's library path (should be the resolved remote path, not the local glob) +>>> print_state.py +[ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + } +] + +=== Subsequent deploy is a no-op on direct +>>> [CLI] bundle deploy +Warn: unable to get spark version for cluster 0717-132531-5opeqon1, err: Not Found +Uploading dist/my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged diff --git a/acceptance/bundle/migrate/auto/artifacts/script b/acceptance/bundle/migrate/auto/artifacts/script new file mode 100644 index 00000000000..d9fe2cf68cc --- /dev/null +++ b/acceptance/bundle/migrate/auto/artifacts/script @@ -0,0 +1,17 @@ +export DATABRICKS_BUNDLE_ENGINE= + +title "Initial deploy on terraform (expands the whl glob and uploads the wheel)" +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy + +title "Redeploy with direct default: auto-migration runs after the build, so library refs are resolved" +trace $CLI bundle deploy + +title "Local state is now direct\n" +trace find .databricks/bundle -name "resources.json" -type f +trace find .databricks/bundle -name "terraform.tfstate*" -type f + +title "Migrated state's library path (should be the resolved remote path, not the local glob)" +trace print_state.py | jq '.. | .libraries? | select(.)' + +title "Subsequent deploy is a no-op on direct" +trace $CLI bundle deploy diff --git a/acceptance/bundle/migrate/engine-config-direct/test.toml b/acceptance/bundle/migrate/auto/artifacts/test.toml similarity index 100% rename from acceptance/bundle/migrate/engine-config-direct/test.toml rename to acceptance/bundle/migrate/auto/artifacts/test.toml diff --git a/acceptance/bundle/migrate/auto-migrate-clean/databricks.yml b/acceptance/bundle/migrate/auto/clean/databricks.yml similarity index 100% rename from acceptance/bundle/migrate/auto-migrate-clean/databricks.yml rename to acceptance/bundle/migrate/auto/clean/databricks.yml diff --git a/acceptance/bundle/migrate/auto-migrate-clean/out.plan_update.direct.json b/acceptance/bundle/migrate/auto/clean/out.plan_update.direct.json similarity index 100% rename from acceptance/bundle/migrate/auto-migrate-clean/out.plan_update.direct.json rename to acceptance/bundle/migrate/auto/clean/out.plan_update.direct.json diff --git a/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml b/acceptance/bundle/migrate/auto/clean/out.test.toml similarity index 100% rename from acceptance/bundle/migrate/auto-migrate-clean/out.test.toml rename to acceptance/bundle/migrate/auto/clean/out.test.toml diff --git a/acceptance/bundle/migrate/auto-migrate-clean/output.txt b/acceptance/bundle/migrate/auto/clean/output.txt similarity index 87% rename from acceptance/bundle/migrate/auto-migrate-clean/output.txt rename to acceptance/bundle/migrate/auto/clean/output.txt index a358b635116..15cb15b183e 100644 --- a/acceptance/bundle/migrate/auto-migrate-clean/output.txt +++ b/acceptance/bundle/migrate/auto/clean/output.txt @@ -15,12 +15,10 @@ direct_drymigrate_warnings false engine: direct >>> [CLI] bundle deploy -Warn: Direct engine selected via bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11 but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy. +Migrated 1 resource to direct deployment engine. Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Files: 1 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged -Migrating state to direct deployment engine (selected via bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11)... -Migrated 1 resource to direct deployment engine. >>> print_migration_telemetry direct_migrated_via_config true diff --git a/acceptance/bundle/migrate/auto-migrate-clean/script b/acceptance/bundle/migrate/auto/clean/script similarity index 100% rename from acceptance/bundle/migrate/auto-migrate-clean/script rename to acceptance/bundle/migrate/auto/clean/script diff --git a/acceptance/bundle/migrate/auto-migrate-clean/test.toml b/acceptance/bundle/migrate/auto/clean/test.toml similarity index 100% rename from acceptance/bundle/migrate/auto-migrate-clean/test.toml rename to acceptance/bundle/migrate/auto/clean/test.toml diff --git a/acceptance/bundle/migrate/auto-migrate-default/databricks.yml b/acceptance/bundle/migrate/auto/default/databricks.yml similarity index 100% rename from acceptance/bundle/migrate/auto-migrate-default/databricks.yml rename to acceptance/bundle/migrate/auto/default/databricks.yml diff --git a/acceptance/bundle/migrate/added/out.test.toml b/acceptance/bundle/migrate/auto/default/out.test.toml similarity index 100% rename from acceptance/bundle/migrate/added/out.test.toml rename to acceptance/bundle/migrate/auto/default/out.test.toml diff --git a/acceptance/bundle/migrate/auto-migrate-default/output.txt b/acceptance/bundle/migrate/auto/default/output.txt similarity index 84% rename from acceptance/bundle/migrate/auto-migrate-default/output.txt rename to acceptance/bundle/migrate/auto/default/output.txt index 33950c186e3..4d92b0eb02d 100644 --- a/acceptance/bundle/migrate/auto-migrate-default/output.txt +++ b/acceptance/bundle/migrate/auto/default/output.txt @@ -12,17 +12,11 @@ direct_drymigrate_warnings false === Redeploy with nothing set: direct is the default, so auto-migration triggers >>> [CLI] bundle deploy -Notice: the direct deployment engine is the default as of CLI v1.14.0. - -This bundle will be automatically migrated to use the direct deployment engine after this deployment. - -Learn more: https://docs.databricks.com/dev-tools/bundles/direct - +Notice: automatically migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). +Migrated 1 resource to direct deployment engine. Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Files: 0 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged -Migrating state to direct deployment engine (selected via default)... -Migrated 1 resource to direct deployment engine. >>> print_migration_telemetry direct_migrated_via_default true diff --git a/acceptance/bundle/migrate/auto-migrate-default/script b/acceptance/bundle/migrate/auto/default/script similarity index 100% rename from acceptance/bundle/migrate/auto-migrate-default/script rename to acceptance/bundle/migrate/auto/default/script diff --git a/acceptance/bundle/migrate/auto/delete-all/databricks.yml b/acceptance/bundle/migrate/auto/delete-all/databricks.yml new file mode 100644 index 00000000000..b13f958f575 --- /dev/null +++ b/acceptance/bundle/migrate/auto/delete-all/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-bundle + +resources: + jobs: + job_a: + name: "Job A" diff --git a/acceptance/bundle/migrate/auto-migrate-default/out.test.toml b/acceptance/bundle/migrate/auto/delete-all/out.test.toml similarity index 100% rename from acceptance/bundle/migrate/auto-migrate-default/out.test.toml rename to acceptance/bundle/migrate/auto/delete-all/out.test.toml diff --git a/acceptance/bundle/migrate/auto/delete-all/output.txt b/acceptance/bundle/migrate/auto/delete-all/output.txt new file mode 100644 index 00000000000..3178149abeb --- /dev/null +++ b/acceptance/bundle/migrate/auto/delete-all/output.txt @@ -0,0 +1,20 @@ + +=== Initial deploy on terraform +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created jobs.job_a +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Remove all resources, then deploy on direct: migrate and delete +>>> [CLI] bundle deploy +Notice: automatically migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). +Migrated 1 resource to direct deployment engine. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deleted jobs.job_a +Files: 1 uploaded, 0 deleted +Resources: 0 created, 0 changed, 1 deleted, 0 unchanged + +=== Follow-up plan is a clean no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/migrate/auto/delete-all/script b/acceptance/bundle/migrate/auto/delete-all/script new file mode 100644 index 00000000000..bf3ae01ff5b --- /dev/null +++ b/acceptance/bundle/migrate/auto/delete-all/script @@ -0,0 +1,15 @@ +export DATABRICKS_BUNDLE_ENGINE= + +title "Initial deploy on terraform" +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy + +# Removing every resource leaves the migration with no config-declared resources, so +# BuildStateFromTF records no WAL entries. The migration must still persist the seeded +# terraform-state resources so the direct plan can delete them (otherwise Finalize would +# write no state file and the migration would fail with a missing resources.json). +title "Remove all resources, then deploy on direct: migrate and delete" +echo '{"bundle": {"name": "test-bundle"}}' > databricks.yml +trace $CLI bundle deploy + +title "Follow-up plan is a clean no-op" +trace $CLI bundle plan | contains.py "0 to delete, 0 unchanged" diff --git a/acceptance/bundle/migrate/engine-config-terraform/test.toml b/acceptance/bundle/migrate/auto/delete-all/test.toml similarity index 100% rename from acceptance/bundle/migrate/engine-config-terraform/test.toml rename to acceptance/bundle/migrate/auto/delete-all/test.toml diff --git a/acceptance/bundle/migrate/auto/delete/databricks.yml b/acceptance/bundle/migrate/auto/delete/databricks.yml new file mode 100644 index 00000000000..858177a2dfb --- /dev/null +++ b/acceptance/bundle/migrate/auto/delete/databricks.yml @@ -0,0 +1,9 @@ +bundle: + name: test-bundle + +resources: + jobs: + job_keep: + name: "Keep Job" + job_drop: # TO_REMOVE + name: "Drop Job" # TO_REMOVE diff --git a/acceptance/bundle/migrate/auto-migrate-direct-only-envvar/out.test.toml b/acceptance/bundle/migrate/auto/delete/out.test.toml similarity index 100% rename from acceptance/bundle/migrate/auto-migrate-direct-only-envvar/out.test.toml rename to acceptance/bundle/migrate/auto/delete/out.test.toml diff --git a/acceptance/bundle/migrate/auto/delete/output.txt b/acceptance/bundle/migrate/auto/delete/output.txt new file mode 100644 index 00000000000..ba881aadcd4 --- /dev/null +++ b/acceptance/bundle/migrate/auto/delete/output.txt @@ -0,0 +1,21 @@ + +=== Initial deploy on terraform with two jobs +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created jobs.job_drop +Created jobs.job_keep +Files: 3 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Drop one job from config, then deploy on direct: migrate two, delete the dropped one +>>> [CLI] bundle deploy +Notice: automatically migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). +Migrated 2 resources to direct deployment engine. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deleted jobs.job_drop +Files: 1 uploaded, 0 deleted +Resources: 0 created, 0 changed, 1 deleted, 1 unchanged + +=== Follow-up plan is a clean no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/migrate/auto/delete/script b/acceptance/bundle/migrate/auto/delete/script new file mode 100644 index 00000000000..39116c88f9e --- /dev/null +++ b/acceptance/bundle/migrate/auto/delete/script @@ -0,0 +1,13 @@ +export DATABRICKS_BUNDLE_ENGINE= + +title "Initial deploy on terraform with two jobs" +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy + +# The migration seeds every terraform-state resource into the direct state, so the +# first direct plan can delete the ones the config no longer declares. +title "Drop one job from config, then deploy on direct: migrate two, delete the dropped one" +grep -v 'TO_REMOVE' databricks.yml > updated.yml && mv updated.yml databricks.yml +trace $CLI bundle deploy + +title "Follow-up plan is a clean no-op" +trace $CLI bundle plan | contains.py "1 unchanged" diff --git a/acceptance/bundle/migrate/profile_arg/test.toml b/acceptance/bundle/migrate/auto/delete/test.toml similarity index 100% rename from acceptance/bundle/migrate/profile_arg/test.toml rename to acceptance/bundle/migrate/auto/delete/test.toml diff --git a/acceptance/bundle/migrate/auto/destroy/databricks.yml b/acceptance/bundle/migrate/auto/destroy/databricks.yml new file mode 100644 index 00000000000..b13f958f575 --- /dev/null +++ b/acceptance/bundle/migrate/auto/destroy/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-bundle + +resources: + jobs: + job_a: + name: "Job A" diff --git a/acceptance/bundle/migrate/auto-migrate-direct-only/out.test.toml b/acceptance/bundle/migrate/auto/destroy/out.test.toml similarity index 100% rename from acceptance/bundle/migrate/auto-migrate-direct-only/out.test.toml rename to acceptance/bundle/migrate/auto/destroy/out.test.toml diff --git a/acceptance/bundle/migrate/auto/destroy/output.txt b/acceptance/bundle/migrate/auto/destroy/output.txt new file mode 100644 index 00000000000..ef5cafac989 --- /dev/null +++ b/acceptance/bundle/migrate/auto/destroy/output.txt @@ -0,0 +1,34 @@ + +=== Initial deploy on terraform +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created jobs.job_a +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Destroy without approval: stays on terraform +>>> musterr [CLI] bundle destroy +Error: this command will destroy all resources deployed by this bundle, including workspace files in the deployment directory. +Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed. +To proceed, use --auto-approve. + +>>> find .databricks/bundle -name resources.json -type f + +>>> find .databricks/bundle -name terraform.tfstate* -type f +.databricks/bundle/default/terraform/terraform.tfstate + +=== Destroy on direct: migrate then destroy +>>> [CLI] bundle destroy --auto-approve +Notice: automatically migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). +Migrated 1 resource to direct deployment engine. +The following resources will be deleted: + delete resources.jobs.job_a + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 1 deleted + +=== Destroy removes both local state files (direct and the superseded terraform state) +>>> find .databricks/bundle -name resources.json -type f + +>>> find .databricks/bundle -name terraform.tfstate* -type f diff --git a/acceptance/bundle/migrate/auto/destroy/script b/acceptance/bundle/migrate/auto/destroy/script new file mode 100644 index 00000000000..61880555741 --- /dev/null +++ b/acceptance/bundle/migrate/auto/destroy/script @@ -0,0 +1,21 @@ +export DATABRICKS_BUNDLE_ENGINE= + +title "Initial deploy on terraform" +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy + +title "Destroy without approval: stays on terraform" +trace musterr $CLI bundle destroy +trace find .databricks/bundle -name "resources.json" -type f +trace find .databricks/bundle -name "terraform.tfstate*" -type f + +# Destroy behaves like deploy: it migrates the terraform state to direct (writing the local +# direct state), plan-checks it, and - once approved - destroys on the direct engine. destroyCore +# then removes the local state files: the superseded terraform state first, then the direct state, +# so a crash between the two never leaves a live terraform.tfstate with no direct state. A declined +# destroy would discard the migration and stay on terraform. +title "Destroy on direct: migrate then destroy" +trace $CLI bundle destroy --auto-approve + +title "Destroy removes both local state files (direct and the superseded terraform state)" +trace find .databricks/bundle -name "resources.json" -type f +trace find .databricks/bundle -name "terraform.tfstate*" -type f diff --git a/acceptance/bundle/migrate/var_arg/test.toml b/acceptance/bundle/migrate/auto/destroy/test.toml similarity index 100% rename from acceptance/bundle/migrate/var_arg/test.toml rename to acceptance/bundle/migrate/auto/destroy/test.toml diff --git a/acceptance/bundle/migrate/added-direct-only/databricks.yml b/acceptance/bundle/migrate/auto/direct-only-envvar/databricks.yml similarity index 100% rename from acceptance/bundle/migrate/added-direct-only/databricks.yml rename to acceptance/bundle/migrate/auto/direct-only-envvar/databricks.yml diff --git a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml b/acceptance/bundle/migrate/auto/direct-only-envvar/out.test.toml similarity index 100% rename from acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml rename to acceptance/bundle/migrate/auto/direct-only-envvar/out.test.toml diff --git a/acceptance/bundle/migrate/auto-migrate-direct-only-envvar/output.txt b/acceptance/bundle/migrate/auto/direct-only-envvar/output.txt similarity index 55% rename from acceptance/bundle/migrate/auto-migrate-direct-only-envvar/output.txt rename to acceptance/bundle/migrate/auto/direct-only-envvar/output.txt index 527b9a89921..cf073e45e81 100644 --- a/acceptance/bundle/migrate/auto-migrate-direct-only-envvar/output.txt +++ b/acceptance/bundle/migrate/auto/direct-only-envvar/output.txt @@ -9,45 +9,27 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Add a direct-only resource to the config >>> update_file.py databricks.yml #instance_pools instance_pools -=== Opt in via env var only: deploy migrates the state, ignoring the direct-only resource +=== Opt in via env var only: deploy migrates the state and creates the direct-only resource >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy -Warn: Direct engine selected via DATABRICKS_BUNDLE_ENGINE environment variable but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy. +Migrated 1 resource to direct deployment engine. Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created instance_pools.pool Files: 1 uploaded, 0 deleted -Resources: 0 created, 0 changed, 0 deleted, 2 unchanged -Migrating state to direct deployment engine (selected via DATABRICKS_BUNDLE_ENGINE environment variable)... -Migrated 1 resource to direct deployment engine. +Resources: 1 created, 0 changed, 0 deleted, 1 unchanged >>> print_migration_telemetry direct_migrated_via_env true -=== The direct-only resource is created on the next deploy +=== Redeploy is a no-op: state is already on the direct engine >>> [CLI] bundle plan -create instance_pools.pool - -Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Created instance_pools.pool Files: 0 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 1 unchanged +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //instance-pools/create -{ - "headers": { - "User-Agent": [ - "cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] interactive/none engine/direct auth/pat" - ] - }, - "method": "POST", - "path": "/api/2.0/instance-pools/create", - "body": { - "idle_instance_autotermination_minutes": 60, - "instance_pool_name": "My Pool", - "node_type_id": "Standard_DS3_v2" - } -} === No further actions planned >>> [CLI] bundle plan diff --git a/acceptance/bundle/migrate/auto-migrate-direct-only-envvar/script b/acceptance/bundle/migrate/auto/direct-only-envvar/script similarity index 79% rename from acceptance/bundle/migrate/auto-migrate-direct-only-envvar/script rename to acceptance/bundle/migrate/auto/direct-only-envvar/script index 702cf582a3a..d927be0e482 100644 --- a/acceptance/bundle/migrate/auto-migrate-direct-only-envvar/script +++ b/acceptance/bundle/migrate/auto/direct-only-envvar/script @@ -7,12 +7,12 @@ rm -f "$OUT_REQUESTS" title "Add a direct-only resource to the config" trace update_file.py databricks.yml "#instance_pools" "instance_pools" -title "Opt in via env var only: deploy migrates the state, ignoring the direct-only resource" +title "Opt in via env var only: deploy migrates the state and creates the direct-only resource" trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy trace print_migration_telemetry rm -f "$OUT_REQUESTS" -title "The direct-only resource is created on the next deploy" +title "Redeploy is a no-op: state is already on the direct engine" trace $CLI bundle plan trace $CLI bundle deploy trace print_requests.py //instance-pools/create diff --git a/acceptance/bundle/migrate/auto-migrate-direct-only-envvar/databricks.yml b/acceptance/bundle/migrate/auto/direct-only/databricks.yml similarity index 100% rename from acceptance/bundle/migrate/auto-migrate-direct-only-envvar/databricks.yml rename to acceptance/bundle/migrate/auto/direct-only/databricks.yml diff --git a/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml b/acceptance/bundle/migrate/auto/direct-only/out.test.toml similarity index 100% rename from acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml rename to acceptance/bundle/migrate/auto/direct-only/out.test.toml diff --git a/acceptance/bundle/migrate/auto-migrate-direct-only/output.txt b/acceptance/bundle/migrate/auto/direct-only/output.txt similarity index 59% rename from acceptance/bundle/migrate/auto-migrate-direct-only/output.txt rename to acceptance/bundle/migrate/auto/direct-only/output.txt index 513b0254a18..ebcf6ee326d 100644 --- a/acceptance/bundle/migrate/auto-migrate-direct-only/output.txt +++ b/acceptance/bundle/migrate/auto/direct-only/output.txt @@ -12,52 +12,45 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> update_file.py databricks.yml name: test-bundle name: test-bundle engine: direct -=== Plan skips the direct-only resource, since this run still uses terraform +=== Plan runs on the direct engine (state migrated in memory), so the direct-only resource is planned >>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged +create instance_pools.pool + +Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged >>> [CLI] bundle plan -o json -json.plan.resources.instance_pools.pool.action = "skip"; +json.plan.resources.instance_pools.pool.action = "create"; +json.plan.resources.instance_pools.pool.new_state.value.idle_instance_autotermination_minutes = 60; +json.plan.resources.instance_pools.pool.new_state.value.instance_pool_name = "My Pool"; +json.plan.resources.instance_pools.pool.new_state.value.node_type_id = "Standard_DS3_v2"; + +=== Plan does not commit state change, terraform stays +>>> find .databricks/bundle -name resources.json -type f -=== Deploy migrates the state, skipping the direct-only resource +>>> find .databricks/bundle -name terraform.tfstate* -type f +.databricks/bundle/default/terraform/terraform.tfstate + +=== Deploy commits the migration and creates the direct-only resource >>> [CLI] bundle deploy -Warn: Direct engine selected via bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11 but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy. +Migrated 1 resource to direct deployment engine. Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created instance_pools.pool Files: 1 uploaded, 0 deleted -Resources: 0 created, 0 changed, 0 deleted, 2 unchanged -Migrating state to direct deployment engine (selected via bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11)... -Migrated 1 resource to direct deployment engine. +Resources: 1 created, 0 changed, 0 deleted, 1 unchanged >>> print_migration_telemetry direct_migrated_via_config true -=== The direct-only resource is created on the next deploy +=== Redeploy is a no-op: state is already on the direct engine >>> [CLI] bundle plan -create instance_pools.pool - -Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Created instance_pools.pool Files: 0 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 1 unchanged +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //instance-pools/create -{ - "headers": { - "User-Agent": [ - "cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] interactive/none engine/direct auth/pat" - ] - }, - "method": "POST", - "path": "/api/2.0/instance-pools/create", - "body": { - "idle_instance_autotermination_minutes": 60, - "instance_pool_name": "My Pool", - "node_type_id": "Standard_DS3_v2" - } -} === No further actions planned >>> [CLI] bundle plan diff --git a/acceptance/bundle/migrate/auto-migrate-direct-only/script b/acceptance/bundle/migrate/auto/direct-only/script similarity index 65% rename from acceptance/bundle/migrate/auto-migrate-direct-only/script rename to acceptance/bundle/migrate/auto/direct-only/script index 0725c42d31c..0d9af1f5cbf 100644 --- a/acceptance/bundle/migrate/auto-migrate-direct-only/script +++ b/acceptance/bundle/migrate/auto/direct-only/script @@ -8,16 +8,19 @@ title "Add a direct-only resource and opt in to the direct engine via config" trace update_file.py databricks.yml "#instance_pools" "instance_pools" trace update_file.py databricks.yml "name: test-bundle" $'name: test-bundle\n engine: direct' -title "Plan skips the direct-only resource, since this run still uses terraform" +title "Plan runs on the direct engine (state migrated in memory), so the direct-only resource is planned" trace $CLI bundle plan trace $CLI bundle plan -o json | nostamp | gron.py | grep instance_pools +title "Plan does not commit state change, terraform stays" +trace find .databricks/bundle -name "resources.json" -type f +trace find .databricks/bundle -name "terraform.tfstate*" -type f -title "Deploy migrates the state, skipping the direct-only resource" +title "Deploy commits the migration and creates the direct-only resource" trace $CLI bundle deploy trace print_migration_telemetry rm -f "$OUT_REQUESTS" -title "The direct-only resource is created on the next deploy" +title "Redeploy is a no-op: state is already on the direct engine" trace $CLI bundle plan trace $CLI bundle deploy trace print_requests.py //instance-pools/create diff --git a/acceptance/bundle/migrate/auto/empty-plan-missing/databricks.yml b/acceptance/bundle/migrate/auto/empty-plan-missing/databricks.yml new file mode 100644 index 00000000000..576d7a9ef25 --- /dev/null +++ b/acceptance/bundle/migrate/auto/empty-plan-missing/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: test-bundle diff --git a/acceptance/bundle/migrate/auto-migrate-plan-failure/out.test.toml b/acceptance/bundle/migrate/auto/empty-plan-missing/out.test.toml similarity index 100% rename from acceptance/bundle/migrate/auto-migrate-plan-failure/out.test.toml rename to acceptance/bundle/migrate/auto/empty-plan-missing/out.test.toml diff --git a/acceptance/bundle/migrate/auto/empty-plan-missing/output.txt b/acceptance/bundle/migrate/auto/empty-plan-missing/output.txt new file mode 100644 index 00000000000..eb6ee6315e8 --- /dev/null +++ b/acceptance/bundle/migrate/auto/empty-plan-missing/output.txt @@ -0,0 +1,15 @@ + +=== deploy --plan with a missing plan file fails after the empty migration prepared the direct state +>>> DATABRICKS_BUNDLE_ENGINE=direct errcode [CLI] bundle deploy --plan does-not-exist.json +Migrated 0 resources to direct deployment engine. +Error: reading plan file: open does-not-exist.json: no such file or directory + +Warn: Migration not committed, keeping Terraform state + +Exit code: 1 + +=== Terraform state is untouched - the empty migration was discarded, not committed or swept +>>> find .databricks/bundle -name resources.json -type f + +>>> find .databricks/bundle -name terraform.tfstate* -type f +.databricks/bundle/default/terraform/terraform.tfstate diff --git a/acceptance/bundle/migrate/auto/empty-plan-missing/script b/acceptance/bundle/migrate/auto/empty-plan-missing/script new file mode 100644 index 00000000000..46daffcbe4e --- /dev/null +++ b/acceptance/bundle/migrate/auto/empty-plan-missing/script @@ -0,0 +1,18 @@ +export DATABRICKS_BUNDLE_ENGINE= + +# Seed an empty terraform state (no databricks_* resources) as the existing state. +mkdir -p .databricks/bundle/default/terraform +cat > .databricks/bundle/default/terraform/terraform.tfstate <<'EOF' +{"version": 4, "serial": 1, "lineage": "test-lineage", "resources": []} +EOF + +title "deploy --plan with a missing plan file fails after the empty migration prepared the direct state" +# The empty-state migration is deferred like any other: it prepares the local direct state but does +# not back up the terraform state until the command commits. A missing --plan file fails +# post-migration and unapproved, so the run discards the prepared migration and stays on terraform - +# proving the empty sweep is gated by commit, not done unconditionally up front. +trace DATABRICKS_BUNDLE_ENGINE=direct errcode $CLI bundle deploy --plan does-not-exist.json + +title "Terraform state is untouched - the empty migration was discarded, not committed or swept" +trace find .databricks/bundle -name "resources.json" -type f +trace find .databricks/bundle -name "terraform.tfstate*" -type f diff --git a/acceptance/bundle/migrate/auto/empty-plan-missing/test.toml b/acceptance/bundle/migrate/auto/empty-plan-missing/test.toml new file mode 100644 index 00000000000..455a6a5c9bf --- /dev/null +++ b/acceptance/bundle/migrate/auto/empty-plan-missing/test.toml @@ -0,0 +1,5 @@ +# Normalize the Windows os.Open error so this migration-discard test runs on all platforms +# (the discard file operations it exercises are worth covering on Windows too). +[[Repls]] +Old = "open does-not-exist.json: The system cannot find the file specified." +New = "open does-not-exist.json: no such file or directory" diff --git a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/databricks.yml b/acceptance/bundle/migrate/auto/empty-tfstate/databricks.yml similarity index 100% rename from acceptance/bundle/migrate/auto-migrate-empty-tfstate/databricks.yml rename to acceptance/bundle/migrate/auto/empty-tfstate/databricks.yml diff --git a/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml b/acceptance/bundle/migrate/auto/empty-tfstate/out.test.toml similarity index 100% rename from acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml rename to acceptance/bundle/migrate/auto/empty-tfstate/out.test.toml diff --git a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt b/acceptance/bundle/migrate/auto/empty-tfstate/output.txt similarity index 64% rename from acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt rename to acceptance/bundle/migrate/auto/empty-tfstate/output.txt index 06e544ad5fc..072e48d97da 100644 --- a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt +++ b/acceptance/bundle/migrate/auto/empty-tfstate/output.txt @@ -1,25 +1,25 @@ -=== Deploy with bundle.engine: direct and an empty terraform state — sweep, don't migrate +=== Deploy with bundle.engine: direct and an empty terraform state — migrate (0 resources) >>> [CLI] bundle deploy -Warn: Direct engine selected via bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11 but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy. +Migrated 0 resources to direct deployment engine. Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Files: 2 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 0 unchanged -Removing empty terraform state; direct engine will be used on the next deploy (selected via bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11)... -=== Terraform state is renamed to .backup; no resources.json (empty state, nothing to persist) +=== Empty state migrated: resources.json written, local terraform state backed up >>> find .databricks/bundle -name resources.json -type f +.databricks/bundle/default/resources.json >>> find .databricks/bundle -name terraform.tfstate* -type f .databricks/bundle/default/terraform/terraform.tfstate.backup -=== Sweep is recorded in telemetry as via-config +=== Migration is recorded in telemetry as via-config >>> print_migration_telemetry direct_migrated_via_config true -=== Sweep also affected the workspace: .backup uploaded, terraform.tfstate deleted +=== The remote had no terraform state, so nothing terraform-related is backed up remotely >>> print_requests.py //workspace-files/import-file //workspace/delete --sort "POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" @@ -28,18 +28,17 @@ direct_migrated_via_config true "POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" "POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json" "POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate.backup" -"POST /api/2.0/workspace/delete" +"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json" "POST /api/2.0/workspace/delete" "POST /api/2.0/workspace/delete" -=== debug states confirms no terraform.tfstate anywhere; only .backup remains +=== debug states confirms no terraform.tfstate anywhere; only the local .backup remains >>> [CLI] bundle debug states --force-pull +resources.json: remote direct state serial=2 lineage="test-lineage" +[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=2 lineage="test-lineage" - -=== Next deploy uses direct engine by default (no more mismatch warning) +=== Next deploy uses direct engine by default (no more mismatch warning, no re-migration) >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Files: 0 uploaded, 0 deleted diff --git a/acceptance/bundle/migrate/auto/empty-tfstate/script b/acceptance/bundle/migrate/auto/empty-tfstate/script new file mode 100644 index 00000000000..cec11c62a22 --- /dev/null +++ b/acceptance/bundle/migrate/auto/empty-tfstate/script @@ -0,0 +1,32 @@ +# A terraform.tfstate with no databricks_* resources goes through the same migration path as a +# populated one: the converter writes an empty direct base state, the approved deploy commits it, +# and the superseded terraform state is backed up. (There is no special "sweep" branch - empty and +# populated states share one code path.) The empty terraform state here exists only locally (never +# pushed), so nothing terraform-related is backed up remotely; the populated tests cover that. + +mkdir -p .databricks/bundle/default/terraform +cat > .databricks/bundle/default/terraform/terraform.tfstate <<'EOF' +{"version": 4, "serial": 1, "lineage": "test-lineage", "resources": []} +EOF + +title "Deploy with bundle.engine: direct and an empty terraform state — migrate (0 resources)" +trace $CLI bundle deploy 2>&1 | contains.py 'Migrated 0 resources to direct deployment engine.' + +title "Empty state migrated: resources.json written, local terraform state backed up\n" +trace find .databricks/bundle -name "resources.json" -type f +trace find .databricks/bundle -name "terraform.tfstate*" -type f + +title "Migration is recorded in telemetry as via-config\n" +trace print_migration_telemetry + +title "The remote had no terraform state, so nothing terraform-related is backed up remotely\n" +trace print_requests.py //workspace-files/import-file //workspace/delete --sort | jq '.method + " " + .path' | contains.py '!terraform.tfstate' + +title "debug states confirms no terraform.tfstate anywhere; only the local .backup remains\n" +trace $CLI bundle debug states --force-pull + +title "Next deploy uses direct engine by default (no more mismatch warning, no re-migration)" +trace $CLI bundle deploy 2>&1 | contains.py '!does not match the existing state' +trace print_migration_telemetry + +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/migrate/auto-migrate-envvar/databricks.yml b/acceptance/bundle/migrate/auto/envvar/databricks.yml similarity index 100% rename from acceptance/bundle/migrate/auto-migrate-envvar/databricks.yml rename to acceptance/bundle/migrate/auto/envvar/databricks.yml diff --git a/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml b/acceptance/bundle/migrate/auto/envvar/out.test.toml similarity index 100% rename from acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml rename to acceptance/bundle/migrate/auto/envvar/out.test.toml diff --git a/acceptance/bundle/migrate/auto-migrate-envvar/output.txt b/acceptance/bundle/migrate/auto/envvar/output.txt similarity index 81% rename from acceptance/bundle/migrate/auto-migrate-envvar/output.txt rename to acceptance/bundle/migrate/auto/envvar/output.txt index a0f04340420..db6abb08814 100644 --- a/acceptance/bundle/migrate/auto-migrate-envvar/output.txt +++ b/acceptance/bundle/migrate/auto/envvar/output.txt @@ -12,16 +12,17 @@ direct_drymigrate_warnings false === Opt in via env var only (no bundle.engine in config): auto-migration triggers >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy -Warn: Direct engine selected via DATABRICKS_BUNDLE_ENGINE environment variable but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy. +Migrated 1 resource to direct deployment engine. Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Files: 0 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged -Migrating state to direct deployment engine (selected via DATABRICKS_BUNDLE_ENGINE environment variable)... -Migrated 1 resource to direct deployment engine. >>> print_migration_telemetry direct_migrated_via_env true +>>> print_requests.py +engine/direct + === Direct-engine state is now in place; terraform state is backed up >>> find .databricks/bundle -name resources.json -type f diff --git a/acceptance/bundle/migrate/auto-migrate-envvar/script b/acceptance/bundle/migrate/auto/envvar/script similarity index 70% rename from acceptance/bundle/migrate/auto-migrate-envvar/script rename to acceptance/bundle/migrate/auto/envvar/script index 8377687a9b6..897fbb32b23 100644 --- a/acceptance/bundle/migrate/auto-migrate-envvar/script +++ b/acceptance/bundle/migrate/auto/envvar/script @@ -8,6 +8,10 @@ rm -f "$OUT_REQUESTS" title "Opt in via env var only (no bundle.engine in config): auto-migration triggers" trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy trace print_migration_telemetry +# The migrated deploy runs on the direct engine, so its requests carry engine/direct only. +# A stale engine/terraform here would mean the user-agent engine tag was appended rather +# than resolved once (the SDK user agent cannot replace an existing dimension). +trace print_requests.py | grep -o 'engine/[a-z]*' | sort -u rm -f "$OUT_REQUESTS" title "Direct-engine state is now in place; terraform state is backed up\n" diff --git a/acceptance/bundle/migrate/auto/pipeline-storage-change/databricks.yml.tmpl b/acceptance/bundle/migrate/auto/pipeline-storage-change/databricks.yml.tmpl new file mode 100644 index 00000000000..cf02ed35516 --- /dev/null +++ b/acceptance/bundle/migrate/auto/pipeline-storage-change/databricks.yml.tmpl @@ -0,0 +1,15 @@ +bundle: + name: uc-pipeline-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + pipelines: + my_pipeline: + name: pipeline-$UNIQUE_NAME + storage: dbfs:/pipelines/$UNIQUE_NAME-init + libraries: + - notebook: + path: ./nb.sql + development: true diff --git a/acceptance/bundle/migrate/auto/pipeline-storage-change/nb.sql b/acceptance/bundle/migrate/auto/pipeline-storage-change/nb.sql new file mode 100644 index 00000000000..199ff507884 --- /dev/null +++ b/acceptance/bundle/migrate/auto/pipeline-storage-change/nb.sql @@ -0,0 +1,2 @@ +-- Databricks notebook source +select 1 diff --git a/acceptance/bundle/migrate/auto/pipeline-storage-change/out.test.toml b/acceptance/bundle/migrate/auto/pipeline-storage-change/out.test.toml new file mode 100644 index 00000000000..d73c45e3119 --- /dev/null +++ b/acceptance/bundle/migrate/auto/pipeline-storage-change/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/auto/pipeline-storage-change/output.txt b/acceptance/bundle/migrate/auto/pipeline-storage-change/output.txt new file mode 100644 index 00000000000..05daf0e43b4 --- /dev/null +++ b/acceptance/bundle/migrate/auto/pipeline-storage-change/output.txt @@ -0,0 +1,65 @@ + +=== Deploy the pipeline on terraform +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Created pipelines.my_pipeline +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Change immutable storage. Without --auto-approve the recreate is refused +>>> DATABRICKS_BUNDLE_ENGINE=direct errcode [CLI] bundle deploy +Migrated 1 resource to direct deployment engine. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... + +This action will result in the deletion or recreation of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the pipelines will +restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline +properties such as the 'catalog' or 'storage' are changed: + recreate resources.pipelines.my_pipeline +Warn: Migration not committed, keeping Terraform state +Error: the deployment requires destructive actions, but the current console does not support prompting. +Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed. +To proceed, use --auto-approve after reviewing the plan above. + +Files: 1 uploaded, 0 deleted + +Exit code: 1 + +>>> print_migration_telemetry +direct_migrate_recreate_planned true + +>>> find .databricks/bundle -name resources.json -type f + +>>> find .databricks/bundle -name *tfstate* -type f +.databricks/bundle/default/terraform/terraform.tfstate + +=== With --auto-approve the recreate is applied on direct +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy --auto-approve +Migrated 1 resource to direct deployment engine. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... + +This action will result in the deletion or recreation of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the pipelines will +restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline +properties such as the 'catalog' or 'storage' are changed: + recreate resources.pipelines.my_pipeline +Recreated pipelines.my_pipeline +Files: 0 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 0 unchanged + +=== Local state is now direct +>>> find .databricks/bundle -name resources.json -type f +.databricks/bundle/default/resources.json + +=== Cleanup +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.pipelines.my_pipeline + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: + delete resources.pipelines.my_pipeline + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 1 deleted diff --git a/acceptance/bundle/migrate/auto/pipeline-storage-change/script b/acceptance/bundle/migrate/auto/pipeline-storage-change/script new file mode 100644 index 00000000000..cdd8059f01e --- /dev/null +++ b/acceptance/bundle/migrate/auto/pipeline-storage-change/script @@ -0,0 +1,35 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + title "Cleanup" + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "Deploy the pipeline on terraform" +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy +rm -f "$OUT_REQUESTS" + +title "Change immutable storage. Without --auto-approve the recreate is refused" +# storage is a recreate_on_changes field on pipelines (not an id field). The migration's plan +# compares the new config value against the deployed remote and plans a recreate. Pipelines are +# in the destructive-approval group, so without --auto-approve the deploy refuses it (exit 1) +# and the migration is not committed. +update_file.py databricks.yml "$UNIQUE_NAME-init" "$UNIQUE_NAME-changed" +trace DATABRICKS_BUNDLE_ENGINE=direct errcode $CLI bundle deploy +trace print_migration_telemetry +trace find .databricks/bundle -name "resources.json" -type f +trace find .databricks/bundle -name "*tfstate*" -type f + +title "With --auto-approve the recreate is applied on direct" +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy --auto-approve + +# No clean-redeploy assertion here: on a real workspace a direct-engine pipeline recreate +# leaves a field the next plan re-updates ("1 changed"), while the testserver converges +# ("1 unchanged"). That post-recreate convergence is a direct-engine pipeline concern +# orthogonal to migration, and a single golden can't match both engines, so the test stops +# after applying the recreate. The schema / registered-model recreate tests cover the +# clean-redeploy path on cloud. + +title "Local state is now direct" +trace find .databricks/bundle -name "resources.json" -type f | sort diff --git a/acceptance/bundle/migrate/auto/pipeline-storage-change/test.toml b/acceptance/bundle/migrate/auto/pipeline-storage-change/test.toml new file mode 100644 index 00000000000..b674c6425f9 --- /dev/null +++ b/acceptance/bundle/migrate/auto/pipeline-storage-change/test.toml @@ -0,0 +1,7 @@ +Cloud = true +RecordRequests = true + +Ignore = [ + ".databricks", + "databricks.yml", +] diff --git a/acceptance/bundle/migrate/auto-migrate-plan-failure/databricks.yml b/acceptance/bundle/migrate/auto/plan-failure/databricks.yml similarity index 100% rename from acceptance/bundle/migrate/auto-migrate-plan-failure/databricks.yml rename to acceptance/bundle/migrate/auto/plan-failure/databricks.yml diff --git a/acceptance/bundle/migrate/basic/out.test.toml b/acceptance/bundle/migrate/auto/plan-failure/out.test.toml similarity index 100% rename from acceptance/bundle/migrate/basic/out.test.toml rename to acceptance/bundle/migrate/auto/plan-failure/out.test.toml diff --git a/acceptance/bundle/migrate/auto-migrate-push-failure/output.txt b/acceptance/bundle/migrate/auto/plan-failure/output.txt similarity index 59% rename from acceptance/bundle/migrate/auto-migrate-push-failure/output.txt rename to acceptance/bundle/migrate/auto/plan-failure/output.txt index cf125b4bd74..4cde68226d9 100644 --- a/acceptance/bundle/migrate/auto-migrate-push-failure/output.txt +++ b/acceptance/bundle/migrate/auto/plan-failure/output.txt @@ -10,39 +10,37 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged direct_drymigrate_success true direct_drymigrate_warnings false -=== Opt in via env var; inject 403 on the resources.json push so commit fails +=== Inject GET failure so the plan check blocks the migration >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy -Warn: Direct engine selected via DATABRICKS_BUNDLE_ENGINE environment variable but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy. +Warn: migration to direct: cannot plan resources.jobs.test_job: reading id="[NUMID]": Fault injected by test. (403 INJECTED): Endpoint: GET [DATABRICKS_URL]/api/2.2/jobs/get?job_id=[NUMID] +HTTP Status: 403 Forbidden +API error_code: INJECTED +API message: Fault injected by test. +Warn: migration to the direct engine failed its plan check; deploying on terraform this time: planning failed Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Files: 0 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged -Migrating state to direct deployment engine (selected via DATABRICKS_BUNDLE_ENGINE environment variable)... -Warn: automatic migration to direct engine failed: pushing direct state to workspace: access denied: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json >>> print_migration_telemetry -direct_migrate_commit_error true +direct_migrate_plan_error true === Local state was NOT rewritten (still terraform) - >>> find .databricks/bundle -name resources.json -type f >>> find .databricks/bundle -name terraform.tfstate* -type f .databricks/bundle/default/terraform/terraform.tfstate -=== Retry: auto-migration succeeds this time +=== Retry: plan check passes this time, migration succeeds >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy -Warn: Direct engine selected via DATABRICKS_BUNDLE_ENGINE environment variable but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy. +Migrated 1 resource to direct deployment engine. Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Files: 0 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged -Migrating state to direct deployment engine (selected via DATABRICKS_BUNDLE_ENGINE environment variable)... -Migrated 1 resource to direct deployment engine. >>> print_migration_telemetry direct_migrated_via_env true === Local state is now direct - >>> find .databricks/bundle -name resources.json -type f .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/migrate/auto-migrate-plan-failure/script b/acceptance/bundle/migrate/auto/plan-failure/script similarity index 82% rename from acceptance/bundle/migrate/auto-migrate-plan-failure/script rename to acceptance/bundle/migrate/auto/plan-failure/script index 0e2e2ead7ed..b8f2344220f 100644 --- a/acceptance/bundle/migrate/auto-migrate-plan-failure/script +++ b/acceptance/bundle/migrate/auto/plan-failure/script @@ -9,9 +9,10 @@ title "Inject GET failure so the plan check blocks the migration" # The plan check calls GET /api/2.2/jobs/get for each resource to compare # local config against remote state. A 403 (non-retryable) causes the plan # to fail → the migration is abandoned and the bundle stays on terraform. -# OFFSET=1: terraform reads the job once during its plan phase; skip that -# call and fail the 2nd one, which is the plan check's DoRead. -fault.py "GET /api/2.2/jobs/get*" 403 1 1 +# The migration runs before any terraform plan, so the plan check's DoRead is +# the first GET here; OFFSET=0 fails it, and TIMES=1 lets terraform's later +# read through so the fallback deploy succeeds. +fault.py "GET /api/2.2/jobs/get*" 403 0 1 trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy trace print_migration_telemetry rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/migrate/auto/plan-missing/databricks.yml b/acceptance/bundle/migrate/auto/plan-missing/databricks.yml new file mode 100644 index 00000000000..7d47b2d4957 --- /dev/null +++ b/acceptance/bundle/migrate/auto/plan-missing/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-bundle + +resources: + jobs: + test_job: + name: "Test Auto-Migrate Plan Missing" diff --git a/acceptance/bundle/migrate/dashboards/out.test.toml b/acceptance/bundle/migrate/auto/plan-missing/out.test.toml similarity index 100% rename from acceptance/bundle/migrate/dashboards/out.test.toml rename to acceptance/bundle/migrate/auto/plan-missing/out.test.toml diff --git a/acceptance/bundle/migrate/auto/plan-missing/output.txt b/acceptance/bundle/migrate/auto/plan-missing/output.txt new file mode 100644 index 00000000000..f2967962d5b --- /dev/null +++ b/acceptance/bundle/migrate/auto/plan-missing/output.txt @@ -0,0 +1,28 @@ + +=== Initial deploy on terraform +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created jobs.test_job +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== deploy --plan with a missing plan file fails after the migration prepared the direct state +>>> DATABRICKS_BUNDLE_ENGINE=direct errcode [CLI] bundle deploy --plan does-not-exist.json +Migrated 1 resource to direct deployment engine. +Error: reading plan file: open does-not-exist.json: no such file or directory + +Warn: Migration not committed, keeping Terraform state + +Exit code: 1 + +=== State is still terraform - the migration was discarded, not committed +>>> find .databricks/bundle -name resources.json -type f + +>>> find .databricks/bundle -name terraform.tfstate* -type f +.databricks/bundle/default/terraform/terraform.tfstate + +=== A plain deploy still runs on terraform (the engine was not silently switched) +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged diff --git a/acceptance/bundle/migrate/auto/plan-missing/script b/acceptance/bundle/migrate/auto/plan-missing/script new file mode 100644 index 00000000000..991d1e74c36 --- /dev/null +++ b/acceptance/bundle/migrate/auto/plan-missing/script @@ -0,0 +1,17 @@ +export DATABRICKS_BUNDLE_ENGINE= + +title "Initial deploy on terraform" +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy + +title "deploy --plan with a missing plan file fails after the migration prepared the direct state" +# The migration runs (writing the local direct state) before the saved plan is loaded, so a +# missing plan file fails post-migration and unapproved. A failed, unapproved command must +# change nothing: the run discards the prepared migration and stays on terraform. +trace DATABRICKS_BUNDLE_ENGINE=direct errcode $CLI bundle deploy --plan does-not-exist.json + +title "State is still terraform - the migration was discarded, not committed" +trace find .databricks/bundle -name "resources.json" -type f +trace find .databricks/bundle -name "terraform.tfstate*" -type f + +title "A plain deploy still runs on terraform (the engine was not silently switched)" +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy diff --git a/acceptance/bundle/migrate/auto/plan-missing/test.toml b/acceptance/bundle/migrate/auto/plan-missing/test.toml new file mode 100644 index 00000000000..455a6a5c9bf --- /dev/null +++ b/acceptance/bundle/migrate/auto/plan-missing/test.toml @@ -0,0 +1,5 @@ +# Normalize the Windows os.Open error so this migration-discard test runs on all platforms +# (the discard file operations it exercises are worth covering on Windows too). +[[Repls]] +Old = "open does-not-exist.json: The system cannot find the file specified." +New = "open does-not-exist.json: no such file or directory" diff --git a/acceptance/bundle/migrate/auto-migrate-push-failure/databricks.yml b/acceptance/bundle/migrate/auto/push-failure/databricks.yml similarity index 100% rename from acceptance/bundle/migrate/auto-migrate-push-failure/databricks.yml rename to acceptance/bundle/migrate/auto/push-failure/databricks.yml diff --git a/acceptance/bundle/migrate/default-python/out.test.toml b/acceptance/bundle/migrate/auto/push-failure/out.test.toml similarity index 100% rename from acceptance/bundle/migrate/default-python/out.test.toml rename to acceptance/bundle/migrate/auto/push-failure/out.test.toml diff --git a/acceptance/bundle/migrate/auto/push-failure/output.txt b/acceptance/bundle/migrate/auto/push-failure/output.txt new file mode 100644 index 00000000000..231075d3522 --- /dev/null +++ b/acceptance/bundle/migrate/auto/push-failure/output.txt @@ -0,0 +1,52 @@ + +=== Initial deploy on terraform +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created jobs.test_job +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_migration_telemetry +direct_drymigrate_success true +direct_drymigrate_warnings false + +=== Opt in via env var; inject 403 on the resources.json push so the deploy fails +>>> DATABRICKS_BUNDLE_ENGINE=direct errcode [CLI] bundle deploy +Migrated 1 resource to direct deployment engine. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Error: access denied: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json (403 INJECTED) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Fstate%2Fresources.json?overwrite=true +HTTP Status: 403 Forbidden +API error_code: INJECTED +API message: Fault injected by test. + +Files: 0 uploaded, 0 deleted + +Exit code: 1 + +>>> print_migration_telemetry + +=== Local state is direct (the migration wrote it) though the remote push did not land + +>>> find .databricks/bundle -name resources.json -type f +.databricks/bundle/default/resources.json + +>>> find .databricks/bundle -name terraform.tfstate* -type f +.databricks/bundle/default/terraform/terraform.tfstate + +=== Retry: the local direct state is used (no re-migration) and the push completes +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +>>> print_migration_telemetry + +=== Retry recovered: local state is direct (the outranked local terraform.tfstate lingers) + +>>> find .databricks/bundle -name resources.json -type f +.databricks/bundle/default/resources.json + +>>> find .databricks/bundle -name terraform.tfstate* -type f +.databricks/bundle/default/terraform/terraform.tfstate diff --git a/acceptance/bundle/migrate/auto-migrate-push-failure/script b/acceptance/bundle/migrate/auto/push-failure/script similarity index 51% rename from acceptance/bundle/migrate/auto-migrate-push-failure/script rename to acceptance/bundle/migrate/auto/push-failure/script index 283398867c7..0e1e52efd37 100644 --- a/acceptance/bundle/migrate/auto-migrate-push-failure/script +++ b/acceptance/bundle/migrate/auto/push-failure/script @@ -5,25 +5,27 @@ trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy trace print_migration_telemetry rm -f "$OUT_REQUESTS" -title "Opt in via env var; inject 403 on the resources.json push so commit fails" -# The auto-migration's commit path uploads resources.json to the workspace -# before renaming any local files. A 403 (non-retryable) fails commitMigration → -# the deploy prints a warning and leaves the local state as terraform. +title "Opt in via env var; inject 403 on the resources.json push so the deploy fails" +# The migration prepares the direct state in memory and in the local state file; the approved +# deploy (deployCore) pushes resources.json to the workspace. A 403 (non-retryable) on that +# push fails the deploy - unlike a declined deploy, the migration is not discarded, so the +# local direct state remains and a retry finishes the push. (A push failure is a hard error on +# any direct deploy, not just a migrating one.) fault.py "POST /api/2.0/workspace-files/import-file/Workspace/Users/$CURRENT_USER_NAME/.bundle/test-bundle/default/state/resources.json" 403 0 1 -trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy +trace DATABRICKS_BUNDLE_ENGINE=direct errcode $CLI bundle deploy trace print_migration_telemetry rm -f "$OUT_REQUESTS" -title "Local state was NOT rewritten (still terraform)\n" +title "Local state is direct (the migration wrote it) though the remote push did not land\n" trace find .databricks/bundle -name "resources.json" -type f trace find .databricks/bundle -name "terraform.tfstate*" -type f -title "Retry: auto-migration succeeds this time" +title "Retry: the local direct state is used (no re-migration) and the push completes" trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy trace print_migration_telemetry rm -f "$OUT_REQUESTS" -title "Local state is now direct\n" +title "Retry recovered: local state is direct (the outranked local terraform.tfstate lingers)\n" trace find .databricks/bundle -name "resources.json" -type f trace find .databricks/bundle -name "terraform.tfstate*" -type f diff --git a/acceptance/bundle/migrate/auto/registered-model-catalog-change/databricks.yml.tmpl b/acceptance/bundle/migrate/auto/registered-model-catalog-change/databricks.yml.tmpl new file mode 100644 index 00000000000..cd4a1173826 --- /dev/null +++ b/acceptance/bundle/migrate/auto/registered-model-catalog-change/databricks.yml.tmpl @@ -0,0 +1,12 @@ +bundle: + name: uc-modelcat-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + registered_models: + my_model: + name: mymodel-$UNIQUE_NAME + catalog_name: main + schema_name: default diff --git a/acceptance/bundle/migrate/auto/registered-model-catalog-change/out.test.toml b/acceptance/bundle/migrate/auto/registered-model-catalog-change/out.test.toml new file mode 100644 index 00000000000..d73c45e3119 --- /dev/null +++ b/acceptance/bundle/migrate/auto/registered-model-catalog-change/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/auto/registered-model-catalog-change/output.txt b/acceptance/bundle/migrate/auto/registered-model-catalog-change/output.txt new file mode 100644 index 00000000000..ed244e9ea1a --- /dev/null +++ b/acceptance/bundle/migrate/auto/registered-model-catalog-change/output.txt @@ -0,0 +1,51 @@ + +>>> [CLI] catalogs create othercat_[UNIQUE_NAME] -o json +{ + "name": "othercat_[UNIQUE_NAME]" +} + +=== Deploy the model on terraform in catalog 'main' +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Created registered_models.my_model +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Move the model to another catalog and deploy on direct +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy +Warn: migration to direct: registered_models.my_model: catalog_name differs between config ("othercat_[UNIQUE_NAME]") and terraform state ("main"); this resource will be recreated. +Migrated 1 resource to direct deployment engine. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Recreated registered_models.my_model +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 0 unchanged + +>>> print_migration_telemetry +direct_migrate_recreate_planned true +direct_migrated_via_env true + +=== Redeploy on direct: the model now matches config, so the plan is a clean no-op +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +=== The model exists in the new catalog and the local state is direct +>>> [CLI] registered-models get othercat_[UNIQUE_NAME].default.mymodel-[UNIQUE_NAME] +{ + "full_name": "othercat_[UNIQUE_NAME].default.mymodel-[UNIQUE_NAME]" +} + +>>> find .databricks/bundle -name resources.json -type f +.databricks/bundle/default/resources.json + +=== Cleanup +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.registered_models.my_model + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 1 deleted + +>>> errcode [CLI] catalogs delete othercat_[UNIQUE_NAME] --force diff --git a/acceptance/bundle/migrate/auto/registered-model-catalog-change/script b/acceptance/bundle/migrate/auto/registered-model-catalog-change/script new file mode 100644 index 00000000000..fa0d8912317 --- /dev/null +++ b/acceptance/bundle/migrate/auto/registered-model-catalog-change/script @@ -0,0 +1,34 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +# catalog_name is an immutable provided id field on registered models; moving the model to +# another catalog recreates it. Create the destination catalog out of band (it auto-provisions +# a "default" schema, which the moved model reuses). +TARGET_CATALOG="othercat_$UNIQUE_NAME" +trace $CLI catalogs create "$TARGET_CATALOG" -o json | jq '{name: (.name // .full_name)}' + +cleanup() { + title "Cleanup" + trace $CLI bundle destroy --auto-approve + trace errcode $CLI catalogs delete "$TARGET_CATALOG" --force +} +trap cleanup EXIT + +title "Deploy the model on terraform in catalog 'main'" +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy +rm -f "$OUT_REQUESTS" + +title "Move the model to another catalog and deploy on direct" +# reconcileIDFields records the deployed catalog, so the migration's plan sees a real change +# and the model recreates. Registered models are not in the destructive-approval group, so the +# deploy is approved without --auto-approve: the migration commits and the recreate moves the +# model to the new catalog. +update_file.py databricks.yml "catalog_name: main" "catalog_name: $TARGET_CATALOG" +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy +trace print_migration_telemetry + +title "Redeploy on direct: the model now matches config, so the plan is a clean no-op" +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy + +title "The model exists in the new catalog and the local state is direct" +trace $CLI registered-models get "$TARGET_CATALOG.default.mymodel-$UNIQUE_NAME" | jq "{full_name}" +trace find .databricks/bundle -name "resources.json" -type f | sort diff --git a/acceptance/bundle/migrate/auto/registered-model-catalog-change/test.toml b/acceptance/bundle/migrate/auto/registered-model-catalog-change/test.toml new file mode 100644 index 00000000000..b674c6425f9 --- /dev/null +++ b/acceptance/bundle/migrate/auto/registered-model-catalog-change/test.toml @@ -0,0 +1,7 @@ +Cloud = true +RecordRequests = true + +Ignore = [ + ".databricks", + "databricks.yml", +] diff --git a/acceptance/bundle/migrate/auto/registered-model-rename/databricks.yml.tmpl b/acceptance/bundle/migrate/auto/registered-model-rename/databricks.yml.tmpl new file mode 100644 index 00000000000..45ced1357d5 --- /dev/null +++ b/acceptance/bundle/migrate/auto/registered-model-rename/databricks.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: uc-migrmodel-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + schemas: + parent: + name: mschema-$UNIQUE_NAME + catalog_name: main + registered_models: + my_model: + name: themodel-$UNIQUE_NAME + catalog_name: main + schema_name: mschema-$UNIQUE_NAME diff --git a/acceptance/bundle/migrate/auto/registered-model-rename/out.test.toml b/acceptance/bundle/migrate/auto/registered-model-rename/out.test.toml new file mode 100644 index 00000000000..d73c45e3119 --- /dev/null +++ b/acceptance/bundle/migrate/auto/registered-model-rename/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/auto/registered-model-rename/output.txt b/acceptance/bundle/migrate/auto/registered-model-rename/output.txt new file mode 100644 index 00000000000..9bf7da02fb2 --- /dev/null +++ b/acceptance/bundle/migrate/auto/registered-model-rename/output.txt @@ -0,0 +1,49 @@ + +=== Deploy the schema and registered model on terraform +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Created registered_models.my_model +Created schemas.parent +Files: 4 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the model (an immutable id field) and deploy on direct +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy +Warn: migration to direct: registered_models.my_model: name differs between config ("themodel-[UNIQUE_NAME]-renamed") and terraform state ("themodel-[UNIQUE_NAME]"); this resource will be recreated. +Migrated 2 resources to direct deployment engine. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Recreated registered_models.my_model +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_migration_telemetry +direct_migrate_recreate_planned true +direct_migrated_via_env true + +=== Redeploy on direct: the model now matches config, so the plan is a clean no-op +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +=== The model exists under the new name and the local state is direct +>>> [CLI] registered-models get main.mschema-[UNIQUE_NAME].themodel-[UNIQUE_NAME]-renamed +{ + "full_name": "main.mschema-[UNIQUE_NAME].themodel-[UNIQUE_NAME]-renamed" +} + +>>> find .databricks/bundle -name resources.json -type f +.databricks/bundle/default/resources.json + +=== Cleanup +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.registered_models.my_model + delete resources.schemas.parent + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.parent + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 2 deleted diff --git a/acceptance/bundle/migrate/auto/registered-model-rename/script b/acceptance/bundle/migrate/auto/registered-model-rename/script new file mode 100644 index 00000000000..b4c809ba96a --- /dev/null +++ b/acceptance/bundle/migrate/auto/registered-model-rename/script @@ -0,0 +1,28 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + title "Cleanup" + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "Deploy the schema and registered model on terraform" +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy +rm -f "$OUT_REQUESTS" + +title "Rename the model (an immutable id field) and deploy on direct" +# name is a provided id field on registered models, so a rename recreates. reconcileIDFields +# seeds the deployed name into the converted state, so the migration's plan sees a real +# recreate. Unlike schemas/volumes, registered models are not in the destructive-approval +# group (a model recreate loses no data), so the deploy is approved without --auto-approve: +# the migration commits and the recreate applies on direct. +update_file.py databricks.yml "themodel-$UNIQUE_NAME" "themodel-$UNIQUE_NAME-renamed" +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy +trace print_migration_telemetry + +title "Redeploy on direct: the model now matches config, so the plan is a clean no-op" +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy + +title "The model exists under the new name and the local state is direct" +trace $CLI registered-models get "main.mschema-$UNIQUE_NAME.themodel-$UNIQUE_NAME-renamed" | jq "{full_name}" +trace find .databricks/bundle -name "resources.json" -type f | sort diff --git a/acceptance/bundle/migrate/auto/registered-model-rename/test.toml b/acceptance/bundle/migrate/auto/registered-model-rename/test.toml new file mode 100644 index 00000000000..b674c6425f9 --- /dev/null +++ b/acceptance/bundle/migrate/auto/registered-model-rename/test.toml @@ -0,0 +1,7 @@ +Cloud = true +RecordRequests = true + +Ignore = [ + ".databricks", + "databricks.yml", +] diff --git a/acceptance/bundle/migrate/auto/schema-catalog-change/databricks.yml.tmpl b/acceptance/bundle/migrate/auto/schema-catalog-change/databricks.yml.tmpl new file mode 100644 index 00000000000..53cf960a034 --- /dev/null +++ b/acceptance/bundle/migrate/auto/schema-catalog-change/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: uc-schemacat-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + schemas: + my_schema: + name: myschema-$UNIQUE_NAME + catalog_name: main diff --git a/acceptance/bundle/migrate/auto/schema-catalog-change/out.test.toml b/acceptance/bundle/migrate/auto/schema-catalog-change/out.test.toml new file mode 100644 index 00000000000..d73c45e3119 --- /dev/null +++ b/acceptance/bundle/migrate/auto/schema-catalog-change/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/auto/schema-catalog-change/output.txt b/acceptance/bundle/migrate/auto/schema-catalog-change/output.txt new file mode 100644 index 00000000000..6282214d96b --- /dev/null +++ b/acceptance/bundle/migrate/auto/schema-catalog-change/output.txt @@ -0,0 +1,73 @@ + +>>> [CLI] catalogs create othercat_[UNIQUE_NAME] -o json +{ + "name": "othercat_[UNIQUE_NAME]" +} + +=== Deploy the schema on terraform in catalog 'main' +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Created schemas.my_schema +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Move the schema to another catalog. Without --auto-approve the recreate is refused +>>> DATABRICKS_BUNDLE_ENGINE=direct errcode [CLI] bundle deploy +Warn: migration to direct: schemas.my_schema: catalog_name differs between config ("othercat_[UNIQUE_NAME]") and terraform state ("main"); this resource will be recreated. +Migrated 1 resource to direct deployment engine. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.my_schema +Warn: Migration not committed, keeping Terraform state +Error: the deployment requires destructive actions, but the current console does not support prompting. +Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed. +To proceed, use --auto-approve after reviewing the plan above. + +Files: 1 uploaded, 0 deleted + +Exit code: 1 + +>>> print_migration_telemetry +direct_migrate_recreate_planned true + +=== With --auto-approve the recreate moves the schema to the new catalog on direct +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy --auto-approve +Warn: migration to direct: schemas.my_schema: catalog_name differs between config ("othercat_[UNIQUE_NAME]") and terraform state ("main"); this resource will be recreated. +Migrated 1 resource to direct deployment engine. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.my_schema +Recreated schemas.my_schema +Files: 0 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 0 unchanged + +=== Redeploy on direct: the schema now matches config, so the plan is a clean no-op +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +=== The schema exists in the new catalog and the local state is direct +>>> [CLI] schemas get othercat_[UNIQUE_NAME].myschema-[UNIQUE_NAME] +{ + "full_name": "othercat_[UNIQUE_NAME].myschema-[UNIQUE_NAME]" +} + +>>> find .databricks/bundle -name resources.json -type f +.databricks/bundle/default/resources.json + +=== Cleanup +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.my_schema + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.my_schema + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 1 deleted + +>>> errcode [CLI] catalogs delete othercat_[UNIQUE_NAME] --force diff --git a/acceptance/bundle/migrate/auto/schema-catalog-change/script b/acceptance/bundle/migrate/auto/schema-catalog-change/script new file mode 100644 index 00000000000..428fc30db33 --- /dev/null +++ b/acceptance/bundle/migrate/auto/schema-catalog-change/script @@ -0,0 +1,36 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +# catalog_name is an immutable provided id field. Moving the schema to another catalog +# recreates it. Create the destination catalog out of band (a catalog cannot be a bundle +# resource here: catalogs are direct-only and this bundle deploys on terraform first). +TARGET_CATALOG="othercat_$UNIQUE_NAME" +trace $CLI catalogs create "$TARGET_CATALOG" -o json | jq '{name: (.name // .full_name)}' + +cleanup() { + title "Cleanup" + trace $CLI bundle destroy --auto-approve + trace errcode $CLI catalogs delete "$TARGET_CATALOG" --force +} +trap cleanup EXIT + +title "Deploy the schema on terraform in catalog 'main'" +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy +rm -f "$OUT_REQUESTS" + +title "Move the schema to another catalog. Without --auto-approve the recreate is refused" +# reconcileIDFields records the deployed catalog, so the migration's plan sees a real change; +# schemas recreate on it. Schemas are in the destructive-approval group, so without +# --auto-approve the deploy refuses the recreate (exit 1) and the migration is not committed. +update_file.py databricks.yml "catalog_name: main" "catalog_name: $TARGET_CATALOG" +trace DATABRICKS_BUNDLE_ENGINE=direct errcode $CLI bundle deploy +trace print_migration_telemetry + +title "With --auto-approve the recreate moves the schema to the new catalog on direct" +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy --auto-approve + +title "Redeploy on direct: the schema now matches config, so the plan is a clean no-op" +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy + +title "The schema exists in the new catalog and the local state is direct" +trace $CLI schemas get "$TARGET_CATALOG.myschema-$UNIQUE_NAME" | jq "{full_name}" +trace find .databricks/bundle -name "resources.json" -type f | sort diff --git a/acceptance/bundle/migrate/auto/schema-catalog-change/test.toml b/acceptance/bundle/migrate/auto/schema-catalog-change/test.toml new file mode 100644 index 00000000000..b674c6425f9 --- /dev/null +++ b/acceptance/bundle/migrate/auto/schema-catalog-change/test.toml @@ -0,0 +1,7 @@ +Cloud = true +RecordRequests = true + +Ignore = [ + ".databricks", + "databricks.yml", +] diff --git a/acceptance/bundle/migrate/auto/schema-name-normalized/databricks.yml.tmpl b/acceptance/bundle/migrate/auto/schema-name-normalized/databricks.yml.tmpl new file mode 100644 index 00000000000..13c8a9a9785 --- /dev/null +++ b/acceptance/bundle/migrate/auto/schema-name-normalized/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: uc-schemaname-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + schemas: + my_schema: + name: MySchema-$UNIQUE_NAME + catalog_name: main diff --git a/acceptance/bundle/migrate/auto/schema-name-normalized/out.test.toml b/acceptance/bundle/migrate/auto/schema-name-normalized/out.test.toml new file mode 100644 index 00000000000..d73c45e3119 --- /dev/null +++ b/acceptance/bundle/migrate/auto/schema-name-normalized/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/auto/schema-name-normalized/output.txt b/acceptance/bundle/migrate/auto/schema-name-normalized/output.txt new file mode 100644 index 00000000000..06d2c87411c --- /dev/null +++ b/acceptance/bundle/migrate/auto/schema-name-normalized/output.txt @@ -0,0 +1,30 @@ + +=== Deploy on terraform; UC lowercases the schema name +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Created schemas.my_schema +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Migrate to direct with the same config (mixed-case name unchanged) +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy +Migrated 1 resource to direct deployment engine. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +=== Follow-up plan on direct converges (the case-only difference is not a change) +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Cleanup +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.my_schema + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.my_schema + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 1 deleted diff --git a/acceptance/bundle/migrate/auto/schema-name-normalized/script b/acceptance/bundle/migrate/auto/schema-name-normalized/script new file mode 100644 index 00000000000..72607f2d689 --- /dev/null +++ b/acceptance/bundle/migrate/auto/schema-name-normalized/script @@ -0,0 +1,21 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + title "Cleanup" + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +# The config uses a mixed-case schema name; UC lowercases the stored name, so the deployed id +# differs from config by case only, without the user changing anything. reconcileIDFields +# treats a case-only difference as backend normalization and keeps the config value, so the +# migration proceeds with no warning and the plan converges. + +title "Deploy on terraform; UC lowercases the schema name" +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy + +title "Migrate to direct with the same config (mixed-case name unchanged)" +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy + +title "Follow-up plan on direct converges (the case-only difference is not a change)" +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan diff --git a/acceptance/bundle/migrate/auto/schema-name-normalized/test.toml b/acceptance/bundle/migrate/auto/schema-name-normalized/test.toml new file mode 100644 index 00000000000..37e1006eb5a --- /dev/null +++ b/acceptance/bundle/migrate/auto/schema-name-normalized/test.toml @@ -0,0 +1,7 @@ +Cloud = true +RecordRequests = false + +Ignore = [ + ".databricks", + "databricks.yml", +] diff --git a/acceptance/bundle/migrate/auto/schema-rename/databricks.yml.tmpl b/acceptance/bundle/migrate/auto/schema-rename/databricks.yml.tmpl new file mode 100644 index 00000000000..b1ea125e2fd --- /dev/null +++ b/acceptance/bundle/migrate/auto/schema-rename/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: uc-migr-schema-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + schemas: + my_schema: + name: myschema-$UNIQUE_NAME + catalog_name: main diff --git a/acceptance/bundle/migrate/auto/schema-rename/out.test.toml b/acceptance/bundle/migrate/auto/schema-rename/out.test.toml new file mode 100644 index 00000000000..d73c45e3119 --- /dev/null +++ b/acceptance/bundle/migrate/auto/schema-rename/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/auto/schema-rename/output.txt b/acceptance/bundle/migrate/auto/schema-rename/output.txt new file mode 100644 index 00000000000..9ce59292f97 --- /dev/null +++ b/acceptance/bundle/migrate/auto/schema-rename/output.txt @@ -0,0 +1,66 @@ + +=== Deploy the schema on terraform +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Created schemas.my_schema +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the schema (an immutable id field). Without --auto-approve the recreate is refused +>>> DATABRICKS_BUNDLE_ENGINE=direct errcode [CLI] bundle deploy +Warn: migration to direct: schemas.my_schema: name differs between config ("myschema-[UNIQUE_NAME]-renamed") and terraform state ("myschema-[UNIQUE_NAME]"); this resource will be recreated. +Migrated 1 resource to direct deployment engine. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.my_schema +Warn: Migration not committed, keeping Terraform state +Error: the deployment requires destructive actions, but the current console does not support prompting. +Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed. +To proceed, use --auto-approve after reviewing the plan above. + +Files: 1 uploaded, 0 deleted + +Exit code: 1 + +>>> print_migration_telemetry +direct_migrate_recreate_planned true + +=== With --auto-approve the recreate is applied on direct under the new name +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy --auto-approve +Warn: migration to direct: schemas.my_schema: name differs between config ("myschema-[UNIQUE_NAME]-renamed") and terraform state ("myschema-[UNIQUE_NAME]"); this resource will be recreated. +Migrated 1 resource to direct deployment engine. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.my_schema +Recreated schemas.my_schema +Files: 0 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 0 unchanged + +=== Redeploy on direct: the schema now matches config, so the plan is a clean no-op +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +=== The schema exists under the new name and the local state is direct +>>> [CLI] schemas get main.myschema-[UNIQUE_NAME]-renamed +{ + "full_name": "main.myschema-[UNIQUE_NAME]-renamed" +} + +>>> find .databricks/bundle -name resources.json -type f +.databricks/bundle/default/resources.json + +=== Cleanup +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.my_schema + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.my_schema + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 1 deleted diff --git a/acceptance/bundle/migrate/auto/schema-rename/script b/acceptance/bundle/migrate/auto/schema-rename/script new file mode 100644 index 00000000000..55c66b60863 --- /dev/null +++ b/acceptance/bundle/migrate/auto/schema-rename/script @@ -0,0 +1,30 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + title "Cleanup" + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "Deploy the schema on terraform" +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy +rm -f "$OUT_REQUESTS" + +title "Rename the schema (an immutable id field). Without --auto-approve the recreate is refused" +# name is a provided id field, so a rename recreates. reconcileIDFields seeds the deployed +# name into the converted state, so the migration's plan sees a real recreate. Schemas are in +# the destructive-approval group, so without --auto-approve the deploy refuses the recreate +# (exit 1) and the migration is not committed - the run stays on terraform. +update_file.py databricks.yml "myschema-$UNIQUE_NAME" "myschema-$UNIQUE_NAME-renamed" +trace DATABRICKS_BUNDLE_ENGINE=direct errcode $CLI bundle deploy +trace print_migration_telemetry + +title "With --auto-approve the recreate is applied on direct under the new name" +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy --auto-approve + +title "Redeploy on direct: the schema now matches config, so the plan is a clean no-op" +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy + +title "The schema exists under the new name and the local state is direct" +trace $CLI schemas get "main.myschema-$UNIQUE_NAME-renamed" | jq "{full_name}" +trace find .databricks/bundle -name "resources.json" -type f | sort diff --git a/acceptance/bundle/migrate/auto/schema-rename/test.toml b/acceptance/bundle/migrate/auto/schema-rename/test.toml new file mode 100644 index 00000000000..b674c6425f9 --- /dev/null +++ b/acceptance/bundle/migrate/auto/schema-rename/test.toml @@ -0,0 +1,7 @@ +Cloud = true +RecordRequests = true + +Ignore = [ + ".databricks", + "databricks.yml", +] diff --git a/acceptance/bundle/migrate/auto/schema-storage-change/databricks.yml b/acceptance/bundle/migrate/auto/schema-storage-change/databricks.yml new file mode 100644 index 00000000000..30d1ade858b --- /dev/null +++ b/acceptance/bundle/migrate/auto/schema-storage-change/databricks.yml @@ -0,0 +1,9 @@ +bundle: + name: test-bundle + +resources: + schemas: + my_schema: + name: myschema + catalog_name: main + storage_root: s3://mybucket/init diff --git a/acceptance/bundle/migrate/engine-config-direct/out.test.toml b/acceptance/bundle/migrate/auto/schema-storage-change/out.test.toml similarity index 100% rename from acceptance/bundle/migrate/engine-config-direct/out.test.toml rename to acceptance/bundle/migrate/auto/schema-storage-change/out.test.toml diff --git a/acceptance/bundle/migrate/auto/schema-storage-change/output.txt b/acceptance/bundle/migrate/auto/schema-storage-change/output.txt new file mode 100644 index 00000000000..a802d3f99b3 --- /dev/null +++ b/acceptance/bundle/migrate/auto/schema-storage-change/output.txt @@ -0,0 +1,57 @@ + +=== Initial deploy on terraform with storage_root s3://mybucket/init +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created schemas.my_schema +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_migration_telemetry +direct_drymigrate_success true +direct_drymigrate_warnings false + +=== Change immutable storage_root; without --auto-approve the recreate is refused +>>> DATABRICKS_BUNDLE_ENGINE=direct errcode [CLI] bundle deploy +Migrated 1 resource to direct deployment engine. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.my_schema +Warn: Migration not committed, keeping Terraform state +Error: the deployment requires destructive actions, but the current console does not support prompting. +Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed. +To proceed, use --auto-approve after reviewing the plan above. + +Files: 1 uploaded, 0 deleted + +Exit code: 1 + +>>> print_migration_telemetry +direct_migrate_recreate_planned true + +=== The migration was not committed: the state is still terraform +>>> find .databricks/bundle -name resources.json -type f + +>>> find .databricks/bundle -name terraform.tfstate* -type f +.databricks/bundle/default/terraform/terraform.tfstate + +=== With --auto-approve the migration commits and the recreate is applied on direct +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy --auto-approve +Migrated 1 resource to direct deployment engine. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.my_schema +Recreated schemas.my_schema +Files: 0 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 0 unchanged + +>>> print_migration_telemetry +direct_migrate_recreate_planned true +direct_migrated_via_env true + +=== Redeploy on direct: the schema now matches config, so the plan is a clean no-op +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged diff --git a/acceptance/bundle/migrate/auto/schema-storage-change/script b/acceptance/bundle/migrate/auto/schema-storage-change/script new file mode 100644 index 00000000000..d852880182c --- /dev/null +++ b/acceptance/bundle/migrate/auto/schema-storage-change/script @@ -0,0 +1,33 @@ +export DATABRICKS_BUNDLE_ENGINE= + +# Testserver-only: an explicit storage_root needs a registered external location on a real +# workspace, so this is not Cloud-enabled. The recreate/refusal class it covers is verified +# on cloud by schema-rename and pipeline-storage-change. + +title "Initial deploy on terraform with storage_root s3://mybucket/init" +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy +trace print_migration_telemetry +rm -f "$OUT_REQUESTS" + +title "Change immutable storage_root; without --auto-approve the recreate is refused" +# storage_root is immutable (recreate_on_changes). The migration prepares the direct state and +# its plan compares the new config value against the deployed remote and plans a recreate +# (direct_migrate_recreate_planned). Schemas are in the destructive-approval group, so without +# --auto-approve the deploy refuses the recreate (exit 1) and the migration is not committed - +# the run stays on terraform. +update_file.py databricks.yml "storage_root: s3://mybucket/init" "storage_root: s3://mybucket/changed" +trace DATABRICKS_BUNDLE_ENGINE=direct errcode $CLI bundle deploy +trace print_migration_telemetry +rm -f "$OUT_REQUESTS" + +title "The migration was not committed: the state is still terraform" +trace find .databricks/bundle -name "resources.json" -type f | sort +trace find .databricks/bundle -name "terraform.tfstate*" -type f | sort + +title "With --auto-approve the migration commits and the recreate is applied on direct" +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy --auto-approve +trace print_migration_telemetry +rm -f "$OUT_REQUESTS" + +title "Redeploy on direct: the schema now matches config, so the plan is a clean no-op" +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy diff --git a/acceptance/bundle/migrate/auto/schema-storage-normalized/databricks.yml b/acceptance/bundle/migrate/auto/schema-storage-normalized/databricks.yml new file mode 100644 index 00000000000..14875f41279 --- /dev/null +++ b/acceptance/bundle/migrate/auto/schema-storage-normalized/databricks.yml @@ -0,0 +1,9 @@ +bundle: + name: test-bundle + +resources: + schemas: + my_schema: + name: myschema + catalog_name: main + storage_root: s3://bucket/path/ diff --git a/acceptance/bundle/migrate/engine-config-terraform/out.test.toml b/acceptance/bundle/migrate/auto/schema-storage-normalized/out.test.toml similarity index 100% rename from acceptance/bundle/migrate/engine-config-terraform/out.test.toml rename to acceptance/bundle/migrate/auto/schema-storage-normalized/out.test.toml diff --git a/acceptance/bundle/migrate/auto/schema-storage-normalized/output.txt b/acceptance/bundle/migrate/auto/schema-storage-normalized/output.txt new file mode 100644 index 00000000000..8bd5e11da74 --- /dev/null +++ b/acceptance/bundle/migrate/auto/schema-storage-normalized/output.txt @@ -0,0 +1,18 @@ + +=== Deploy on terraform; UC strips the trailing slash from storage_root +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created schemas.my_schema +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Migrate to direct with the same config (storage_root still has the trailing slash) +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy +Migrated 1 resource to direct deployment engine. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +=== Follow-up plan on direct: does it converge or keep seeing a diff? +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/migrate/auto/schema-storage-normalized/script b/acceptance/bundle/migrate/auto/schema-storage-normalized/script new file mode 100644 index 00000000000..ad6ccc2bb64 --- /dev/null +++ b/acceptance/bundle/migrate/auto/schema-storage-normalized/script @@ -0,0 +1,19 @@ +export DATABRICKS_BUNDLE_ENGINE= + +# Testserver-only: an explicit storage_root needs a registered external location on a real +# workspace, so this is not Cloud-enabled. The normalization-converge class it covers is +# verified on cloud by schema-name-normalized / volume-name-normalized. + +# storage_root is immutable (recreate_on_changes) and UC strips a trailing slash. The config +# keeps the trailing slash ("s3://bucket/path/") but the deployed value is normalized +# ("s3://bucket/path"), a difference the user did not make. normalize_slash absorbs it, so the +# migration proceeds with no warning or recreate and the plan converges. + +title "Deploy on terraform; UC strips the trailing slash from storage_root" +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy + +title "Migrate to direct with the same config (storage_root still has the trailing slash)" +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy + +title "Follow-up plan on direct: does it converge or keep seeing a diff?" +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan diff --git a/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/databricks.yml b/acceptance/bundle/migrate/auto/tfbackup-failure/databricks.yml similarity index 100% rename from acceptance/bundle/migrate/auto-migrate-tfbackup-failure/databricks.yml rename to acceptance/bundle/migrate/auto/tfbackup-failure/databricks.yml diff --git a/acceptance/bundle/migrate/grants/out.test.toml b/acceptance/bundle/migrate/auto/tfbackup-failure/out.test.toml similarity index 100% rename from acceptance/bundle/migrate/grants/out.test.toml rename to acceptance/bundle/migrate/auto/tfbackup-failure/out.test.toml diff --git a/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/output.txt b/acceptance/bundle/migrate/auto/tfbackup-failure/output.txt similarity index 80% rename from acceptance/bundle/migrate/auto-migrate-tfbackup-failure/output.txt rename to acceptance/bundle/migrate/auto/tfbackup-failure/output.txt index dd63e04347d..c68f6949251 100644 --- a/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/output.txt +++ b/acceptance/bundle/migrate/auto/tfbackup-failure/output.txt @@ -12,13 +12,10 @@ direct_drymigrate_warnings false === Opt in via env var; inject 403 on the DELETE of remote terraform.tfstate >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy -Warn: Direct engine selected via DATABRICKS_BUNDLE_ENGINE environment variable but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy. +Migrated 1 resource to direct deployment engine. Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Files: 0 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged -Migrating state to direct deployment engine (selected via DATABRICKS_BUNDLE_ENGINE environment variable)... -Warn: backing up terraform state: could not delete terraform.tfstate: Fault injected by test. -Migrated 1 resource to direct deployment engine. >>> print_migration_telemetry direct_migrated_via_env true @@ -36,7 +33,6 @@ ID Type Language Path [NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json [NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json [NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json -[NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate [NUMID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate.backup === Retry: deploys on already-migrated direct state diff --git a/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/script b/acceptance/bundle/migrate/auto/tfbackup-failure/script similarity index 68% rename from acceptance/bundle/migrate/auto-migrate-tfbackup-failure/script rename to acceptance/bundle/migrate/auto/tfbackup-failure/script index ae45d9e4b6d..b57f04a5321 100644 --- a/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/script +++ b/acceptance/bundle/migrate/auto/tfbackup-failure/script @@ -6,12 +6,12 @@ trace print_migration_telemetry rm -f "$OUT_REQUESTS" title "Opt in via env var; inject 403 on the DELETE of remote terraform.tfstate" -# resources.json is uploaded successfully; then we back up terraform.tfstate to -# .backup and delete the original. Fault the second workspace/delete call -# (skipping the first one, which is the deploy's own artifacts cleanup) to -# fail the terraform.tfstate delete. The delete failure is best-effort: -# migration succeeds with a warning and the .backup is kept. -fault.py "POST /api/2.0/workspace/delete" 403 1 1 +# The approved deploy pushes resources.json, then FinalizeDeferredMigration best-effort backs +# up and deletes the remote terraform.tfstate. That delete is the first workspace/delete of the +# run (the no-op deploy issues none), so OFFSET=0 faults it. The delete failure is best-effort — +# resources.json already outranks the terraform state by serial — so the migration succeeds +# with a warning and the .backup is kept. +fault.py "POST /api/2.0/workspace/delete" 403 0 1 trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy trace print_migration_telemetry rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/migrate/auto/volume-name-normalized/databricks.yml.tmpl b/acceptance/bundle/migrate/auto/volume-name-normalized/databricks.yml.tmpl new file mode 100644 index 00000000000..c64198ef3ca --- /dev/null +++ b/acceptance/bundle/migrate/auto/volume-name-normalized/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: uc-volname-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + volumes: + my_volume: + catalog_name: main + schema_name: default + name: MyVolume-$UNIQUE_NAME + volume_type: MANAGED diff --git a/acceptance/bundle/migrate/auto/volume-name-normalized/out.test.toml b/acceptance/bundle/migrate/auto/volume-name-normalized/out.test.toml new file mode 100644 index 00000000000..d73c45e3119 --- /dev/null +++ b/acceptance/bundle/migrate/auto/volume-name-normalized/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/auto/volume-name-normalized/output.txt b/acceptance/bundle/migrate/auto/volume-name-normalized/output.txt new file mode 100644 index 00000000000..197eb9edec5 --- /dev/null +++ b/acceptance/bundle/migrate/auto/volume-name-normalized/output.txt @@ -0,0 +1,33 @@ + +=== Deploy on terraform; UC lowercases the volume name +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Created volumes.my_volume +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Migrate to direct with the same config (mixed-case name unchanged) +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy +Migrated 1 resource to direct deployment engine. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +=== Follow-up plan on direct converges (the case-only difference is not a change) +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Cleanup +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.volumes.my_volume + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.my_volume + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 1 deleted diff --git a/acceptance/bundle/migrate/auto/volume-name-normalized/script b/acceptance/bundle/migrate/auto/volume-name-normalized/script new file mode 100644 index 00000000000..c9a14dab412 --- /dev/null +++ b/acceptance/bundle/migrate/auto/volume-name-normalized/script @@ -0,0 +1,21 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + title "Cleanup" + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +# name is an immutable provided id field. UC lowercases it, so the deployed id differs from +# config by case only, without the user changing anything. reconcileIDFields treats the +# case-only difference as backend normalization and keeps the config value, so the migration +# proceeds with no warning and the plan converges. + +title "Deploy on terraform; UC lowercases the volume name" +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy + +title "Migrate to direct with the same config (mixed-case name unchanged)" +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy + +title "Follow-up plan on direct converges (the case-only difference is not a change)" +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan diff --git a/acceptance/bundle/migrate/auto/volume-name-normalized/test.toml b/acceptance/bundle/migrate/auto/volume-name-normalized/test.toml new file mode 100644 index 00000000000..37e1006eb5a --- /dev/null +++ b/acceptance/bundle/migrate/auto/volume-name-normalized/test.toml @@ -0,0 +1,7 @@ +Cloud = true +RecordRequests = false + +Ignore = [ + ".databricks", + "databricks.yml", +] diff --git a/acceptance/bundle/migrate/auto/volume-rename/databricks.yml.tmpl b/acceptance/bundle/migrate/auto/volume-rename/databricks.yml.tmpl new file mode 100644 index 00000000000..10bfccbb9e1 --- /dev/null +++ b/acceptance/bundle/migrate/auto/volume-rename/databricks.yml.tmpl @@ -0,0 +1,17 @@ +bundle: + name: uc-migrvol-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + schemas: + parent: + name: pschema-$UNIQUE_NAME + catalog_name: main + volumes: + my_volume: + name: thevolume-$UNIQUE_NAME + catalog_name: main + schema_name: pschema-$UNIQUE_NAME + volume_type: MANAGED diff --git a/acceptance/bundle/migrate/auto/volume-rename/out.test.toml b/acceptance/bundle/migrate/auto/volume-rename/out.test.toml new file mode 100644 index 00000000000..d73c45e3119 --- /dev/null +++ b/acceptance/bundle/migrate/auto/volume-rename/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/auto/volume-rename/output.txt b/acceptance/bundle/migrate/auto/volume-rename/output.txt new file mode 100644 index 00000000000..2dcd60bc3c0 --- /dev/null +++ b/acceptance/bundle/migrate/auto/volume-rename/output.txt @@ -0,0 +1,51 @@ + +=== Deploy the schema and volume on terraform +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Created schemas.parent +Created volumes.my_volume +Files: 4 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the volume (an updatable id field) and deploy on direct +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy +Warn: migration to direct: volumes.my_volume: name differs between config ("thevolume-[UNIQUE_NAME]-renamed") and terraform state ("thevolume-[UNIQUE_NAME]"); this resource will be renamed. +Migrated 2 resources to direct deployment engine. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Updated volumes.my_volume +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +=== Retry on direct: state now matches the backend, so the migration succeeds +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +=== The volume exists under the new name and the local state is direct +>>> [CLI] volumes read main.pschema-[UNIQUE_NAME].thevolume-[UNIQUE_NAME]-renamed +{ + "full_name": "main.pschema-[UNIQUE_NAME].thevolume-[UNIQUE_NAME]-renamed" +} + +>>> find .databricks/bundle -name resources.json -type f +.databricks/bundle/default/resources.json + +=== Cleanup +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.parent + delete resources.volumes.my_volume + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.parent + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.my_volume + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 2 deleted diff --git a/acceptance/bundle/migrate/auto/volume-rename/script b/acceptance/bundle/migrate/auto/volume-rename/script new file mode 100644 index 00000000000..f5a1a39ee36 --- /dev/null +++ b/acceptance/bundle/migrate/auto/volume-rename/script @@ -0,0 +1,26 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + title "Cleanup" + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "Deploy the schema and volume on terraform" +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy + +title "Rename the volume (an updatable id field) and deploy on direct" +# Unlike a schema (whose name recreates), a volume name is an updatable id field: it is +# renamed in place (UpdateWithID). The migration seeds the deployed name into the converted +# state, so its plan sees a rename and the migration succeeds; the deploy then renames the +# volume rather than the migration snapshotting the new name as already applied. Verifies the +# updatable-id path against the real backend. +update_file.py databricks.yml "thevolume-$UNIQUE_NAME" "thevolume-$UNIQUE_NAME-renamed" +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy + +title "Retry on direct: state now matches the backend, so the migration succeeds" +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy + +title "The volume exists under the new name and the local state is direct" +trace $CLI volumes read "main.pschema-$UNIQUE_NAME.thevolume-$UNIQUE_NAME-renamed" | jq "{full_name}" +trace find .databricks/bundle -name "resources.json" -type f | sort diff --git a/acceptance/bundle/migrate/auto/volume-rename/test.toml b/acceptance/bundle/migrate/auto/volume-rename/test.toml new file mode 100644 index 00000000000..37e1006eb5a --- /dev/null +++ b/acceptance/bundle/migrate/auto/volume-rename/test.toml @@ -0,0 +1,7 @@ +Cloud = true +RecordRequests = false + +Ignore = [ + ".databricks", + "databricks.yml", +] diff --git a/acceptance/bundle/migrate/auto/volume-type-change/databricks.yml b/acceptance/bundle/migrate/auto/volume-type-change/databricks.yml new file mode 100644 index 00000000000..c90a9506ef9 --- /dev/null +++ b/acceptance/bundle/migrate/auto/volume-type-change/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: test-bundle + +resources: + volumes: + my_volume: + catalog_name: main + schema_name: default + name: myvolume + volume_type: MANAGED diff --git a/acceptance/bundle/migrate/permissions/out.test.toml b/acceptance/bundle/migrate/auto/volume-type-change/out.test.toml similarity index 100% rename from acceptance/bundle/migrate/permissions/out.test.toml rename to acceptance/bundle/migrate/auto/volume-type-change/out.test.toml diff --git a/acceptance/bundle/migrate/auto/volume-type-change/output.txt b/acceptance/bundle/migrate/auto/volume-type-change/output.txt new file mode 100644 index 00000000000..38bd86a0cc2 --- /dev/null +++ b/acceptance/bundle/migrate/auto/volume-type-change/output.txt @@ -0,0 +1,29 @@ + +=== Deploy on terraform with volume_type MANAGED +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created volumes.my_volume +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Change volume_type (immutable); without --auto-approve the recreate is refused +>>> DATABRICKS_BUNDLE_ENGINE=direct errcode [CLI] bundle deploy +Migrated 1 resource to direct deployment engine. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... + +This action will result in the deletion or recreation of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + recreate resources.volumes.my_volume +Warn: Migration not committed, keeping Terraform state +Error: the deployment requires destructive actions, but the current console does not support prompting. +Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed. +To proceed, use --auto-approve after reviewing the plan above. + +Files: 1 uploaded, 0 deleted + +Exit code: 1 + +>>> print_migration_telemetry +direct_migrate_recreate_planned true diff --git a/acceptance/bundle/migrate/auto/volume-type-change/script b/acceptance/bundle/migrate/auto/volume-type-change/script new file mode 100644 index 00000000000..26b253f2d59 --- /dev/null +++ b/acceptance/bundle/migrate/auto/volume-type-change/script @@ -0,0 +1,19 @@ +export DATABRICKS_BUNDLE_ENGINE= + +# Testserver-only: changing volume_type to EXTERNAL needs a storage_location backed by a +# registered external location on a real workspace, so this is not Cloud-enabled. The +# recreate class it covers is verified on cloud by schema-rename / registered-model-rename. + +# volume_type is both a provided id field and a recreate_on_changes field. reconcileIDFields +# records the deployed value, so the migration's plan sees a real change and recreates. Volumes +# are in the destructive-approval group, so without --auto-approve the deploy refuses the +# recreate (exit 1) and the migration is not committed. + +title "Deploy on terraform with volume_type MANAGED" +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy +rm -f "$OUT_REQUESTS" + +title "Change volume_type (immutable); without --auto-approve the recreate is refused" +update_file.py databricks.yml "volume_type: MANAGED" "volume_type: EXTERNAL" +trace DATABRICKS_BUNDLE_ENGINE=direct errcode $CLI bundle deploy +trace print_migration_telemetry diff --git a/acceptance/bundle/migrate/auto-migrate-direct-only/databricks.yml b/acceptance/bundle/migrate/command/added-direct-only/databricks.yml similarity index 100% rename from acceptance/bundle/migrate/auto-migrate-direct-only/databricks.yml rename to acceptance/bundle/migrate/command/added-direct-only/databricks.yml diff --git a/acceptance/bundle/migrate/profile_arg/out.test.toml b/acceptance/bundle/migrate/command/added-direct-only/out.test.toml similarity index 100% rename from acceptance/bundle/migrate/profile_arg/out.test.toml rename to acceptance/bundle/migrate/command/added-direct-only/out.test.toml diff --git a/acceptance/bundle/migrate/added-direct-only/output.txt b/acceptance/bundle/migrate/command/added-direct-only/output.txt similarity index 100% rename from acceptance/bundle/migrate/added-direct-only/output.txt rename to acceptance/bundle/migrate/command/added-direct-only/output.txt diff --git a/acceptance/bundle/migrate/added-direct-only/script b/acceptance/bundle/migrate/command/added-direct-only/script similarity index 100% rename from acceptance/bundle/migrate/added-direct-only/script rename to acceptance/bundle/migrate/command/added-direct-only/script diff --git a/acceptance/bundle/migrate/added/databricks.yml b/acceptance/bundle/migrate/command/added/databricks.yml similarity index 100% rename from acceptance/bundle/migrate/added/databricks.yml rename to acceptance/bundle/migrate/command/added/databricks.yml diff --git a/acceptance/bundle/migrate/removed/out.test.toml b/acceptance/bundle/migrate/command/added/out.test.toml similarity index 100% rename from acceptance/bundle/migrate/removed/out.test.toml rename to acceptance/bundle/migrate/command/added/out.test.toml diff --git a/acceptance/bundle/migrate/added/output.txt b/acceptance/bundle/migrate/command/added/output.txt similarity index 100% rename from acceptance/bundle/migrate/added/output.txt rename to acceptance/bundle/migrate/command/added/output.txt diff --git a/acceptance/bundle/migrate/added/script b/acceptance/bundle/migrate/command/added/script similarity index 100% rename from acceptance/bundle/migrate/added/script rename to acceptance/bundle/migrate/command/added/script diff --git a/acceptance/bundle/migrate/basic/databricks.yml b/acceptance/bundle/migrate/command/basic/databricks.yml similarity index 100% rename from acceptance/bundle/migrate/basic/databricks.yml rename to acceptance/bundle/migrate/command/basic/databricks.yml diff --git a/acceptance/bundle/migrate/basic/notebook.py b/acceptance/bundle/migrate/command/basic/notebook.py similarity index 100% rename from acceptance/bundle/migrate/basic/notebook.py rename to acceptance/bundle/migrate/command/basic/notebook.py diff --git a/acceptance/bundle/migrate/basic/out.new_state.json b/acceptance/bundle/migrate/command/basic/out.new_state.json similarity index 100% rename from acceptance/bundle/migrate/basic/out.new_state.json rename to acceptance/bundle/migrate/command/basic/out.new_state.json diff --git a/acceptance/bundle/migrate/basic/out.original_state.json b/acceptance/bundle/migrate/command/basic/out.original_state.json similarity index 100% rename from acceptance/bundle/migrate/basic/out.original_state.json rename to acceptance/bundle/migrate/command/basic/out.original_state.json diff --git a/acceptance/bundle/migrate/basic/out.plan_update.json b/acceptance/bundle/migrate/command/basic/out.plan_update.json similarity index 100% rename from acceptance/bundle/migrate/basic/out.plan_update.json rename to acceptance/bundle/migrate/command/basic/out.plan_update.json diff --git a/acceptance/bundle/migrate/runas/out.test.toml b/acceptance/bundle/migrate/command/basic/out.test.toml similarity index 100% rename from acceptance/bundle/migrate/runas/out.test.toml rename to acceptance/bundle/migrate/command/basic/out.test.toml diff --git a/acceptance/bundle/migrate/basic/output.txt b/acceptance/bundle/migrate/command/basic/output.txt similarity index 100% rename from acceptance/bundle/migrate/basic/output.txt rename to acceptance/bundle/migrate/command/basic/output.txt diff --git a/acceptance/bundle/migrate/basic/pipeline.py b/acceptance/bundle/migrate/command/basic/pipeline.py similarity index 100% rename from acceptance/bundle/migrate/basic/pipeline.py rename to acceptance/bundle/migrate/command/basic/pipeline.py diff --git a/acceptance/bundle/migrate/basic/script b/acceptance/bundle/migrate/command/basic/script similarity index 100% rename from acceptance/bundle/migrate/basic/script rename to acceptance/bundle/migrate/command/basic/script diff --git a/acceptance/bundle/migrate/basic/test.toml b/acceptance/bundle/migrate/command/basic/test.toml similarity index 100% rename from acceptance/bundle/migrate/basic/test.toml rename to acceptance/bundle/migrate/command/basic/test.toml diff --git a/acceptance/bundle/migrate/dashboards/databricks.yml b/acceptance/bundle/migrate/command/dashboards/databricks.yml similarity index 100% rename from acceptance/bundle/migrate/dashboards/databricks.yml rename to acceptance/bundle/migrate/command/dashboards/databricks.yml diff --git a/acceptance/bundle/migrate/dashboards/out.new_state.json b/acceptance/bundle/migrate/command/dashboards/out.new_state.json similarity index 100% rename from acceptance/bundle/migrate/dashboards/out.new_state.json rename to acceptance/bundle/migrate/command/dashboards/out.new_state.json diff --git a/acceptance/bundle/migrate/dashboards/out.original_state.json b/acceptance/bundle/migrate/command/dashboards/out.original_state.json similarity index 100% rename from acceptance/bundle/migrate/dashboards/out.original_state.json rename to acceptance/bundle/migrate/command/dashboards/out.original_state.json diff --git a/acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json b/acceptance/bundle/migrate/command/dashboards/out.plan_after_migrate.json similarity index 100% rename from acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json rename to acceptance/bundle/migrate/command/dashboards/out.plan_after_migrate.json diff --git a/acceptance/bundle/migrate/var_arg/out.test.toml b/acceptance/bundle/migrate/command/dashboards/out.test.toml similarity index 100% rename from acceptance/bundle/migrate/var_arg/out.test.toml rename to acceptance/bundle/migrate/command/dashboards/out.test.toml diff --git a/acceptance/bundle/migrate/dashboards/output.txt b/acceptance/bundle/migrate/command/dashboards/output.txt similarity index 100% rename from acceptance/bundle/migrate/dashboards/output.txt rename to acceptance/bundle/migrate/command/dashboards/output.txt diff --git a/acceptance/bundle/migrate/dashboards/sample-dashboard.lvdash.json b/acceptance/bundle/migrate/command/dashboards/sample-dashboard.lvdash.json similarity index 100% rename from acceptance/bundle/migrate/dashboards/sample-dashboard.lvdash.json rename to acceptance/bundle/migrate/command/dashboards/sample-dashboard.lvdash.json diff --git a/acceptance/bundle/migrate/dashboards/script b/acceptance/bundle/migrate/command/dashboards/script similarity index 100% rename from acceptance/bundle/migrate/dashboards/script rename to acceptance/bundle/migrate/command/dashboards/script diff --git a/acceptance/bundle/migrate/default-python/databricks.yml b/acceptance/bundle/migrate/command/default-python/databricks.yml similarity index 100% rename from acceptance/bundle/migrate/default-python/databricks.yml rename to acceptance/bundle/migrate/command/default-python/databricks.yml diff --git a/acceptance/bundle/migrate/default-python/input.json b/acceptance/bundle/migrate/command/default-python/input.json similarity index 100% rename from acceptance/bundle/migrate/default-python/input.json rename to acceptance/bundle/migrate/command/default-python/input.json diff --git a/acceptance/bundle/migrate/default-python/notebook.py b/acceptance/bundle/migrate/command/default-python/notebook.py similarity index 100% rename from acceptance/bundle/migrate/default-python/notebook.py rename to acceptance/bundle/migrate/command/default-python/notebook.py diff --git a/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json b/acceptance/bundle/migrate/command/default-python/out.plan_after_deploy.json similarity index 100% rename from acceptance/bundle/migrate/default-python/out.plan_after_deploy.json rename to acceptance/bundle/migrate/command/default-python/out.plan_after_deploy.json diff --git a/acceptance/bundle/migrate/default-python/out.plan_after_migration.json b/acceptance/bundle/migrate/command/default-python/out.plan_after_migration.json similarity index 100% rename from acceptance/bundle/migrate/default-python/out.plan_after_migration.json rename to acceptance/bundle/migrate/command/default-python/out.plan_after_migration.json diff --git a/acceptance/bundle/migrate/default-python/out.state_after_migration.json b/acceptance/bundle/migrate/command/default-python/out.state_after_migration.json similarity index 100% rename from acceptance/bundle/migrate/default-python/out.state_after_migration.json rename to acceptance/bundle/migrate/command/default-python/out.state_after_migration.json diff --git a/acceptance/bundle/migrate/default-python/out.state_original.json b/acceptance/bundle/migrate/command/default-python/out.state_original.json similarity index 100% rename from acceptance/bundle/migrate/default-python/out.state_original.json rename to acceptance/bundle/migrate/command/default-python/out.state_original.json diff --git a/acceptance/bundle/migrate/command/default-python/out.test.toml b/acceptance/bundle/migrate/command/default-python/out.test.toml new file mode 100644 index 00000000000..27ec2a7fcd6 --- /dev/null +++ b/acceptance/bundle/migrate/command/default-python/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/default-python/output.txt b/acceptance/bundle/migrate/command/default-python/output.txt similarity index 100% rename from acceptance/bundle/migrate/default-python/output.txt rename to acceptance/bundle/migrate/command/default-python/output.txt diff --git a/acceptance/bundle/migrate/default-python/pipeline.py b/acceptance/bundle/migrate/command/default-python/pipeline.py similarity index 100% rename from acceptance/bundle/migrate/default-python/pipeline.py rename to acceptance/bundle/migrate/command/default-python/pipeline.py diff --git a/acceptance/bundle/migrate/default-python/script b/acceptance/bundle/migrate/command/default-python/script similarity index 100% rename from acceptance/bundle/migrate/default-python/script rename to acceptance/bundle/migrate/command/default-python/script diff --git a/acceptance/bundle/migrate/default-python/test.toml b/acceptance/bundle/migrate/command/default-python/test.toml similarity index 100% rename from acceptance/bundle/migrate/default-python/test.toml rename to acceptance/bundle/migrate/command/default-python/test.toml diff --git a/acceptance/bundle/migrate/engine-config-direct/databricks.yml b/acceptance/bundle/migrate/command/engine-config-direct/databricks.yml similarity index 100% rename from acceptance/bundle/migrate/engine-config-direct/databricks.yml rename to acceptance/bundle/migrate/command/engine-config-direct/databricks.yml diff --git a/acceptance/bundle/migrate/command/engine-config-direct/out.test.toml b/acceptance/bundle/migrate/command/engine-config-direct/out.test.toml new file mode 100644 index 00000000000..27ec2a7fcd6 --- /dev/null +++ b/acceptance/bundle/migrate/command/engine-config-direct/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/engine-config-direct/output.txt b/acceptance/bundle/migrate/command/engine-config-direct/output.txt similarity index 100% rename from acceptance/bundle/migrate/engine-config-direct/output.txt rename to acceptance/bundle/migrate/command/engine-config-direct/output.txt diff --git a/acceptance/bundle/migrate/engine-config-direct/script b/acceptance/bundle/migrate/command/engine-config-direct/script similarity index 100% rename from acceptance/bundle/migrate/engine-config-direct/script rename to acceptance/bundle/migrate/command/engine-config-direct/script diff --git a/acceptance/bundle/migrate/command/engine-config-direct/test.toml b/acceptance/bundle/migrate/command/engine-config-direct/test.toml new file mode 100644 index 00000000000..a030353d571 --- /dev/null +++ b/acceptance/bundle/migrate/command/engine-config-direct/test.toml @@ -0,0 +1 @@ +RecordRequests = false diff --git a/acceptance/bundle/migrate/engine-config-terraform/databricks.yml b/acceptance/bundle/migrate/command/engine-config-terraform/databricks.yml similarity index 100% rename from acceptance/bundle/migrate/engine-config-terraform/databricks.yml rename to acceptance/bundle/migrate/command/engine-config-terraform/databricks.yml diff --git a/acceptance/bundle/migrate/command/engine-config-terraform/out.test.toml b/acceptance/bundle/migrate/command/engine-config-terraform/out.test.toml new file mode 100644 index 00000000000..27ec2a7fcd6 --- /dev/null +++ b/acceptance/bundle/migrate/command/engine-config-terraform/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/engine-config-terraform/output.txt b/acceptance/bundle/migrate/command/engine-config-terraform/output.txt similarity index 100% rename from acceptance/bundle/migrate/engine-config-terraform/output.txt rename to acceptance/bundle/migrate/command/engine-config-terraform/output.txt diff --git a/acceptance/bundle/migrate/engine-config-terraform/script b/acceptance/bundle/migrate/command/engine-config-terraform/script similarity index 100% rename from acceptance/bundle/migrate/engine-config-terraform/script rename to acceptance/bundle/migrate/command/engine-config-terraform/script diff --git a/acceptance/bundle/migrate/command/engine-config-terraform/test.toml b/acceptance/bundle/migrate/command/engine-config-terraform/test.toml new file mode 100644 index 00000000000..a030353d571 --- /dev/null +++ b/acceptance/bundle/migrate/command/engine-config-terraform/test.toml @@ -0,0 +1 @@ +RecordRequests = false diff --git a/acceptance/bundle/migrate/grants/databricks.yml b/acceptance/bundle/migrate/command/grants/databricks.yml similarity index 100% rename from acceptance/bundle/migrate/grants/databricks.yml rename to acceptance/bundle/migrate/command/grants/databricks.yml diff --git a/acceptance/bundle/migrate/grants/out.new_state.json b/acceptance/bundle/migrate/command/grants/out.new_state.json similarity index 100% rename from acceptance/bundle/migrate/grants/out.new_state.json rename to acceptance/bundle/migrate/command/grants/out.new_state.json diff --git a/acceptance/bundle/migrate/grants/out.original_state.json b/acceptance/bundle/migrate/command/grants/out.original_state.json similarity index 100% rename from acceptance/bundle/migrate/grants/out.original_state.json rename to acceptance/bundle/migrate/command/grants/out.original_state.json diff --git a/acceptance/bundle/migrate/command/grants/out.test.toml b/acceptance/bundle/migrate/command/grants/out.test.toml new file mode 100644 index 00000000000..27ec2a7fcd6 --- /dev/null +++ b/acceptance/bundle/migrate/command/grants/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/grants/output.txt b/acceptance/bundle/migrate/command/grants/output.txt similarity index 100% rename from acceptance/bundle/migrate/grants/output.txt rename to acceptance/bundle/migrate/command/grants/output.txt diff --git a/acceptance/bundle/migrate/grants/script b/acceptance/bundle/migrate/command/grants/script similarity index 100% rename from acceptance/bundle/migrate/grants/script rename to acceptance/bundle/migrate/command/grants/script diff --git a/acceptance/bundle/migrate/permissions/databricks.yml b/acceptance/bundle/migrate/command/permissions/databricks.yml similarity index 100% rename from acceptance/bundle/migrate/permissions/databricks.yml rename to acceptance/bundle/migrate/command/permissions/databricks.yml diff --git a/acceptance/bundle/migrate/permissions/notebook.py b/acceptance/bundle/migrate/command/permissions/notebook.py similarity index 100% rename from acceptance/bundle/migrate/permissions/notebook.py rename to acceptance/bundle/migrate/command/permissions/notebook.py diff --git a/acceptance/bundle/migrate/permissions/out.new_state.json b/acceptance/bundle/migrate/command/permissions/out.new_state.json similarity index 100% rename from acceptance/bundle/migrate/permissions/out.new_state.json rename to acceptance/bundle/migrate/command/permissions/out.new_state.json diff --git a/acceptance/bundle/migrate/permissions/out.original_state.json b/acceptance/bundle/migrate/command/permissions/out.original_state.json similarity index 100% rename from acceptance/bundle/migrate/permissions/out.original_state.json rename to acceptance/bundle/migrate/command/permissions/out.original_state.json diff --git a/acceptance/bundle/migrate/command/permissions/out.test.toml b/acceptance/bundle/migrate/command/permissions/out.test.toml new file mode 100644 index 00000000000..27ec2a7fcd6 --- /dev/null +++ b/acceptance/bundle/migrate/command/permissions/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/permissions/output.txt b/acceptance/bundle/migrate/command/permissions/output.txt similarity index 100% rename from acceptance/bundle/migrate/permissions/output.txt rename to acceptance/bundle/migrate/command/permissions/output.txt diff --git a/acceptance/bundle/migrate/permissions/pipeline.py b/acceptance/bundle/migrate/command/permissions/pipeline.py similarity index 100% rename from acceptance/bundle/migrate/permissions/pipeline.py rename to acceptance/bundle/migrate/command/permissions/pipeline.py diff --git a/acceptance/bundle/migrate/permissions/script b/acceptance/bundle/migrate/command/permissions/script similarity index 100% rename from acceptance/bundle/migrate/permissions/script rename to acceptance/bundle/migrate/command/permissions/script diff --git a/acceptance/bundle/migrate/permissions/test.toml b/acceptance/bundle/migrate/command/permissions/test.toml similarity index 100% rename from acceptance/bundle/migrate/permissions/test.toml rename to acceptance/bundle/migrate/command/permissions/test.toml diff --git a/acceptance/bundle/migrate/profile_arg/databricks.yml b/acceptance/bundle/migrate/command/profile_arg/databricks.yml similarity index 100% rename from acceptance/bundle/migrate/profile_arg/databricks.yml rename to acceptance/bundle/migrate/command/profile_arg/databricks.yml diff --git a/acceptance/bundle/migrate/profile_arg/notebook.py b/acceptance/bundle/migrate/command/profile_arg/notebook.py similarity index 100% rename from acceptance/bundle/migrate/profile_arg/notebook.py rename to acceptance/bundle/migrate/command/profile_arg/notebook.py diff --git a/acceptance/bundle/migrate/command/profile_arg/out.test.toml b/acceptance/bundle/migrate/command/profile_arg/out.test.toml new file mode 100644 index 00000000000..27ec2a7fcd6 --- /dev/null +++ b/acceptance/bundle/migrate/command/profile_arg/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/profile_arg/output.txt b/acceptance/bundle/migrate/command/profile_arg/output.txt similarity index 100% rename from acceptance/bundle/migrate/profile_arg/output.txt rename to acceptance/bundle/migrate/command/profile_arg/output.txt diff --git a/acceptance/bundle/migrate/profile_arg/pipeline.py b/acceptance/bundle/migrate/command/profile_arg/pipeline.py similarity index 100% rename from acceptance/bundle/migrate/profile_arg/pipeline.py rename to acceptance/bundle/migrate/command/profile_arg/pipeline.py diff --git a/acceptance/bundle/migrate/profile_arg/script b/acceptance/bundle/migrate/command/profile_arg/script similarity index 100% rename from acceptance/bundle/migrate/profile_arg/script rename to acceptance/bundle/migrate/command/profile_arg/script diff --git a/acceptance/bundle/migrate/command/profile_arg/test.toml b/acceptance/bundle/migrate/command/profile_arg/test.toml new file mode 100644 index 00000000000..a030353d571 --- /dev/null +++ b/acceptance/bundle/migrate/command/profile_arg/test.toml @@ -0,0 +1 @@ +RecordRequests = false diff --git a/acceptance/bundle/migrate/removed/databricks.yml b/acceptance/bundle/migrate/command/removed/databricks.yml similarity index 100% rename from acceptance/bundle/migrate/removed/databricks.yml rename to acceptance/bundle/migrate/command/removed/databricks.yml diff --git a/acceptance/bundle/migrate/command/removed/out.test.toml b/acceptance/bundle/migrate/command/removed/out.test.toml new file mode 100644 index 00000000000..27ec2a7fcd6 --- /dev/null +++ b/acceptance/bundle/migrate/command/removed/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/removed/output.txt b/acceptance/bundle/migrate/command/removed/output.txt similarity index 100% rename from acceptance/bundle/migrate/removed/output.txt rename to acceptance/bundle/migrate/command/removed/output.txt diff --git a/acceptance/bundle/migrate/removed/script b/acceptance/bundle/migrate/command/removed/script similarity index 100% rename from acceptance/bundle/migrate/removed/script rename to acceptance/bundle/migrate/command/removed/script diff --git a/acceptance/bundle/migrate/runas/databricks.yml.tmpl b/acceptance/bundle/migrate/command/runas/databricks.yml.tmpl similarity index 100% rename from acceptance/bundle/migrate/runas/databricks.yml.tmpl rename to acceptance/bundle/migrate/command/runas/databricks.yml.tmpl diff --git a/acceptance/bundle/migrate/runas/out.create_requests.json b/acceptance/bundle/migrate/command/runas/out.create_requests.json similarity index 100% rename from acceptance/bundle/migrate/runas/out.create_requests.json rename to acceptance/bundle/migrate/command/runas/out.create_requests.json diff --git a/acceptance/bundle/migrate/runas/out.new_state.json b/acceptance/bundle/migrate/command/runas/out.new_state.json similarity index 100% rename from acceptance/bundle/migrate/runas/out.new_state.json rename to acceptance/bundle/migrate/command/runas/out.new_state.json diff --git a/acceptance/bundle/migrate/runas/out.old_state.json b/acceptance/bundle/migrate/command/runas/out.old_state.json similarity index 100% rename from acceptance/bundle/migrate/runas/out.old_state.json rename to acceptance/bundle/migrate/command/runas/out.old_state.json diff --git a/acceptance/bundle/migrate/runas/out.pipelines_get.json b/acceptance/bundle/migrate/command/runas/out.pipelines_get.json similarity index 100% rename from acceptance/bundle/migrate/runas/out.pipelines_get.json rename to acceptance/bundle/migrate/command/runas/out.pipelines_get.json diff --git a/acceptance/bundle/migrate/runas/out.plan.json b/acceptance/bundle/migrate/command/runas/out.plan.json similarity index 100% rename from acceptance/bundle/migrate/runas/out.plan.json rename to acceptance/bundle/migrate/command/runas/out.plan.json diff --git a/acceptance/bundle/migrate/command/runas/out.test.toml b/acceptance/bundle/migrate/command/runas/out.test.toml new file mode 100644 index 00000000000..27ec2a7fcd6 --- /dev/null +++ b/acceptance/bundle/migrate/command/runas/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/runas/output.txt b/acceptance/bundle/migrate/command/runas/output.txt similarity index 98% rename from acceptance/bundle/migrate/runas/output.txt rename to acceptance/bundle/migrate/command/runas/output.txt index 71faf2c5903..ac50c6617d1 100644 --- a/acceptance/bundle/migrate/runas/output.txt +++ b/acceptance/bundle/migrate/command/runas/output.txt @@ -51,6 +51,7 @@ Consider using a adding a top-level permissions section such as the following: See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. in databricks.yml:5:3 +Notice: automatically migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). >>> [CLI] pipelines get [UUID] diff --git a/acceptance/bundle/migrate/runas/script b/acceptance/bundle/migrate/command/runas/script similarity index 100% rename from acceptance/bundle/migrate/runas/script rename to acceptance/bundle/migrate/command/runas/script diff --git a/acceptance/bundle/migrate/runas/sql.sql b/acceptance/bundle/migrate/command/runas/sql.sql similarity index 100% rename from acceptance/bundle/migrate/runas/sql.sql rename to acceptance/bundle/migrate/command/runas/sql.sql diff --git a/acceptance/bundle/migrate/runas/test.toml b/acceptance/bundle/migrate/command/runas/test.toml similarity index 100% rename from acceptance/bundle/migrate/runas/test.toml rename to acceptance/bundle/migrate/command/runas/test.toml diff --git a/acceptance/bundle/migrate/var_arg/databricks.yml b/acceptance/bundle/migrate/command/var_arg/databricks.yml similarity index 100% rename from acceptance/bundle/migrate/var_arg/databricks.yml rename to acceptance/bundle/migrate/command/var_arg/databricks.yml diff --git a/acceptance/bundle/migrate/var_arg/notebook.py b/acceptance/bundle/migrate/command/var_arg/notebook.py similarity index 100% rename from acceptance/bundle/migrate/var_arg/notebook.py rename to acceptance/bundle/migrate/command/var_arg/notebook.py diff --git a/acceptance/bundle/migrate/command/var_arg/out.test.toml b/acceptance/bundle/migrate/command/var_arg/out.test.toml new file mode 100644 index 00000000000..27ec2a7fcd6 --- /dev/null +++ b/acceptance/bundle/migrate/command/var_arg/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/var_arg/output.txt b/acceptance/bundle/migrate/command/var_arg/output.txt similarity index 100% rename from acceptance/bundle/migrate/var_arg/output.txt rename to acceptance/bundle/migrate/command/var_arg/output.txt diff --git a/acceptance/bundle/migrate/var_arg/script b/acceptance/bundle/migrate/command/var_arg/script similarity index 100% rename from acceptance/bundle/migrate/var_arg/script rename to acceptance/bundle/migrate/command/var_arg/script diff --git a/acceptance/bundle/migrate/command/var_arg/test.toml b/acceptance/bundle/migrate/command/var_arg/test.toml new file mode 100644 index 00000000000..a030353d571 --- /dev/null +++ b/acceptance/bundle/migrate/command/var_arg/test.toml @@ -0,0 +1 @@ +RecordRequests = false diff --git a/acceptance/bundle/state/state_present/output.txt b/acceptance/bundle/state/state_present/output.txt index 38d568259f6..0a511b4dc9f 100644 --- a/acceptance/bundle/state/state_present/output.txt +++ b/acceptance/bundle/state/state_present/output.txt @@ -11,10 +11,11 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat" >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle plan +Notice: automatically migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_requests.py --get //api/2.1/unity-catalog/schemas -"databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat" +"cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID] interactive/none engine/direct auth/pat" === Adding resources.json with lower serial does not change anything >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan @@ -24,6 +25,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle plan +Notice: automatically migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Adding resources.json with higher serial, env var mismatch warns but uses state engine @@ -56,19 +58,17 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy -Warn: Direct engine selected via DATABRICKS_BUNDLE_ENGINE environment variable but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy. +Migrated 1 resource to direct deployment engine. Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created schemas.foo Files: 3 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -Migrating state to direct deployment engine (selected via DATABRICKS_BUNDLE_ENGINE environment variable)... -Migrated 1 resource to direct deployment engine. >>> print_requests.py //api/2.1/unity-catalog/schemas -"databricks-tf-provider/[TF_PROVIDER_VERSION] databricks-sdk-go/[SDK_VERSION] go/1.25.8 os/[OS] cli/[CLI_VERSION] terraform/1.5.5 sdk/sdkv2 resource/schema auth/pat" +"cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] interactive/none engine/direct auth/pat" >>> print_state.py -7 +6 >>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... @@ -85,11 +85,11 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged "cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] interactive/none engine/direct auth/pat" >>> print_state.py -7 +6 >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states -[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=7 lineage="test-lineage" +[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=6 lineage="test-lineage" >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states --force-pull -resources.json: remote direct state serial=7 lineage="test-lineage" -[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=7 lineage="test-lineage" +resources.json: remote direct state serial=6 lineage="test-lineage" +[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=6 lineage="test-lineage" diff --git a/acceptance/bundle/state/state_present/script b/acceptance/bundle/state/state_present/script index 5607b0baec3..76fe0af6a16 100644 --- a/acceptance/bundle/state/state_present/script +++ b/acceptance/bundle/state/state_present/script @@ -8,9 +8,9 @@ echo '{"version": 4, "serial": 1, "lineage": "test-lineage"}' > .databricks/bund trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy trace print_requests.py //api/2.1/unity-catalog/schemas | jq '.headers["User-Agent"][0]' | contains.py 'terraform' 'tf-provider' '!direct' -# An empty env var still reads the engine off the state, so plan runs on terraform. +# An empty env var defaults to direct, which migrates the terraform state in memory (no commit), so plan runs on direct. trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -trace print_requests.py --get //api/2.1/unity-catalog/schemas | jq '.headers["User-Agent"][0]' | contains.py 'terraform' 'tf-provider' '!direct' +trace print_requests.py --get //api/2.1/unity-catalog/schemas | jq '.headers["User-Agent"][0]' | contains.py 'engine/direct' '!tf-provider' title 'Adding resources.json with lower serial does not change anything' rm .databricks/bundle/default/terraform/terraform.tfstate @@ -31,17 +31,17 @@ trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle destroy --auto-approve rm "$OUT_REQUESTS" # The destroy above ran on the higher-serial direct state and removed it, leaving only -# the terraform state the deploy created, so this deploy runs on terraform and migrates. +# the terraform state the deploy created. This deploy migrates it to direct and commits, so it runs on direct. trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy -trace print_requests.py //api/2.1/unity-catalog/schemas | jq '.headers["User-Agent"][0]' | contains.py 'terraform' 'tf-provider' '!engine/direct' -trace print_state.py | jq .serial | contains.py "7" +trace print_requests.py //api/2.1/unity-catalog/schemas | jq '.headers["User-Agent"][0]' | contains.py 'engine/direct' '!tf-provider' +trace print_state.py | jq .serial | contains.py "6" trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy #trace print_requests.py --get //api/2.1/unity-catalog/schemas | jq '.headers["User-Agent"][0]' | contains.py 'engine/direct' '!terraform' '!tf-provider' trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle deploy trace print_requests.py --get //api/2.1/unity-catalog/schemas | jq '.headers["User-Agent"][0]' | contains.py 'engine/direct' '!terraform' '!tf-provider' -trace print_state.py | jq .serial | contains.py "7" +trace print_state.py | jq .serial | contains.py "6" trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle debug states trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle debug states --force-pull diff --git a/acceptance/bundle/state/state_present/test.toml b/acceptance/bundle/state/state_present/test.toml index 041121b226d..9d5ef1f1b48 100644 --- a/acceptance/bundle/state/state_present/test.toml +++ b/acceptance/bundle/state/state_present/test.toml @@ -4,3 +4,5 @@ EnvMatrix.DMS = [""] RecordRequests = true IncludeRequestHeaders = ["User-Agent"] + +Ignore = [".databricks"] diff --git a/bundle/bundle.go b/bundle/bundle.go index ce158b3e26c..ff4d2cd579f 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -209,6 +209,14 @@ type Bundle struct { // which runs on the migrated state, creates them. MigratingToDirect bool + // MigrationDeferred is set when a deploy prepared the terraform→direct migration in + // memory but has not committed it yet: the deploy commits the converted state itself + // (deployCore) only once it is approved, and finalizes the migration (terraform-state + // cleanup) afterwards. A declined deploy discards it, leaving the terraform state in + // place. Distinguishes the deferred deploy path from destroy's early commit and from a + // trivial empty-state migration, both of which commit up front. + MigrationDeferred bool + // Quiet is the output verbosity reduction requested via -q/--quiet, which is // repeatable: QuietSummary drops the per-resource lines, QuietAll additionally // drops the summary and progress lines, leaving warnings and errors. diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 00fe443ccc5..ff363ebbe24 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -872,6 +872,37 @@ func (db *DeploymentState) UpgradeToWrite() error { return appendJSONLine(db.walFile, walHead) } +// DiscardWrite abandons a state opened for write without persisting it: it closes and +// removes the WAL and resets the in-memory state, so nothing lands on disk. Used when a +// deploy that opened the state for write is then declined - e.g. a deferred terraform→direct +// migration whose deploy the user did not approve - so no resources.json is written and the +// run stays on the previous engine, and no orphan WAL is left to block the next attempt. +func (db *DeploymentState) DiscardWrite(ctx context.Context) { + db.mu.Lock() + defer db.mu.Unlock() + + if db.walFile != nil { + if err := db.walFile.Close(); err != nil { + log.Warnf(ctx, "Error closing WAL file while discarding state: %s", err) + } + db.walFile = nil + } + if db.Path != "" { + _ = os.Remove(db.Path + walSuffix) + } + db.reset() +} + +// Persist writes the current in-memory state to its file immediately, without a WAL cycle. +// Used to materialize a freshly opened state that produces no WAL entries - e.g. an empty +// migrated state - as a base file on disk, so a later write has a base and Finalize's +// header-only WAL leaves it intact. +func (db *DeploymentState) Persist() error { + db.mu.Lock() + defer db.mu.Unlock() + return db.unlockedSave() +} + // IsOpen reports whether the state has been opened (for read or write). It lets // callers probe the state without risking the panic in AssertOpenedForReadOrWrite, // e.g. code paths shared with the terraform engine where the state DB is never opened. diff --git a/bundle/metrics/metrics.go b/bundle/metrics/metrics.go index 27169e44a10..73241aa2882 100644 --- a/bundle/metrics/metrics.go +++ b/bundle/metrics/metrics.go @@ -34,6 +34,10 @@ const ( DirectMigrateWarnings = "direct_migrate_warnings" // True when the post-convert plan check failed; the migration was not committed. DirectMigratePlanError = "direct_migrate_plan_error" + // True when the migrated state's first plan would recreate (destroy + create) an + // existing resource. The migration still commits; the recreate is a genuine pending + // change gated by the deploy's --auto-approve, and this only records how often it happens. + DirectMigrateRecreatePlanned = "direct_migrate_recreate_planned" // Recorded when an automatic post-deploy migration to the direct engine // actually ran (state was rewritten). Exactly one of the three keys is true; diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index 61a00ac18ed..ac67bd02ee6 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -205,6 +205,13 @@ func BuildStateFromTF( } } + // Reconcile id-composing fields with the deployed terraform state so a pending id + // change surfaces in the plan instead of being snapshotted as already applied. A no-op + // for permissions/grants sub-nodes: their adapters declare no id fields. + if err := reconcileIDFields(ctx, adapter, srcGroup, srcName, sv.Value, tfAttrs, warnPrefix); err != nil { + return warningsSeen, fmt.Errorf("%s: reconciling id fields: %w", node, err) + } + // Compact hashed_fields fields so the migrated state stays small. Not needed for // correctness — the first plan (CalculatePlan) compacts the saved state on read. compacted, err := dresources.CompactState(adapter.ResourceConfig(), sv.Value) @@ -219,3 +226,69 @@ func BuildStateFromTF( return warningsSeen, nil } + +// reconcileIDFields aligns each id-composing field (provided_id_fields, updatable_id_fields) +// in the migrated state with the deployed terraform state. The state is otherwise seeded from +// config; for id fields, recording the current config would snapshot a pending change as +// already applied and silently drift from the backend, so: +// +// - When config and the deployed value differ only by backend normalization (identifier +// case, trailing slash), keep the config value: the difference is not a real change and +// recording either side converges. +// - Otherwise the user genuinely changed the id in config without deploying it. Record the +// deployed value so the migrated state matches what a direct deploy of the last-applied +// config would hold, and warn: the next plan surfaces the change as a recreate (provided-id +// fields) or a rename (updatable-id fields). +func reconcileIDFields(ctx context.Context, adapter *dresources.Adapter, group, name string, stateValue any, tfAttrs TFStateAttrs, warnPrefix string) error { + cfg := adapter.ResourceConfig() + if cfg == nil { + return nil + } + // A provided-id change recreates the resource; an updatable-id change renames it in place. + for _, kind := range []struct { + rules []dresources.FieldRule + action string + }{ + {cfg.ProvidedIDFields, "recreated"}, + {cfg.UpdatableIDFields, "renamed"}, + } { + for _, rule := range kind.rules { + path, err := structpath.ParsePath(rule.Field.String()) + if err != nil { + continue + } + configVal, err := structaccess.Get(stateValue, path) + if err != nil { + continue + } + deployedVal, err := LookupTFField(tfAttrs, group, name, path) + if err != nil { + continue + } + + // Every id-composing field is a string (a name, catalog_name, storage path, ...). + configStr, ok1 := configVal.(string) + deployedStr, ok2 := deployedVal.(string) + if !ok1 || !ok2 { + return fmt.Errorf("id field %q: expected string values, got config %T and deployed %T", rule.Field.String(), configVal, deployedVal) + } + + // UC identifier names are case-insensitive and UC strips trailing slashes from + // storage paths, so a config value that differs from the deployed one only by that + // normalization is not a real change: keep the config value (it converges). + if strings.EqualFold(strings.TrimRight(configStr, "/"), strings.TrimRight(deployedStr, "/")) { + continue + } + + // A genuine change the user has not deployed yet. Record the deployed value so the + // migrated state matches the last-applied config, and warn so the recreate/rename in + // the plan that follows the migration is not a surprise. + log.Warnf(ctx, "%s%s.%s: %s differs between config (%q) and terraform state (%q); this resource will be %s.", + warnPrefix, group, name, rule.Field.String(), configStr, deployedStr, kind.action) + if err := structaccess.Set(stateValue, path, deployedVal); err != nil { + return fmt.Errorf("setting id field %q: %w", rule.Field.String(), err) + } + } + } + return nil +} diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 8a8ef56be25..37f15b03328 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -323,6 +323,14 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // No version was created, so the deferred CompleteVersion is a no-op and the version // number is left for the next deploy. Both the user declining and a console that // cannot prompt land here. + // + // A deferred terraform→direct migration wrote the local direct state but has not + // committed it (deployCore below never ran): discard it and stay on the terraform + // engine, so a declined deploy changes nothing. + if b.MigrationDeferred { + statemgmt.DiscardDeferredMigration(ctx, b) + log.Warnf(ctx, "Migration not committed, keeping Terraform state") + } if err != nil { logdiag.LogError(ctx, err) return @@ -331,6 +339,13 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } + // Approved: the deploy is committing (deployCore below writes and pushes the state), so a + // prepared migration is no longer "not committed". Clear the flag before deployCore so + // ProcessBundleRet's cleanup does not discard it and a post-approval failure retries on + // direct rather than reverting to terraform; keep migrating for the finalize below. + migrating := b.MigrationDeferred + b.MigrationDeferred = false + // Create the deployment now that the plan is approved, so a declined deploy leaves none behind. // A first deploy's id did not exist at plan time - the version and any existing id were stamped // then - so stamp the one just created into the plan the apply reads. @@ -370,6 +385,13 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } + // A deferred terraform→direct migration is committed by deployCore above, which wrote and + // pushed the converted direct state. Now that the deploy succeeded, finalize the migration + // by cleaning up the superseded terraform state and recording the migration source. + if migrating { + statemgmt.FinalizeDeferredMigration(ctx, b, requestedEngine) + } + // Report what was deployed, mirroring "bundle plan". Printed before the // postdeploy script so the deploy's own report is not interleaved with // post-deploy output: the script's lines and the migration's below both follow @@ -381,24 +403,6 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand logDeploySummary(ctx, b, plan, stateEngine) bundle.ApplyContext(ctx, b, scripts.Execute(config.ScriptPostDeploy)) - - // Migrate the state to the direct engine, if the user opted in (via - // bundle.engine or DATABRICKS_BUNDLE_ENGINE) and a dry-run of the migration - // comes back clean. Without the opt-in, or when the dry-run reports problems, - // nothing is written: only the outcome is recorded in telemetry, and the - // deploy is unaffected. - // - // Last, after the deploy has reported what it did: this is post-deploy work, - // and its warnings read as belonging to the deploy if they precede the - // summary. - // - // Gated on the deploy alone, which the early return above already guarantees - // — not on the postdeploy script. The resources were applied before that - // script ran, so the state is worth migrating even if it failed, the same - // reasoning that prints the summary ahead of it. - if !stateEngine.IsDirect() { - statemgmt.MigrateToDirect(ctx, b, requestedEngine) - } } func RunPlan(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) *deployplan.Plan { diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 5cb2d615b58..487d9c23356 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -17,6 +17,7 @@ import ( "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dms" @@ -140,7 +141,7 @@ func approvalForDestroy(ctx context.Context, b *bundle.Bundle, plan *deployplan. return cmdio.AskYesOrNo(ctx, "Would you like to proceed?") } -func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, engine engine.EngineType) { +func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, engine engine.EngineType, migrating bool) { if engine.IsDirect() { // Not reported per resource: destroy names them up front for consent and then // reports only a count, so there is no per-resource output to report into. @@ -206,6 +207,19 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, e return } + // A destroy that migrated from terraform left the superseded local terraform state behind + // (files.Delete already removed the remote one). Remove it before the direct state below: a + // crash between the two must never leave a live local terraform.tfstate with no direct state, + // which the next deploy would pick up. Gated on migrating, not just engine.IsDirect(): a + // non-migrating direct destroy may run alongside a separate, still-valid terraform state that + // a later deploy is meant to migrate, and must not delete it. + if migrating { + _, localTerraformPath := b.StateFilenameTerraform(ctx) + if err := os.Remove(localTerraformPath); err != nil && !errors.Is(err, fs.ErrNotExist) { + logdiag.LogError(ctx, err) + } + } + // Remove the local state file now that the deployment is gone. Destroy only // deletes the remote state; leaving the local file behind keeps its lineage // around, so a later fresh deploy of the same bundle (e.g. from another @@ -363,6 +377,14 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { } if hasApproval { + // Approved: the destroy is committing (destroyCore runs on the direct state), so a + // prepared migration is no longer "not committed". Clear the flag so ProcessBundleRet's + // cleanup does not discard it, and a post-approval failure keeps the direct state rather + // than reverting to terraform. Keep migrating so destroyCore retires the superseded + // terraform state only when this destroy actually migrated. + migrating := b.MigrationDeferred + b.MigrationDeferred = false + if engine.IsDirect() { // Upgrade from read (opened by process.go) to write mode if err := b.DeploymentBundle.StateDB.UpgradeToWrite(); err != nil { @@ -383,8 +405,15 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { return } } - destroyCore(ctx, b, plan, engine) + destroyCore(ctx, b, plan, engine, migrating) } else { + // A deferred terraform→direct migration wrote the local direct state but has not + // committed it (destroyCore never ran): discard it and stay on the terraform engine, + // so a declined destroy changes nothing. + if b.MigrationDeferred { + statemgmt.DiscardDeferredMigration(ctx, b) + log.Warnf(ctx, "Migration not committed, keeping Terraform state") + } cmdio.LogString(ctx, "Destroy cancelled!") } } diff --git a/bundle/statemgmt/direct_migration.go b/bundle/statemgmt/direct_migration.go index bbfcef47bfc..2aec9554a30 100644 --- a/bundle/statemgmt/direct_migration.go +++ b/bundle/statemgmt/direct_migration.go @@ -8,12 +8,13 @@ import ( "io/fs" "os" "path/filepath" + "slices" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/config/mutator/resourcemutator" - "github.com/databricks/cli/bundle/deploy" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" @@ -21,144 +22,250 @@ import ( "github.com/databricks/cli/bundle/migrate" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" ) -// warnPrefix labels warnings emitted by the post-deploy dry-run so they are not -// confused with warnings from the user-invoked `bundle migrate` command. -const warnPrefix = "post-deploy dry-run migration to direct: " - -// feedbackNotice is appended after the dry-run warnings to reassure the user the -// deploy is unaffected and to ask them to report the warnings. -const feedbackNotice = `The warnings above are from a dry-run migration to the direct deployment engine (https://docs.databricks.com/aws/en/dev-tools/bundles/direct). -Your deployment is not affected and works normally, but you may experience these issues when migrating to the direct deployment engine. -Please forward these warnings to dabs-feedback@databricks.com` - -// autoMigrateStoppedNotice is emitted when the direct engine is selected but the -// dry-run migration surfaced errors or warnings, so the automatic post-deploy -// migration is skipped. -const autoMigrateStoppedNotice = `Direct engine was selected but the migration reported issues; automatic migration to the direct deployment engine is stopped. Address the issues above or run "databricks bundle deployment migrate" manually.` +// warnPrefix labels warnings emitted while converting terraform state to the direct +// engine, distinguishing them from the user-invoked "bundle deployment migrate". +const warnPrefix = "migration to direct: " + +// MigrateMode selects how MigrateTerraformState commits the converted state. +type MigrateMode int + +const ( + // MigratePlan converts the state and loads it into memory only (for `bundle plan` and + // `bundle run`): nothing is written or pushed and no plan check runs. + MigratePlan MigrateMode = iota + + // MigrateDeferred converts the state, plan-checks it (falling back to terraform if the + // check fails), writes the local direct state file, and sets b.MigrationDeferred. Used by + // the commands that apply changes (deploy, destroy): the command commits the state itself + // once it is approved (deployCore / destroyCore) and then calls FinalizeDeferredMigration + // to clean up the terraform state; a declined command discards it (DiscardDeferredMigration) + // and stays on terraform. + MigrateDeferred +) -// MigrateToDirect performs a dry-run migration of the just-deployed terraform -// state to the direct engine and records the outcome in deploy telemetry. +// MigrateTerraformState converts the bundle's terraform state to a direct-engine state +// and opens b.DeploymentBundle.StateDB with it. Returns false when there is no terraform +// state to migrate (the caller then opens the direct state normally). // -// The converted state is written to a temporary file. If the dry-run is clean -// and requestedEngine resolves to "direct" (which is the default, and can also -// be set explicitly via bundle.engine or the DATABRICKS_BUNDLE_ENGINE env var), -// the temp state is committed (renamed to resources.json, terraform.tfstate is -// backed up, and the new state is pushed to the workspace). Otherwise the temp -// state is deleted and only telemetry is recorded. Any failure is surfaced as a -// warning so it never fails a deploy that already succeeded. -func MigrateToDirect(ctx context.Context, b *bundle.Bundle, requestedEngine engine.EngineSetting) { +// MigratePlan loads the converted state in memory and writes nothing. MigrateDeferred also +// plan-checks it, writes the local direct state file, and sets b.MigrationDeferred so the +// approved command commits it and calls FinalizeDeferredMigration; a declined command calls +// DiscardDeferredMigration. The workspace is never touched here - it is committed by the +// command's own state push after approval. +// +// Any failure (parsing, conversion, the plan check) is non-fatal: a warning is emitted, false +// is returned, and the caller proceeds on the terraform engine, which is still in place - so a +// failed migration never blocks a command that would succeed. An empty terraform state goes +// through the same path: the converter writes an empty base state file, so there is no special +// case here. +func MigrateTerraformState(ctx context.Context, b *bundle.Bundle, requiredEngine engine.EngineSetting, mode MigrateMode) (bool, error) { _, localTerraformPath := b.StateFilenameTerraform(ctx) tfState, err := migrate.ParseTFStateFull(ctx, localTerraformPath) if err != nil { - log.Warnf(ctx, "%sfailed to parse terraform state: %v", warnPrefix, err) - if requestedEngine.Type == engine.EngineDirect { - b.Metrics.SetBoolValue(metrics.DirectMigrateError, true) - log.Warnf(ctx, "%s", autoMigrateStoppedNotice) - } else { - b.Metrics.SetBoolValue(metrics.DirectDryMigrateSuccess, false) - } - return + b.Metrics.SetBoolValue(metrics.DirectMigrateError, true) + log.Warnf(ctx, "could not parse terraform state for migration to the direct engine; deploying on terraform this time: %v", err) + return false, nil } - if tfState == nil { - // No terraform state file to migrate; nothing to do either way. - return + return false, nil } - // A terraform.tfstate that has no databricks_* resources AND no managed - // resources of any kind has no state to migrate. Gate the sweep on both: - // Attrs is empty when the file has zero managed resources; IDs is empty - // when nothing maps to a known DABs group. Sweeping when Attrs is - // non-empty (unknown TF resource types in the state) would destroy state - // the CLI doesn't recognize, so restrict the sweep to the truly-empty - // case and let the populated path handle everything else. - if len(tfState.IDs) == 0 && len(tfState.Attrs) == 0 { - if requestedEngine.Type != engine.EngineDirect { - recordDryRunNoop(b, requestedEngine) - return - } - cmdio.LogString(ctx, "Removing empty terraform state; direct engine will be used on the next deploy (selected via "+requestedEngine.Source+")...") - if err := backupTerraformState(ctx, b); err != nil { - b.Metrics.SetBoolValue(metrics.DirectMigrateCommitError, true) - log.Warnf(ctx, "automatic migration to direct engine failed: %v", err) - return - } - recordAutoMigrateSource(b, requestedEngine) - return - } + _, localDirectPath := b.StateFilenameDirect(ctx) tempStatePath, resourceCount, hasWarnings, cfg, err := convertTFStateToDirect(ctx, b, tfState) if tempStatePath != "" { - // The temp file sits next to the real resources.json path (same - // filesystem, so finalizeLocalMigration's os.Rename works even when - // os.TempDir() is on a different volume). Clean up the temp file - // and its WAL sibling — finalizeLocalMigration renames the state file out - // of the way on success, so these Removes are no-ops in that case. + // Always remove the temp state and its WAL. A successful commit renames the temp + // file into place first, so these become no-ops; on any failure they clean up the + // leftovers so a later run's UpgradeToWrite (which opens the WAL with O_EXCL) does + // not trip over them. defer func() { _ = os.Remove(tempStatePath) _ = os.Remove(tempStatePath + ".wal") }() } - - if err != nil { - log.Warnf(ctx, "%s%v", warnPrefix, err) + if hasWarnings { + b.Metrics.SetBoolValue(metrics.DirectMigrateWarnings, true) } - if hasWarnings || err != nil { - log.Warnf(ctx, "%s", feedbackNotice) + if err != nil { + b.Metrics.SetBoolValue(metrics.DirectMigrateError, true) + log.Warnf(ctx, "could not convert terraform state to the direct engine; deploying on terraform this time: %v", err) + return false, nil + } + + // Plan-check the converted state for deploy and destroy (not plan): validate it can be + // planned - falling back to terraform if not - and record the recreate metric. + if mode == MigrateDeferred { + plan, err := checkPlanOnTempState(ctx, b, tempStatePath, cfg) + if err != nil { + b.Metrics.SetBoolValue(metrics.DirectMigratePlanError, true) + log.Warnf(ctx, "migration to the direct engine failed its plan check; deploying on terraform this time: %v", err) + return false, nil + } + + // Record when the migrated state's first plan would recreate a resource, but still + // migrate: such a recreate is a genuine pending config change (the same one + // terraform would apply), and the command's approval flow gates it behind + // --auto-approve exactly like any other recreate. The metric lets us observe how + // often a migration carries a recreate. + if recreated := recreatedResources(plan); len(recreated) > 0 { + b.Metrics.SetBoolValue(metrics.DirectMigrateRecreatePlanned, true) + log.Infof(ctx, "migration to the direct engine will recreate %v; the command's approval gates this", recreated) + } } - // The direct engine was not selected (the user opted out with - // engine: terraform) — the conversion was only a dry run for fleet-wide - // telemetry, so record dry-run outcome only. - if requestedEngine.Type != engine.EngineDirect { - b.Metrics.SetBoolValue(metrics.DirectDryMigrateSuccess, err == nil) - b.Metrics.SetBoolValue(metrics.DirectDryMigrateWarnings, hasWarnings) - return + if mode == MigratePlan { + // Plan never persists the state, so load the converted state into memory only. It was + // just written by this CLI, so it is at the current schema version and needs no migration. + raw, err := os.ReadFile(tempStatePath) + if err != nil { + return false, fmt.Errorf("reading migrated state: %w", err) + } + var data dstate.Database + if err := json.Unmarshal(raw, &data); err != nil { + return false, fmt.Errorf("parsing migrated state: %w", err) + } + b.DeploymentBundle.StateDB.OpenWithData(localDirectPath, data) + return true, nil } - // From here on, direct is the engine to migrate to: use the migrate_* telemetry keys. - if err != nil { - b.Metrics.SetBoolValue(metrics.DirectMigrateError, true) + // MigrateDeferred: place the converted state in the local direct state file so the command + // (deployCore / destroyCore) has a base to persist even when the deploy is a no-op - an + // in-memory state is dropped by Finalize when there are no operations. The remote push and + // terraform-state cleanup are deferred to the approved command (its own state push + + // FinalizeDeferredMigration), so the workspace is untouched until then; a declined command + // removes this local file (DiscardDeferredMigration) and stays on terraform. + if err := os.MkdirAll(filepath.Dir(localDirectPath), 0o700); err != nil { + return false, fmt.Errorf("creating local state directory for migration: %w", err) } - if hasWarnings { - b.Metrics.SetBoolValue(metrics.DirectMigrateWarnings, true) + if err := os.Rename(tempStatePath, localDirectPath); err != nil { + return false, fmt.Errorf("writing local direct state for migration: %w", err) + } + // The local direct state now exists but is not committed. Mark it so any exit before the + // command commits - a decline, or any pre-apply error (including the Open below) - discards + // it (see DiscardDeferredMigration, invoked from ProcessBundleRet's deferred cleanup) and + // the run stays on terraform. + b.MigrationDeferred = true + if err := b.DeploymentBundle.StateDB.Open(ctx, localDirectPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{}); err != nil { + return false, fmt.Errorf("opening migrated local state: %w", err) } - if err != nil || hasWarnings { - log.Warnf(ctx, "%s", autoMigrateStoppedNotice) - return + // The command commits this state once approved and then FinalizeDeferredMigration cleans up + // terraform. Report the migration now; a not-committed run adds a "not committed" warning. + suffix := "s" + if resourceCount == 1 { + suffix = "" } + cmdio.LogString(ctx, fmt.Sprintf("Migrated %d resource%s to direct deployment engine.", resourceCount, suffix)) + return true, nil +} - if planErr := checkPlanOnTempState(ctx, b, tempStatePath, cfg); planErr != nil { - log.Warnf(ctx, "%s%v", warnPrefix, planErr) - log.Warnf(ctx, "%s", feedbackNotice) - b.Metrics.SetBoolValue(metrics.DirectMigratePlanError, true) - log.Warnf(ctx, "%s", autoMigrateStoppedNotice) +// CleanupTerraformStateAfterMigration backs up the now-superseded terraform state, remote and +// local, best-effort. Called after a deploy commits the migrated direct state (via +// FinalizeDeferredMigration), so a later run does not pick up the stale terraform state (its +// resources are gone) instead of the direct one. Destroy does not use this: it removes the local +// terraform state directly in destroyCore and files.Delete removes the remote one. +func CleanupTerraformStateAfterMigration(ctx context.Context, b *bundle.Bundle) { + BackupRemoteTerraformState(ctx, b) + _, localTerraformPath := b.StateFilenameTerraform(ctx) + if err := os.Rename(localTerraformPath, localTerraformPath+".backup"); err != nil && !errors.Is(err, fs.ErrNotExist) { + log.Warnf(ctx, "automatic migration to direct engine: could not back up local terraform state: %v", err) + } +} + +// FinalizeDeferredMigration completes a deploy's MigrateDeferred migration after deployCore has +// committed the converted direct state: it cleans up the terraform state and records which +// source triggered the migration. The deploy phase calls it only once the deploy is approved +// and applied, so a declined deploy leaves the terraform state untouched. The "Migrated N +// resources" summary was already printed when the converted state was prepared. (Destroy does not +// call this: it retires the terraform state in destroyCore, and a destroy that migrates only to +// tear down is not migration adoption worth recording as a source.) +func FinalizeDeferredMigration(ctx context.Context, b *bundle.Bundle, requiredEngine engine.EngineSetting) { + CleanupTerraformStateAfterMigration(ctx, b) + recordAutoMigrateSource(b, requiredEngine) +} + +// DiscardDeferredMigration undoes a MigrateDeferred migration that was never committed - the +// command was declined or failed before it applied anything. It removes the local direct state +// file and WAL the migration wrote and resets the in-memory state, so nothing is committed and +// the run stays on the terraform state (the remote workspace was never touched). Clearing +// b.MigrationDeferred makes it idempotent: ProcessBundleRet's deferred cleanup calls it on any +// non-committed exit, and the deploy/destroy decline paths call it explicitly. +func DiscardDeferredMigration(ctx context.Context, b *bundle.Bundle) { + if !b.MigrationDeferred { return } + b.MigrationDeferred = false + // DiscardWrite closes and removes the WAL and resets the in-memory state. + b.DeploymentBundle.StateDB.DiscardWrite(ctx) + _, localDirectPath := b.StateFilenameDirect(ctx) + if err := os.Remove(localDirectPath); err != nil && !errors.Is(err, fs.ErrNotExist) { + log.Warnf(ctx, "could not remove local direct state after a discarded migration: %v", err) + } +} - cmdio.LogString(ctx, "Migrating state to direct deployment engine (selected via "+requestedEngine.Source+")...") +// recordAutoMigrateSource sets exactly one of the migrated-via-* telemetry keys, per how +// the direct engine was selected. ConfigType is set only when bundle.engine populated it, +// so it distinguishes a durable opt-in (via_config) from an env-var-only one (via_env); +// IsDefault covers the population that asked for nothing. +func recordAutoMigrateSource(b *bundle.Bundle, requiredEngine engine.EngineSetting) { + switch { + case requiredEngine.IsDefault: + b.Metrics.SetBoolValue(metrics.DirectAutoMigrateViaDefault, true) + case requiredEngine.ConfigType == engine.EngineDirect: + b.Metrics.SetBoolValue(metrics.DirectAutoMigrateViaConfig, true) + default: + b.Metrics.SetBoolValue(metrics.DirectAutoMigrateViaEnv, true) + } +} - if err := commitMigration(ctx, b, tempStatePath, resourceCount); err != nil { - b.Metrics.SetBoolValue(metrics.DirectMigrateCommitError, true) - log.Warnf(ctx, "automatic migration to direct engine failed: %v", err) +// DryRunMigrationTelemetry converts the terraform state to the direct engine WITHOUT +// committing, purely to record direct_drymigrate_* telemetry for deploys that opted out +// of the direct engine (engine: terraform). It mirrors the actual migration's conversion +// (but never runs the plan check, and never touches any state) so the fleet-wide "could +// this bundle migrate?" signal is preserved. It is called after a terraform deploy, so +// mutating b.Config during the conversion is harmless, and it swallows failures because +// the deploy already succeeded. DirectDryMigrateSuccess reflects only whether the state +// conversion succeeded. +func DryRunMigrationTelemetry(ctx context.Context, b *bundle.Bundle) { + _, localTerraformPath := b.StateFilenameTerraform(ctx) + tfState, err := migrate.ParseTFStateFull(ctx, localTerraformPath) + if err != nil { + b.Metrics.SetBoolValue(metrics.DirectDryMigrateSuccess, false) + return + } + if tfState == nil { + return + } + // An empty terraform state has nothing to convert, so the dry run trivially succeeds. + if len(tfState.IDs) == 0 && len(tfState.Attrs) == 0 { + b.Metrics.SetBoolValue(metrics.DirectDryMigrateSuccess, true) + b.Metrics.SetBoolValue(metrics.DirectDryMigrateWarnings, false) return } - recordAutoMigrateSource(b, requestedEngine) + tempStatePath, _, hasWarnings, _, err := convertTFStateToDirect(ctx, b, tfState) + if tempStatePath != "" { + defer func() { + _ = os.Remove(tempStatePath) + _ = os.Remove(tempStatePath + ".wal") + }() + } + b.Metrics.SetBoolValue(metrics.DirectDryMigrateSuccess, err == nil) + b.Metrics.SetBoolValue(metrics.DirectDryMigrateWarnings, hasWarnings) } // checkPlanOnTempState opens the migrated state at tempStatePath in read mode, -// runs a full plan against it, and returns a non-nil error if the plan fails. -// Individual planning errors are emitted as warnings with warnPrefix so they -// are visible without failing the deploy. The plan is run in an isolated -// context so its diagnostics do not affect the deploy's own error state. -func checkPlanOnTempState(ctx context.Context, b *bundle.Bundle, tempStatePath string, cfg *config.Root) error { +// runs a full plan against it, and returns the plan (and a non-nil error if the +// plan fails). Individual planning errors are emitted as warnings with warnPrefix +// so they are visible without failing the deploy. The plan is run in an isolated +// context so its diagnostics do not affect the deploy's own error state. The +// returned plan lets the caller inspect the planned actions (e.g. reject a +// migration that would recreate a resource). +func checkPlanOnTempState(ctx context.Context, b *bundle.Bundle, tempStatePath string, cfg *config.Root) (*deployplan.Plan, error) { planCtx := logdiag.IsolatedContext(ctx) logdiag.SetCollect(planCtx, true) defer func() { @@ -176,87 +283,37 @@ func checkPlanOnTempState(ctx context.Context, b *bundle.Bundle, tempStatePath s // This plan is not created with the deployment history feature enabled, // so we can safely pass false for withDeploymentHistory. if err := planBundle.StateDB.Open(planCtx, tempStatePath, false, false, dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{}); err != nil { - return fmt.Errorf("opening migrated state for plan check: %w", err) + return nil, fmt.Errorf("opening migrated state for plan check: %w", err) } - _, err := planBundle.CalculatePlan(planCtx, b.WorkspaceClient(ctx), cfg) - return err -} - -// recordDryRunNoop records dry-run telemetry for a no-op case (no state, or -// state with no managed resources) when direct was NOT selected. On the -// migrating paths the caller uses direct_migrate_* keys instead. -func recordDryRunNoop(b *bundle.Bundle, requestedEngine engine.EngineSetting) { - if requestedEngine.Type == engine.EngineDirect { - return - } - b.Metrics.SetBoolValue(metrics.DirectDryMigrateSuccess, true) - b.Metrics.SetBoolValue(metrics.DirectDryMigrateWarnings, false) -} - -// recordAutoMigrateSource sets exactly one of the migrated-via-* telemetry -// keys. requestedEngine.Type may resolve to direct from the config, the env var, -// or the default (config wins over env in ResolveEngineSetting). ConfigType is -// set only when the config populated the setting, so it's the correct signal for -// "was this a durable opt-in?" — env-only opt-ins are the ones with -// ConfigType == EngineNotSet and IsDefault false. -func recordAutoMigrateSource(b *bundle.Bundle, requestedEngine engine.EngineSetting) { - switch { - case requestedEngine.IsDefault: - b.Metrics.SetBoolValue(metrics.DirectAutoMigrateViaDefault, true) - case requestedEngine.ConfigType == engine.EngineDirect: - b.Metrics.SetBoolValue(metrics.DirectAutoMigrateViaConfig, true) - default: - b.Metrics.SetBoolValue(metrics.DirectAutoMigrateViaEnv, true) - } + return planBundle.CalculatePlan(planCtx, b.WorkspaceClient(ctx), cfg) } -// backupTerraformState moves the terraform state to .backup both remotely -// (read → write .backup → delete) and locally (rename). Every step must -// succeed, so callers get an accurate error path — a stale terraform state -// left anywhere lets it win over remote direct in PullResourcesState when -// AlwaysPull is off. Missing files (both local and remote) are treated as -// no-ops, so this helper is safe to call whether or not any state exists. -// Contrast with BackupRemoteTerraformState, which only handles the remote -// half and swallows errors via log.Warnf for best-effort direct-engine -// cleanup on unrelated code paths. -func backupTerraformState(ctx context.Context, b *bundle.Bundle) error { - f, err := deploy.StateFiler(ctx, b) - if err != nil { - return err - } - remoteTerraformPath, localTerraformPath := b.StateFilenameTerraform(ctx) - reader, err := f.Read(ctx, remoteTerraformPath) - if err != nil && !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("reading remote terraform state %s: %w", remoteTerraformPath, err) - } - if err == nil { - defer reader.Close() - if err := f.Write(ctx, remoteTerraformPath+".backup", reader, filer.OverwriteIfExists); err != nil { - return fmt.Errorf("writing remote terraform backup: %w", err) - } - if err := f.Delete(ctx, remoteTerraformPath); err != nil { - return fmt.Errorf("deleting remote terraform state: %w", err) +// recreatedResources returns the sorted keys of resources the plan would recreate +// (destroy + create). Used only to record the direct_migrate_recreate_planned metric: +// the migration proceeds regardless, and the deploy's approval flow gates the recreate +// behind --auto-approve just like any other recreate. +func recreatedResources(plan *deployplan.Plan) []string { + var keys []string + for key, entry := range plan.Plan { + if entry.Action == deployplan.Recreate { + keys = append(keys, key) } } - - if err := os.Rename(localTerraformPath, localTerraformPath+".backup"); err != nil && !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("renaming local terraform state to %s.backup: %w", localTerraformPath, err) - } - return nil + slices.Sort(keys) + return keys } // convertTFStateToDirect converts the given terraform state to the direct engine state, // returning the path to the converted state file, the number of resources // migrated, whether any warnings were emitted, and the bundle config with // terraform interpolation reversed (needed by the caller to run a plan against -// the converted state). Callers must ensure tfState is non-nil and has at least -// one resource ID (the empty and nil cases are handled by MigrateToDirect -// directly, since they take different commit paths). The caller is responsible -// for deleting the temp state's parent directory when it is done with the file. +// the converted state). Callers must ensure tfState is non-nil; an empty state +// (no resource IDs or attrs) yields an empty base state file. The caller is +// responsible for deleting the temp state's parent directory when it is done with the file. func convertTFStateToDirect(ctx context.Context, b *bundle.Bundle, tfState *migrate.TFState) (string, int, bool, *config.Root, error) { // Write the converted state to a sibling of the final resources.json - // path so finalizeLocalMigration's os.Rename stays within one filesystem + // path so commitMigration's os.Rename stays within one filesystem // (os.TempDir() often lives on a different volume from the project; // cross-filesystem Rename fails with EXDEV). The state DB creates the // file and its .wal itself, so a deterministic sibling name is enough @@ -286,20 +343,19 @@ func convertTFStateToDirect(ctx context.Context, b *bundle.Bundle, tfState *migr } }() - state := make(map[string]dstate.ResourceEntry) - for key, id := range tfState.IDs { - state[key] = dstate.ResourceEntry{ - ID: id, - State: json.RawMessage("{}"), + var stateDB dstate.DeploymentState + stateDB.OpenWithData(tempStatePath, dstate.NewDatabase(tfState.Lineage, tfState.Serial+1)) + + // An empty terraform state seeds and builds no WAL entries below, so Finalize would persist + // no file. Write the base file now so the migration always yields one; the deferred commit + // builds on it and a crash mid-apply stays recoverable. The header-only Finalize leaves it + // intact. + if len(tfState.IDs) == 0 && len(tfState.Attrs) == 0 { + if err := stateDB.Persist(); err != nil { + return tempStatePath, resourceCount, false, nil, fmt.Errorf("persisting empty migrated state: %w", err) } } - migratedDB := dstate.NewDatabase(tfState.Lineage, tfState.Serial+1) - migratedDB.State = state - - var stateDB dstate.DeploymentState - stateDB.OpenWithData(tempStatePath, migratedDB) - // Apply SecretScopeFixups so the config matches what the direct engine expects. // This adds MANAGE ACL for the current user to all secret scopes, ensuring // the migrated state and config agree on .permissions entries. @@ -333,6 +389,19 @@ func convertTFStateToDirect(ctx context.Context, b *bundle.Bundle, tfState *migr return tempStatePath, resourceCount, false, nil, fmt.Errorf("upgrading state for apply: %w", err) } + // Seed every terraform-state resource into the WAL so the migrated state persists + // all deployed resources, including ones the current config no longer declares. + // BuildStateFromTF overwrites the config-declared entries below with their full + // state (the later WAL entry wins on replay); the rest keep this minimal entry, + // which is enough for the first direct plan to delete them. Without this, a config + // that dropped every resource would record no WAL entries at all, so Finalize would + // persist no state file and the migration would fail with a missing resources.json. + for key, id := range tfState.IDs { + if err := stateDB.SaveState(ctx, key, id, json.RawMessage("{}"), nil); err != nil { + return tempStatePath, resourceCount, false, nil, fmt.Errorf("seeding migrated state for %s: %w", key, err) + } + } + // warnPrefix labels the conversion's warnings as coming from the background dry run. hasWarnings, err := migrate.BuildStateFromTF(ctx, &uninterpolatedConfig, adapters, &stateDB, tfState.Attrs, tfState.IDs, warnPrefix) if err != nil { @@ -350,68 +419,3 @@ func convertTFStateToDirect(ctx context.Context, b *bundle.Bundle, tfState *migr return tempStatePath, resourceCount, hasWarnings, &uninterpolatedConfig, nil } - -// commitMigration pushes the converted direct state to the workspace (the -// hard commit — once resources.json lands with serial tf+1, it outranks any -// leftover terraform state regardless of whether the cleanup steps below -// succeed), then best-effort backs up and removes the terraform state files. -// Any cleanup failure is logged as a warning and does not affect the -// migration outcome. -func commitMigration(ctx context.Context, b *bundle.Bundle, tempStatePath string, resourceCount int) error { - _, localDirectPath := b.StateFilenameDirect(ctx) - - // A stat error other than "not exist" (e.g. permission denied) is not - // "file is missing"; treat it as a hard failure to avoid renaming over - // something we couldn't read. - if _, err := os.Stat(localDirectPath); err == nil { - return fmt.Errorf("state file %s already exists", localDirectPath) - } else if !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("stat %s: %w", localDirectPath, err) - } - - if err := pushDirectState(ctx, b, tempStatePath); err != nil { - return fmt.Errorf("pushing direct state to workspace: %w", err) - } - - // Migration is committed. The remaining steps are best-effort cleanup: - // leftover terraform state files cannot affect engine selection because - // resources.json has a higher serial number. - BackupRemoteTerraformState(ctx, b) - - _, localTerraformPath := b.StateFilenameTerraform(ctx) - if err := os.Rename(localTerraformPath, localTerraformPath+".backup"); err != nil && !errors.Is(err, fs.ErrNotExist) { - log.Warnf(ctx, "automatic migration to direct engine: could not back up local terraform state: %v", err) - } - - if err := os.MkdirAll(filepath.Dir(localDirectPath), 0o700); err != nil { - log.Warnf(ctx, "automatic migration to direct engine: could not create local state directory: %v", err) - } else if err := os.Rename(tempStatePath, localDirectPath); err != nil { - log.Warnf(ctx, "automatic migration to direct engine: could not write local direct state: %v", err) - } - - suffix := "s" - if resourceCount == 1 { - suffix = "" - } - cmdio.LogString(ctx, fmt.Sprintf("Migrated %d resource%s to direct deployment engine.", resourceCount, suffix)) - return nil -} - -// pushDirectState uploads the direct-engine state file to the workspace. -// The caller passes the temp file produced by the dry-run; it is renamed -// into place locally by commitMigration after this upload succeeds. -func pushDirectState(ctx context.Context, b *bundle.Bundle, localPath string) error { - f, err := deploy.StateFiler(ctx, b) - if err != nil { - return err - } - - remoteDirectPath, _ := b.StateFilenameDirect(ctx) - local, err := os.Open(localPath) - if err != nil { - return err - } - defer local.Close() - - return f.Write(ctx, remoteDirectPath, local, filer.CreateParentDirectories, filer.OverwriteIfExists) -} diff --git a/cmd/bundle/deploy.go b/cmd/bundle/deploy.go index f248111e8b0..41d1bd3b13f 100644 --- a/cmd/bundle/deploy.go +++ b/cmd/bundle/deploy.go @@ -67,12 +67,13 @@ See https://docs.databricks.com/en/dev-tools/bundles/index.html for more informa b.Config.Bundle.Deployment.FailOnActiveRuns = failOnActiveRuns } }, - Verbose: verbose, - AlwaysPull: true, - FastValidate: true, - Build: true, - Deploy: true, - ReadPlanPath: readPlanPath, + Verbose: verbose, + AlwaysPull: true, + FastValidate: true, + Build: true, + Deploy: true, + CommitStateMigration: true, + ReadPlanPath: readPlanPath, }) return err diff --git a/cmd/bundle/destroy.go b/cmd/bundle/destroy.go index 90dec11a1ea..41310f20aff 100644 --- a/cmd/bundle/destroy.go +++ b/cmd/bundle/destroy.go @@ -80,6 +80,7 @@ func CommandBundleDestroy(cmd *cobra.Command, args []string, autoApprove, forceD // Skip context initialization if already initialized by parent command SkipInitContext: skipInitContext, AlwaysPull: true, + CommitStateMigration: true, SkipEnforcingDeploymentHistorySetting: true, PostStateFunc: func(ctx context.Context, b *bundle.Bundle, stateDesc *statemgmt.StateDesc) error { phases.Destroy(ctx, b, stateDesc.Engine) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 4af01b84037..8bbccdf29a0 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -73,6 +73,12 @@ type ProcessOptions struct { // Implies ReadState ErrorOnEmptyState bool + // If true, an auto-migration from the terraform engine to the direct engine is + // committed (resources.json written and pushed, terraform.tfstate backed up) rather + // than kept in memory. Set by the commands that apply changes to remote resources + // (deploy, destroy); plan and read-only commands migrate in memory only. + CommitStateMigration bool + // If true, configure outputHandler for phases.Deploy Verbose bool @@ -226,7 +232,16 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if logdiag.HasError(ctx) { return b, stateDesc, root.ErrAlreadyPrinted } - ctx = useragent.InContext(ctx, "engine", string(stateDesc.Engine)) + + b.MigratingToDirect = requiredEngine.Type == engine.EngineDirect && !stateDesc.Engine.IsDirect() + + // Tag the user agent with the engine this run actually uses. On the auto-migration + // path the engine is only final after the migration runs (below), so leave it unset + // here and tag there; setting it once rather than appending avoids a stale + // engine/terraform tag alongside engine/direct. + if !b.MigratingToDirect { + ctx = useragent.InContext(ctx, "engine", string(stateDesc.Engine)) + } cmd.SetContext(ctx) if stateDesc.Engine.IsDirect() { resolveDeploymentHistory(ctx, b, stateDesc) @@ -235,36 +250,6 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle // Record the engine the resolved state uses now, so deploy telemetry reports // it even when the deploy fails or is cancelled before deployCore runs. b.Metrics.StateEngine = stateDesc.Engine.ThisOrDefault() - - b.MigratingToDirect = requiredEngine.Type == engine.EngineDirect && !stateDesc.Engine.IsDirect() - - // Announce the auto-migration path here (only on deploy) so the user - // isn't surprised when MigrateToDirect commits state changes at the - // end. PullResourcesState is shared with non-deploy commands like - // `bundle debug states`, which would otherwise print the same hint - // even though they will not migrate. - if opts.Deploy && b.MigratingToDirect { - if requiredEngine.IsDefault { - // The user did not ask for direct; it is the default. Frame the - // auto-migration as an informational notice rather than a warning, - // and do not claim the user selected anything. - cmdio.LogString(ctx, "Notice: the direct deployment engine is the default as of CLI v1.14.0.\n\n"+ - "This bundle will be automatically migrated to use the direct deployment engine after this deployment.\n\n"+ - "Learn more: https://docs.databricks.com/dev-tools/bundles/direct\n") - } else { - log.Warnf(ctx, "Direct engine selected via %s but the existing state uses %q. Deploying on %q; will attempt to migrate the state to the direct engine after this deploy.", requiredEngine.Source, stateDesc.Engine, stateDesc.Engine) - } - } - - // --select is only supported by the direct engine, which tracks resource - // dependencies in the plan graph (used to expand the selection transitively). - // The engine is only known for certain after the state is pulled, so reject it - // here rather than silently planning/deploying every resource on terraform. - if len(b.Select) > 0 && !stateDesc.Engine.IsDirect() { - logdiag.LogError(ctx, errors.New("--select is only supported with the direct engine. See https://docs.databricks.com/aws/en/dev-tools/bundles/direct")) - return b, stateDesc, root.ErrAlreadyPrinted - } - } // --plan applies a precomputed plan, so it skips Build and PreDeployChecks; a plain @@ -323,17 +308,77 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } // Resolve the deployment state after phases.Build, in one place, so the order reads - // build → resolve-state → deploy. This is behavior-preserving today (nothing between - // FastValidate and Build consumes the opened state, and the auto-migration still runs - // post-deploy). It prepares for moving the terraform→direct migration before deploy, - // which must run after Build so the converted state records resolved library and - // ${artifacts.*} paths rather than unexpanded globs. + // build → migrate → open → (plan/deploy). Running the migration after the build lets + // its conversion and plan check see library and ${artifacts.*} references resolved by + // the build, and record resolved remote paths in the migrated state. Commands without + // a build phase (destroy, summary, ...) reach here with the build skipped, so migration + // and state opening still happen in this single block. var plan *deployplan.Plan if shouldReadState { + // needsState is narrower than shouldReadState: it drops the pull-only flags + // (ReadState, AlwaysPull) so read-only consumers like "bundle debug states" pull + // and display the state without triggering a migration, and adds PostStateFunc for + // commands that operate on the state (destroy, run). It gates migrating and opening + // the direct state. + needsState := opts.InitIDs || opts.ErrorOnEmptyState || opts.Deploy || opts.ReadPlanPath != "" || opts.PreDeployChecks || opts.PostStateFunc != nil + + // Migrate a Terraform state to the direct engine: when the direct engine is + // requested (the default) and the existing state still uses Terraform, convert it so + // the run proceeds on the direct engine. deploy/destroy commit the migration + // (resources.json written and pushed, terraform.tfstate backed up); plan and summary + // keep it in memory. Deriving commit from opts.Deploy keeps every deploy entry point + // (bundle, pipelines, apps) consistent. If the migration's plan check fails, + // MigrateTerraformState leaves the Terraform state intact and the run falls back to + // the terraform engine. Read-only commands set none of these options and keep + // reading the Terraform state as-is. + // A deploy/destroy that prepares the migration but never commits it - declined, or any + // error before it applies (a missing --plan file, a validation error, prevent_destroy) + // - must not leave the bundle silently migrated. The commit paths clear + // b.MigrationDeferred; this discards whatever a not-committed run left, so it stays on + // terraform. Registered before the migration so a failure inside it is covered too. + defer func() { + if b.MigrationDeferred { + statemgmt.DiscardDeferredMigration(ctx, b) + log.Warnf(ctx, "Migration not committed, keeping Terraform state") + } + }() + + if b.MigratingToDirect && needsState { + // deploy and destroy defer the commit to their phase (after approval); other + // state-reading commands (plan, run) migrate in memory only. + mode := statemgmt.MigratePlan + if opts.Deploy || opts.CommitStateMigration { + mode = statemgmt.MigrateDeferred + } + if err := migrateTerraformToDirect(ctx, b, stateDesc, requiredEngine, mode); err != nil { + logdiag.LogError(ctx, err) + return b, stateDesc, root.ErrAlreadyPrinted + } + } + + // Tag the user agent with the engine this run actually uses. The tag is left unset + // after the pull on the auto-migration path (see above) because the engine is only + // final here, after the migration ran, was skipped, or fell back. Setting it once + // (rather than appending) avoids a stale engine/terraform tag alongside engine/direct. + if b.MigratingToDirect { + ctx = useragent.InContext(ctx, "engine", string(stateDesc.Engine)) + cmd.SetContext(ctx) + } + + // --select is only supported by the direct engine, which tracks resource + // dependencies in the plan graph (used to expand the selection transitively). + // Validate once the engine is final (after any migration above), rather than + // silently planning/deploying every resource on terraform. + if len(b.Select) > 0 && !stateDesc.Engine.IsDirect() { + logdiag.LogError(ctx, errors.New("--select is only supported with the direct engine. See https://docs.databricks.com/aws/en/dev-tools/bundles/direct")) + return b, stateDesc, root.ErrAlreadyPrinted + } + // Open direct engine state once for all subsequent operations (ExportState, CalculatePlan, Apply, etc.) - needDirectState := stateDesc.Engine.IsDirect() && (opts.InitIDs || opts.ErrorOnEmptyState || opts.Deploy || opts.ReadPlanPath != "" || opts.PreDeployChecks || opts.PostStateFunc != nil) + // A migrated-from-Terraform state is already open (seeded in memory above), so skip the disk open. + needDirectState := stateDesc.Engine.IsDirect() && needsState var localPath string - if needDirectState { + if needDirectState && !b.DeploymentBundle.StateDB.IsOpen() { _, localPath = b.StateFilenameDirect(ctx) if !stateDesc.IsDMS() { if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{}); err != nil { @@ -508,13 +553,25 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, stateDesc, root.ErrAlreadyPrinted } - if b != nil && stateDesc != nil && stateDesc.Engine.IsDirect() && stateDesc.HasRemoteTerraformState() { + // A migrating deploy already backed up terraform.tfstate when it committed the + // converted state above; this handles a plain direct deploy that still finds a + // lingering remote terraform state. + if b != nil && stateDesc != nil && stateDesc.Engine.IsDirect() && !b.MigratingToDirect && stateDesc.HasRemoteTerraformState() { statemgmt.BackupRemoteTerraformState(ctx, b) if logdiag.HasError(ctx) { return b, stateDesc, root.ErrAlreadyPrinted } } + + // The user opted out of the direct engine (engine: terraform), so no migration ran. + // Do a throwaway conversion of the just-deployed terraform state to record + // direct_drymigrate_* telemetry — the fleet-wide "could this bundle migrate?" + // signal. Runs after the deploy so mutating b.Config during the conversion is + // harmless, and only when the deploy succeeded on a terraform state. + if stateDesc != nil && requiredEngine.Type == engine.EngineTerraform && !stateDesc.Engine.IsDirect() { + statemgmt.DryRunMigrationTelemetry(ctx, b) + } } if opts.PostStateFunc != nil { @@ -526,6 +583,31 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, stateDesc, nil } +// migrateTerraformToDirect converts the bundle's Terraform state to the direct engine. +// On success it advances stateDesc and metrics to the direct engine. Any failure before +// the migration commits (parse, conversion, plan check, or push) leaves the Terraform +// state intact (migrated=false) so the caller proceeds on the terraform engine and the +// deploy still runs; only a failure after the commit returns an error. The caller tags the +// user agent with the resolved stateDesc.Engine afterwards. +// +// mode selects how the converted state is committed: MigrateDeferred (deploy, destroy) writes +// the local state and lets the approved command commit it, and MigratePlan (plan, run) keeps +// it in memory only. See MigrateTerraformState. +func migrateTerraformToDirect(ctx context.Context, b *bundle.Bundle, stateDesc *statemgmt.StateDesc, requiredEngine engine.EngineSetting, mode statemgmt.MigrateMode) error { + if requiredEngine.IsDefault { + cmdio.LogString(ctx, "Notice: automatically migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765).") + } + migrated, err := statemgmt.MigrateTerraformState(ctx, b, requiredEngine, mode) + if err != nil { + return fmt.Errorf("migrating Terraform state to the direct engine: %w", err) + } + if migrated { + stateDesc.Engine = engine.EngineDirect + b.Metrics.StateEngine = engine.EngineDirect + } + return nil +} + // ResolveEngineSetting determines the effective engine setting by combining bundle config and env var. // Priority: bundle.engine config > DATABRICKS_BUNDLE_ENGINE env var > engine.Default. func ResolveEngineSetting(ctx context.Context, b *bundle.Bundle) (engine.EngineSetting, error) {