diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 2c13750d4b..578e2f5708 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -15,6 +15,7 @@ * engine/direct: Fix bind and unbind for non-Terraform resources ([#4850](https://github.com/databricks/cli/pull/4850)) * engine/direct: Fix deploying removed principals ([#4824](https://github.com/databricks/cli/pull/4824)) * engine/direct: Fix secret scope permissions migration from Terraform to Direct engine ([#4866](https://github.com/databricks/cli/pull/4866)) +* Fix `bundle deployment bind` to always pull remote state before modifying ([#4892](https://github.com/databricks/cli/pull/4892)) ### Dependency updates diff --git a/acceptance/bundle/deployment/bind/cross-engine/databricks.yml b/acceptance/bundle/deployment/bind/cross-engine/databricks.yml new file mode 100644 index 0000000000..9deddbdf30 --- /dev/null +++ b/acceptance/bundle/deployment/bind/cross-engine/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: cross-engine-bind + +resources: + pipelines: + pipeline_1: + name: Pipeline 1 + + pipeline_2: + name: Pipeline 2 diff --git a/acceptance/bundle/deployment/bind/cross-engine/databricks_empty.yml b/acceptance/bundle/deployment/bind/cross-engine/databricks_empty.yml new file mode 100644 index 0000000000..87dfce028e --- /dev/null +++ b/acceptance/bundle/deployment/bind/cross-engine/databricks_empty.yml @@ -0,0 +1,2 @@ +bundle: + name: cross-engine-bind diff --git a/acceptance/bundle/deployment/bind/cross-engine/databricks_p1.yml b/acceptance/bundle/deployment/bind/cross-engine/databricks_p1.yml new file mode 100644 index 0000000000..8d99284161 --- /dev/null +++ b/acceptance/bundle/deployment/bind/cross-engine/databricks_p1.yml @@ -0,0 +1,7 @@ +bundle: + name: cross-engine-bind + +resources: + pipelines: + pipeline_1: + name: Pipeline 1 diff --git a/acceptance/bundle/deployment/bind/cross-engine/out.test.toml b/acceptance/bundle/deployment/bind/cross-engine/out.test.toml new file mode 100644 index 0000000000..54146af564 --- /dev/null +++ b/acceptance/bundle/deployment/bind/cross-engine/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/cross-engine/output.txt b/acceptance/bundle/deployment/bind/cross-engine/output.txt new file mode 100644 index 0000000000..bee49be1ad --- /dev/null +++ b/acceptance/bundle/deployment/bind/cross-engine/output.txt @@ -0,0 +1,32 @@ + +=== Step 1: Deploy empty bundle with terraform engine +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cross-engine-bind/default/files... +Deploying resources... +Deployment complete! + +=== Step 2: Add pipeline_1 and bind with terraform engine +>>> [CLI] bundle deployment bind pipeline_1 [PIPELINE_1_ID] --auto-approve +Updating deployment state... +Successfully bound pipeline with an id '[PIPELINE_1_ID]' +Run 'bundle deploy' to deploy changes to your workspace + +=== Step 3: Remove local state +>>> rm -rf .databricks + +=== Step 4: Add pipeline_2 config and bind with direct engine +>>> [CLI] bundle deployment bind pipeline_2 [PIPELINE_2_ID] --auto-approve +Updating deployment state... +Successfully bound pipeline with an id '[PIPELINE_2_ID]' +Run 'bundle deploy' to deploy changes to your workspace + +=== Step 5: Run summary +>>> errcode [CLI] bundle summary +Error: lineage mismatch in state files + +Available state files: +- [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate: local terraform state serial=1 lineage="[UUID]" +- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=2 lineage="[UUID]" + + +Exit code: 1 diff --git a/acceptance/bundle/deployment/bind/cross-engine/script b/acceptance/bundle/deployment/bind/cross-engine/script new file mode 100755 index 0000000000..f08ebd0278 --- /dev/null +++ b/acceptance/bundle/deployment/bind/cross-engine/script @@ -0,0 +1,25 @@ +title "Step 1: Deploy empty bundle with terraform engine" +cp databricks_empty.yml databricks.yml +DATABRICKS_BUNDLE_ENGINE=terraform trace $CLI bundle deploy + +title "Step 2: Add pipeline_1 and bind with terraform engine" +cp databricks_p1.yml databricks.yml +p1_id=$($CLI pipelines create --json '{"name": "External Pipeline 1"}' | jq -r '.pipeline_id') +add_repl.py "$p1_id" PIPELINE_1_ID +DATABRICKS_BUNDLE_ENGINE=terraform trace $CLI bundle deployment bind pipeline_1 $p1_id --auto-approve + +title "Step 3: Remove local state" +trace rm -rf .databricks + +title "Step 4: Add pipeline_2 config and bind with direct engine" +cat >> databricks.yml <>> errcode [CLI] bundle deployment bind job_2 [EXTERNAL_JOB_ID] --auto-approve +Error: Resource already managed + +The bundle is already managing a resource for resources.jobs.job_2 with ID '[JOB_2_ID]'. +To bind to a different resource with ID '[EXTERNAL_JOB_ID]', you must first unbind the existing resource. + + +Exit code: 1 diff --git a/acceptance/bundle/deployment/bind/job/stale-state/out.bind.terraform.txt b/acceptance/bundle/deployment/bind/job/stale-state/out.bind.terraform.txt new file mode 100644 index 0000000000..88dabf9f20 --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/stale-state/out.bind.terraform.txt @@ -0,0 +1,14 @@ + +>>> errcode [CLI] bundle deployment bind job_2 [EXTERNAL_JOB_ID] --auto-approve +Error: terraform import: exit status 1 + +Error: Resource already managed by Terraform + +Terraform is already managing a remote object for databricks_job.job_2. To +import to this address you must first remove the existing object from the +state. + + + + +Exit code: 1 diff --git a/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml b/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml new file mode 100644 index 0000000000..d560f1de04 --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/stale-state/output.txt b/acceptance/bundle/deployment/bind/job/stale-state/output.txt new file mode 100644 index 0000000000..815586e6d4 --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/stale-state/output.txt @@ -0,0 +1,21 @@ + +=== Step 1: Deploy with job_1 only +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/stale_state_test/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Step 2: Save stale local state (has only job_1, serial=1) +=== Step 3: Add job_2 to config and deploy again +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/stale_state_test/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Step 4: Record deployed job_2 ID from stateDeployed job_2 ID: [JOB_2_ID] + +=== Step 5: Restore stale local state (only job_1, serial=1) +=== Step 6: Create external job and try to bind (should fail: remote state already has job_2) +External job ID: [EXTERNAL_JOB_ID] diff --git a/acceptance/bundle/deployment/bind/job/stale-state/script b/acceptance/bundle/deployment/bind/job/stale-state/script new file mode 100755 index 0000000000..32a2d3fb47 --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/stale-state/script @@ -0,0 +1,27 @@ +if [ "$DATABRICKS_BUNDLE_ENGINE" = "direct" ]; then + state_file=".databricks/bundle/default/resources.json" +else + state_file=".databricks/bundle/default/terraform/terraform.tfstate" +fi + +title "Step 1: Deploy with job_1 only" +trace $CLI bundle deploy + +title "Step 2: Save stale local state (has only job_1, serial=1)" +cp "$state_file" stale_state.json + +title "Step 3: Add job_2 to config and deploy again" +cp databricks_v2.yml databricks.yml +trace $CLI bundle deploy + +title "Step 4: Record deployed job_2 ID from state" +echo "Deployed job_2 ID: $(read_id.py job_2)" + +title "Step 5: Restore stale local state (only job_1, serial=1)" +cp stale_state.json "$state_file" + +title "Step 6: Create external job and try to bind (should fail: remote state already has job_2)\n" +job_id=$($CLI jobs create --json '{"name": "External Job"}' | jq -r '.job_id') +add_repl.py "$job_id" EXTERNAL_JOB_ID +echo "External job ID: $job_id" +trace errcode $CLI bundle deployment bind job_2 $job_id --auto-approve &> out.bind.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/deployment/bind/job/stale-state/test.toml b/acceptance/bundle/deployment/bind/job/stale-state/test.toml new file mode 100644 index 0000000000..df072247ba --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/stale-state/test.toml @@ -0,0 +1,9 @@ +Cloud = false + +Ignore = [ + "databricks_v2.yml", + "stale_state.json", +] + +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/cmd/apps/import.go b/cmd/apps/import.go index de6529cf3c..7aae1fe43a 100644 --- a/cmd/apps/import.go +++ b/cmd/apps/import.go @@ -298,7 +298,7 @@ func runImport(ctx context.Context, w *databricks.WorkspaceClient, appName, outp var err error b, err = bundleutils.ProcessBundle(bindCmd, bundleutils.ProcessOptions{ SkipInitContext: true, - ReadState: true, + AlwaysPull: true, InitFunc: func(b *bundle.Bundle) { b.Config.Bundle.Deployment.Lock.Force = false }, diff --git a/cmd/bundle/deployment/bind_resource.go b/cmd/bundle/deployment/bind_resource.go index 4fcee8b07f..8a766f240c 100644 --- a/cmd/bundle/deployment/bind_resource.go +++ b/cmd/bundle/deployment/bind_resource.go @@ -18,7 +18,7 @@ import ( func BindResource(cmd *cobra.Command, resourceKey, resourceId string, autoApprove, forceLock, skipInitContext bool) error { b, err := utils.ProcessBundle(cmd, utils.ProcessOptions{ SkipInitContext: skipInitContext, - ReadState: true, + AlwaysPull: true, InitFunc: func(b *bundle.Bundle) { b.Config.Bundle.Deployment.Lock.Force = forceLock },