From 0eed91bba0ace1da58778e60eb9cd00867747d23 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Thu, 2 Apr 2026 14:23:47 +0200 Subject: [PATCH 1/4] Fix yaml sync snapshot when bundle is deployed without any resources --- .../empty_deploy/databricks.yml.tmpl | 2 + .../empty_deploy/output.txt | 11 ++++++ .../config-remote-sync/empty_deploy/script | 10 +++++ .../config-remote-sync/empty_deploy/test.toml | 10 +++++ .../statemgmt/upload_state_for_yaml_sync.go | 37 +++++++++++-------- 5 files changed, 55 insertions(+), 15 deletions(-) create mode 100644 acceptance/bundle/config-remote-sync/empty_deploy/databricks.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/empty_deploy/output.txt create mode 100644 acceptance/bundle/config-remote-sync/empty_deploy/script create mode 100644 acceptance/bundle/config-remote-sync/empty_deploy/test.toml diff --git a/acceptance/bundle/config-remote-sync/empty_deploy/databricks.yml.tmpl b/acceptance/bundle/config-remote-sync/empty_deploy/databricks.yml.tmpl new file mode 100644 index 0000000000..244a11badf --- /dev/null +++ b/acceptance/bundle/config-remote-sync/empty_deploy/databricks.yml.tmpl @@ -0,0 +1,2 @@ +bundle: + name: test-bundle-$UNIQUE_NAME diff --git a/acceptance/bundle/config-remote-sync/empty_deploy/output.txt b/acceptance/bundle/config-remote-sync/empty_deploy/output.txt new file mode 100644 index 0000000000..919accb661 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/empty_deploy/output.txt @@ -0,0 +1,11 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Deployment complete! + +>>> [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/config-remote-sync/empty_deploy/script b/acceptance/bundle/config-remote-sync/empty_deploy/script new file mode 100644 index 0000000000..ea933590c4 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/empty_deploy/script @@ -0,0 +1,10 @@ +#!/bin/bash + +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy diff --git a/acceptance/bundle/config-remote-sync/empty_deploy/test.toml b/acceptance/bundle/config-remote-sync/empty_deploy/test.toml new file mode 100644 index 0000000000..2a5dcf77c8 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/empty_deploy/test.toml @@ -0,0 +1,10 @@ +Cloud = true + +RecordRequests = false +Ignore = [".databricks", "databricks.yml"] + +[Env] +DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true" + +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/bundle/statemgmt/upload_state_for_yaml_sync.go b/bundle/statemgmt/upload_state_for_yaml_sync.go index 7d7c766743..2b2488c41e 100644 --- a/bundle/statemgmt/upload_state_for_yaml_sync.go +++ b/bundle/statemgmt/upload_state_for_yaml_sync.go @@ -54,8 +54,8 @@ func (m *uploadStateForYamlSync) Apply(ctx context.Context, b *bundle.Bundle) di _, snapshotPath := b.StateFilenameConfigSnapshot(ctx) - diags := m.convertState(ctx, b, snapshotPath) - if diags.HasError() { + created, diags := m.convertState(ctx, b, snapshotPath) + if diags.HasError() || !created { return diags } @@ -92,10 +92,22 @@ func uploadState(ctx context.Context, b *bundle.Bundle) error { return nil } -func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bundle, snapshotPath string) (diags diag.Diagnostics) { +func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bundle, snapshotPath string) (bool, diag.Diagnostics) { terraformResources, err := terraform.ParseResourcesState(ctx, b) if err != nil { - return diag.FromErr(err) + return false, diag.FromErr(fmt.Errorf("failed to parse terraform state for config snapshot: %w", err)) + } + + // ParseResourcesState returns nil when the terraform state file doesn't exist + // (e.g. first deploy with no resources). + if terraformResources == nil { + return false, nil + } + + _, localTerraformPath := b.StateFilenameTerraform(ctx) + data, err := os.ReadFile(localTerraformPath) + if err != nil { + return false, diag.FromErr(fmt.Errorf("failed to read terraform state for config snapshot: %w", err)) } state := make(map[string]dstate.ResourceEntry) @@ -111,18 +123,12 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun } } - _, localTerraformPath := b.StateFilenameTerraform(ctx) - data, err := os.ReadFile(localTerraformPath) - if err != nil { - return diag.FromErr(err) - } - var tfState struct { Lineage string `json:"lineage"` Serial int `json:"serial"` } if err := json.Unmarshal(data, &tfState); err != nil { - return diag.FromErr(err) + return false, diag.FromErr(err) } migratedDB := dstate.NewDatabase(tfState.Lineage, tfState.Serial+1) @@ -141,7 +147,7 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun interpolatedRoot := b.Config.Value() uninterpolatedRoot, err := reverseInterpolate(interpolatedRoot) if err != nil { - return diag.FromErr(fmt.Errorf("failed to reverse interpolation: %w", err)) + return false, diag.FromErr(fmt.Errorf("failed to reverse interpolation: %w", err)) } var uninterpolatedConfig config.Root @@ -149,18 +155,19 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun return uninterpolatedRoot, nil }) if err != nil { - return diag.FromErr(fmt.Errorf("failed to create uninterpolated config: %w", err)) + return false, diag.FromErr(fmt.Errorf("failed to create uninterpolated config: %w", err)) } plan, err := deploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(), &uninterpolatedConfig, snapshotPath) if err != nil { - return diag.FromErr(err) + return false, diag.FromErr(err) } for _, entry := range plan.Plan { entry.Action = deployplan.Update } + var diags diag.Diagnostics for key := range plan.Plan { etag := etags[key] if etag == "" { @@ -178,7 +185,7 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun deploymentBundle.Apply(ctx, b.WorkspaceClient(), plan, direct.MigrateMode(true)) - return diags + return true, diags } // reverseInterpolate reverses the terraform.Interpolate transformation. From f6819272f7d7e8d51123eb3997ce98a4515f0250 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Thu, 2 Apr 2026 14:27:37 +0200 Subject: [PATCH 2/4] Don't stop deploy when snapshot is not created --- bundle/statemgmt/upload_state_for_yaml_sync.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bundle/statemgmt/upload_state_for_yaml_sync.go b/bundle/statemgmt/upload_state_for_yaml_sync.go index 2b2488c41e..9295d8aebe 100644 --- a/bundle/statemgmt/upload_state_for_yaml_sync.go +++ b/bundle/statemgmt/upload_state_for_yaml_sync.go @@ -55,13 +55,16 @@ func (m *uploadStateForYamlSync) Apply(ctx context.Context, b *bundle.Bundle) di _, snapshotPath := b.StateFilenameConfigSnapshot(ctx) created, diags := m.convertState(ctx, b, snapshotPath) - if diags.HasError() || !created { - return diags + if diags.HasError() { + return diag.Warningf("Failed to create config snapshot: %v", diags.Error()) + } + if !created { + return nil } err := uploadState(ctx, b) if err != nil { - return diags.Extend(diag.Warningf("Failed to upload config snapshot to workspace: %v", err)) + return diag.Warningf("Failed to upload config snapshot to workspace: %v", err) } log.Infof(ctx, "Config snapshot created at %s", snapshotPath) From af91585ee5c5721d18d410f3c565c2bbbc3d5d33 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Thu, 2 Apr 2026 17:19:27 +0200 Subject: [PATCH 3/4] Use existing tests --- .../empty_deploy/databricks.yml.tmpl | 2 -- .../bundle/config-remote-sync/empty_deploy/output.txt | 11 ----------- .../bundle/config-remote-sync/empty_deploy/script | 10 ---------- .../bundle/config-remote-sync/empty_deploy/test.toml | 10 ---------- acceptance/bundle/deploy/empty-bundle/out.test.toml | 1 + acceptance/bundle/deploy/empty-bundle/test.toml | 5 ++++- 6 files changed, 5 insertions(+), 34 deletions(-) delete mode 100644 acceptance/bundle/config-remote-sync/empty_deploy/databricks.yml.tmpl delete mode 100644 acceptance/bundle/config-remote-sync/empty_deploy/output.txt delete mode 100644 acceptance/bundle/config-remote-sync/empty_deploy/script delete mode 100644 acceptance/bundle/config-remote-sync/empty_deploy/test.toml diff --git a/acceptance/bundle/config-remote-sync/empty_deploy/databricks.yml.tmpl b/acceptance/bundle/config-remote-sync/empty_deploy/databricks.yml.tmpl deleted file mode 100644 index 244a11badf..0000000000 --- a/acceptance/bundle/config-remote-sync/empty_deploy/databricks.yml.tmpl +++ /dev/null @@ -1,2 +0,0 @@ -bundle: - name: test-bundle-$UNIQUE_NAME diff --git a/acceptance/bundle/config-remote-sync/empty_deploy/output.txt b/acceptance/bundle/config-remote-sync/empty_deploy/output.txt deleted file mode 100644 index 919accb661..0000000000 --- a/acceptance/bundle/config-remote-sync/empty_deploy/output.txt +++ /dev/null @@ -1,11 +0,0 @@ - ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... -Deploying resources... -Deployment complete! - ->>> [CLI] bundle destroy --auto-approve -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default - -Deleting files... -Destroy complete! diff --git a/acceptance/bundle/config-remote-sync/empty_deploy/script b/acceptance/bundle/config-remote-sync/empty_deploy/script deleted file mode 100644 index ea933590c4..0000000000 --- a/acceptance/bundle/config-remote-sync/empty_deploy/script +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -envsubst < databricks.yml.tmpl > databricks.yml - -cleanup() { - trace $CLI bundle destroy --auto-approve -} -trap cleanup EXIT - -trace $CLI bundle deploy diff --git a/acceptance/bundle/config-remote-sync/empty_deploy/test.toml b/acceptance/bundle/config-remote-sync/empty_deploy/test.toml deleted file mode 100644 index 2a5dcf77c8..0000000000 --- a/acceptance/bundle/config-remote-sync/empty_deploy/test.toml +++ /dev/null @@ -1,10 +0,0 @@ -Cloud = true - -RecordRequests = false -Ignore = [".databricks", "databricks.yml"] - -[Env] -DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true" - -[EnvMatrix] -DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/deploy/empty-bundle/out.test.toml b/acceptance/bundle/deploy/empty-bundle/out.test.toml index 01ed6822af..0940cf4b56 100644 --- a/acceptance/bundle/deploy/empty-bundle/out.test.toml +++ b/acceptance/bundle/deploy/empty-bundle/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true [EnvMatrix] + DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = ["", "true"] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/empty-bundle/test.toml b/acceptance/bundle/deploy/empty-bundle/test.toml index 5265ac0b23..f64800a163 100644 --- a/acceptance/bundle/deploy/empty-bundle/test.toml +++ b/acceptance/bundle/deploy/empty-bundle/test.toml @@ -1 +1,4 @@ -Cloud=true +Cloud = true + +[EnvMatrix] +DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = ["", "true"] From 9c5b798df50cfff208dd2859099a429c277595d5 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Thu, 2 Apr 2026 17:19:36 +0200 Subject: [PATCH 4/4] Remove diags --- .../statemgmt/upload_state_for_yaml_sync.go | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/bundle/statemgmt/upload_state_for_yaml_sync.go b/bundle/statemgmt/upload_state_for_yaml_sync.go index 9295d8aebe..c3da05727c 100644 --- a/bundle/statemgmt/upload_state_for_yaml_sync.go +++ b/bundle/statemgmt/upload_state_for_yaml_sync.go @@ -54,21 +54,23 @@ func (m *uploadStateForYamlSync) Apply(ctx context.Context, b *bundle.Bundle) di _, snapshotPath := b.StateFilenameConfigSnapshot(ctx) - created, diags := m.convertState(ctx, b, snapshotPath) - if diags.HasError() { - return diag.Warningf("Failed to create config snapshot: %v", diags.Error()) + created, err := m.convertState(ctx, b, snapshotPath) + if err != nil { + log.Warnf(ctx, "Failed to create config snapshot: %v", err) + return nil } if !created { return nil } - err := uploadState(ctx, b) + err = uploadState(ctx, b) if err != nil { - return diag.Warningf("Failed to upload config snapshot to workspace: %v", err) + log.Warnf(ctx, "Failed to upload config snapshot: %v", err) + return nil } log.Infof(ctx, "Config snapshot created at %s", snapshotPath) - return diags + return nil } func uploadState(ctx context.Context, b *bundle.Bundle) error { @@ -95,10 +97,10 @@ func uploadState(ctx context.Context, b *bundle.Bundle) error { return nil } -func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bundle, snapshotPath string) (bool, diag.Diagnostics) { +func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bundle, snapshotPath string) (bool, error) { terraformResources, err := terraform.ParseResourcesState(ctx, b) if err != nil { - return false, diag.FromErr(fmt.Errorf("failed to parse terraform state for config snapshot: %w", err)) + return false, fmt.Errorf("failed to parse terraform state: %w", err) } // ParseResourcesState returns nil when the terraform state file doesn't exist @@ -110,7 +112,7 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun _, localTerraformPath := b.StateFilenameTerraform(ctx) data, err := os.ReadFile(localTerraformPath) if err != nil { - return false, diag.FromErr(fmt.Errorf("failed to read terraform state for config snapshot: %w", err)) + return false, fmt.Errorf("failed to read terraform state: %w", err) } state := make(map[string]dstate.ResourceEntry) @@ -131,7 +133,7 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun Serial int `json:"serial"` } if err := json.Unmarshal(data, &tfState); err != nil { - return false, diag.FromErr(err) + return false, err } migratedDB := dstate.NewDatabase(tfState.Lineage, tfState.Serial+1) @@ -144,13 +146,13 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun }, } - // Get the dynamic value from b.Config and reverse the interpolation + // Get the dynamic value from b.Config and reverse the interpolation. // b.Config has been modified by terraform.Interpolate which converts bundle-style - // references (${resources.pipelines.x.id}) to terraform-style (${databricks_pipeline.x.id}) + // references (${resources.pipelines.x.id}) to terraform-style (${databricks_pipeline.x.id}). interpolatedRoot := b.Config.Value() uninterpolatedRoot, err := reverseInterpolate(interpolatedRoot) if err != nil { - return false, diag.FromErr(fmt.Errorf("failed to reverse interpolation: %w", err)) + return false, fmt.Errorf("failed to reverse interpolation: %w", err) } var uninterpolatedConfig config.Root @@ -158,19 +160,18 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun return uninterpolatedRoot, nil }) if err != nil { - return false, diag.FromErr(fmt.Errorf("failed to create uninterpolated config: %w", err)) + return false, fmt.Errorf("failed to create uninterpolated config: %w", err) } plan, err := deploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(), &uninterpolatedConfig, snapshotPath) if err != nil { - return false, diag.FromErr(err) + return false, err } for _, entry := range plan.Plan { entry.Action = deployplan.Update } - var diags diag.Diagnostics for key := range plan.Plan { etag := etags[key] if etag == "" { @@ -182,13 +183,13 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun } err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etag) if err != nil { - diags = diags.Extend(diag.Warningf("Failed to set etag on %q: %v", key, err)) + log.Warnf(ctx, "Failed to set etag on %q: %v", key, err) } } deploymentBundle.Apply(ctx, b.WorkspaceClient(), plan, direct.MigrateMode(true)) - return true, diags + return true, nil } // reverseInterpolate reverses the terraform.Interpolate transformation.