Represent struct fields and map keys; fix ss_default to ignore map keys

acceptance/bin/edit_resource.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""
+Implements get / update / set for a given resource.
+
+Fetches a given resource by id from the backend, executes Python code read from stdin and updates the resource.
+"""
+
+import sys
+import os
+import subprocess
+import argparse
+import json
+import pprint
+
+sys.path.insert(0, os.path.dirname(__file__))
+import util
+from util import run_json, run
+
+
+CLI = os.environ["CLI"]
+
+
+# Each class should be named after CLI command group and implement get(id) and set(id, value) methods:
+
+
+class jobs:
+    def get(self, job_id):
+        return run_json([CLI, "jobs", "get", job_id])["settings"]
+
+    def set(self, job_id, value):
+        payload = {"job_id": job_id, "new_settings": value}
+        return run([CLI, "jobs", "reset", job_id, "--json", json.dumps(payload)])
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("type")
+    parser.add_argument("id")
+    parser.add_argument("-v", "--verbose", action="store_true")
+    args = parser.parse_args()
+
+    util.VERBOSE = args.verbose
+
+    script = sys.stdin.read()
+
+    klass = globals()[args.type]
+    instance = klass()
+
+    data = instance.get(args.id)
+    my_locals = {"r": data}
+
+    try:
+        exec(script, locals=my_locals)
+    except Exception:
+        pprint.pprint(my_locals)
+        raise
+
+    instance.set(args.id, my_locals["r"])
+
+
+if __name__ == "__main__":
+    main()

acceptance/bin/util.py

@@ -0,0 +1,34 @@
+import sys
+import subprocess
+import json
+import shlex
+
+
+VERBOSE = False
+
+
+class RunError(Exception):
+    pass
+
+
+def run_json(cmd):
+    if VERBOSE:
+        print("+ " + " ".join([shlex.quote(x) for x in cmd]), file=sys.stderr, flush=True)
+    result = subprocess.run(cmd, stdout=subprocess.PIPE, encoding="utf-8")
+    if VERBOSE and result.stdout:
+        print(result.stdout, flush=True)
+    if result.returncode != 0:
+        raise RunError(f"{cmd} failed with code {result.returncode}\n{result.stdout}".strip())
+    try:
+        return json.loads(result.stdout)
+    except Exception as ex:
+        raise RunError(f"{cmd} returned non-json: {ex}\n{result.stdout}")
+
+
+def run(cmd):
+    if VERBOSE:
+        print("+ " + " ".join([shlex.quote(x) for x in cmd]), file=sys.stderr, flush=True)
+    result = subprocess.run(cmd)
+    if result.returncode != 0:
+        raise RunError(f"{cmd} failed with code {result.returncode}")
+    return result

acceptance/bundle/migrate/basic/out.plan_update.json

@@ -175,7 +175,7 @@
           "name": {
             "action": "update"
           },
-          "tags.myjob_name": {
+          "tags['myjob_name']": {
             "action": "update"
           }
         },

acceptance/bundle/resources/jobs/remote_add_tag/databricks.yml

@@ -0,0 +1,14 @@
+resources:
+  jobs:
+    foo:
+      name: "My Wheel Job"
+      tags:
+        tag1: tag value
+      tasks:
+        - task_key: TestTask
+          python_wheel_task:
+            package_name: "my_test_code"
+            entry_point: "run"
+          environment_key: test_env
+          libraries:
+            - whl: hello.whl

acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.direct.json

@@ -0,0 +1,100 @@
+{
+  "plan": {
+    "resources.jobs.foo": {
+      "action": "update",
+      "new_state": {
+        "value": {
+          "deployment": {
+            "kind": "BUNDLE",
+            "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
+          },
+          "edit_mode": "UI_LOCKED",
+          "format": "MULTI_TASK",
+          "max_concurrent_runs": 1,
+          "name": "My Wheel Job",
+          "queue": {
+            "enabled": true
+          },
+          "tags": {
+            "tag1": "tag value"
+          },
+          "tasks": [
+            {
+              "environment_key": "test_env",
+              "libraries": [
+                {
+                  "whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/hello.whl"
+                }
+              ],
+              "python_wheel_task": {
+                "entry_point": "run",
+                "package_name": "my_test_code"
+              },
+              "task_key": "TestTask"
+            }
+          ]
+        }
+      },
+      "remote_state": {
+        "created_time": [UNIX_TIME_MILLIS],
+        "creator_user_name": "[USERNAME]",
+        "job_id": [JOB_ID],
+        "run_as_user_name": "[USERNAME]",
+        "settings": {
+          "deployment": {
+            "kind": "BUNDLE",
+            "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
+          },
+          "edit_mode": "UI_LOCKED",
+          "email_notifications": {},
+          "format": "MULTI_TASK",
+          "max_concurrent_runs": 1,
+          "name": "My Wheel Job",
+          "queue": {
+            "enabled": true
+          },
+          "tags": {
+            "new_tag": "new_value",
+            "tag1": "tag value"
+          },
+          "tasks": [
+            {
+              "environment_key": "test_env",
+              "libraries": [
+                {
+                  "whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/hello.whl"
+                }
+              ],
+              "python_wheel_task": {
+                "entry_point": "run",
+                "package_name": "my_test_code"
+              },
+              "task_key": "TestTask"
+            }
+          ],
+          "timeout_seconds": 0,
+          "webhook_notifications": {}
+        }
+      },
+      "changes": {
+        "remote": {
+          "email_notifications": {
+            "action": "skip",
+            "reason": "server_side_default"
+          },
+          "tags['new_tag']": {
+            "action": "update"
+          },
+          "timeout_seconds": {
+            "action": "skip",
+            "reason": "server_side_default"
+          },
+          "webhook_notifications": {
+            "action": "skip",
+            "reason": "server_side_default"
+          }
+        }
+      }
+    }
+  }
+}

acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.terraform.json

@@ -0,0 +1,7 @@
+{
+  "plan": {
+    "resources.jobs.foo": {
+      "action": "update"
+    }
+  }
+}

acceptance/bundle/resources/jobs/remote_add_tag/output.txt

@@ -0,0 +1,11 @@
+create jobs.foo
+
+Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged
+Uploading hello.whl...
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+update jobs.foo
+
+Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged

acceptance/bundle/resources/jobs/remote_add_tag/script

@@ -0,0 +1,12 @@
+touch hello.whl
+$CLI bundle plan
+$CLI bundle deploy
+job_id="$(read_id.py jobs foo)"
+echo "$job_id:JOB_ID" >> ACC_REPLS
+
+edit_resource.py jobs $job_id <<EOF
+r["tags"]["new_tag"] = "new_value"
+EOF
+
+$CLI bundle plan
+$CLI bundle plan -o json > out.plan_post_update.$DATABRICKS_BUNDLE_ENGINE.json

acceptance/bundle/resources/jobs/remote_add_tag/test.toml

@@ -0,0 +1,2 @@
+RecordRequests = false
+Ignore = [".databricks", "hello.whl"]

bundle/direct/bundle_plan.go

@@ -260,12 +260,11 @@ func interpretOldStateVsRemoteState(ctx context.Context, adapter *dresources.Ada
 	m := make(map[string]deployplan.Trigger)
 
 	for _, ch := range diff {
-		if ch.Old == nil && ch.Path.IsStringKey() {
+		if ch.Old == nil && ch.Path.IsDotString() {
 			// The field was not set by us, but comes from the remote state.
 			// This could either be server-side default or a policy.
 			// In any case, this is not a change we should react to.
-			// Note, we only consider StringKeys here, because indexes and key-value pairs refer to slices and we want to react to new element in slices.
-			// Note, IsStringKey is also too broad - it currently covers struct fields and map keys, we don't want to include map keys here.
+			// Note, we only consider struct fields here. Adding/removing elements to/from maps and slices should trigger updates.
 			m[ch.Path.String()] = deployplan.Trigger{
 				Action: deployplan.ActionTypeSkipString,
 				Reason: "server_side_default",

libs/structs/structdiff/diff.go

@@ -205,7 +205,7 @@ func diffStruct(ctx *diffContext, path *structpath.PathNode, s1, s2 reflect.Valu
 		if fieldName == "" {
 			fieldName = sf.Name
 		}
-		node := structpath.NewStringKey(path, fieldName)
+		node := structpath.NewDotString(path, fieldName)
 
 		v1Field := s1.Field(i)
 		v2Field := s2.Field(i)
@@ -257,7 +257,7 @@ func diffMapStringKey(ctx *diffContext, path *structpath.PathNode, m1, m2 reflec
 		k := keySet[ks]
 		v1 := m1.MapIndex(k)
 		v2 := m2.MapIndex(k)
-		node := structpath.NewStringKey(path, ks)
+		node := structpath.NewBracketString(path, ks)
 		if err := diffValues(ctx, node, v1, v2, changes); err != nil {
 			return err
 		}

libs/structs/structdiff/diff_test.go

@@ -147,7 +147,7 @@ func TestGetStructDiff(t *testing.T) {
 			name: "map diff",
 			a:    A{M: map[string]int{"a": 1}},
 			b:    A{M: map[string]int{"a": 2}},
-			want: []ResolvedChange{{Field: "m.a", Old: 1, New: 2}},
+			want: []ResolvedChange{{Field: "m['a']", Old: 1, New: 2}},
 		},
 		{
 			name: "slice diff",
@@ -258,9 +258,9 @@ func TestGetStructDiff(t *testing.T) {
 			a:    map[string]C{"key1": {Title: "title", ForceSendFields: []string{"Name", "IsEnabled", "Title"}}},
 			b:    map[string]C{"key1": {Title: "title", ForceSendFields: []string{"Age"}}},
 			want: []ResolvedChange{
-				{Field: "key1.name", Old: "", New: nil},
-				{Field: "key1.age", Old: nil, New: 0},
-				{Field: "key1.is_enabled", Old: false, New: nil},
+				{Field: "['key1'].name", Old: "", New: nil},
+				{Field: "['key1'].age", Old: nil, New: 0},
+				{Field: "['key1'].is_enabled", Old: false, New: nil},
 			},
 		},
 
@@ -270,9 +270,9 @@ func TestGetStructDiff(t *testing.T) {
 			a:    map[string]*C{"key1": {Title: "title", ForceSendFields: []string{"Name", "IsEnabled", "Title"}}},
 			b:    map[string]*C{"key1": {Title: "title", ForceSendFields: []string{"Age"}}},
 			want: []ResolvedChange{
-				{Field: "key1.name", Old: "", New: nil},
-				{Field: "key1.age", Old: nil, New: 0},
-				{Field: "key1.is_enabled", Old: false, New: nil},
+				{Field: "['key1'].name", Old: "", New: nil},
+				{Field: "['key1'].age", Old: nil, New: 0},
+				{Field: "['key1'].is_enabled", Old: false, New: nil},
 			},
 		},