Skip to content

Commit 7783e65

Browse files
denikdenik
authored
Represent struct fields and map keys; fix ss_default to ignore map keys (#4041)
## Changes - Update structpath to distinguish between fields (.field) and map keys (["field"]). Note, when it comes to references, we still accept any syntax. However, structdiff and structwalk accurately represent map keys now. - Update server_side_default logic to ignore map keys. ## Why It was never the intention for map entries to have "server_side_default" feature. (Similar to slice indices in #4038). Issue appeared when we stopped distinguishing between fields and map keys in struct path #3640 Note, originally the difference between fields and map keys was removed so that a) users can use either syntax without worrying about matching the type. This is still the case, structaccess still treats those two the same way. b) "resources.jobs.foo" is printed with dots and not like 'resources.jobs["foo"]'. This is still the case, we don't use structpath's String() for this. ## Tests New regression test.
1 parent a3f9821 commit 7783e65

File tree

18 files changed

+333
-42
lines changed

18 files changed

+333
-42
lines changed

‎acceptance/bin/edit_resource.py‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/usr/bin/env python3
2+
"""
3+
Implements get / update / set for a given resource.
4+
5+
Fetches a given resource by id from the backend, executes Python code read from stdin and updates the resource.
6+
"""
7+
8+
import sys
9+
import os
10+
import subprocess
11+
import argparse
12+
import json
13+
import pprint
14+
15+
sys.path.insert(0, os.path.dirname(__file__))
16+
import util
17+
from util import run_json, run
18+
19+
20+
CLI = os.environ["CLI"]
21+
22+
23+
# Each class should be named after CLI command group and implement get(id) and set(id, value) methods:
24+
25+
26+
class jobs:
27+
def get(self, job_id):
28+
return run_json([CLI, "jobs", "get", job_id])["settings"]
29+
30+
def set(self, job_id, value):
31+
payload = {"job_id": job_id, "new_settings": value}
32+
return run([CLI, "jobs", "reset", job_id, "--json", json.dumps(payload)])
33+
34+
35+
def main():
36+
parser = argparse.ArgumentParser()
37+
parser.add_argument("type")
38+
parser.add_argument("id")
39+
parser.add_argument("-v", "--verbose", action="store_true")
40+
args = parser.parse_args()
41+
42+
util.VERBOSE = args.verbose
43+
44+
script = sys.stdin.read()
45+
46+
klass = globals()[args.type]
47+
instance = klass()
48+
49+
data = instance.get(args.id)
50+
my_locals = {"r": data}
51+
52+
try:
53+
exec(script, locals=my_locals)
54+
except Exception:
55+
pprint.pprint(my_locals)
56+
raise
57+
58+
instance.set(args.id, my_locals["r"])
59+
60+
61+
if __name__ == "__main__":
62+
main()

‎acceptance/bin/util.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
import sys
2+
import subprocess
3+
import json
4+
import shlex
5+
6+
7+
VERBOSE = False
8+
9+
10+
class RunError(Exception):
11+
pass
12+
13+
14+
def run_json(cmd):
15+
if VERBOSE:
16+
print("+ " + " ".join([shlex.quote(x) for x in cmd]), file=sys.stderr, flush=True)
17+
result = subprocess.run(cmd, stdout=subprocess.PIPE, encoding="utf-8")
18+
if VERBOSE and result.stdout:
19+
print(result.stdout, flush=True)
20+
if result.returncode != 0:
21+
raise RunError(f"{cmd} failed with code {result.returncode}\n{result.stdout}".strip())
22+
try:
23+
return json.loads(result.stdout)
24+
except Exception as ex:
25+
raise RunError(f"{cmd} returned non-json: {ex}\n{result.stdout}")
26+
27+
28+
def run(cmd):
29+
if VERBOSE:
30+
print("+ " + " ".join([shlex.quote(x) for x in cmd]), file=sys.stderr, flush=True)
31+
result = subprocess.run(cmd)
32+
if result.returncode != 0:
33+
raise RunError(f"{cmd} failed with code {result.returncode}")
34+
return result

‎acceptance/bundle/migrate/basic/out.plan_update.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,7 @@
175175
"name": {
176176
"action": "update"
177177
},
178-
"tags.myjob_name": {
178+
"tags['myjob_name']": {
179179
"action": "update"
180180
}
181181
},

‎acceptance/bundle/resources/jobs/remote_add_tag/databricks.yml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
resources:
2+
jobs:
3+
foo:
4+
name: "My Wheel Job"
5+
tags:
6+
tag1: tag value
7+
tasks:
8+
- task_key: TestTask
9+
python_wheel_task:
10+
package_name: "my_test_code"
11+
entry_point: "run"
12+
environment_key: test_env
13+
libraries:
14+
- whl: hello.whl

‎acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.direct.json‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
{
2+
"plan": {
3+
"resources.jobs.foo": {
4+
"action": "update",
5+
"new_state": {
6+
"value": {
7+
"deployment": {
8+
"kind": "BUNDLE",
9+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
10+
},
11+
"edit_mode": "UI_LOCKED",
12+
"format": "MULTI_TASK",
13+
"max_concurrent_runs": 1,
14+
"name": "My Wheel Job",
15+
"queue": {
16+
"enabled": true
17+
},
18+
"tags": {
19+
"tag1": "tag value"
20+
},
21+
"tasks": [
22+
{
23+
"environment_key": "test_env",
24+
"libraries": [
25+
{
26+
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/hello.whl"
27+
}
28+
],
29+
"python_wheel_task": {
30+
"entry_point": "run",
31+
"package_name": "my_test_code"
32+
},
33+
"task_key": "TestTask"
34+
}
35+
]
36+
}
37+
},
38+
"remote_state": {
39+
"created_time": [UNIX_TIME_MILLIS],
40+
"creator_user_name": "[USERNAME]",
41+
"job_id": [JOB_ID],
42+
"run_as_user_name": "[USERNAME]",
43+
"settings": {
44+
"deployment": {
45+
"kind": "BUNDLE",
46+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
47+
},
48+
"edit_mode": "UI_LOCKED",
49+
"email_notifications": {},
50+
"format": "MULTI_TASK",
51+
"max_concurrent_runs": 1,
52+
"name": "My Wheel Job",
53+
"queue": {
54+
"enabled": true
55+
},
56+
"tags": {
57+
"new_tag": "new_value",
58+
"tag1": "tag value"
59+
},
60+
"tasks": [
61+
{
62+
"environment_key": "test_env",
63+
"libraries": [
64+
{
65+
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/hello.whl"
66+
}
67+
],
68+
"python_wheel_task": {
69+
"entry_point": "run",
70+
"package_name": "my_test_code"
71+
},
72+
"task_key": "TestTask"
73+
}
74+
],
75+
"timeout_seconds": 0,
76+
"webhook_notifications": {}
77+
}
78+
},
79+
"changes": {
80+
"remote": {
81+
"email_notifications": {
82+
"action": "skip",
83+
"reason": "server_side_default"
84+
},
85+
"tags['new_tag']": {
86+
"action": "update"
87+
},
88+
"timeout_seconds": {
89+
"action": "skip",
90+
"reason": "server_side_default"
91+
},
92+
"webhook_notifications": {
93+
"action": "skip",
94+
"reason": "server_side_default"
95+
}
96+
}
97+
}
98+
}
99+
}
100+
}

‎acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.terraform.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"plan": {
3+
"resources.jobs.foo": {
4+
"action": "update"
5+
}
6+
}
7+
}

‎acceptance/bundle/resources/jobs/remote_add_tag/out.test.toml‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎acceptance/bundle/resources/jobs/remote_add_tag/output.txt‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
create jobs.foo
2+
3+
Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged
4+
Uploading hello.whl...
5+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
6+
Deploying resources...
7+
Updating deployment state...
8+
Deployment complete!
9+
update jobs.foo
10+
11+
Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged

‎acceptance/bundle/resources/jobs/remote_add_tag/script‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
touch hello.whl
2+
$CLI bundle plan
3+
$CLI bundle deploy
4+
job_id="$(read_id.py jobs foo)"
5+
echo "$job_id:JOB_ID" >> ACC_REPLS
6+
7+
edit_resource.py jobs $job_id <<EOF
8+
r["tags"]["new_tag"] = "new_value"
9+
EOF
10+
11+
$CLI bundle plan
12+
$CLI bundle plan -o json > out.plan_post_update.$DATABRICKS_BUNDLE_ENGINE.json

‎acceptance/bundle/resources/jobs/remote_add_tag/test.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
RecordRequests = false
2+
Ignore = [".databricks", "hello.whl"]

0 commit comments

Comments
 (0)