Skip to content

Commit ad2bc94

Browse files
ilyakuz-dbilyakuz-db
committed
Merge branch 'main' of github.com:databricks/cli into fix-yaml-json-formatting
2 parents 1d95659 + c1bc97e commit ad2bc94

File tree

87 files changed

+1644
-633
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

87 files changed

+1644
-633
lines changed

‎acceptance/bin/gron.py‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
#!/usr/bin/env python3
2+
import json
3+
import sys
4+
from pathlib import Path
5+
6+
sys.path.insert(0, str(Path(__file__).parent))
7+
from print_requests import read_json_many
8+
9+
10+
def gron(obj, path="json"):
11+
"""Flatten JSON into greppable assignments.
12+
13+
The path parameter defaults to "json" to match the original gron tool,
14+
which treats the input as a JavaScript variable named "json".
15+
16+
Container declarations are only printed for empty dicts/lists.
17+
This differs from https://github.com/tomnomnom/gron which always prints
18+
container declarations to allow reconstruction via ungron. We don't need
19+
reversibility - we only care about making JSON greppable.
20+
21+
>>> gron({"name": "Tom", "age": 30})
22+
json.name = "Tom";
23+
json.age = 30;
24+
25+
>>> gron({"items": ["apple", "banana"]})
26+
json.items[0] = "apple";
27+
json.items[1] = "banana";
28+
29+
>>> gron({"tasks": [{"libraries": [{"whl": "file.whl"}]}]})
30+
json.tasks[0].libraries[0].whl = "file.whl";
31+
32+
>>> gron({"empty": {}, "items": []})
33+
json.empty = {};
34+
json.items = [];
35+
"""
36+
if isinstance(obj, dict):
37+
if not obj:
38+
print(f"{path} = {{}};")
39+
else:
40+
for key in obj:
41+
gron(obj[key], f"{path}.{key}")
42+
elif isinstance(obj, list):
43+
if not obj:
44+
print(f"{path} = [];")
45+
else:
46+
for i, item in enumerate(obj):
47+
gron(item, f"{path}[{i}]")
48+
else:
49+
print(f"{path} = {json.dumps(obj)};")
50+
51+
52+
def main():
53+
if len(sys.argv) > 1:
54+
with open(sys.argv[1]) as f:
55+
content = f.read()
56+
data = read_json_many(content)
57+
if len(data) == 1:
58+
data = data[0]
59+
else:
60+
content = sys.stdin.read()
61+
data = read_json_many(content)
62+
if len(data) == 1:
63+
data = data[0]
64+
65+
gron(data)
66+
67+
68+
if __name__ == "__main__":
69+
main()

‎acceptance/bundle/artifacts/whl_change_version/output.txt‎

Lines changed: 2 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -64,21 +64,7 @@ dist/my_test_code-0.2.0-py3-none-any.whl
6464

6565
=== Expecting 1 wheel in libraries section in /jobs/reset
6666
>>> jq -s .[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks out.requests.txt
67-
[
68-
{
69-
"existing_cluster_id": "0717-aaaaa-bbbbbb",
70-
"libraries": [
71-
{
72-
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl"
73-
}
74-
],
75-
"python_wheel_task": {
76-
"entry_point": "run",
77-
"package_name": "my_test_code"
78-
},
79-
"task_key": "TestTask"
80-
}
81-
]
67+
json[0].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl";
8268

8369
=== Expecting 1 wheel to be uploaded
8470
>>> jq .path out.requests.txt
@@ -110,21 +96,7 @@ dist/my_test_code-0.2.0-py3-none-any.whl
11096

11197
=== Expecting 1 wheel in libraries section in /jobs/reset
11298
>>> jq -s .[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks out.requests.txt
113-
[
114-
{
115-
"existing_cluster_id": "0717-aaaaa-bbbbbb",
116-
"libraries": [
117-
{
118-
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl"
119-
}
120-
],
121-
"python_wheel_task": {
122-
"entry_point": "run",
123-
"package_name": "my_test_code"
124-
},
125-
"task_key": "TestTask"
126-
}
127-
]
99+
json[0].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl";
128100

129101
=== Expecting 1 wheel to be uploaded
130102
>>> jq .path out.requests.txt

‎acceptance/bundle/artifacts/whl_change_version/script‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ trace $CLI bundle deploy
1717
trace find.py --expect 2 whl # there are now 2 wheels on disk
1818

1919
title "Expecting 1 wheel in libraries section in /jobs/reset"
20-
trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' out.requests.txt
20+
trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' out.requests.txt | gron.py | grep -w libraries
2121

2222
title "Expecting 1 wheel to be uploaded"
2323
trace jq .path out.requests.txt | grep import | sort
@@ -31,7 +31,7 @@ trace $CLI bundle deploy
3131
trace find.py --expect 2 whl
3232

3333
title "Expecting 1 wheel in libraries section in /jobs/reset"
34-
trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' out.requests.txt
34+
trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' out.requests.txt | gron.py | grep -w libraries
3535

3636
title "Expecting 1 wheel to be uploaded"
3737
trace jq .path out.requests.txt | grep import | sort

‎acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,14 +104,18 @@
104104
"run_as_user_name": "[USERNAME]",
105105
"tasks": [
106106
{
107+
"email_notifications": {},
107108
"environment_key": "test_env",
108109
"python_wheel_task": {
109110
"entry_point": "run",
110111
"package_name": "my_test_code"
111112
},
112-
"task_key": "ServerlessTestTask"
113+
"run_if": "ALL_SUCCESS",
114+
"task_key": "ServerlessTestTask",
115+
"timeout_seconds": 0
113116
},
114117
{
118+
"email_notifications": {},
115119
"existing_cluster_id": "0717-132531-5opeqon1",
116120
"for_each_task": {
117121
"inputs": "[1]",
@@ -141,7 +145,9 @@
141145
"entry_point": "run",
142146
"package_name": "my_test_code"
143147
},
144-
"task_key": "TestTask"
148+
"run_if": "ALL_SUCCESS",
149+
"task_key": "TestTask",
150+
"timeout_seconds": 0
145151
}
146152
],
147153
"timeout_seconds": 0,
@@ -159,6 +165,26 @@
159165
"new": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl",
160166
"remote": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl"
161167
},
168+
"tasks[task_key='ServerlessTestTask'].email_notifications": {
169+
"action": "skip",
170+
"reason": "empty_struct",
171+
"remote": {}
172+
},
173+
"tasks[task_key='ServerlessTestTask'].run_if": {
174+
"action": "skip",
175+
"reason": "server_side_default",
176+
"remote": "ALL_SUCCESS"
177+
},
178+
"tasks[task_key='ServerlessTestTask'].timeout_seconds": {
179+
"action": "skip",
180+
"reason": "server_side_default",
181+
"remote": 0
182+
},
183+
"tasks[task_key='TestTask'].email_notifications": {
184+
"action": "skip",
185+
"reason": "empty_struct",
186+
"remote": {}
187+
},
162188
"tasks[task_key='TestTask'].for_each_task.task.libraries[0].whl": {
163189
"action": "update",
164190
"old": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl",
@@ -171,6 +197,16 @@
171197
"new": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl",
172198
"remote": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl"
173199
},
200+
"tasks[task_key='TestTask'].run_if": {
201+
"action": "skip",
202+
"reason": "server_side_default",
203+
"remote": "ALL_SUCCESS"
204+
},
205+
"tasks[task_key='TestTask'].timeout_seconds": {
206+
"action": "skip",
207+
"reason": "server_side_default",
208+
"remote": 0
209+
},
174210
"timeout_seconds": {
175211
"action": "skip",
176212
"reason": "server_side_default",

‎acceptance/bundle/artifacts/whl_dynamic/output.txt‎

Lines changed: 6 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -53,48 +53,9 @@ my_test_code-0.0.1+[UNIX_TIME_NANOS][1].dist-info/RECORD
5353

5454
=== Expecting 2 patched wheels in libraries section in /jobs/create
5555
>>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.tasks out.requests.txt
56-
[
57-
{
58-
"environment_key": "test_env",
59-
"python_wheel_task": {
60-
"entry_point": "run",
61-
"package_name": "my_test_code"
62-
},
63-
"task_key": "ServerlessTestTask"
64-
},
65-
{
66-
"existing_cluster_id": "0717-132531-5opeqon1",
67-
"for_each_task": {
68-
"inputs": "[1]",
69-
"task": {
70-
"existing_cluster_id": "0717-132531-5opeqon1",
71-
"libraries": [
72-
{
73-
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl"
74-
}
75-
],
76-
"python_wheel_task": {
77-
"entry_point": "run",
78-
"package_name": "my_test_code"
79-
},
80-
"task_key": "SubTask"
81-
}
82-
},
83-
"libraries": [
84-
{
85-
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl"
86-
},
87-
{
88-
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"
89-
}
90-
],
91-
"python_wheel_task": {
92-
"entry_point": "run",
93-
"package_name": "my_test_code"
94-
},
95-
"task_key": "TestTask"
96-
}
97-
]
56+
json[1].for_each_task.task.libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl";
57+
json[1].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl";
58+
json[1].libraries[1].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl";
9859

9960
=== Expecting 2 patched wheels to be uploaded
10061
>>> jq .path
@@ -128,48 +89,9 @@ my_test_code-0.0.1+[UNIX_TIME_NANOS][2].dist-info/RECORD
12889

12990
=== Expecting 2 patched wheels in libraries section in /jobs/reset
13091
>>> jq -s .[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks out.requests.txt
131-
[
132-
{
133-
"environment_key": "test_env",
134-
"python_wheel_task": {
135-
"entry_point": "run",
136-
"package_name": "my_test_code"
137-
},
138-
"task_key": "ServerlessTestTask"
139-
},
140-
{
141-
"existing_cluster_id": "0717-132531-5opeqon1",
142-
"for_each_task": {
143-
"inputs": "[1]",
144-
"task": {
145-
"existing_cluster_id": "0717-132531-5opeqon1",
146-
"libraries": [
147-
{
148-
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl"
149-
}
150-
],
151-
"python_wheel_task": {
152-
"entry_point": "run",
153-
"package_name": "my_test_code"
154-
},
155-
"task_key": "SubTask"
156-
}
157-
},
158-
"libraries": [
159-
{
160-
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl"
161-
},
162-
{
163-
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"
164-
}
165-
],
166-
"python_wheel_task": {
167-
"entry_point": "run",
168-
"package_name": "my_test_code"
169-
},
170-
"task_key": "TestTask"
171-
}
172-
]
92+
json[1].for_each_task.task.libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl";
93+
json[1].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl";
94+
json[1].libraries[1].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl";
17395

17496
=== Expecting 2 pached wheels to be uploaded (Bad: it is currently uploaded twice)
17597
>>> jq .path

‎acceptance/bundle/artifacts/whl_dynamic/script‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ title "Verify contents of the zip file"
1616
trace find.py --expect 1 '.databricks/.*my_test_code.*whl' | xargs ziplist.py
1717

1818
title "Expecting 2 patched wheels in libraries section in /jobs/create"
19-
trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt
19+
trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt | gron.py | grep -w libraries
2020

2121
title "Expecting 2 patched wheels to be uploaded"
2222
trace jq .path < out.requests.txt | grep import | grep whl | sort
@@ -32,7 +32,7 @@ title "Verify contents, it should now have new_module.py"
3232
trace find.py --expect 1 '.databricks/.*my_test_code.*whl' | xargs ziplist.py
3333

3434
title "Expecting 2 patched wheels in libraries section in /jobs/reset"
35-
trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' out.requests.txt
35+
trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' out.requests.txt | gron.py | grep -w libraries
3636

3737
title "Expecting 2 pached wheels to be uploaded (Bad: it is currently uploaded twice)"
3838
trace jq .path < out.requests.txt | grep import | grep whl | sort

‎acceptance/bundle/deploy/experimental-python/output.txt‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,14 @@ Deployment complete!
2626
},
2727
"tasks": [
2828
{
29+
"email_notifications": {},
2930
"notebook_task": {
3031
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_notebook",
3132
"source": "WORKSPACE"
3233
},
33-
"task_key": "my_notebook"
34+
"run_if": "ALL_SUCCESS",
35+
"task_key": "my_notebook",
36+
"timeout_seconds": 0
3437
}
3538
],
3639
"timeout_seconds": 0,

‎acceptance/bundle/deploy/python-notebook/output.txt‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,14 @@ Deployment complete!
2626
},
2727
"tasks": [
2828
{
29+
"email_notifications": {},
2930
"notebook_task": {
3031
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_notebook",
3132
"source": "WORKSPACE"
3233
},
33-
"task_key": "my_notebook"
34+
"run_if": "ALL_SUCCESS",
35+
"task_key": "my_notebook",
36+
"timeout_seconds": 0
3437
}
3538
],
3639
"timeout_seconds": 0,

‎acceptance/bundle/invariant/configs/job_with_task.yml.tmpl‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
bundle:
2+
name: test-bundle-$UNIQUE_NAME
3+
4+
resources:
5+
jobs:
6+
foo:
7+
name: test-job-$UNIQUE_NAME
8+
tasks:
9+
- task_key: only_task
10+
notebook_task:
11+
notebook_path: /Shared/notebook
12+
new_cluster:
13+
spark_version: $DEFAULT_SPARK_VERSION
14+
node_type_id: $NODE_TYPE_ID
15+
num_workers: 1

‎acceptance/bundle/invariant/migrate/out.test.toml‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)