diff --git a/acceptance/bundle/generate/genie_space/databricks.yml b/acceptance/bundle/generate/genie_space/databricks.yml new file mode 100644 index 00000000000..c533b2b6f98 --- /dev/null +++ b/acceptance/bundle/generate/genie_space/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: genie-space-generate diff --git a/acceptance/bundle/generate/genie_space/genie_space.json.tmpl b/acceptance/bundle/generate/genie_space/genie_space.json.tmpl new file mode 100644 index 00000000000..2056f3061ef --- /dev/null +++ b/acceptance/bundle/generate/genie_space/genie_space.json.tmpl @@ -0,0 +1,7 @@ +{ + "title": "test genie space", + "description": "test description", + "parent_path": "/Workspace/test-$UNIQUE_NAME", + "warehouse_id": "test-warehouse-id", + "serialized_space": "{\"tables\":[],\"questions\":[]}" +} diff --git a/acceptance/bundle/generate/genie_space/out.test.toml b/acceptance/bundle/generate/genie_space/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/generate/genie_space/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.geniespace.json b/acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.geniespace.json new file mode 100644 index 00000000000..2c12b3c032c --- /dev/null +++ b/acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.geniespace.json @@ -0,0 +1,4 @@ +{ + "questions": [], + "tables": [] +} diff --git a/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml b/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml new file mode 100644 index 00000000000..1471a901344 --- /dev/null +++ b/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml @@ -0,0 +1,8 @@ +resources: + genie_spaces: + test_genie_space: + title: "test genie space" + warehouse_id: test-warehouse-id + file_path: ../genie_space/test_genie_space.geniespace.json + description: test description + parent_path: /Workspace/test-[UNIQUE_NAME] diff --git a/acceptance/bundle/generate/genie_space/output.txt b/acceptance/bundle/generate/genie_space/output.txt new file mode 100644 index 00000000000..a313a51adc3 --- /dev/null +++ b/acceptance/bundle/generate/genie_space/output.txt @@ -0,0 +1,6 @@ + +>>> [CLI] workspace mkdirs /Workspace/test-[UNIQUE_NAME] + +>>> [CLI] bundle generate genie-space --existing-id [GENIE_SPACE_ID] --genie-space-dir out/genie_space --resource-dir out/resource +Writing genie space to out/genie_space/test_genie_space.geniespace.json +Writing configuration to out/resource/test_genie_space.genie_space.yml diff --git a/acceptance/bundle/generate/genie_space/script b/acceptance/bundle/generate/genie_space/script new file mode 100644 index 00000000000..8e2fa32170a --- /dev/null +++ b/acceptance/bundle/generate/genie_space/script @@ -0,0 +1,8 @@ +trace $CLI workspace mkdirs /Workspace/test-$UNIQUE_NAME + +# create a genie space to import +envsubst < genie_space.json.tmpl > genie_space.json +genie_space_id=$($CLI genie create-space --json @genie_space.json | jq -r '.space_id') +rm genie_space.json + +trace $CLI bundle generate genie-space --existing-id $genie_space_id --genie-space-dir out/genie_space --resource-dir out/resource diff --git a/acceptance/bundle/generate/genie_space/test.toml b/acceptance/bundle/generate/genie_space/test.toml new file mode 100644 index 00000000000..e389c33c277 --- /dev/null +++ b/acceptance/bundle/generate/genie_space/test.toml @@ -0,0 +1,10 @@ +[[Repls]] +Old = '\\\\' +New = '/' + +[[Repls]] +Old = "[0-9a-f]{32}" +New = "[GENIE_SPACE_ID]" + +[Env] +MSYS_NO_PATHCONV = "1" diff --git a/acceptance/bundle/help/bundle-generate/output.txt b/acceptance/bundle/help/bundle-generate/output.txt index 97e8667ac78..39ce9293539 100644 --- a/acceptance/bundle/help/bundle-generate/output.txt +++ b/acceptance/bundle/help/bundle-generate/output.txt @@ -30,6 +30,7 @@ Available Commands: alert Generate configuration for an alert app Generate bundle configuration for a Databricks app dashboard Generate configuration for a dashboard + genie-space Generate configuration for a Genie space job Generate bundle configuration for a job pipeline Generate bundle configuration for a pipeline diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 07cbaac3a63..b6f0f177b3b 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -734,6 +734,25 @@ resources.external_locations.*.grants[*] catalog.PrivilegeAssignment ALL resources.external_locations.*.grants[*].principal string ALL resources.external_locations.*.grants[*].privileges []catalog.Privilege ALL resources.external_locations.*.grants[*].privileges[*] catalog.Privilege ALL +resources.genie_spaces.*.description string ALL +resources.genie_spaces.*.etag string ALL +resources.genie_spaces.*.file_path string INPUT +resources.genie_spaces.*.id string INPUT +resources.genie_spaces.*.lifecycle resources.Lifecycle INPUT +resources.genie_spaces.*.lifecycle.prevent_destroy bool INPUT +resources.genie_spaces.*.modified_status string INPUT +resources.genie_spaces.*.parent_path string ALL +resources.genie_spaces.*.serialized_space any ALL +resources.genie_spaces.*.space_id string ALL +resources.genie_spaces.*.title string ALL +resources.genie_spaces.*.url string INPUT +resources.genie_spaces.*.warehouse_id string ALL +resources.genie_spaces.*.permissions.object_id string ALL +resources.genie_spaces.*.permissions[*] dresources.StatePermission ALL +resources.genie_spaces.*.permissions[*].group_name string ALL +resources.genie_spaces.*.permissions[*].level iam.PermissionLevel ALL +resources.genie_spaces.*.permissions[*].service_principal_name string ALL +resources.genie_spaces.*.permissions[*].user_name string ALL resources.jobs.*.budget_policy_id string ALL resources.jobs.*.continuous *jobs.Continuous ALL resources.jobs.*.continuous.pause_status jobs.PauseStatus ALL diff --git a/acceptance/bundle/resources/genie_spaces/inline/databricks.yml.tmpl b/acceptance/bundle/resources/genie_spaces/inline/databricks.yml.tmpl new file mode 100644 index 00000000000..ea045097468 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/databricks.yml.tmpl @@ -0,0 +1,22 @@ +bundle: + name: deploy-genie-space-inline-$UNIQUE_NAME + +resources: + genie_spaces: + sales_analytics: + title: "Sales Analytics Inline Genie" + description: "Inline serialized_space test" + warehouse_id: "test-warehouse-id" + parent_path: /Users/$CURRENT_USER_NAME + serialized_space: + version: 1 + config: + sample_questions: + - id: "sq-001" + question: ["What is the total revenue?"] + data_sources: + tables: + - identifier: "main.sales.orders" + column_configs: + - column_name: "amount" + get_example_values: true diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.plan.json b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json new file mode 100644 index 00000000000..8cfcc539a44 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json @@ -0,0 +1,28 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "plan": { + "resources.genie_spaces.sales_analytics": { + "action": "skip", + "remote_state": { + "description": "Inline serialized_space test", + "etag": "1", + "parent_path": "/Workspace/Users/[USERNAME]", + "serialized_space": "{\"config\":{\"sample_questions\":[{\"id\":\"sq-001\",\"question\":[\"What is the total revenue?\"]}]},\"data_sources\":{\"tables\":[{\"column_configs\":[{\"column_name\":\"amount\",\"get_example_values\":true}],\"identifier\":\"main.sales.orders\"}]},\"version\":1}", + "space_id": "[GENIE_SPACE_ID]", + "title": "Sales Analytics Inline Genie", + "warehouse_id": "test-warehouse-id" + }, + "changes": { + "etag": { + "action": "skip", + "reason": "custom", + "old": "1", + "remote": "1" + } + } + } + } +} diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.test.toml b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/inline/output.txt b/acceptance/bundle/resources/genie_spaces/inline/output.txt new file mode 100644 index 00000000000..99fef197dd0 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/output.txt @@ -0,0 +1,17 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-genie-space-inline-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.genie_spaces.sales_analytics + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-genie-space-inline-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/genie_spaces/inline/script b/acceptance/bundle/resources/genie_spaces/inline/script new file mode 100644 index 00000000000..8f2f625b7b0 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/script @@ -0,0 +1,18 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy +GENIE_SPACE_ID=$($CLI bundle summary --output json | jq -r '.resources.genie_spaces.sales_analytics.id') + +# Capture the genie space ID as a replacement. +echo "$GENIE_SPACE_ID:GENIE_SPACE_ID" >> ACC_REPLS + +# Plan after deploy must be drift-free aside from input_only fields. +# Without normalization the inline serialized_space leaves a map in the +# config struct while state holds a string, and structdiff reports false +# drift on every plan. +trace $CLI bundle plan -o json > out.plan.json diff --git a/acceptance/bundle/resources/genie_spaces/inline/test.toml b/acceptance/bundle/resources/genie_spaces/inline/test.toml new file mode 100644 index 00000000000..bbdf2380b2d --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/test.toml @@ -0,0 +1,6 @@ +Local = true +RecordRequests = false + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_update/databricks.yml.tmpl b/acceptance/bundle/resources/genie_spaces/parent_path_update/databricks.yml.tmpl new file mode 100644 index 00000000000..d479c3eef35 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/parent_path_update/databricks.yml.tmpl @@ -0,0 +1,10 @@ +bundle: + name: update-genie-space-$UNIQUE_NAME + +resources: + genie_spaces: + update_target: + title: "Update Target" + warehouse_id: "test-warehouse-id" + parent_path: PARENT_PATH_PLACEHOLDER + serialized_space: "{}" diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml b/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_update/output.txt b/acceptance/bundle/resources/genie_spaces/parent_path_update/output.txt new file mode 100644 index 00000000000..76f2dd99b32 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/parent_path_update/output.txt @@ -0,0 +1,21 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-genie-space-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Plan after changing parent_path should show update +>>> [CLI] bundle plan +update genie_spaces.update_target + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.genie_spaces.update_target + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/update-genie-space-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_update/script b/acceptance/bundle/resources/genie_spaces/parent_path_update/script new file mode 100644 index 00000000000..cc7359b67d9 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/parent_path_update/script @@ -0,0 +1,14 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +# Deploy with the original parent_path. +envsubst < databricks.yml.tmpl | sed "s|PARENT_PATH_PLACEHOLDER|/Users/$CURRENT_USER_NAME/genie-old|" > databricks.yml +trace $CLI bundle deploy + +# Change parent_path. The Genie update API moves the space to the new parent, +# so the plan should show an update rather than a recreate (delete + create). +envsubst < databricks.yml.tmpl | sed "s|PARENT_PATH_PLACEHOLDER|/Users/$CURRENT_USER_NAME/genie-new|" > databricks.yml +title "Plan after changing parent_path should show update" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_update/test.toml b/acceptance/bundle/resources/genie_spaces/parent_path_update/test.toml new file mode 100644 index 00000000000..bbdf2380b2d --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/parent_path_update/test.toml @@ -0,0 +1,6 @@ +Local = true +RecordRequests = false + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/databricks.yml.tmpl b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/databricks.yml.tmpl new file mode 100644 index 00000000000..fe135e8b627 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: recreate-gone-genie-space-$UNIQUE_NAME + +resources: + genie_spaces: + sales_analytics: + title: "Recreate When Gone" + warehouse_id: "test-warehouse-id" + parent_path: /Users/$CURRENT_USER_NAME + serialized_space: + version: 1 diff --git a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/output.txt b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/output.txt new file mode 100644 index 00000000000..aeaa5d7964d --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/output.txt @@ -0,0 +1,20 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-gone-genie-space-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] genie trash-space [GENIE_SPACE_ID] + +=== Plan after remote deletion should recreate the space +>>> [CLI] bundle plan +create genie_spaces.sales_analytics + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/recreate-gone-genie-space-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/script b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/script new file mode 100644 index 00000000000..c965df9fcc1 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/script @@ -0,0 +1,18 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy +GENIE_SPACE_ID=$($CLI bundle summary --output json | jq -r '.resources.genie_spaces.sales_analytics.id') +echo "$GENIE_SPACE_ID:GENIE_SPACE_ID" >> ACC_REPLS + +# Delete the space out-of-band, simulating a UI deletion. The GET API then +# returns 403; the resource maps that to "gone" so the plan recreates the +# space instead of failing. +trace $CLI genie trash-space $GENIE_SPACE_ID + +title "Plan after remote deletion should recreate the space" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/test.toml b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/test.toml new file mode 100644 index 00000000000..bbdf2380b2d --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/test.toml @@ -0,0 +1,6 @@ +Local = true +RecordRequests = false + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/bundle/resources/genie_spaces/simple/databricks.yml.tmpl b/acceptance/bundle/resources/genie_spaces/simple/databricks.yml.tmpl new file mode 100644 index 00000000000..f30e4991de9 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: deploy-genie-space-test-$UNIQUE_NAME + +resources: + genie_spaces: + sales_analytics: + title: "Sales Analytics Genie" + description: "AI assistant for sales data analysis" + warehouse_id: "test-warehouse-id" + parent_path: /Users/$CURRENT_USER_NAME + file_path: "sales_analytics.geniespace.json" diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.plan.json b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json new file mode 100644 index 00000000000..77d3669d8ab --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json @@ -0,0 +1,28 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "plan": { + "resources.genie_spaces.sales_analytics": { + "action": "skip", + "remote_state": { + "description": "AI assistant for sales data analysis", + "etag": "1", + "parent_path": "/Workspace/Users/[USERNAME]", + "serialized_space": "{\n \"benchmarks\": {\n \"questions\": [\n {\n \"answer\": [\n {\n \"content\": [\n \"SELECT\\n\",\n \" name,\\n\",\n \" country\\n\",\n \"FROM main.default.countries\\n\",\n \"ORDER BY name\"\n ],\n \"format\": \"SQL\"\n }\n ],\n \"id\": \"[NUMID]\",\n \"question\": [\n \"Show all names and countries\"\n ]\n }\n ]\n },\n \"config\": {\n \"sample_questions\": [\n {\n \"id\": \"[NUMID]\",\n \"question\": [\n \"List the names and countries\"\n ]\n },\n {\n \"id\": \"[NUMID]\",\n \"question\": [\n \"Which names are in Canada?\"\n ]\n }\n ]\n },\n \"data_sources\": {\n \"tables\": [\n {\n \"column_configs\": [\n {\n \"column_name\": \"country\",\n \"enable_entity_matching\": true,\n \"enable_format_assistance\": true\n },\n {\n \"column_name\": \"name\",\n \"enable_entity_matching\": true,\n \"enable_format_assistance\": true\n }\n ],\n \"identifier\": \"main.default.countries\"\n }\n ]\n },\n \"instructions\": {\n \"example_question_sqls\": [\n {\n \"id\": \"[NUMID]\",\n \"question\": [\n \"List the names and countries\"\n ],\n \"sql\": [\n \"SELECT\\n\",\n \" name,\\n\",\n \" country\\n\",\n \"FROM main.default.countries\\n\",\n \"ORDER BY name\"\n ]\n }\n ],\n \"text_instructions\": [\n {\n \"content\": [\n \"This genie space answers simple questions about people and their countries.\\n\",\n \"Use only the main.default.countries table.\\n\",\n \"Prefer returning the name and country columns directly.\"\n ],\n \"id\": \"[NUMID]\"\n }\n ]\n },\n \"version\": 2\n}\n", + "space_id": "[GENIE_SPACE_ID]", + "title": "Sales Analytics Genie", + "warehouse_id": "test-warehouse-id" + }, + "changes": { + "etag": { + "action": "skip", + "reason": "custom", + "old": "1", + "remote": "1" + } + } + } + } +} diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.test.toml b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/simple/output.txt b/acceptance/bundle/resources/genie_spaces/simple/output.txt new file mode 100644 index 00000000000..f65d9583e18 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/output.txt @@ -0,0 +1,24 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-genie-space-test-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] genie get-space [GENIE_SPACE_ID] +{ + "title": "Sales Analytics Genie", + "description": "AI assistant for sales data analysis", + "warehouse_id": "test-warehouse-id" +} + +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.genie_spaces.sales_analytics + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-genie-space-test-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.geniespace.json b/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.geniespace.json new file mode 100644 index 00000000000..fb62b7c4859 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.geniespace.json @@ -0,0 +1,87 @@ +{ + "benchmarks": { + "questions": [ + { + "answer": [ + { + "content": [ + "SELECT\n", + " name,\n", + " country\n", + "FROM main.default.countries\n", + "ORDER BY name" + ], + "format": "SQL" + } + ], + "id": "88888888888888888888888888888888", + "question": [ + "Show all names and countries" + ] + } + ] + }, + "config": { + "sample_questions": [ + { + "id": "11111111111111111111111111111111", + "question": [ + "List the names and countries" + ] + }, + { + "id": "22222222222222222222222222222222", + "question": [ + "Which names are in Canada?" + ] + } + ] + }, + "data_sources": { + "tables": [ + { + "column_configs": [ + { + "column_name": "country", + "enable_entity_matching": true, + "enable_format_assistance": true + }, + { + "column_name": "name", + "enable_entity_matching": true, + "enable_format_assistance": true + } + ], + "identifier": "main.default.countries" + } + ] + }, + "instructions": { + "example_question_sqls": [ + { + "id": "44444444444444444444444444444444", + "question": [ + "List the names and countries" + ], + "sql": [ + "SELECT\n", + " name,\n", + " country\n", + "FROM main.default.countries\n", + "ORDER BY name" + ] + } + ], + "text_instructions": [ + { + "content": [ + "This genie space answers simple questions about people and their countries.\n", + "Use only the main.default.countries table.\n", + "Prefer returning the name and country columns directly." + ], + "id": "33333333333333333333333333333333" + } + ] + }, + "version": 2 +} diff --git a/acceptance/bundle/resources/genie_spaces/simple/script b/acceptance/bundle/resources/genie_spaces/simple/script new file mode 100644 index 00000000000..4db9ef03078 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/script @@ -0,0 +1,17 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy +GENIE_SPACE_ID=$($CLI bundle summary --output json | jq -r '.resources.genie_spaces.sales_analytics.id') + +# Capture the genie space ID as a replacement. +echo "$GENIE_SPACE_ID:GENIE_SPACE_ID" >> ACC_REPLS + +trace $CLI genie get-space $GENIE_SPACE_ID | jq '{title, description, warehouse_id}' + +# Verify that there is no drift right after deploy. +trace $CLI bundle plan -o json > out.plan.json diff --git a/acceptance/bundle/resources/genie_spaces/simple/test.toml b/acceptance/bundle/resources/genie_spaces/simple/test.toml new file mode 100644 index 00000000000..bbdf2380b2d --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/test.toml @@ -0,0 +1,6 @@ +Local = true +RecordRequests = false + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/bundle/resources/genie_spaces/test.toml b/acceptance/bundle/resources/genie_spaces/test.toml new file mode 100644 index 00000000000..7f397c47833 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/test.toml @@ -0,0 +1,2 @@ +# Genie spaces are only deployed via the direct deployment engine. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/databricks.yml b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/databricks.yml new file mode 100644 index 00000000000..d9e1c56c35c --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: test-bundle + +resources: + genie_spaces: + foo: + title: "Permissions Test Space" + warehouse_id: test-warehouse-id + parent_path: /Workspace/Users/tester@databricks.com + serialized_space: "{}" + permissions: + - level: CAN_READ + user_name: viewer@example.com + - level: CAN_MANAGE + group_name: data-team + - level: CAN_MANAGE + service_principal_name: f37d18cd-98a8-4db5-8112-12dd0a6bfe38 + - level: CAN_MANAGE + user_name: tester@databricks.com diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json new file mode 100644 index 00000000000..5e1e8dd17b3 --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json @@ -0,0 +1,52 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.genie_spaces.foo": { + "action": "create", + "new_state": { + "value": { + "parent_path": "/Workspace/Users/[USERNAME]", + "serialized_space": "{}", + "title": "Permissions Test Space", + "warehouse_id": "test-warehouse-id" + } + } + }, + "resources.genie_spaces.foo.permissions": { + "depends_on": [ + { + "node": "resources.genie_spaces.foo", + "label": "${resources.genie_spaces.foo.id}" + } + ], + "action": "create", + "new_state": { + "value": { + "object_id": "", + "__embed__": [ + { + "level": "CAN_READ", + "user_name": "viewer@example.com" + }, + { + "level": "CAN_MANAGE", + "group_name": "data-team" + }, + { + "level": "CAN_MANAGE", + "service_principal_name": "[UUID]" + }, + { + "level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] + }, + "vars": { + "object_id": "/genie/${resources.genie_spaces.foo.id}" + } + } + } + } +} diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.deploy.direct.json new file mode 100644 index 00000000000..abcba718ab7 --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.deploy.direct.json @@ -0,0 +1,24 @@ +{ + "method": "PUT", + "path": "/api/2.0/permissions/genie/[FOO_ID]", + "body": { + "access_control_list": [ + { + "permission_level": "CAN_READ", + "user_name": "viewer@example.com" + }, + { + "group_name": "data-team", + "permission_level": "CAN_MANAGE" + }, + { + "permission_level": "CAN_MANAGE", + "service_principal_name": "[UUID]" + }, + { + "permission_level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] + } +} diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.destroy.direct.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/output.txt b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/output.txt new file mode 100644 index 00000000000..e4e8cf0189d --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/output.txt @@ -0,0 +1,35 @@ + +>>> [CLI] bundle validate -o json +[ + { + "level": "CAN_READ", + "user_name": "viewer@example.com" + }, + { + "group_name": "data-team", + "level": "CAN_MANAGE" + }, + { + "level": "CAN_MANAGE", + "service_principal_name": "[UUID]" + }, + { + "level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } +] + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.genie_spaces.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script new file mode 100644 index 00000000000..1b20af07543 --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script @@ -0,0 +1,18 @@ +trace $CLI bundle validate -o json | jq .resources.genie_spaces.foo.permissions +rm out.requests.txt + +$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json + +print_requests() { + jq -c < out.requests.txt | jq 'select(.method != "GET" and (.path | contains("permissions")))' + rm out.requests.txt +} + +rm out.requests.txt +trace $CLI bundle deploy +# Genie space IDs are random; normalize them in the recorded requests. +replace_ids.py +print_requests > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json + +trace $CLI bundle destroy --auto-approve +print_requests > out.requests.destroy.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/permissions/genie_spaces/test.toml b/acceptance/bundle/resources/permissions/genie_spaces/test.toml new file mode 100644 index 00000000000..390388f04aa --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/test.toml @@ -0,0 +1,4 @@ +Env.RESOURCE = "genie_spaces" # for ../_script + +# Genie spaces only support the direct deployment engine. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/permissions/output.txt b/acceptance/bundle/resources/permissions/output.txt index 85eea2e6e9c..12a35fcfd86 100644 --- a/acceptance/bundle/resources/permissions/output.txt +++ b/acceptance/bundle/resources/permissions/output.txt @@ -99,6 +99,8 @@ DIFF experiments/current_can_manage/out.requests.destroy.direct.json + "path": "/api/2.0/permissions/experiments/[NUMID]" + } +] +DIRECT_ONLY genie_spaces/current_can_manage/out.requests.deploy.direct.json +DIRECT_ONLY genie_spaces/current_can_manage/out.requests.destroy.direct.json MATCH jobs/current_can_manage/out.requests.deploy.direct.json DIFF jobs/current_can_manage/out.requests.destroy.direct.json --- jobs/current_can_manage/out.requests.destroy.direct.json diff --git a/acceptance/bundle/validate/genie_space_complex/databricks.yml b/acceptance/bundle/validate/genie_space_complex/databricks.yml new file mode 100644 index 00000000000..94c5fc7188c --- /dev/null +++ b/acceptance/bundle/validate/genie_space_complex/databricks.yml @@ -0,0 +1,51 @@ +bundle: + name: genie-space-complex + +workspace: + resource_path: /foo/bar + +resources: + genie_spaces: + # Test with all features enabled + full_featured: + warehouse_id: "my-warehouse-1234" + title: "Full Featured Genie Space" + description: "A comprehensive test of all genie space features" + file_path: ./full_featured.geniespace.json + + # Test with inline serialized_space (YAML syntax) + inline_yaml: + warehouse_id: "my-warehouse-1234" + title: "Inline YAML Genie Space" + serialized_space: + version: 1 + data_sources: + tables: + - identifier: main.schema.table1 + column_configs: + - column_name: id + get_example_values: true + build_value_dictionary: true + - column_name: name + get_example_values: true + instructions: + text_instructions: + - id: inst-001 + content: + - "This is a text instruction.\n" + - "It spans multiple lines." + example_question_sqls: + - id: eq-001 + question: + - "How many records are there?" + sql: + - "SELECT COUNT(*) FROM main.schema.table1" + + # Test with empty but valid structure + minimal_valid: + warehouse_id: "my-warehouse-1234" + title: "Minimal Valid" + serialized_space: + version: 1 + data_sources: + tables: [] diff --git a/acceptance/bundle/validate/genie_space_complex/full_featured.geniespace.json b/acceptance/bundle/validate/genie_space_complex/full_featured.geniespace.json new file mode 100644 index 00000000000..9c9221d8328 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_complex/full_featured.geniespace.json @@ -0,0 +1,160 @@ +{ + "version": 1, + "config": { + "sample_questions": [ + { + "id": "sq-001", + "question": ["What is the total revenue?"] + }, + { + "id": "sq-002", + "question": ["Show me the top customers"] + } + ] + }, + "data_sources": { + "tables": [ + { + "identifier": "main.sales.orders", + "column_configs": [ + { + "column_name": "order_id", + "get_example_values": true, + "build_value_dictionary": true + }, + { + "column_name": "customer_id", + "get_example_values": true + }, + { + "column_name": "amount", + "get_example_values": false + } + ] + }, + { + "identifier": "main.sales.customers" + } + ] + }, + "instructions": { + "text_instructions": [ + { + "id": "ti-001", + "content": [ + "This genie space analyzes sales data.\n", + "Always filter by date when querying orders.\n", + "Use customer_name instead of customer_id in results." + ] + } + ], + "example_question_sqls": [ + { + "id": "eq-001", + "question": ["What are the top customers by revenue?"], + "sql": [ + "SELECT\n", + " c.customer_name,\n", + " SUM(o.amount) AS total_revenue\n", + "FROM main.sales.orders o\n", + "JOIN main.sales.customers c ON o.customer_id = c.id\n", + "WHERE o.order_date >= :start_date\n", + "GROUP BY c.customer_name\n", + "ORDER BY total_revenue DESC\n", + "LIMIT :limit" + ], + "parameters": [ + { + "name": "start_date", + "type_hint": "STRING", + "description": ["Start date for the analysis period"], + "default_value": { + "values": ["2024-01-01"] + } + }, + { + "name": "limit", + "type_hint": "INTEGER", + "description": ["Number of customers to return"], + "default_value": { + "values": ["10"] + } + } + ] + }, + { + "id": "eq-002", + "question": ["Calculate daily revenue"], + "sql": [ + "SELECT\n", + " order_date,\n", + " SUM(amount) AS daily_revenue\n", + "FROM main.sales.orders\n", + "GROUP BY order_date\n", + "ORDER BY order_date" + ] + } + ], + "sql_snippets": { + "measures": [ + { + "id": "m-001", + "alias": "total_revenue", + "sql": ["SUM(orders.amount)"], + "display_name": "Total Revenue" + }, + { + "id": "m-002", + "alias": "order_count", + "sql": ["COUNT(orders.order_id)"], + "display_name": "Order Count" + } + ] + }, + "sql_functions": [ + { + "id": "sf-001", + "identifier": "main.analytics.calculate_churn" + } + ] + }, + "benchmarks": { + "questions": [ + { + "id": "bq-001", + "question": ["What is the monthly revenue trend?"], + "answer": [ + { + "format": "SQL", + "content": [ + "SELECT\n", + " DATE_TRUNC('month', order_date) AS month,\n", + " SUM(amount) AS revenue\n", + "FROM main.sales.orders\n", + "GROUP BY 1\n", + "ORDER BY 1" + ] + } + ] + }, + { + "id": "bq-002", + "question": ["Which customers have the highest average order value?"], + "answer": [ + { + "format": "SQL", + "content": [ + "SELECT\n", + " customer_id,\n", + " AVG(amount) AS avg_order_value\n", + "FROM main.sales.orders\n", + "GROUP BY customer_id\n", + "ORDER BY avg_order_value DESC\n", + "LIMIT 10" + ] + } + ] + } + ] + } +} diff --git a/acceptance/bundle/validate/genie_space_complex/out.test.toml b/acceptance/bundle/validate/genie_space_complex/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_complex/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_complex/output.txt b/acceptance/bundle/validate/genie_space_complex/output.txt new file mode 100644 index 00000000000..4a0747210cc --- /dev/null +++ b/acceptance/bundle/validate/genie_space_complex/output.txt @@ -0,0 +1,22 @@ +{ + "full_featured": { + "title": "Full Featured Genie Space", + "warehouse_id": "my-warehouse-1234", + "serialized_space_is_string": true + }, + "inline_yaml": { + "title": "Inline YAML Genie Space", + "serialized_space_type": "string", + "parsed": { + "tables_count": 1, + "has_column_configs": true, + "has_text_instructions": true + } + }, + "minimal_valid": { + "title": "Minimal Valid", + "parsed": { + "tables_count": 0 + } + } +} diff --git a/acceptance/bundle/validate/genie_space_complex/script b/acceptance/bundle/validate/genie_space_complex/script new file mode 100644 index 00000000000..4feab1e373b --- /dev/null +++ b/acceptance/bundle/validate/genie_space_complex/script @@ -0,0 +1,26 @@ +# Validate complex genie spaces. ConfigureGenieSpaceSerializedSpace +# normalizes inline serialized_space YAML to a JSON string so the field has +# the same shape as the file_path code path; the script parses the JSON +# back to verify that the original structure is preserved. +$CLI bundle validate -o json | jq '{ + full_featured: .resources.genie_spaces.full_featured | { + title, + warehouse_id, + serialized_space_is_string: (.serialized_space | type == "string") + }, + inline_yaml: .resources.genie_spaces.inline_yaml | { + title, + serialized_space_type: (.serialized_space | type), + parsed: (.serialized_space | fromjson) | { + tables_count: (.data_sources.tables | length), + has_column_configs: ((.data_sources.tables[0].column_configs | length) > 0), + has_text_instructions: ((.instructions.text_instructions | length) > 0) + } + }, + minimal_valid: .resources.genie_spaces.minimal_valid | { + title, + parsed: (.serialized_space | fromjson) | { + tables_count: (.data_sources.tables | length) + } + } +}' diff --git a/acceptance/bundle/validate/genie_space_defaults/databricks.yml b/acceptance/bundle/validate/genie_space_defaults/databricks.yml new file mode 100644 index 00000000000..7e4b87d35e6 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_defaults/databricks.yml @@ -0,0 +1,30 @@ +bundle: + name: test-bundle + +workspace: + resource_path: /foo/bar + +resources: + genie_spaces: + empty_string: + warehouse_id: "my-warehouse-1234" + title: "empty-string" + serialized_space: "{}" + + # unchanged + parent_path: "" + + non_empty_string: + warehouse_id: "my-warehouse-1234" + title: "non-empty-string" + serialized_space: "{}" + + # unchanged + parent_path: "already-set" + + default_parent_path: + warehouse_id: "my-warehouse-1234" + title: "default-parent-path" + serialized_space: "{}" + + # parent_path set to default diff --git a/acceptance/bundle/validate/genie_space_defaults/out.test.toml b/acceptance/bundle/validate/genie_space_defaults/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_defaults/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_defaults/output.txt b/acceptance/bundle/validate/genie_space_defaults/output.txt new file mode 100644 index 00000000000..13105c91b76 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_defaults/output.txt @@ -0,0 +1,14 @@ +{ + "default_parent_path": { + "title": "default-parent-path", + "parent_path": "/Workspace/foo/bar" + }, + "empty_string": { + "title": "empty-string", + "parent_path": "/Workspace" + }, + "non_empty_string": { + "title": "non-empty-string", + "parent_path": "/Workspace/already-set" + } +} diff --git a/acceptance/bundle/validate/genie_space_defaults/script b/acceptance/bundle/validate/genie_space_defaults/script new file mode 100644 index 00000000000..eebc582f136 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_defaults/script @@ -0,0 +1 @@ +$CLI bundle validate -o json | jq '.resources.genie_spaces | map_values({title: .title, parent_path: .parent_path})' diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/contents.geniespace.json b/acceptance/bundle/validate/genie_space_file_path_and_inline/contents.geniespace.json new file mode 100644 index 00000000000..cb608f6e9c4 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/contents.geniespace.json @@ -0,0 +1 @@ +{"version": 1} diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/databricks.yml b/acceptance/bundle/validate/genie_space_file_path_and_inline/databricks.yml new file mode 100644 index 00000000000..ca57e978d83 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: genie-space-file-path-and-inline + +resources: + genie_spaces: + both_set: + title: "Both set" + warehouse_id: "test-warehouse-id" + file_path: "./contents.geniespace.json" + serialized_space: + version: 1 diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/output.txt b/acceptance/bundle/validate/genie_space_file_path_and_inline/output.txt new file mode 100644 index 00000000000..589a8333c37 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/output.txt @@ -0,0 +1,10 @@ +Warning: both file_path and serialized_space are set; file_path will be used and serialized_space will be ignored + in databricks.yml:11:9 + +Name: genie-space-file-path-and-inline +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/genie-space-file-path-and-inline/default + +Found 1 warning diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/script b/acceptance/bundle/validate/genie_space_file_path_and_inline/script new file mode 100644 index 00000000000..72555b332a4 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/script @@ -0,0 +1 @@ +$CLI bundle validate diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/test.toml b/acceptance/bundle/validate/genie_space_file_path_and_inline/test.toml new file mode 100644 index 00000000000..97900adac7a --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = false + +# Genie spaces only support direct deployment engine. +[Env] +DATABRICKS_BUNDLE_ENGINE = "direct" diff --git a/acceptance/experimental/open/output.txt b/acceptance/experimental/open/output.txt index af12759c8a4..c395bb4967e 100644 --- a/acceptance/experimental/open/output.txt +++ b/acceptance/experimental/open/output.txt @@ -9,7 +9,7 @@ === unknown resource type >>> [CLI] experimental open --url unknown 123 -Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses +Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses Exit code: 1 @@ -23,6 +23,7 @@ dashboards database_catalogs database_instances experiments +genie_spaces jobs model_serving_endpoints models diff --git a/bundle/config/mutator/paths/genie_space_paths_visitor.go b/bundle/config/mutator/paths/genie_space_paths_visitor.go new file mode 100644 index 00000000000..edd6ff2d8df --- /dev/null +++ b/bundle/config/mutator/paths/genie_space_paths_visitor.go @@ -0,0 +1,18 @@ +package paths + +import ( + "github.com/databricks/cli/libs/dyn" +) + +func VisitGenieSpacePaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { + pattern := dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("genie_spaces"), + dyn.AnyKey(), + dyn.Key("file_path"), + ) + + return dyn.MapByPattern(value, pattern, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + return fn(path, TranslateModeLocalRelative, value) + }) +} diff --git a/bundle/config/mutator/paths/visitor.go b/bundle/config/mutator/paths/visitor.go index 0e3d59d43f5..bdf42188fde 100644 --- a/bundle/config/mutator/paths/visitor.go +++ b/bundle/config/mutator/paths/visitor.go @@ -15,6 +15,7 @@ func VisitPaths(root dyn.Value, fn VisitFunc) (dyn.Value, error) { VisitArtifactPaths, VisitAlertPaths, VisitDashboardPaths, + VisitGenieSpacePaths, VisitPipelinePaths, VisitPipelineLibrariesPaths, } diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go index fd019479d77..cbb05d7d622 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go @@ -47,6 +47,10 @@ var ( permissions.CAN_MANAGE: "CAN_MANAGE", permissions.CAN_VIEW: "CAN_READ", }, + "genie_spaces": { + permissions.CAN_MANAGE: "CAN_MANAGE", + permissions.CAN_VIEW: "CAN_READ", + }, "apps": { permissions.CAN_MANAGE: "CAN_MANAGE", permissions.CAN_VIEW: "CAN_USE", diff --git a/bundle/config/mutator/resourcemutator/apply_presets.go b/bundle/config/mutator/resourcemutator/apply_presets.go index 663adf4a281..79c0ebbec3c 100644 --- a/bundle/config/mutator/resourcemutator/apply_presets.go +++ b/bundle/config/mutator/resourcemutator/apply_presets.go @@ -237,6 +237,14 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos dashboard.DisplayName = prefix + dashboard.DisplayName } + // Genie Spaces: Prefix + for _, genieSpace := range r.GenieSpaces { + if genieSpace == nil { + continue + } + genieSpace.Title = prefix + genieSpace.Title + } + // Apps: No presets // Alerts: Prefix, TriggerPauseStatus diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index f8063ddf346..440221001a7 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -154,6 +154,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, }, + GenieSpaces: map[string]*resources.GenieSpace{ + "geniespace1": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "geniespace1", + }, + }, + }, Apps: map[string]*resources.App{ "app1": { App: apps.App{ @@ -350,6 +357,9 @@ func TestProcessTargetModeDevelopment(t *testing.T) { // Dashboards assert.Equal(t, "[dev lennart] dashboard1", b.Config.Resources.Dashboards["dashboard1"].DisplayName) + // Genie Spaces + assert.Equal(t, "[dev lennart] geniespace1", b.Config.Resources.GenieSpaces["geniespace1"].Title) + // Alert 1: has schedule without pause status set - should be paused assert.Equal(t, "[dev lennart] alert1", b.Config.Resources.Alerts["alert1"].DisplayName) assert.Equal(t, sql.SchedulePauseStatusPaused, b.Config.Resources.Alerts["alert1"].Schedule.PauseStatus) diff --git a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go new file mode 100644 index 00000000000..a9327217c08 --- /dev/null +++ b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go @@ -0,0 +1,81 @@ +package resourcemutator + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +const serializedSpaceFieldName = "serialized_space" + +type configureGenieSpaceSerializedSpace struct{} + +func ConfigureGenieSpaceSerializedSpace() bundle.Mutator { + return &configureGenieSpaceSerializedSpace{} +} + +func (c configureGenieSpaceSerializedSpace) Name() string { + return "ConfigureGenieSpaceSerializedSpace" +} + +func (c configureGenieSpaceSerializedSpace) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + pattern := dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("genie_spaces"), + dyn.AnyKey(), + ) + + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + filePath, hasFilePath := v.Get(filePathFieldName).AsString() + ss := v.Get(serializedSpaceFieldName) + + if hasFilePath { + if ss.IsValid() && ss.Kind() != dyn.KindNil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: "both file_path and serialized_space are set; file_path will be used and serialized_space will be ignored", + Locations: ss.Locations(), + }) + } + contents, err := b.SyncRoot.ReadFile(filePath) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to read serialized genie space from file_path %s: %w", filePath, err) + } + return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(contents))) + } + + // Marshal an inline structured serialized_space to a JSON string so + // both config-side and state-side carry the same plain string. + // Otherwise YAML decodes small ints as Go `int` while state JSON + // round-trip decodes them as `float64`, and structdiff reports + // false drift on every plan. + switch ss.Kind() { + case dyn.KindNil, dyn.KindString: + return v, nil + case dyn.KindMap, dyn.KindSequence: + jsonBytes, err := json.Marshal(ss.AsAny()) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to marshal inline serialized_space: %w", err) + } + return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(jsonBytes))) + default: + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("serialized_space must be a string, map, or sequence, got %s", ss.Kind()), + Locations: ss.Locations(), + }) + return v, nil + } + }) + }) + + diags = diags.Extend(diag.FromErr(err)) + return diags +} diff --git a/bundle/config/mutator/resourcemutator/genie_space_fixups.go b/bundle/config/mutator/resourcemutator/genie_space_fixups.go new file mode 100644 index 00000000000..85e1bb7e745 --- /dev/null +++ b/bundle/config/mutator/resourcemutator/genie_space_fixups.go @@ -0,0 +1,30 @@ +package resourcemutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" +) + +type genieSpaceFixups struct{} + +func GenieSpaceFixups() bundle.Mutator { + return &genieSpaceFixups{} +} + +func (m *genieSpaceFixups) Name() string { + return "GenieSpaceFixups" +} + +func (m *genieSpaceFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + for _, genieSpace := range b.Config.Resources.GenieSpaces { + if genieSpace == nil { + continue + } + + genieSpace.ParentPath = ensureWorkspacePrefix(genieSpace.ParentPath) + } + + return nil +} diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 209bbcb06a0..45740f53599 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -52,6 +52,7 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { }{ {"resources.dashboards.*.parent_path", b.Config.Workspace.ResourcePath}, {"resources.dashboards.*.embed_credentials", false}, + {"resources.genie_spaces.*.parent_path", b.Config.Workspace.ResourcePath}, {"resources.volumes.*.volume_type", "MANAGED"}, {"resources.alerts.*.parent_path", b.Config.Workspace.ResourcePath}, @@ -115,6 +116,11 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { // Ensures dashboard parent paths have the required /Workspace prefix DashboardFixups(), + // Reads (typed): b.Config.Resources.GenieSpaces (checks genie space configurations) + // Updates (typed): b.Config.Resources.GenieSpaces[].ParentPath (ensures /Workspace prefix is present) + // Ensures genie space parent paths have the required /Workspace prefix + GenieSpaceFixups(), + // Reads (typed): b.Config.Permissions (validates permission levels) // Reads (dynamic): resources.{jobs,pipelines,experiments,models,model_serving_endpoints,dashboards,apps,vector_search_endpoints,...}.*.permissions (reads existing permissions) // Updates (dynamic): resources.{jobs,pipelines,experiments,models,model_serving_endpoints,dashboards,apps,vector_search_endpoints,...}.*.permissions (adds permissions from bundle-level configuration) @@ -182,6 +188,10 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) { // Drops (dynamic): resources.dashboards.*.file_path ConfigureDashboardSerializedDashboard(), + // Reads (dynamic): resources.genie_spaces.*.file_path + // Updates (dynamic): resources.genie_spaces.*.serialized_space + ConfigureGenieSpaceSerializedSpace(), + // Reads (typed): resources.alerts.*.file_path // Updates (typed): resources.alerts.* (loads alert configuration from .dbalert.json file) mutator.LoadDBAlertFiles(), diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 9ef2db077ac..af1470848d7 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -41,6 +41,7 @@ func allResourceTypes(t *testing.T) []string { "database_instances", "experiments", "external_locations", + "genie_spaces", "jobs", "model_serving_endpoints", "models", @@ -183,6 +184,7 @@ var allowList = []string{ "postgres_synced_tables", "registered_models", "experiments", + "genie_spaces", "schemas", "secret_scopes", "sql_warehouses", diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index a0bb76e2e6e..b36ec094447 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -368,6 +368,7 @@ func (m *translatePathsDashboards) Apply(ctx context.Context, b *bundle.Bundle) return applyTranslations(ctx, b, t, []func(context.Context, dyn.Value) (dyn.Value, error){ t.applyDashboardTranslations, + t.applyGenieSpaceTranslations, }) } diff --git a/bundle/config/mutator/translate_paths_genie_spaces.go b/bundle/config/mutator/translate_paths_genie_spaces.go new file mode 100644 index 00000000000..279d97be01c --- /dev/null +++ b/bundle/config/mutator/translate_paths_genie_spaces.go @@ -0,0 +1,21 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle/config/mutator/paths" + "github.com/databricks/cli/libs/dyn" +) + +func (t *translateContext) applyGenieSpaceTranslations(ctx context.Context, v dyn.Value) (dyn.Value, error) { + // Convert the `file_path` field to a local absolute path. + // We load the file at this path and use its contents for the genie space contents. + + return paths.VisitGenieSpacePaths(v, func(p dyn.Path, mode paths.TranslateMode, v dyn.Value) (dyn.Value, error) { + opts := translateOptions{ + Mode: mode, + } + + return t.rewriteValue(ctx, p, v, t.b.BundleRootPath, opts) + }) +} diff --git a/bundle/config/mutator/translate_paths_genie_spaces_test.go b/bundle/config/mutator/translate_paths_genie_spaces_test.go new file mode 100644 index 00000000000..a1ac0b160b1 --- /dev/null +++ b/bundle/config/mutator/translate_paths_genie_spaces_test.go @@ -0,0 +1,55 @@ +package mutator_test + +import ( + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/vfs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTranslatePathsGenieSpaces_FilePathRelativeSubDirectory(t *testing.T) { + dir := t.TempDir() + touchEmptyFile(t, filepath.Join(dir, "src", "my_space.geniespace.json")) + + b := &bundle.Bundle{ + SyncRootPath: dir, + BundleRootPath: dir, + SyncRoot: vfs.MustNew(dir), + Config: config.Root{ + Resources: config.Resources{ + GenieSpaces: map[string]*resources.GenieSpace{ + "genie_space": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "My Genie Space", + }, + FilePath: "../src/my_space.geniespace.json", + }, + }, + }, + }, + } + + bundletest.SetLocation(b, "resources.genie_spaces", []dyn.Location{{ + File: filepath.Join(dir, "resources", "genie_space.yml"), + }}) + + // Genie space paths reuse the dashboard translator; there is no separate + // genie_space mutator. The dashboard translator walks all resource types + // that need path translation, so calling it covers genie_spaces too. + diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePathsDashboards()) + require.NoError(t, diags.Error()) + + assert.Equal( + t, + filepath.ToSlash(filepath.Join("src", "my_space.geniespace.json")), + b.Config.Resources.GenieSpaces["genie_space"].FilePath, + ) +} diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 2e840a5ef80..3dc7dc295d3 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -25,6 +25,7 @@ type Resources struct { ExternalLocations map[string]*resources.ExternalLocation `json:"external_locations,omitempty"` Clusters map[string]*resources.Cluster `json:"clusters,omitempty"` Dashboards map[string]*resources.Dashboard `json:"dashboards,omitempty"` + GenieSpaces map[string]*resources.GenieSpace `json:"genie_spaces,omitempty"` Apps map[string]*resources.App `json:"apps,omitempty"` SecretScopes map[string]*resources.SecretScope `json:"secret_scopes,omitempty"` Alerts map[string]*resources.Alert `json:"alerts,omitempty"` @@ -104,6 +105,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["external_locations"], r.ExternalLocations), collectResourceMap(descriptions["clusters"], r.Clusters), collectResourceMap(descriptions["dashboards"], r.Dashboards), + collectResourceMap(descriptions["genie_spaces"], r.GenieSpaces), collectResourceMap(descriptions["volumes"], r.Volumes), collectResourceMap(descriptions["apps"], r.Apps), collectResourceMap(descriptions["alerts"], r.Alerts), @@ -162,6 +164,7 @@ func SupportedResources() map[string]resources.ResourceDescription { "external_locations": (&resources.ExternalLocation{}).ResourceDescription(), "clusters": (&resources.Cluster{}).ResourceDescription(), "dashboards": (&resources.Dashboard{}).ResourceDescription(), + "genie_spaces": (&resources.GenieSpace{}).ResourceDescription(), "volumes": (&resources.Volume{}).ResourceDescription(), "apps": (&resources.App{}).ResourceDescription(), "secret_scopes": (&resources.SecretScope{}).ResourceDescription(), diff --git a/bundle/config/resources/genie_space.go b/bundle/config/resources/genie_space.go new file mode 100644 index 00000000000..b0a5efbf304 --- /dev/null +++ b/bundle/config/resources/genie_space.go @@ -0,0 +1,100 @@ +package resources + +import ( + "context" + "net/url" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/workspaceurls" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/dashboards" +) + +type GenieSpaceConfig struct { + // Description of the Genie Space + Description string `json:"description,omitempty"` + // Etag for change detection. The bundle persists the value the backend + // returned on the last Create/Update and uses it both as an If-Match for + // the next Update and as the signal for `bundle plan` to detect remote + // drift (see OverrideChangeDesc in bundle/direct/dresources/genie_space.go). + // Mirrors dashboards.DashboardConfig.Etag. + Etag string `json:"etag,omitempty"` + // Genie space ID + SpaceId string `json:"space_id,omitempty"` + // Title of the Genie Space + Title string `json:"title,omitempty"` + // Warehouse associated with the Genie Space + WarehouseId string `json:"warehouse_id,omitempty"` + // Parent folder path where the space will be registered + ParentPath string `json:"parent_path,omitempty"` + + ForceSendFields []string `json:"-" url:"-"` + + // ============================================== + // === overrides over [dashboards.GenieSpace] === + // ============================================== + + // SerializedSpace holds the contents of the Genie Space in serialized JSON form. + // Even though the SDK represents this as a string, we override it as any to allow for inlining as YAML. + // If the value is a string, it is used as is. + // If it is not a string, its contents is marshalled as JSON. + SerializedSpace any `json:"serialized_space,omitempty"` +} + +func (c *GenieSpaceConfig) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c GenieSpaceConfig) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +type GenieSpace struct { + BaseResource + GenieSpaceConfig + + Permissions []Permission `json:"permissions,omitempty"` + + // FilePath points to the local `.geniespace.json` file containing the Genie Space definition. + // This is inlined into serialized_space during deployment. The file_path is kept around + // as metadata which is needed for `databricks bundle generate genie-space --resource ` to work. + // This is not part of GenieSpaceConfig because we don't need to store this in the resource state. + FilePath string `json:"file_path,omitempty"` +} + +func (*GenieSpace) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { + _, err := w.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: id, + }) + if err != nil { + log.Debugf(ctx, "genie space %s does not exist", id) + return false, err + } + return true, nil +} + +func (*GenieSpace) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "genie_space", + PluralName: "genie_spaces", + SingularTitle: "Genie Space", + PluralTitle: "Genie Spaces", + } +} + +func (r *GenieSpace) InitializeURL(baseURL url.URL) { + if r.ID == "" { + return + } + + r.URL = workspaceurls.ResourceURL(baseURL, "genie_spaces", r.ID) +} + +func (r *GenieSpace) GetName() string { + return r.Title +} + +func (r *GenieSpace) GetURL() string { + return r.URL +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 4f51476536f..8e610ba9418 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -201,6 +201,9 @@ func TestResourcesBindSupport(t *testing.T) { Dashboards: map[string]*resources.Dashboard{ "my_dashboard": {}, }, + GenieSpaces: map[string]*resources.GenieSpace{ + "my_genie_space": {}, + }, Volumes: map[string]*resources.Volume{ "my_volume": { CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{}, @@ -328,6 +331,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockSchemasAPI().EXPECT().GetByFullName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockClustersAPI().EXPECT().GetByClusterId(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockLakeviewAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockGenieAPI().EXPECT().GetSpace(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockVolumesAPI().EXPECT().Read(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockAppsAPI().EXPECT().GetByName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockAlertsV2API().EXPECT().GetAlertById(mock.Anything, mock.Anything).Return(nil, nil) diff --git a/bundle/deploy/terraform/lifecycle_test.go b/bundle/deploy/terraform/lifecycle_test.go index 2986a9cc8de..b60bff612c7 100644 --- a/bundle/deploy/terraform/lifecycle_test.go +++ b/bundle/deploy/terraform/lifecycle_test.go @@ -17,6 +17,7 @@ func TestConvertLifecycleForAllResources(t *testing.T) { ignoredResources := []string{ "catalogs", "external_locations", + "genie_spaces", "vector_search_endpoints", "vector_search_indexes", } diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index a263a25f127..6cc1eb55437 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -29,6 +29,7 @@ var SupportedResources = map[string]any{ "clusters": (*ResourceCluster)(nil), "registered_models": (*ResourceRegisteredModel)(nil), "dashboards": (*ResourceDashboard)(nil), + "genie_spaces": (*ResourceGenieSpace)(nil), "secret_scopes": (*ResourceSecretScope)(nil), "model_serving_endpoints": (*ResourceModelServingEndpoint)(nil), "quality_monitors": (*ResourceQualityMonitor)(nil), @@ -49,6 +50,7 @@ var SupportedResources = map[string]any{ "secret_scopes.permissions": (*ResourceSecretScopeAcls)(nil), "model_serving_endpoints.permissions": (*ResourcePermissions)(nil), "dashboards.permissions": (*ResourcePermissions)(nil), + "genie_spaces.permissions": (*ResourcePermissions)(nil), "vector_search_endpoints.permissions": (*ResourcePermissions)(nil), // Grants diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index f18a84d0efc..30adb4640cc 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -254,6 +254,15 @@ var testConfig map[string]any = map[string]any{ }, }, + "genie_spaces": &resources.GenieSpace{ + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "my-genie-space", + WarehouseId: "test-warehouse-id", + ParentPath: "/Workspace/Users/user@example.com", + SerializedSpace: "{}", + }, + }, + "vector_search_endpoints": &resources.VectorSearchEndpoint{ CreateEndpoint: vectorsearch.CreateEndpoint{ Name: "my-endpoint", @@ -495,6 +504,25 @@ var testDeps = map[string]prepareWorkspace{ }, nil }, + "genie_spaces.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { + resp, err := client.Genie.CreateSpace(ctx, dashboards.GenieCreateSpaceRequest{ + Title: "genie-space-permissions", + WarehouseId: "test-warehouse-id", + SerializedSpace: "{}", + }) + if err != nil { + return nil, err + } + + return &PermissionsState{ + ObjectID: "/genie/" + resp.SpaceId, + EmbeddedSlice: []StatePermission{{ + Level: "CAN_MANAGE", + UserName: "user@example.com", + }}, + }, nil + }, + "model_serving_endpoints.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { waiter, err := client.ServingEndpoints.Create(ctx, serving.CreateServingEndpoint{ Name: "endpoint-permissions", diff --git a/bundle/direct/dresources/apitypes.yml b/bundle/direct/dresources/apitypes.yml index 7d478be47f7..11cba516a4d 100644 --- a/bundle/direct/dresources/apitypes.yml +++ b/bundle/direct/dresources/apitypes.yml @@ -4,6 +4,8 @@ # Set a value to null to remove a type: # jobs: null +genie_spaces: dashboards.GenieSpace + postgres_branches: postgres.BranchSpec postgres_endpoints: postgres.EndpointSpec diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go new file mode 100644 index 00000000000..84aedf5059f --- /dev/null +++ b/bundle/direct/dresources/genie_space.go @@ -0,0 +1,307 @@ +package dresources + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/utils" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/dashboards" +) + +var pathSerializedSpace = structpath.MustParsePath("serialized_space") + +// ResourceGenieSpace mirrors the dashboard resource pattern (see dashboard.go), +// with these intentional divergences: +// - No Published wrapper: Genie spaces have no publish lifecycle, so +// PrepareState returns the config directly. +// - RemapState filters fewer fields: Genie has no LifecycleState / CreateTime / +// Path / UpdateTime output-only fields to scrub. +// - DoUpdate omits serialized_space when unchanged: serialized_space is in +// ignore_remote_changes (see resources.yml), so a UI edit produces no plan +// entry. Sending the local body anyway would clobber the UI edit on every +// unrelated update. +// - DoCreate has expanded missing-parent-path detection: see +// isMissingGenieParentPathError below. +// +// Permissions follow the standard /permissions/genie/{id} endpoint and are wired +// up via the generic permissions adapter (permissions.go). +type ResourceGenieSpace struct { + client *databricks.WorkspaceClient +} + +func (*ResourceGenieSpace) New(client *databricks.WorkspaceClient) *ResourceGenieSpace { + return &ResourceGenieSpace{client: client} +} + +func (*ResourceGenieSpace) PrepareState(input *resources.GenieSpace) *resources.GenieSpaceConfig { + return &input.GenieSpaceConfig +} + +func (r *ResourceGenieSpace) RemapState(state *resources.GenieSpaceConfig) *resources.GenieSpaceConfig { + forceSendFields := utils.FilterFields[resources.GenieSpaceConfig](state.ForceSendFields, []string{ + "SpaceId", + "SerializedSpace", + }...) + + return &resources.GenieSpaceConfig{ + Description: state.Description, + Etag: state.Etag, + Title: state.Title, + WarehouseId: state.WarehouseId, + ParentPath: state.ParentPath, + SerializedSpace: state.SerializedSpace, + + ForceSendFields: forceSendFields, + + // Clear output only fields. They should not show up on remote diff computation. + SpaceId: "", + } +} + +func (r *ResourceGenieSpace) DoRead(ctx context.Context, id string) (*resources.GenieSpaceConfig, error) { + space, err := r.client.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: id, + IncludeSerializedSpace: true, // otherwise etag isn't returned + ForceSendFields: nil, + }) + if err != nil { + return nil, genieSpaceGoneError(err) + } + return responseToGenieSpaceConfig(space, space.SerializedSpace), nil +} + +func prepareGenieSpaceRequest(config *resources.GenieSpaceConfig) (string, error) { + v := config.SerializedSpace + if serializedSpace, ok := v.(string); ok { + return serializedSpace, nil + } else if v != nil { + b, err := json.Marshal(v) + if err != nil { + return "", fmt.Errorf("failed to marshal serialized_space: %w", err) + } + return string(b), nil + } + return "", nil +} + +func responseToGenieSpaceConfig(space *dashboards.GenieSpace, serializedSpace string) *resources.GenieSpaceConfig { + forceSendFields := utils.FilterFields[resources.GenieSpaceConfig](space.ForceSendFields) + + return &resources.GenieSpaceConfig{ + Description: space.Description, + Etag: space.Etag, + Title: space.Title, + WarehouseId: space.WarehouseId, + ParentPath: ensureWorkspacePrefix(space.ParentPath), + SerializedSpace: serializedSpace, + + // Output only fields + SpaceId: space.SpaceId, + ForceSendFields: forceSendFields, + } +} + +// isMissingGenieParentPathError reports whether the given Create error means +// "the parent workspace folder does not exist", so DoCreate can mkdir and retry. +// +// Dashboard handles the equivalent condition with a plain apierr.IsMissing +// check (see ResourceDashboard.DoCreate). Genie cannot, because it surfaces +// the same condition in two different shapes depending on the workspace's +// backend version: +// +// 1. Standard missing-resource error: HTTP 404, ErrorCode RESOURCE_DOES_NOT_EXIST. +// Caught by apierr.IsMissing. Observed on workspaces running the newer +// Genie service implementation. +// 2. HTTP 400 with ErrorCode INVALID_PARAMETER_VALUE and a message of the +// form "Tree node with path '' does not exist". Observed on +// workspaces still backed by the legacy implementation during integration +// testing in early 2026 (aws-prod-ucws and azure-prod-ucws clusters at +// the time). The string match is intentional: there is no distinct error +// code to key on. +// +// Both forms unambiguously mean "create the parent and retry once". +func isMissingGenieParentPathError(err error) bool { + if apierr.IsMissing(err) { + return true + } + + apiErr, ok := errors.AsType[*apierr.APIError](err) + if !ok { + return false + } + + return apiErr.StatusCode == http.StatusBadRequest && + apiErr.ErrorCode == "INVALID_PARAMETER_VALUE" && + strings.Contains(apiErr.Message, "Tree node with path") && + strings.Contains(apiErr.Message, "does not exist") +} + +// genieSpaceGoneError maps the Genie API's "space does not exist" response to +// the framework's gone sentinel (apierr.ErrResourceDoesNotExist). The Genie API +// returns 403 (not 404) for a missing space, so without this translation +// isResourceGone would treat a remotely-deleted space as a hard permission +// error instead of recreating it (on read) or tolerating it (on delete). +func genieSpaceGoneError(err error) error { + if errors.Is(err, apierr.ErrPermissionDenied) { + return errors.Join(err, apierr.ErrResourceDoesNotExist) + } + return err +} + +func (r *ResourceGenieSpace) DoCreate(ctx context.Context, config *resources.GenieSpaceConfig) (string, *resources.GenieSpaceConfig, error) { + serializedSpace, err := prepareGenieSpaceRequest(config) + if err != nil { + return "", nil, err + } + + req := dashboards.GenieCreateSpaceRequest{ + Description: config.Description, + Title: config.Title, + WarehouseId: config.WarehouseId, + ParentPath: config.ParentPath, + SerializedSpace: serializedSpace, + + ForceSendFields: utils.FilterFields[dashboards.GenieCreateSpaceRequest](config.ForceSendFields), + } + + createResp, err := r.client.Genie.CreateSpace(ctx, req) + + // Retry once after creating the parent directory when the workspace folder + // is missing. Genie can surface this either as a standard missing-resource + // error or as INVALID_PARAMETER_VALUE with a "Tree node ... does not exist" + // message depending on the backend. + if err != nil && isMissingGenieParentPathError(err) { + err = r.client.Workspace.MkdirsByPath(ctx, config.ParentPath) //nolint:staticcheck // Deprecated in SDK v0.127.0. Migration to WorkspaceHierarchyService tracked separately. + if err != nil { + return "", nil, fmt.Errorf("failed to create parent directory: %w", err) + } + createResp, err = r.client.Genie.CreateSpace(ctx, req) + } + if err != nil { + return "", nil, err + } + + // Persist the etag in state. The deploy framework saves `config` (the input + // to DoCreate) as the state record, so mutating it here is what gets the + // backend-returned etag onto disk for the next plan's drift check. + // Matches the dashboard pattern (dashboard.go DoCreate). + config.Etag = createResp.Etag + + return createResp.SpaceId, responseToGenieSpaceConfig(createResp, serializedSpace), nil +} + +func (r *ResourceGenieSpace) DoUpdate(ctx context.Context, id string, config *resources.GenieSpaceConfig, entry *PlanEntry) (*resources.GenieSpaceConfig, error) { + serializedSpace, err := prepareGenieSpaceRequest(config) + if err != nil { + return nil, err + } + + // serialized_space is in ignore_remote_changes (we cannot diff structured + // local YAML against remote JSON), so a UI edit produces no plan entry. + // If we still sent the unchanged local body on every update, the next + // update triggered by another field would clobber the UI edit. Only + // send it when the user actually changed it locally. + var excludeForceSend []string + sentSerialized := true + if !hasUpdate(entry, pathSerializedSpace) { + serializedSpace = "" + sentSerialized = false + excludeForceSend = append(excludeForceSend, "SerializedSpace") + } + + updateResp, err := r.client.Genie.UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ + SpaceId: id, + Description: config.Description, + Title: config.Title, + WarehouseId: config.WarehouseId, + ParentPath: config.ParentPath, + SerializedSpace: serializedSpace, + // Send the etag we last observed. The backend uses it as an If-Match + // guard against concurrent writes, and OverrideChangeDesc uses the + // post-update etag to detect drift on subsequent plans. + Etag: config.Etag, + + ForceSendFields: utils.FilterFields[dashboards.GenieUpdateSpaceRequest](config.ForceSendFields, excludeForceSend...), + }) + if err != nil { + return nil, err + } + + // Persist the new etag in state (see DoCreate for the rationale). + config.Etag = updateResp.Etag + + // Decide what to record as the new state's serialized_space. + // - If we sent a new body, use it. + // - If we omitted it (UI-edit protection above) but the API echoed back + // a value, record that — it's the most up-to-date view we have. + // - If neither side carries a value, keep whatever was already in state. + // Otherwise RemapState would blank the field on every unrelated update. + respSerialized := serializedSpace + if !sentSerialized { + respSerialized = updateResp.SerializedSpace + if respSerialized == "" { + if prior, ok := config.SerializedSpace.(string); ok { + respSerialized = prior + } + } + } + + return responseToGenieSpaceConfig(updateResp, respSerialized), nil +} + +// OverrideChangeDesc handles the etag field. The user never sets it directly; +// we compare the stored etag against the remote one and Skip if they match. +// This mirrors ResourceDashboard.OverrideChangeDesc. +func (r *ResourceGenieSpace) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, _ *resources.GenieSpaceConfig) error { + switch path.String() { + case "etag": + // change.New is always nil for etag because it's not present in the + // user-authored config. Compare stored etag with remote one to decide + // whether anything changed out-of-band since the last deploy. + if change.Old == change.Remote { + change.Action = deployplan.Skip + } else { + change.Action = deployplan.Update + } + } + return nil +} + +// hasUpdate reports whether entry has an Update-action change at the given path. +// HasChange alone matches Skip-action changes too, which we cannot use to drive +// request shaping for fields covered by ignore_remote_changes. +func hasUpdate(entry *PlanEntry, path *structpath.PathNode) bool { + if entry == nil { + return false + } + for s, change := range entry.Changes { + if change.Action != deployplan.Update { + continue + } + node, err := structpath.ParsePath(s) + if err != nil { + continue + } + if node.HasPrefix(path) { + return true + } + } + return false +} + +func (r *ResourceGenieSpace) DoDelete(ctx context.Context, id string, _ *resources.GenieSpaceConfig) error { + // TrashSpace returns 403 when the space is already gone; map that to the + // gone sentinel so deletion is idempotent. + return genieSpaceGoneError(r.client.Genie.TrashSpace(ctx, dashboards.GenieTrashSpaceRequest{ + SpaceId: id, + })) +} diff --git a/bundle/direct/dresources/genie_space_test.go b/bundle/direct/dresources/genie_space_test.go new file mode 100644 index 00000000000..a72e50d9fe7 --- /dev/null +++ b/bundle/direct/dresources/genie_space_test.go @@ -0,0 +1,309 @@ +package dresources + +import ( + "errors" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestIsMissingGenieParentPathError(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + { + name: "standard missing error", + err: &apierr.APIError{ + StatusCode: 404, + ErrorCode: "NOT_FOUND", + Message: "not found", + }, + want: true, + }, + { + name: "invalid parameter tree node missing error", + err: &apierr.APIError{ + StatusCode: 400, + ErrorCode: "INVALID_PARAMETER_VALUE", + Message: "NOT_FOUND: Tree node with path /Workspace/foo does not exist", + }, + want: true, + }, + { + name: "other invalid parameter error", + err: &apierr.APIError{ + StatusCode: 400, + ErrorCode: "INVALID_PARAMETER_VALUE", + Message: "some other validation failure", + }, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, isMissingGenieParentPathError(tt.err)) + }) + } +} + +func TestGenieSpaceDoCreateRetriesWhenParentPathLooksMissing(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + req := dashboards.GenieCreateSpaceRequest{ + Title: "test genie space", + Description: "test description", + ParentPath: "/Workspace/test-parent", + WarehouseId: "test-warehouse-id", + SerializedSpace: "{}", + } + + m.GetMockGenieAPI().EXPECT(). + CreateSpace(ctx, req). + Return(nil, &apierr.APIError{ + StatusCode: 400, + ErrorCode: "INVALID_PARAMETER_VALUE", + Message: "NOT_FOUND: Tree node with path /Workspace/test-parent does not exist", + }). + Once() + + m.GetMockWorkspaceAPI().EXPECT(). + MkdirsByPath(ctx, "/Workspace/test-parent"). + Return(nil). + Once() + + m.GetMockGenieAPI().EXPECT(). + CreateSpace(ctx, req). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id", + Title: "test genie space", + Description: "test description", + WarehouseId: "test-warehouse-id", + SerializedSpace: "{}", + }, nil). + Once() + + id, state, err := r.DoCreate(ctx, &resources.GenieSpaceConfig{ + Title: "test genie space", + Description: "test description", + ParentPath: "/Workspace/test-parent", + WarehouseId: "test-warehouse-id", + SerializedSpace: "{}", + }) + require.NoError(t, err) + assert.Equal(t, "space-id", id) + require.NotNil(t, state) + assert.Equal(t, "test genie space", state.Title) +} + +func TestGenieSpaceDoUpdateOmitsSerializedSpaceWhenUnchanged(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + // Plan entry indicates only title changed; serialized_space is absent. + entry := &deployplan.PlanEntry{ + Changes: deployplan.Changes{ + "title": {Action: deployplan.Update, Old: "old", New: "new"}, + }, + } + + m.GetMockGenieAPI().EXPECT(). + UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ + SpaceId: "space-id", + Title: "new", + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id", + Title: "new", + SerializedSpace: "{\"remote\":\"edit\"}", + }, nil). + Once() + + state, err := r.DoUpdate(ctx, "space-id", &resources.GenieSpaceConfig{ + Title: "new", + SerializedSpace: "{\"local\":\"stale\"}", + }, entry) + require.NoError(t, err) + require.NotNil(t, state) + assert.Equal(t, "{\"remote\":\"edit\"}", state.SerializedSpace) +} + +func TestGenieSpaceDoUpdateSendsSerializedSpaceWhenChanged(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + entry := &deployplan.PlanEntry{ + Changes: deployplan.Changes{ + "serialized_space": {Action: deployplan.Update, Old: "{}", New: "{\"v\":1}"}, + }, + } + + m.GetMockGenieAPI().EXPECT(). + UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ + SpaceId: "space-id", + SerializedSpace: "{\"v\":1}", + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id", + SerializedSpace: "{\"v\":1}", + }, nil). + Once() + + state, err := r.DoUpdate(ctx, "space-id", &resources.GenieSpaceConfig{ + SerializedSpace: "{\"v\":1}", + }, entry) + require.NoError(t, err) + require.NotNil(t, state) + assert.Equal(t, "{\"v\":1}", state.SerializedSpace) +} + +func TestGenieSpaceDoUpdateRoundTripsEtag(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + entry := &deployplan.PlanEntry{ + Changes: deployplan.Changes{ + "title": {Action: deployplan.Update, Old: "old", New: "new"}, + }, + } + + m.GetMockGenieAPI().EXPECT(). + UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ + SpaceId: "space-id", + Title: "new", + Etag: "etag-7", + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id", + Title: "new", + Etag: "etag-8", + }, nil). + Once() + + state, err := r.DoUpdate(ctx, "space-id", &resources.GenieSpaceConfig{ + Title: "new", + Etag: "etag-7", + }, entry) + require.NoError(t, err) + require.NotNil(t, state) + assert.Equal(t, "etag-8", state.Etag) +} + +func TestGenieSpaceDoUpdateKeepsPriorSerializedSpaceWhenBothEmpty(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + // Only title changed; serialized_space should be omitted from the request. + entry := &deployplan.PlanEntry{ + Changes: deployplan.Changes{ + "title": {Action: deployplan.Update, Old: "old", New: "new"}, + }, + } + + m.GetMockGenieAPI().EXPECT(). + UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ + SpaceId: "space-id", + Title: "new", + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id", + Title: "new", + // API also omits serialized_space; we should keep the prior local value. + }, nil). + Once() + + state, err := r.DoUpdate(ctx, "space-id", &resources.GenieSpaceConfig{ + Title: "new", + SerializedSpace: "{\"keep\":\"me\"}", + }, entry) + require.NoError(t, err) + require.NotNil(t, state) + assert.Equal(t, "{\"keep\":\"me\"}", state.SerializedSpace) +} + +func TestGenieSpaceOverrideChangeDescEtag(t *testing.T) { + r := &ResourceGenieSpace{} + etagPath := structpath.MustParsePath("etag") + + t.Run("Skip when stored matches remote", func(t *testing.T) { + change := &ChangeDesc{Old: "etag-7", Remote: "etag-7"} + require.NoError(t, r.OverrideChangeDesc(t.Context(), etagPath, change, nil)) + assert.Equal(t, deployplan.Skip, change.Action) + }) + + t.Run("Update when stored differs from remote", func(t *testing.T) { + change := &ChangeDesc{Old: "etag-7", Remote: "etag-8"} + require.NoError(t, r.OverrideChangeDesc(t.Context(), etagPath, change, nil)) + assert.Equal(t, deployplan.Update, change.Action) + }) + + t.Run("Other paths are untouched", func(t *testing.T) { + titlePath := structpath.MustParsePath("title") + change := &ChangeDesc{Action: deployplan.Update, Old: "a", Remote: "b"} + require.NoError(t, r.OverrideChangeDesc(t.Context(), titlePath, change, nil)) + assert.Equal(t, deployplan.Update, change.Action) + }) +} + +func TestGenieSpaceGoneErrorMapsForbidden(t *testing.T) { + // The Genie API returns 403 (not 404) for a missing space; it must surface + // as the framework's gone sentinel so a deleted space is recreated (read) + // or tolerated (delete). + forbidden := &apierr.APIError{StatusCode: 403, Message: "dataRoom is not user-facing"} + assert.ErrorIs(t, genieSpaceGoneError(forbidden), apierr.ErrResourceDoesNotExist) + + // Unrelated errors pass through unchanged. + other := errors.New("boom") + assert.Equal(t, other, genieSpaceGoneError(other)) + + // A nil error stays nil. + assert.NoError(t, genieSpaceGoneError(nil)) +} + +func TestGenieSpaceDoReadTreatsForbiddenAsGone(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + m.GetMockGenieAPI().EXPECT(). + GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: "space-id", + IncludeSerializedSpace: true, + }). + Return(nil, &apierr.APIError{StatusCode: 403, Message: "dataRoom is not user-facing"}). + Once() + + _, err := r.DoRead(ctx, "space-id") + require.Error(t, err) + assert.ErrorIs(t, err, apierr.ErrResourceDoesNotExist) +} + +func TestGenieSpaceDoDeleteToleratesForbidden(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + m.GetMockGenieAPI().EXPECT(). + TrashSpace(ctx, dashboards.GenieTrashSpaceRequest{SpaceId: "space-id"}). + Return(&apierr.APIError{StatusCode: 403, Message: "dataRoom is not user-facing"}). + Once() + + err := r.DoDelete(ctx, "space-id", nil) + require.Error(t, err) + assert.ErrorIs(t, err, apierr.ErrResourceDoesNotExist) +} diff --git a/bundle/direct/dresources/permissions.go b/bundle/direct/dresources/permissions.go index 91ca9000aaf..6e1fa79d811 100644 --- a/bundle/direct/dresources/permissions.go +++ b/bundle/direct/dresources/permissions.go @@ -18,6 +18,7 @@ var permissionResourceToObjectType = map[string]string{ "apps": "/apps/", "clusters": "/clusters/", "dashboards": "/dashboards/", + "genie_spaces": "/genie/", "database_instances": "/database-instances/", "postgres_projects": "/database-projects/", "jobs": "/jobs/", diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 8da2d5fee50..c3d8f559d25 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -359,6 +359,13 @@ resources: - field: dataset_schema reason: input_only + genie_spaces: + ignore_remote_changes: + # serialized_space locally (structured YAML) and remotely (JSON string) will differ + # textually, so we cannot meaningfully compare them for drift. + - field: serialized_space + reason: etag_based + apps: recreate_on_changes: - field: name diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index c9fefb7c1f2..fee09ed119f 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -85,6 +85,9 @@ var knownMissingInStateType = map[string][]string{ "dashboards": { "file_path", }, + "genie_spaces": { + "file_path", + }, "secret_scopes": { "backend_type", "keyvault_metadata", diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 5b2a70adbb3..7f719674ebb 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -439,9 +439,14 @@ func ExportStateFromData(data Database) resourcestate.ExportedResourcesMap { result := make(resourcestate.ExportedResourcesMap) for key, entry := range data.State { var etag string - // Extract etag for dashboards. - // covered by test case: bundle/deploy/dashboard/detect-change - if strings.Contains(key, ".dashboards.") && len(entry.State) > 0 { + // Extract etag for resources that use it for drift detection + // (dashboards and genie_spaces). Both follow the same pattern of + // persisting the backend-returned etag in state and comparing it + // against the remote on the next plan via OverrideChangeDesc. + // covered by test cases: + // - bundle/deploy/dashboard/detect-change + // - bundle/resources/genie_spaces/simple + if (strings.Contains(key, ".dashboards.") || strings.Contains(key, ".genie_spaces.")) && len(entry.State) > 0 { var holder struct { Etag string `json:"etag"` } diff --git a/bundle/docsgen/output/reference.md b/bundle/docsgen/output/reference.md index 16b7020cecd..92228810a38 100644 --- a/bundle/docsgen/output/reference.md +++ b/bundle/docsgen/output/reference.md @@ -1,7 +1,7 @@ --- description: 'Configuration reference for databricks.yml' last_update: - date: 2026-06-03 + date: 2026-06-04 --- @@ -479,6 +479,10 @@ resources: - Map - See [\_](#resourcesexternal_locations). +- - `genie_spaces` + - Map + - See [\_](#resourcesgenie_spaces). + - - `jobs` - Map - The job definitions for the bundle, where each key is the name of the job. See [\_](/dev-tools/bundles/resources.md#jobs). @@ -1502,6 +1506,122 @@ external_locations: ::: +### resources.genie_spaces + +**`Type: Map`** + + + +```yaml +genie_spaces: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `description` + - String + - Description of the Genie space shown alongside the title in the Databricks UI. + +- - `etag` + - String + - + +- - `file_path` + - String + - Local path to a `.geniespace.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. + +- - `lifecycle` + - Map + - See [\_](#resourcesgenie_spacesnamelifecycle). + +- - `parent_path` + - String + - Workspace folder under which to create the Genie space. Immutable: changing this field recreates the resource. + +- - `permissions` + - Sequence + - See [\_](#resourcesgenie_spacesnamepermissions). + +- - `serialized_space` + - Any + - Serialized Genie space body. May be provided inline as a JSON string (or YAML that will be marshalled to JSON) or referenced via `file_path`. To round-trip an existing space into a bundle, use `databricks bundle generate genie-space`. + +- - `space_id` + - String + - + +- - `title` + - String + - Title of the Genie space shown in the Databricks UI. + +- - `warehouse_id` + - String + - ID of the SQL warehouse used to run queries for this Genie space. + +::: + + +### resources.genie_spaces._name_.lifecycle + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + +### resources.genie_spaces._name_.permissions + +**`Type: Sequence`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `group_name` + - String + - The name of the group that has the permission set in level. + +- - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + +- - `service_principal_name` + - String + - The name of the service principal that has the permission set in level. + +- - `user_name` + - String + - The name of the user that has the permission set in level. + +::: + + ### resources.pipelines **`Type: Map`** @@ -2018,6 +2138,10 @@ postgres_projects: - String - +- - `purge_on_delete` + - Boolean + - + ::: @@ -3026,6 +3150,10 @@ The resource definitions for the target. - Map - See [\_](#targetsnameresourcesexternal_locations). +- - `genie_spaces` + - Map + - See [\_](#targetsnameresourcesgenie_spaces). + - - `jobs` - Map - The job definitions for the bundle, where each key is the name of the job. See [\_](/dev-tools/bundles/resources.md#jobs). @@ -4049,6 +4177,122 @@ external_locations: ::: +### targets._name_.resources.genie_spaces + +**`Type: Map`** + + + +```yaml +genie_spaces: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `description` + - String + - Description of the Genie space shown alongside the title in the Databricks UI. + +- - `etag` + - String + - + +- - `file_path` + - String + - Local path to a `.geniespace.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. + +- - `lifecycle` + - Map + - See [\_](#targetsnameresourcesgenie_spacesnamelifecycle). + +- - `parent_path` + - String + - Workspace folder under which to create the Genie space. Immutable: changing this field recreates the resource. + +- - `permissions` + - Sequence + - See [\_](#targetsnameresourcesgenie_spacesnamepermissions). + +- - `serialized_space` + - Any + - Serialized Genie space body. May be provided inline as a JSON string (or YAML that will be marshalled to JSON) or referenced via `file_path`. To round-trip an existing space into a bundle, use `databricks bundle generate genie-space`. + +- - `space_id` + - String + - + +- - `title` + - String + - Title of the Genie space shown in the Databricks UI. + +- - `warehouse_id` + - String + - ID of the SQL warehouse used to run queries for this Genie space. + +::: + + +### targets._name_.resources.genie_spaces._name_.lifecycle + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + +### targets._name_.resources.genie_spaces._name_.permissions + +**`Type: Sequence`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `group_name` + - String + - The name of the group that has the permission set in level. + +- - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + +- - `service_principal_name` + - String + - The name of the service principal that has the permission set in level. + +- - `user_name` + - String + - The name of the user that has the permission set in level. + +::: + + ### targets._name_.resources.pipelines **`Type: Map`** @@ -4565,6 +4809,10 @@ postgres_projects: - String - +- - `purge_on_delete` + - Boolean + - + ::: diff --git a/bundle/docsgen/output/resources.md b/bundle/docsgen/output/resources.md index e4f902e6146..bce0e3f9fb7 100644 --- a/bundle/docsgen/output/resources.md +++ b/bundle/docsgen/output/resources.md @@ -1,7 +1,7 @@ --- description: 'Learn about resources supported by Declarative Automation Bundles and how to configure them.' last_update: - date: 2026-06-03 + date: 2026-06-04 --- @@ -3086,6 +3086,122 @@ The privileges assigned to the principal. ::: +## genie_spaces + +**`Type: Map`** + + + +```yaml +genie_spaces: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `description` + - String + - Description of the Genie space shown alongside the title in the Databricks UI. + +- - `etag` + - String + - + +- - `file_path` + - String + - Local path to a `.geniespace.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. + +- - `lifecycle` + - Map + - See [\_](#genie_spacesnamelifecycle). + +- - `parent_path` + - String + - Workspace folder under which to create the Genie space. Immutable: changing this field recreates the resource. + +- - `permissions` + - Sequence + - See [\_](#genie_spacesnamepermissions). + +- - `serialized_space` + - Any + - Serialized Genie space body. May be provided inline as a JSON string (or YAML that will be marshalled to JSON) or referenced via `file_path`. To round-trip an existing space into a bundle, use `databricks bundle generate genie-space`. + +- - `space_id` + - String + - + +- - `title` + - String + - Title of the Genie space shown in the Databricks UI. + +- - `warehouse_id` + - String + - ID of the SQL warehouse used to run queries for this Genie space. + +::: + + +### genie_spaces._name_.lifecycle + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + +### genie_spaces._name_.permissions + +**`Type: Sequence`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `group_name` + - String + - The name of the group that has the permission set in level. + +- - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + +- - `service_principal_name` + - String + - The name of the service principal that has the permission set in level. + +- - `user_name` + - String + - The name of the user that has the permission set in level. + +::: + + ## jobs **`Type: Map`** @@ -10771,6 +10887,10 @@ postgres_projects: - String - +- - `purge_on_delete` + - Boolean + - + ::: @@ -11909,6 +12029,10 @@ Lifecycle is a struct that contains the lifecycle settings for a resource. It co - Boolean - Lifecycle setting to prevent the resource from being destroyed. +- - `started` + - Boolean + - Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. + ::: diff --git a/bundle/generate/genie_space.go b/bundle/generate/genie_space.go new file mode 100644 index 00000000000..bf00abbc66e --- /dev/null +++ b/bundle/generate/genie_space.go @@ -0,0 +1,42 @@ +package generate + +import ( + "path" + "strings" + + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/databricks-sdk-go/service/dashboards" +) + +func ConvertGenieSpaceToValue(genieSpace *dashboards.GenieSpace, filePath string) (dyn.Value, error) { + // Emit only the fields a user authors in a bundle. serialized_space is + // written to a separate file and referenced via file_path, and output-only + // fields (e.g. space_id, etag) must not appear in the generated config, so + // we build the value field by field rather than marshaling the struct. + dv := map[string]dyn.Value{ + "title": dyn.NewValue(genieSpace.Title, []dyn.Location{{Line: 1}}), + "warehouse_id": dyn.NewValue(genieSpace.WarehouseId, []dyn.Location{{Line: 2}}), + "file_path": dyn.NewValue(filePath, []dyn.Location{{Line: 3}}), + } + + if genieSpace.Description != "" { + dv["description"] = dyn.NewValue(genieSpace.Description, []dyn.Location{{Line: 4}}) + } + + if genieSpace.ParentPath != "" { + dv["parent_path"] = dyn.NewValue(ensureWorkspacePrefix(genieSpace.ParentPath), []dyn.Location{{Line: 5}}) + } + + return dyn.V(dv), nil +} + +// ensureWorkspacePrefix re-adds the /Workspace prefix that the Genie GET API +// strips from parent_path, so the generated config matches the convention used +// in hand-written bundles and in deployment state (mirrors the equivalent +// helper in bundle/direct/dresources/dashboard.go). +func ensureWorkspacePrefix(parentPath string) string { + if parentPath == "/Workspace" || strings.HasPrefix(parentPath, "/Workspace/") { + return parentPath + } + return path.Join("/Workspace", parentPath) +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index fd5f4275d12..69d7c9d025d 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -195,6 +195,9 @@ github.com/databricks/cli/bundle/config.Resources: "external_locations": "description": |- PLACEHOLDER + "genie_spaces": + "description": |- + PLACEHOLDER "jobs": "description": |- The job definitions for the bundle, where each key is the name of the job. @@ -668,6 +671,37 @@ github.com/databricks/cli/bundle/config/resources.ExternalLocation: "url": "description": |- PLACEHOLDER +github.com/databricks/cli/bundle/config/resources.GenieSpace: + "description": + "description": |- + Description of the Genie space shown alongside the title in the Databricks UI. + "etag": + "description": |- + PLACEHOLDER + "file_path": + "description": |- + Local path to a `.geniespace.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. + "lifecycle": + "description": |- + PLACEHOLDER + "parent_path": + "description": |- + Workspace folder under which to create the Genie space. Immutable: changing this field recreates the resource. + "permissions": + "description": |- + PLACEHOLDER + "serialized_space": + "description": |- + Serialized Genie space body. May be provided inline as a JSON string (or YAML that will be marshalled to JSON) or referenced via `file_path`. To round-trip an existing space into a bundle, use `databricks bundle generate genie-space`. + "space_id": + "description": |- + PLACEHOLDER + "title": + "description": |- + Title of the Genie space shown in the Databricks UI. + "warehouse_id": + "description": |- + ID of the SQL warehouse used to run queries for this Genie space. github.com/databricks/cli/bundle/config/resources.JobPermission: "group_name": "description": |- diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 476eaaf3637..1a8b4369e83 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -760,6 +760,56 @@ } ] }, + "resources.GenieSpace": { + "oneOf": [ + { + "type": "object", + "properties": { + "description": { + "description": "Description of the Genie space shown alongside the title in the Databricks UI.", + "$ref": "#/$defs/string" + }, + "etag": { + "$ref": "#/$defs/string" + }, + "file_path": { + "description": "Local path to a `.geniespace.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`.", + "$ref": "#/$defs/string" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "parent_path": { + "description": "Workspace folder under which to create the Genie space. Immutable: changing this field recreates the resource.", + "$ref": "#/$defs/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "serialized_space": { + "description": "Serialized Genie space body. May be provided inline as a JSON string (or YAML that will be marshalled to JSON) or referenced via `file_path`. To round-trip an existing space into a bundle, use `databricks bundle generate genie-space`.", + "$ref": "#/$defs/interface" + }, + "space_id": { + "$ref": "#/$defs/string" + }, + "title": { + "description": "Title of the Genie space shown in the Databricks UI.", + "$ref": "#/$defs/string" + }, + "warehouse_id": { + "description": "ID of the SQL warehouse used to run queries for this Genie space.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Job": { "oneOf": [ { @@ -2655,6 +2705,9 @@ "external_locations": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.ExternalLocation" }, + "genie_spaces": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.GenieSpace" + }, "jobs": { "description": "The job definitions for the bundle, where each key is the name of the job.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Job", @@ -12206,6 +12259,20 @@ } ] }, + "resources.GenieSpace": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.GenieSpace" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Job": { "oneOf": [ { diff --git a/bundle/schema/jsonschema_for_docs.json b/bundle/schema/jsonschema_for_docs.json index 198d9191196..04e3930f36c 100644 --- a/bundle/schema/jsonschema_for_docs.json +++ b/bundle/schema/jsonschema_for_docs.json @@ -741,6 +741,39 @@ "url" ] }, + "resources.GenieSpace": { + "type": "object", + "properties": { + "description": { + "$ref": "#/$defs/string" + }, + "file_path": { + "$ref": "#/$defs/string" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "parent_path": { + "$ref": "#/$defs/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "serialized_space": { + "$ref": "#/$defs/interface" + }, + "space_id": { + "$ref": "#/$defs/string" + }, + "title": { + "$ref": "#/$defs/string" + }, + "warehouse_id": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, "resources.Job": { "type": "object", "properties": { @@ -2615,6 +2648,9 @@ "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.ExternalLocation", "x-since-version": "v0.289.0" }, + "genie_spaces": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.GenieSpace" + }, "jobs": { "description": "The job definitions for the bundle, where each key is the name of the job.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Job", @@ -10211,6 +10247,12 @@ "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.ExternalLocation" } }, + "resources.GenieSpace": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.GenieSpace" + } + }, "resources.Job": { "type": "object", "additionalProperties": { diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 57f31450d93..672cd9855b2 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -39,6 +39,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.volumes.test_volume": {ID: "1"}, "resources.clusters.test_cluster": {ID: "1"}, "resources.dashboards.test_dashboard": {ID: "1"}, + "resources.genie_spaces.test_genie_space": {ID: "1"}, "resources.apps.test_app": {ID: "app1"}, "resources.secret_scopes.test_secret_scope": {ID: "secret_scope1"}, "resources.sql_warehouses.test_sql_warehouse": {ID: "1"}, @@ -96,6 +97,9 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "1", config.Resources.Dashboards["test_dashboard"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Dashboards["test_dashboard"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.GenieSpaces["test_genie_space"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.GenieSpaces["test_genie_space"].ModifiedStatus) + assert.Equal(t, "app1", config.Resources.Apps["test_app"].ID) assert.Empty(t, config.Resources.Apps["test_app"].Name) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Apps["test_app"].ModifiedStatus) @@ -228,6 +232,13 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + GenieSpaces: map[string]*resources.GenieSpace{ + "test_genie_space": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "test_genie_space", + }, + }, + }, Apps: map[string]*resources.App{ "test_app": { App: apps.App{ @@ -374,6 +385,9 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { assert.Empty(t, config.Resources.Dashboards["test_dashboard"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Dashboards["test_dashboard"].ModifiedStatus) + assert.Empty(t, config.Resources.GenieSpaces["test_genie_space"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.GenieSpaces["test_genie_space"].ModifiedStatus) + assert.Empty(t, config.Resources.Apps["test_app"].Name) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Apps["test_app"].ModifiedStatus) @@ -571,6 +585,18 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + GenieSpaces: map[string]*resources.GenieSpace{ + "test_genie_space": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "test_genie_space", + }, + }, + "test_genie_space_new": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "test_genie_space_new", + }, + }, + }, Apps: map[string]*resources.App{ "test_app": { App: apps.App{ @@ -767,6 +793,8 @@ func TestStateToBundleModifiedResources(t *testing.T) { "resources.clusters.test_cluster_old": {ID: "2"}, "resources.dashboards.test_dashboard": {ID: "1"}, "resources.dashboards.test_dashboard_old": {ID: "2"}, + "resources.genie_spaces.test_genie_space": {ID: "1"}, + "resources.genie_spaces.test_genie_space_old": {ID: "2"}, "resources.apps.test_app": {ID: "test_app"}, "resources.apps.test_app_old": {ID: "test_app_old"}, "resources.secret_scopes.test_secret_scope": {ID: "test_secret_scope"}, @@ -877,6 +905,13 @@ func TestStateToBundleModifiedResources(t *testing.T) { assert.Empty(t, config.Resources.Dashboards["test_dashboard_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Dashboards["test_dashboard_new"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.GenieSpaces["test_genie_space"].ID) + assert.Empty(t, config.Resources.GenieSpaces["test_genie_space"].ModifiedStatus) + assert.Equal(t, "2", config.Resources.GenieSpaces["test_genie_space_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.GenieSpaces["test_genie_space_old"].ModifiedStatus) + assert.Empty(t, config.Resources.GenieSpaces["test_genie_space_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.GenieSpaces["test_genie_space_new"].ModifiedStatus) + assert.Equal(t, "test_app", config.Resources.Apps["test_app"].Name) assert.Empty(t, config.Resources.Apps["test_app"].ModifiedStatus) assert.Equal(t, "test_app_old", config.Resources.Apps["test_app_old"].ID) diff --git a/cmd/bundle/generate.go b/cmd/bundle/generate.go index 9caf7fa1e37..12452293bc6 100644 --- a/cmd/bundle/generate.go +++ b/cmd/bundle/generate.go @@ -40,6 +40,7 @@ Use --bind to automatically bind the generated resource to the existing workspac cmd.AddCommand(generate.NewGenerateDashboardCommand()) cmd.AddCommand(generate.NewGenerateAlertCommand()) cmd.AddCommand(generate.NewGenerateAppCommand()) + cmd.AddCommand(generate.NewGenerateGenieSpaceCommand()) cmd.PersistentFlags().StringVar(&key, "key", "", `resource key to use for the generated configuration`) return cmd } diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 7af4e01e92f..250d05e3693 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -275,7 +275,11 @@ func waitForChanges(ctx context.Context, w *databricks.WorkspaceClient, dashboar break } - time.Sleep(1 * time.Second) + select { + case <-ctx.Done(): + return + case <-time.After(1 * time.Second): + } } } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go new file mode 100644 index 00000000000..542dfd50b35 --- /dev/null +++ b/cmd/bundle/generate/genie_space.go @@ -0,0 +1,530 @@ +package generate + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "maps" + "os" + "path" + "path/filepath" + "slices" + "time" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/bundle/generate" + "github.com/databricks/cli/bundle/phases" + "github.com/databricks/cli/bundle/resources" + "github.com/databricks/cli/bundle/statemgmt" + "github.com/databricks/cli/cmd/bundle/deployment" + "github.com/databricks/cli/cmd/bundle/utils" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/yamlsaver" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/textutil" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/spf13/cobra" + "go.yaml.in/yaml/v3" +) + +const genieSpaceWatchInterval = 1 * time.Second + +type genieSpace struct { + // Lookup flags for one-time generate. + existingPath string + existingID string + + // Lookup flag for existing bundle resource. + resource string + + // Where to write the configuration and genie space representation. + resourceDir string + genieSpaceDir string + + // Force overwrite of existing files. + force bool + + // Watch for changes to the genie space. + watch bool + + // Relative path from the resource directory to the genie space directory. + relativeGenieSpaceDir string + + // Command. + cmd *cobra.Command + + // Automatically bind the generated resource to the existing resource. + bind bool + + // Output and error streams. + out io.Writer + err io.Writer +} + +func (g *genieSpace) resolveID(ctx context.Context, b *bundle.Bundle) string { + switch { + case g.existingPath != "": + return g.resolveFromPath(ctx, b) + case g.existingID != "": + return g.resolveFromID(ctx, b) + } + + logdiag.LogError(ctx, errors.New("expected one of --existing-path, --existing-id")) + return "" +} + +func (g *genieSpace) resolveFromPath(ctx context.Context, b *bundle.Bundle) string { + w := b.WorkspaceClient(ctx) + obj, err := w.Workspace.GetStatusByPath(ctx, g.existingPath) //nolint:staticcheck // Deprecated in SDK v0.127.0. Migration to WorkspaceHierarchyService tracked separately. + if err != nil { + if apierr.IsMissing(err) { + logdiag.LogError(ctx, fmt.Errorf("genie space %q not found", path.Base(g.existingPath))) + return "" + } + + logdiag.LogError(ctx, err) + return "" + } + + if obj.ResourceId == "" { + logdiag.LogDiag(ctx, diag.Diagnostic{ + Severity: diag.Error, + Summary: "expected a non-empty genie space resource ID", + }) + return "" + } + + return obj.ResourceId +} + +func (g *genieSpace) resolveFromID(ctx context.Context, b *bundle.Bundle) string { + w := b.WorkspaceClient(ctx) + obj, err := w.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: g.existingID, + }) + if err != nil { + // The Genie API returns 403 (not 404) when a space does not exist. + if apierr.IsMissing(err) || errors.Is(err, apierr.ErrPermissionDenied) { + logdiag.LogError(ctx, fmt.Errorf("genie space with ID %s not found", g.existingID)) + return "" + } + logdiag.LogError(ctx, err) + return "" + } + + return obj.SpaceId +} + +func (g *genieSpace) saveSerializedGenieSpace(ctx context.Context, b *bundle.Bundle, genieSpace *dashboards.GenieSpace, filename string) error { + if genieSpace.SerializedSpace == "" { + return fmt.Errorf("genie space response did not include serialized_space; refusing to write %s", filepath.ToSlash(filename)) + } + + // Unmarshal and remarshal the serialized genie space to ensure it is formatted correctly. + // The result will have alphabetically sorted keys and be indented. + data, err := remarshalJSON([]byte(genieSpace.SerializedSpace)) + if err != nil { + return err + } + + // Make sure the output directory exists. + if err := os.MkdirAll(filepath.Dir(filename), 0o755); err != nil { + return err + } + + // Clean the filename to ensure it is a valid path (and can be used on this OS). + filename = filepath.Clean(filename) + + // Attempt to make the path relative to the bundle root. + rel, err := filepath.Rel(b.BundleRootPath, filename) + if err != nil { + rel = filename + } + + // Verify that the file does not already exist. + info, err := os.Stat(filename) + if err == nil { + if info.IsDir() { + return fmt.Errorf("%s is a directory", filepath.ToSlash(rel)) + } + if !g.force { + return fmt.Errorf("%s already exists. Use --force to overwrite", filepath.ToSlash(rel)) + } + } + + cmdio.LogString(ctx, "Writing genie space to "+filepath.ToSlash(rel)) + return os.WriteFile(filename, data, 0o644) +} + +func (g *genieSpace) saveConfiguration(ctx context.Context, b *bundle.Bundle, genieSpace *dashboards.GenieSpace, key string) error { + // Save serialized genie space definition to the genie space directory. + genieSpaceBasename := key + ".geniespace.json" + genieSpacePath := filepath.Join(g.genieSpaceDir, genieSpaceBasename) + err := g.saveSerializedGenieSpace(ctx, b, genieSpace, genieSpacePath) + if err != nil { + return err + } + + // Synthesize resource configuration. + v, err := generate.ConvertGenieSpaceToValue(genieSpace, path.Join(g.relativeGenieSpaceDir, genieSpaceBasename)) + if err != nil { + return err + } + + result := map[string]dyn.Value{ + "resources": dyn.V(map[string]dyn.Value{ + "genie_spaces": dyn.V(map[string]dyn.Value{ + key: v, + }), + }), + } + + // Make sure the output directory exists. + if err := os.MkdirAll(g.resourceDir, 0o755); err != nil { + return err + } + + // Save the configuration to the resource directory. + resourcePath := filepath.Join(g.resourceDir, key+".genie_space.yml") + saver := yamlsaver.NewSaverWithStyle(map[string]yaml.Style{ + "title": yaml.DoubleQuotedStyle, + }) + + // Attempt to make the path relative to the bundle root. + rel, err := filepath.Rel(b.BundleRootPath, resourcePath) + if err != nil { + rel = resourcePath + } + + cmdio.LogString(ctx, "Writing configuration to "+filepath.ToSlash(rel)) + err = saver.SaveAsYAML(result, resourcePath, g.force) + if err != nil { + return err + } + + return nil +} + +func (g *genieSpace) updateGenieSpaceForResource(ctx context.Context, b *bundle.Bundle) { + resource, ok := b.Config.Resources.GenieSpaces[g.resource] + if !ok { + logdiag.LogError(ctx, fmt.Errorf("genie space resource %q is not defined", g.resource)) + return + } + + if resource.FilePath == "" { + logdiag.LogError(ctx, fmt.Errorf("genie space resource %q has no file path defined", g.resource)) + return + } + + genieSpaceID := resource.ID + genieSpacePath := resource.FilePath + + w := b.WorkspaceClient(ctx) + + first := true + for { + genieSpace, err := w.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: genieSpaceID, + IncludeSerializedSpace: true, + }) + if err != nil { + logdiag.LogError(ctx, err) + return + } + + // Genie has no remote modification timestamp we can poll. Compare + // the canonicalized remote body against the on-disk body and only + // re-save when they differ. The first iteration always saves, to + // match the prior behavior of an unconditional initial sync. + shouldSave := first + if !first { + differs, err := genieSpaceBodyDiffersFromDisk(genieSpace.SerializedSpace, genieSpacePath) + if err != nil { + logdiag.LogError(ctx, err) + return + } + shouldSave = differs + } + + if shouldSave { + if err := g.saveSerializedGenieSpace(ctx, b, genieSpace, genieSpacePath); err != nil { + logdiag.LogError(ctx, err) + return + } + } + + if !g.watch { + return + } + + first = false + select { + case <-ctx.Done(): + return + case <-time.After(genieSpaceWatchInterval): + } + } +} + +// genieSpaceBodyDiffersFromDisk reports whether the canonicalized remote +// serialized_space differs from the contents of filename. +func genieSpaceBodyDiffersFromDisk(remoteSerialized, filename string) (bool, error) { + if remoteSerialized == "" { + return false, nil + } + canonical, err := remarshalJSON([]byte(remoteSerialized)) + if err != nil { + return false, err + } + onDisk, err := os.ReadFile(filename) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return true, nil + } + return false, err + } + return !bytes.Equal(canonical, onDisk), nil +} + +func (g *genieSpace) generateForExisting(ctx context.Context, b *bundle.Bundle, genieSpaceID string) { + w := b.WorkspaceClient(ctx) + genieSpace, err := w.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: genieSpaceID, + IncludeSerializedSpace: true, + }) + if err != nil { + logdiag.LogError(ctx, err) + return + } + + key := g.cmd.Flag("key").Value.String() + if key == "" { + key = textutil.NormalizeString(genieSpace.Title) + } + err = g.saveConfiguration(ctx, b, genieSpace, key) + if err != nil { + logdiag.LogError(ctx, err) + return + } + + if g.bind { + err = deployment.BindResource(g.cmd, key, genieSpaceID, true, false, true) + if err != nil { + logdiag.LogError(ctx, err) + return + } + cmdio.LogString(ctx, fmt.Sprintf("Successfully bound genie space with an id '%s'", genieSpaceID)) + } +} + +func (g *genieSpace) initialize(ctx context.Context, b *bundle.Bundle) { + // Make the paths absolute if they aren't already. + if !filepath.IsAbs(g.resourceDir) { + g.resourceDir = filepath.Join(b.BundleRootPath, g.resourceDir) + } + if !filepath.IsAbs(g.genieSpaceDir) { + g.genieSpaceDir = filepath.Join(b.BundleRootPath, g.genieSpaceDir) + } + + // Make sure we know how the genie space path is relative to the resource path. + rel, err := filepath.Rel(g.resourceDir, g.genieSpaceDir) + if err != nil { + logdiag.LogError(ctx, err) + return + } + + g.relativeGenieSpaceDir = filepath.ToSlash(rel) +} + +func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { + phases.Initialize(ctx, b) + if logdiag.HasError(ctx) { + return + } + + requiredEngine, err := utils.ResolveEngineSetting(ctx, b) + if err != nil { + logdiag.LogError(ctx, err) + return + } + ctx, stateDesc := statemgmt.PullResourcesState(ctx, b, statemgmt.AlwaysPull(true), requiredEngine) + if logdiag.HasError(ctx) { + return + } + + var state statemgmt.ExportedResourcesMap + if stateDesc.Engine.IsDirect() { + _, localPath := b.StateFilenameDirect(ctx) + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + logdiag.LogError(ctx, err) + return + } + state = b.DeploymentBundle.ExportState(ctx) + } else { + var err error + state, err = terraform.ParseResourcesState(ctx, b) + if err != nil { + logdiag.LogError(ctx, err) + return + } + } + + bundle.ApplySeqContext(ctx, b, + statemgmt.Load(state), + ) + if logdiag.HasError(ctx) { + return + } + + g.updateGenieSpaceForResource(ctx, b) +} + +func (g *genieSpace) runForExisting(ctx context.Context, b *bundle.Bundle) { + // Resolve the ID of the genie space to generate configuration for. + genieSpaceID := g.resolveID(ctx, b) + if logdiag.HasError(ctx) { + return + } + + g.generateForExisting(ctx, b, genieSpaceID) +} + +func (g *genieSpace) RunE(cmd *cobra.Command, args []string) error { + ctx := logdiag.InitContext(cmd.Context()) + cmd.SetContext(ctx) + + b := root.MustConfigureBundle(cmd) + if b == nil || logdiag.HasError(ctx) { + return root.ErrAlreadyPrinted + } + + g.initialize(ctx, b) + if logdiag.HasError(ctx) { + return root.ErrAlreadyPrinted + } + + if g.resource != "" { + g.runForResource(ctx, b) + } else { + g.runForExisting(ctx, b) + } + + if logdiag.HasError(ctx) { + return root.ErrAlreadyPrinted + } + + return nil +} + +// filterGenieSpaces returns a filter that only includes genie spaces. +func filterGenieSpaces(ref resources.Reference) bool { + return ref.Description.SingularName == "genie_space" +} + +// genieSpaceResourceCompletion executes to autocomplete the argument to the resource flag. +func genieSpaceResourceCompletion(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + b := root.MustConfigureBundle(cmd) + if logdiag.HasError(cmd.Context()) { + return nil, cobra.ShellCompDirectiveError + } + + if b == nil { + return nil, cobra.ShellCompDirectiveNoFileComp + } + + return slices.Collect(maps.Keys(resources.Completions(b, filterGenieSpaces))), cobra.ShellCompDirectiveNoFileComp +} + +func NewGenerateGenieSpaceCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "genie-space", + Short: "Generate configuration for a Genie space", + Long: `Generate bundle configuration for an existing Databricks Genie space. + +This command downloads an existing Genie space and creates bundle files +that you can use to deploy the Genie space to other environments or manage it as code. + +Examples: + # Import Genie space by workspace path + databricks bundle generate genie-space --existing-path /Users/me/my-genie-space + + # Import Genie space by ID + databricks bundle generate genie-space --existing-id abc123 + + # Watch for changes to keep bundle in sync with UI modifications + databricks bundle generate genie-space --resource my_genie_space --watch --force + +What gets generated: +- Genie space configuration YAML file with settings and a reference to the Genie space definition +- Genie space definition (.geniespace.json) file with the serialized space content + +Sync workflow for Genie space development: +When developing Genie spaces, you can modify them in the Databricks UI and sync +changes back to your bundle: + +1. Make changes to Genie space in the Databricks UI +2. Run: databricks bundle generate genie-space --resource my_genie_space --force +3. Commit changes to version control +4. Deploy to other environments with: databricks bundle deploy --target prod + +The --watch flag continuously polls for remote changes and updates your local +bundle files automatically, useful during active Genie space development.`, + } + + g := &genieSpace{ + out: cmd.OutOrStdout(), + err: cmd.ErrOrStderr(), + } + + // Lookup flags. + cmd.Flags().StringVar(&g.existingPath, "existing-path", "", `workspace path of the Genie space to generate configuration for`) + cmd.Flags().StringVar(&g.existingID, "existing-id", "", `ID of the Genie space to generate configuration for`) + cmd.Flags().StringVar(&g.resource, "resource", "", `resource key of Genie space to watch for changes`) + + // Alias lookup flags that include the resource type name. + cmd.Flags().StringVar(&g.existingPath, "existing-genie-space-path", "", `workspace path of the Genie space to generate configuration for`) + cmd.Flags().StringVar(&g.existingID, "existing-genie-space-id", "", `ID of the Genie space to generate configuration for`) + cmd.Flags().MarkHidden("existing-genie-space-path") + cmd.Flags().MarkHidden("existing-genie-space-id") + + // Output flags. + cmd.Flags().StringVarP(&g.resourceDir, "resource-dir", "d", "resources", `directory to write the configuration to`) + cmd.Flags().StringVarP(&g.genieSpaceDir, "genie-space-dir", "s", "src", `directory to write the Genie space representation to`) + cmd.Flags().BoolVarP(&g.force, "force", "f", false, `force overwrite existing files in the output directory`) + + cmd.Flags().BoolVarP(&g.bind, "bind", "b", false, `automatically bind the generated Genie space config to the existing Genie space`) + cmd.Flags().MarkHidden("bind") + + // Exactly one of the lookup flags must be provided. + cmd.MarkFlagsOneRequired( + "existing-path", + "existing-id", + "resource", + ) + + // Watch flag. This is relevant only in combination with the resource flag. + cmd.Flags().BoolVar(&g.watch, "watch", false, `watch for changes to the Genie space and update the configuration`) + + // Make sure the watch flag is only used with the existing-resource flag. + cmd.MarkFlagsMutuallyExclusive("watch", "existing-path") + cmd.MarkFlagsMutuallyExclusive("watch", "existing-id") + + // Make sure the bind flag is only used with the existing-resource flag. + cmd.MarkFlagsMutuallyExclusive("bind", "resource") + + // Completion for the resource flag. + cmd.RegisterFlagCompletionFunc("resource", genieSpaceResourceCompletion) + + cmd.RunE = g.RunE + g.cmd = cmd + return cmd +} diff --git a/cmd/bundle/generate/genie_space_test.go b/cmd/bundle/generate/genie_space_test.go new file mode 100644 index 00000000000..746b2b1c3af --- /dev/null +++ b/cmd/bundle/generate/genie_space_test.go @@ -0,0 +1,126 @@ +package generate + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func newGenieSpaceTestBundle(t *testing.T, m *mocks.MockWorkspaceClient, filePath string) *bundle.Bundle { + t.Helper() + b := &bundle.Bundle{ + BundleRootPath: t.TempDir(), + Config: config.Root{ + Resources: config.Resources{ + GenieSpaces: map[string]*resources.GenieSpace{ + "my_space": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "My Space", + }, + FilePath: filePath, + }, + }, + }, + }, + } + b.Config.Resources.GenieSpaces["my_space"].ID = "space-id-1" + b.SetWorkpaceClient(m.WorkspaceClient) + return b +} + +func TestGenieSpace_UpdateForResource_WritesFileWhenNotWatching(t *testing.T) { + dir := t.TempDir() + filePath := filepath.Join(dir, "my_space.geniespace.json") + + m := mocks.NewMockWorkspaceClient(t) + m.GetMockGenieAPI().EXPECT(). + GetSpace(mock.Anything, dashboards.GenieGetSpaceRequest{ + SpaceId: "space-id-1", + IncludeSerializedSpace: true, + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id-1", + Title: "My Space", + SerializedSpace: `{"version":1}`, + }, nil). + Once() + + g := &genieSpace{ + resource: "my_space", + force: true, + } + b := newGenieSpaceTestBundle(t, m, filePath) + + ctx, _ := cmdio.NewTestContextWithStdout(t.Context()) + ctx = logdiag.InitContext(ctx) + logdiag.SetCollect(ctx, true) + g.updateGenieSpaceForResource(ctx, b) + + require.Empty(t, logdiag.FlushCollected(ctx)) + + contents, err := os.ReadFile(filePath) + require.NoError(t, err) + assert.Contains(t, string(contents), `"version"`) +} + +func TestGenieSpace_UpdateForResource_WatchExitsOnCancel(t *testing.T) { + dir := t.TempDir() + filePath := filepath.Join(dir, "my_space.geniespace.json") + + m := mocks.NewMockWorkspaceClient(t) + // Allow any number of GetSpace calls; we don't know how many fire before cancel. + m.GetMockGenieAPI().EXPECT(). + GetSpace(mock.Anything, dashboards.GenieGetSpaceRequest{ + SpaceId: "space-id-1", + IncludeSerializedSpace: true, + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id-1", + Title: "My Space", + SerializedSpace: `{"version":1}`, + }, nil). + Maybe() + + g := &genieSpace{ + resource: "my_space", + force: true, + watch: true, + } + b := newGenieSpaceTestBundle(t, m, filePath) + + base, _ := cmdio.NewTestContextWithStdout(t.Context()) + ctx, cancel := context.WithCancel(logdiag.InitContext(base)) + logdiag.SetCollect(ctx, true) + + done := make(chan struct{}) + go func() { + g.updateGenieSpaceForResource(ctx, b) + close(done) + }() + + // First iteration always saves. Wait until the file lands, then cancel. + require.Eventually(t, func() bool { + _, err := os.Stat(filePath) + return err == nil + }, 2*time.Second, 10*time.Millisecond, "expected initial save to write file") + + cancel() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("watch loop did not exit promptly after ctx cancel") + } +} diff --git a/cmd/experimental/workspace_open_test.go b/cmd/experimental/workspace_open_test.go index f81e79be52b..9589b48a126 100644 --- a/cmd/experimental/workspace_open_test.go +++ b/cmd/experimental/workspace_open_test.go @@ -67,7 +67,7 @@ func TestBuildWorkspaceURLFragmentBasedResources(t *testing.T) { func TestBuildWorkspaceURLUnknownResourceType(t *testing.T) { _, err := workspaceurls.BuildResourceURL("https://myworkspace.databricks.com", "unknown", "123", "") assert.ErrorContains(t, err, "unknown resource type \"unknown\"") - assert.ErrorContains(t, err, "alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses") + assert.ErrorContains(t, err, "alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses") } func TestBuildWorkspaceURLHostWithTrailingSlash(t *testing.T) { @@ -115,6 +115,7 @@ func TestWorkspaceOpenCommandCompletion(t *testing.T) { "database_catalogs", "database_instances", "experiments", + "genie_spaces", "jobs", "model_serving_endpoints", "models", @@ -145,7 +146,7 @@ func TestWorkspaceOpenCommandCompletionSecondArg(t *testing.T) { func TestWorkspaceOpenCommandHelpText(t *testing.T) { cmd := newWorkspaceOpenCommand() - assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses.") + assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses.") assert.Contains(t, cmd.Long, "databricks experimental open jobs 123456789") assert.Contains(t, cmd.Long, "databricks experimental open notebooks /Users/user@example.com/my-notebook") assert.Contains(t, cmd.Long, "databricks experimental open registered_models catalog.schema.my_model") diff --git a/libs/structs/structwalk/walktype_test.go b/libs/structs/structwalk/walktype_test.go index 5db64efc0d7..51069fc1f42 100644 --- a/libs/structs/structwalk/walktype_test.go +++ b/libs/structs/structwalk/walktype_test.go @@ -136,7 +136,7 @@ func TestTypeJobSettings(t *testing.T) { func TestTypeRoot(t *testing.T) { testStruct(t, reflect.TypeFor[config.Root](), - 5000, 5800, // 5651 after SDK v0.136.0 bump + 5000, 6000, // 5814 after genie_space resource added map[string]any{ "bundle.target": "", `variables.*.lookup.dashboard`: "", diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index ff70f6b0505..6c3766ccaeb 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -141,6 +141,7 @@ type FakeWorkspace struct { Volumes map[string]catalog.VolumeInfo Dashboards map[string]fakeDashboard PublishedDashboards map[string]dashboards.PublishedDashboard + GenieSpaces map[string]dashboards.GenieSpace SqlWarehouses map[string]sql.GetWarehouseResponse Alerts map[string]sql.AlertV2 Experiments map[string]ml.GetExperimentResponse @@ -288,6 +289,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { Volumes: map[string]catalog.VolumeInfo{}, Dashboards: map[string]fakeDashboard{}, PublishedDashboards: map[string]dashboards.PublishedDashboard{}, + GenieSpaces: map[string]dashboards.GenieSpace{}, SqlWarehouses: map[string]sql.GetWarehouseResponse{ TestDefaultWarehouseId: { Id: TestDefaultWarehouseId, diff --git a/libs/testserver/genie_spaces.go b/libs/testserver/genie_spaces.go new file mode 100644 index 00000000000..cded1497611 --- /dev/null +++ b/libs/testserver/genie_spaces.go @@ -0,0 +1,208 @@ +package testserver + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "strconv" + "strings" + + "github.com/databricks/databricks-sdk-go/service/dashboards" +) + +// generateGenieSpaceId returns a random 32-character hex string. +func generateGenieSpaceId() (string, error) { + randomBytes := make([]byte, 16) + _, err := rand.Read(randomBytes) + if err != nil { + return "", err + } + return hex.EncodeToString(randomBytes), nil +} + +func (s *FakeWorkspace) GenieSpaceCreate(req Request) Response { + defer s.LockUnlock()() + + var createReq dashboards.GenieCreateSpaceRequest + if err := json.Unmarshal(req.Body, &createReq); err != nil { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "message": "Invalid request body: " + err.Error(), + }, + } + } + + // Default to user's home directory if parent_path is not provided (matches cloud behavior) + if createReq.ParentPath == "" { + createReq.ParentPath = "/Users/" + s.CurrentUser().UserName + } + + spaceId, err := generateGenieSpaceId() + if err != nil { + return Response{ + StatusCode: 500, + Body: map[string]string{ + "message": "Failed to generate genie space ID", + }, + } + } + + // Strip the /Workspace prefix from parent_path before storing. This matches + // the remote behavior: the GET API returns parent_path without the prefix, + // mirroring DashboardCreate. + if strings.HasPrefix(createReq.ParentPath, "/Workspace/") { + createReq.ParentPath = strings.TrimPrefix(createReq.ParentPath, "/Workspace") + } + + genieSpace := dashboards.GenieSpace{ + SpaceId: spaceId, + Title: createReq.Title, + Description: createReq.Description, + ParentPath: createReq.ParentPath, + WarehouseId: createReq.WarehouseId, + SerializedSpace: createReq.SerializedSpace, + // Mirror libs/testserver/dashboards.go: initialize etag to a numeric + // string so each subsequent update can bump it monotonically. + Etag: "1", + } + + s.GenieSpaces[spaceId] = genieSpace + + // Genie spaces are not exposed as workspace files ("dataRoom is not + // user-facing"), so unlike dashboards we do not register a workspace path + // entry — there is nothing to resolve via the Workspace API. + + return Response{ + Body: genieSpace, + } +} + +func (s *FakeWorkspace) GenieSpaceGet(req Request) Response { + defer s.LockUnlock()() + + spaceId := req.Vars["space_id"] + genieSpace, ok := s.GenieSpaces[spaceId] + if !ok { + // The real API returns 403 (not 404) when a Genie space does not exist. + return Response{ + StatusCode: 403, + Body: map[string]string{ + "message": "Genie space not found", + }, + } + } + + // The GET API only returns the etag when serialized_space is requested + // (include_serialized_space=true). genieSpace is a copy, so clearing the + // field here only affects the response. + if req.URL.Query().Get("include_serialized_space") != "true" { + genieSpace.Etag = "" + } + + return Response{ + Body: genieSpace, + } +} + +func (s *FakeWorkspace) GenieSpaceUpdate(req Request) Response { + defer s.LockUnlock()() + + spaceId := req.Vars["space_id"] + genieSpace, ok := s.GenieSpaces[spaceId] + if !ok { + // The real API returns 403 (not 404) when a Genie space does not exist. + return Response{ + StatusCode: 403, + Body: map[string]string{ + "message": "Genie space not found", + }, + } + } + + var updateReq dashboards.GenieUpdateSpaceRequest + if err := json.Unmarshal(req.Body, &updateReq); err != nil { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "message": "Invalid request body: " + err.Error(), + }, + } + } + + // Optimistic concurrency: if the caller sent an etag, it must match the + // current one. Empty etag means apply unconditionally. + if updateReq.Etag != "" && updateReq.Etag != genieSpace.Etag { + return Response{ + StatusCode: 409, + Body: map[string]string{ + "message": "Etag mismatch: expected " + genieSpace.Etag + ", got " + updateReq.Etag, + }, + } + } + + prev := genieSpace + if updateReq.Title != "" { + genieSpace.Title = updateReq.Title + } + if updateReq.Description != "" { + genieSpace.Description = updateReq.Description + } + if updateReq.WarehouseId != "" { + genieSpace.WarehouseId = updateReq.WarehouseId + } + if updateReq.ParentPath != "" { + parentPath := updateReq.ParentPath + if strings.HasPrefix(parentPath, "/Workspace/") { + parentPath = strings.TrimPrefix(parentPath, "/Workspace") + } + genieSpace.ParentPath = parentPath + } + if updateReq.SerializedSpace != "" { + genieSpace.SerializedSpace = updateReq.SerializedSpace + } + + // The backend bumps the etag only when serialized_space changes; updates to + // other fields (title, description, warehouse_id, parent_path) leave it + // unchanged. This mirrors the GET API, which only returns the etag + // alongside serialized_space. + if prev.SerializedSpace != genieSpace.SerializedSpace { + prevEtag, err := strconv.Atoi(genieSpace.Etag) + if err != nil { + return Response{ + StatusCode: 500, + Body: map[string]string{ + "message": "Invalid stored etag: " + genieSpace.Etag, + }, + } + } + genieSpace.Etag = strconv.Itoa(prevEtag + 1) + } + + s.GenieSpaces[spaceId] = genieSpace + + return Response{ + Body: genieSpace, + } +} + +func (s *FakeWorkspace) GenieSpaceTrash(req Request) Response { + defer s.LockUnlock()() + + spaceId := req.Vars["space_id"] + if _, ok := s.GenieSpaces[spaceId]; !ok { + // The real API returns 403 (not 404) when a Genie space does not exist. + return Response{ + StatusCode: 403, + Body: map[string]string{ + "message": "Genie space not found", + }, + } + } + + delete(s.GenieSpaces, spaceId) + + return Response{ + StatusCode: 200, + } +} diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index a5dea385abe..e0d621269dc 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -309,6 +309,20 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.DashboardUnpublish(req) }) + // Genie Spaces: + server.Handle("GET", "/api/2.0/genie/spaces/{space_id}", func(req Request) any { + return req.Workspace.GenieSpaceGet(req) + }) + server.Handle("POST", "/api/2.0/genie/spaces", func(req Request) any { + return req.Workspace.GenieSpaceCreate(req) + }) + server.Handle("PATCH", "/api/2.0/genie/spaces/{space_id}", func(req Request) any { + return req.Workspace.GenieSpaceUpdate(req) + }) + server.Handle("DELETE", "/api/2.0/genie/spaces/{space_id}", func(req Request) any { + return req.Workspace.GenieSpaceTrash(req) + }) + // Pipelines: server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}", func(req Request) any { diff --git a/libs/testserver/permissions.go b/libs/testserver/permissions.go index e7983b1afa7..312c88e9029 100644 --- a/libs/testserver/permissions.go +++ b/libs/testserver/permissions.go @@ -25,6 +25,7 @@ var requestObjectTypeToObjectType = map[string]string{ "sql/alerts": "alert", "sql/queries": "query", "dashboards": "dashboard", + "genie": "genie-space", "experiments": "mlflowExperiment", "registered-models": "registered-model", "serving-endpoints": "serving-endpoint", diff --git a/libs/workspaceurls/urls.go b/libs/workspaceurls/urls.go index be9a41fc959..46930c92239 100644 --- a/libs/workspaceurls/urls.go +++ b/libs/workspaceurls/urls.go @@ -16,6 +16,7 @@ var resourceURLPatterns = map[string]string{ "database_catalogs": "explore/data/%s", "database_instances": "compute/database-instances/%s", "experiments": "ml/experiments/%s", + "genie_spaces": "genie/rooms/%s", "jobs": "jobs/%s", "models": "ml/models/%s", "model_serving_endpoints": "ml/endpoints/%s", diff --git a/libs/workspaceurls/urls_test.go b/libs/workspaceurls/urls_test.go index af7b5b08ea4..d0a86a0f247 100644 --- a/libs/workspaceurls/urls_test.go +++ b/libs/workspaceurls/urls_test.go @@ -111,6 +111,7 @@ func TestResourceURL(t *testing.T) { {"jobs", "jobs", "123", "https://host.com/jobs/123"}, {"experiments", "experiments", "exp-1", "https://host.com/ml/experiments/exp-1"}, {"dashboards", "dashboards", "d-1", "https://host.com/dashboardsv3/d-1/published"}, + {"genie_spaces", "genie_spaces", "space-1", "https://host.com/genie/rooms/space-1"}, {"notebooks", "notebooks", "12345", "https://host.com/#notebook/12345"}, {"notebooks with path", "notebooks", "/Users/u/nb", "https://host.com/#notebook//Users/u/nb"}, {"registered_models normalizes dots", "registered_models", "cat.sch.model", "https://host.com/explore/data/models/cat/sch/model"},