From 9e46a655e5fc974a00d940c9c6b8e0c856dcd562 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Tue, 21 Apr 2026 13:13:59 +0200 Subject: [PATCH 01/42] bundle: add genie_spaces resource for direct deploy Adds first-class support for Genie spaces as a bundle resource, complete with CRUD via direct-mode deploy, `bundle generate genie-space` to import an existing space, permissions handling, and acceptance tests. The resource configuration follows the dashboards pattern: a `file_path` field points to a local `.genie.json` file whose contents are inlined into `serialized_space` during deployment. The parent_path defaults to `${workspace.resource_path}` and is normalized to the `/Workspace` prefix, matching the API's expected form. Co-authored-by: Isaac --- .../generate/genie_space/databricks.yml | 2 + .../genie_space/genie_space.json.tmpl | 7 + .../bundle/generate/genie_space/out.test.toml | 5 + .../genie_space/test_genie_space.genie.json | 4 + .../resource/test_genie_space.genie_space.yml | 7 + .../bundle/generate/genie_space/output.txt | 6 + acceptance/bundle/generate/genie_space/script | 8 + .../bundle/generate/genie_space/test.toml | 10 + .../bundle/help/bundle-generate/output.txt | 1 + acceptance/bundle/refschema/out.fields.txt | 18 + .../genie_spaces/simple/databricks.yml.tmpl | 11 + .../genie_spaces/simple/out.plan.json | 26 + .../genie_spaces/simple/out.test.toml | 6 + .../resources/genie_spaces/simple/output.txt | 24 + .../simple/sales_analytics.genie.json | 136 +++ .../resources/genie_spaces/simple/script | 17 + .../resources/genie_spaces/simple/test.toml | 10 + .../bundle/resources/genie_spaces/test.toml | 3 + .../genie_space_complex/databricks.yml | 51 ++ .../full_featured.genie.json | 160 ++++ .../genie_space_complex/out.test.toml | 5 + .../validate/genie_space_complex/output.txt | 18 + .../validate/genie_space_complex/script | 19 + .../genie_space_defaults/databricks.yml | 30 + .../genie_space_defaults/out.test.toml | 5 + .../validate/genie_space_defaults/output.txt | 14 + .../validate/genie_space_defaults/script | 1 + acceptance/experimental/open/output.txt | 3 +- .../paths/genie_space_paths_visitor.go | 18 + .../apply_bundle_permissions.go | 4 + .../mutator/resourcemutator/apply_presets.go | 8 + .../resourcemutator/apply_target_mode_test.go | 10 + .../configure_genie_space_serialized_space.go | 53 ++ .../resourcemutator/genie_space_fixups.go | 30 + .../resourcemutator/resource_mutator.go | 10 + .../mutator/resourcemutator/run_as_test.go | 2 + bundle/config/mutator/translate_paths.go | 1 + .../mutator/translate_paths_genie_spaces.go | 22 + bundle/config/resources.go | 3 + bundle/config/resources/genie_space.go | 94 ++ bundle/config/resources_test.go | 4 + bundle/deploy/terraform/lifecycle_test.go | 1 + bundle/direct/dresources/all.go | 2 + bundle/direct/dresources/all_test.go | 19 + bundle/direct/dresources/apitypes.yml | 2 + bundle/direct/dresources/genie_space.go | 164 ++++ bundle/direct/dresources/permissions.go | 1 + bundle/direct/dresources/resources.yml | 13 + bundle/direct/dresources/type_test.go | 3 + bundle/docsgen/output/reference.md | 832 ++---------------- bundle/docsgen/output/resources.md | 112 +++ bundle/generate/genie_space.go | 22 + bundle/internal/schema/annotations.yml | 31 + bundle/schema/jsonschema.json | 58 ++ bundle/schema/jsonschema_for_docs.json | 42 + bundle/statemgmt/state_load_test.go | 35 + cmd/bundle/generate.go | 1 + cmd/bundle/generate/genie_space.go | 503 +++++++++++ cmd/experimental/workspace_open_test.go | 5 +- libs/testserver/fake_workspace.go | 2 + libs/testserver/genie_spaces.go | 162 ++++ libs/testserver/handlers.go | 14 + libs/testserver/permissions.go | 1 + libs/workspaceurls/urls.go | 1 + 64 files changed, 2118 insertions(+), 774 deletions(-) create mode 100644 acceptance/bundle/generate/genie_space/databricks.yml create mode 100644 acceptance/bundle/generate/genie_space/genie_space.json.tmpl create mode 100644 acceptance/bundle/generate/genie_space/out.test.toml create mode 100644 acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.genie.json create mode 100644 acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml create mode 100644 acceptance/bundle/generate/genie_space/output.txt create mode 100644 acceptance/bundle/generate/genie_space/script create mode 100644 acceptance/bundle/generate/genie_space/test.toml create mode 100644 acceptance/bundle/resources/genie_spaces/simple/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/genie_spaces/simple/out.plan.json create mode 100644 acceptance/bundle/resources/genie_spaces/simple/out.test.toml create mode 100644 acceptance/bundle/resources/genie_spaces/simple/output.txt create mode 100644 acceptance/bundle/resources/genie_spaces/simple/sales_analytics.genie.json create mode 100644 acceptance/bundle/resources/genie_spaces/simple/script create mode 100644 acceptance/bundle/resources/genie_spaces/simple/test.toml create mode 100644 acceptance/bundle/resources/genie_spaces/test.toml create mode 100644 acceptance/bundle/validate/genie_space_complex/databricks.yml create mode 100644 acceptance/bundle/validate/genie_space_complex/full_featured.genie.json create mode 100644 acceptance/bundle/validate/genie_space_complex/out.test.toml create mode 100644 acceptance/bundle/validate/genie_space_complex/output.txt create mode 100644 acceptance/bundle/validate/genie_space_complex/script create mode 100644 acceptance/bundle/validate/genie_space_defaults/databricks.yml create mode 100644 acceptance/bundle/validate/genie_space_defaults/out.test.toml create mode 100644 acceptance/bundle/validate/genie_space_defaults/output.txt create mode 100644 acceptance/bundle/validate/genie_space_defaults/script create mode 100644 bundle/config/mutator/paths/genie_space_paths_visitor.go create mode 100644 bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go create mode 100644 bundle/config/mutator/resourcemutator/genie_space_fixups.go create mode 100644 bundle/config/mutator/translate_paths_genie_spaces.go create mode 100644 bundle/config/resources/genie_space.go create mode 100644 bundle/direct/dresources/genie_space.go create mode 100644 bundle/generate/genie_space.go create mode 100644 cmd/bundle/generate/genie_space.go create mode 100644 libs/testserver/genie_spaces.go diff --git a/acceptance/bundle/generate/genie_space/databricks.yml b/acceptance/bundle/generate/genie_space/databricks.yml new file mode 100644 index 00000000000..c533b2b6f98 --- /dev/null +++ b/acceptance/bundle/generate/genie_space/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: genie-space-generate diff --git a/acceptance/bundle/generate/genie_space/genie_space.json.tmpl b/acceptance/bundle/generate/genie_space/genie_space.json.tmpl new file mode 100644 index 00000000000..2056f3061ef --- /dev/null +++ b/acceptance/bundle/generate/genie_space/genie_space.json.tmpl @@ -0,0 +1,7 @@ +{ + "title": "test genie space", + "description": "test description", + "parent_path": "/Workspace/test-$UNIQUE_NAME", + "warehouse_id": "test-warehouse-id", + "serialized_space": "{\"tables\":[],\"questions\":[]}" +} diff --git a/acceptance/bundle/generate/genie_space/out.test.toml b/acceptance/bundle/generate/genie_space/out.test.toml new file mode 100644 index 00000000000..d560f1de043 --- /dev/null +++ b/acceptance/bundle/generate/genie_space/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.genie.json b/acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.genie.json new file mode 100644 index 00000000000..2c12b3c032c --- /dev/null +++ b/acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.genie.json @@ -0,0 +1,4 @@ +{ + "questions": [], + "tables": [] +} diff --git a/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml b/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml new file mode 100644 index 00000000000..14764931f1b --- /dev/null +++ b/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml @@ -0,0 +1,7 @@ +resources: + genie_spaces: + test_genie_space: + title: "test genie space" + warehouse_id: test-warehouse-id + file_path: ../genie_space/test_genie_space.genie.json + description: test description diff --git a/acceptance/bundle/generate/genie_space/output.txt b/acceptance/bundle/generate/genie_space/output.txt new file mode 100644 index 00000000000..985366ada68 --- /dev/null +++ b/acceptance/bundle/generate/genie_space/output.txt @@ -0,0 +1,6 @@ + +>>> [CLI] workspace mkdirs /Workspace/test-[UNIQUE_NAME] + +>>> [CLI] bundle generate genie-space --existing-id [GENIE_SPACE_ID] --genie-space-dir out/genie_space --resource-dir out/resource +Writing genie space to out/genie_space/test_genie_space.genie.json +Writing configuration to out/resource/test_genie_space.genie_space.yml diff --git a/acceptance/bundle/generate/genie_space/script b/acceptance/bundle/generate/genie_space/script new file mode 100644 index 00000000000..8e2fa32170a --- /dev/null +++ b/acceptance/bundle/generate/genie_space/script @@ -0,0 +1,8 @@ +trace $CLI workspace mkdirs /Workspace/test-$UNIQUE_NAME + +# create a genie space to import +envsubst < genie_space.json.tmpl > genie_space.json +genie_space_id=$($CLI genie create-space --json @genie_space.json | jq -r '.space_id') +rm genie_space.json + +trace $CLI bundle generate genie-space --existing-id $genie_space_id --genie-space-dir out/genie_space --resource-dir out/resource diff --git a/acceptance/bundle/generate/genie_space/test.toml b/acceptance/bundle/generate/genie_space/test.toml new file mode 100644 index 00000000000..e389c33c277 --- /dev/null +++ b/acceptance/bundle/generate/genie_space/test.toml @@ -0,0 +1,10 @@ +[[Repls]] +Old = '\\\\' +New = '/' + +[[Repls]] +Old = "[0-9a-f]{32}" +New = "[GENIE_SPACE_ID]" + +[Env] +MSYS_NO_PATHCONV = "1" diff --git a/acceptance/bundle/help/bundle-generate/output.txt b/acceptance/bundle/help/bundle-generate/output.txt index 97e8667ac78..39ce9293539 100644 --- a/acceptance/bundle/help/bundle-generate/output.txt +++ b/acceptance/bundle/help/bundle-generate/output.txt @@ -30,6 +30,7 @@ Available Commands: alert Generate configuration for an alert app Generate bundle configuration for a Databricks app dashboard Generate configuration for a dashboard + genie-space Generate configuration for a Genie space job Generate bundle configuration for a job pipeline Generate bundle configuration for a pipeline diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 07cbaac3a63..42735528ec5 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -734,6 +734,24 @@ resources.external_locations.*.grants[*] catalog.PrivilegeAssignment ALL resources.external_locations.*.grants[*].principal string ALL resources.external_locations.*.grants[*].privileges []catalog.Privilege ALL resources.external_locations.*.grants[*].privileges[*] catalog.Privilege ALL +resources.genie_spaces.*.description string ALL +resources.genie_spaces.*.file_path string INPUT +resources.genie_spaces.*.id string INPUT +resources.genie_spaces.*.lifecycle resources.Lifecycle INPUT +resources.genie_spaces.*.lifecycle.prevent_destroy bool INPUT +resources.genie_spaces.*.modified_status string INPUT +resources.genie_spaces.*.parent_path string ALL +resources.genie_spaces.*.serialized_space any ALL +resources.genie_spaces.*.space_id string ALL +resources.genie_spaces.*.title string ALL +resources.genie_spaces.*.url string INPUT +resources.genie_spaces.*.warehouse_id string ALL +resources.genie_spaces.*.permissions.object_id string ALL +resources.genie_spaces.*.permissions[*] dresources.StatePermission ALL +resources.genie_spaces.*.permissions[*].group_name string ALL +resources.genie_spaces.*.permissions[*].level iam.PermissionLevel ALL +resources.genie_spaces.*.permissions[*].service_principal_name string ALL +resources.genie_spaces.*.permissions[*].user_name string ALL resources.jobs.*.budget_policy_id string ALL resources.jobs.*.continuous *jobs.Continuous ALL resources.jobs.*.continuous.pause_status jobs.PauseStatus ALL diff --git a/acceptance/bundle/resources/genie_spaces/simple/databricks.yml.tmpl b/acceptance/bundle/resources/genie_spaces/simple/databricks.yml.tmpl new file mode 100644 index 00000000000..26ed410751a --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: deploy-genie-space-test-$UNIQUE_NAME + +resources: + genie_spaces: + sales_analytics: + title: "Sales Analytics Genie" + description: "AI assistant for sales data analysis" + warehouse_id: "test-warehouse-id" + parent_path: /Users/$CURRENT_USER_NAME + file_path: "sales_analytics.genie.json" diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.plan.json b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json new file mode 100644 index 00000000000..19f063bb002 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json @@ -0,0 +1,26 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "plan": { + "resources.genie_spaces.sales_analytics": { + "action": "skip", + "remote_state": { + "description": "AI assistant for sales data analysis", + "serialized_space": "{\n \"version\": 1,\n \"config\": {\n \"sample_questions\": [\n {\n \"id\": \"sq-001\",\n \"question\": [\"What is the total revenue?\"]\n },\n {\n \"id\": \"sq-002\",\n \"question\": [\"Show me the top customers\"]\n }\n ]\n },\n \"data_sources\": {\n \"tables\": [\n {\n \"identifier\": \"main.sales.orders\",\n \"column_configs\": [\n {\n \"column_name\": \"order_id\",\n \"get_example_values\": true,\n \"build_value_dictionary\": true\n },\n {\n \"column_name\": \"customer_id\",\n \"get_example_values\": true\n },\n {\n \"column_name\": \"amount\",\n \"get_example_values\": false\n }\n ]\n },\n {\n \"identifier\": \"main.sales.customers\"\n }\n ]\n },\n \"instructions\": {\n \"text_instructions\": [\n {\n \"id\": \"ti-001\",\n \"content\": [\n \"This genie space analyzes sales data.\\n\",\n \"Always filter by date when querying orders.\\n\",\n \"Use customer_name instead of customer_id in results.\"\n ]\n }\n ],\n \"example_question_sqls\": [\n {\n \"id\": \"eq-001\",\n \"question\": [\"What are the top customers by revenue?\"],\n \"sql\": [\n \"SELECT\\n\",\n \" c.customer_name,\\n\",\n \" SUM(o.amount) AS total_revenue\\n\",\n \"FROM main.sales.orders o\\n\",\n \"JOIN main.sales.customers c ON o.customer_id = c.id\\n\",\n \"WHERE o.order_date \u003e= :start_date\\n\",\n \"GROUP BY c.customer_name\\n\",\n \"ORDER BY total_revenue DESC\\n\",\n \"LIMIT :limit\"\n ],\n \"parameters\": [\n {\n \"name\": \"start_date\",\n \"type_hint\": \"STRING\",\n \"description\": [\"Start date for the analysis period\"],\n \"default_value\": {\n \"values\": [\"2024-01-01\"]\n }\n },\n {\n \"name\": \"limit\",\n \"type_hint\": \"INTEGER\",\n \"description\": [\"Number of customers to return\"],\n \"default_value\": {\n \"values\": [\"10\"]\n }\n }\n ]\n },\n {\n \"id\": \"eq-002\",\n \"question\": [\"Calculate daily revenue\"],\n \"sql\": [\n \"SELECT\\n\",\n \" order_date,\\n\",\n \" SUM(amount) AS daily_revenue\\n\",\n \"FROM main.sales.orders\\n\",\n \"GROUP BY order_date\\n\",\n \"ORDER BY order_date\"\n ]\n }\n ],\n \"sql_snippets\": {\n \"measures\": [\n {\n \"id\": \"m-001\",\n \"alias\": \"total_revenue\",\n \"sql\": [\"SUM(orders.amount)\"],\n \"display_name\": \"Total Revenue\"\n }\n ]\n },\n \"sql_functions\": [\n {\n \"id\": \"sf-001\",\n \"identifier\": \"main.analytics.calculate_churn\"\n }\n ]\n },\n \"benchmarks\": {\n \"questions\": [\n {\n \"id\": \"bq-001\",\n \"question\": [\"What is the monthly revenue trend?\"],\n \"answer\": [\n {\n \"format\": \"SQL\",\n \"content\": [\n \"SELECT\\n\",\n \" DATE_TRUNC('month', order_date) AS month,\\n\",\n \" SUM(amount) AS revenue\\n\",\n \"FROM main.sales.orders\\n\",\n \"GROUP BY 1\\n\",\n \"ORDER BY 1\"\n ]\n }\n ]\n }\n ]\n }\n}\n", + "space_id": "[GENIE_SPACE_ID]", + "title": "Sales Analytics Genie", + "warehouse_id": "test-warehouse-id" + }, + "changes": { + "parent_path": { + "action": "skip", + "reason": "input_only", + "old": "/Workspace/Users/[USERNAME]", + "new": "/Workspace/Users/[USERNAME]" + } + } + } + } +} diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.test.toml b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml new file mode 100644 index 00000000000..496668716fa --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] + Ignore = ["databricks.yml"] diff --git a/acceptance/bundle/resources/genie_spaces/simple/output.txt b/acceptance/bundle/resources/genie_spaces/simple/output.txt new file mode 100644 index 00000000000..f65d9583e18 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/output.txt @@ -0,0 +1,24 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-genie-space-test-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] genie get-space [GENIE_SPACE_ID] +{ + "title": "Sales Analytics Genie", + "description": "AI assistant for sales data analysis", + "warehouse_id": "test-warehouse-id" +} + +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.genie_spaces.sales_analytics + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-genie-space-test-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.genie.json b/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.genie.json new file mode 100644 index 00000000000..5d59dff96d5 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.genie.json @@ -0,0 +1,136 @@ +{ + "version": 1, + "config": { + "sample_questions": [ + { + "id": "sq-001", + "question": ["What is the total revenue?"] + }, + { + "id": "sq-002", + "question": ["Show me the top customers"] + } + ] + }, + "data_sources": { + "tables": [ + { + "identifier": "main.sales.orders", + "column_configs": [ + { + "column_name": "order_id", + "get_example_values": true, + "build_value_dictionary": true + }, + { + "column_name": "customer_id", + "get_example_values": true + }, + { + "column_name": "amount", + "get_example_values": false + } + ] + }, + { + "identifier": "main.sales.customers" + } + ] + }, + "instructions": { + "text_instructions": [ + { + "id": "ti-001", + "content": [ + "This genie space analyzes sales data.\n", + "Always filter by date when querying orders.\n", + "Use customer_name instead of customer_id in results." + ] + } + ], + "example_question_sqls": [ + { + "id": "eq-001", + "question": ["What are the top customers by revenue?"], + "sql": [ + "SELECT\n", + " c.customer_name,\n", + " SUM(o.amount) AS total_revenue\n", + "FROM main.sales.orders o\n", + "JOIN main.sales.customers c ON o.customer_id = c.id\n", + "WHERE o.order_date >= :start_date\n", + "GROUP BY c.customer_name\n", + "ORDER BY total_revenue DESC\n", + "LIMIT :limit" + ], + "parameters": [ + { + "name": "start_date", + "type_hint": "STRING", + "description": ["Start date for the analysis period"], + "default_value": { + "values": ["2024-01-01"] + } + }, + { + "name": "limit", + "type_hint": "INTEGER", + "description": ["Number of customers to return"], + "default_value": { + "values": ["10"] + } + } + ] + }, + { + "id": "eq-002", + "question": ["Calculate daily revenue"], + "sql": [ + "SELECT\n", + " order_date,\n", + " SUM(amount) AS daily_revenue\n", + "FROM main.sales.orders\n", + "GROUP BY order_date\n", + "ORDER BY order_date" + ] + } + ], + "sql_snippets": { + "measures": [ + { + "id": "m-001", + "alias": "total_revenue", + "sql": ["SUM(orders.amount)"], + "display_name": "Total Revenue" + } + ] + }, + "sql_functions": [ + { + "id": "sf-001", + "identifier": "main.analytics.calculate_churn" + } + ] + }, + "benchmarks": { + "questions": [ + { + "id": "bq-001", + "question": ["What is the monthly revenue trend?"], + "answer": [ + { + "format": "SQL", + "content": [ + "SELECT\n", + " DATE_TRUNC('month', order_date) AS month,\n", + " SUM(amount) AS revenue\n", + "FROM main.sales.orders\n", + "GROUP BY 1\n", + "ORDER BY 1" + ] + } + ] + } + ] + } +} diff --git a/acceptance/bundle/resources/genie_spaces/simple/script b/acceptance/bundle/resources/genie_spaces/simple/script new file mode 100644 index 00000000000..4db9ef03078 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/script @@ -0,0 +1,17 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy +GENIE_SPACE_ID=$($CLI bundle summary --output json | jq -r '.resources.genie_spaces.sales_analytics.id') + +# Capture the genie space ID as a replacement. +echo "$GENIE_SPACE_ID:GENIE_SPACE_ID" >> ACC_REPLS + +trace $CLI genie get-space $GENIE_SPACE_ID | jq '{title, description, warehouse_id}' + +# Verify that there is no drift right after deploy. +trace $CLI bundle plan -o json > out.plan.json diff --git a/acceptance/bundle/resources/genie_spaces/simple/test.toml b/acceptance/bundle/resources/genie_spaces/simple/test.toml new file mode 100644 index 00000000000..18a07a1b00b --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/test.toml @@ -0,0 +1,10 @@ +Local = true +RecordRequests = false + +# Genie spaces only support direct deployment engine (no Terraform provider yet) +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/bundle/resources/genie_spaces/test.toml b/acceptance/bundle/resources/genie_spaces/test.toml new file mode 100644 index 00000000000..2569ef7dcb1 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/test.toml @@ -0,0 +1,3 @@ +# Genie spaces are only deployed via direct deployment engine +[Env] +DATABRICKS_BUNDLE_ENGINE = "direct" diff --git a/acceptance/bundle/validate/genie_space_complex/databricks.yml b/acceptance/bundle/validate/genie_space_complex/databricks.yml new file mode 100644 index 00000000000..eae5cdc3f10 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_complex/databricks.yml @@ -0,0 +1,51 @@ +bundle: + name: genie-space-complex + +workspace: + resource_path: /foo/bar + +resources: + genie_spaces: + # Test with all features enabled + full_featured: + warehouse_id: "my-warehouse-1234" + title: "Full Featured Genie Space" + description: "A comprehensive test of all genie space features" + file_path: ./full_featured.genie.json + + # Test with inline serialized_space (YAML syntax) + inline_yaml: + warehouse_id: "my-warehouse-1234" + title: "Inline YAML Genie Space" + serialized_space: + version: 1 + data_sources: + tables: + - identifier: main.schema.table1 + column_configs: + - column_name: id + get_example_values: true + build_value_dictionary: true + - column_name: name + get_example_values: true + instructions: + text_instructions: + - id: inst-001 + content: + - "This is a text instruction.\n" + - "It spans multiple lines." + example_question_sqls: + - id: eq-001 + question: + - "How many records are there?" + sql: + - "SELECT COUNT(*) FROM main.schema.table1" + + # Test with empty but valid structure + minimal_valid: + warehouse_id: "my-warehouse-1234" + title: "Minimal Valid" + serialized_space: + version: 1 + data_sources: + tables: [] diff --git a/acceptance/bundle/validate/genie_space_complex/full_featured.genie.json b/acceptance/bundle/validate/genie_space_complex/full_featured.genie.json new file mode 100644 index 00000000000..9c9221d8328 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_complex/full_featured.genie.json @@ -0,0 +1,160 @@ +{ + "version": 1, + "config": { + "sample_questions": [ + { + "id": "sq-001", + "question": ["What is the total revenue?"] + }, + { + "id": "sq-002", + "question": ["Show me the top customers"] + } + ] + }, + "data_sources": { + "tables": [ + { + "identifier": "main.sales.orders", + "column_configs": [ + { + "column_name": "order_id", + "get_example_values": true, + "build_value_dictionary": true + }, + { + "column_name": "customer_id", + "get_example_values": true + }, + { + "column_name": "amount", + "get_example_values": false + } + ] + }, + { + "identifier": "main.sales.customers" + } + ] + }, + "instructions": { + "text_instructions": [ + { + "id": "ti-001", + "content": [ + "This genie space analyzes sales data.\n", + "Always filter by date when querying orders.\n", + "Use customer_name instead of customer_id in results." + ] + } + ], + "example_question_sqls": [ + { + "id": "eq-001", + "question": ["What are the top customers by revenue?"], + "sql": [ + "SELECT\n", + " c.customer_name,\n", + " SUM(o.amount) AS total_revenue\n", + "FROM main.sales.orders o\n", + "JOIN main.sales.customers c ON o.customer_id = c.id\n", + "WHERE o.order_date >= :start_date\n", + "GROUP BY c.customer_name\n", + "ORDER BY total_revenue DESC\n", + "LIMIT :limit" + ], + "parameters": [ + { + "name": "start_date", + "type_hint": "STRING", + "description": ["Start date for the analysis period"], + "default_value": { + "values": ["2024-01-01"] + } + }, + { + "name": "limit", + "type_hint": "INTEGER", + "description": ["Number of customers to return"], + "default_value": { + "values": ["10"] + } + } + ] + }, + { + "id": "eq-002", + "question": ["Calculate daily revenue"], + "sql": [ + "SELECT\n", + " order_date,\n", + " SUM(amount) AS daily_revenue\n", + "FROM main.sales.orders\n", + "GROUP BY order_date\n", + "ORDER BY order_date" + ] + } + ], + "sql_snippets": { + "measures": [ + { + "id": "m-001", + "alias": "total_revenue", + "sql": ["SUM(orders.amount)"], + "display_name": "Total Revenue" + }, + { + "id": "m-002", + "alias": "order_count", + "sql": ["COUNT(orders.order_id)"], + "display_name": "Order Count" + } + ] + }, + "sql_functions": [ + { + "id": "sf-001", + "identifier": "main.analytics.calculate_churn" + } + ] + }, + "benchmarks": { + "questions": [ + { + "id": "bq-001", + "question": ["What is the monthly revenue trend?"], + "answer": [ + { + "format": "SQL", + "content": [ + "SELECT\n", + " DATE_TRUNC('month', order_date) AS month,\n", + " SUM(amount) AS revenue\n", + "FROM main.sales.orders\n", + "GROUP BY 1\n", + "ORDER BY 1" + ] + } + ] + }, + { + "id": "bq-002", + "question": ["Which customers have the highest average order value?"], + "answer": [ + { + "format": "SQL", + "content": [ + "SELECT\n", + " customer_id,\n", + " AVG(amount) AS avg_order_value\n", + "FROM main.sales.orders\n", + "GROUP BY customer_id\n", + "ORDER BY avg_order_value DESC\n", + "LIMIT 10" + ] + } + ] + } + ] + } +} diff --git a/acceptance/bundle/validate/genie_space_complex/out.test.toml b/acceptance/bundle/validate/genie_space_complex/out.test.toml new file mode 100644 index 00000000000..d560f1de043 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_complex/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_complex/output.txt b/acceptance/bundle/validate/genie_space_complex/output.txt new file mode 100644 index 00000000000..2fe58959fc8 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_complex/output.txt @@ -0,0 +1,18 @@ +{ + "full_featured": { + "title": "Full Featured Genie Space", + "warehouse_id": "my-warehouse-1234", + "serialized_space_is_string": true + }, + "inline_yaml": { + "title": "Inline YAML Genie Space", + "serialized_space_type": "object", + "tables_count": 1, + "has_column_configs": true, + "has_text_instructions": true + }, + "minimal_valid": { + "title": "Minimal Valid", + "tables_count": 0 + } +} diff --git a/acceptance/bundle/validate/genie_space_complex/script b/acceptance/bundle/validate/genie_space_complex/script new file mode 100644 index 00000000000..01e1fed72bc --- /dev/null +++ b/acceptance/bundle/validate/genie_space_complex/script @@ -0,0 +1,19 @@ +# Validate complex genie spaces and check the serialized_space structure is preserved +$CLI bundle validate -o json | jq '{ + full_featured: .resources.genie_spaces.full_featured | { + title, + warehouse_id, + serialized_space_is_string: (.serialized_space | type == "string") + }, + inline_yaml: .resources.genie_spaces.inline_yaml | { + title, + serialized_space_type: (.serialized_space | type), + tables_count: (.serialized_space.data_sources.tables | length), + has_column_configs: ((.serialized_space.data_sources.tables[0].column_configs | length) > 0), + has_text_instructions: ((.serialized_space.instructions.text_instructions | length) > 0) + }, + minimal_valid: .resources.genie_spaces.minimal_valid | { + title, + tables_count: (.serialized_space.data_sources.tables | length) + } +}' diff --git a/acceptance/bundle/validate/genie_space_defaults/databricks.yml b/acceptance/bundle/validate/genie_space_defaults/databricks.yml new file mode 100644 index 00000000000..7e4b87d35e6 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_defaults/databricks.yml @@ -0,0 +1,30 @@ +bundle: + name: test-bundle + +workspace: + resource_path: /foo/bar + +resources: + genie_spaces: + empty_string: + warehouse_id: "my-warehouse-1234" + title: "empty-string" + serialized_space: "{}" + + # unchanged + parent_path: "" + + non_empty_string: + warehouse_id: "my-warehouse-1234" + title: "non-empty-string" + serialized_space: "{}" + + # unchanged + parent_path: "already-set" + + default_parent_path: + warehouse_id: "my-warehouse-1234" + title: "default-parent-path" + serialized_space: "{}" + + # parent_path set to default diff --git a/acceptance/bundle/validate/genie_space_defaults/out.test.toml b/acceptance/bundle/validate/genie_space_defaults/out.test.toml new file mode 100644 index 00000000000..d560f1de043 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_defaults/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_defaults/output.txt b/acceptance/bundle/validate/genie_space_defaults/output.txt new file mode 100644 index 00000000000..13105c91b76 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_defaults/output.txt @@ -0,0 +1,14 @@ +{ + "default_parent_path": { + "title": "default-parent-path", + "parent_path": "/Workspace/foo/bar" + }, + "empty_string": { + "title": "empty-string", + "parent_path": "/Workspace" + }, + "non_empty_string": { + "title": "non-empty-string", + "parent_path": "/Workspace/already-set" + } +} diff --git a/acceptance/bundle/validate/genie_space_defaults/script b/acceptance/bundle/validate/genie_space_defaults/script new file mode 100644 index 00000000000..eebc582f136 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_defaults/script @@ -0,0 +1 @@ +$CLI bundle validate -o json | jq '.resources.genie_spaces | map_values({title: .title, parent_path: .parent_path})' diff --git a/acceptance/experimental/open/output.txt b/acceptance/experimental/open/output.txt index af12759c8a4..c395bb4967e 100644 --- a/acceptance/experimental/open/output.txt +++ b/acceptance/experimental/open/output.txt @@ -9,7 +9,7 @@ === unknown resource type >>> [CLI] experimental open --url unknown 123 -Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses +Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses Exit code: 1 @@ -23,6 +23,7 @@ dashboards database_catalogs database_instances experiments +genie_spaces jobs model_serving_endpoints models diff --git a/bundle/config/mutator/paths/genie_space_paths_visitor.go b/bundle/config/mutator/paths/genie_space_paths_visitor.go new file mode 100644 index 00000000000..edd6ff2d8df --- /dev/null +++ b/bundle/config/mutator/paths/genie_space_paths_visitor.go @@ -0,0 +1,18 @@ +package paths + +import ( + "github.com/databricks/cli/libs/dyn" +) + +func VisitGenieSpacePaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { + pattern := dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("genie_spaces"), + dyn.AnyKey(), + dyn.Key("file_path"), + ) + + return dyn.MapByPattern(value, pattern, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + return fn(path, TranslateModeLocalRelative, value) + }) +} diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go index fd019479d77..cbb05d7d622 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go @@ -47,6 +47,10 @@ var ( permissions.CAN_MANAGE: "CAN_MANAGE", permissions.CAN_VIEW: "CAN_READ", }, + "genie_spaces": { + permissions.CAN_MANAGE: "CAN_MANAGE", + permissions.CAN_VIEW: "CAN_READ", + }, "apps": { permissions.CAN_MANAGE: "CAN_MANAGE", permissions.CAN_VIEW: "CAN_USE", diff --git a/bundle/config/mutator/resourcemutator/apply_presets.go b/bundle/config/mutator/resourcemutator/apply_presets.go index 663adf4a281..79c0ebbec3c 100644 --- a/bundle/config/mutator/resourcemutator/apply_presets.go +++ b/bundle/config/mutator/resourcemutator/apply_presets.go @@ -237,6 +237,14 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos dashboard.DisplayName = prefix + dashboard.DisplayName } + // Genie Spaces: Prefix + for _, genieSpace := range r.GenieSpaces { + if genieSpace == nil { + continue + } + genieSpace.Title = prefix + genieSpace.Title + } + // Apps: No presets // Alerts: Prefix, TriggerPauseStatus diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index f8063ddf346..440221001a7 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -154,6 +154,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, }, + GenieSpaces: map[string]*resources.GenieSpace{ + "geniespace1": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "geniespace1", + }, + }, + }, Apps: map[string]*resources.App{ "app1": { App: apps.App{ @@ -350,6 +357,9 @@ func TestProcessTargetModeDevelopment(t *testing.T) { // Dashboards assert.Equal(t, "[dev lennart] dashboard1", b.Config.Resources.Dashboards["dashboard1"].DisplayName) + // Genie Spaces + assert.Equal(t, "[dev lennart] geniespace1", b.Config.Resources.GenieSpaces["geniespace1"].Title) + // Alert 1: has schedule without pause status set - should be paused assert.Equal(t, "[dev lennart] alert1", b.Config.Resources.Alerts["alert1"].DisplayName) assert.Equal(t, sql.SchedulePauseStatusPaused, b.Config.Resources.Alerts["alert1"].Schedule.PauseStatus) diff --git a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go new file mode 100644 index 00000000000..bf129b4060d --- /dev/null +++ b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go @@ -0,0 +1,53 @@ +package resourcemutator + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +const serializedSpaceFieldName = "serialized_space" + +type configureGenieSpaceSerializedSpace struct{} + +func ConfigureGenieSpaceSerializedSpace() bundle.Mutator { + return &configureGenieSpaceSerializedSpace{} +} + +func (c configureGenieSpaceSerializedSpace) Name() string { + return "ConfigureGenieSpaceSerializedSpace" +} + +func (c configureGenieSpaceSerializedSpace) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + pattern := dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("genie_spaces"), + dyn.AnyKey(), + ) + + // Configure serialized_space field for all genie spaces. + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + // Include "serialized_space" field if "file_path" is set. + path, ok := v.Get(filePathFieldName).AsString() + if !ok { + return v, nil + } + + contents, err := b.SyncRoot.ReadFile(path) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to read serialized genie space from file_path %s: %w", path, err) + } + + return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(contents))) + }) + }) + + diags = diags.Extend(diag.FromErr(err)) + return diags +} diff --git a/bundle/config/mutator/resourcemutator/genie_space_fixups.go b/bundle/config/mutator/resourcemutator/genie_space_fixups.go new file mode 100644 index 00000000000..85e1bb7e745 --- /dev/null +++ b/bundle/config/mutator/resourcemutator/genie_space_fixups.go @@ -0,0 +1,30 @@ +package resourcemutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" +) + +type genieSpaceFixups struct{} + +func GenieSpaceFixups() bundle.Mutator { + return &genieSpaceFixups{} +} + +func (m *genieSpaceFixups) Name() string { + return "GenieSpaceFixups" +} + +func (m *genieSpaceFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + for _, genieSpace := range b.Config.Resources.GenieSpaces { + if genieSpace == nil { + continue + } + + genieSpace.ParentPath = ensureWorkspacePrefix(genieSpace.ParentPath) + } + + return nil +} diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 209bbcb06a0..45740f53599 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -52,6 +52,7 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { }{ {"resources.dashboards.*.parent_path", b.Config.Workspace.ResourcePath}, {"resources.dashboards.*.embed_credentials", false}, + {"resources.genie_spaces.*.parent_path", b.Config.Workspace.ResourcePath}, {"resources.volumes.*.volume_type", "MANAGED"}, {"resources.alerts.*.parent_path", b.Config.Workspace.ResourcePath}, @@ -115,6 +116,11 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { // Ensures dashboard parent paths have the required /Workspace prefix DashboardFixups(), + // Reads (typed): b.Config.Resources.GenieSpaces (checks genie space configurations) + // Updates (typed): b.Config.Resources.GenieSpaces[].ParentPath (ensures /Workspace prefix is present) + // Ensures genie space parent paths have the required /Workspace prefix + GenieSpaceFixups(), + // Reads (typed): b.Config.Permissions (validates permission levels) // Reads (dynamic): resources.{jobs,pipelines,experiments,models,model_serving_endpoints,dashboards,apps,vector_search_endpoints,...}.*.permissions (reads existing permissions) // Updates (dynamic): resources.{jobs,pipelines,experiments,models,model_serving_endpoints,dashboards,apps,vector_search_endpoints,...}.*.permissions (adds permissions from bundle-level configuration) @@ -182,6 +188,10 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) { // Drops (dynamic): resources.dashboards.*.file_path ConfigureDashboardSerializedDashboard(), + // Reads (dynamic): resources.genie_spaces.*.file_path + // Updates (dynamic): resources.genie_spaces.*.serialized_space + ConfigureGenieSpaceSerializedSpace(), + // Reads (typed): resources.alerts.*.file_path // Updates (typed): resources.alerts.* (loads alert configuration from .dbalert.json file) mutator.LoadDBAlertFiles(), diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 9ef2db077ac..af1470848d7 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -41,6 +41,7 @@ func allResourceTypes(t *testing.T) []string { "database_instances", "experiments", "external_locations", + "genie_spaces", "jobs", "model_serving_endpoints", "models", @@ -183,6 +184,7 @@ var allowList = []string{ "postgres_synced_tables", "registered_models", "experiments", + "genie_spaces", "schemas", "secret_scopes", "sql_warehouses", diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index a0bb76e2e6e..b36ec094447 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -368,6 +368,7 @@ func (m *translatePathsDashboards) Apply(ctx context.Context, b *bundle.Bundle) return applyTranslations(ctx, b, t, []func(context.Context, dyn.Value) (dyn.Value, error){ t.applyDashboardTranslations, + t.applyGenieSpaceTranslations, }) } diff --git a/bundle/config/mutator/translate_paths_genie_spaces.go b/bundle/config/mutator/translate_paths_genie_spaces.go new file mode 100644 index 00000000000..b6a3a652427 --- /dev/null +++ b/bundle/config/mutator/translate_paths_genie_spaces.go @@ -0,0 +1,22 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle/config/mutator/paths" + + "github.com/databricks/cli/libs/dyn" +) + +func (t *translateContext) applyGenieSpaceTranslations(ctx context.Context, v dyn.Value) (dyn.Value, error) { + // Convert the `file_path` field to a local absolute path. + // We load the file at this path and use its contents for the genie space contents. + + return paths.VisitGenieSpacePaths(v, func(p dyn.Path, mode paths.TranslateMode, v dyn.Value) (dyn.Value, error) { + opts := translateOptions{ + Mode: mode, + } + + return t.rewriteValue(ctx, p, v, t.b.BundleRootPath, opts) + }) +} diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 2e840a5ef80..3dc7dc295d3 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -25,6 +25,7 @@ type Resources struct { ExternalLocations map[string]*resources.ExternalLocation `json:"external_locations,omitempty"` Clusters map[string]*resources.Cluster `json:"clusters,omitempty"` Dashboards map[string]*resources.Dashboard `json:"dashboards,omitempty"` + GenieSpaces map[string]*resources.GenieSpace `json:"genie_spaces,omitempty"` Apps map[string]*resources.App `json:"apps,omitempty"` SecretScopes map[string]*resources.SecretScope `json:"secret_scopes,omitempty"` Alerts map[string]*resources.Alert `json:"alerts,omitempty"` @@ -104,6 +105,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["external_locations"], r.ExternalLocations), collectResourceMap(descriptions["clusters"], r.Clusters), collectResourceMap(descriptions["dashboards"], r.Dashboards), + collectResourceMap(descriptions["genie_spaces"], r.GenieSpaces), collectResourceMap(descriptions["volumes"], r.Volumes), collectResourceMap(descriptions["apps"], r.Apps), collectResourceMap(descriptions["alerts"], r.Alerts), @@ -162,6 +164,7 @@ func SupportedResources() map[string]resources.ResourceDescription { "external_locations": (&resources.ExternalLocation{}).ResourceDescription(), "clusters": (&resources.Cluster{}).ResourceDescription(), "dashboards": (&resources.Dashboard{}).ResourceDescription(), + "genie_spaces": (&resources.GenieSpace{}).ResourceDescription(), "volumes": (&resources.Volume{}).ResourceDescription(), "apps": (&resources.App{}).ResourceDescription(), "secret_scopes": (&resources.SecretScope{}).ResourceDescription(), diff --git a/bundle/config/resources/genie_space.go b/bundle/config/resources/genie_space.go new file mode 100644 index 00000000000..276ab12b17d --- /dev/null +++ b/bundle/config/resources/genie_space.go @@ -0,0 +1,94 @@ +package resources + +import ( + "context" + "net/url" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/workspaceurls" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/dashboards" +) + +type GenieSpaceConfig struct { + // Description of the Genie Space + Description string `json:"description,omitempty"` + // Genie space ID + SpaceId string `json:"space_id,omitempty"` + // Title of the Genie Space + Title string `json:"title,omitempty"` + // Warehouse associated with the Genie Space + WarehouseId string `json:"warehouse_id,omitempty"` + // Parent folder path where the space will be registered + ParentPath string `json:"parent_path,omitempty"` + + ForceSendFields []string `json:"-" url:"-"` + + // ============================================== + // === overrides over [dashboards.GenieSpace] === + // ============================================== + + // SerializedSpace holds the contents of the Genie Space in serialized JSON form. + // Even though the SDK represents this as a string, we override it as any to allow for inlining as YAML. + // If the value is a string, it is used as is. + // If it is not a string, its contents is marshalled as JSON. + SerializedSpace any `json:"serialized_space,omitempty"` +} + +func (c *GenieSpaceConfig) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c GenieSpaceConfig) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +type GenieSpace struct { + BaseResource + GenieSpaceConfig + + Permissions []Permission `json:"permissions,omitempty"` + + // FilePath points to the local `.genie.json` file containing the Genie Space definition. + // This is inlined into serialized_space during deployment. The file_path is kept around + // as metadata which is needed for `databricks bundle generate genie-space --resource ` to work. + // This is not part of GenieSpaceConfig because we don't need to store this in the resource state. + FilePath string `json:"file_path,omitempty"` +} + +func (*GenieSpace) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { + _, err := w.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: id, + }) + if err != nil { + log.Debugf(ctx, "genie space %s does not exist", id) + return false, err + } + return true, nil +} + +func (*GenieSpace) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "genie_space", + PluralName: "genie_spaces", + SingularTitle: "Genie Space", + PluralTitle: "Genie Spaces", + } +} + +func (r *GenieSpace) InitializeURL(baseURL url.URL) { + if r.ID == "" { + return + } + + r.URL = workspaceurls.ResourceURL(baseURL, "genie_spaces", r.ID) +} + +func (r *GenieSpace) GetName() string { + return r.Title +} + +func (r *GenieSpace) GetURL() string { + return r.URL +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 4f51476536f..8e610ba9418 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -201,6 +201,9 @@ func TestResourcesBindSupport(t *testing.T) { Dashboards: map[string]*resources.Dashboard{ "my_dashboard": {}, }, + GenieSpaces: map[string]*resources.GenieSpace{ + "my_genie_space": {}, + }, Volumes: map[string]*resources.Volume{ "my_volume": { CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{}, @@ -328,6 +331,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockSchemasAPI().EXPECT().GetByFullName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockClustersAPI().EXPECT().GetByClusterId(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockLakeviewAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockGenieAPI().EXPECT().GetSpace(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockVolumesAPI().EXPECT().Read(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockAppsAPI().EXPECT().GetByName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockAlertsV2API().EXPECT().GetAlertById(mock.Anything, mock.Anything).Return(nil, nil) diff --git a/bundle/deploy/terraform/lifecycle_test.go b/bundle/deploy/terraform/lifecycle_test.go index 2986a9cc8de..b60bff612c7 100644 --- a/bundle/deploy/terraform/lifecycle_test.go +++ b/bundle/deploy/terraform/lifecycle_test.go @@ -17,6 +17,7 @@ func TestConvertLifecycleForAllResources(t *testing.T) { ignoredResources := []string{ "catalogs", "external_locations", + "genie_spaces", "vector_search_endpoints", "vector_search_indexes", } diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index a263a25f127..6cc1eb55437 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -29,6 +29,7 @@ var SupportedResources = map[string]any{ "clusters": (*ResourceCluster)(nil), "registered_models": (*ResourceRegisteredModel)(nil), "dashboards": (*ResourceDashboard)(nil), + "genie_spaces": (*ResourceGenieSpace)(nil), "secret_scopes": (*ResourceSecretScope)(nil), "model_serving_endpoints": (*ResourceModelServingEndpoint)(nil), "quality_monitors": (*ResourceQualityMonitor)(nil), @@ -49,6 +50,7 @@ var SupportedResources = map[string]any{ "secret_scopes.permissions": (*ResourceSecretScopeAcls)(nil), "model_serving_endpoints.permissions": (*ResourcePermissions)(nil), "dashboards.permissions": (*ResourcePermissions)(nil), + "genie_spaces.permissions": (*ResourcePermissions)(nil), "vector_search_endpoints.permissions": (*ResourcePermissions)(nil), // Grants diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index f18a84d0efc..b57fc4c5610 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -495,6 +495,25 @@ var testDeps = map[string]prepareWorkspace{ }, nil }, + "genie_spaces.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { + resp, err := client.Genie.CreateSpace(ctx, dashboards.GenieCreateSpaceRequest{ + Title: "genie-space-permissions", + WarehouseId: "test-warehouse-id", + SerializedSpace: "{}", + }) + if err != nil { + return nil, err + } + + return &PermissionsState{ + ObjectID: "/genie/spaces/" + resp.SpaceId, + EmbeddedSlice: []StatePermission{{ + Level: "CAN_MANAGE", + UserName: "user@example.com", + }}, + }, nil + }, + "model_serving_endpoints.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { waiter, err := client.ServingEndpoints.Create(ctx, serving.CreateServingEndpoint{ Name: "endpoint-permissions", diff --git a/bundle/direct/dresources/apitypes.yml b/bundle/direct/dresources/apitypes.yml index 7d478be47f7..11cba516a4d 100644 --- a/bundle/direct/dresources/apitypes.yml +++ b/bundle/direct/dresources/apitypes.yml @@ -4,6 +4,8 @@ # Set a value to null to remove a type: # jobs: null +genie_spaces: dashboards.GenieSpace + postgres_branches: postgres.BranchSpec postgres_endpoints: postgres.EndpointSpec diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go new file mode 100644 index 00000000000..490849cfbdf --- /dev/null +++ b/bundle/direct/dresources/genie_space.go @@ -0,0 +1,164 @@ +package dresources + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/utils" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/dashboards" +) + +type ResourceGenieSpace struct { + client *databricks.WorkspaceClient +} + +func (*ResourceGenieSpace) New(client *databricks.WorkspaceClient) *ResourceGenieSpace { + return &ResourceGenieSpace{client: client} +} + +func (*ResourceGenieSpace) PrepareState(input *resources.GenieSpace) *resources.GenieSpaceConfig { + return &input.GenieSpaceConfig +} + +func (r *ResourceGenieSpace) RemapState(state *resources.GenieSpaceConfig) *resources.GenieSpaceConfig { + forceSendFields := utils.FilterFields[resources.GenieSpaceConfig](state.ForceSendFields, []string{ + "SpaceId", + "SerializedSpace", + }...) + + return &resources.GenieSpaceConfig{ + Description: state.Description, + Title: state.Title, + WarehouseId: state.WarehouseId, + ParentPath: state.ParentPath, + SerializedSpace: state.SerializedSpace, + + ForceSendFields: forceSendFields, + + // Clear output only fields. They should not show up on remote diff computation. + SpaceId: "", + } +} + +func (r *ResourceGenieSpace) DoRead(ctx context.Context, id string) (*resources.GenieSpaceConfig, error) { + space, err := r.client.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: id, + IncludeSerializedSpace: true, + ForceSendFields: nil, + }) + if err != nil { + return nil, err + } + + forceSendFields := utils.FilterFields[resources.GenieSpaceConfig](space.ForceSendFields) + + return &resources.GenieSpaceConfig{ + Description: space.Description, + Title: space.Title, + WarehouseId: space.WarehouseId, + ParentPath: "", + SerializedSpace: space.SerializedSpace, + + // Output only fields + SpaceId: space.SpaceId, + ForceSendFields: forceSendFields, + }, nil +} + +func prepareGenieSpaceRequest(config *resources.GenieSpaceConfig) (string, error) { + v := config.SerializedSpace + if serializedSpace, ok := v.(string); ok { + return serializedSpace, nil + } else if v != nil { + b, err := json.Marshal(v) + if err != nil { + return "", fmt.Errorf("failed to marshal serialized_space: %w", err) + } + return string(b), nil + } + return "", nil +} + +func responseToGenieSpaceConfig(space *dashboards.GenieSpace, serializedSpace string) *resources.GenieSpaceConfig { + forceSendFields := utils.FilterFields[resources.GenieSpaceConfig](space.ForceSendFields) + + return &resources.GenieSpaceConfig{ + Description: space.Description, + Title: space.Title, + WarehouseId: space.WarehouseId, + ParentPath: "", + SerializedSpace: serializedSpace, + + // Output only fields + SpaceId: space.SpaceId, + ForceSendFields: forceSendFields, + } +} + +func (r *ResourceGenieSpace) DoCreate(ctx context.Context, config *resources.GenieSpaceConfig) (string, *resources.GenieSpaceConfig, error) { + serializedSpace, err := prepareGenieSpaceRequest(config) + if err != nil { + return "", nil, err + } + + req := dashboards.GenieCreateSpaceRequest{ + Description: config.Description, + Title: config.Title, + WarehouseId: config.WarehouseId, + ParentPath: config.ParentPath, + SerializedSpace: serializedSpace, + + ForceSendFields: utils.FilterFields[dashboards.GenieCreateSpaceRequest](config.ForceSendFields), + } + + createResp, err := r.client.Genie.CreateSpace(ctx, req) + + // The API returns 404 if the parent directory doesn't exist. + // Create it and retry once. + if err != nil && apierr.IsMissing(err) { + err = r.client.Workspace.MkdirsByPath(ctx, config.ParentPath) //nolint:staticcheck // Deprecated in SDK v0.127.0. Migration to WorkspaceHierarchyService tracked separately. + if err != nil { + return "", nil, fmt.Errorf("failed to create parent directory: %w", err) + } + createResp, err = r.client.Genie.CreateSpace(ctx, req) + } + if err != nil { + return "", nil, err + } + + return createResp.SpaceId, responseToGenieSpaceConfig(createResp, serializedSpace), nil +} + +func (r *ResourceGenieSpace) DoUpdate(ctx context.Context, id string, config *resources.GenieSpaceConfig, _ *PlanEntry) (*resources.GenieSpaceConfig, error) { + serializedSpace, err := prepareGenieSpaceRequest(config) + if err != nil { + return nil, err + } + + updateResp, err := r.client.Genie.UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ + SpaceId: id, + Description: config.Description, + Title: config.Title, + WarehouseId: config.WarehouseId, + SerializedSpace: serializedSpace, + // Etag is for optimistic concurrency; we apply updates unconditionally. + Etag: "", + + ForceSendFields: utils.FilterFields[dashboards.GenieUpdateSpaceRequest](config.ForceSendFields), + }) + if err != nil { + return nil, err + } + + return responseToGenieSpaceConfig(updateResp, serializedSpace), nil +} + +func (r *ResourceGenieSpace) DoDelete(ctx context.Context, id string) error { + return r.client.Genie.TrashSpace(ctx, dashboards.GenieTrashSpaceRequest{ + SpaceId: id, + }) +} diff --git a/bundle/direct/dresources/permissions.go b/bundle/direct/dresources/permissions.go index 91ca9000aaf..9c1b26de750 100644 --- a/bundle/direct/dresources/permissions.go +++ b/bundle/direct/dresources/permissions.go @@ -18,6 +18,7 @@ var permissionResourceToObjectType = map[string]string{ "apps": "/apps/", "clusters": "/clusters/", "dashboards": "/dashboards/", + "genie_spaces": "/genie/spaces/", "database_instances": "/database-instances/", "postgres_projects": "/database-projects/", "jobs": "/jobs/", diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 8da2d5fee50..fb4164c545b 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -359,6 +359,19 @@ resources: - field: dataset_schema reason: input_only + genie_spaces: + recreate_on_changes: + - field: parent_path + reason: immutable + ignore_remote_changes: + # serialized_space locally (structured YAML) and remotely (JSON string) will differ + # textually, so we cannot meaningfully compare them for drift. + - field: serialized_space + reason: input_only + # parent_path is not reliably returned by the GET Genie space API. + - field: parent_path + reason: input_only + apps: recreate_on_changes: - field: name diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index c9fefb7c1f2..fee09ed119f 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -85,6 +85,9 @@ var knownMissingInStateType = map[string][]string{ "dashboards": { "file_path", }, + "genie_spaces": { + "file_path", + }, "secret_scopes": { "backend_type", "keyvault_metadata", diff --git a/bundle/docsgen/output/reference.md b/bundle/docsgen/output/reference.md index 16b7020cecd..5f8df77dfe9 100644 --- a/bundle/docsgen/output/reference.md +++ b/bundle/docsgen/output/reference.md @@ -1,7 +1,7 @@ --- description: 'Configuration reference for databricks.yml' last_update: - date: 2026-06-03 + date: 2026-05-11 --- @@ -242,10 +242,6 @@ Defines attributes for experimental features. - Boolean - Whether to use a Python wheel wrapper. -- - `record_deployment_history` - - Boolean - - Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments. - - - `scripts` - Map - The commands to run. @@ -479,6 +475,10 @@ resources: - Map - See [\_](#resourcesexternal_locations). +- - `genie_spaces` + - Map + - See [\_](#resourcesgenie_spaces). + - - `jobs` - Map - The job definitions for the bundle, where each key is the name of the job. See [\_](/dev-tools/bundles/resources.md#jobs). @@ -493,16 +493,12 @@ resources: - - `pipelines` - Map - - The pipeline definitions for the bundle, where each key is the name of the pipeline. See [\_](/dev-tools/bundles/resources.md#pipelines). See [\_](#resourcespipelines). + - The pipeline definitions for the bundle, where each key is the name of the pipeline. See [\_](/dev-tools/bundles/resources.md#pipelines). - - `postgres_branches` - Map - See [\_](#resourcespostgres_branches). -- - `postgres_catalogs` - - Map - - The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch. See [\_](#resourcespostgres_catalogs). - - - `postgres_endpoints` - Map - See [\_](#resourcespostgres_endpoints). @@ -511,10 +507,6 @@ resources: - Map - See [\_](#resourcespostgres_projects). -- - `postgres_synced_tables` - - Map - - The Postgres synced table definitions for the bundle, where each key is the name of the synced table. Each entry continuously replicates a Unity Catalog Delta source table into a Postgres table on a Lakebase Autoscaling instance. See [\_](#resourcespostgres_synced_tables). - - - `quality_monitors` - Map - The quality monitor definitions for the bundle, where each key is the name of the quality monitor. See [\_](/dev-tools/bundles/resources.md#quality_monitors). @@ -543,10 +535,6 @@ resources: - Map - See [\_](#resourcesvector_search_endpoints). -- - `vector_search_indexes` - - Map - - See [\_](#resourcesvector_search_indexes). - - - `volumes` - Map - The volume definitions for the bundle, where each key is the name of the volume. See [\_](/dev-tools/bundles/resources.md#volumes). @@ -737,14 +725,6 @@ apps: - String - -- - `compute_max_instances` - - Integer - - - -- - `compute_min_instances` - - Integer - - - - - `compute_size` - String - @@ -1502,16 +1482,16 @@ external_locations: ::: -### resources.pipelines +### resources.genie_spaces **`Type: Map`** -The pipeline definitions for the bundle, where each key is the name of the pipeline. See [\_](/dev-tools/bundles/resources.md#pipelines). + ```yaml -pipelines: - : - : +genie_spaces: + : + : ``` @@ -1521,146 +1501,46 @@ pipelines: - Type - Description -- - `allow_duplicate_names` - - Boolean - - - -- - `budget_policy_id` - - String - - - -- - `catalog` - - String - - - -- - `channel` - - String - - - -- - `clusters` - - Sequence - - See [\_](#resourcespipelinesnameclusters). - -- - `configuration` - - Map - - - -- - `continuous` - - Boolean - - - -- - `deployment` - - Map - - See [\_](#resourcespipelinesnamedeployment). - -- - `development` - - Boolean - - - -- - `dry_run` - - Boolean - - - -- - `edition` +- - `description` - String - -- - `environment` - - Map - - See [\_](#resourcespipelinesnameenvironment). - -- - `event_log` - - Map - - See [\_](#resourcespipelinesnameevent_log). - -- - `filters` - - Map - - See [\_](#resourcespipelinesnamefilters). - -- - `gateway_definition` - - Map - - See [\_](#resourcespipelinesnamegateway_definition). - -- - `id` +- - `file_path` - String - -- - `ingestion_definition` - - Map - - See [\_](#resourcespipelinesnameingestion_definition). - -- - `libraries` - - Sequence - - See [\_](#resourcespipelinesnamelibraries). - - - `lifecycle` - Map - - See [\_](#resourcespipelinesnamelifecycle). + - See [\_](#resourcesgenie_spacesnamelifecycle). -- - `name` +- - `parent_path` - String - -- - `notifications` - - Sequence - - See [\_](#resourcespipelinesnamenotifications). - -- - `parameters` - - Map - - - - - `permissions` - Sequence - - See [\_](#resourcespipelinesnamepermissions). - -- - `photon` - - Boolean - - - -- - `restart_window` - - Map - - See [\_](#resourcespipelinesnamerestart_window). - -- - `root_path` - - String - - - -- - `run_as` - - Map - - See [\_](#resourcespipelinesnamerun_as). - -- - `schema` - - String - - + - See [\_](#resourcesgenie_spacesnamepermissions). -- - `serverless` - - Boolean +- - `serialized_space` + - Any - -- - `storage` +- - `space_id` - String - -- - `tags` - - Map - - - -- - `target` +- - `title` - String - -- - `trigger` - - Map - - See [\_](#resourcespipelinesnametrigger). - -- - `usage_policy_id` +- - `warehouse_id` - String - ::: -### resources.pipelines._name_.lifecycle +### resources.genie_spaces._name_.lifecycle **`Type: Map`** @@ -1681,7 +1561,7 @@ pipelines: ::: -### resources.pipelines._name_.permissions +### resources.genie_spaces._name_.permissions **`Type: Sequence`** @@ -1697,19 +1577,19 @@ pipelines: - - `group_name` - String - - + - The name of the group that has the permission set in level. - - `level` - String - - + - The allowed permission for user, group, service principal defined for this permission. - - `service_principal_name` - String - - + - The name of the service principal that has the permission set in level. - - `user_name` - String - - + - The name of the user that has the permission set in level. ::: @@ -1757,10 +1637,6 @@ postgres_branches: - String - -- - `replace_existing` - - Boolean - - - - - `source_branch` - String - @@ -1788,69 +1664,6 @@ postgres_branches: -:::list-table - -- - Key - - Type - - Description - -- - `prevent_destroy` - - Boolean - - Lifecycle setting to prevent the resource from being destroyed. - -::: - - -### resources.postgres_catalogs - -**`Type: Map`** - -The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch. - -```yaml -postgres_catalogs: - : - : -``` - - -:::list-table - -- - Key - - Type - - Description - -- - `branch` - - String - - - -- - `catalog_id` - - String - - - -- - `create_database_if_missing` - - Boolean - - - -- - `lifecycle` - - Map - - See [\_](#resourcespostgres_catalogsnamelifecycle). - -- - `postgres_database` - - String - - - -::: - - -### resources.postgres_catalogs._name_.lifecycle - -**`Type: Map`** - - - - - :::list-table - - Key @@ -1919,10 +1732,6 @@ postgres_endpoints: - String - -- - `replace_existing` - - Boolean - - - - - `settings` - Map - See [\_](#resourcespostgres_endpointsnamesettings). @@ -2075,93 +1884,6 @@ postgres_projects: ::: -### resources.postgres_synced_tables - -**`Type: Map`** - -The Postgres synced table definitions for the bundle, where each key is the name of the synced table. Each entry continuously replicates a Unity Catalog Delta source table into a Postgres table on a Lakebase Autoscaling instance. - -```yaml -postgres_synced_tables: - : - : -``` - - -:::list-table - -- - Key - - Type - - Description - -- - `branch` - - String - - - -- - `create_database_objects_if_missing` - - Boolean - - - -- - `existing_pipeline_id` - - String - - - -- - `lifecycle` - - Map - - See [\_](#resourcespostgres_synced_tablesnamelifecycle). - -- - `new_pipeline_spec` - - Map - - See [\_](#resourcespostgres_synced_tablesnamenew_pipeline_spec). - -- - `postgres_database` - - String - - - -- - `primary_key_columns` - - Sequence - - - -- - `scheduling_policy` - - String - - - -- - `source_table_full_name` - - String - - - -- - `synced_table_id` - - String - - - -- - `timeseries_key` - - String - - - -::: - - -### resources.postgres_synced_tables._name_.lifecycle - -**`Type: Map`** - - - - - -:::list-table - -- - Key - - Type - - Description - -- - `prevent_destroy` - - Boolean - - Lifecycle setting to prevent the resource from being destroyed. - -::: - - ### resources.secret_scopes **`Type: Map`** @@ -2441,17 +2163,12 @@ vector_search_endpoints: ::: -### resources.vector_search_indexes +## run_as **`Type: Map`** - +The identity to use when running Declarative Automation Bundles resources. See [\_](/dev-tools/bundles/run-as.md). -```yaml -vector_search_indexes: - : - : -``` :::list-table @@ -2460,93 +2177,19 @@ vector_search_indexes: - Type - Description -- - `delta_sync_index_spec` - - Map - - See [\_](#resourcesvector_search_indexesnamedelta_sync_index_spec). - -- - `direct_access_index_spec` - - Map - - See [\_](#resourcesvector_search_indexesnamedirect_access_index_spec). - -- - `endpoint_name` +- - `group_name` - String - -- - `grants` - - Sequence - - See [\_](#resourcesvector_search_indexesnamegrants). - -- - `index_subtype` +- - `service_principal_name` - String - - + - The application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role. -- - `index_type` +- - `user_name` - String - - + - The email of an active workspace user. Non-admin users can only set this field to their own email. -- - `lifecycle` - - Map - - See [\_](#resourcesvector_search_indexesnamelifecycle). - -- - `name` - - String - - - -- - `primary_key` - - String - - - -::: - - -### resources.vector_search_indexes._name_.lifecycle - -**`Type: Map`** - - - - - -:::list-table - -- - Key - - Type - - Description - -- - `prevent_destroy` - - Boolean - - Lifecycle setting to prevent the resource from being destroyed. - -::: - - -## run_as - -**`Type: Map`** - -The identity to use when running Declarative Automation Bundles resources. See [\_](/dev-tools/bundles/run-as.md). - - - -:::list-table - -- - Key - - Type - - Description - -- - `group_name` - - String - - - -- - `service_principal_name` - - String - - The application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role. - -- - `user_name` - - String - - The email of an active workspace user. Non-admin users can only set this field to their own email. - -::: +::: ## scripts @@ -3026,6 +2669,10 @@ The resource definitions for the target. - Map - See [\_](#targetsnameresourcesexternal_locations). +- - `genie_spaces` + - Map + - See [\_](#targetsnameresourcesgenie_spaces). + - - `jobs` - Map - The job definitions for the bundle, where each key is the name of the job. See [\_](/dev-tools/bundles/resources.md#jobs). @@ -3040,16 +2687,12 @@ The resource definitions for the target. - - `pipelines` - Map - - The pipeline definitions for the bundle, where each key is the name of the pipeline. See [\_](/dev-tools/bundles/resources.md#pipelines). See [\_](#targetsnameresourcespipelines). + - The pipeline definitions for the bundle, where each key is the name of the pipeline. See [\_](/dev-tools/bundles/resources.md#pipelines). - - `postgres_branches` - Map - See [\_](#targetsnameresourcespostgres_branches). -- - `postgres_catalogs` - - Map - - The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch. See [\_](#targetsnameresourcespostgres_catalogs). - - - `postgres_endpoints` - Map - See [\_](#targetsnameresourcespostgres_endpoints). @@ -3058,10 +2701,6 @@ The resource definitions for the target. - Map - See [\_](#targetsnameresourcespostgres_projects). -- - `postgres_synced_tables` - - Map - - The Postgres synced table definitions for the bundle, where each key is the name of the synced table. Each entry continuously replicates a Unity Catalog Delta source table into a Postgres table on a Lakebase Autoscaling instance. See [\_](#targetsnameresourcespostgres_synced_tables). - - - `quality_monitors` - Map - The quality monitor definitions for the bundle, where each key is the name of the quality monitor. See [\_](/dev-tools/bundles/resources.md#quality_monitors). @@ -3090,10 +2729,6 @@ The resource definitions for the target. - Map - See [\_](#targetsnameresourcesvector_search_endpoints). -- - `vector_search_indexes` - - Map - - See [\_](#targetsnameresourcesvector_search_indexes). - - - `volumes` - Map - The volume definitions for the bundle, where each key is the name of the volume. See [\_](/dev-tools/bundles/resources.md#volumes). @@ -3284,14 +2919,6 @@ apps: - String - -- - `compute_max_instances` - - Integer - - - -- - `compute_min_instances` - - Integer - - - - - `compute_size` - String - @@ -4049,16 +3676,16 @@ external_locations: ::: -### targets._name_.resources.pipelines +### targets._name_.resources.genie_spaces **`Type: Map`** -The pipeline definitions for the bundle, where each key is the name of the pipeline. See [\_](/dev-tools/bundles/resources.md#pipelines). + ```yaml -pipelines: - : - : +genie_spaces: + : + : ``` @@ -4068,146 +3695,46 @@ pipelines: - Type - Description -- - `allow_duplicate_names` - - Boolean - - - -- - `budget_policy_id` - - String - - - -- - `catalog` - - String - - - -- - `channel` - - String - - - -- - `clusters` - - Sequence - - See [\_](#targetsnameresourcespipelinesnameclusters). - -- - `configuration` - - Map - - - -- - `continuous` - - Boolean - - - -- - `deployment` - - Map - - See [\_](#targetsnameresourcespipelinesnamedeployment). - -- - `development` - - Boolean - - - -- - `dry_run` - - Boolean - - - -- - `edition` +- - `description` - String - -- - `environment` - - Map - - See [\_](#targetsnameresourcespipelinesnameenvironment). - -- - `event_log` - - Map - - See [\_](#targetsnameresourcespipelinesnameevent_log). - -- - `filters` - - Map - - See [\_](#targetsnameresourcespipelinesnamefilters). - -- - `gateway_definition` - - Map - - See [\_](#targetsnameresourcespipelinesnamegateway_definition). - -- - `id` +- - `file_path` - String - -- - `ingestion_definition` - - Map - - See [\_](#targetsnameresourcespipelinesnameingestion_definition). - -- - `libraries` - - Sequence - - See [\_](#targetsnameresourcespipelinesnamelibraries). - - - `lifecycle` - Map - - See [\_](#targetsnameresourcespipelinesnamelifecycle). + - See [\_](#targetsnameresourcesgenie_spacesnamelifecycle). -- - `name` +- - `parent_path` - String - -- - `notifications` - - Sequence - - See [\_](#targetsnameresourcespipelinesnamenotifications). - -- - `parameters` - - Map - - - - - `permissions` - Sequence - - See [\_](#targetsnameresourcespipelinesnamepermissions). - -- - `photon` - - Boolean - - + - See [\_](#targetsnameresourcesgenie_spacesnamepermissions). -- - `restart_window` - - Map - - See [\_](#targetsnameresourcespipelinesnamerestart_window). - -- - `root_path` - - String - - - -- - `run_as` - - Map - - See [\_](#targetsnameresourcespipelinesnamerun_as). - -- - `schema` - - String - - - -- - `serverless` - - Boolean +- - `serialized_space` + - Any - -- - `storage` +- - `space_id` - String - -- - `tags` - - Map - - - -- - `target` +- - `title` - String - -- - `trigger` - - Map - - See [\_](#targetsnameresourcespipelinesnametrigger). - -- - `usage_policy_id` +- - `warehouse_id` - String - ::: -### targets._name_.resources.pipelines._name_.lifecycle +### targets._name_.resources.genie_spaces._name_.lifecycle **`Type: Map`** @@ -4228,7 +3755,7 @@ pipelines: ::: -### targets._name_.resources.pipelines._name_.permissions +### targets._name_.resources.genie_spaces._name_.permissions **`Type: Sequence`** @@ -4244,19 +3771,19 @@ pipelines: - - `group_name` - String - - + - The name of the group that has the permission set in level. - - `level` - String - - + - The allowed permission for user, group, service principal defined for this permission. - - `service_principal_name` - String - - + - The name of the service principal that has the permission set in level. - - `user_name` - String - - + - The name of the user that has the permission set in level. ::: @@ -4304,10 +3831,6 @@ postgres_branches: - String - -- - `replace_existing` - - Boolean - - - - - `source_branch` - String - @@ -4335,69 +3858,6 @@ postgres_branches: -:::list-table - -- - Key - - Type - - Description - -- - `prevent_destroy` - - Boolean - - Lifecycle setting to prevent the resource from being destroyed. - -::: - - -### targets._name_.resources.postgres_catalogs - -**`Type: Map`** - -The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch. - -```yaml -postgres_catalogs: - : - : -``` - - -:::list-table - -- - Key - - Type - - Description - -- - `branch` - - String - - - -- - `catalog_id` - - String - - - -- - `create_database_if_missing` - - Boolean - - - -- - `lifecycle` - - Map - - See [\_](#targetsnameresourcespostgres_catalogsnamelifecycle). - -- - `postgres_database` - - String - - - -::: - - -### targets._name_.resources.postgres_catalogs._name_.lifecycle - -**`Type: Map`** - - - - - :::list-table - - Key @@ -4466,10 +3926,6 @@ postgres_endpoints: - String - -- - `replace_existing` - - Boolean - - - - - `settings` - Map - See [\_](#targetsnameresourcespostgres_endpointsnamesettings). @@ -4622,93 +4078,6 @@ postgres_projects: ::: -### targets._name_.resources.postgres_synced_tables - -**`Type: Map`** - -The Postgres synced table definitions for the bundle, where each key is the name of the synced table. Each entry continuously replicates a Unity Catalog Delta source table into a Postgres table on a Lakebase Autoscaling instance. - -```yaml -postgres_synced_tables: - : - : -``` - - -:::list-table - -- - Key - - Type - - Description - -- - `branch` - - String - - - -- - `create_database_objects_if_missing` - - Boolean - - - -- - `existing_pipeline_id` - - String - - - -- - `lifecycle` - - Map - - See [\_](#targetsnameresourcespostgres_synced_tablesnamelifecycle). - -- - `new_pipeline_spec` - - Map - - See [\_](#targetsnameresourcespostgres_synced_tablesnamenew_pipeline_spec). - -- - `postgres_database` - - String - - - -- - `primary_key_columns` - - Sequence - - - -- - `scheduling_policy` - - String - - - -- - `source_table_full_name` - - String - - - -- - `synced_table_id` - - String - - - -- - `timeseries_key` - - String - - - -::: - - -### targets._name_.resources.postgres_synced_tables._name_.lifecycle - -**`Type: Map`** - - - - - -:::list-table - -- - Key - - Type - - Description - -- - `prevent_destroy` - - Boolean - - Lifecycle setting to prevent the resource from being destroyed. - -::: - - ### targets._name_.resources.secret_scopes **`Type: Map`** @@ -4988,85 +4357,6 @@ vector_search_endpoints: ::: -### targets._name_.resources.vector_search_indexes - -**`Type: Map`** - - - -```yaml -vector_search_indexes: - : - : -``` - - -:::list-table - -- - Key - - Type - - Description - -- - `delta_sync_index_spec` - - Map - - See [\_](#targetsnameresourcesvector_search_indexesnamedelta_sync_index_spec). - -- - `direct_access_index_spec` - - Map - - See [\_](#targetsnameresourcesvector_search_indexesnamedirect_access_index_spec). - -- - `endpoint_name` - - String - - - -- - `grants` - - Sequence - - See [\_](#targetsnameresourcesvector_search_indexesnamegrants). - -- - `index_subtype` - - String - - - -- - `index_type` - - String - - - -- - `lifecycle` - - Map - - See [\_](#targetsnameresourcesvector_search_indexesnamelifecycle). - -- - `name` - - String - - - -- - `primary_key` - - String - - - -::: - - -### targets._name_.resources.vector_search_indexes._name_.lifecycle - -**`Type: Map`** - - - - - -:::list-table - -- - Key - - Type - - Description - -- - `prevent_destroy` - - Boolean - - Lifecycle setting to prevent the resource from being destroyed. - -::: - - ### targets._name_.run_as **`Type: Map`** diff --git a/bundle/docsgen/output/resources.md b/bundle/docsgen/output/resources.md index e4f902e6146..95ece375b97 100644 --- a/bundle/docsgen/output/resources.md +++ b/bundle/docsgen/output/resources.md @@ -3086,6 +3086,118 @@ The privileges assigned to the principal. ::: +## genie_spaces + +**`Type: Map`** + + + +```yaml +genie_spaces: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `description` + - String + - + +- - `file_path` + - String + - + +- - `lifecycle` + - Map + - See [\_](#genie_spacesnamelifecycle). + +- - `parent_path` + - String + - + +- - `permissions` + - Sequence + - See [\_](#genie_spacesnamepermissions). + +- - `serialized_space` + - Any + - + +- - `space_id` + - String + - + +- - `title` + - String + - + +- - `warehouse_id` + - String + - + +::: + + +### genie_spaces._name_.lifecycle + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + +### genie_spaces._name_.permissions + +**`Type: Sequence`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `group_name` + - String + - The name of the group that has the permission set in level. + +- - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + +- - `service_principal_name` + - String + - The name of the service principal that has the permission set in level. + +- - `user_name` + - String + - The name of the user that has the permission set in level. + +::: + + ## jobs **`Type: Map`** diff --git a/bundle/generate/genie_space.go b/bundle/generate/genie_space.go new file mode 100644 index 00000000000..ebb1098b2b1 --- /dev/null +++ b/bundle/generate/genie_space.go @@ -0,0 +1,22 @@ +package generate + +import ( + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/databricks-sdk-go/service/dashboards" +) + +func ConvertGenieSpaceToValue(genieSpace *dashboards.GenieSpace, filePath string) (dyn.Value, error) { + // The majority of fields of the genie space struct are read-only. + // We copy the relevant fields manually. + dv := map[string]dyn.Value{ + "title": dyn.NewValue(genieSpace.Title, []dyn.Location{{Line: 1}}), + "warehouse_id": dyn.NewValue(genieSpace.WarehouseId, []dyn.Location{{Line: 2}}), + "file_path": dyn.NewValue(filePath, []dyn.Location{{Line: 3}}), + } + + if genieSpace.Description != "" { + dv["description"] = dyn.NewValue(genieSpace.Description, []dyn.Location{{Line: 4}}) + } + + return dyn.V(dv), nil +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index fd5f4275d12..fd9e3b6e78f 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -195,6 +195,9 @@ github.com/databricks/cli/bundle/config.Resources: "external_locations": "description": |- PLACEHOLDER + "genie_spaces": + "description": |- + PLACEHOLDER "jobs": "description": |- The job definitions for the bundle, where each key is the name of the job. @@ -668,6 +671,34 @@ github.com/databricks/cli/bundle/config/resources.ExternalLocation: "url": "description": |- PLACEHOLDER +github.com/databricks/cli/bundle/config/resources.GenieSpace: + "description": + "description": |- + PLACEHOLDER + "file_path": + "description": |- + PLACEHOLDER + "lifecycle": + "description": |- + PLACEHOLDER + "parent_path": + "description": |- + PLACEHOLDER + "permissions": + "description": |- + PLACEHOLDER + "serialized_space": + "description": |- + PLACEHOLDER + "space_id": + "description": |- + PLACEHOLDER + "title": + "description": |- + PLACEHOLDER + "warehouse_id": + "description": |- + PLACEHOLDER github.com/databricks/cli/bundle/config/resources.JobPermission: "group_name": "description": |- diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 476eaaf3637..9c229c20a2d 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -760,6 +760,47 @@ } ] }, + "resources.GenieSpace": { + "oneOf": [ + { + "type": "object", + "properties": { + "description": { + "$ref": "#/$defs/string" + }, + "file_path": { + "$ref": "#/$defs/string" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "parent_path": { + "$ref": "#/$defs/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "serialized_space": { + "$ref": "#/$defs/interface" + }, + "space_id": { + "$ref": "#/$defs/string" + }, + "title": { + "$ref": "#/$defs/string" + }, + "warehouse_id": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Job": { "oneOf": [ { @@ -2655,6 +2696,9 @@ "external_locations": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.ExternalLocation" }, + "genie_spaces": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.GenieSpace" + }, "jobs": { "description": "The job definitions for the bundle, where each key is the name of the job.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Job", @@ -12206,6 +12250,20 @@ } ] }, + "resources.GenieSpace": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.GenieSpace" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Job": { "oneOf": [ { diff --git a/bundle/schema/jsonschema_for_docs.json b/bundle/schema/jsonschema_for_docs.json index 198d9191196..04e3930f36c 100644 --- a/bundle/schema/jsonschema_for_docs.json +++ b/bundle/schema/jsonschema_for_docs.json @@ -741,6 +741,39 @@ "url" ] }, + "resources.GenieSpace": { + "type": "object", + "properties": { + "description": { + "$ref": "#/$defs/string" + }, + "file_path": { + "$ref": "#/$defs/string" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "parent_path": { + "$ref": "#/$defs/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "serialized_space": { + "$ref": "#/$defs/interface" + }, + "space_id": { + "$ref": "#/$defs/string" + }, + "title": { + "$ref": "#/$defs/string" + }, + "warehouse_id": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, "resources.Job": { "type": "object", "properties": { @@ -2615,6 +2648,9 @@ "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.ExternalLocation", "x-since-version": "v0.289.0" }, + "genie_spaces": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.GenieSpace" + }, "jobs": { "description": "The job definitions for the bundle, where each key is the name of the job.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Job", @@ -10211,6 +10247,12 @@ "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.ExternalLocation" } }, + "resources.GenieSpace": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.GenieSpace" + } + }, "resources.Job": { "type": "object", "additionalProperties": { diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 57f31450d93..203635b5b7a 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -39,6 +39,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.volumes.test_volume": {ID: "1"}, "resources.clusters.test_cluster": {ID: "1"}, "resources.dashboards.test_dashboard": {ID: "1"}, + "resources.genie_spaces.test_genie_space": {ID: "1"}, "resources.apps.test_app": {ID: "app1"}, "resources.secret_scopes.test_secret_scope": {ID: "secret_scope1"}, "resources.sql_warehouses.test_sql_warehouse": {ID: "1"}, @@ -96,6 +97,9 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "1", config.Resources.Dashboards["test_dashboard"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Dashboards["test_dashboard"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.GenieSpaces["test_genie_space"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.GenieSpaces["test_genie_space"].ModifiedStatus) + assert.Equal(t, "app1", config.Resources.Apps["test_app"].ID) assert.Empty(t, config.Resources.Apps["test_app"].Name) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Apps["test_app"].ModifiedStatus) @@ -228,6 +232,13 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + GenieSpaces: map[string]*resources.GenieSpace{ + "test_genie_space": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "test_genie_space", + }, + }, + }, Apps: map[string]*resources.App{ "test_app": { App: apps.App{ @@ -374,6 +385,9 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { assert.Empty(t, config.Resources.Dashboards["test_dashboard"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Dashboards["test_dashboard"].ModifiedStatus) + assert.Empty(t, config.Resources.GenieSpaces["test_genie_space"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.GenieSpaces["test_genie_space"].ModifiedStatus) + assert.Empty(t, config.Resources.Apps["test_app"].Name) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Apps["test_app"].ModifiedStatus) @@ -571,6 +585,18 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + GenieSpaces: map[string]*resources.GenieSpace{ + "test_genie_space": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "test_genie_space", + }, + }, + "test_genie_space_new": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "test_genie_space_new", + }, + }, + }, Apps: map[string]*resources.App{ "test_app": { App: apps.App{ @@ -767,6 +793,8 @@ func TestStateToBundleModifiedResources(t *testing.T) { "resources.clusters.test_cluster_old": {ID: "2"}, "resources.dashboards.test_dashboard": {ID: "1"}, "resources.dashboards.test_dashboard_old": {ID: "2"}, + "resources.genie_spaces.test_genie_space": {ID: "1"}, + "resources.genie_spaces.test_genie_space_old": {ID: "2"}, "resources.apps.test_app": {ID: "test_app"}, "resources.apps.test_app_old": {ID: "test_app_old"}, "resources.secret_scopes.test_secret_scope": {ID: "test_secret_scope"}, @@ -877,6 +905,13 @@ func TestStateToBundleModifiedResources(t *testing.T) { assert.Empty(t, config.Resources.Dashboards["test_dashboard_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Dashboards["test_dashboard_new"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.GenieSpaces["test_genie_space"].ID) + assert.Equal(t, "", config.Resources.GenieSpaces["test_genie_space"].ModifiedStatus) + assert.Equal(t, "2", config.Resources.GenieSpaces["test_genie_space_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.GenieSpaces["test_genie_space_old"].ModifiedStatus) + assert.Equal(t, "", config.Resources.GenieSpaces["test_genie_space_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.GenieSpaces["test_genie_space_new"].ModifiedStatus) + assert.Equal(t, "test_app", config.Resources.Apps["test_app"].Name) assert.Empty(t, config.Resources.Apps["test_app"].ModifiedStatus) assert.Equal(t, "test_app_old", config.Resources.Apps["test_app_old"].ID) diff --git a/cmd/bundle/generate.go b/cmd/bundle/generate.go index 9caf7fa1e37..12452293bc6 100644 --- a/cmd/bundle/generate.go +++ b/cmd/bundle/generate.go @@ -40,6 +40,7 @@ Use --bind to automatically bind the generated resource to the existing workspac cmd.AddCommand(generate.NewGenerateDashboardCommand()) cmd.AddCommand(generate.NewGenerateAlertCommand()) cmd.AddCommand(generate.NewGenerateAppCommand()) + cmd.AddCommand(generate.NewGenerateGenieSpaceCommand()) cmd.PersistentFlags().StringVar(&key, "key", "", `resource key to use for the generated configuration`) return cmd } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go new file mode 100644 index 00000000000..c6fb9e0651b --- /dev/null +++ b/cmd/bundle/generate/genie_space.go @@ -0,0 +1,503 @@ +package generate + +import ( + "context" + "errors" + "fmt" + "io" + "maps" + "os" + "path" + "path/filepath" + "slices" + "time" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/generate" + "github.com/databricks/cli/bundle/phases" + "github.com/databricks/cli/bundle/resources" + "github.com/databricks/cli/bundle/statemgmt" + "github.com/databricks/cli/cmd/bundle/deployment" + "github.com/databricks/cli/cmd/bundle/utils" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/yamlsaver" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/textutil" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/spf13/cobra" + "go.yaml.in/yaml/v3" +) + +type genieSpace struct { + // Lookup flags for one-time generate. + existingPath string + existingID string + + // Lookup flag for existing bundle resource. + resource string + + // Where to write the configuration and genie space representation. + resourceDir string + genieSpaceDir string + + // Force overwrite of existing files. + force bool + + // Watch for changes to the genie space. + watch bool + + // Relative path from the resource directory to the genie space directory. + relativeGenieSpaceDir string + + // Command. + cmd *cobra.Command + + // Automatically bind the generated resource to the existing resource. + bind bool + + // Output and error streams. + out io.Writer + err io.Writer +} + +func (g *genieSpace) resolveID(ctx context.Context, b *bundle.Bundle) string { + switch { + case g.existingPath != "": + return g.resolveFromPath(ctx, b) + case g.existingID != "": + return g.resolveFromID(ctx, b) + } + + logdiag.LogError(ctx, errors.New("expected one of --existing-path, --existing-id")) + return "" +} + +func (g *genieSpace) resolveFromPath(ctx context.Context, b *bundle.Bundle) string { + w := b.WorkspaceClient(ctx) + obj, err := w.Workspace.GetStatusByPath(ctx, g.existingPath) //nolint:staticcheck // Deprecated in SDK v0.127.0. Migration to WorkspaceHierarchyService tracked separately. + if err != nil { + if apierr.IsMissing(err) { + logdiag.LogError(ctx, fmt.Errorf("genie space %q not found", path.Base(g.existingPath))) + return "" + } + + logdiag.LogError(ctx, err) + return "" + } + + if obj.ResourceId == "" { + logdiag.LogDiag(ctx, diag.Diagnostic{ + Severity: diag.Error, + Summary: "expected a non-empty genie space resource ID", + }) + return "" + } + + return obj.ResourceId +} + +func (g *genieSpace) resolveFromID(ctx context.Context, b *bundle.Bundle) string { + w := b.WorkspaceClient(ctx) + obj, err := w.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: g.existingID, + }) + if err != nil { + if apierr.IsMissing(err) { + logdiag.LogError(ctx, fmt.Errorf("genie space with ID %s not found", g.existingID)) + return "" + } + logdiag.LogError(ctx, err) + return "" + } + + return obj.SpaceId +} + +func (g *genieSpace) saveSerializedGenieSpace(ctx context.Context, b *bundle.Bundle, genieSpace *dashboards.GenieSpace, filename string) error { + // Unmarshal and remarshal the serialized genie space to ensure it is formatted correctly. + // The result will have alphabetically sorted keys and be indented. + data, err := remarshalJSON([]byte(genieSpace.SerializedSpace)) + if err != nil { + return err + } + + // Make sure the output directory exists. + if err := os.MkdirAll(filepath.Dir(filename), 0o755); err != nil { + return err + } + + // Clean the filename to ensure it is a valid path (and can be used on this OS). + filename = filepath.Clean(filename) + + // Attempt to make the path relative to the bundle root. + rel, err := filepath.Rel(b.BundleRootPath, filename) + if err != nil { + rel = filename + } + + // Verify that the file does not already exist. + info, err := os.Stat(filename) + if err == nil { + if info.IsDir() { + return fmt.Errorf("%s is a directory", filepath.ToSlash(rel)) + } + if !g.force { + return fmt.Errorf("%s already exists. Use --force to overwrite", filepath.ToSlash(rel)) + } + } + + cmdio.LogString(ctx, "Writing genie space to "+filepath.ToSlash(rel)) + return os.WriteFile(filename, data, 0o644) +} + +func (g *genieSpace) saveConfiguration(ctx context.Context, b *bundle.Bundle, genieSpace *dashboards.GenieSpace, key string) error { + // Save serialized genie space definition to the genie space directory. + genieSpaceBasename := key + ".genie.json" + genieSpacePath := filepath.Join(g.genieSpaceDir, genieSpaceBasename) + err := g.saveSerializedGenieSpace(ctx, b, genieSpace, genieSpacePath) + if err != nil { + return err + } + + // Synthesize resource configuration. + v, err := generate.ConvertGenieSpaceToValue(genieSpace, path.Join(g.relativeGenieSpaceDir, genieSpaceBasename)) + if err != nil { + return err + } + + result := map[string]dyn.Value{ + "resources": dyn.V(map[string]dyn.Value{ + "genie_spaces": dyn.V(map[string]dyn.Value{ + key: v, + }), + }), + } + + // Make sure the output directory exists. + if err := os.MkdirAll(g.resourceDir, 0o755); err != nil { + return err + } + + // Save the configuration to the resource directory. + resourcePath := filepath.Join(g.resourceDir, key+".genie_space.yml") + saver := yamlsaver.NewSaverWithStyle(map[string]yaml.Style{ + "title": yaml.DoubleQuotedStyle, + }) + + // Attempt to make the path relative to the bundle root. + rel, err := filepath.Rel(b.BundleRootPath, resourcePath) + if err != nil { + rel = resourcePath + } + + cmdio.LogString(ctx, "Writing configuration to "+filepath.ToSlash(rel)) + err = saver.SaveAsYAML(result, resourcePath, g.force) + if err != nil { + return err + } + + return nil +} + +func waitForGenieSpaceChanges(ctx context.Context, w *databricks.WorkspaceClient, genieSpacePath string, lastModified int64) { + for { + obj, err := w.Workspace.GetStatusByPath(ctx, genieSpacePath) //nolint:staticcheck // Deprecated in SDK v0.127.0. Migration to WorkspaceHierarchyService tracked separately. + if err != nil { + logdiag.LogError(ctx, err) + return + } + + if obj.ModifiedAt > lastModified { + break + } + + time.Sleep(1 * time.Second) + } +} + +func (g *genieSpace) updateGenieSpaceForResource(ctx context.Context, b *bundle.Bundle) { + resource, ok := b.Config.Resources.GenieSpaces[g.resource] + if !ok { + logdiag.LogError(ctx, fmt.Errorf("genie space resource %q is not defined", g.resource)) + return + } + + if resource.FilePath == "" { + logdiag.LogError(ctx, fmt.Errorf("genie space resource %q has no file path defined", g.resource)) + return + } + + genieSpaceID := resource.ID + genieSpacePath := resource.FilePath + + w := b.WorkspaceClient(ctx) + + var lastModified int64 + for { + genieSpace, err := w.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: genieSpaceID, + IncludeSerializedSpace: true, + }) + if err != nil { + logdiag.LogError(ctx, err) + return + } + + // Get workspace status to check modification time. + obj, err := w.Workspace.GetStatusByPath(ctx, "/Workspace"+genieSpacePath) //nolint:staticcheck // Deprecated in SDK v0.127.0. Migration to WorkspaceHierarchyService tracked separately. + if err != nil && !apierr.IsMissing(err) { + obj, err = w.Workspace.GetStatusByPath(ctx, genieSpacePath) //nolint:staticcheck // Deprecated in SDK v0.127.0. Migration to WorkspaceHierarchyService tracked separately. + } + + var currentModified int64 + if err == nil { + currentModified = obj.ModifiedAt + } + + if lastModified == 0 || currentModified > lastModified { + err = g.saveSerializedGenieSpace(ctx, b, genieSpace, genieSpacePath) + if err != nil { + logdiag.LogError(ctx, err) + return + } + } + + if !g.watch { + return + } + + lastModified = currentModified + + if obj != nil { + waitForGenieSpaceChanges(ctx, w, obj.Path, lastModified) + } else { + time.Sleep(1 * time.Second) + } + } +} + +func (g *genieSpace) generateForExisting(ctx context.Context, b *bundle.Bundle, genieSpaceID string) { + w := b.WorkspaceClient(ctx) + genieSpace, err := w.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: genieSpaceID, + IncludeSerializedSpace: true, + }) + if err != nil { + logdiag.LogError(ctx, err) + return + } + + key := textutil.NormalizeString(genieSpace.Title) + err = g.saveConfiguration(ctx, b, genieSpace, key) + if err != nil { + logdiag.LogError(ctx, err) + } + + if g.bind { + err = deployment.BindResource(g.cmd, key, genieSpaceID, true, false, true) + if err != nil { + logdiag.LogError(ctx, err) + return + } + cmdio.LogString(ctx, fmt.Sprintf("Successfully bound genie space with an id '%s'", genieSpaceID)) + } +} + +func (g *genieSpace) initialize(ctx context.Context, b *bundle.Bundle) { + // Make the paths absolute if they aren't already. + if !filepath.IsAbs(g.resourceDir) { + g.resourceDir = filepath.Join(b.BundleRootPath, g.resourceDir) + } + if !filepath.IsAbs(g.genieSpaceDir) { + g.genieSpaceDir = filepath.Join(b.BundleRootPath, g.genieSpaceDir) + } + + // Make sure we know how the genie space path is relative to the resource path. + rel, err := filepath.Rel(g.resourceDir, g.genieSpaceDir) + if err != nil { + logdiag.LogError(ctx, err) + return + } + + g.relativeGenieSpaceDir = filepath.ToSlash(rel) +} + +func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { + phases.Initialize(ctx, b) + if logdiag.HasError(ctx) { + return + } + + requiredEngine, err := utils.ResolveEngineSetting(ctx, b) + if err != nil { + logdiag.LogError(ctx, err) + return + } + ctx, stateDesc := statemgmt.PullResourcesState(ctx, b, statemgmt.AlwaysPull(true), requiredEngine) + if logdiag.HasError(ctx) { + return + } + + if stateDesc.Engine.IsDirect() { + _, localPath := b.StateFilenameDirect(ctx) + if err := b.DeploymentBundle.StateDB.Open(localPath); err != nil { + logdiag.LogError(ctx, err) + return + } + } + + bundle.ApplySeqContext(ctx, b, + statemgmt.Load(stateDesc.Engine), + ) + if logdiag.HasError(ctx) { + return + } + + g.updateGenieSpaceForResource(ctx, b) +} + +func (g *genieSpace) runForExisting(ctx context.Context, b *bundle.Bundle) { + // Resolve the ID of the genie space to generate configuration for. + genieSpaceID := g.resolveID(ctx, b) + if logdiag.HasError(ctx) { + return + } + + g.generateForExisting(ctx, b, genieSpaceID) +} + +func (g *genieSpace) RunE(cmd *cobra.Command, args []string) error { + ctx := logdiag.InitContext(cmd.Context()) + cmd.SetContext(ctx) + + b := root.MustConfigureBundle(cmd) + if b == nil || logdiag.HasError(ctx) { + return root.ErrAlreadyPrinted + } + + g.initialize(ctx, b) + if logdiag.HasError(ctx) { + return root.ErrAlreadyPrinted + } + + if g.resource != "" { + g.runForResource(ctx, b) + } else { + g.runForExisting(ctx, b) + } + + if logdiag.HasError(ctx) { + return root.ErrAlreadyPrinted + } + + return nil +} + +// filterGenieSpaces returns a filter that only includes genie spaces. +func filterGenieSpaces(ref resources.Reference) bool { + return ref.Description.SingularName == "genie_space" +} + +// genieSpaceResourceCompletion executes to autocomplete the argument to the resource flag. +func genieSpaceResourceCompletion(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + b := root.MustConfigureBundle(cmd) + if logdiag.HasError(cmd.Context()) { + return nil, cobra.ShellCompDirectiveError + } + + if b == nil { + return nil, cobra.ShellCompDirectiveNoFileComp + } + + return slices.Collect(maps.Keys(resources.Completions(b, filterGenieSpaces))), cobra.ShellCompDirectiveNoFileComp +} + +func NewGenerateGenieSpaceCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "genie-space", + Short: "Generate configuration for a Genie space", + Long: `Generate bundle configuration for an existing Databricks Genie space. + +This command downloads an existing Genie space and creates bundle files +that you can use to deploy the Genie space to other environments or manage it as code. + +Examples: + # Import Genie space by workspace path + databricks bundle generate genie-space --existing-path /Users/me/my-genie-space + + # Import Genie space by ID + databricks bundle generate genie-space --existing-id abc123 + + # Watch for changes to keep bundle in sync with UI modifications + databricks bundle generate genie-space --resource my_genie_space --watch --force + +What gets generated: +- Genie space configuration YAML file with settings and a reference to the Genie space definition +- Genie space definition (.genie.json) file with the serialized space content + +Sync workflow for Genie space development: +When developing Genie spaces, you can modify them in the Databricks UI and sync +changes back to your bundle: + +1. Make changes to Genie space in the Databricks UI +2. Run: databricks bundle generate genie-space --resource my_genie_space --force +3. Commit changes to version control +4. Deploy to other environments with: databricks bundle deploy --target prod + +The --watch flag continuously polls for remote changes and updates your local +bundle files automatically, useful during active Genie space development.`, + } + + g := &genieSpace{ + out: cmd.OutOrStdout(), + err: cmd.ErrOrStderr(), + } + + // Lookup flags. + cmd.Flags().StringVar(&g.existingPath, "existing-path", "", `workspace path of the Genie space to generate configuration for`) + cmd.Flags().StringVar(&g.existingID, "existing-id", "", `ID of the Genie space to generate configuration for`) + cmd.Flags().StringVar(&g.resource, "resource", "", `resource key of Genie space to watch for changes`) + + // Alias lookup flags that include the resource type name. + cmd.Flags().StringVar(&g.existingPath, "existing-genie-space-path", "", `workspace path of the Genie space to generate configuration for`) + cmd.Flags().StringVar(&g.existingID, "existing-genie-space-id", "", `ID of the Genie space to generate configuration for`) + cmd.Flags().MarkHidden("existing-genie-space-path") + cmd.Flags().MarkHidden("existing-genie-space-id") + + // Output flags. + cmd.Flags().StringVarP(&g.resourceDir, "resource-dir", "d", "resources", `directory to write the configuration to`) + cmd.Flags().StringVarP(&g.genieSpaceDir, "genie-space-dir", "s", "src", `directory to write the Genie space representation to`) + cmd.Flags().BoolVarP(&g.force, "force", "f", false, `force overwrite existing files in the output directory`) + + cmd.Flags().BoolVarP(&g.bind, "bind", "b", false, `automatically bind the generated Genie space config to the existing Genie space`) + cmd.Flags().MarkHidden("bind") + + // Exactly one of the lookup flags must be provided. + cmd.MarkFlagsOneRequired( + "existing-path", + "existing-id", + "resource", + ) + + // Watch flag. This is relevant only in combination with the resource flag. + cmd.Flags().BoolVar(&g.watch, "watch", false, `watch for changes to the Genie space and update the configuration`) + + // Make sure the watch flag is only used with the existing-resource flag. + cmd.MarkFlagsMutuallyExclusive("watch", "existing-path") + cmd.MarkFlagsMutuallyExclusive("watch", "existing-id") + + // Make sure the bind flag is only used with the existing-resource flag. + cmd.MarkFlagsMutuallyExclusive("bind", "resource") + + // Completion for the resource flag. + cmd.RegisterFlagCompletionFunc("resource", genieSpaceResourceCompletion) + + cmd.RunE = g.RunE + g.cmd = cmd + return cmd +} diff --git a/cmd/experimental/workspace_open_test.go b/cmd/experimental/workspace_open_test.go index f81e79be52b..9589b48a126 100644 --- a/cmd/experimental/workspace_open_test.go +++ b/cmd/experimental/workspace_open_test.go @@ -67,7 +67,7 @@ func TestBuildWorkspaceURLFragmentBasedResources(t *testing.T) { func TestBuildWorkspaceURLUnknownResourceType(t *testing.T) { _, err := workspaceurls.BuildResourceURL("https://myworkspace.databricks.com", "unknown", "123", "") assert.ErrorContains(t, err, "unknown resource type \"unknown\"") - assert.ErrorContains(t, err, "alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses") + assert.ErrorContains(t, err, "alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses") } func TestBuildWorkspaceURLHostWithTrailingSlash(t *testing.T) { @@ -115,6 +115,7 @@ func TestWorkspaceOpenCommandCompletion(t *testing.T) { "database_catalogs", "database_instances", "experiments", + "genie_spaces", "jobs", "model_serving_endpoints", "models", @@ -145,7 +146,7 @@ func TestWorkspaceOpenCommandCompletionSecondArg(t *testing.T) { func TestWorkspaceOpenCommandHelpText(t *testing.T) { cmd := newWorkspaceOpenCommand() - assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses.") + assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses.") assert.Contains(t, cmd.Long, "databricks experimental open jobs 123456789") assert.Contains(t, cmd.Long, "databricks experimental open notebooks /Users/user@example.com/my-notebook") assert.Contains(t, cmd.Long, "databricks experimental open registered_models catalog.schema.my_model") diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index ff70f6b0505..6c3766ccaeb 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -141,6 +141,7 @@ type FakeWorkspace struct { Volumes map[string]catalog.VolumeInfo Dashboards map[string]fakeDashboard PublishedDashboards map[string]dashboards.PublishedDashboard + GenieSpaces map[string]dashboards.GenieSpace SqlWarehouses map[string]sql.GetWarehouseResponse Alerts map[string]sql.AlertV2 Experiments map[string]ml.GetExperimentResponse @@ -288,6 +289,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { Volumes: map[string]catalog.VolumeInfo{}, Dashboards: map[string]fakeDashboard{}, PublishedDashboards: map[string]dashboards.PublishedDashboard{}, + GenieSpaces: map[string]dashboards.GenieSpace{}, SqlWarehouses: map[string]sql.GetWarehouseResponse{ TestDefaultWarehouseId: { Id: TestDefaultWarehouseId, diff --git a/libs/testserver/genie_spaces.go b/libs/testserver/genie_spaces.go new file mode 100644 index 00000000000..7bd242912d3 --- /dev/null +++ b/libs/testserver/genie_spaces.go @@ -0,0 +1,162 @@ +package testserver + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "path" + "strings" + + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/databricks/databricks-sdk-go/service/workspace" +) + +// generateGenieSpaceId returns a random 32-character hex string. +func generateGenieSpaceId() (string, error) { + randomBytes := make([]byte, 16) + _, err := rand.Read(randomBytes) + if err != nil { + return "", err + } + return hex.EncodeToString(randomBytes), nil +} + +func (s *FakeWorkspace) GenieSpaceCreate(req Request) Response { + defer s.LockUnlock()() + + var createReq dashboards.GenieCreateSpaceRequest + if err := json.Unmarshal(req.Body, &createReq); err != nil { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "message": "Invalid request body: " + err.Error(), + }, + } + } + + spaceId, err := generateGenieSpaceId() + if err != nil { + return Response{ + StatusCode: 500, + Body: map[string]string{ + "message": "Failed to generate genie space ID", + }, + } + } + + genieSpace := dashboards.GenieSpace{ + SpaceId: spaceId, + Title: createReq.Title, + Description: createReq.Description, + WarehouseId: createReq.WarehouseId, + SerializedSpace: createReq.SerializedSpace, + } + + s.GenieSpaces[spaceId] = genieSpace + + // Register in the workspace files for path lookup. + if createReq.ParentPath != "" { + workspacePath := createReq.ParentPath + if !strings.HasPrefix(workspacePath, "/Workspace") { + workspacePath = path.Join("/Workspace", workspacePath) + } + workspacePath = path.Join(workspacePath, createReq.Title+".genie") + + s.files[workspacePath] = FileEntry{ + Info: workspace.ObjectInfo{ + ObjectType: "FILE", + Path: workspacePath, + ResourceId: spaceId, + }, + Data: []byte(createReq.SerializedSpace), + } + } + + return Response{ + Body: genieSpace, + } +} + +func (s *FakeWorkspace) GenieSpaceGet(req Request) Response { + defer s.LockUnlock()() + + spaceId := req.Vars["space_id"] + genieSpace, ok := s.GenieSpaces[spaceId] + if !ok { + return Response{ + StatusCode: 404, + Body: map[string]string{ + "message": "Genie space not found", + }, + } + } + + return Response{ + Body: genieSpace, + } +} + +func (s *FakeWorkspace) GenieSpaceUpdate(req Request) Response { + defer s.LockUnlock()() + + spaceId := req.Vars["space_id"] + genieSpace, ok := s.GenieSpaces[spaceId] + if !ok { + return Response{ + StatusCode: 404, + Body: map[string]string{ + "message": "Genie space not found", + }, + } + } + + var updateReq dashboards.GenieUpdateSpaceRequest + if err := json.Unmarshal(req.Body, &updateReq); err != nil { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "message": "Invalid request body: " + err.Error(), + }, + } + } + + if updateReq.Title != "" { + genieSpace.Title = updateReq.Title + } + if updateReq.Description != "" { + genieSpace.Description = updateReq.Description + } + if updateReq.WarehouseId != "" { + genieSpace.WarehouseId = updateReq.WarehouseId + } + if updateReq.SerializedSpace != "" { + genieSpace.SerializedSpace = updateReq.SerializedSpace + } + + s.GenieSpaces[spaceId] = genieSpace + + return Response{ + Body: genieSpace, + } +} + +func (s *FakeWorkspace) GenieSpaceTrash(req Request) Response { + defer s.LockUnlock()() + + spaceId := req.Vars["space_id"] + _, ok := s.GenieSpaces[spaceId] + if !ok { + return Response{ + StatusCode: 404, + Body: map[string]string{ + "message": "Genie space not found", + }, + } + } + + delete(s.GenieSpaces, spaceId) + + return Response{ + StatusCode: 200, + } +} diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index a5dea385abe..e0d621269dc 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -309,6 +309,20 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.DashboardUnpublish(req) }) + // Genie Spaces: + server.Handle("GET", "/api/2.0/genie/spaces/{space_id}", func(req Request) any { + return req.Workspace.GenieSpaceGet(req) + }) + server.Handle("POST", "/api/2.0/genie/spaces", func(req Request) any { + return req.Workspace.GenieSpaceCreate(req) + }) + server.Handle("PATCH", "/api/2.0/genie/spaces/{space_id}", func(req Request) any { + return req.Workspace.GenieSpaceUpdate(req) + }) + server.Handle("DELETE", "/api/2.0/genie/spaces/{space_id}", func(req Request) any { + return req.Workspace.GenieSpaceTrash(req) + }) + // Pipelines: server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}", func(req Request) any { diff --git a/libs/testserver/permissions.go b/libs/testserver/permissions.go index e7983b1afa7..962825dd037 100644 --- a/libs/testserver/permissions.go +++ b/libs/testserver/permissions.go @@ -25,6 +25,7 @@ var requestObjectTypeToObjectType = map[string]string{ "sql/alerts": "alert", "sql/queries": "query", "dashboards": "dashboard", + "genie/spaces": "genie-space", "experiments": "mlflowExperiment", "registered-models": "registered-model", "serving-endpoints": "serving-endpoint", diff --git a/libs/workspaceurls/urls.go b/libs/workspaceurls/urls.go index be9a41fc959..46930c92239 100644 --- a/libs/workspaceurls/urls.go +++ b/libs/workspaceurls/urls.go @@ -16,6 +16,7 @@ var resourceURLPatterns = map[string]string{ "database_catalogs": "explore/data/%s", "database_instances": "compute/database-instances/%s", "experiments": "ml/experiments/%s", + "genie_spaces": "genie/rooms/%s", "jobs": "jobs/%s", "models": "ml/models/%s", "model_serving_endpoints": "ml/endpoints/%s", From 517d22e33775e481521355f3d2dc0e61d6ef4141 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 30 Apr 2026 23:10:24 +0200 Subject: [PATCH 02/42] bundle: retry genie_space create on 400 INVALID_PARAMETER_VALUE missing-parent-path errors Genie surfaces a missing parent folder inconsistently across environments: some workspaces return a standard 404 missing-resource error, while others return 400 INVALID_PARAMETER_VALUE with a NOT_FOUND "Tree node ... does not exist" message embedded in the text. Treat both forms as "create the parent directory and retry once". Co-authored-by: Isaac --- bundle/direct/dresources/genie_space.go | 30 +++++- bundle/direct/dresources/genie_space_test.go | 105 +++++++++++++++++++ 2 files changed, 132 insertions(+), 3 deletions(-) create mode 100644 bundle/direct/dresources/genie_space_test.go diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go index 490849cfbdf..8cd26897322 100644 --- a/bundle/direct/dresources/genie_space.go +++ b/bundle/direct/dresources/genie_space.go @@ -3,7 +3,9 @@ package dresources import ( "context" "encoding/json" + "errors" "fmt" + "strings" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/utils" @@ -99,6 +101,26 @@ func responseToGenieSpaceConfig(space *dashboards.GenieSpace, serializedSpace st } } +func isMissingGenieParentPathError(err error) bool { + if apierr.IsMissing(err) { + return true + } + + var apiErr *apierr.APIError + if !errors.As(err, &apiErr) { + return false + } + + // Genie reports a missing parent folder inconsistently across environments. + // Some workspaces return a standard missing-resource error, while others + // return INVALID_PARAMETER_VALUE with a NOT_FOUND message embedded in the + // text. Treat both forms as "create the parent directory and retry once". + return apiErr.StatusCode == 400 && + apiErr.ErrorCode == "INVALID_PARAMETER_VALUE" && + strings.Contains(apiErr.Message, "Tree node with path") && + strings.Contains(apiErr.Message, "does not exist") +} + func (r *ResourceGenieSpace) DoCreate(ctx context.Context, config *resources.GenieSpaceConfig) (string, *resources.GenieSpaceConfig, error) { serializedSpace, err := prepareGenieSpaceRequest(config) if err != nil { @@ -117,9 +139,11 @@ func (r *ResourceGenieSpace) DoCreate(ctx context.Context, config *resources.Gen createResp, err := r.client.Genie.CreateSpace(ctx, req) - // The API returns 404 if the parent directory doesn't exist. - // Create it and retry once. - if err != nil && apierr.IsMissing(err) { + // Retry once after creating the parent directory when the workspace folder + // is missing. Genie can surface this either as a standard missing-resource + // error or as INVALID_PARAMETER_VALUE with a "Tree node ... does not exist" + // message depending on the backend. + if err != nil && isMissingGenieParentPathError(err) { err = r.client.Workspace.MkdirsByPath(ctx, config.ParentPath) //nolint:staticcheck // Deprecated in SDK v0.127.0. Migration to WorkspaceHierarchyService tracked separately. if err != nil { return "", nil, fmt.Errorf("failed to create parent directory: %w", err) diff --git a/bundle/direct/dresources/genie_space_test.go b/bundle/direct/dresources/genie_space_test.go new file mode 100644 index 00000000000..c731322ca8e --- /dev/null +++ b/bundle/direct/dresources/genie_space_test.go @@ -0,0 +1,105 @@ +package dresources + +import ( + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestIsMissingGenieParentPathError(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + { + name: "standard missing error", + err: &apierr.APIError{ + StatusCode: 404, + ErrorCode: "NOT_FOUND", + Message: "not found", + }, + want: true, + }, + { + name: "invalid parameter tree node missing error", + err: &apierr.APIError{ + StatusCode: 400, + ErrorCode: "INVALID_PARAMETER_VALUE", + Message: "NOT_FOUND: Tree node with path /Workspace/foo does not exist", + }, + want: true, + }, + { + name: "other invalid parameter error", + err: &apierr.APIError{ + StatusCode: 400, + ErrorCode: "INVALID_PARAMETER_VALUE", + Message: "some other validation failure", + }, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, isMissingGenieParentPathError(tt.err)) + }) + } +} + +func TestGenieSpaceDoCreateRetriesWhenParentPathLooksMissing(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + req := dashboards.GenieCreateSpaceRequest{ + Title: "test genie space", + Description: "test description", + ParentPath: "/Workspace/test-parent", + WarehouseId: "test-warehouse-id", + SerializedSpace: "{}", + } + + m.GetMockGenieAPI().EXPECT(). + CreateSpace(ctx, req). + Return(nil, &apierr.APIError{ + StatusCode: 400, + ErrorCode: "INVALID_PARAMETER_VALUE", + Message: "NOT_FOUND: Tree node with path /Workspace/test-parent does not exist", + }). + Once() + + m.GetMockWorkspaceAPI().EXPECT(). + MkdirsByPath(ctx, "/Workspace/test-parent"). + Return(nil). + Once() + + m.GetMockGenieAPI().EXPECT(). + CreateSpace(ctx, req). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id", + Title: "test genie space", + Description: "test description", + WarehouseId: "test-warehouse-id", + SerializedSpace: "{}", + }, nil). + Once() + + id, state, err := r.DoCreate(ctx, &resources.GenieSpaceConfig{ + Title: "test genie space", + Description: "test description", + ParentPath: "/Workspace/test-parent", + WarehouseId: "test-warehouse-id", + SerializedSpace: "{}", + }) + require.NoError(t, err) + assert.Equal(t, "space-id", id) + require.NotNil(t, state) + assert.Equal(t, "test genie space", state.Title) +} From fc4af0d37fa58f527e03797e85361c05cca9d2be Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 30 Apr 2026 23:10:38 +0200 Subject: [PATCH 03/42] bundle: register genie_space file_path translation in unified path visitor VisitGenieSpacePaths existed but was never called by VisitPaths, so NormalizePaths did not rewrite genie_space file_path values from "relative to YAML location" to "relative to bundle root" before applyGenieSpaceTranslations resolved them. The result was that generator output like "../src/.genie.json" failed on deploy with "path ... is not contained in sync root path". Co-authored-by: Isaac --- bundle/config/mutator/paths/visitor.go | 1 + .../mutator/translate_paths_genie_spaces.go | 1 - .../translate_paths_genie_spaces_test.go | 52 +++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 bundle/config/mutator/translate_paths_genie_spaces_test.go diff --git a/bundle/config/mutator/paths/visitor.go b/bundle/config/mutator/paths/visitor.go index 0e3d59d43f5..bdf42188fde 100644 --- a/bundle/config/mutator/paths/visitor.go +++ b/bundle/config/mutator/paths/visitor.go @@ -15,6 +15,7 @@ func VisitPaths(root dyn.Value, fn VisitFunc) (dyn.Value, error) { VisitArtifactPaths, VisitAlertPaths, VisitDashboardPaths, + VisitGenieSpacePaths, VisitPipelinePaths, VisitPipelineLibrariesPaths, } diff --git a/bundle/config/mutator/translate_paths_genie_spaces.go b/bundle/config/mutator/translate_paths_genie_spaces.go index b6a3a652427..279d97be01c 100644 --- a/bundle/config/mutator/translate_paths_genie_spaces.go +++ b/bundle/config/mutator/translate_paths_genie_spaces.go @@ -4,7 +4,6 @@ import ( "context" "github.com/databricks/cli/bundle/config/mutator/paths" - "github.com/databricks/cli/libs/dyn" ) diff --git a/bundle/config/mutator/translate_paths_genie_spaces_test.go b/bundle/config/mutator/translate_paths_genie_spaces_test.go new file mode 100644 index 00000000000..a5295400905 --- /dev/null +++ b/bundle/config/mutator/translate_paths_genie_spaces_test.go @@ -0,0 +1,52 @@ +package mutator_test + +import ( + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/vfs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTranslatePathsGenieSpaces_FilePathRelativeSubDirectory(t *testing.T) { + dir := t.TempDir() + touchEmptyFile(t, filepath.Join(dir, "src", "my_space.genie.json")) + + b := &bundle.Bundle{ + SyncRootPath: dir, + BundleRootPath: dir, + SyncRoot: vfs.MustNew(dir), + Config: config.Root{ + Resources: config.Resources{ + GenieSpaces: map[string]*resources.GenieSpace{ + "genie_space": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "My Genie Space", + }, + FilePath: "../src/my_space.genie.json", + }, + }, + }, + }, + } + + bundletest.SetLocation(b, "resources.genie_spaces", []dyn.Location{{ + File: filepath.Join(dir, "resources", "genie_space.yml"), + }}) + + diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePathsDashboards()) + require.NoError(t, diags.Error()) + + assert.Equal( + t, + filepath.ToSlash(filepath.Join("src", "my_space.genie.json")), + b.Config.Resources.GenieSpaces["genie_space"].FilePath, + ) +} From 0145b87dbfd1d3ac4a2fc5b8e1dc064d9cadbec3 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 30 Apr 2026 23:25:14 +0200 Subject: [PATCH 04/42] bundle: normalize inline serialized_space to JSON string before plan Inline YAML serialized_space stayed as a structured value (map[string]any with int leaves) in the config struct, while state held the JSON string that was sent to the API. structdiff compared an `any` field with reflect.DeepEqual, which reports map != string, so every plan after deploy showed a false update for the genie_space. Marshal inline serialized_space to its JSON string in ConfigureGenieSpaceSerializedSpace, mirroring the file_path code path, so config-side and state-side carry the same type. The genie_space_complex validate test is updated to reflect that serialized_space is now a string regardless of input form, and a new acceptance test under resources/genie_spaces/inline asserts that a deploy + plan cycle is drift-free for inline serialized_space. Co-authored-by: Isaac --- .../genie_spaces/inline/databricks.yml.tmpl | 22 +++++++++++++ .../genie_spaces/inline/out.plan.json | 28 +++++++++++++++++ .../genie_spaces/inline/out.test.toml | 6 ++++ .../resources/genie_spaces/inline/output.txt | 17 ++++++++++ .../resources/genie_spaces/inline/script | 18 +++++++++++ .../resources/genie_spaces/inline/test.toml | 10 ++++++ .../validate/genie_space_complex/output.txt | 14 ++++++--- .../validate/genie_space_complex/script | 17 +++++++--- .../configure_genie_space_serialized_space.go | 31 +++++++++++++------ 9 files changed, 143 insertions(+), 20 deletions(-) create mode 100644 acceptance/bundle/resources/genie_spaces/inline/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/genie_spaces/inline/out.plan.json create mode 100644 acceptance/bundle/resources/genie_spaces/inline/out.test.toml create mode 100644 acceptance/bundle/resources/genie_spaces/inline/output.txt create mode 100644 acceptance/bundle/resources/genie_spaces/inline/script create mode 100644 acceptance/bundle/resources/genie_spaces/inline/test.toml diff --git a/acceptance/bundle/resources/genie_spaces/inline/databricks.yml.tmpl b/acceptance/bundle/resources/genie_spaces/inline/databricks.yml.tmpl new file mode 100644 index 00000000000..ea045097468 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/databricks.yml.tmpl @@ -0,0 +1,22 @@ +bundle: + name: deploy-genie-space-inline-$UNIQUE_NAME + +resources: + genie_spaces: + sales_analytics: + title: "Sales Analytics Inline Genie" + description: "Inline serialized_space test" + warehouse_id: "test-warehouse-id" + parent_path: /Users/$CURRENT_USER_NAME + serialized_space: + version: 1 + config: + sample_questions: + - id: "sq-001" + question: ["What is the total revenue?"] + data_sources: + tables: + - identifier: "main.sales.orders" + column_configs: + - column_name: "amount" + get_example_values: true diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.plan.json b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json new file mode 100644 index 00000000000..49a077a1d4f --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json @@ -0,0 +1,28 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "plan": { + "resources.genie_spaces.sales_analytics": { + "action": "skip", + "remote_state": { + "description": "Inline serialized_space test", + "parent_path": "", + "serialized_space": "{\"config\":{\"sample_questions\":[{\"id\":\"sq-001\",\"question\":[\"What is the total revenue?\"]}]},\"data_sources\":{\"tables\":[{\"column_configs\":[{\"column_name\":\"amount\",\"get_example_values\":true}],\"identifier\":\"main.sales.orders\"}]},\"version\":1}", + "space_id": "[GENIE_SPACE_ID]", + "title": "Sales Analytics Inline Genie", + "warehouse_id": "test-warehouse-id" + }, + "changes": { + "parent_path": { + "action": "skip", + "reason": "input_only", + "old": "/Workspace/Users/[USERNAME]", + "new": "/Workspace/Users/[USERNAME]", + "remote": "" + } + } + } + } +} diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.test.toml b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml new file mode 100644 index 00000000000..496668716fa --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] + Ignore = ["databricks.yml"] diff --git a/acceptance/bundle/resources/genie_spaces/inline/output.txt b/acceptance/bundle/resources/genie_spaces/inline/output.txt new file mode 100644 index 00000000000..99fef197dd0 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/output.txt @@ -0,0 +1,17 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-genie-space-inline-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.genie_spaces.sales_analytics + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-genie-space-inline-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/genie_spaces/inline/script b/acceptance/bundle/resources/genie_spaces/inline/script new file mode 100644 index 00000000000..8f2f625b7b0 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/script @@ -0,0 +1,18 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy +GENIE_SPACE_ID=$($CLI bundle summary --output json | jq -r '.resources.genie_spaces.sales_analytics.id') + +# Capture the genie space ID as a replacement. +echo "$GENIE_SPACE_ID:GENIE_SPACE_ID" >> ACC_REPLS + +# Plan after deploy must be drift-free aside from input_only fields. +# Without normalization the inline serialized_space leaves a map in the +# config struct while state holds a string, and structdiff reports false +# drift on every plan. +trace $CLI bundle plan -o json > out.plan.json diff --git a/acceptance/bundle/resources/genie_spaces/inline/test.toml b/acceptance/bundle/resources/genie_spaces/inline/test.toml new file mode 100644 index 00000000000..18a07a1b00b --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/test.toml @@ -0,0 +1,10 @@ +Local = true +RecordRequests = false + +# Genie spaces only support direct deployment engine (no Terraform provider yet) +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/bundle/validate/genie_space_complex/output.txt b/acceptance/bundle/validate/genie_space_complex/output.txt index 2fe58959fc8..4a0747210cc 100644 --- a/acceptance/bundle/validate/genie_space_complex/output.txt +++ b/acceptance/bundle/validate/genie_space_complex/output.txt @@ -6,13 +6,17 @@ }, "inline_yaml": { "title": "Inline YAML Genie Space", - "serialized_space_type": "object", - "tables_count": 1, - "has_column_configs": true, - "has_text_instructions": true + "serialized_space_type": "string", + "parsed": { + "tables_count": 1, + "has_column_configs": true, + "has_text_instructions": true + } }, "minimal_valid": { "title": "Minimal Valid", - "tables_count": 0 + "parsed": { + "tables_count": 0 + } } } diff --git a/acceptance/bundle/validate/genie_space_complex/script b/acceptance/bundle/validate/genie_space_complex/script index 01e1fed72bc..4feab1e373b 100644 --- a/acceptance/bundle/validate/genie_space_complex/script +++ b/acceptance/bundle/validate/genie_space_complex/script @@ -1,4 +1,7 @@ -# Validate complex genie spaces and check the serialized_space structure is preserved +# Validate complex genie spaces. ConfigureGenieSpaceSerializedSpace +# normalizes inline serialized_space YAML to a JSON string so the field has +# the same shape as the file_path code path; the script parses the JSON +# back to verify that the original structure is preserved. $CLI bundle validate -o json | jq '{ full_featured: .resources.genie_spaces.full_featured | { title, @@ -8,12 +11,16 @@ $CLI bundle validate -o json | jq '{ inline_yaml: .resources.genie_spaces.inline_yaml | { title, serialized_space_type: (.serialized_space | type), - tables_count: (.serialized_space.data_sources.tables | length), - has_column_configs: ((.serialized_space.data_sources.tables[0].column_configs | length) > 0), - has_text_instructions: ((.serialized_space.instructions.text_instructions | length) > 0) + parsed: (.serialized_space | fromjson) | { + tables_count: (.data_sources.tables | length), + has_column_configs: ((.data_sources.tables[0].column_configs | length) > 0), + has_text_instructions: ((.instructions.text_instructions | length) > 0) + } }, minimal_valid: .resources.genie_spaces.minimal_valid | { title, - tables_count: (.serialized_space.data_sources.tables | length) + parsed: (.serialized_space | fromjson) | { + tables_count: (.data_sources.tables | length) + } } }' diff --git a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go index bf129b4060d..b8382ca6e7c 100644 --- a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go +++ b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go @@ -2,6 +2,7 @@ package resourcemutator import ( "context" + "encoding/json" "fmt" "github.com/databricks/cli/bundle" @@ -30,21 +31,31 @@ func (c configureGenieSpaceSerializedSpace) Apply(_ context.Context, b *bundle.B dyn.AnyKey(), ) - // Configure serialized_space field for all genie spaces. err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - // Include "serialized_space" field if "file_path" is set. - path, ok := v.Get(filePathFieldName).AsString() - if !ok { - return v, nil + if path, ok := v.Get(filePathFieldName).AsString(); ok { + contents, err := b.SyncRoot.ReadFile(path) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to read serialized genie space from file_path %s: %w", path, err) + } + return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(contents))) } - contents, err := b.SyncRoot.ReadFile(path) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to read serialized genie space from file_path %s: %w", path, err) + // Marshal an inline structured serialized_space to a JSON string so + // both config-side and state-side carry the same plain string. + // Otherwise YAML decodes small ints as Go `int` while state JSON + // round-trip decodes them as `float64`, and structdiff reports + // false drift on every plan. + ss := v.Get(serializedSpaceFieldName) + switch ss.Kind() { + case dyn.KindMap, dyn.KindSequence: + jsonBytes, err := json.Marshal(ss.AsAny()) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to marshal inline serialized_space: %w", err) + } + return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(jsonBytes))) } - - return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(contents))) + return v, nil }) }) From 5535c46ff4862b610ca2f49a995141baed31844b Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 30 Apr 2026 23:29:13 +0200 Subject: [PATCH 05/42] bundle: reject genie_space permissions during plan Databricks workspaces do not expose a permissions endpoint for Genie Spaces (PUT /permissions/genie/spaces/ returns 404 ENDPOINT_NOT_FOUND). Without an upfront check the deploy creates the space first and then errors when applying permissions, leaving partial state behind. Add ValidateGenieSpacePermissions to the PreDeployChecks pipeline so both per-resource permissions and bundle-level permissions propagated by ApplyBundlePermissions surface a clear validation error before any API call is made. Co-authored-by: Isaac --- .../databricks.yml | 21 ++++++++ .../out.test.toml | 5 ++ .../output.txt | 22 ++++++++ .../script | 1 + .../test.toml | 8 +++ .../validate_genie_space_permissions.go | 44 +++++++++++++++ .../validate_genie_space_permissions_test.go | 54 +++++++++++++++++++ bundle/phases/plan.go | 1 + 8 files changed, 156 insertions(+) create mode 100644 acceptance/bundle/validate/genie_space_permissions_unsupported/databricks.yml create mode 100644 acceptance/bundle/validate/genie_space_permissions_unsupported/out.test.toml create mode 100644 acceptance/bundle/validate/genie_space_permissions_unsupported/output.txt create mode 100644 acceptance/bundle/validate/genie_space_permissions_unsupported/script create mode 100644 acceptance/bundle/validate/genie_space_permissions_unsupported/test.toml create mode 100644 bundle/config/mutator/validate_genie_space_permissions.go create mode 100644 bundle/config/mutator/validate_genie_space_permissions_test.go diff --git a/acceptance/bundle/validate/genie_space_permissions_unsupported/databricks.yml b/acceptance/bundle/validate/genie_space_permissions_unsupported/databricks.yml new file mode 100644 index 00000000000..71126ef7dcb --- /dev/null +++ b/acceptance/bundle/validate/genie_space_permissions_unsupported/databricks.yml @@ -0,0 +1,21 @@ +bundle: + name: genie-space-permissions-unsupported + +resources: + genie_spaces: + inline_perms: + title: "Inline Perms" + warehouse_id: "test-warehouse-id" + serialized_space: "{}" + permissions: + - level: CAN_MANAGE + user_name: someone@example.com + + bundle_perms: + title: "Bundle Perms" + warehouse_id: "test-warehouse-id" + serialized_space: "{}" + +permissions: + - level: CAN_MANAGE + user_name: someone@example.com diff --git a/acceptance/bundle/validate/genie_space_permissions_unsupported/out.test.toml b/acceptance/bundle/validate/genie_space_permissions_unsupported/out.test.toml new file mode 100644 index 00000000000..54146af5645 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_permissions_unsupported/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/genie_space_permissions_unsupported/output.txt b/acceptance/bundle/validate/genie_space_permissions_unsupported/output.txt new file mode 100644 index 00000000000..7224e5d1fea --- /dev/null +++ b/acceptance/bundle/validate/genie_space_permissions_unsupported/output.txt @@ -0,0 +1,22 @@ +Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups +If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. + +Consider using a adding a top-level permissions section such as the following: + + permissions: + - user_name: [USERNAME] + level: CAN_MANAGE + +See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. + in databricks.yml:20:3 + +Error: Genie Space permissions are not supported + +Databricks workspaces do not expose a permissions endpoint for Genie Spaces, so a deploy with permissions configured would create the space and then fail. Remove the permissions block, or remove the Genie Space from any bundle-level permissions, until the API adds support. + +Error: Genie Space permissions are not supported + +Databricks workspaces do not expose a permissions endpoint for Genie Spaces, so a deploy with permissions configured would create the space and then fail. Remove the permissions block, or remove the Genie Space from any bundle-level permissions, until the API adds support. + + +Exit code: 1 diff --git a/acceptance/bundle/validate/genie_space_permissions_unsupported/script b/acceptance/bundle/validate/genie_space_permissions_unsupported/script new file mode 100644 index 00000000000..b260e836a71 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_permissions_unsupported/script @@ -0,0 +1 @@ +$CLI bundle plan diff --git a/acceptance/bundle/validate/genie_space_permissions_unsupported/test.toml b/acceptance/bundle/validate/genie_space_permissions_unsupported/test.toml new file mode 100644 index 00000000000..c304e3eba17 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_permissions_unsupported/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = false + +Ignore = [".databricks"] + +# Genie spaces only support direct deployment engine. +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/config/mutator/validate_genie_space_permissions.go b/bundle/config/mutator/validate_genie_space_permissions.go new file mode 100644 index 00000000000..e2efb517705 --- /dev/null +++ b/bundle/config/mutator/validate_genie_space_permissions.go @@ -0,0 +1,44 @@ +package mutator + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" +) + +type validateGenieSpacePermissions struct{} + +// ValidateGenieSpacePermissions errors if any genie_space resource has +// permissions configured. The Databricks workspace API does not expose +// PUT /permissions/genie/spaces/, so the deploy would create the +// space and then fail when applying permissions, leaving partial state. +// Bundle-level permissions are propagated to genie_spaces by +// ApplyBundlePermissions and are caught here as well. +func ValidateGenieSpacePermissions() bundle.Mutator { + return &validateGenieSpacePermissions{} +} + +func (m *validateGenieSpacePermissions) Name() string { + return "ValidateGenieSpacePermissions" +} + +func (m *validateGenieSpacePermissions) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + for key, space := range b.Config.Resources.GenieSpaces { + if space == nil || len(space.Permissions) == 0 { + continue + } + + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "Genie Space permissions are not supported", + Detail: "Databricks workspaces do not expose a permissions endpoint for Genie Spaces, so a deploy with permissions configured would create the space and then fail. Remove the permissions block, or remove the Genie Space from any bundle-level permissions, until the API adds support.", + Locations: b.Config.GetLocations(fmt.Sprintf("resources.genie_spaces.%s.permissions", key)), + }) + } + + return diags +} diff --git a/bundle/config/mutator/validate_genie_space_permissions_test.go b/bundle/config/mutator/validate_genie_space_permissions_test.go new file mode 100644 index 00000000000..0ccf143815e --- /dev/null +++ b/bundle/config/mutator/validate_genie_space_permissions_test.go @@ -0,0 +1,54 @@ +package mutator_test + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/iam" + "github.com/stretchr/testify/assert" +) + +func TestValidateGenieSpacePermissions_NoPermissions(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + GenieSpaces: map[string]*resources.GenieSpace{ + "my_space": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "My Space", + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ValidateGenieSpacePermissions()) + assert.Empty(t, diags) +} + +func TestValidateGenieSpacePermissions_WithPermissionsErrors(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + GenieSpaces: map[string]*resources.GenieSpace{ + "my_space": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "My Space", + }, + Permissions: []resources.Permission{ + {Level: iam.PermissionLevel("CAN_MANAGE"), UserName: "user@example.com"}, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ValidateGenieSpacePermissions()) + assert.Len(t, diags, 1) + assert.Equal(t, "Genie Space permissions are not supported", diags[0].Summary) +} diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 0af9394f243..3e63b8f607b 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -25,6 +25,7 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine deploy.StatePull(), mutator.ValidateGitDetails(), mutator.ValidateDirectOnlyResources(engine), + mutator.ValidateGenieSpacePermissions(), mutator.ValidateLifecycleStarted(engine), statemgmt.CheckRunningResource(engine), ) From 133a64c5b516a691edd3e8cd99d9b01079cf6039 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 30 Apr 2026 23:33:28 +0200 Subject: [PATCH 06/42] bundle: warn on dual genie_space sources and clarify direct-only error Two minor follow-ups to the genie_spaces work: - ConfigureGenieSpaceSerializedSpace silently let file_path win when a user also set serialized_space inline. Emit a warning that points at the inline block so the user knows their YAML is being dropped on the floor. - ValidateDirectOnlyResources only mentioned the DATABRICKS_BUNDLE_ENGINE env var as a way to opt into direct mode, even though 'bundle.engine: direct' in databricks.yml is the more common entry point. Mention both. Co-authored-by: Isaac --- .../contents.genie.json | 1 + .../databricks.yml | 11 +++++++++++ .../out.test.toml | 5 +++++ .../genie_space_file_path_and_inline/output.txt | 10 ++++++++++ .../genie_space_file_path_and_inline/script | 1 + .../genie_space_file_path_and_inline/test.toml | 6 ++++++ .../configure_genie_space_serialized_space.go | 17 +++++++++++++---- 7 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 acceptance/bundle/validate/genie_space_file_path_and_inline/contents.genie.json create mode 100644 acceptance/bundle/validate/genie_space_file_path_and_inline/databricks.yml create mode 100644 acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml create mode 100644 acceptance/bundle/validate/genie_space_file_path_and_inline/output.txt create mode 100644 acceptance/bundle/validate/genie_space_file_path_and_inline/script create mode 100644 acceptance/bundle/validate/genie_space_file_path_and_inline/test.toml diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/contents.genie.json b/acceptance/bundle/validate/genie_space_file_path_and_inline/contents.genie.json new file mode 100644 index 00000000000..cb608f6e9c4 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/contents.genie.json @@ -0,0 +1 @@ +{"version": 1} diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/databricks.yml b/acceptance/bundle/validate/genie_space_file_path_and_inline/databricks.yml new file mode 100644 index 00000000000..dea909bf2f9 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: genie-space-file-path-and-inline + +resources: + genie_spaces: + both_set: + title: "Both set" + warehouse_id: "test-warehouse-id" + file_path: "./contents.genie.json" + serialized_space: + version: 1 diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml new file mode 100644 index 00000000000..54146af5645 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/output.txt b/acceptance/bundle/validate/genie_space_file_path_and_inline/output.txt new file mode 100644 index 00000000000..589a8333c37 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/output.txt @@ -0,0 +1,10 @@ +Warning: both file_path and serialized_space are set; file_path will be used and serialized_space will be ignored + in databricks.yml:11:9 + +Name: genie-space-file-path-and-inline +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/genie-space-file-path-and-inline/default + +Found 1 warning diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/script b/acceptance/bundle/validate/genie_space_file_path_and_inline/script new file mode 100644 index 00000000000..72555b332a4 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/script @@ -0,0 +1 @@ +$CLI bundle validate diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/test.toml b/acceptance/bundle/validate/genie_space_file_path_and_inline/test.toml new file mode 100644 index 00000000000..4fc5284af50 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = false + +# Genie spaces only support direct deployment engine. +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go index b8382ca6e7c..3144bf7b614 100644 --- a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go +++ b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go @@ -33,10 +33,20 @@ func (c configureGenieSpaceSerializedSpace) Apply(_ context.Context, b *bundle.B err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - if path, ok := v.Get(filePathFieldName).AsString(); ok { - contents, err := b.SyncRoot.ReadFile(path) + filePath, hasFilePath := v.Get(filePathFieldName).AsString() + ss := v.Get(serializedSpaceFieldName) + + if hasFilePath { + if ss.IsValid() && ss.Kind() != dyn.KindNil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: "both file_path and serialized_space are set; file_path will be used and serialized_space will be ignored", + Locations: ss.Locations(), + }) + } + contents, err := b.SyncRoot.ReadFile(filePath) if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to read serialized genie space from file_path %s: %w", path, err) + return dyn.InvalidValue, fmt.Errorf("failed to read serialized genie space from file_path %s: %w", filePath, err) } return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(contents))) } @@ -46,7 +56,6 @@ func (c configureGenieSpaceSerializedSpace) Apply(_ context.Context, b *bundle.B // Otherwise YAML decodes small ints as Go `int` while state JSON // round-trip decodes them as `float64`, and structdiff reports // false drift on every plan. - ss := v.Get(serializedSpaceFieldName) switch ss.Kind() { case dyn.KindMap, dyn.KindSequence: jsonBytes, err := json.Marshal(ss.AsAny()) From 5decb0091e395d0066da0cfe7f3951972d34261e Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 30 Apr 2026 23:40:43 +0200 Subject: [PATCH 07/42] bundle: round-trip parent_path through generate genie-space When the Genie API returns parent_path on GetSpace, propagate it through bundle generate genie-space so the produced YAML deploys back to the same workspace folder. The testserver is updated to mirror that response shape so the acceptance fixture exercises the new path. Filter ParentPath out of ForceSendFields in DoRead and responseToGenieSpaceConfig: we deliberately clear ParentPath in the returned GenieSpaceConfig because the GET API does not reliably include it, but the SDK still surfaces it in ForceSendFields when the field appeared on the wire. Without this filter, deploy state serialization force-emits parent_path: "" even though the field is logically unset, producing spurious output diffs. Co-authored-by: Isaac --- .../out/resource/test_genie_space.genie_space.yml | 1 + .../resources/genie_spaces/inline/out.plan.json | 4 +--- bundle/direct/dresources/genie_space.go | 12 ++++++++++-- bundle/generate/genie_space.go | 4 ++++ libs/testserver/genie_spaces.go | 1 + 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml b/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml index 14764931f1b..a73945364ce 100644 --- a/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml +++ b/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml @@ -5,3 +5,4 @@ resources: warehouse_id: test-warehouse-id file_path: ../genie_space/test_genie_space.genie.json description: test description + parent_path: /Workspace/test-[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.plan.json b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json index 49a077a1d4f..41f6aa983de 100644 --- a/acceptance/bundle/resources/genie_spaces/inline/out.plan.json +++ b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json @@ -8,7 +8,6 @@ "action": "skip", "remote_state": { "description": "Inline serialized_space test", - "parent_path": "", "serialized_space": "{\"config\":{\"sample_questions\":[{\"id\":\"sq-001\",\"question\":[\"What is the total revenue?\"]}]},\"data_sources\":{\"tables\":[{\"column_configs\":[{\"column_name\":\"amount\",\"get_example_values\":true}],\"identifier\":\"main.sales.orders\"}]},\"version\":1}", "space_id": "[GENIE_SPACE_ID]", "title": "Sales Analytics Inline Genie", @@ -19,8 +18,7 @@ "action": "skip", "reason": "input_only", "old": "/Workspace/Users/[USERNAME]", - "new": "/Workspace/Users/[USERNAME]", - "remote": "" + "new": "/Workspace/Users/[USERNAME]" } } } diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go index 8cd26897322..d2569af7a61 100644 --- a/bundle/direct/dresources/genie_space.go +++ b/bundle/direct/dresources/genie_space.go @@ -56,7 +56,11 @@ func (r *ResourceGenieSpace) DoRead(ctx context.Context, id string) (*resources. return nil, err } - forceSendFields := utils.FilterFields[resources.GenieSpaceConfig](space.ForceSendFields) + // Drop ParentPath from ForceSendFields. We always clear ParentPath + // below because the GET Genie space API does not reliably return it, + // and keeping it in ForceSendFields would force-emit parent_path: "" + // in state output even though the field is logically unset. + forceSendFields := utils.FilterFields[resources.GenieSpaceConfig](space.ForceSendFields, "ParentPath") return &resources.GenieSpaceConfig{ Description: space.Description, @@ -86,7 +90,11 @@ func prepareGenieSpaceRequest(config *resources.GenieSpaceConfig) (string, error } func responseToGenieSpaceConfig(space *dashboards.GenieSpace, serializedSpace string) *resources.GenieSpaceConfig { - forceSendFields := utils.FilterFields[resources.GenieSpaceConfig](space.ForceSendFields) + // Drop ParentPath from ForceSendFields. We always clear ParentPath + // below because the GET Genie space API does not reliably return it, + // and keeping it in ForceSendFields would force-emit parent_path: "" + // in state output even though the field is logically unset. + forceSendFields := utils.FilterFields[resources.GenieSpaceConfig](space.ForceSendFields, "ParentPath") return &resources.GenieSpaceConfig{ Description: space.Description, diff --git a/bundle/generate/genie_space.go b/bundle/generate/genie_space.go index ebb1098b2b1..5bd11761a49 100644 --- a/bundle/generate/genie_space.go +++ b/bundle/generate/genie_space.go @@ -18,5 +18,9 @@ func ConvertGenieSpaceToValue(genieSpace *dashboards.GenieSpace, filePath string dv["description"] = dyn.NewValue(genieSpace.Description, []dyn.Location{{Line: 4}}) } + if genieSpace.ParentPath != "" { + dv["parent_path"] = dyn.NewValue(genieSpace.ParentPath, []dyn.Location{{Line: 5}}) + } + return dyn.V(dv), nil } diff --git a/libs/testserver/genie_spaces.go b/libs/testserver/genie_spaces.go index 7bd242912d3..0b11b119fcf 100644 --- a/libs/testserver/genie_spaces.go +++ b/libs/testserver/genie_spaces.go @@ -48,6 +48,7 @@ func (s *FakeWorkspace) GenieSpaceCreate(req Request) Response { SpaceId: spaceId, Title: createReq.Title, Description: createReq.Description, + ParentPath: createReq.ParentPath, WarehouseId: createReq.WarehouseId, SerializedSpace: createReq.SerializedSpace, } From 786ded73015d27ae43ed697abed9b2c1524013ba Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 30 Apr 2026 23:42:59 +0200 Subject: [PATCH 08/42] bundle: lint cleanups for genie_space changes - Replace switch-with-fallthrough on dyn.Kind with a guard clause to satisfy the exhaustive linter without listing every Kind variant. - Use http.StatusBadRequest in isMissingGenieParentPathError instead of a magic 400 (auto-fix from golangci-lint). Co-authored-by: Isaac --- .../configure_genie_space_serialized_space.go | 15 +++++++-------- bundle/direct/dresources/genie_space.go | 3 ++- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go index 3144bf7b614..fe746cbd98e 100644 --- a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go +++ b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go @@ -56,15 +56,14 @@ func (c configureGenieSpaceSerializedSpace) Apply(_ context.Context, b *bundle.B // Otherwise YAML decodes small ints as Go `int` while state JSON // round-trip decodes them as `float64`, and structdiff reports // false drift on every plan. - switch ss.Kind() { - case dyn.KindMap, dyn.KindSequence: - jsonBytes, err := json.Marshal(ss.AsAny()) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to marshal inline serialized_space: %w", err) - } - return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(jsonBytes))) + if ss.Kind() != dyn.KindMap && ss.Kind() != dyn.KindSequence { + return v, nil + } + jsonBytes, err := json.Marshal(ss.AsAny()) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to marshal inline serialized_space: %w", err) } - return v, nil + return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(jsonBytes))) }) }) diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go index d2569af7a61..e719c55b4ff 100644 --- a/bundle/direct/dresources/genie_space.go +++ b/bundle/direct/dresources/genie_space.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "net/http" "strings" "github.com/databricks/cli/bundle/config/resources" @@ -123,7 +124,7 @@ func isMissingGenieParentPathError(err error) bool { // Some workspaces return a standard missing-resource error, while others // return INVALID_PARAMETER_VALUE with a NOT_FOUND message embedded in the // text. Treat both forms as "create the parent directory and retry once". - return apiErr.StatusCode == 400 && + return apiErr.StatusCode == http.StatusBadRequest && apiErr.ErrorCode == "INVALID_PARAMETER_VALUE" && strings.Contains(apiErr.Message, "Tree node with path") && strings.Contains(apiErr.Message, "does not exist") From cd3f101eaa5854c2dc1fc349f7ce4c3d71cc6f41 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 7 May 2026 11:41:36 +0200 Subject: [PATCH 09/42] bundle: skip genie_space serialized_space in update when unchanged locally serialized_space is in ignore_remote_changes because we cannot diff a structured local YAML body against a remote JSON string. That makes UI edits invisible at plan time, but the unconditional UpdateSpace request was still sending the local body, so any later update to title or description would silently overwrite UI changes. Use the plan entry to detect whether the user actually changed serialized_space locally; only include it in the update request when the change is an Update action (not a Skip from ignore_remote_changes). Co-authored-by: Isaac --- bundle/direct/dresources/genie_space.go | 50 ++++++++++++++- bundle/direct/dresources/genie_space_test.go | 64 ++++++++++++++++++++ 2 files changed, 111 insertions(+), 3 deletions(-) diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go index e719c55b4ff..f8305ffac4d 100644 --- a/bundle/direct/dresources/genie_space.go +++ b/bundle/direct/dresources/genie_space.go @@ -9,12 +9,16 @@ import ( "strings" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/dashboards" ) +var pathSerializedSpace = structpath.MustParsePath("serialized_space") + type ResourceGenieSpace struct { client *databricks.WorkspaceClient } @@ -166,12 +170,23 @@ func (r *ResourceGenieSpace) DoCreate(ctx context.Context, config *resources.Gen return createResp.SpaceId, responseToGenieSpaceConfig(createResp, serializedSpace), nil } -func (r *ResourceGenieSpace) DoUpdate(ctx context.Context, id string, config *resources.GenieSpaceConfig, _ *PlanEntry) (*resources.GenieSpaceConfig, error) { +func (r *ResourceGenieSpace) DoUpdate(ctx context.Context, id string, config *resources.GenieSpaceConfig, entry *PlanEntry) (*resources.GenieSpaceConfig, error) { serializedSpace, err := prepareGenieSpaceRequest(config) if err != nil { return nil, err } + // serialized_space is in ignore_remote_changes (we cannot diff structured + // local YAML against remote JSON), so a UI edit produces no plan entry. + // If we still sent the unchanged local body on every update, the next + // update triggered by another field would clobber the UI edit. Only + // send it when the user actually changed it locally. + excludeForceSend := []string{} + if !hasUpdate(entry, pathSerializedSpace) { + serializedSpace = "" + excludeForceSend = append(excludeForceSend, "SerializedSpace") + } + updateResp, err := r.client.Genie.UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ SpaceId: id, Description: config.Description, @@ -181,13 +196,42 @@ func (r *ResourceGenieSpace) DoUpdate(ctx context.Context, id string, config *re // Etag is for optimistic concurrency; we apply updates unconditionally. Etag: "", - ForceSendFields: utils.FilterFields[dashboards.GenieUpdateSpaceRequest](config.ForceSendFields), + ForceSendFields: utils.FilterFields[dashboards.GenieUpdateSpaceRequest](config.ForceSendFields, excludeForceSend...), }) if err != nil { return nil, err } - return responseToGenieSpaceConfig(updateResp, serializedSpace), nil + // When the request omitted serialized_space, use the value the response + // echoes back so RemapState records the latest body. + respSerialized := serializedSpace + if respSerialized == "" { + respSerialized = updateResp.SerializedSpace + } + + return responseToGenieSpaceConfig(updateResp, respSerialized), nil +} + +// hasUpdate reports whether entry has an Update-action change at the given path. +// HasChange alone matches Skip-action changes too, which we cannot use to drive +// request shaping for fields covered by ignore_remote_changes. +func hasUpdate(entry *PlanEntry, path *structpath.PathNode) bool { + if entry == nil { + return false + } + for s, change := range entry.Changes { + if change.Action != deployplan.Update { + continue + } + node, err := structpath.ParsePath(s) + if err != nil { + continue + } + if node.HasPrefix(path) { + return true + } + } + return false } func (r *ResourceGenieSpace) DoDelete(ctx context.Context, id string) error { diff --git a/bundle/direct/dresources/genie_space_test.go b/bundle/direct/dresources/genie_space_test.go index c731322ca8e..7db9f5c4b16 100644 --- a/bundle/direct/dresources/genie_space_test.go +++ b/bundle/direct/dresources/genie_space_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/dashboards" @@ -103,3 +104,66 @@ func TestGenieSpaceDoCreateRetriesWhenParentPathLooksMissing(t *testing.T) { require.NotNil(t, state) assert.Equal(t, "test genie space", state.Title) } + +func TestGenieSpaceDoUpdateOmitsSerializedSpaceWhenUnchanged(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + // Plan entry indicates only title changed; serialized_space is absent. + entry := &deployplan.PlanEntry{ + Changes: deployplan.Changes{ + "title": {Action: deployplan.Update, Old: "old", New: "new"}, + }, + } + + m.GetMockGenieAPI().EXPECT(). + UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ + SpaceId: "space-id", + Title: "new", + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id", + Title: "new", + SerializedSpace: "{\"remote\":\"edit\"}", + }, nil). + Once() + + state, err := r.DoUpdate(ctx, "space-id", &resources.GenieSpaceConfig{ + Title: "new", + SerializedSpace: "{\"local\":\"stale\"}", + }, entry) + require.NoError(t, err) + require.NotNil(t, state) + assert.Equal(t, "{\"remote\":\"edit\"}", state.SerializedSpace) +} + +func TestGenieSpaceDoUpdateSendsSerializedSpaceWhenChanged(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + entry := &deployplan.PlanEntry{ + Changes: deployplan.Changes{ + "serialized_space": {Action: deployplan.Update, Old: "{}", New: "{\"v\":1}"}, + }, + } + + m.GetMockGenieAPI().EXPECT(). + UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ + SpaceId: "space-id", + SerializedSpace: "{\"v\":1}", + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id", + SerializedSpace: "{\"v\":1}", + }, nil). + Once() + + state, err := r.DoUpdate(ctx, "space-id", &resources.GenieSpaceConfig{ + SerializedSpace: "{\"v\":1}", + }, entry) + require.NoError(t, err) + require.NotNil(t, state) + assert.Equal(t, "{\"v\":1}", state.SerializedSpace) +} From 732641c4306b7eee67dfd8540443bbdfeedfdbf7 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 7 May 2026 11:43:46 +0200 Subject: [PATCH 10/42] bundle: fix watch loop in generate genie-space The previous implementation polled w.Workspace.GetStatusByPath using resource.FilePath, which is the local relative path (e.g. "src/foo.genie.json"). Both lookups (with and without the "/Workspace" prefix) were invalid for the workspace API, so currentModified stayed at 0 and the file never updated past the first iteration. Genie has no remote modification timestamp on the response, so use content comparison instead: canonicalize the just-fetched serialized_space and compare against the on-disk body, re-saving only when they differ. The first iteration still always saves, preserving the prior unconditional initial sync. Co-authored-by: Isaac --- cmd/bundle/generate/genie_space.go | 73 ++++++++++++++++-------------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index c6fb9e0651b..2c2aa9d106c 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -1,6 +1,7 @@ package generate import ( + "bytes" "context" "errors" "fmt" @@ -26,13 +27,14 @@ import ( "github.com/databricks/cli/libs/dyn/yamlsaver" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/textutil" - "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/dashboards" "github.com/spf13/cobra" "go.yaml.in/yaml/v3" ) +const genieSpaceWatchInterval = 1 * time.Second + type genieSpace struct { // Lookup flags for one-time generate. existingPath string @@ -204,22 +206,6 @@ func (g *genieSpace) saveConfiguration(ctx context.Context, b *bundle.Bundle, ge return nil } -func waitForGenieSpaceChanges(ctx context.Context, w *databricks.WorkspaceClient, genieSpacePath string, lastModified int64) { - for { - obj, err := w.Workspace.GetStatusByPath(ctx, genieSpacePath) //nolint:staticcheck // Deprecated in SDK v0.127.0. Migration to WorkspaceHierarchyService tracked separately. - if err != nil { - logdiag.LogError(ctx, err) - return - } - - if obj.ModifiedAt > lastModified { - break - } - - time.Sleep(1 * time.Second) - } -} - func (g *genieSpace) updateGenieSpaceForResource(ctx context.Context, b *bundle.Bundle) { resource, ok := b.Config.Resources.GenieSpaces[g.resource] if !ok { @@ -237,7 +223,7 @@ func (g *genieSpace) updateGenieSpaceForResource(ctx context.Context, b *bundle. w := b.WorkspaceClient(ctx) - var lastModified int64 + first := true for { genieSpace, err := w.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ SpaceId: genieSpaceID, @@ -248,20 +234,22 @@ func (g *genieSpace) updateGenieSpaceForResource(ctx context.Context, b *bundle. return } - // Get workspace status to check modification time. - obj, err := w.Workspace.GetStatusByPath(ctx, "/Workspace"+genieSpacePath) //nolint:staticcheck // Deprecated in SDK v0.127.0. Migration to WorkspaceHierarchyService tracked separately. - if err != nil && !apierr.IsMissing(err) { - obj, err = w.Workspace.GetStatusByPath(ctx, genieSpacePath) //nolint:staticcheck // Deprecated in SDK v0.127.0. Migration to WorkspaceHierarchyService tracked separately. - } - - var currentModified int64 - if err == nil { - currentModified = obj.ModifiedAt + // Genie has no remote modification timestamp we can poll. Compare + // the canonicalized remote body against the on-disk body and only + // re-save when they differ. The first iteration always saves, to + // match the prior behavior of an unconditional initial sync. + shouldSave := first + if !first { + differs, err := genieSpaceBodyDiffersFromDisk(genieSpace.SerializedSpace, genieSpacePath) + if err != nil { + logdiag.LogError(ctx, err) + return + } + shouldSave = differs } - if lastModified == 0 || currentModified > lastModified { - err = g.saveSerializedGenieSpace(ctx, b, genieSpace, genieSpacePath) - if err != nil { + if shouldSave { + if err := g.saveSerializedGenieSpace(ctx, b, genieSpace, genieSpacePath); err != nil { logdiag.LogError(ctx, err) return } @@ -271,14 +259,29 @@ func (g *genieSpace) updateGenieSpaceForResource(ctx context.Context, b *bundle. return } - lastModified = currentModified + first = false + time.Sleep(genieSpaceWatchInterval) + } +} - if obj != nil { - waitForGenieSpaceChanges(ctx, w, obj.Path, lastModified) - } else { - time.Sleep(1 * time.Second) +// genieSpaceBodyDiffersFromDisk reports whether the canonicalized remote +// serialized_space differs from the contents of filename. +func genieSpaceBodyDiffersFromDisk(remoteSerialized, filename string) (bool, error) { + if remoteSerialized == "" { + return false, nil + } + canonical, err := remarshalJSON([]byte(remoteSerialized)) + if err != nil { + return false, err + } + onDisk, err := os.ReadFile(filename) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return true, nil } + return false, err } + return !bytes.Equal(canonical, onDisk), nil } func (g *genieSpace) generateForExisting(ctx context.Context, b *bundle.Bundle, genieSpaceID string) { From 41f36a99648201eb00be224ee994672f3aec5de3 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 7 May 2026 11:44:40 +0200 Subject: [PATCH 11/42] bundle: honor --key flag in generate genie-space The parent generate command exposes --key as a persistent flag, but the genie-space subcommand was always deriving the key from the remote title. Read the flag value and fall back to the title-derived key only when not provided. Co-authored-by: Isaac --- cmd/bundle/generate/genie_space.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 2c2aa9d106c..50242169cf9 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -295,7 +295,10 @@ func (g *genieSpace) generateForExisting(ctx context.Context, b *bundle.Bundle, return } - key := textutil.NormalizeString(genieSpace.Title) + key := g.cmd.Flag("key").Value.String() + if key == "" { + key = textutil.NormalizeString(genieSpace.Title) + } err = g.saveConfiguration(ctx, b, genieSpace, key) if err != nil { logdiag.LogError(ctx, err) From 0b27cc427d706caed98f5286468f8717fdd5f81c Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 7 May 2026 11:45:31 +0200 Subject: [PATCH 12/42] bundle: guard empty serialized_space in generate genie-space Calling json.Unmarshal on an empty serialized_space surfaces a confusing "unexpected end of JSON input" error and writes nothing useful. Bail out early with a clear message that names the target file. Co-authored-by: Isaac --- cmd/bundle/generate/genie_space.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 50242169cf9..8b74ce2a057 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -121,6 +121,10 @@ func (g *genieSpace) resolveFromID(ctx context.Context, b *bundle.Bundle) string } func (g *genieSpace) saveSerializedGenieSpace(ctx context.Context, b *bundle.Bundle, genieSpace *dashboards.GenieSpace, filename string) error { + if genieSpace.SerializedSpace == "" { + return fmt.Errorf("Genie space response did not include serialized_space; refusing to write %s", filepath.ToSlash(filename)) + } + // Unmarshal and remarshal the serialized genie space to ensure it is formatted correctly. // The result will have alphabetically sorted keys and be indented. data, err := remarshalJSON([]byte(genieSpace.SerializedSpace)) From 2ff9273ae9d4302bcfc128bd1cdf4d1f69c24ce0 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 7 May 2026 11:46:22 +0200 Subject: [PATCH 13/42] bundle: collapse DoRead into responseToGenieSpaceConfig for genie_space DoRead duplicated the field copy and the ParentPath-drop comment that already lives in responseToGenieSpaceConfig. Reuse the helper directly so the two stay in sync. Co-authored-by: Isaac --- bundle/direct/dresources/genie_space.go | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go index f8305ffac4d..08ce1b95a94 100644 --- a/bundle/direct/dresources/genie_space.go +++ b/bundle/direct/dresources/genie_space.go @@ -60,24 +60,7 @@ func (r *ResourceGenieSpace) DoRead(ctx context.Context, id string) (*resources. if err != nil { return nil, err } - - // Drop ParentPath from ForceSendFields. We always clear ParentPath - // below because the GET Genie space API does not reliably return it, - // and keeping it in ForceSendFields would force-emit parent_path: "" - // in state output even though the field is logically unset. - forceSendFields := utils.FilterFields[resources.GenieSpaceConfig](space.ForceSendFields, "ParentPath") - - return &resources.GenieSpaceConfig{ - Description: space.Description, - Title: space.Title, - WarehouseId: space.WarehouseId, - ParentPath: "", - SerializedSpace: space.SerializedSpace, - - // Output only fields - SpaceId: space.SpaceId, - ForceSendFields: forceSendFields, - }, nil + return responseToGenieSpaceConfig(space, space.SerializedSpace), nil } func prepareGenieSpaceRequest(config *resources.GenieSpaceConfig) (string, error) { From 04ec5fa4b55610e4f45f93fd8633facf94b9ad3e Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 7 May 2026 11:47:45 +0200 Subject: [PATCH 14/42] bundle: fill genie_space schema annotations The user-facing fields (title, description, warehouse_id, parent_path, file_path, serialized_space) had PLACEHOLDER descriptions, leaving the generated reference and resources docs blank. Fill them in with short descriptions and regenerate the schema and docs output. Co-authored-by: Isaac --- bundle/docsgen/output/reference.md | 24 ++++++++++++------------ bundle/docsgen/output/resources.md | 12 ++++++------ bundle/internal/schema/annotations.yml | 12 ++++++------ bundle/schema/jsonschema.json | 6 ++++++ 4 files changed, 30 insertions(+), 24 deletions(-) diff --git a/bundle/docsgen/output/reference.md b/bundle/docsgen/output/reference.md index 5f8df77dfe9..3d89c58e8e8 100644 --- a/bundle/docsgen/output/reference.md +++ b/bundle/docsgen/output/reference.md @@ -1503,11 +1503,11 @@ genie_spaces: - - `description` - String - - + - Description of the Genie space shown alongside the title in the Databricks UI. - - `file_path` - String - - + - Local path to a `.genie.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. - - `lifecycle` - Map @@ -1515,7 +1515,7 @@ genie_spaces: - - `parent_path` - String - - + - Workspace folder under which to create the Genie space. Immutable: changing this field recreates the resource. - - `permissions` - Sequence @@ -1523,7 +1523,7 @@ genie_spaces: - - `serialized_space` - Any - - + - Serialized Genie space body. May be provided inline as a JSON string (or YAML that will be marshalled to JSON) or referenced via `file_path`. To round-trip an existing space into a bundle, use `databricks bundle generate genie-space`. - - `space_id` - String @@ -1531,11 +1531,11 @@ genie_spaces: - - `title` - String - - + - Title of the Genie space shown in the Databricks UI. - - `warehouse_id` - String - - + - ID of the SQL warehouse used to run queries for this Genie space. ::: @@ -3697,11 +3697,11 @@ genie_spaces: - - `description` - String - - + - Description of the Genie space shown alongside the title in the Databricks UI. - - `file_path` - String - - + - Local path to a `.genie.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. - - `lifecycle` - Map @@ -3709,7 +3709,7 @@ genie_spaces: - - `parent_path` - String - - + - Workspace folder under which to create the Genie space. Immutable: changing this field recreates the resource. - - `permissions` - Sequence @@ -3717,7 +3717,7 @@ genie_spaces: - - `serialized_space` - Any - - + - Serialized Genie space body. May be provided inline as a JSON string (or YAML that will be marshalled to JSON) or referenced via `file_path`. To round-trip an existing space into a bundle, use `databricks bundle generate genie-space`. - - `space_id` - String @@ -3725,11 +3725,11 @@ genie_spaces: - - `title` - String - - + - Title of the Genie space shown in the Databricks UI. - - `warehouse_id` - String - - + - ID of the SQL warehouse used to run queries for this Genie space. ::: diff --git a/bundle/docsgen/output/resources.md b/bundle/docsgen/output/resources.md index 95ece375b97..484064dd36e 100644 --- a/bundle/docsgen/output/resources.md +++ b/bundle/docsgen/output/resources.md @@ -3107,11 +3107,11 @@ genie_spaces: - - `description` - String - - + - Description of the Genie space shown alongside the title in the Databricks UI. - - `file_path` - String - - + - Local path to a `.genie.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. - - `lifecycle` - Map @@ -3119,7 +3119,7 @@ genie_spaces: - - `parent_path` - String - - + - Workspace folder under which to create the Genie space. Immutable: changing this field recreates the resource. - - `permissions` - Sequence @@ -3127,7 +3127,7 @@ genie_spaces: - - `serialized_space` - Any - - + - Serialized Genie space body. May be provided inline as a JSON string (or YAML that will be marshalled to JSON) or referenced via `file_path`. To round-trip an existing space into a bundle, use `databricks bundle generate genie-space`. - - `space_id` - String @@ -3135,11 +3135,11 @@ genie_spaces: - - `title` - String - - + - Title of the Genie space shown in the Databricks UI. - - `warehouse_id` - String - - + - ID of the SQL warehouse used to run queries for this Genie space. ::: diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index fd9e3b6e78f..6194cf078e0 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -674,31 +674,31 @@ github.com/databricks/cli/bundle/config/resources.ExternalLocation: github.com/databricks/cli/bundle/config/resources.GenieSpace: "description": "description": |- - PLACEHOLDER + Description of the Genie space shown alongside the title in the Databricks UI. "file_path": "description": |- - PLACEHOLDER + Local path to a `.genie.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. "lifecycle": "description": |- PLACEHOLDER "parent_path": "description": |- - PLACEHOLDER + Workspace folder under which to create the Genie space. Immutable: changing this field recreates the resource. "permissions": "description": |- PLACEHOLDER "serialized_space": "description": |- - PLACEHOLDER + Serialized Genie space body. May be provided inline as a JSON string (or YAML that will be marshalled to JSON) or referenced via `file_path`. To round-trip an existing space into a bundle, use `databricks bundle generate genie-space`. "space_id": "description": |- PLACEHOLDER "title": "description": |- - PLACEHOLDER + Title of the Genie space shown in the Databricks UI. "warehouse_id": "description": |- - PLACEHOLDER + ID of the SQL warehouse used to run queries for this Genie space. github.com/databricks/cli/bundle/config/resources.JobPermission: "group_name": "description": |- diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 9c229c20a2d..4ba9eb7c43c 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -766,30 +766,36 @@ "type": "object", "properties": { "description": { + "description": "Description of the Genie space shown alongside the title in the Databricks UI.", "$ref": "#/$defs/string" }, "file_path": { + "description": "Local path to a `.genie.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`.", "$ref": "#/$defs/string" }, "lifecycle": { "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" }, "parent_path": { + "description": "Workspace folder under which to create the Genie space. Immutable: changing this field recreates the resource.", "$ref": "#/$defs/string" }, "permissions": { "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" }, "serialized_space": { + "description": "Serialized Genie space body. May be provided inline as a JSON string (or YAML that will be marshalled to JSON) or referenced via `file_path`. To round-trip an existing space into a bundle, use `databricks bundle generate genie-space`.", "$ref": "#/$defs/interface" }, "space_id": { "$ref": "#/$defs/string" }, "title": { + "description": "Title of the Genie space shown in the Databricks UI.", "$ref": "#/$defs/string" }, "warehouse_id": { + "description": "ID of the SQL warehouse used to run queries for this Genie space.", "$ref": "#/$defs/string" } }, From 3303b8d29fa575ea02e44e017b378ce6fdfbbd31 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 7 May 2026 11:48:55 +0200 Subject: [PATCH 15/42] bundle: lint cleanups for genie_space review fixes Lowercase the genie_space error message to satisfy ST1005 and let the linter convert an empty []string{} to a nil slice. Co-authored-by: Isaac --- bundle/direct/dresources/genie_space.go | 2 +- cmd/bundle/generate/genie_space.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go index 08ce1b95a94..1eaa1b1995c 100644 --- a/bundle/direct/dresources/genie_space.go +++ b/bundle/direct/dresources/genie_space.go @@ -164,7 +164,7 @@ func (r *ResourceGenieSpace) DoUpdate(ctx context.Context, id string, config *re // If we still sent the unchanged local body on every update, the next // update triggered by another field would clobber the UI edit. Only // send it when the user actually changed it locally. - excludeForceSend := []string{} + var excludeForceSend []string if !hasUpdate(entry, pathSerializedSpace) { serializedSpace = "" excludeForceSend = append(excludeForceSend, "SerializedSpace") diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 8b74ce2a057..f4c344667e7 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -122,7 +122,7 @@ func (g *genieSpace) resolveFromID(ctx context.Context, b *bundle.Bundle) string func (g *genieSpace) saveSerializedGenieSpace(ctx context.Context, b *bundle.Bundle, genieSpace *dashboards.GenieSpace, filename string) error { if genieSpace.SerializedSpace == "" { - return fmt.Errorf("Genie space response did not include serialized_space; refusing to write %s", filepath.ToSlash(filename)) + return fmt.Errorf("genie space response did not include serialized_space; refusing to write %s", filepath.ToSlash(filename)) } // Unmarshal and remarshal the serialized genie space to ensure it is formatted correctly. From b1f759541a76b8a9cea702a7836b38f42ffc3c25 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Mon, 18 May 2026 15:02:52 +0200 Subject: [PATCH 16/42] Regenerate test toml --- acceptance/bundle/generate/genie_space/out.test.toml | 4 +--- .../bundle/resources/genie_spaces/inline/out.test.toml | 6 ++---- .../bundle/resources/genie_spaces/simple/out.test.toml | 6 ++---- .../bundle/validate/genie_space_complex/out.test.toml | 4 +--- .../bundle/validate/genie_space_defaults/out.test.toml | 4 +--- .../validate/genie_space_file_path_and_inline/out.test.toml | 4 +--- .../genie_space_permissions_unsupported/out.test.toml | 4 +--- 7 files changed, 9 insertions(+), 23 deletions(-) diff --git a/acceptance/bundle/generate/genie_space/out.test.toml b/acceptance/bundle/generate/genie_space/out.test.toml index d560f1de043..f784a183258 100644 --- a/acceptance/bundle/generate/genie_space/out.test.toml +++ b/acceptance/bundle/generate/genie_space/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false - -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.test.toml b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml index 496668716fa..03d89cd7d57 100644 --- a/acceptance/bundle/resources/genie_spaces/inline/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false - -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["direct"] - Ignore = ["databricks.yml"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.Ignore = ["databricks.yml"] diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.test.toml b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml index 496668716fa..03d89cd7d57 100644 --- a/acceptance/bundle/resources/genie_spaces/simple/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false - -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["direct"] - Ignore = ["databricks.yml"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.Ignore = ["databricks.yml"] diff --git a/acceptance/bundle/validate/genie_space_complex/out.test.toml b/acceptance/bundle/validate/genie_space_complex/out.test.toml index d560f1de043..f784a183258 100644 --- a/acceptance/bundle/validate/genie_space_complex/out.test.toml +++ b/acceptance/bundle/validate/genie_space_complex/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false - -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_defaults/out.test.toml b/acceptance/bundle/validate/genie_space_defaults/out.test.toml index d560f1de043..f784a183258 100644 --- a/acceptance/bundle/validate/genie_space_defaults/out.test.toml +++ b/acceptance/bundle/validate/genie_space_defaults/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false - -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml index 54146af5645..e90b6d5d1ba 100644 --- a/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false - -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/genie_space_permissions_unsupported/out.test.toml b/acceptance/bundle/validate/genie_space_permissions_unsupported/out.test.toml index 54146af5645..e90b6d5d1ba 100644 --- a/acceptance/bundle/validate/genie_space_permissions_unsupported/out.test.toml +++ b/acceptance/bundle/validate/genie_space_permissions_unsupported/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false - -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] From 98912296f6970c5062fb8bd2d1c70173ba936fab Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 20 May 2026 12:50:24 +0200 Subject: [PATCH 17/42] Agent review --- ....json => test_genie_space.geniespace.json} | 0 .../resource/test_genie_space.genie_space.yml | 2 +- .../bundle/generate/genie_space/output.txt | 2 +- .../genie_spaces/inline/out.plan.json | 7 + .../genie_spaces/inline/out.test.toml | 1 - .../resources/genie_spaces/inline/test.toml | 4 - .../parent_path_recreate/databricks.yml.tmpl | 10 + .../parent_path_recreate}/out.test.toml | 0 .../parent_path_recreate/output.txt | 21 + .../genie_spaces/parent_path_recreate/script | 14 + .../parent_path_recreate/test.toml | 6 + .../genie_spaces/simple/databricks.yml.tmpl | 2 +- .../genie_spaces/simple/out.plan.json | 7 + .../genie_spaces/simple/out.test.toml | 1 - ...e.json => sales_analytics.geniespace.json} | 0 .../resources/genie_spaces/simple/test.toml | 4 - .../bundle/resources/genie_spaces/test.toml | 5 +- .../current_can_manage/databricks.yml | 19 + .../current_can_manage/out.plan.direct.json | 52 ++ .../out.requests.deploy.direct.json | 24 + .../out.requests.destroy.direct.json | 0 .../current_can_manage/out.test.toml | 3 + .../current_can_manage/output.txt | 35 + .../genie_spaces/current_can_manage/script | 18 + .../permissions/genie_spaces/test.toml | 4 + .../genie_space_complex/databricks.yml | 2 +- ...nie.json => full_featured.geniespace.json} | 0 ...ts.genie.json => contents.geniespace.json} | 0 .../databricks.yml | 2 +- .../out.test.toml | 2 +- .../test.toml | 4 +- .../databricks.yml | 21 - .../output.txt | 22 - .../script | 1 - .../test.toml | 8 - .../configure_genie_space_serialized_space.go | 21 +- .../translate_paths_genie_spaces_test.go | 9 +- .../validate_genie_space_permissions.go | 44 -- .../validate_genie_space_permissions_test.go | 54 -- bundle/config/resources/genie_space.go | 8 +- bundle/direct/dresources/all_test.go | 9 + bundle/direct/dresources/genie_space.go | 93 ++- bundle/direct/dresources/genie_space_test.go | 91 +++ bundle/direct/dstate/state.go | 11 +- bundle/docsgen/output/reference.md | 14 +- bundle/docsgen/output/resources.md | 611 ++---------------- bundle/internal/schema/annotations.yml | 5 +- bundle/phases/plan.go | 1 - bundle/schema/jsonschema.json | 5 +- cmd/bundle/generate/dashboard.go | 6 +- cmd/bundle/generate/genie_space.go | 11 +- cmd/bundle/generate/genie_space_test.go | 126 ++++ libs/testserver/genie_spaces.go | 52 +- libs/workspaceurls/urls_test.go | 1 + 54 files changed, 712 insertions(+), 763 deletions(-) rename acceptance/bundle/generate/genie_space/out/genie_space/{test_genie_space.genie.json => test_genie_space.geniespace.json} (100%) create mode 100644 acceptance/bundle/resources/genie_spaces/parent_path_recreate/databricks.yml.tmpl rename acceptance/bundle/{validate/genie_space_permissions_unsupported => resources/genie_spaces/parent_path_recreate}/out.test.toml (100%) create mode 100644 acceptance/bundle/resources/genie_spaces/parent_path_recreate/output.txt create mode 100644 acceptance/bundle/resources/genie_spaces/parent_path_recreate/script create mode 100644 acceptance/bundle/resources/genie_spaces/parent_path_recreate/test.toml rename acceptance/bundle/resources/genie_spaces/simple/{sales_analytics.genie.json => sales_analytics.geniespace.json} (100%) create mode 100644 acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/databricks.yml create mode 100644 acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json create mode 100644 acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.deploy.direct.json create mode 100644 acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.destroy.direct.json create mode 100644 acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml create mode 100644 acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/output.txt create mode 100644 acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script create mode 100644 acceptance/bundle/resources/permissions/genie_spaces/test.toml rename acceptance/bundle/validate/genie_space_complex/{full_featured.genie.json => full_featured.geniespace.json} (100%) rename acceptance/bundle/validate/genie_space_file_path_and_inline/{contents.genie.json => contents.geniespace.json} (100%) delete mode 100644 acceptance/bundle/validate/genie_space_permissions_unsupported/databricks.yml delete mode 100644 acceptance/bundle/validate/genie_space_permissions_unsupported/output.txt delete mode 100644 acceptance/bundle/validate/genie_space_permissions_unsupported/script delete mode 100644 acceptance/bundle/validate/genie_space_permissions_unsupported/test.toml delete mode 100644 bundle/config/mutator/validate_genie_space_permissions.go delete mode 100644 bundle/config/mutator/validate_genie_space_permissions_test.go create mode 100644 cmd/bundle/generate/genie_space_test.go diff --git a/acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.genie.json b/acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.geniespace.json similarity index 100% rename from acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.genie.json rename to acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.geniespace.json diff --git a/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml b/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml index a73945364ce..1471a901344 100644 --- a/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml +++ b/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml @@ -3,6 +3,6 @@ resources: test_genie_space: title: "test genie space" warehouse_id: test-warehouse-id - file_path: ../genie_space/test_genie_space.genie.json + file_path: ../genie_space/test_genie_space.geniespace.json description: test description parent_path: /Workspace/test-[UNIQUE_NAME] diff --git a/acceptance/bundle/generate/genie_space/output.txt b/acceptance/bundle/generate/genie_space/output.txt index 985366ada68..a313a51adc3 100644 --- a/acceptance/bundle/generate/genie_space/output.txt +++ b/acceptance/bundle/generate/genie_space/output.txt @@ -2,5 +2,5 @@ >>> [CLI] workspace mkdirs /Workspace/test-[UNIQUE_NAME] >>> [CLI] bundle generate genie-space --existing-id [GENIE_SPACE_ID] --genie-space-dir out/genie_space --resource-dir out/resource -Writing genie space to out/genie_space/test_genie_space.genie.json +Writing genie space to out/genie_space/test_genie_space.geniespace.json Writing configuration to out/resource/test_genie_space.genie_space.yml diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.plan.json b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json index 41f6aa983de..8d6f7aabc78 100644 --- a/acceptance/bundle/resources/genie_spaces/inline/out.plan.json +++ b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json @@ -8,12 +8,19 @@ "action": "skip", "remote_state": { "description": "Inline serialized_space test", + "etag": "1", "serialized_space": "{\"config\":{\"sample_questions\":[{\"id\":\"sq-001\",\"question\":[\"What is the total revenue?\"]}]},\"data_sources\":{\"tables\":[{\"column_configs\":[{\"column_name\":\"amount\",\"get_example_values\":true}],\"identifier\":\"main.sales.orders\"}]},\"version\":1}", "space_id": "[GENIE_SPACE_ID]", "title": "Sales Analytics Inline Genie", "warehouse_id": "test-warehouse-id" }, "changes": { + "etag": { + "action": "skip", + "reason": "custom", + "old": "1", + "remote": "1" + }, "parent_path": { "action": "skip", "reason": "input_only", diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.test.toml b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml index 03d89cd7d57..e90b6d5d1ba 100644 --- a/acceptance/bundle/resources/genie_spaces/inline/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml @@ -1,4 +1,3 @@ Local = true Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.Ignore = ["databricks.yml"] diff --git a/acceptance/bundle/resources/genie_spaces/inline/test.toml b/acceptance/bundle/resources/genie_spaces/inline/test.toml index 18a07a1b00b..bbdf2380b2d 100644 --- a/acceptance/bundle/resources/genie_spaces/inline/test.toml +++ b/acceptance/bundle/resources/genie_spaces/inline/test.toml @@ -1,10 +1,6 @@ Local = true RecordRequests = false -# Genie spaces only support direct deployment engine (no Terraform provider yet) -[EnvMatrix] -DATABRICKS_BUNDLE_ENGINE = ["direct"] - Ignore = [ "databricks.yml", ] diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_recreate/databricks.yml.tmpl b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/databricks.yml.tmpl new file mode 100644 index 00000000000..1eea5d933e3 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/databricks.yml.tmpl @@ -0,0 +1,10 @@ +bundle: + name: recreate-genie-space-$UNIQUE_NAME + +resources: + genie_spaces: + recreate_target: + title: "Recreate Target" + warehouse_id: "test-warehouse-id" + parent_path: PARENT_PATH_PLACEHOLDER + serialized_space: "{}" diff --git a/acceptance/bundle/validate/genie_space_permissions_unsupported/out.test.toml b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/out.test.toml similarity index 100% rename from acceptance/bundle/validate/genie_space_permissions_unsupported/out.test.toml rename to acceptance/bundle/resources/genie_spaces/parent_path_recreate/out.test.toml diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_recreate/output.txt b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/output.txt new file mode 100644 index 00000000000..e90257bf971 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/output.txt @@ -0,0 +1,21 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-genie-space-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Plan after changing parent_path should show recreate +>>> [CLI] bundle plan +recreate genie_spaces.recreate_target + +Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.genie_spaces.recreate_target + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/recreate-genie-space-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_recreate/script b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/script new file mode 100644 index 00000000000..d5f04a795ab --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/script @@ -0,0 +1,14 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +# Deploy with the original parent_path. +envsubst < databricks.yml.tmpl | sed "s|PARENT_PATH_PLACEHOLDER|/Users/$CURRENT_USER_NAME/genie-old|" > databricks.yml +trace $CLI bundle deploy + +# Change parent_path. parent_path is recreate_on_changes in resources.yml, +# so the plan should show a recreate (delete + create) rather than an update. +envsubst < databricks.yml.tmpl | sed "s|PARENT_PATH_PLACEHOLDER|/Users/$CURRENT_USER_NAME/genie-new|" > databricks.yml +title "Plan after changing parent_path should show recreate" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_recreate/test.toml b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/test.toml new file mode 100644 index 00000000000..bbdf2380b2d --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/test.toml @@ -0,0 +1,6 @@ +Local = true +RecordRequests = false + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/bundle/resources/genie_spaces/simple/databricks.yml.tmpl b/acceptance/bundle/resources/genie_spaces/simple/databricks.yml.tmpl index 26ed410751a..f30e4991de9 100644 --- a/acceptance/bundle/resources/genie_spaces/simple/databricks.yml.tmpl +++ b/acceptance/bundle/resources/genie_spaces/simple/databricks.yml.tmpl @@ -8,4 +8,4 @@ resources: description: "AI assistant for sales data analysis" warehouse_id: "test-warehouse-id" parent_path: /Users/$CURRENT_USER_NAME - file_path: "sales_analytics.genie.json" + file_path: "sales_analytics.geniespace.json" diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.plan.json b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json index 19f063bb002..b5583b47c06 100644 --- a/acceptance/bundle/resources/genie_spaces/simple/out.plan.json +++ b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json @@ -8,12 +8,19 @@ "action": "skip", "remote_state": { "description": "AI assistant for sales data analysis", + "etag": "1", "serialized_space": "{\n \"version\": 1,\n \"config\": {\n \"sample_questions\": [\n {\n \"id\": \"sq-001\",\n \"question\": [\"What is the total revenue?\"]\n },\n {\n \"id\": \"sq-002\",\n \"question\": [\"Show me the top customers\"]\n }\n ]\n },\n \"data_sources\": {\n \"tables\": [\n {\n \"identifier\": \"main.sales.orders\",\n \"column_configs\": [\n {\n \"column_name\": \"order_id\",\n \"get_example_values\": true,\n \"build_value_dictionary\": true\n },\n {\n \"column_name\": \"customer_id\",\n \"get_example_values\": true\n },\n {\n \"column_name\": \"amount\",\n \"get_example_values\": false\n }\n ]\n },\n {\n \"identifier\": \"main.sales.customers\"\n }\n ]\n },\n \"instructions\": {\n \"text_instructions\": [\n {\n \"id\": \"ti-001\",\n \"content\": [\n \"This genie space analyzes sales data.\\n\",\n \"Always filter by date when querying orders.\\n\",\n \"Use customer_name instead of customer_id in results.\"\n ]\n }\n ],\n \"example_question_sqls\": [\n {\n \"id\": \"eq-001\",\n \"question\": [\"What are the top customers by revenue?\"],\n \"sql\": [\n \"SELECT\\n\",\n \" c.customer_name,\\n\",\n \" SUM(o.amount) AS total_revenue\\n\",\n \"FROM main.sales.orders o\\n\",\n \"JOIN main.sales.customers c ON o.customer_id = c.id\\n\",\n \"WHERE o.order_date \u003e= :start_date\\n\",\n \"GROUP BY c.customer_name\\n\",\n \"ORDER BY total_revenue DESC\\n\",\n \"LIMIT :limit\"\n ],\n \"parameters\": [\n {\n \"name\": \"start_date\",\n \"type_hint\": \"STRING\",\n \"description\": [\"Start date for the analysis period\"],\n \"default_value\": {\n \"values\": [\"2024-01-01\"]\n }\n },\n {\n \"name\": \"limit\",\n \"type_hint\": \"INTEGER\",\n \"description\": [\"Number of customers to return\"],\n \"default_value\": {\n \"values\": [\"10\"]\n }\n }\n ]\n },\n {\n \"id\": \"eq-002\",\n \"question\": [\"Calculate daily revenue\"],\n \"sql\": [\n \"SELECT\\n\",\n \" order_date,\\n\",\n \" SUM(amount) AS daily_revenue\\n\",\n \"FROM main.sales.orders\\n\",\n \"GROUP BY order_date\\n\",\n \"ORDER BY order_date\"\n ]\n }\n ],\n \"sql_snippets\": {\n \"measures\": [\n {\n \"id\": \"m-001\",\n \"alias\": \"total_revenue\",\n \"sql\": [\"SUM(orders.amount)\"],\n \"display_name\": \"Total Revenue\"\n }\n ]\n },\n \"sql_functions\": [\n {\n \"id\": \"sf-001\",\n \"identifier\": \"main.analytics.calculate_churn\"\n }\n ]\n },\n \"benchmarks\": {\n \"questions\": [\n {\n \"id\": \"bq-001\",\n \"question\": [\"What is the monthly revenue trend?\"],\n \"answer\": [\n {\n \"format\": \"SQL\",\n \"content\": [\n \"SELECT\\n\",\n \" DATE_TRUNC('month', order_date) AS month,\\n\",\n \" SUM(amount) AS revenue\\n\",\n \"FROM main.sales.orders\\n\",\n \"GROUP BY 1\\n\",\n \"ORDER BY 1\"\n ]\n }\n ]\n }\n ]\n }\n}\n", "space_id": "[GENIE_SPACE_ID]", "title": "Sales Analytics Genie", "warehouse_id": "test-warehouse-id" }, "changes": { + "etag": { + "action": "skip", + "reason": "custom", + "old": "1", + "remote": "1" + }, "parent_path": { "action": "skip", "reason": "input_only", diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.test.toml b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml index 03d89cd7d57..e90b6d5d1ba 100644 --- a/acceptance/bundle/resources/genie_spaces/simple/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml @@ -1,4 +1,3 @@ Local = true Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.Ignore = ["databricks.yml"] diff --git a/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.genie.json b/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.geniespace.json similarity index 100% rename from acceptance/bundle/resources/genie_spaces/simple/sales_analytics.genie.json rename to acceptance/bundle/resources/genie_spaces/simple/sales_analytics.geniespace.json diff --git a/acceptance/bundle/resources/genie_spaces/simple/test.toml b/acceptance/bundle/resources/genie_spaces/simple/test.toml index 18a07a1b00b..bbdf2380b2d 100644 --- a/acceptance/bundle/resources/genie_spaces/simple/test.toml +++ b/acceptance/bundle/resources/genie_spaces/simple/test.toml @@ -1,10 +1,6 @@ Local = true RecordRequests = false -# Genie spaces only support direct deployment engine (no Terraform provider yet) -[EnvMatrix] -DATABRICKS_BUNDLE_ENGINE = ["direct"] - Ignore = [ "databricks.yml", ] diff --git a/acceptance/bundle/resources/genie_spaces/test.toml b/acceptance/bundle/resources/genie_spaces/test.toml index 2569ef7dcb1..7f397c47833 100644 --- a/acceptance/bundle/resources/genie_spaces/test.toml +++ b/acceptance/bundle/resources/genie_spaces/test.toml @@ -1,3 +1,2 @@ -# Genie spaces are only deployed via direct deployment engine -[Env] -DATABRICKS_BUNDLE_ENGINE = "direct" +# Genie spaces are only deployed via the direct deployment engine. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/databricks.yml b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/databricks.yml new file mode 100644 index 00000000000..d9e1c56c35c --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: test-bundle + +resources: + genie_spaces: + foo: + title: "Permissions Test Space" + warehouse_id: test-warehouse-id + parent_path: /Workspace/Users/tester@databricks.com + serialized_space: "{}" + permissions: + - level: CAN_READ + user_name: viewer@example.com + - level: CAN_MANAGE + group_name: data-team + - level: CAN_MANAGE + service_principal_name: f37d18cd-98a8-4db5-8112-12dd0a6bfe38 + - level: CAN_MANAGE + user_name: tester@databricks.com diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json new file mode 100644 index 00000000000..dba8cde6d93 --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json @@ -0,0 +1,52 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.genie_spaces.foo": { + "action": "create", + "new_state": { + "value": { + "parent_path": "/Workspace/Users/[USERNAME]", + "serialized_space": "{}", + "title": "Permissions Test Space", + "warehouse_id": "test-warehouse-id" + } + } + }, + "resources.genie_spaces.foo.permissions": { + "depends_on": [ + { + "node": "resources.genie_spaces.foo", + "label": "${resources.genie_spaces.foo.id}" + } + ], + "action": "create", + "new_state": { + "value": { + "object_id": "", + "__embed__": [ + { + "level": "CAN_READ", + "user_name": "viewer@example.com" + }, + { + "level": "CAN_MANAGE", + "group_name": "data-team" + }, + { + "level": "CAN_MANAGE", + "service_principal_name": "[UUID]" + }, + { + "level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] + }, + "vars": { + "object_id": "/genie/spaces/${resources.genie_spaces.foo.id}" + } + } + } + } +} diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.deploy.direct.json new file mode 100644 index 00000000000..c112ccb7e76 --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.deploy.direct.json @@ -0,0 +1,24 @@ +{ + "method": "PUT", + "path": "/api/2.0/permissions/genie/spaces/[FOO_ID]", + "body": { + "access_control_list": [ + { + "permission_level": "CAN_READ", + "user_name": "viewer@example.com" + }, + { + "group_name": "data-team", + "permission_level": "CAN_MANAGE" + }, + { + "permission_level": "CAN_MANAGE", + "service_principal_name": "[UUID]" + }, + { + "permission_level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] + } +} diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.destroy.direct.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/output.txt b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/output.txt new file mode 100644 index 00000000000..e4e8cf0189d --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/output.txt @@ -0,0 +1,35 @@ + +>>> [CLI] bundle validate -o json +[ + { + "level": "CAN_READ", + "user_name": "viewer@example.com" + }, + { + "group_name": "data-team", + "level": "CAN_MANAGE" + }, + { + "level": "CAN_MANAGE", + "service_principal_name": "[UUID]" + }, + { + "level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } +] + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.genie_spaces.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script new file mode 100644 index 00000000000..1b20af07543 --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script @@ -0,0 +1,18 @@ +trace $CLI bundle validate -o json | jq .resources.genie_spaces.foo.permissions +rm out.requests.txt + +$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json + +print_requests() { + jq -c < out.requests.txt | jq 'select(.method != "GET" and (.path | contains("permissions")))' + rm out.requests.txt +} + +rm out.requests.txt +trace $CLI bundle deploy +# Genie space IDs are random; normalize them in the recorded requests. +replace_ids.py +print_requests > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json + +trace $CLI bundle destroy --auto-approve +print_requests > out.requests.destroy.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/permissions/genie_spaces/test.toml b/acceptance/bundle/resources/permissions/genie_spaces/test.toml new file mode 100644 index 00000000000..390388f04aa --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/test.toml @@ -0,0 +1,4 @@ +Env.RESOURCE = "genie_spaces" # for ../_script + +# Genie spaces only support the direct deployment engine. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/genie_space_complex/databricks.yml b/acceptance/bundle/validate/genie_space_complex/databricks.yml index eae5cdc3f10..94c5fc7188c 100644 --- a/acceptance/bundle/validate/genie_space_complex/databricks.yml +++ b/acceptance/bundle/validate/genie_space_complex/databricks.yml @@ -11,7 +11,7 @@ resources: warehouse_id: "my-warehouse-1234" title: "Full Featured Genie Space" description: "A comprehensive test of all genie space features" - file_path: ./full_featured.genie.json + file_path: ./full_featured.geniespace.json # Test with inline serialized_space (YAML syntax) inline_yaml: diff --git a/acceptance/bundle/validate/genie_space_complex/full_featured.genie.json b/acceptance/bundle/validate/genie_space_complex/full_featured.geniespace.json similarity index 100% rename from acceptance/bundle/validate/genie_space_complex/full_featured.genie.json rename to acceptance/bundle/validate/genie_space_complex/full_featured.geniespace.json diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/contents.genie.json b/acceptance/bundle/validate/genie_space_file_path_and_inline/contents.geniespace.json similarity index 100% rename from acceptance/bundle/validate/genie_space_file_path_and_inline/contents.genie.json rename to acceptance/bundle/validate/genie_space_file_path_and_inline/contents.geniespace.json diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/databricks.yml b/acceptance/bundle/validate/genie_space_file_path_and_inline/databricks.yml index dea909bf2f9..ca57e978d83 100644 --- a/acceptance/bundle/validate/genie_space_file_path_and_inline/databricks.yml +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/databricks.yml @@ -6,6 +6,6 @@ resources: both_set: title: "Both set" warehouse_id: "test-warehouse-id" - file_path: "./contents.genie.json" + file_path: "./contents.geniespace.json" serialized_space: version: 1 diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml index e90b6d5d1ba..f784a183258 100644 --- a/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/test.toml b/acceptance/bundle/validate/genie_space_file_path_and_inline/test.toml index 4fc5284af50..97900adac7a 100644 --- a/acceptance/bundle/validate/genie_space_file_path_and_inline/test.toml +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/test.toml @@ -2,5 +2,5 @@ Local = true Cloud = false # Genie spaces only support direct deployment engine. -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["direct"] +[Env] +DATABRICKS_BUNDLE_ENGINE = "direct" diff --git a/acceptance/bundle/validate/genie_space_permissions_unsupported/databricks.yml b/acceptance/bundle/validate/genie_space_permissions_unsupported/databricks.yml deleted file mode 100644 index 71126ef7dcb..00000000000 --- a/acceptance/bundle/validate/genie_space_permissions_unsupported/databricks.yml +++ /dev/null @@ -1,21 +0,0 @@ -bundle: - name: genie-space-permissions-unsupported - -resources: - genie_spaces: - inline_perms: - title: "Inline Perms" - warehouse_id: "test-warehouse-id" - serialized_space: "{}" - permissions: - - level: CAN_MANAGE - user_name: someone@example.com - - bundle_perms: - title: "Bundle Perms" - warehouse_id: "test-warehouse-id" - serialized_space: "{}" - -permissions: - - level: CAN_MANAGE - user_name: someone@example.com diff --git a/acceptance/bundle/validate/genie_space_permissions_unsupported/output.txt b/acceptance/bundle/validate/genie_space_permissions_unsupported/output.txt deleted file mode 100644 index 7224e5d1fea..00000000000 --- a/acceptance/bundle/validate/genie_space_permissions_unsupported/output.txt +++ /dev/null @@ -1,22 +0,0 @@ -Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups -If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. - -Consider using a adding a top-level permissions section such as the following: - - permissions: - - user_name: [USERNAME] - level: CAN_MANAGE - -See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. - in databricks.yml:20:3 - -Error: Genie Space permissions are not supported - -Databricks workspaces do not expose a permissions endpoint for Genie Spaces, so a deploy with permissions configured would create the space and then fail. Remove the permissions block, or remove the Genie Space from any bundle-level permissions, until the API adds support. - -Error: Genie Space permissions are not supported - -Databricks workspaces do not expose a permissions endpoint for Genie Spaces, so a deploy with permissions configured would create the space and then fail. Remove the permissions block, or remove the Genie Space from any bundle-level permissions, until the API adds support. - - -Exit code: 1 diff --git a/acceptance/bundle/validate/genie_space_permissions_unsupported/script b/acceptance/bundle/validate/genie_space_permissions_unsupported/script deleted file mode 100644 index b260e836a71..00000000000 --- a/acceptance/bundle/validate/genie_space_permissions_unsupported/script +++ /dev/null @@ -1 +0,0 @@ -$CLI bundle plan diff --git a/acceptance/bundle/validate/genie_space_permissions_unsupported/test.toml b/acceptance/bundle/validate/genie_space_permissions_unsupported/test.toml deleted file mode 100644 index c304e3eba17..00000000000 --- a/acceptance/bundle/validate/genie_space_permissions_unsupported/test.toml +++ /dev/null @@ -1,8 +0,0 @@ -Local = true -Cloud = false - -Ignore = [".databricks"] - -# Genie spaces only support direct deployment engine. -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go index fe746cbd98e..a9327217c08 100644 --- a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go +++ b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go @@ -56,14 +56,23 @@ func (c configureGenieSpaceSerializedSpace) Apply(_ context.Context, b *bundle.B // Otherwise YAML decodes small ints as Go `int` while state JSON // round-trip decodes them as `float64`, and structdiff reports // false drift on every plan. - if ss.Kind() != dyn.KindMap && ss.Kind() != dyn.KindSequence { + switch ss.Kind() { + case dyn.KindNil, dyn.KindString: + return v, nil + case dyn.KindMap, dyn.KindSequence: + jsonBytes, err := json.Marshal(ss.AsAny()) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to marshal inline serialized_space: %w", err) + } + return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(jsonBytes))) + default: + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("serialized_space must be a string, map, or sequence, got %s", ss.Kind()), + Locations: ss.Locations(), + }) return v, nil } - jsonBytes, err := json.Marshal(ss.AsAny()) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to marshal inline serialized_space: %w", err) - } - return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(jsonBytes))) }) }) diff --git a/bundle/config/mutator/translate_paths_genie_spaces_test.go b/bundle/config/mutator/translate_paths_genie_spaces_test.go index a5295400905..a1ac0b160b1 100644 --- a/bundle/config/mutator/translate_paths_genie_spaces_test.go +++ b/bundle/config/mutator/translate_paths_genie_spaces_test.go @@ -17,7 +17,7 @@ import ( func TestTranslatePathsGenieSpaces_FilePathRelativeSubDirectory(t *testing.T) { dir := t.TempDir() - touchEmptyFile(t, filepath.Join(dir, "src", "my_space.genie.json")) + touchEmptyFile(t, filepath.Join(dir, "src", "my_space.geniespace.json")) b := &bundle.Bundle{ SyncRootPath: dir, @@ -30,7 +30,7 @@ func TestTranslatePathsGenieSpaces_FilePathRelativeSubDirectory(t *testing.T) { GenieSpaceConfig: resources.GenieSpaceConfig{ Title: "My Genie Space", }, - FilePath: "../src/my_space.genie.json", + FilePath: "../src/my_space.geniespace.json", }, }, }, @@ -41,12 +41,15 @@ func TestTranslatePathsGenieSpaces_FilePathRelativeSubDirectory(t *testing.T) { File: filepath.Join(dir, "resources", "genie_space.yml"), }}) + // Genie space paths reuse the dashboard translator; there is no separate + // genie_space mutator. The dashboard translator walks all resource types + // that need path translation, so calling it covers genie_spaces too. diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePathsDashboards()) require.NoError(t, diags.Error()) assert.Equal( t, - filepath.ToSlash(filepath.Join("src", "my_space.genie.json")), + filepath.ToSlash(filepath.Join("src", "my_space.geniespace.json")), b.Config.Resources.GenieSpaces["genie_space"].FilePath, ) } diff --git a/bundle/config/mutator/validate_genie_space_permissions.go b/bundle/config/mutator/validate_genie_space_permissions.go deleted file mode 100644 index e2efb517705..00000000000 --- a/bundle/config/mutator/validate_genie_space_permissions.go +++ /dev/null @@ -1,44 +0,0 @@ -package mutator - -import ( - "context" - "fmt" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" -) - -type validateGenieSpacePermissions struct{} - -// ValidateGenieSpacePermissions errors if any genie_space resource has -// permissions configured. The Databricks workspace API does not expose -// PUT /permissions/genie/spaces/, so the deploy would create the -// space and then fail when applying permissions, leaving partial state. -// Bundle-level permissions are propagated to genie_spaces by -// ApplyBundlePermissions and are caught here as well. -func ValidateGenieSpacePermissions() bundle.Mutator { - return &validateGenieSpacePermissions{} -} - -func (m *validateGenieSpacePermissions) Name() string { - return "ValidateGenieSpacePermissions" -} - -func (m *validateGenieSpacePermissions) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { - var diags diag.Diagnostics - - for key, space := range b.Config.Resources.GenieSpaces { - if space == nil || len(space.Permissions) == 0 { - continue - } - - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "Genie Space permissions are not supported", - Detail: "Databricks workspaces do not expose a permissions endpoint for Genie Spaces, so a deploy with permissions configured would create the space and then fail. Remove the permissions block, or remove the Genie Space from any bundle-level permissions, until the API adds support.", - Locations: b.Config.GetLocations(fmt.Sprintf("resources.genie_spaces.%s.permissions", key)), - }) - } - - return diags -} diff --git a/bundle/config/mutator/validate_genie_space_permissions_test.go b/bundle/config/mutator/validate_genie_space_permissions_test.go deleted file mode 100644 index 0ccf143815e..00000000000 --- a/bundle/config/mutator/validate_genie_space_permissions_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package mutator_test - -import ( - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/mutator" - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/databricks-sdk-go/service/iam" - "github.com/stretchr/testify/assert" -) - -func TestValidateGenieSpacePermissions_NoPermissions(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - GenieSpaces: map[string]*resources.GenieSpace{ - "my_space": { - GenieSpaceConfig: resources.GenieSpaceConfig{ - Title: "My Space", - }, - }, - }, - }, - }, - } - - diags := bundle.Apply(t.Context(), b, mutator.ValidateGenieSpacePermissions()) - assert.Empty(t, diags) -} - -func TestValidateGenieSpacePermissions_WithPermissionsErrors(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - GenieSpaces: map[string]*resources.GenieSpace{ - "my_space": { - GenieSpaceConfig: resources.GenieSpaceConfig{ - Title: "My Space", - }, - Permissions: []resources.Permission{ - {Level: iam.PermissionLevel("CAN_MANAGE"), UserName: "user@example.com"}, - }, - }, - }, - }, - }, - } - - diags := bundle.Apply(t.Context(), b, mutator.ValidateGenieSpacePermissions()) - assert.Len(t, diags, 1) - assert.Equal(t, "Genie Space permissions are not supported", diags[0].Summary) -} diff --git a/bundle/config/resources/genie_space.go b/bundle/config/resources/genie_space.go index 276ab12b17d..b0a5efbf304 100644 --- a/bundle/config/resources/genie_space.go +++ b/bundle/config/resources/genie_space.go @@ -14,6 +14,12 @@ import ( type GenieSpaceConfig struct { // Description of the Genie Space Description string `json:"description,omitempty"` + // Etag for change detection. The bundle persists the value the backend + // returned on the last Create/Update and uses it both as an If-Match for + // the next Update and as the signal for `bundle plan` to detect remote + // drift (see OverrideChangeDesc in bundle/direct/dresources/genie_space.go). + // Mirrors dashboards.DashboardConfig.Etag. + Etag string `json:"etag,omitempty"` // Genie space ID SpaceId string `json:"space_id,omitempty"` // Title of the Genie Space @@ -50,7 +56,7 @@ type GenieSpace struct { Permissions []Permission `json:"permissions,omitempty"` - // FilePath points to the local `.genie.json` file containing the Genie Space definition. + // FilePath points to the local `.geniespace.json` file containing the Genie Space definition. // This is inlined into serialized_space during deployment. The file_path is kept around // as metadata which is needed for `databricks bundle generate genie-space --resource ` to work. // This is not part of GenieSpaceConfig because we don't need to store this in the resource state. diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index b57fc4c5610..4f66e907736 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -254,6 +254,15 @@ var testConfig map[string]any = map[string]any{ }, }, + "genie_spaces": &resources.GenieSpace{ + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "my-genie-space", + WarehouseId: "test-warehouse-id", + ParentPath: "/Workspace/Users/user@example.com", + SerializedSpace: "{}", + }, + }, + "vector_search_endpoints": &resources.VectorSearchEndpoint{ CreateEndpoint: vectorsearch.CreateEndpoint{ Name: "my-endpoint", diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go index 1eaa1b1995c..395466fda2b 100644 --- a/bundle/direct/dresources/genie_space.go +++ b/bundle/direct/dresources/genie_space.go @@ -19,6 +19,24 @@ import ( var pathSerializedSpace = structpath.MustParsePath("serialized_space") +// ResourceGenieSpace mirrors the dashboard resource pattern (see dashboard.go), +// with these intentional divergences: +// - No Published wrapper: Genie spaces have no publish lifecycle, so +// PrepareState returns the config directly. +// - RemapState filters fewer fields: Genie has no LifecycleState / CreateTime / +// Path / UpdateTime output-only fields to scrub. +// - DoRead clears ParentPath: the GET API does not reliably return parent_path, +// so we drop it from ForceSendFields and zero the value rather than re-adding +// a "/Workspace" prefix the way dashboard.go does in ensureWorkspacePrefix. +// - DoUpdate omits serialized_space when unchanged: serialized_space is in +// ignore_remote_changes (see resources.yml), so a UI edit produces no plan +// entry. Sending the local body anyway would clobber the UI edit on every +// unrelated update. +// - DoCreate has expanded missing-parent-path detection: see +// isMissingGenieParentPathError below. +// +// Permissions follow the standard /permissions/genie/{id} endpoint and are wired +// up via the generic permissions adapter (permissions.go). type ResourceGenieSpace struct { client *databricks.WorkspaceClient } @@ -39,6 +57,7 @@ func (r *ResourceGenieSpace) RemapState(state *resources.GenieSpaceConfig) *reso return &resources.GenieSpaceConfig{ Description: state.Description, + Etag: state.Etag, Title: state.Title, WarehouseId: state.WarehouseId, ParentPath: state.ParentPath, @@ -86,6 +105,7 @@ func responseToGenieSpaceConfig(space *dashboards.GenieSpace, serializedSpace st return &resources.GenieSpaceConfig{ Description: space.Description, + Etag: space.Etag, Title: space.Title, WarehouseId: space.WarehouseId, ParentPath: "", @@ -97,6 +117,25 @@ func responseToGenieSpaceConfig(space *dashboards.GenieSpace, serializedSpace st } } +// isMissingGenieParentPathError reports whether the given Create error means +// "the parent workspace folder does not exist", so DoCreate can mkdir and retry. +// +// Dashboard handles the equivalent condition with a plain apierr.IsMissing +// check (see ResourceDashboard.DoCreate). Genie cannot, because it surfaces +// the same condition in two different shapes depending on the workspace's +// backend version: +// +// 1. Standard missing-resource error: HTTP 404, ErrorCode RESOURCE_DOES_NOT_EXIST. +// Caught by apierr.IsMissing. Observed on workspaces running the newer +// Genie service implementation. +// 2. HTTP 400 with ErrorCode INVALID_PARAMETER_VALUE and a message of the +// form "Tree node with path '' does not exist". Observed on +// workspaces still backed by the legacy implementation during integration +// testing in early 2026 (aws-prod-ucws and azure-prod-ucws clusters at +// the time). The string match is intentional: there is no distinct error +// code to key on. +// +// Both forms unambiguously mean "create the parent and retry once". func isMissingGenieParentPathError(err error) bool { if apierr.IsMissing(err) { return true @@ -107,10 +146,6 @@ func isMissingGenieParentPathError(err error) bool { return false } - // Genie reports a missing parent folder inconsistently across environments. - // Some workspaces return a standard missing-resource error, while others - // return INVALID_PARAMETER_VALUE with a NOT_FOUND message embedded in the - // text. Treat both forms as "create the parent directory and retry once". return apiErr.StatusCode == http.StatusBadRequest && apiErr.ErrorCode == "INVALID_PARAMETER_VALUE" && strings.Contains(apiErr.Message, "Tree node with path") && @@ -150,6 +185,12 @@ func (r *ResourceGenieSpace) DoCreate(ctx context.Context, config *resources.Gen return "", nil, err } + // Persist the etag in state. The deploy framework saves `config` (the input + // to DoCreate) as the state record, so mutating it here is what gets the + // backend-returned etag onto disk for the next plan's drift check. + // Matches the dashboard pattern (dashboard.go DoCreate). + config.Etag = createResp.Etag + return createResp.SpaceId, responseToGenieSpaceConfig(createResp, serializedSpace), nil } @@ -165,8 +206,10 @@ func (r *ResourceGenieSpace) DoUpdate(ctx context.Context, id string, config *re // update triggered by another field would clobber the UI edit. Only // send it when the user actually changed it locally. var excludeForceSend []string + sentSerialized := true if !hasUpdate(entry, pathSerializedSpace) { serializedSpace = "" + sentSerialized = false excludeForceSend = append(excludeForceSend, "SerializedSpace") } @@ -176,8 +219,10 @@ func (r *ResourceGenieSpace) DoUpdate(ctx context.Context, id string, config *re Title: config.Title, WarehouseId: config.WarehouseId, SerializedSpace: serializedSpace, - // Etag is for optimistic concurrency; we apply updates unconditionally. - Etag: "", + // Send the etag we last observed. The backend uses it as an If-Match + // guard against concurrent writes, and OverrideChangeDesc uses the + // post-update etag to detect drift on subsequent plans. + Etag: config.Etag, ForceSendFields: utils.FilterFields[dashboards.GenieUpdateSpaceRequest](config.ForceSendFields, excludeForceSend...), }) @@ -185,16 +230,46 @@ func (r *ResourceGenieSpace) DoUpdate(ctx context.Context, id string, config *re return nil, err } - // When the request omitted serialized_space, use the value the response - // echoes back so RemapState records the latest body. + // Persist the new etag in state (see DoCreate for the rationale). + config.Etag = updateResp.Etag + + // Decide what to record as the new state's serialized_space. + // - If we sent a new body, use it. + // - If we omitted it (UI-edit protection above) but the API echoed back + // a value, record that — it's the most up-to-date view we have. + // - If neither side carries a value, keep whatever was already in state. + // Otherwise RemapState would blank the field on every unrelated update. respSerialized := serializedSpace - if respSerialized == "" { + if !sentSerialized { respSerialized = updateResp.SerializedSpace + if respSerialized == "" { + if prior, ok := config.SerializedSpace.(string); ok { + respSerialized = prior + } + } } return responseToGenieSpaceConfig(updateResp, respSerialized), nil } +// OverrideChangeDesc handles the etag field. The user never sets it directly; +// we compare the stored etag against the remote one and Skip if they match. +// This mirrors ResourceDashboard.OverrideChangeDesc. +func (r *ResourceGenieSpace) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, _ *resources.GenieSpaceConfig) error { + switch path.String() { + case "etag": + // change.New is always nil for etag because it's not present in the + // user-authored config. Compare stored etag with remote one to decide + // whether anything changed out-of-band since the last deploy. + if change.Old == change.Remote { + change.Action = deployplan.Skip + } else { + change.Action = deployplan.Update + } + } + return nil +} + // hasUpdate reports whether entry has an Update-action change at the given path. // HasChange alone matches Skip-action changes too, which we cannot use to drive // request shaping for fields covered by ignore_remote_changes. diff --git a/bundle/direct/dresources/genie_space_test.go b/bundle/direct/dresources/genie_space_test.go index 7db9f5c4b16..742deff6cb4 100644 --- a/bundle/direct/dresources/genie_space_test.go +++ b/bundle/direct/dresources/genie_space_test.go @@ -5,6 +5,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/dashboards" @@ -167,3 +168,93 @@ func TestGenieSpaceDoUpdateSendsSerializedSpaceWhenChanged(t *testing.T) { require.NotNil(t, state) assert.Equal(t, "{\"v\":1}", state.SerializedSpace) } + +func TestGenieSpaceDoUpdateRoundTripsEtag(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + entry := &deployplan.PlanEntry{ + Changes: deployplan.Changes{ + "title": {Action: deployplan.Update, Old: "old", New: "new"}, + }, + } + + m.GetMockGenieAPI().EXPECT(). + UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ + SpaceId: "space-id", + Title: "new", + Etag: "etag-7", + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id", + Title: "new", + Etag: "etag-8", + }, nil). + Once() + + state, err := r.DoUpdate(ctx, "space-id", &resources.GenieSpaceConfig{ + Title: "new", + Etag: "etag-7", + }, entry) + require.NoError(t, err) + require.NotNil(t, state) + assert.Equal(t, "etag-8", state.Etag) +} + +func TestGenieSpaceDoUpdateKeepsPriorSerializedSpaceWhenBothEmpty(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + // Only title changed; serialized_space should be omitted from the request. + entry := &deployplan.PlanEntry{ + Changes: deployplan.Changes{ + "title": {Action: deployplan.Update, Old: "old", New: "new"}, + }, + } + + m.GetMockGenieAPI().EXPECT(). + UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ + SpaceId: "space-id", + Title: "new", + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id", + Title: "new", + // API also omits serialized_space; we should keep the prior local value. + }, nil). + Once() + + state, err := r.DoUpdate(ctx, "space-id", &resources.GenieSpaceConfig{ + Title: "new", + SerializedSpace: "{\"keep\":\"me\"}", + }, entry) + require.NoError(t, err) + require.NotNil(t, state) + assert.Equal(t, "{\"keep\":\"me\"}", state.SerializedSpace) +} + +func TestGenieSpaceOverrideChangeDescEtag(t *testing.T) { + r := &ResourceGenieSpace{} + etagPath := structpath.MustParsePath("etag") + + t.Run("Skip when stored matches remote", func(t *testing.T) { + change := &ChangeDesc{Old: "etag-7", Remote: "etag-7"} + require.NoError(t, r.OverrideChangeDesc(t.Context(), etagPath, change, nil)) + assert.Equal(t, deployplan.Skip, change.Action) + }) + + t.Run("Update when stored differs from remote", func(t *testing.T) { + change := &ChangeDesc{Old: "etag-7", Remote: "etag-8"} + require.NoError(t, r.OverrideChangeDesc(t.Context(), etagPath, change, nil)) + assert.Equal(t, deployplan.Update, change.Action) + }) + + t.Run("Other paths are untouched", func(t *testing.T) { + titlePath := structpath.MustParsePath("title") + change := &ChangeDesc{Action: deployplan.Update, Old: "a", Remote: "b"} + require.NoError(t, r.OverrideChangeDesc(t.Context(), titlePath, change, nil)) + assert.Equal(t, deployplan.Update, change.Action) + }) +} diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 5b2a70adbb3..7f719674ebb 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -439,9 +439,14 @@ func ExportStateFromData(data Database) resourcestate.ExportedResourcesMap { result := make(resourcestate.ExportedResourcesMap) for key, entry := range data.State { var etag string - // Extract etag for dashboards. - // covered by test case: bundle/deploy/dashboard/detect-change - if strings.Contains(key, ".dashboards.") && len(entry.State) > 0 { + // Extract etag for resources that use it for drift detection + // (dashboards and genie_spaces). Both follow the same pattern of + // persisting the backend-returned etag in state and comparing it + // against the remote on the next plan via OverrideChangeDesc. + // covered by test cases: + // - bundle/deploy/dashboard/detect-change + // - bundle/resources/genie_spaces/simple + if (strings.Contains(key, ".dashboards.") || strings.Contains(key, ".genie_spaces.")) && len(entry.State) > 0 { var holder struct { Etag string `json:"etag"` } diff --git a/bundle/docsgen/output/reference.md b/bundle/docsgen/output/reference.md index 3d89c58e8e8..b3e86d6f305 100644 --- a/bundle/docsgen/output/reference.md +++ b/bundle/docsgen/output/reference.md @@ -1,7 +1,7 @@ --- description: 'Configuration reference for databricks.yml' last_update: - date: 2026-05-11 + date: 2026-05-20 --- @@ -1505,9 +1505,13 @@ genie_spaces: - String - Description of the Genie space shown alongside the title in the Databricks UI. +- - `etag` + - String + - + - - `file_path` - String - - Local path to a `.genie.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. + - Local path to a `.geniespace.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. - - `lifecycle` - Map @@ -3699,9 +3703,13 @@ genie_spaces: - String - Description of the Genie space shown alongside the title in the Databricks UI. +- - `etag` + - String + - + - - `file_path` - String - - Local path to a `.genie.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. + - Local path to a `.geniespace.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. - - `lifecycle` - Map diff --git a/bundle/docsgen/output/resources.md b/bundle/docsgen/output/resources.md index 484064dd36e..4d740961a6a 100644 --- a/bundle/docsgen/output/resources.md +++ b/bundle/docsgen/output/resources.md @@ -1,7 +1,7 @@ --- description: 'Learn about resources supported by Declarative Automation Bundles and how to configure them.' last_update: - date: 2026-06-03 + date: 2026-05-20 --- @@ -532,14 +532,6 @@ apps: - String - -- - `compute_max_instances` - - Integer - - - -- - `compute_min_instances` - - Integer - - - - - `compute_size` - String - @@ -556,6 +548,10 @@ apps: - Map - Git repository configuration for app deployments. When specified, deployments can reference code from this repository by providing only the git reference (branch, tag, or commit). See [\_](#appsnamegit_repository). +- - `git_source` + - Map + - Git source configuration for app deployments. Specifies which git reference (branch, tag, or commit) to use when deploying the app. Used in conjunction with git_repository to deploy code directly from git. The source_code_path within git_source specifies the relative path to the app code within the repository. See [\_](#appsnamegit_source). + - - `lifecycle` - Map - Lifecycle is a struct that contains the lifecycle settings for a resource. It controls the behavior of the resource when it is deployed or destroyed. See [\_](#appsnamelifecycle). @@ -572,6 +568,10 @@ apps: - Sequence - Resources for the app. See [\_](#appsnameresources). +- - `source_code_path` + - String + - + - - `telemetry_export_destinations` - Sequence - See [\_](#appsnametelemetry_export_destinations). @@ -667,6 +667,41 @@ reference code from this repository by providing only the git reference (branch, ::: +### apps._name_.git_source + +**`Type: Map`** + +Git source configuration for app deployments. Specifies which git reference (branch, tag, or commit) +to use when deploying the app. Used in conjunction with git_repository to deploy code directly from git. +The source_code_path within git_source specifies the relative path to the app code within the repository. + + + +:::list-table + +- - Key + - Type + - Description + +- - `branch` + - String + - Git branch to checkout. + +- - `commit` + - String + - Git commit SHA to checkout. + +- - `source_code_path` + - String + - Relative path to the app source code within the Git repository. If not specified, the root of the repository is used. + +- - `tag` + - String + - Git tag to checkout. + +::: + + ### apps._name_.lifecycle **`Type: Map`** @@ -1340,7 +1375,7 @@ clusters: - - `data_security_mode` - String - - Data security mode decides what data governance model to use when accessing data from a cluster. * `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration. * `DATA_SECURITY_MODE_STANDARD`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other’s data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited. * `DATA_SECURITY_MODE_DEDICATED`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. The following modes are legacy aliases for the above modes: * `USER_ISOLATION`: Legacy alias for `DATA_SECURITY_MODE_STANDARD`. * `SINGLE_USER`: Legacy alias for `DATA_SECURITY_MODE_DEDICATED`. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters. * `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled. + - Data security mode decides what data governance model to use when accessing data from a cluster. The following modes can only be used when `kind = CLASSIC_PREVIEW`. * `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration. * `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. * `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`. The following modes can be used regardless of `kind`. * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode. * `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. * `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters. * `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled. - - `docker_image` - Map @@ -2088,10 +2123,6 @@ Lifecycle is a struct that contains the lifecycle settings for a resource. It co - Boolean - Lifecycle setting to prevent the resource from being destroyed. -- - `started` - - Boolean - - Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. - ::: @@ -3109,9 +3140,13 @@ genie_spaces: - String - Description of the Genie space shown alongside the title in the Databricks UI. +- - `etag` + - String + - + - - `file_path` - String - - Local path to a `.genie.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. + - Local path to a `.geniespace.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. - - `lifecycle` - Map @@ -3380,10 +3415,6 @@ Deployment information for jobs managed by external sources. - Type - Description -- - `deployment_id` - - String - - - - - `kind` - String - The kind of deployment that manages the job. * `BUNDLE`: The job is managed by Databricks Asset Bundle. * `SYSTEM_MANAGED`: The job is managed by Databricks and is read-only. @@ -3392,10 +3423,6 @@ Deployment information for jobs managed by external sources. - String - Path of the file that contains deployment metadata. -- - `version_id` - - String - - - ::: @@ -3484,7 +3511,7 @@ In this minimal environment spec, only pip and java dependencies are supported. - - `base_environment` - String - - The base environment this environment is built on top of. A base environment defines the environment version and a list of dependencies for serverless compute. The value can be a file path to a custom `env.yaml` file (e.g., `/Workspace/path/to/env.yaml`). Support for a Databricks-provided base environment ID (e.g., `workspace-base-environments/databricks_ai_v4`) and workspace base environment ID (e.g., `workspace-base-environments/dbe_b849b66e-b31a-4cb5-b161-1f2b10877fb7`) is in Beta. Either `environment_version` or `base_environment` can be provided. For more information about Databricks-provided base environments, see the [list workspace base environments](:method:Environments/ListWorkspaceBaseEnvironments) API. For more information, see + - The base environment this environment is built on top of. A base environment defines the environment version and a list of dependencies for serverless compute. The value can be a file path to a custom `env.yaml` file (e.g., `/Workspace/path/to/env.yaml`). Support for a Databricks-provided base environment ID (e.g., `workspace-base-environments/databricks_ai_v4`) and workspace base environment ID (e.g., `workspace-base-environments/dbe_b849b66e-b31a-4cb5-b161-1f2b10877fb7`) is in Beta. Either `environment_version` or `base_environment` can be provided. For more information, see - - `client` - String @@ -3719,7 +3746,7 @@ If new_cluster, a description of a cluster that is created for each task. - - `data_security_mode` - String - - Data security mode decides what data governance model to use when accessing data from a cluster. * `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration. * `DATA_SECURITY_MODE_STANDARD`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other’s data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited. * `DATA_SECURITY_MODE_DEDICATED`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. The following modes are legacy aliases for the above modes: * `USER_ISOLATION`: Legacy alias for `DATA_SECURITY_MODE_STANDARD`. * `SINGLE_USER`: Legacy alias for `DATA_SECURITY_MODE_DEDICATED`. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters. * `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled. + - Data security mode decides what data governance model to use when accessing data from a cluster. The following modes can only be used when `kind = CLASSIC_PREVIEW`. * `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration. * `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. * `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`. The following modes can be used regardless of `kind`. * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode. * `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. * `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters. * `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled. - - `docker_image` - Map @@ -4713,10 +4740,6 @@ Read endpoints return only 100 tasks. If more than 100 tasks are available, you - Boolean - An option to disable auto optimization in serverless -- - `disabled` - - Boolean - - An optional flag to disable the task. If set to true, the task will not run even if it is part of a job. - - - `email_notifications` - Map - An optional set of email addresses that is notified when runs of this task begin or complete as well as when this task is deleted. The default behavior is to not send any emails. See [\_](#jobsnametasksemail_notifications). @@ -5414,7 +5437,7 @@ If new_cluster, a description of a new cluster that is created for each run. - - `data_security_mode` - String - - Data security mode decides what data governance model to use when accessing data from a cluster. * `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration. * `DATA_SECURITY_MODE_STANDARD`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other’s data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited. * `DATA_SECURITY_MODE_DEDICATED`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. The following modes are legacy aliases for the above modes: * `USER_ISOLATION`: Legacy alias for `DATA_SECURITY_MODE_STANDARD`. * `SINGLE_USER`: Legacy alias for `DATA_SECURITY_MODE_DEDICATED`. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters. * `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled. + - Data security mode decides what data governance model to use when accessing data from a cluster. The following modes can only be used when `kind = CLASSIC_PREVIEW`. * `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration. * `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. * `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`. The following modes can be used regardless of `kind`. * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode. * `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. * `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters. * `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled. - - `docker_image` - Map @@ -6255,30 +6278,10 @@ The task triggers a pipeline update when the `pipeline_task` field is present. O - Boolean - If true, triggers a full refresh on the spark declarative pipeline. -- - `full_refresh_selection` - - Sequence - - A list of tables to update with fullRefresh. - -- - `parameters` - - Map - - - - - `pipeline_id` - String - The full name of the pipeline task to execute. -- - `refresh_flow_selection` - - Sequence - - Flow names to selectively refresh. These are unioned with other selective refresh options (refresh_selection, full_refresh_selection) to determine the final set of flows to refresh. - -- - `refresh_selection` - - Sequence - - A list of tables to update without fullRefresh. - -- - `reset_checkpoint_selection` - - Sequence - - A list of streaming flows to reset checkpoints without clearing data. - ::: @@ -6469,22 +6472,6 @@ Controls whether the pipeline should perform a full refresh - Boolean - If true, triggers a full refresh on the spark declarative pipeline. -- - `full_refresh_selection` - - Sequence - - A list of tables to update with fullRefresh. - -- - `refresh_flow_selection` - - Sequence - - Flow names to selectively refresh. These are unioned with other selective refresh options (refresh_selection, full_refresh_selection) to determine the final set of flows to refresh. - -- - `refresh_selection` - - Sequence - - A list of tables to update without fullRefresh. - -- - `reset_checkpoint_selection` - - Sequence - - A list of streaming flows to reset checkpoints without clearing data. - ::: @@ -8513,10 +8500,6 @@ pipelines: - Sequence - List of notification settings for this pipeline. See [\_](#pipelinesnamenotifications). -- - `parameters` - - Map - - - - - `permissions` - Sequence - See [\_](#pipelinesnamepermissions). @@ -9209,10 +9192,6 @@ Deployment type of this pipeline. - Type - Description -- - `deployment_id` - - String - - - - - `kind` - String - The deployment method that manages the pipeline. @@ -9221,10 +9200,6 @@ Deployment type of this pipeline. - String - The path to the file containing metadata about the deployment. -- - `version_id` - - String - - - ::: @@ -10572,10 +10547,6 @@ postgres_branches: - String - -- - `replace_existing` - - Boolean - - - - - `source_branch` - String - @@ -10603,69 +10574,6 @@ postgres_branches: -:::list-table - -- - Key - - Type - - Description - -- - `prevent_destroy` - - Boolean - - Lifecycle setting to prevent the resource from being destroyed. - -::: - - -## postgres_catalogs - -**`Type: Map`** - -The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch. - -```yaml -postgres_catalogs: - : - : -``` - - -:::list-table - -- - Key - - Type - - Description - -- - `branch` - - String - - - -- - `catalog_id` - - String - - - -- - `create_database_if_missing` - - Boolean - - - -- - `lifecycle` - - Map - - See [\_](#postgres_catalogsnamelifecycle). - -- - `postgres_database` - - String - - - -::: - - -### postgres_catalogs._name_.lifecycle - -**`Type: Map`** - - - - - :::list-table - - Key @@ -10734,10 +10642,6 @@ postgres_endpoints: - String - -- - `replace_existing` - - Boolean - - - - - `settings` - Map - A collection of settings for a compute endpoint. See [\_](#postgres_endpointsnamesettings). @@ -11002,122 +10906,6 @@ A collection of settings for a compute endpoint. ::: -## postgres_synced_tables - -**`Type: Map`** - -The Postgres synced table definitions for the bundle, where each key is the name of the synced table. Each entry continuously replicates a Unity Catalog Delta source table into a Postgres table on a Lakebase Autoscaling instance. - -```yaml -postgres_synced_tables: - : - : -``` - - -:::list-table - -- - Key - - Type - - Description - -- - `branch` - - String - - - -- - `create_database_objects_if_missing` - - Boolean - - - -- - `existing_pipeline_id` - - String - - - -- - `lifecycle` - - Map - - See [\_](#postgres_synced_tablesnamelifecycle). - -- - `new_pipeline_spec` - - Map - - See [\_](#postgres_synced_tablesnamenew_pipeline_spec). - -- - `postgres_database` - - String - - - -- - `primary_key_columns` - - Sequence - - - -- - `scheduling_policy` - - String - - Scheduling policy of the synced table's underlying pipeline. - -- - `source_table_full_name` - - String - - - -- - `synced_table_id` - - String - - - -- - `timeseries_key` - - String - - - -::: - - -### postgres_synced_tables._name_.lifecycle - -**`Type: Map`** - - - - - -:::list-table - -- - Key - - Type - - Description - -- - `prevent_destroy` - - Boolean - - Lifecycle setting to prevent the resource from being destroyed. - -::: - - -### postgres_synced_tables._name_.new_pipeline_spec - -**`Type: Map`** - - - - - -:::list-table - -- - Key - - Type - - Description - -- - `budget_policy_id` - - String - - Budget policy to set on the newly created pipeline. - -- - `storage_catalog` - - String - - UC catalog for the pipeline to store intermediate files (checkpoints, event logs etc). This needs to be a standard catalog where the user has permissions to create Delta tables. - -- - `storage_schema` - - String - - UC schema for the pipeline to store intermediate files (checkpoints, event logs etc). This needs to be in the standard catalog where the user has permissions to create Delta tables. - -::: - - ## quality_monitors **`Type: Map`** @@ -12349,299 +12137,6 @@ vector_search_endpoints: ::: -## vector_search_indexes - -**`Type: Map`** - - - -```yaml -vector_search_indexes: - : - : -``` - - -:::list-table - -- - Key - - Type - - Description - -- - `delta_sync_index_spec` - - Map - - See [\_](#vector_search_indexesnamedelta_sync_index_spec). - -- - `direct_access_index_spec` - - Map - - See [\_](#vector_search_indexesnamedirect_access_index_spec). - -- - `endpoint_name` - - String - - - -- - `grants` - - Sequence - - See [\_](#vector_search_indexesnamegrants). - -- - `index_subtype` - - String - - - -- - `index_type` - - String - - - -- - `lifecycle` - - Map - - See [\_](#vector_search_indexesnamelifecycle). - -- - `name` - - String - - - -- - `primary_key` - - String - - - -::: - - -### vector_search_indexes._name_.delta_sync_index_spec - -**`Type: Map`** - - - - - -:::list-table - -- - Key - - Type - - Description - -- - `columns_to_index` - - Sequence - - - -- - `columns_to_sync` - - Sequence - - - -- - `embedding_source_columns` - - Sequence - - See [\_](#vector_search_indexesnamedelta_sync_index_specembedding_source_columns). - -- - `embedding_vector_columns` - - Sequence - - See [\_](#vector_search_indexesnamedelta_sync_index_specembedding_vector_columns). - -- - `embedding_writeback_table` - - String - - - -- - `pipeline_type` - - String - - - -- - `source_table` - - String - - - -::: - - -### vector_search_indexes._name_.delta_sync_index_spec.embedding_source_columns - -**`Type: Sequence`** - - - - - -:::list-table - -- - Key - - Type - - Description - -- - `embedding_model_endpoint_name` - - String - - - -- - `model_endpoint_name_for_query` - - String - - - -- - `name` - - String - - - -::: - - -### vector_search_indexes._name_.delta_sync_index_spec.embedding_vector_columns - -**`Type: Sequence`** - - - - - -:::list-table - -- - Key - - Type - - Description - -- - `embedding_dimension` - - Integer - - - -- - `name` - - String - - - -::: - - -### vector_search_indexes._name_.direct_access_index_spec - -**`Type: Map`** - - - - - -:::list-table - -- - Key - - Type - - Description - -- - `embedding_source_columns` - - Sequence - - See [\_](#vector_search_indexesnamedirect_access_index_specembedding_source_columns). - -- - `embedding_vector_columns` - - Sequence - - See [\_](#vector_search_indexesnamedirect_access_index_specembedding_vector_columns). - -- - `schema_json` - - String - - - -::: - - -### vector_search_indexes._name_.direct_access_index_spec.embedding_source_columns - -**`Type: Sequence`** - - - - - -:::list-table - -- - Key - - Type - - Description - -- - `embedding_model_endpoint_name` - - String - - - -- - `model_endpoint_name_for_query` - - String - - - -- - `name` - - String - - - -::: - - -### vector_search_indexes._name_.direct_access_index_spec.embedding_vector_columns - -**`Type: Sequence`** - - - - - -:::list-table - -- - Key - - Type - - Description - -- - `embedding_dimension` - - Integer - - - -- - `name` - - String - - - -::: - - -### vector_search_indexes._name_.grants - -**`Type: Sequence`** - - - - - -:::list-table - -- - Key - - Type - - Description - -- - `principal` - - String - - The principal (user email address or group name). For deleted principals, `principal` is empty while `principal_id` is populated. - -- - `privileges` - - Sequence - - The privileges assigned to the principal. - -::: - - -### vector_search_indexes._name_.grants.privileges - -**`Type: Sequence`** - -The privileges assigned to the principal. - - -### vector_search_indexes._name_.lifecycle - -**`Type: Map`** - - - - - -:::list-table - -- - Key - - Type - - Description - -- - `prevent_destroy` - - Boolean - - Lifecycle setting to prevent the resource from being destroyed. - -::: - - ## volumes **`Type: Map`** diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 6194cf078e0..69d7c9d025d 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -675,9 +675,12 @@ github.com/databricks/cli/bundle/config/resources.GenieSpace: "description": "description": |- Description of the Genie space shown alongside the title in the Databricks UI. + "etag": + "description": |- + PLACEHOLDER "file_path": "description": |- - Local path to a `.genie.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. + Local path to a `.geniespace.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. "lifecycle": "description": |- PLACEHOLDER diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 3e63b8f607b..0af9394f243 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -25,7 +25,6 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine deploy.StatePull(), mutator.ValidateGitDetails(), mutator.ValidateDirectOnlyResources(engine), - mutator.ValidateGenieSpacePermissions(), mutator.ValidateLifecycleStarted(engine), statemgmt.CheckRunningResource(engine), ) diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 4ba9eb7c43c..1a8b4369e83 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -769,8 +769,11 @@ "description": "Description of the Genie space shown alongside the title in the Databricks UI.", "$ref": "#/$defs/string" }, + "etag": { + "$ref": "#/$defs/string" + }, "file_path": { - "description": "Local path to a `.genie.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`.", + "description": "Local path to a `.geniespace.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`.", "$ref": "#/$defs/string" }, "lifecycle": { diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 7af4e01e92f..250d05e3693 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -275,7 +275,11 @@ func waitForChanges(ctx context.Context, w *databricks.WorkspaceClient, dashboar break } - time.Sleep(1 * time.Second) + select { + case <-ctx.Done(): + return + case <-time.After(1 * time.Second): + } } } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index f4c344667e7..9cc415ba43d 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -163,7 +163,7 @@ func (g *genieSpace) saveSerializedGenieSpace(ctx context.Context, b *bundle.Bun func (g *genieSpace) saveConfiguration(ctx context.Context, b *bundle.Bundle, genieSpace *dashboards.GenieSpace, key string) error { // Save serialized genie space definition to the genie space directory. - genieSpaceBasename := key + ".genie.json" + genieSpaceBasename := key + ".geniespace.json" genieSpacePath := filepath.Join(g.genieSpaceDir, genieSpaceBasename) err := g.saveSerializedGenieSpace(ctx, b, genieSpace, genieSpacePath) if err != nil { @@ -264,7 +264,11 @@ func (g *genieSpace) updateGenieSpaceForResource(ctx context.Context, b *bundle. } first = false - time.Sleep(genieSpaceWatchInterval) + select { + case <-ctx.Done(): + return + case <-time.After(genieSpaceWatchInterval): + } } } @@ -306,6 +310,7 @@ func (g *genieSpace) generateForExisting(ctx context.Context, b *bundle.Bundle, err = g.saveConfiguration(ctx, b, genieSpace, key) if err != nil { logdiag.LogError(ctx, err) + return } if g.bind { @@ -448,7 +453,7 @@ Examples: What gets generated: - Genie space configuration YAML file with settings and a reference to the Genie space definition -- Genie space definition (.genie.json) file with the serialized space content +- Genie space definition (.geniespace.json) file with the serialized space content Sync workflow for Genie space development: When developing Genie spaces, you can modify them in the Databricks UI and sync diff --git a/cmd/bundle/generate/genie_space_test.go b/cmd/bundle/generate/genie_space_test.go new file mode 100644 index 00000000000..746b2b1c3af --- /dev/null +++ b/cmd/bundle/generate/genie_space_test.go @@ -0,0 +1,126 @@ +package generate + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func newGenieSpaceTestBundle(t *testing.T, m *mocks.MockWorkspaceClient, filePath string) *bundle.Bundle { + t.Helper() + b := &bundle.Bundle{ + BundleRootPath: t.TempDir(), + Config: config.Root{ + Resources: config.Resources{ + GenieSpaces: map[string]*resources.GenieSpace{ + "my_space": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "My Space", + }, + FilePath: filePath, + }, + }, + }, + }, + } + b.Config.Resources.GenieSpaces["my_space"].ID = "space-id-1" + b.SetWorkpaceClient(m.WorkspaceClient) + return b +} + +func TestGenieSpace_UpdateForResource_WritesFileWhenNotWatching(t *testing.T) { + dir := t.TempDir() + filePath := filepath.Join(dir, "my_space.geniespace.json") + + m := mocks.NewMockWorkspaceClient(t) + m.GetMockGenieAPI().EXPECT(). + GetSpace(mock.Anything, dashboards.GenieGetSpaceRequest{ + SpaceId: "space-id-1", + IncludeSerializedSpace: true, + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id-1", + Title: "My Space", + SerializedSpace: `{"version":1}`, + }, nil). + Once() + + g := &genieSpace{ + resource: "my_space", + force: true, + } + b := newGenieSpaceTestBundle(t, m, filePath) + + ctx, _ := cmdio.NewTestContextWithStdout(t.Context()) + ctx = logdiag.InitContext(ctx) + logdiag.SetCollect(ctx, true) + g.updateGenieSpaceForResource(ctx, b) + + require.Empty(t, logdiag.FlushCollected(ctx)) + + contents, err := os.ReadFile(filePath) + require.NoError(t, err) + assert.Contains(t, string(contents), `"version"`) +} + +func TestGenieSpace_UpdateForResource_WatchExitsOnCancel(t *testing.T) { + dir := t.TempDir() + filePath := filepath.Join(dir, "my_space.geniespace.json") + + m := mocks.NewMockWorkspaceClient(t) + // Allow any number of GetSpace calls; we don't know how many fire before cancel. + m.GetMockGenieAPI().EXPECT(). + GetSpace(mock.Anything, dashboards.GenieGetSpaceRequest{ + SpaceId: "space-id-1", + IncludeSerializedSpace: true, + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id-1", + Title: "My Space", + SerializedSpace: `{"version":1}`, + }, nil). + Maybe() + + g := &genieSpace{ + resource: "my_space", + force: true, + watch: true, + } + b := newGenieSpaceTestBundle(t, m, filePath) + + base, _ := cmdio.NewTestContextWithStdout(t.Context()) + ctx, cancel := context.WithCancel(logdiag.InitContext(base)) + logdiag.SetCollect(ctx, true) + + done := make(chan struct{}) + go func() { + g.updateGenieSpaceForResource(ctx, b) + close(done) + }() + + // First iteration always saves. Wait until the file lands, then cancel. + require.Eventually(t, func() bool { + _, err := os.Stat(filePath) + return err == nil + }, 2*time.Second, 10*time.Millisecond, "expected initial save to write file") + + cancel() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("watch loop did not exit promptly after ctx cancel") + } +} diff --git a/libs/testserver/genie_spaces.go b/libs/testserver/genie_spaces.go index 0b11b119fcf..98f5cdb381a 100644 --- a/libs/testserver/genie_spaces.go +++ b/libs/testserver/genie_spaces.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "encoding/json" "path" + "strconv" "strings" "github.com/databricks/databricks-sdk-go/service/dashboards" @@ -51,6 +52,9 @@ func (s *FakeWorkspace) GenieSpaceCreate(req Request) Response { ParentPath: createReq.ParentPath, WarehouseId: createReq.WarehouseId, SerializedSpace: createReq.SerializedSpace, + // Mirror libs/testserver/dashboards.go: initialize etag to a numeric + // string so each subsequent update can bump it monotonically. + Etag: "1", } s.GenieSpaces[spaceId] = genieSpace @@ -61,7 +65,7 @@ func (s *FakeWorkspace) GenieSpaceCreate(req Request) Response { if !strings.HasPrefix(workspacePath, "/Workspace") { workspacePath = path.Join("/Workspace", workspacePath) } - workspacePath = path.Join(workspacePath, createReq.Title+".genie") + workspacePath = path.Join(workspacePath, createReq.Title+".geniespace") s.files[workspacePath] = FileEntry{ Info: workspace.ObjectInfo{ @@ -121,6 +125,18 @@ func (s *FakeWorkspace) GenieSpaceUpdate(req Request) Response { } } + // Optimistic concurrency: if the caller sent an etag, it must match the + // current one. Empty etag means apply unconditionally. + if updateReq.Etag != "" && updateReq.Etag != genieSpace.Etag { + return Response{ + StatusCode: 409, + Body: map[string]string{ + "message": "Etag mismatch: expected " + genieSpace.Etag + ", got " + updateReq.Etag, + }, + } + } + + prev := genieSpace if updateReq.Title != "" { genieSpace.Title = updateReq.Title } @@ -134,6 +150,26 @@ func (s *FakeWorkspace) GenieSpaceUpdate(req Request) Response { genieSpace.SerializedSpace = updateReq.SerializedSpace } + // Bump the etag only when the update actually changes user-visible state. + // Matches dashboard's behavior (libs/testserver/dashboards.go) and keeps + // no-op updates idempotent so TestAll can pass the same config to Create + // and Update without observing spurious drift. + if prev.Title != genieSpace.Title || + prev.Description != genieSpace.Description || + prev.WarehouseId != genieSpace.WarehouseId || + prev.SerializedSpace != genieSpace.SerializedSpace { + prevEtag, err := strconv.Atoi(genieSpace.Etag) + if err != nil { + return Response{ + StatusCode: 500, + Body: map[string]string{ + "message": "Invalid stored etag: " + genieSpace.Etag, + }, + } + } + genieSpace.Etag = strconv.Itoa(prevEtag + 1) + } + s.GenieSpaces[spaceId] = genieSpace return Response{ @@ -145,7 +181,7 @@ func (s *FakeWorkspace) GenieSpaceTrash(req Request) Response { defer s.LockUnlock()() spaceId := req.Vars["space_id"] - _, ok := s.GenieSpaces[spaceId] + genieSpace, ok := s.GenieSpaces[spaceId] if !ok { return Response{ StatusCode: 404, @@ -157,6 +193,18 @@ func (s *FakeWorkspace) GenieSpaceTrash(req Request) Response { delete(s.GenieSpaces, spaceId) + // Also remove the synthetic workspace file entry registered by + // GenieSpaceCreate, so a trash+recreate flow does not resolve to stale + // state via the workspace path index. + if genieSpace.ParentPath != "" { + workspacePath := genieSpace.ParentPath + if !strings.HasPrefix(workspacePath, "/Workspace") { + workspacePath = path.Join("/Workspace", workspacePath) + } + workspacePath = path.Join(workspacePath, genieSpace.Title+".geniespace") + delete(s.files, workspacePath) + } + return Response{ StatusCode: 200, } diff --git a/libs/workspaceurls/urls_test.go b/libs/workspaceurls/urls_test.go index af7b5b08ea4..d0a86a0f247 100644 --- a/libs/workspaceurls/urls_test.go +++ b/libs/workspaceurls/urls_test.go @@ -111,6 +111,7 @@ func TestResourceURL(t *testing.T) { {"jobs", "jobs", "123", "https://host.com/jobs/123"}, {"experiments", "experiments", "exp-1", "https://host.com/ml/experiments/exp-1"}, {"dashboards", "dashboards", "d-1", "https://host.com/dashboardsv3/d-1/published"}, + {"genie_spaces", "genie_spaces", "space-1", "https://host.com/genie/rooms/space-1"}, {"notebooks", "notebooks", "12345", "https://host.com/#notebook/12345"}, {"notebooks with path", "notebooks", "/Users/u/nb", "https://host.com/#notebook//Users/u/nb"}, {"registered_models normalizes dots", "registered_models", "cat.sch.model", "https://host.com/explore/data/models/cat/sch/model"}, From 0a9f3cbc574e02cdb69952a753b53631a50bcc82 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 20 May 2026 15:08:41 +0200 Subject: [PATCH 18/42] bundle: update genie_space test fixture to v2 export format The simple acceptance test fixture was a v1 serialized_space sample that the Genie backend now rejects with 409 ABORTED ("The export format has changed since this export was taken"). Bumps version to 2 and replaces get_example_values / build_value_dictionary with the v2-equivalent enable_format_assistance / enable_entity_matching, matching the format that bundle generate genie-space now produces. Co-authored-by: Isaac --- .../genie_spaces/simple/out.plan.json | 4 +- .../simple/sales_analytics.geniespace.json | 155 ++++++------------ 2 files changed, 55 insertions(+), 104 deletions(-) diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.plan.json b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json index b5583b47c06..0e97968436e 100644 --- a/acceptance/bundle/resources/genie_spaces/simple/out.plan.json +++ b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json @@ -2,14 +2,14 @@ "plan_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", - "serial": 1, + "serial": 6, "plan": { "resources.genie_spaces.sales_analytics": { "action": "skip", "remote_state": { "description": "AI assistant for sales data analysis", "etag": "1", - "serialized_space": "{\n \"version\": 1,\n \"config\": {\n \"sample_questions\": [\n {\n \"id\": \"sq-001\",\n \"question\": [\"What is the total revenue?\"]\n },\n {\n \"id\": \"sq-002\",\n \"question\": [\"Show me the top customers\"]\n }\n ]\n },\n \"data_sources\": {\n \"tables\": [\n {\n \"identifier\": \"main.sales.orders\",\n \"column_configs\": [\n {\n \"column_name\": \"order_id\",\n \"get_example_values\": true,\n \"build_value_dictionary\": true\n },\n {\n \"column_name\": \"customer_id\",\n \"get_example_values\": true\n },\n {\n \"column_name\": \"amount\",\n \"get_example_values\": false\n }\n ]\n },\n {\n \"identifier\": \"main.sales.customers\"\n }\n ]\n },\n \"instructions\": {\n \"text_instructions\": [\n {\n \"id\": \"ti-001\",\n \"content\": [\n \"This genie space analyzes sales data.\\n\",\n \"Always filter by date when querying orders.\\n\",\n \"Use customer_name instead of customer_id in results.\"\n ]\n }\n ],\n \"example_question_sqls\": [\n {\n \"id\": \"eq-001\",\n \"question\": [\"What are the top customers by revenue?\"],\n \"sql\": [\n \"SELECT\\n\",\n \" c.customer_name,\\n\",\n \" SUM(o.amount) AS total_revenue\\n\",\n \"FROM main.sales.orders o\\n\",\n \"JOIN main.sales.customers c ON o.customer_id = c.id\\n\",\n \"WHERE o.order_date \u003e= :start_date\\n\",\n \"GROUP BY c.customer_name\\n\",\n \"ORDER BY total_revenue DESC\\n\",\n \"LIMIT :limit\"\n ],\n \"parameters\": [\n {\n \"name\": \"start_date\",\n \"type_hint\": \"STRING\",\n \"description\": [\"Start date for the analysis period\"],\n \"default_value\": {\n \"values\": [\"2024-01-01\"]\n }\n },\n {\n \"name\": \"limit\",\n \"type_hint\": \"INTEGER\",\n \"description\": [\"Number of customers to return\"],\n \"default_value\": {\n \"values\": [\"10\"]\n }\n }\n ]\n },\n {\n \"id\": \"eq-002\",\n \"question\": [\"Calculate daily revenue\"],\n \"sql\": [\n \"SELECT\\n\",\n \" order_date,\\n\",\n \" SUM(amount) AS daily_revenue\\n\",\n \"FROM main.sales.orders\\n\",\n \"GROUP BY order_date\\n\",\n \"ORDER BY order_date\"\n ]\n }\n ],\n \"sql_snippets\": {\n \"measures\": [\n {\n \"id\": \"m-001\",\n \"alias\": \"total_revenue\",\n \"sql\": [\"SUM(orders.amount)\"],\n \"display_name\": \"Total Revenue\"\n }\n ]\n },\n \"sql_functions\": [\n {\n \"id\": \"sf-001\",\n \"identifier\": \"main.analytics.calculate_churn\"\n }\n ]\n },\n \"benchmarks\": {\n \"questions\": [\n {\n \"id\": \"bq-001\",\n \"question\": [\"What is the monthly revenue trend?\"],\n \"answer\": [\n {\n \"format\": \"SQL\",\n \"content\": [\n \"SELECT\\n\",\n \" DATE_TRUNC('month', order_date) AS month,\\n\",\n \" SUM(amount) AS revenue\\n\",\n \"FROM main.sales.orders\\n\",\n \"GROUP BY 1\\n\",\n \"ORDER BY 1\"\n ]\n }\n ]\n }\n ]\n }\n}\n", + "serialized_space": "{\n \"benchmarks\": {\n \"questions\": [\n {\n \"answer\": [\n {\n \"content\": [\n \"SELECT\\n\",\n \" name,\\n\",\n \" country\\n\",\n \"FROM main.default.countries\\n\",\n \"ORDER BY name\"\n ],\n \"format\": \"SQL\"\n }\n ],\n \"id\": \"[NUMID]\",\n \"question\": [\n \"Show all names and countries\"\n ]\n }\n ]\n },\n \"config\": {\n \"sample_questions\": [\n {\n \"id\": \"[NUMID]\",\n \"question\": [\n \"List the names and countries\"\n ]\n },\n {\n \"id\": \"[NUMID]\",\n \"question\": [\n \"Which names are in Canada?\"\n ]\n }\n ]\n },\n \"data_sources\": {\n \"tables\": [\n {\n \"column_configs\": [\n {\n \"column_name\": \"country\",\n \"enable_entity_matching\": true,\n \"enable_format_assistance\": true\n },\n {\n \"column_name\": \"name\",\n \"enable_entity_matching\": true,\n \"enable_format_assistance\": true\n }\n ],\n \"identifier\": \"main.default.countries\"\n }\n ]\n },\n \"instructions\": {\n \"example_question_sqls\": [\n {\n \"id\": \"[NUMID]\",\n \"question\": [\n \"List the names and countries\"\n ],\n \"sql\": [\n \"SELECT\\n\",\n \" name,\\n\",\n \" country\\n\",\n \"FROM main.default.countries\\n\",\n \"ORDER BY name\"\n ]\n }\n ],\n \"text_instructions\": [\n {\n \"content\": [\n \"This genie space answers simple questions about people and their countries.\\n\",\n \"Use only the main.default.countries table.\\n\",\n \"Prefer returning the name and country columns directly.\"\n ],\n \"id\": \"[NUMID]\"\n }\n ]\n },\n \"version\": 2\n}\n", "space_id": "[GENIE_SPACE_ID]", "title": "Sales Analytics Genie", "warehouse_id": "test-warehouse-id" diff --git a/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.geniespace.json b/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.geniespace.json index 5d59dff96d5..fb62b7c4859 100644 --- a/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.geniespace.json +++ b/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.geniespace.json @@ -1,136 +1,87 @@ { - "version": 1, + "benchmarks": { + "questions": [ + { + "answer": [ + { + "content": [ + "SELECT\n", + " name,\n", + " country\n", + "FROM main.default.countries\n", + "ORDER BY name" + ], + "format": "SQL" + } + ], + "id": "88888888888888888888888888888888", + "question": [ + "Show all names and countries" + ] + } + ] + }, "config": { "sample_questions": [ { - "id": "sq-001", - "question": ["What is the total revenue?"] + "id": "11111111111111111111111111111111", + "question": [ + "List the names and countries" + ] }, { - "id": "sq-002", - "question": ["Show me the top customers"] + "id": "22222222222222222222222222222222", + "question": [ + "Which names are in Canada?" + ] } ] }, "data_sources": { "tables": [ { - "identifier": "main.sales.orders", "column_configs": [ { - "column_name": "order_id", - "get_example_values": true, - "build_value_dictionary": true - }, - { - "column_name": "customer_id", - "get_example_values": true + "column_name": "country", + "enable_entity_matching": true, + "enable_format_assistance": true }, { - "column_name": "amount", - "get_example_values": false + "column_name": "name", + "enable_entity_matching": true, + "enable_format_assistance": true } - ] - }, - { - "identifier": "main.sales.customers" + ], + "identifier": "main.default.countries" } ] }, "instructions": { - "text_instructions": [ - { - "id": "ti-001", - "content": [ - "This genie space analyzes sales data.\n", - "Always filter by date when querying orders.\n", - "Use customer_name instead of customer_id in results." - ] - } - ], "example_question_sqls": [ { - "id": "eq-001", - "question": ["What are the top customers by revenue?"], - "sql": [ - "SELECT\n", - " c.customer_name,\n", - " SUM(o.amount) AS total_revenue\n", - "FROM main.sales.orders o\n", - "JOIN main.sales.customers c ON o.customer_id = c.id\n", - "WHERE o.order_date >= :start_date\n", - "GROUP BY c.customer_name\n", - "ORDER BY total_revenue DESC\n", - "LIMIT :limit" + "id": "44444444444444444444444444444444", + "question": [ + "List the names and countries" ], - "parameters": [ - { - "name": "start_date", - "type_hint": "STRING", - "description": ["Start date for the analysis period"], - "default_value": { - "values": ["2024-01-01"] - } - }, - { - "name": "limit", - "type_hint": "INTEGER", - "description": ["Number of customers to return"], - "default_value": { - "values": ["10"] - } - } - ] - }, - { - "id": "eq-002", - "question": ["Calculate daily revenue"], "sql": [ "SELECT\n", - " order_date,\n", - " SUM(amount) AS daily_revenue\n", - "FROM main.sales.orders\n", - "GROUP BY order_date\n", - "ORDER BY order_date" + " name,\n", + " country\n", + "FROM main.default.countries\n", + "ORDER BY name" ] } ], - "sql_snippets": { - "measures": [ - { - "id": "m-001", - "alias": "total_revenue", - "sql": ["SUM(orders.amount)"], - "display_name": "Total Revenue" - } - ] - }, - "sql_functions": [ + "text_instructions": [ { - "id": "sf-001", - "identifier": "main.analytics.calculate_churn" + "content": [ + "This genie space answers simple questions about people and their countries.\n", + "Use only the main.default.countries table.\n", + "Prefer returning the name and country columns directly." + ], + "id": "33333333333333333333333333333333" } ] }, - "benchmarks": { - "questions": [ - { - "id": "bq-001", - "question": ["What is the monthly revenue trend?"], - "answer": [ - { - "format": "SQL", - "content": [ - "SELECT\n", - " DATE_TRUNC('month', order_date) AS month,\n", - " SUM(amount) AS revenue\n", - "FROM main.sales.orders\n", - "GROUP BY 1\n", - "ORDER BY 1" - ] - } - ] - } - ] - } + "version": 2 } From 03350e8f86bf7a328fced2ee526eff5297514713 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 20 May 2026 15:11:17 +0200 Subject: [PATCH 19/42] bundle: align genie_space generate with new dstate.DB Open signature The state DB API gained context, withRecovery and withWrite arguments on origin/main; mirror the dashboard generate command and use the same arguments. Also regenerates the simple acceptance plan output to pick up the WAL-implementation serial increment. Co-authored-by: Isaac --- .../resources/genie_spaces/simple/out.plan.json | 2 +- cmd/bundle/generate/genie_space.go | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.plan.json b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json index 0e97968436e..5a64ae32bd2 100644 --- a/acceptance/bundle/resources/genie_spaces/simple/out.plan.json +++ b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json @@ -2,7 +2,7 @@ "plan_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", - "serial": 6, + "serial": 7, "plan": { "resources.genie_spaces.sales_analytics": { "action": "skip", diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 9cc415ba43d..4d67db210c2 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -14,6 +14,8 @@ import ( "time" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/generate" "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/bundle/resources" @@ -358,16 +360,25 @@ func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { return } + var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(localPath); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + logdiag.LogError(ctx, err) + return + } + state = b.DeploymentBundle.ExportState(ctx) + } else { + var err error + state, err = terraform.ParseResourcesState(ctx, b) + if err != nil { logdiag.LogError(ctx, err) return } } bundle.ApplySeqContext(ctx, b, - statemgmt.Load(stateDesc.Engine), + statemgmt.Load(state), ) if logdiag.HasError(ctx) { return From e5d49dd478ddbd3dda66e815ff3bfde412b04628 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 4 Jun 2026 06:45:35 +0200 Subject: [PATCH 20/42] bundle: integrate genie_space after rebase onto main Adapt genie_space to changes that landed on main while this branch was open: - ResourceGenieSpace.DoDelete now takes the third state parameter (_ *resources.GenieSpaceConfig) to match main's IResource.DoDelete signature. Without it the adapter failed to initialize at runtime ("param count mismatch: interface 3, concrete 2"). - Bump the structwalk config.Root field-count guard from 5800 to 6000 to account for the new genie_space fields (count is now 5814). - Regenerate bundle docs, JSON schema field reference, and the genie plan fixture against the rebased tree. Co-authored-by: Isaac --- acceptance/bundle/refschema/out.fields.txt | 1 + .../genie_spaces/simple/out.plan.json | 2 +- bundle/direct/dresources/genie_space.go | 2 +- bundle/docsgen/output/reference.md | 1032 ++++++++++++++++- bundle/docsgen/output/resources.md | 613 +++++++++- libs/structs/structwalk/walktype_test.go | 2 +- 6 files changed, 1560 insertions(+), 92 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 42735528ec5..b6f0f177b3b 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -735,6 +735,7 @@ resources.external_locations.*.grants[*].principal string ALL resources.external_locations.*.grants[*].privileges []catalog.Privilege ALL resources.external_locations.*.grants[*].privileges[*] catalog.Privilege ALL resources.genie_spaces.*.description string ALL +resources.genie_spaces.*.etag string ALL resources.genie_spaces.*.file_path string INPUT resources.genie_spaces.*.id string INPUT resources.genie_spaces.*.lifecycle resources.Lifecycle INPUT diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.plan.json b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json index 5a64ae32bd2..2a260f21a04 100644 --- a/acceptance/bundle/resources/genie_spaces/simple/out.plan.json +++ b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json @@ -2,7 +2,7 @@ "plan_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", - "serial": 7, + "serial": 1, "plan": { "resources.genie_spaces.sales_analytics": { "action": "skip", diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go index 395466fda2b..b45cde61a0b 100644 --- a/bundle/direct/dresources/genie_space.go +++ b/bundle/direct/dresources/genie_space.go @@ -292,7 +292,7 @@ func hasUpdate(entry *PlanEntry, path *structpath.PathNode) bool { return false } -func (r *ResourceGenieSpace) DoDelete(ctx context.Context, id string) error { +func (r *ResourceGenieSpace) DoDelete(ctx context.Context, id string, _ *resources.GenieSpaceConfig) error { return r.client.Genie.TrashSpace(ctx, dashboards.GenieTrashSpaceRequest{ SpaceId: id, }) diff --git a/bundle/docsgen/output/reference.md b/bundle/docsgen/output/reference.md index b3e86d6f305..92228810a38 100644 --- a/bundle/docsgen/output/reference.md +++ b/bundle/docsgen/output/reference.md @@ -1,7 +1,7 @@ --- description: 'Configuration reference for databricks.yml' last_update: - date: 2026-05-20 + date: 2026-06-04 --- @@ -242,6 +242,10 @@ Defines attributes for experimental features. - Boolean - Whether to use a Python wheel wrapper. +- - `record_deployment_history` + - Boolean + - Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments. + - - `scripts` - Map - The commands to run. @@ -493,12 +497,16 @@ resources: - - `pipelines` - Map - - The pipeline definitions for the bundle, where each key is the name of the pipeline. See [\_](/dev-tools/bundles/resources.md#pipelines). + - The pipeline definitions for the bundle, where each key is the name of the pipeline. See [\_](/dev-tools/bundles/resources.md#pipelines). See [\_](#resourcespipelines). - - `postgres_branches` - Map - See [\_](#resourcespostgres_branches). +- - `postgres_catalogs` + - Map + - The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch. See [\_](#resourcespostgres_catalogs). + - - `postgres_endpoints` - Map - See [\_](#resourcespostgres_endpoints). @@ -507,6 +515,10 @@ resources: - Map - See [\_](#resourcespostgres_projects). +- - `postgres_synced_tables` + - Map + - The Postgres synced table definitions for the bundle, where each key is the name of the synced table. Each entry continuously replicates a Unity Catalog Delta source table into a Postgres table on a Lakebase Autoscaling instance. See [\_](#resourcespostgres_synced_tables). + - - `quality_monitors` - Map - The quality monitor definitions for the bundle, where each key is the name of the quality monitor. See [\_](/dev-tools/bundles/resources.md#quality_monitors). @@ -535,6 +547,10 @@ resources: - Map - See [\_](#resourcesvector_search_endpoints). +- - `vector_search_indexes` + - Map + - See [\_](#resourcesvector_search_indexes). + - - `volumes` - Map - The volume definitions for the bundle, where each key is the name of the volume. See [\_](/dev-tools/bundles/resources.md#volumes). @@ -725,6 +741,14 @@ apps: - String - +- - `compute_max_instances` + - Integer + - + +- - `compute_min_instances` + - Integer + - + - - `compute_size` - String - @@ -1598,6 +1622,218 @@ genie_spaces: ::: +### resources.pipelines + +**`Type: Map`** + +The pipeline definitions for the bundle, where each key is the name of the pipeline. See [\_](/dev-tools/bundles/resources.md#pipelines). + +```yaml +pipelines: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `allow_duplicate_names` + - Boolean + - + +- - `budget_policy_id` + - String + - + +- - `catalog` + - String + - + +- - `channel` + - String + - + +- - `clusters` + - Sequence + - See [\_](#resourcespipelinesnameclusters). + +- - `configuration` + - Map + - + +- - `continuous` + - Boolean + - + +- - `deployment` + - Map + - See [\_](#resourcespipelinesnamedeployment). + +- - `development` + - Boolean + - + +- - `dry_run` + - Boolean + - + +- - `edition` + - String + - + +- - `environment` + - Map + - See [\_](#resourcespipelinesnameenvironment). + +- - `event_log` + - Map + - See [\_](#resourcespipelinesnameevent_log). + +- - `filters` + - Map + - See [\_](#resourcespipelinesnamefilters). + +- - `gateway_definition` + - Map + - See [\_](#resourcespipelinesnamegateway_definition). + +- - `id` + - String + - + +- - `ingestion_definition` + - Map + - See [\_](#resourcespipelinesnameingestion_definition). + +- - `libraries` + - Sequence + - See [\_](#resourcespipelinesnamelibraries). + +- - `lifecycle` + - Map + - See [\_](#resourcespipelinesnamelifecycle). + +- - `name` + - String + - + +- - `notifications` + - Sequence + - See [\_](#resourcespipelinesnamenotifications). + +- - `parameters` + - Map + - + +- - `permissions` + - Sequence + - See [\_](#resourcespipelinesnamepermissions). + +- - `photon` + - Boolean + - + +- - `restart_window` + - Map + - See [\_](#resourcespipelinesnamerestart_window). + +- - `root_path` + - String + - + +- - `run_as` + - Map + - See [\_](#resourcespipelinesnamerun_as). + +- - `schema` + - String + - + +- - `serverless` + - Boolean + - + +- - `storage` + - String + - + +- - `tags` + - Map + - + +- - `target` + - String + - + +- - `trigger` + - Map + - See [\_](#resourcespipelinesnametrigger). + +- - `usage_policy_id` + - String + - + +::: + + +### resources.pipelines._name_.lifecycle + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + +### resources.pipelines._name_.permissions + +**`Type: Sequence`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `group_name` + - String + - + +- - `level` + - String + - + +- - `service_principal_name` + - String + - + +- - `user_name` + - String + - + +::: + + ### resources.postgres_branches **`Type: Map`** @@ -1641,6 +1877,10 @@ postgres_branches: - String - +- - `replace_existing` + - Boolean + - + - - `source_branch` - String - @@ -1668,6 +1908,69 @@ postgres_branches: +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + +### resources.postgres_catalogs + +**`Type: Map`** + +The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch. + +```yaml +postgres_catalogs: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `branch` + - String + - + +- - `catalog_id` + - String + - + +- - `create_database_if_missing` + - Boolean + - + +- - `lifecycle` + - Map + - See [\_](#resourcespostgres_catalogsnamelifecycle). + +- - `postgres_database` + - String + - + +::: + + +### resources.postgres_catalogs._name_.lifecycle + +**`Type: Map`** + + + + + :::list-table - - Key @@ -1736,6 +2039,10 @@ postgres_endpoints: - String - +- - `replace_existing` + - Boolean + - + - - `settings` - Map - See [\_](#resourcespostgres_endpointsnamesettings). @@ -1831,6 +2138,10 @@ postgres_projects: - String - +- - `purge_on_delete` + - Boolean + - + ::: @@ -1888,16 +2199,16 @@ postgres_projects: ::: -### resources.secret_scopes +### resources.postgres_synced_tables **`Type: Map`** -The secret scope definitions for the bundle, where each key is the name of the secret scope. See [\_](/dev-tools/bundles/resources.md#secret_scopes). +The Postgres synced table definitions for the bundle, where each key is the name of the synced table. Each entry continuously replicates a Unity Catalog Delta source table into a Postgres table on a Lakebase Autoscaling instance. ```yaml -secret_scopes: - : - : +postgres_synced_tables: + : + : ``` @@ -1907,13 +2218,100 @@ secret_scopes: - Type - Description -- - `backend_type` +- - `branch` - String - - The backend type the scope will be created with. If not specified, will default to `DATABRICKS` + - + +- - `create_database_objects_if_missing` + - Boolean + - + +- - `existing_pipeline_id` + - String + - + +- - `lifecycle` + - Map + - See [\_](#resourcespostgres_synced_tablesnamelifecycle). + +- - `new_pipeline_spec` + - Map + - See [\_](#resourcespostgres_synced_tablesnamenew_pipeline_spec). + +- - `postgres_database` + - String + - + +- - `primary_key_columns` + - Sequence + - + +- - `scheduling_policy` + - String + - + +- - `source_table_full_name` + - String + - + +- - `synced_table_id` + - String + - + +- - `timeseries_key` + - String + - + +::: + + +### resources.postgres_synced_tables._name_.lifecycle + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + +### resources.secret_scopes + +**`Type: Map`** + +The secret scope definitions for the bundle, where each key is the name of the secret scope. See [\_](/dev-tools/bundles/resources.md#secret_scopes). + +```yaml +secret_scopes: + : + : +``` -- - `keyvault_metadata` - - Map - - The metadata for the secret scope if the `backend_type` is `AZURE_KEYVAULT`. See [\_](#resourcessecret_scopesnamekeyvault_metadata). + +:::list-table + +- - Key + - Type + - Description + +- - `backend_type` + - String + - The backend type the scope will be created with. If not specified, will default to `DATABRICKS` + +- - `keyvault_metadata` + - Map + - The metadata for the secret scope if the `backend_type` is `AZURE_KEYVAULT`. See [\_](#resourcessecret_scopesnamekeyvault_metadata). - - `lifecycle` - Map @@ -2167,6 +2565,85 @@ vector_search_endpoints: ::: +### resources.vector_search_indexes + +**`Type: Map`** + + + +```yaml +vector_search_indexes: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `delta_sync_index_spec` + - Map + - See [\_](#resourcesvector_search_indexesnamedelta_sync_index_spec). + +- - `direct_access_index_spec` + - Map + - See [\_](#resourcesvector_search_indexesnamedirect_access_index_spec). + +- - `endpoint_name` + - String + - + +- - `grants` + - Sequence + - See [\_](#resourcesvector_search_indexesnamegrants). + +- - `index_subtype` + - String + - + +- - `index_type` + - String + - + +- - `lifecycle` + - Map + - See [\_](#resourcesvector_search_indexesnamelifecycle). + +- - `name` + - String + - + +- - `primary_key` + - String + - + +::: + + +### resources.vector_search_indexes._name_.lifecycle + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + ## run_as **`Type: Map`** @@ -2691,12 +3168,16 @@ The resource definitions for the target. - - `pipelines` - Map - - The pipeline definitions for the bundle, where each key is the name of the pipeline. See [\_](/dev-tools/bundles/resources.md#pipelines). + - The pipeline definitions for the bundle, where each key is the name of the pipeline. See [\_](/dev-tools/bundles/resources.md#pipelines). See [\_](#targetsnameresourcespipelines). - - `postgres_branches` - Map - See [\_](#targetsnameresourcespostgres_branches). +- - `postgres_catalogs` + - Map + - The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch. See [\_](#targetsnameresourcespostgres_catalogs). + - - `postgres_endpoints` - Map - See [\_](#targetsnameresourcespostgres_endpoints). @@ -2705,6 +3186,10 @@ The resource definitions for the target. - Map - See [\_](#targetsnameresourcespostgres_projects). +- - `postgres_synced_tables` + - Map + - The Postgres synced table definitions for the bundle, where each key is the name of the synced table. Each entry continuously replicates a Unity Catalog Delta source table into a Postgres table on a Lakebase Autoscaling instance. See [\_](#targetsnameresourcespostgres_synced_tables). + - - `quality_monitors` - Map - The quality monitor definitions for the bundle, where each key is the name of the quality monitor. See [\_](/dev-tools/bundles/resources.md#quality_monitors). @@ -2733,6 +3218,10 @@ The resource definitions for the target. - Map - See [\_](#targetsnameresourcesvector_search_endpoints). +- - `vector_search_indexes` + - Map + - See [\_](#targetsnameresourcesvector_search_indexes). + - - `volumes` - Map - The volume definitions for the bundle, where each key is the name of the volume. See [\_](/dev-tools/bundles/resources.md#volumes). @@ -2923,6 +3412,14 @@ apps: - String - +- - `compute_max_instances` + - Integer + - + +- - `compute_min_instances` + - Integer + - + - - `compute_size` - String - @@ -3796,16 +4293,16 @@ genie_spaces: ::: -### targets._name_.resources.postgres_branches +### targets._name_.resources.pipelines **`Type: Map`** - +The pipeline definitions for the bundle, where each key is the name of the pipeline. See [\_](/dev-tools/bundles/resources.md#pipelines). ```yaml -postgres_branches: - : - : +pipelines: + : + : ``` @@ -3815,50 +4312,146 @@ postgres_branches: - Type - Description -- - `branch_id` +- - `allow_duplicate_names` + - Boolean + - + +- - `budget_policy_id` - String - -- - `expire_time` +- - `catalog` + - String + - + +- - `channel` + - String + - + +- - `clusters` + - Sequence + - See [\_](#targetsnameresourcespipelinesnameclusters). + +- - `configuration` - Map - -- - `is_protected` +- - `continuous` + - Boolean + - + +- - `deployment` + - Map + - See [\_](#targetsnameresourcespipelinesnamedeployment). + +- - `development` + - Boolean + - + +- - `dry_run` - Boolean - +- - `edition` + - String + - + +- - `environment` + - Map + - See [\_](#targetsnameresourcespipelinesnameenvironment). + +- - `event_log` + - Map + - See [\_](#targetsnameresourcespipelinesnameevent_log). + +- - `filters` + - Map + - See [\_](#targetsnameresourcespipelinesnamefilters). + +- - `gateway_definition` + - Map + - See [\_](#targetsnameresourcespipelinesnamegateway_definition). + +- - `id` + - String + - + +- - `ingestion_definition` + - Map + - See [\_](#targetsnameresourcespipelinesnameingestion_definition). + +- - `libraries` + - Sequence + - See [\_](#targetsnameresourcespipelinesnamelibraries). + - - `lifecycle` - Map - - See [\_](#targetsnameresourcespostgres_branchesnamelifecycle). + - See [\_](#targetsnameresourcespipelinesnamelifecycle). -- - `no_expiry` +- - `name` + - String + - + +- - `notifications` + - Sequence + - See [\_](#targetsnameresourcespipelinesnamenotifications). + +- - `parameters` + - Map + - + +- - `permissions` + - Sequence + - See [\_](#targetsnameresourcespipelinesnamepermissions). + +- - `photon` - Boolean - -- - `parent` +- - `restart_window` + - Map + - See [\_](#targetsnameresourcespipelinesnamerestart_window). + +- - `root_path` - String - -- - `source_branch` +- - `run_as` + - Map + - See [\_](#targetsnameresourcespipelinesnamerun_as). + +- - `schema` - String - -- - `source_branch_lsn` +- - `serverless` + - Boolean + - + +- - `storage` - String - -- - `source_branch_time` +- - `tags` - Map - -- - `ttl` +- - `target` + - String + - + +- - `trigger` + - Map + - See [\_](#targetsnameresourcespipelinesnametrigger). + +- - `usage_policy_id` - String - ::: -### targets._name_.resources.postgres_branches._name_.lifecycle +### targets._name_.resources.pipelines._name_.lifecycle **`Type: Map`** @@ -3879,17 +4472,12 @@ postgres_branches: ::: -### targets._name_.resources.postgres_endpoints +### targets._name_.resources.pipelines._name_.permissions -**`Type: Map`** +**`Type: Sequence`** -```yaml -postgres_endpoints: - : - : -``` :::list-table @@ -3898,15 +4486,203 @@ postgres_endpoints: - Type - Description -- - `autoscaling_limit_max_cu` - - Any +- - `group_name` + - String - -- - `autoscaling_limit_min_cu` - - Any +- - `level` + - String - -- - `disabled` +- - `service_principal_name` + - String + - + +- - `user_name` + - String + - + +::: + + +### targets._name_.resources.postgres_branches + +**`Type: Map`** + + + +```yaml +postgres_branches: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `branch_id` + - String + - + +- - `expire_time` + - Map + - + +- - `is_protected` + - Boolean + - + +- - `lifecycle` + - Map + - See [\_](#targetsnameresourcespostgres_branchesnamelifecycle). + +- - `no_expiry` + - Boolean + - + +- - `parent` + - String + - + +- - `replace_existing` + - Boolean + - + +- - `source_branch` + - String + - + +- - `source_branch_lsn` + - String + - + +- - `source_branch_time` + - Map + - + +- - `ttl` + - String + - + +::: + + +### targets._name_.resources.postgres_branches._name_.lifecycle + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + +### targets._name_.resources.postgres_catalogs + +**`Type: Map`** + +The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch. + +```yaml +postgres_catalogs: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `branch` + - String + - + +- - `catalog_id` + - String + - + +- - `create_database_if_missing` + - Boolean + - + +- - `lifecycle` + - Map + - See [\_](#targetsnameresourcespostgres_catalogsnamelifecycle). + +- - `postgres_database` + - String + - + +::: + + +### targets._name_.resources.postgres_catalogs._name_.lifecycle + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + +### targets._name_.resources.postgres_endpoints + +**`Type: Map`** + + + +```yaml +postgres_endpoints: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `autoscaling_limit_max_cu` + - Any + - + +- - `autoscaling_limit_min_cu` + - Any + - + +- - `disabled` - Boolean - @@ -3934,6 +4710,10 @@ postgres_endpoints: - String - +- - `replace_existing` + - Boolean + - + - - `settings` - Map - See [\_](#targetsnameresourcespostgres_endpointsnamesettings). @@ -4029,6 +4809,10 @@ postgres_projects: - String - +- - `purge_on_delete` + - Boolean + - + ::: @@ -4086,6 +4870,93 @@ postgres_projects: ::: +### targets._name_.resources.postgres_synced_tables + +**`Type: Map`** + +The Postgres synced table definitions for the bundle, where each key is the name of the synced table. Each entry continuously replicates a Unity Catalog Delta source table into a Postgres table on a Lakebase Autoscaling instance. + +```yaml +postgres_synced_tables: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `branch` + - String + - + +- - `create_database_objects_if_missing` + - Boolean + - + +- - `existing_pipeline_id` + - String + - + +- - `lifecycle` + - Map + - See [\_](#targetsnameresourcespostgres_synced_tablesnamelifecycle). + +- - `new_pipeline_spec` + - Map + - See [\_](#targetsnameresourcespostgres_synced_tablesnamenew_pipeline_spec). + +- - `postgres_database` + - String + - + +- - `primary_key_columns` + - Sequence + - + +- - `scheduling_policy` + - String + - + +- - `source_table_full_name` + - String + - + +- - `synced_table_id` + - String + - + +- - `timeseries_key` + - String + - + +::: + + +### targets._name_.resources.postgres_synced_tables._name_.lifecycle + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + ### targets._name_.resources.secret_scopes **`Type: Map`** @@ -4365,6 +5236,85 @@ vector_search_endpoints: ::: +### targets._name_.resources.vector_search_indexes + +**`Type: Map`** + + + +```yaml +vector_search_indexes: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `delta_sync_index_spec` + - Map + - See [\_](#targetsnameresourcesvector_search_indexesnamedelta_sync_index_spec). + +- - `direct_access_index_spec` + - Map + - See [\_](#targetsnameresourcesvector_search_indexesnamedirect_access_index_spec). + +- - `endpoint_name` + - String + - + +- - `grants` + - Sequence + - See [\_](#targetsnameresourcesvector_search_indexesnamegrants). + +- - `index_subtype` + - String + - + +- - `index_type` + - String + - + +- - `lifecycle` + - Map + - See [\_](#targetsnameresourcesvector_search_indexesnamelifecycle). + +- - `name` + - String + - + +- - `primary_key` + - String + - + +::: + + +### targets._name_.resources.vector_search_indexes._name_.lifecycle + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + ### targets._name_.run_as **`Type: Map`** diff --git a/bundle/docsgen/output/resources.md b/bundle/docsgen/output/resources.md index 4d740961a6a..bce0e3f9fb7 100644 --- a/bundle/docsgen/output/resources.md +++ b/bundle/docsgen/output/resources.md @@ -1,7 +1,7 @@ --- description: 'Learn about resources supported by Declarative Automation Bundles and how to configure them.' last_update: - date: 2026-05-20 + date: 2026-06-04 --- @@ -532,6 +532,14 @@ apps: - String - +- - `compute_max_instances` + - Integer + - + +- - `compute_min_instances` + - Integer + - + - - `compute_size` - String - @@ -548,10 +556,6 @@ apps: - Map - Git repository configuration for app deployments. When specified, deployments can reference code from this repository by providing only the git reference (branch, tag, or commit). See [\_](#appsnamegit_repository). -- - `git_source` - - Map - - Git source configuration for app deployments. Specifies which git reference (branch, tag, or commit) to use when deploying the app. Used in conjunction with git_repository to deploy code directly from git. The source_code_path within git_source specifies the relative path to the app code within the repository. See [\_](#appsnamegit_source). - - - `lifecycle` - Map - Lifecycle is a struct that contains the lifecycle settings for a resource. It controls the behavior of the resource when it is deployed or destroyed. See [\_](#appsnamelifecycle). @@ -568,10 +572,6 @@ apps: - Sequence - Resources for the app. See [\_](#appsnameresources). -- - `source_code_path` - - String - - - - - `telemetry_export_destinations` - Sequence - See [\_](#appsnametelemetry_export_destinations). @@ -667,41 +667,6 @@ reference code from this repository by providing only the git reference (branch, ::: -### apps._name_.git_source - -**`Type: Map`** - -Git source configuration for app deployments. Specifies which git reference (branch, tag, or commit) -to use when deploying the app. Used in conjunction with git_repository to deploy code directly from git. -The source_code_path within git_source specifies the relative path to the app code within the repository. - - - -:::list-table - -- - Key - - Type - - Description - -- - `branch` - - String - - Git branch to checkout. - -- - `commit` - - String - - Git commit SHA to checkout. - -- - `source_code_path` - - String - - Relative path to the app source code within the Git repository. If not specified, the root of the repository is used. - -- - `tag` - - String - - Git tag to checkout. - -::: - - ### apps._name_.lifecycle **`Type: Map`** @@ -1375,7 +1340,7 @@ clusters: - - `data_security_mode` - String - - Data security mode decides what data governance model to use when accessing data from a cluster. The following modes can only be used when `kind = CLASSIC_PREVIEW`. * `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration. * `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. * `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`. The following modes can be used regardless of `kind`. * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode. * `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. * `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters. * `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled. + - Data security mode decides what data governance model to use when accessing data from a cluster. * `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration. * `DATA_SECURITY_MODE_STANDARD`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other’s data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited. * `DATA_SECURITY_MODE_DEDICATED`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. The following modes are legacy aliases for the above modes: * `USER_ISOLATION`: Legacy alias for `DATA_SECURITY_MODE_STANDARD`. * `SINGLE_USER`: Legacy alias for `DATA_SECURITY_MODE_DEDICATED`. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters. * `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled. - - `docker_image` - Map @@ -2123,6 +2088,10 @@ Lifecycle is a struct that contains the lifecycle settings for a resource. It co - Boolean - Lifecycle setting to prevent the resource from being destroyed. +- - `started` + - Boolean + - Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. + ::: @@ -3415,6 +3384,10 @@ Deployment information for jobs managed by external sources. - Type - Description +- - `deployment_id` + - String + - + - - `kind` - String - The kind of deployment that manages the job. * `BUNDLE`: The job is managed by Databricks Asset Bundle. * `SYSTEM_MANAGED`: The job is managed by Databricks and is read-only. @@ -3423,6 +3396,10 @@ Deployment information for jobs managed by external sources. - String - Path of the file that contains deployment metadata. +- - `version_id` + - String + - + ::: @@ -3511,7 +3488,7 @@ In this minimal environment spec, only pip and java dependencies are supported. - - `base_environment` - String - - The base environment this environment is built on top of. A base environment defines the environment version and a list of dependencies for serverless compute. The value can be a file path to a custom `env.yaml` file (e.g., `/Workspace/path/to/env.yaml`). Support for a Databricks-provided base environment ID (e.g., `workspace-base-environments/databricks_ai_v4`) and workspace base environment ID (e.g., `workspace-base-environments/dbe_b849b66e-b31a-4cb5-b161-1f2b10877fb7`) is in Beta. Either `environment_version` or `base_environment` can be provided. For more information, see + - The base environment this environment is built on top of. A base environment defines the environment version and a list of dependencies for serverless compute. The value can be a file path to a custom `env.yaml` file (e.g., `/Workspace/path/to/env.yaml`). Support for a Databricks-provided base environment ID (e.g., `workspace-base-environments/databricks_ai_v4`) and workspace base environment ID (e.g., `workspace-base-environments/dbe_b849b66e-b31a-4cb5-b161-1f2b10877fb7`) is in Beta. Either `environment_version` or `base_environment` can be provided. For more information about Databricks-provided base environments, see the [list workspace base environments](:method:Environments/ListWorkspaceBaseEnvironments) API. For more information, see - - `client` - String @@ -3746,7 +3723,7 @@ If new_cluster, a description of a cluster that is created for each task. - - `data_security_mode` - String - - Data security mode decides what data governance model to use when accessing data from a cluster. The following modes can only be used when `kind = CLASSIC_PREVIEW`. * `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration. * `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. * `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`. The following modes can be used regardless of `kind`. * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode. * `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. * `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters. * `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled. + - Data security mode decides what data governance model to use when accessing data from a cluster. * `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration. * `DATA_SECURITY_MODE_STANDARD`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other’s data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited. * `DATA_SECURITY_MODE_DEDICATED`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. The following modes are legacy aliases for the above modes: * `USER_ISOLATION`: Legacy alias for `DATA_SECURITY_MODE_STANDARD`. * `SINGLE_USER`: Legacy alias for `DATA_SECURITY_MODE_DEDICATED`. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters. * `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled. - - `docker_image` - Map @@ -4740,6 +4717,10 @@ Read endpoints return only 100 tasks. If more than 100 tasks are available, you - Boolean - An option to disable auto optimization in serverless +- - `disabled` + - Boolean + - An optional flag to disable the task. If set to true, the task will not run even if it is part of a job. + - - `email_notifications` - Map - An optional set of email addresses that is notified when runs of this task begin or complete as well as when this task is deleted. The default behavior is to not send any emails. See [\_](#jobsnametasksemail_notifications). @@ -5437,7 +5418,7 @@ If new_cluster, a description of a new cluster that is created for each run. - - `data_security_mode` - String - - Data security mode decides what data governance model to use when accessing data from a cluster. The following modes can only be used when `kind = CLASSIC_PREVIEW`. * `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration. * `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. * `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`. The following modes can be used regardless of `kind`. * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode. * `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. * `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters. * `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled. + - Data security mode decides what data governance model to use when accessing data from a cluster. * `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration. * `DATA_SECURITY_MODE_STANDARD`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other’s data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited. * `DATA_SECURITY_MODE_DEDICATED`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. The following modes are legacy aliases for the above modes: * `USER_ISOLATION`: Legacy alias for `DATA_SECURITY_MODE_STANDARD`. * `SINGLE_USER`: Legacy alias for `DATA_SECURITY_MODE_DEDICATED`. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters. * `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled. - - `docker_image` - Map @@ -6278,10 +6259,30 @@ The task triggers a pipeline update when the `pipeline_task` field is present. O - Boolean - If true, triggers a full refresh on the spark declarative pipeline. +- - `full_refresh_selection` + - Sequence + - A list of tables to update with fullRefresh. + +- - `parameters` + - Map + - + - - `pipeline_id` - String - The full name of the pipeline task to execute. +- - `refresh_flow_selection` + - Sequence + - Flow names to selectively refresh. These are unioned with other selective refresh options (refresh_selection, full_refresh_selection) to determine the final set of flows to refresh. + +- - `refresh_selection` + - Sequence + - A list of tables to update without fullRefresh. + +- - `reset_checkpoint_selection` + - Sequence + - A list of streaming flows to reset checkpoints without clearing data. + ::: @@ -6472,6 +6473,22 @@ Controls whether the pipeline should perform a full refresh - Boolean - If true, triggers a full refresh on the spark declarative pipeline. +- - `full_refresh_selection` + - Sequence + - A list of tables to update with fullRefresh. + +- - `refresh_flow_selection` + - Sequence + - Flow names to selectively refresh. These are unioned with other selective refresh options (refresh_selection, full_refresh_selection) to determine the final set of flows to refresh. + +- - `refresh_selection` + - Sequence + - A list of tables to update without fullRefresh. + +- - `reset_checkpoint_selection` + - Sequence + - A list of streaming flows to reset checkpoints without clearing data. + ::: @@ -8500,6 +8517,10 @@ pipelines: - Sequence - List of notification settings for this pipeline. See [\_](#pipelinesnamenotifications). +- - `parameters` + - Map + - + - - `permissions` - Sequence - See [\_](#pipelinesnamepermissions). @@ -9192,6 +9213,10 @@ Deployment type of this pipeline. - Type - Description +- - `deployment_id` + - String + - + - - `kind` - String - The deployment method that manages the pipeline. @@ -9200,6 +9225,10 @@ Deployment type of this pipeline. - String - The path to the file containing metadata about the deployment. +- - `version_id` + - String + - + ::: @@ -10547,6 +10576,10 @@ postgres_branches: - String - +- - `replace_existing` + - Boolean + - + - - `source_branch` - String - @@ -10574,6 +10607,69 @@ postgres_branches: +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + +## postgres_catalogs + +**`Type: Map`** + +The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch. + +```yaml +postgres_catalogs: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `branch` + - String + - + +- - `catalog_id` + - String + - + +- - `create_database_if_missing` + - Boolean + - + +- - `lifecycle` + - Map + - See [\_](#postgres_catalogsnamelifecycle). + +- - `postgres_database` + - String + - + +::: + + +### postgres_catalogs._name_.lifecycle + +**`Type: Map`** + + + + + :::list-table - - Key @@ -10642,6 +10738,10 @@ postgres_endpoints: - String - +- - `replace_existing` + - Boolean + - + - - `settings` - Map - A collection of settings for a compute endpoint. See [\_](#postgres_endpointsnamesettings). @@ -10787,6 +10887,10 @@ postgres_projects: - String - +- - `purge_on_delete` + - Boolean + - + ::: @@ -10906,6 +11010,122 @@ A collection of settings for a compute endpoint. ::: +## postgres_synced_tables + +**`Type: Map`** + +The Postgres synced table definitions for the bundle, where each key is the name of the synced table. Each entry continuously replicates a Unity Catalog Delta source table into a Postgres table on a Lakebase Autoscaling instance. + +```yaml +postgres_synced_tables: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `branch` + - String + - + +- - `create_database_objects_if_missing` + - Boolean + - + +- - `existing_pipeline_id` + - String + - + +- - `lifecycle` + - Map + - See [\_](#postgres_synced_tablesnamelifecycle). + +- - `new_pipeline_spec` + - Map + - See [\_](#postgres_synced_tablesnamenew_pipeline_spec). + +- - `postgres_database` + - String + - + +- - `primary_key_columns` + - Sequence + - + +- - `scheduling_policy` + - String + - Scheduling policy of the synced table's underlying pipeline. + +- - `source_table_full_name` + - String + - + +- - `synced_table_id` + - String + - + +- - `timeseries_key` + - String + - + +::: + + +### postgres_synced_tables._name_.lifecycle + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + +### postgres_synced_tables._name_.new_pipeline_spec + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `budget_policy_id` + - String + - Budget policy to set on the newly created pipeline. + +- - `storage_catalog` + - String + - UC catalog for the pipeline to store intermediate files (checkpoints, event logs etc). This needs to be a standard catalog where the user has permissions to create Delta tables. + +- - `storage_schema` + - String + - UC schema for the pipeline to store intermediate files (checkpoints, event logs etc). This needs to be in the standard catalog where the user has permissions to create Delta tables. + +::: + + ## quality_monitors **`Type: Map`** @@ -11809,6 +12029,10 @@ Lifecycle is a struct that contains the lifecycle settings for a resource. It co - Boolean - Lifecycle setting to prevent the resource from being destroyed. +- - `started` + - Boolean + - Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. + ::: @@ -12137,6 +12361,299 @@ vector_search_endpoints: ::: +## vector_search_indexes + +**`Type: Map`** + + + +```yaml +vector_search_indexes: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `delta_sync_index_spec` + - Map + - See [\_](#vector_search_indexesnamedelta_sync_index_spec). + +- - `direct_access_index_spec` + - Map + - See [\_](#vector_search_indexesnamedirect_access_index_spec). + +- - `endpoint_name` + - String + - + +- - `grants` + - Sequence + - See [\_](#vector_search_indexesnamegrants). + +- - `index_subtype` + - String + - + +- - `index_type` + - String + - + +- - `lifecycle` + - Map + - See [\_](#vector_search_indexesnamelifecycle). + +- - `name` + - String + - + +- - `primary_key` + - String + - + +::: + + +### vector_search_indexes._name_.delta_sync_index_spec + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `columns_to_index` + - Sequence + - + +- - `columns_to_sync` + - Sequence + - + +- - `embedding_source_columns` + - Sequence + - See [\_](#vector_search_indexesnamedelta_sync_index_specembedding_source_columns). + +- - `embedding_vector_columns` + - Sequence + - See [\_](#vector_search_indexesnamedelta_sync_index_specembedding_vector_columns). + +- - `embedding_writeback_table` + - String + - + +- - `pipeline_type` + - String + - + +- - `source_table` + - String + - + +::: + + +### vector_search_indexes._name_.delta_sync_index_spec.embedding_source_columns + +**`Type: Sequence`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `embedding_model_endpoint_name` + - String + - + +- - `model_endpoint_name_for_query` + - String + - + +- - `name` + - String + - + +::: + + +### vector_search_indexes._name_.delta_sync_index_spec.embedding_vector_columns + +**`Type: Sequence`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `embedding_dimension` + - Integer + - + +- - `name` + - String + - + +::: + + +### vector_search_indexes._name_.direct_access_index_spec + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `embedding_source_columns` + - Sequence + - See [\_](#vector_search_indexesnamedirect_access_index_specembedding_source_columns). + +- - `embedding_vector_columns` + - Sequence + - See [\_](#vector_search_indexesnamedirect_access_index_specembedding_vector_columns). + +- - `schema_json` + - String + - + +::: + + +### vector_search_indexes._name_.direct_access_index_spec.embedding_source_columns + +**`Type: Sequence`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `embedding_model_endpoint_name` + - String + - + +- - `model_endpoint_name_for_query` + - String + - + +- - `name` + - String + - + +::: + + +### vector_search_indexes._name_.direct_access_index_spec.embedding_vector_columns + +**`Type: Sequence`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `embedding_dimension` + - Integer + - + +- - `name` + - String + - + +::: + + +### vector_search_indexes._name_.grants + +**`Type: Sequence`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `principal` + - String + - The principal (user email address or group name). For deleted principals, `principal` is empty while `principal_id` is populated. + +- - `privileges` + - Sequence + - The privileges assigned to the principal. + +::: + + +### vector_search_indexes._name_.grants.privileges + +**`Type: Sequence`** + +The privileges assigned to the principal. + + +### vector_search_indexes._name_.lifecycle + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + ## volumes **`Type: Map`** diff --git a/libs/structs/structwalk/walktype_test.go b/libs/structs/structwalk/walktype_test.go index 5db64efc0d7..51069fc1f42 100644 --- a/libs/structs/structwalk/walktype_test.go +++ b/libs/structs/structwalk/walktype_test.go @@ -136,7 +136,7 @@ func TestTypeJobSettings(t *testing.T) { func TestTypeRoot(t *testing.T) { testStruct(t, reflect.TypeFor[config.Root](), - 5000, 5800, // 5651 after SDK v0.136.0 bump + 5000, 6000, // 5814 after genie_space resource added map[string]any{ "bundle.target": "", `variables.*.lookup.dashboard`: "", From 2b186cb4d58273644d41155d97b734a90c736b67 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 4 Jun 2026 08:21:18 +0200 Subject: [PATCH 21/42] bundle: address genie_space PR review comments Apply the unresolved self-review comments on PR #5282: - Permissions use object type "genie" (endpoint /permissions/genie/{id}) rather than "genie/spaces" (permissions.go, all_test.go, and the testserver object-type map). - parent_path is a normal, updatable field now that the Genie GET API returns it (without the /Workspace prefix) and the update API accepts it: * DoRead re-adds the /Workspace prefix via the shared ensureWorkspacePrefix (mirrors ResourceDashboard) instead of clearing the field. * DoUpdate sends parent_path so the backend can move the space. * Removed parent_path from recreate_on_changes and ignore_remote_changes. * The testserver strips the /Workspace prefix on write to match the real API, and `generate genie-space` re-adds it so the generated config keeps the conventional form. * Renamed the parent_path_recreate acceptance test to parent_path_update: changing parent_path now plans an update, not a recreate. - serialized_space uses reason: etag_based (matching dashboards) since drift is detected via etag. Also switches isMissingGenieParentPathError to errors.AsType[T] (forbidigo rule from the SDK bump on main) and lets testifylint rewrite the genie assertions in state_load_test.go to assert.Empty. Co-authored-by: Isaac --- .../genie_spaces/inline/out.plan.json | 7 +------ .../parent_path_recreate/output.txt | 21 ------------------- .../databricks.yml.tmpl | 6 +++--- .../out.test.toml | 0 .../parent_path_update/output.txt | 21 +++++++++++++++++++ .../script | 6 +++--- .../test.toml | 0 .../genie_spaces/simple/out.plan.json | 7 +------ .../current_can_manage/out.plan.direct.json | 2 +- .../out.requests.deploy.direct.json | 2 +- bundle/direct/dresources/all_test.go | 2 +- bundle/direct/dresources/genie_space.go | 16 +++++--------- bundle/direct/dresources/permissions.go | 2 +- bundle/direct/dresources/resources.yml | 8 +------ bundle/generate/genie_space.go | 16 +++++++++++++- bundle/statemgmt/state_load_test.go | 4 ++-- libs/testserver/genie_spaces.go | 15 +++++++++++++ libs/testserver/permissions.go | 2 +- 18 files changed, 72 insertions(+), 65 deletions(-) delete mode 100644 acceptance/bundle/resources/genie_spaces/parent_path_recreate/output.txt rename acceptance/bundle/resources/genie_spaces/{parent_path_recreate => parent_path_update}/databricks.yml.tmpl (61%) rename acceptance/bundle/resources/genie_spaces/{parent_path_recreate => parent_path_update}/out.test.toml (100%) create mode 100644 acceptance/bundle/resources/genie_spaces/parent_path_update/output.txt rename acceptance/bundle/resources/genie_spaces/{parent_path_recreate => parent_path_update}/script (65%) rename acceptance/bundle/resources/genie_spaces/{parent_path_recreate => parent_path_update}/test.toml (100%) diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.plan.json b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json index 8d6f7aabc78..8cfcc539a44 100644 --- a/acceptance/bundle/resources/genie_spaces/inline/out.plan.json +++ b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json @@ -9,6 +9,7 @@ "remote_state": { "description": "Inline serialized_space test", "etag": "1", + "parent_path": "/Workspace/Users/[USERNAME]", "serialized_space": "{\"config\":{\"sample_questions\":[{\"id\":\"sq-001\",\"question\":[\"What is the total revenue?\"]}]},\"data_sources\":{\"tables\":[{\"column_configs\":[{\"column_name\":\"amount\",\"get_example_values\":true}],\"identifier\":\"main.sales.orders\"}]},\"version\":1}", "space_id": "[GENIE_SPACE_ID]", "title": "Sales Analytics Inline Genie", @@ -20,12 +21,6 @@ "reason": "custom", "old": "1", "remote": "1" - }, - "parent_path": { - "action": "skip", - "reason": "input_only", - "old": "/Workspace/Users/[USERNAME]", - "new": "/Workspace/Users/[USERNAME]" } } } diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_recreate/output.txt b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/output.txt deleted file mode 100644 index e90257bf971..00000000000 --- a/acceptance/bundle/resources/genie_spaces/parent_path_recreate/output.txt +++ /dev/null @@ -1,21 +0,0 @@ - ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-genie-space-[UNIQUE_NAME]/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! - -=== Plan after changing parent_path should show recreate ->>> [CLI] bundle plan -recreate genie_spaces.recreate_target - -Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.genie_spaces.recreate_target - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/recreate-genie-space-[UNIQUE_NAME]/default - -Deleting files... -Destroy complete! diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_recreate/databricks.yml.tmpl b/acceptance/bundle/resources/genie_spaces/parent_path_update/databricks.yml.tmpl similarity index 61% rename from acceptance/bundle/resources/genie_spaces/parent_path_recreate/databricks.yml.tmpl rename to acceptance/bundle/resources/genie_spaces/parent_path_update/databricks.yml.tmpl index 1eea5d933e3..d479c3eef35 100644 --- a/acceptance/bundle/resources/genie_spaces/parent_path_recreate/databricks.yml.tmpl +++ b/acceptance/bundle/resources/genie_spaces/parent_path_update/databricks.yml.tmpl @@ -1,10 +1,10 @@ bundle: - name: recreate-genie-space-$UNIQUE_NAME + name: update-genie-space-$UNIQUE_NAME resources: genie_spaces: - recreate_target: - title: "Recreate Target" + update_target: + title: "Update Target" warehouse_id: "test-warehouse-id" parent_path: PARENT_PATH_PLACEHOLDER serialized_space: "{}" diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_recreate/out.test.toml b/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml similarity index 100% rename from acceptance/bundle/resources/genie_spaces/parent_path_recreate/out.test.toml rename to acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_update/output.txt b/acceptance/bundle/resources/genie_spaces/parent_path_update/output.txt new file mode 100644 index 00000000000..76f2dd99b32 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/parent_path_update/output.txt @@ -0,0 +1,21 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-genie-space-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Plan after changing parent_path should show update +>>> [CLI] bundle plan +update genie_spaces.update_target + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.genie_spaces.update_target + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/update-genie-space-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_recreate/script b/acceptance/bundle/resources/genie_spaces/parent_path_update/script similarity index 65% rename from acceptance/bundle/resources/genie_spaces/parent_path_recreate/script rename to acceptance/bundle/resources/genie_spaces/parent_path_update/script index d5f04a795ab..cc7359b67d9 100644 --- a/acceptance/bundle/resources/genie_spaces/parent_path_recreate/script +++ b/acceptance/bundle/resources/genie_spaces/parent_path_update/script @@ -7,8 +7,8 @@ trap cleanup EXIT envsubst < databricks.yml.tmpl | sed "s|PARENT_PATH_PLACEHOLDER|/Users/$CURRENT_USER_NAME/genie-old|" > databricks.yml trace $CLI bundle deploy -# Change parent_path. parent_path is recreate_on_changes in resources.yml, -# so the plan should show a recreate (delete + create) rather than an update. +# Change parent_path. The Genie update API moves the space to the new parent, +# so the plan should show an update rather than a recreate (delete + create). envsubst < databricks.yml.tmpl | sed "s|PARENT_PATH_PLACEHOLDER|/Users/$CURRENT_USER_NAME/genie-new|" > databricks.yml -title "Plan after changing parent_path should show recreate" +title "Plan after changing parent_path should show update" trace $CLI bundle plan diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_recreate/test.toml b/acceptance/bundle/resources/genie_spaces/parent_path_update/test.toml similarity index 100% rename from acceptance/bundle/resources/genie_spaces/parent_path_recreate/test.toml rename to acceptance/bundle/resources/genie_spaces/parent_path_update/test.toml diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.plan.json b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json index 2a260f21a04..77d3669d8ab 100644 --- a/acceptance/bundle/resources/genie_spaces/simple/out.plan.json +++ b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json @@ -9,6 +9,7 @@ "remote_state": { "description": "AI assistant for sales data analysis", "etag": "1", + "parent_path": "/Workspace/Users/[USERNAME]", "serialized_space": "{\n \"benchmarks\": {\n \"questions\": [\n {\n \"answer\": [\n {\n \"content\": [\n \"SELECT\\n\",\n \" name,\\n\",\n \" country\\n\",\n \"FROM main.default.countries\\n\",\n \"ORDER BY name\"\n ],\n \"format\": \"SQL\"\n }\n ],\n \"id\": \"[NUMID]\",\n \"question\": [\n \"Show all names and countries\"\n ]\n }\n ]\n },\n \"config\": {\n \"sample_questions\": [\n {\n \"id\": \"[NUMID]\",\n \"question\": [\n \"List the names and countries\"\n ]\n },\n {\n \"id\": \"[NUMID]\",\n \"question\": [\n \"Which names are in Canada?\"\n ]\n }\n ]\n },\n \"data_sources\": {\n \"tables\": [\n {\n \"column_configs\": [\n {\n \"column_name\": \"country\",\n \"enable_entity_matching\": true,\n \"enable_format_assistance\": true\n },\n {\n \"column_name\": \"name\",\n \"enable_entity_matching\": true,\n \"enable_format_assistance\": true\n }\n ],\n \"identifier\": \"main.default.countries\"\n }\n ]\n },\n \"instructions\": {\n \"example_question_sqls\": [\n {\n \"id\": \"[NUMID]\",\n \"question\": [\n \"List the names and countries\"\n ],\n \"sql\": [\n \"SELECT\\n\",\n \" name,\\n\",\n \" country\\n\",\n \"FROM main.default.countries\\n\",\n \"ORDER BY name\"\n ]\n }\n ],\n \"text_instructions\": [\n {\n \"content\": [\n \"This genie space answers simple questions about people and their countries.\\n\",\n \"Use only the main.default.countries table.\\n\",\n \"Prefer returning the name and country columns directly.\"\n ],\n \"id\": \"[NUMID]\"\n }\n ]\n },\n \"version\": 2\n}\n", "space_id": "[GENIE_SPACE_ID]", "title": "Sales Analytics Genie", @@ -20,12 +21,6 @@ "reason": "custom", "old": "1", "remote": "1" - }, - "parent_path": { - "action": "skip", - "reason": "input_only", - "old": "/Workspace/Users/[USERNAME]", - "new": "/Workspace/Users/[USERNAME]" } } } diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json index dba8cde6d93..5e1e8dd17b3 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json @@ -44,7 +44,7 @@ ] }, "vars": { - "object_id": "/genie/spaces/${resources.genie_spaces.foo.id}" + "object_id": "/genie/${resources.genie_spaces.foo.id}" } } } diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.deploy.direct.json index c112ccb7e76..abcba718ab7 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.deploy.direct.json @@ -1,6 +1,6 @@ { "method": "PUT", - "path": "/api/2.0/permissions/genie/spaces/[FOO_ID]", + "path": "/api/2.0/permissions/genie/[FOO_ID]", "body": { "access_control_list": [ { diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 4f66e907736..30adb4640cc 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -515,7 +515,7 @@ var testDeps = map[string]prepareWorkspace{ } return &PermissionsState{ - ObjectID: "/genie/spaces/" + resp.SpaceId, + ObjectID: "/genie/" + resp.SpaceId, EmbeddedSlice: []StatePermission{{ Level: "CAN_MANAGE", UserName: "user@example.com", diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go index b45cde61a0b..154a90ddcf7 100644 --- a/bundle/direct/dresources/genie_space.go +++ b/bundle/direct/dresources/genie_space.go @@ -25,9 +25,6 @@ var pathSerializedSpace = structpath.MustParsePath("serialized_space") // PrepareState returns the config directly. // - RemapState filters fewer fields: Genie has no LifecycleState / CreateTime / // Path / UpdateTime output-only fields to scrub. -// - DoRead clears ParentPath: the GET API does not reliably return parent_path, -// so we drop it from ForceSendFields and zero the value rather than re-adding -// a "/Workspace" prefix the way dashboard.go does in ensureWorkspacePrefix. // - DoUpdate omits serialized_space when unchanged: serialized_space is in // ignore_remote_changes (see resources.yml), so a UI edit produces no plan // entry. Sending the local body anyway would clobber the UI edit on every @@ -97,18 +94,14 @@ func prepareGenieSpaceRequest(config *resources.GenieSpaceConfig) (string, error } func responseToGenieSpaceConfig(space *dashboards.GenieSpace, serializedSpace string) *resources.GenieSpaceConfig { - // Drop ParentPath from ForceSendFields. We always clear ParentPath - // below because the GET Genie space API does not reliably return it, - // and keeping it in ForceSendFields would force-emit parent_path: "" - // in state output even though the field is logically unset. - forceSendFields := utils.FilterFields[resources.GenieSpaceConfig](space.ForceSendFields, "ParentPath") + forceSendFields := utils.FilterFields[resources.GenieSpaceConfig](space.ForceSendFields) return &resources.GenieSpaceConfig{ Description: space.Description, Etag: space.Etag, Title: space.Title, WarehouseId: space.WarehouseId, - ParentPath: "", + ParentPath: ensureWorkspacePrefix(space.ParentPath), SerializedSpace: serializedSpace, // Output only fields @@ -141,8 +134,8 @@ func isMissingGenieParentPathError(err error) bool { return true } - var apiErr *apierr.APIError - if !errors.As(err, &apiErr) { + apiErr, ok := errors.AsType[*apierr.APIError](err) + if !ok { return false } @@ -218,6 +211,7 @@ func (r *ResourceGenieSpace) DoUpdate(ctx context.Context, id string, config *re Description: config.Description, Title: config.Title, WarehouseId: config.WarehouseId, + ParentPath: config.ParentPath, SerializedSpace: serializedSpace, // Send the etag we last observed. The backend uses it as an If-Match // guard against concurrent writes, and OverrideChangeDesc uses the diff --git a/bundle/direct/dresources/permissions.go b/bundle/direct/dresources/permissions.go index 9c1b26de750..6e1fa79d811 100644 --- a/bundle/direct/dresources/permissions.go +++ b/bundle/direct/dresources/permissions.go @@ -18,7 +18,7 @@ var permissionResourceToObjectType = map[string]string{ "apps": "/apps/", "clusters": "/clusters/", "dashboards": "/dashboards/", - "genie_spaces": "/genie/spaces/", + "genie_spaces": "/genie/", "database_instances": "/database-instances/", "postgres_projects": "/database-projects/", "jobs": "/jobs/", diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index fb4164c545b..c3d8f559d25 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -360,17 +360,11 @@ resources: reason: input_only genie_spaces: - recreate_on_changes: - - field: parent_path - reason: immutable ignore_remote_changes: # serialized_space locally (structured YAML) and remotely (JSON string) will differ # textually, so we cannot meaningfully compare them for drift. - field: serialized_space - reason: input_only - # parent_path is not reliably returned by the GET Genie space API. - - field: parent_path - reason: input_only + reason: etag_based apps: recreate_on_changes: diff --git a/bundle/generate/genie_space.go b/bundle/generate/genie_space.go index 5bd11761a49..f11092d5f81 100644 --- a/bundle/generate/genie_space.go +++ b/bundle/generate/genie_space.go @@ -1,6 +1,9 @@ package generate import ( + "path" + "strings" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/dashboards" ) @@ -19,8 +22,19 @@ func ConvertGenieSpaceToValue(genieSpace *dashboards.GenieSpace, filePath string } if genieSpace.ParentPath != "" { - dv["parent_path"] = dyn.NewValue(genieSpace.ParentPath, []dyn.Location{{Line: 5}}) + dv["parent_path"] = dyn.NewValue(ensureWorkspacePrefix(genieSpace.ParentPath), []dyn.Location{{Line: 5}}) } return dyn.V(dv), nil } + +// ensureWorkspacePrefix re-adds the /Workspace prefix that the Genie GET API +// strips from parent_path, so the generated config matches the convention used +// in hand-written bundles and in deployment state (mirrors the equivalent +// helper in bundle/direct/dresources/dashboard.go). +func ensureWorkspacePrefix(parentPath string) string { + if parentPath == "/Workspace" || strings.HasPrefix(parentPath, "/Workspace/") { + return parentPath + } + return path.Join("/Workspace", parentPath) +} diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 203635b5b7a..672cd9855b2 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -906,10 +906,10 @@ func TestStateToBundleModifiedResources(t *testing.T) { assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Dashboards["test_dashboard_new"].ModifiedStatus) assert.Equal(t, "1", config.Resources.GenieSpaces["test_genie_space"].ID) - assert.Equal(t, "", config.Resources.GenieSpaces["test_genie_space"].ModifiedStatus) + assert.Empty(t, config.Resources.GenieSpaces["test_genie_space"].ModifiedStatus) assert.Equal(t, "2", config.Resources.GenieSpaces["test_genie_space_old"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.GenieSpaces["test_genie_space_old"].ModifiedStatus) - assert.Equal(t, "", config.Resources.GenieSpaces["test_genie_space_new"].ID) + assert.Empty(t, config.Resources.GenieSpaces["test_genie_space_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.GenieSpaces["test_genie_space_new"].ModifiedStatus) assert.Equal(t, "test_app", config.Resources.Apps["test_app"].Name) diff --git a/libs/testserver/genie_spaces.go b/libs/testserver/genie_spaces.go index 98f5cdb381a..aac3eabb019 100644 --- a/libs/testserver/genie_spaces.go +++ b/libs/testserver/genie_spaces.go @@ -45,6 +45,13 @@ func (s *FakeWorkspace) GenieSpaceCreate(req Request) Response { } } + // Strip the /Workspace prefix from parent_path before storing. This matches + // the remote behavior: the GET API returns parent_path without the prefix, + // mirroring DashboardCreate. + if strings.HasPrefix(createReq.ParentPath, "/Workspace/") { + createReq.ParentPath = strings.TrimPrefix(createReq.ParentPath, "/Workspace") + } + genieSpace := dashboards.GenieSpace{ SpaceId: spaceId, Title: createReq.Title, @@ -146,6 +153,13 @@ func (s *FakeWorkspace) GenieSpaceUpdate(req Request) Response { if updateReq.WarehouseId != "" { genieSpace.WarehouseId = updateReq.WarehouseId } + if updateReq.ParentPath != "" { + parentPath := updateReq.ParentPath + if strings.HasPrefix(parentPath, "/Workspace/") { + parentPath = strings.TrimPrefix(parentPath, "/Workspace") + } + genieSpace.ParentPath = parentPath + } if updateReq.SerializedSpace != "" { genieSpace.SerializedSpace = updateReq.SerializedSpace } @@ -157,6 +171,7 @@ func (s *FakeWorkspace) GenieSpaceUpdate(req Request) Response { if prev.Title != genieSpace.Title || prev.Description != genieSpace.Description || prev.WarehouseId != genieSpace.WarehouseId || + prev.ParentPath != genieSpace.ParentPath || prev.SerializedSpace != genieSpace.SerializedSpace { prevEtag, err := strconv.Atoi(genieSpace.Etag) if err != nil { diff --git a/libs/testserver/permissions.go b/libs/testserver/permissions.go index 962825dd037..312c88e9029 100644 --- a/libs/testserver/permissions.go +++ b/libs/testserver/permissions.go @@ -25,7 +25,7 @@ var requestObjectTypeToObjectType = map[string]string{ "sql/alerts": "alert", "sql/queries": "query", "dashboards": "dashboard", - "genie/spaces": "genie-space", + "genie": "genie-space", "experiments": "mlflowExperiment", "registered-models": "registered-model", "serving-endpoints": "serving-endpoint", From a595ca4eb7b1af613ea2c8add2d45f6e19aa6216 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 4 Jun 2026 08:44:17 +0200 Subject: [PATCH 22/42] bundle: regenerate permissions analyze_requests output for genie_spaces The genie_spaces permissions acceptance test is direct-only, so analyze_requests.py lists its request files as DIRECT_ONLY. Regenerate bundle/resources/permissions/output.txt to include those entries (the aggregate test wasn't refreshed when the genie_spaces permissions case was added). Co-authored-by: Isaac --- acceptance/bundle/resources/permissions/output.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/acceptance/bundle/resources/permissions/output.txt b/acceptance/bundle/resources/permissions/output.txt index 85eea2e6e9c..12a35fcfd86 100644 --- a/acceptance/bundle/resources/permissions/output.txt +++ b/acceptance/bundle/resources/permissions/output.txt @@ -99,6 +99,8 @@ DIFF experiments/current_can_manage/out.requests.destroy.direct.json + "path": "/api/2.0/permissions/experiments/[NUMID]" + } +] +DIRECT_ONLY genie_spaces/current_can_manage/out.requests.deploy.direct.json +DIRECT_ONLY genie_spaces/current_can_manage/out.requests.destroy.direct.json MATCH jobs/current_can_manage/out.requests.deploy.direct.json DIFF jobs/current_can_manage/out.requests.destroy.direct.json --- jobs/current_can_manage/out.requests.destroy.direct.json From dcce42ee515ae6224fd6d3dd2a923d8a8df0b77d Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 4 Jun 2026 09:26:56 +0200 Subject: [PATCH 23/42] bundle: default genie_space testserver parent_path to home dir Mirror DashboardCreate: when a Genie space is created without a parent_path, default it to the user's home directory (/Users/), matching cloud behavior. Co-authored-by: Isaac --- libs/testserver/genie_spaces.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libs/testserver/genie_spaces.go b/libs/testserver/genie_spaces.go index aac3eabb019..1bdbae28aab 100644 --- a/libs/testserver/genie_spaces.go +++ b/libs/testserver/genie_spaces.go @@ -35,6 +35,11 @@ func (s *FakeWorkspace) GenieSpaceCreate(req Request) Response { } } + // Default to user's home directory if parent_path is not provided (matches cloud behavior) + if createReq.ParentPath == "" { + createReq.ParentPath = "/Users/" + s.CurrentUser().UserName + } + spaceId, err := generateGenieSpaceId() if err != nil { return Response{ From 9118a3b87712ce0b01f98efe16391e851a3bddb8 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 4 Jun 2026 09:44:35 +0200 Subject: [PATCH 24/42] bundle: testserver GetSpace returns etag only with serialized_space Mirror the real Genie GET API: the etag is only included in the response when include_serialized_space=true. ResourceGenieSpace.DoRead already requests serialized_space, so it still receives the etag for drift detection; this guards against a regression that drops that flag. Co-authored-by: Isaac --- libs/testserver/genie_spaces.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/libs/testserver/genie_spaces.go b/libs/testserver/genie_spaces.go index 1bdbae28aab..3af8cd533c6 100644 --- a/libs/testserver/genie_spaces.go +++ b/libs/testserver/genie_spaces.go @@ -108,6 +108,13 @@ func (s *FakeWorkspace) GenieSpaceGet(req Request) Response { } } + // The GET API only returns the etag when serialized_space is requested + // (include_serialized_space=true). genieSpace is a copy, so clearing the + // field here only affects the response. + if req.URL.Query().Get("include_serialized_space") != "true" { + genieSpace.Etag = "" + } + return Response{ Body: genieSpace, } From 1c1aa88c7f7de8f7e8c17e6343f03ae236966b41 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 4 Jun 2026 09:50:59 +0200 Subject: [PATCH 25/42] bundle: testserver returns 403 for missing genie space The real Genie API returns 403 (not 404) when a space does not exist on get/update/trash. Align the testserver so tests exercise that behavior. Co-authored-by: Isaac --- libs/testserver/genie_spaces.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/libs/testserver/genie_spaces.go b/libs/testserver/genie_spaces.go index 3af8cd533c6..f6a5227cbba 100644 --- a/libs/testserver/genie_spaces.go +++ b/libs/testserver/genie_spaces.go @@ -100,8 +100,9 @@ func (s *FakeWorkspace) GenieSpaceGet(req Request) Response { spaceId := req.Vars["space_id"] genieSpace, ok := s.GenieSpaces[spaceId] if !ok { + // The real API returns 403 (not 404) when a Genie space does not exist. return Response{ - StatusCode: 404, + StatusCode: 403, Body: map[string]string{ "message": "Genie space not found", }, @@ -126,8 +127,9 @@ func (s *FakeWorkspace) GenieSpaceUpdate(req Request) Response { spaceId := req.Vars["space_id"] genieSpace, ok := s.GenieSpaces[spaceId] if !ok { + // The real API returns 403 (not 404) when a Genie space does not exist. return Response{ - StatusCode: 404, + StatusCode: 403, Body: map[string]string{ "message": "Genie space not found", }, @@ -210,8 +212,9 @@ func (s *FakeWorkspace) GenieSpaceTrash(req Request) Response { spaceId := req.Vars["space_id"] genieSpace, ok := s.GenieSpaces[spaceId] if !ok { + // The real API returns 403 (not 404) when a Genie space does not exist. return Response{ - StatusCode: 404, + StatusCode: 403, Body: map[string]string{ "message": "Genie space not found", }, From b80768c83a4c24047a0fa6d781b14d4a0d183c7a Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 4 Jun 2026 10:02:42 +0200 Subject: [PATCH 26/42] bundle: testserver bumps genie etag only on serialized_space change The Genie backend updates a space's etag only when serialized_space changes, not when other fields (title, description, warehouse_id, parent_path) change. Align the testserver update handler. Co-authored-by: Isaac --- libs/testserver/genie_spaces.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/libs/testserver/genie_spaces.go b/libs/testserver/genie_spaces.go index f6a5227cbba..2a45422f20b 100644 --- a/libs/testserver/genie_spaces.go +++ b/libs/testserver/genie_spaces.go @@ -178,15 +178,11 @@ func (s *FakeWorkspace) GenieSpaceUpdate(req Request) Response { genieSpace.SerializedSpace = updateReq.SerializedSpace } - // Bump the etag only when the update actually changes user-visible state. - // Matches dashboard's behavior (libs/testserver/dashboards.go) and keeps - // no-op updates idempotent so TestAll can pass the same config to Create - // and Update without observing spurious drift. - if prev.Title != genieSpace.Title || - prev.Description != genieSpace.Description || - prev.WarehouseId != genieSpace.WarehouseId || - prev.ParentPath != genieSpace.ParentPath || - prev.SerializedSpace != genieSpace.SerializedSpace { + // The backend bumps the etag only when serialized_space changes; updates to + // other fields (title, description, warehouse_id, parent_path) leave it + // unchanged. This mirrors the GET API, which only returns the etag + // alongside serialized_space. + if prev.SerializedSpace != genieSpace.SerializedSpace { prevEtag, err := strconv.Atoi(genieSpace.Etag) if err != nil { return Response{ From c82791c910a6df75bae43115f0ff79cf0ea70401 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 4 Jun 2026 10:16:20 +0200 Subject: [PATCH 27/42] bundle: testserver does not register genie space as workspace file Genie space files are not user-facing ("dataRoom is not user-facing"), so the Workspace export/get-status APIs cannot resolve them. Stop registering the synthetic workspace file entry on create (and the matching trash cleanup); nothing in DABs resolves a genie space via the Workspace API. Co-authored-by: Isaac --- libs/testserver/genie_spaces.go | 37 ++++----------------------------- 1 file changed, 4 insertions(+), 33 deletions(-) diff --git a/libs/testserver/genie_spaces.go b/libs/testserver/genie_spaces.go index 2a45422f20b..cded1497611 100644 --- a/libs/testserver/genie_spaces.go +++ b/libs/testserver/genie_spaces.go @@ -4,12 +4,10 @@ import ( "crypto/rand" "encoding/hex" "encoding/json" - "path" "strconv" "strings" "github.com/databricks/databricks-sdk-go/service/dashboards" - "github.com/databricks/databricks-sdk-go/service/workspace" ) // generateGenieSpaceId returns a random 32-character hex string. @@ -71,23 +69,9 @@ func (s *FakeWorkspace) GenieSpaceCreate(req Request) Response { s.GenieSpaces[spaceId] = genieSpace - // Register in the workspace files for path lookup. - if createReq.ParentPath != "" { - workspacePath := createReq.ParentPath - if !strings.HasPrefix(workspacePath, "/Workspace") { - workspacePath = path.Join("/Workspace", workspacePath) - } - workspacePath = path.Join(workspacePath, createReq.Title+".geniespace") - - s.files[workspacePath] = FileEntry{ - Info: workspace.ObjectInfo{ - ObjectType: "FILE", - Path: workspacePath, - ResourceId: spaceId, - }, - Data: []byte(createReq.SerializedSpace), - } - } + // Genie spaces are not exposed as workspace files ("dataRoom is not + // user-facing"), so unlike dashboards we do not register a workspace path + // entry — there is nothing to resolve via the Workspace API. return Response{ Body: genieSpace, @@ -206,8 +190,7 @@ func (s *FakeWorkspace) GenieSpaceTrash(req Request) Response { defer s.LockUnlock()() spaceId := req.Vars["space_id"] - genieSpace, ok := s.GenieSpaces[spaceId] - if !ok { + if _, ok := s.GenieSpaces[spaceId]; !ok { // The real API returns 403 (not 404) when a Genie space does not exist. return Response{ StatusCode: 403, @@ -219,18 +202,6 @@ func (s *FakeWorkspace) GenieSpaceTrash(req Request) Response { delete(s.GenieSpaces, spaceId) - // Also remove the synthetic workspace file entry registered by - // GenieSpaceCreate, so a trash+recreate flow does not resolve to stale - // state via the workspace path index. - if genieSpace.ParentPath != "" { - workspacePath := genieSpace.ParentPath - if !strings.HasPrefix(workspacePath, "/Workspace") { - workspacePath = path.Join("/Workspace", workspacePath) - } - workspacePath = path.Join(workspacePath, genieSpace.Title+".geniespace") - delete(s.files, workspacePath) - } - return Response{ StatusCode: 200, } From fe19eb25305dd6ba1e5c10297d27b454dcb84935 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 4 Jun 2026 10:51:19 +0200 Subject: [PATCH 28/42] bundle: handle genie space 403-for-missing on read/delete/generate The Genie API returns 403 (not 404) when a space does not exist, so the framework's 404-based isResourceGone/apierr.IsMissing checks would treat a deleted space as a hard error. Map the 403 to the gone sentinel: - DoRead surfaces a deleted space as gone, so the plan recreates it instead of failing. - DoDelete tolerates trashing an already-gone space (idempotent). - generate genie-space --existing-id reports "not found" instead of a raw 403. Adds unit tests for the translation (read/delete) and an acceptance test that deletes a deployed space out-of-band and verifies the next plan recreates it. Co-authored-by: Isaac --- .../recreate_when_gone/databricks.yml.tmpl | 11 +++++ .../recreate_when_gone/out.test.toml | 3 ++ .../recreate_when_gone/output.txt | 20 ++++++++ .../genie_spaces/recreate_when_gone/script | 18 +++++++ .../genie_spaces/recreate_when_gone/test.toml | 6 +++ bundle/direct/dresources/genie_space.go | 20 ++++++-- bundle/direct/dresources/genie_space_test.go | 49 +++++++++++++++++++ cmd/bundle/generate/genie_space.go | 3 +- 8 files changed, 126 insertions(+), 4 deletions(-) create mode 100644 acceptance/bundle/resources/genie_spaces/recreate_when_gone/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml create mode 100644 acceptance/bundle/resources/genie_spaces/recreate_when_gone/output.txt create mode 100644 acceptance/bundle/resources/genie_spaces/recreate_when_gone/script create mode 100644 acceptance/bundle/resources/genie_spaces/recreate_when_gone/test.toml diff --git a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/databricks.yml.tmpl b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/databricks.yml.tmpl new file mode 100644 index 00000000000..fe135e8b627 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: recreate-gone-genie-space-$UNIQUE_NAME + +resources: + genie_spaces: + sales_analytics: + title: "Recreate When Gone" + warehouse_id: "test-warehouse-id" + parent_path: /Users/$CURRENT_USER_NAME + serialized_space: + version: 1 diff --git a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/output.txt b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/output.txt new file mode 100644 index 00000000000..aeaa5d7964d --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/output.txt @@ -0,0 +1,20 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-gone-genie-space-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] genie trash-space [GENIE_SPACE_ID] + +=== Plan after remote deletion should recreate the space +>>> [CLI] bundle plan +create genie_spaces.sales_analytics + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/recreate-gone-genie-space-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/script b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/script new file mode 100644 index 00000000000..c965df9fcc1 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/script @@ -0,0 +1,18 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy +GENIE_SPACE_ID=$($CLI bundle summary --output json | jq -r '.resources.genie_spaces.sales_analytics.id') +echo "$GENIE_SPACE_ID:GENIE_SPACE_ID" >> ACC_REPLS + +# Delete the space out-of-band, simulating a UI deletion. The GET API then +# returns 403; the resource maps that to "gone" so the plan recreates the +# space instead of failing. +trace $CLI genie trash-space $GENIE_SPACE_ID + +title "Plan after remote deletion should recreate the space" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/test.toml b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/test.toml new file mode 100644 index 00000000000..bbdf2380b2d --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/test.toml @@ -0,0 +1,6 @@ +Local = true +RecordRequests = false + +Ignore = [ + "databricks.yml", +] diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go index 154a90ddcf7..7eeeb3b01ad 100644 --- a/bundle/direct/dresources/genie_space.go +++ b/bundle/direct/dresources/genie_space.go @@ -74,7 +74,7 @@ func (r *ResourceGenieSpace) DoRead(ctx context.Context, id string) (*resources. ForceSendFields: nil, }) if err != nil { - return nil, err + return nil, genieSpaceGoneError(err) } return responseToGenieSpaceConfig(space, space.SerializedSpace), nil } @@ -145,6 +145,18 @@ func isMissingGenieParentPathError(err error) bool { strings.Contains(apiErr.Message, "does not exist") } +// genieSpaceGoneError maps the Genie API's "space does not exist" response to +// the framework's gone sentinel (apierr.ErrResourceDoesNotExist). The Genie API +// returns 403 (not 404) for a missing space, so without this translation +// isResourceGone would treat a remotely-deleted space as a hard permission +// error instead of recreating it (on read) or tolerating it (on delete). +func genieSpaceGoneError(err error) error { + if errors.Is(err, apierr.ErrPermissionDenied) { + return errors.Join(err, apierr.ErrResourceDoesNotExist) + } + return err +} + func (r *ResourceGenieSpace) DoCreate(ctx context.Context, config *resources.GenieSpaceConfig) (string, *resources.GenieSpaceConfig, error) { serializedSpace, err := prepareGenieSpaceRequest(config) if err != nil { @@ -287,7 +299,9 @@ func hasUpdate(entry *PlanEntry, path *structpath.PathNode) bool { } func (r *ResourceGenieSpace) DoDelete(ctx context.Context, id string, _ *resources.GenieSpaceConfig) error { - return r.client.Genie.TrashSpace(ctx, dashboards.GenieTrashSpaceRequest{ + // TrashSpace returns 403 when the space is already gone; map that to the + // gone sentinel so deletion is idempotent. + return genieSpaceGoneError(r.client.Genie.TrashSpace(ctx, dashboards.GenieTrashSpaceRequest{ SpaceId: id, - }) + })) } diff --git a/bundle/direct/dresources/genie_space_test.go b/bundle/direct/dresources/genie_space_test.go index 742deff6cb4..a72e50d9fe7 100644 --- a/bundle/direct/dresources/genie_space_test.go +++ b/bundle/direct/dresources/genie_space_test.go @@ -1,6 +1,7 @@ package dresources import ( + "errors" "testing" "github.com/databricks/cli/bundle/config/resources" @@ -258,3 +259,51 @@ func TestGenieSpaceOverrideChangeDescEtag(t *testing.T) { assert.Equal(t, deployplan.Update, change.Action) }) } + +func TestGenieSpaceGoneErrorMapsForbidden(t *testing.T) { + // The Genie API returns 403 (not 404) for a missing space; it must surface + // as the framework's gone sentinel so a deleted space is recreated (read) + // or tolerated (delete). + forbidden := &apierr.APIError{StatusCode: 403, Message: "dataRoom is not user-facing"} + assert.ErrorIs(t, genieSpaceGoneError(forbidden), apierr.ErrResourceDoesNotExist) + + // Unrelated errors pass through unchanged. + other := errors.New("boom") + assert.Equal(t, other, genieSpaceGoneError(other)) + + // A nil error stays nil. + assert.NoError(t, genieSpaceGoneError(nil)) +} + +func TestGenieSpaceDoReadTreatsForbiddenAsGone(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + m.GetMockGenieAPI().EXPECT(). + GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: "space-id", + IncludeSerializedSpace: true, + }). + Return(nil, &apierr.APIError{StatusCode: 403, Message: "dataRoom is not user-facing"}). + Once() + + _, err := r.DoRead(ctx, "space-id") + require.Error(t, err) + assert.ErrorIs(t, err, apierr.ErrResourceDoesNotExist) +} + +func TestGenieSpaceDoDeleteToleratesForbidden(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + m.GetMockGenieAPI().EXPECT(). + TrashSpace(ctx, dashboards.GenieTrashSpaceRequest{SpaceId: "space-id"}). + Return(&apierr.APIError{StatusCode: 403, Message: "dataRoom is not user-facing"}). + Once() + + err := r.DoDelete(ctx, "space-id", nil) + require.Error(t, err) + assert.ErrorIs(t, err, apierr.ErrResourceDoesNotExist) +} diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 4d67db210c2..542dfd50b35 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -111,7 +111,8 @@ func (g *genieSpace) resolveFromID(ctx context.Context, b *bundle.Bundle) string SpaceId: g.existingID, }) if err != nil { - if apierr.IsMissing(err) { + // The Genie API returns 403 (not 404) when a space does not exist. + if apierr.IsMissing(err) || errors.Is(err, apierr.ErrPermissionDenied) { logdiag.LogError(ctx, fmt.Errorf("genie space with ID %s not found", g.existingID)) return "" } From 67607088d6333a97391cc59eba92bd81d47304a6 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 4 Jun 2026 10:59:01 +0200 Subject: [PATCH 29/42] bundle: clarify ConvertGenieSpaceToValue field-copy comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The old comment ("majority of fields ... are read-only") was inaccurate and confusing — title, warehouse_id, description, parent_path, and serialized_space are all user-settable. Explain the actual reason for the manual field-by-field copy: serialized_space is written to a separate file (referenced via file_path) and output-only fields (space_id, etag) must be excluded from the generated config. Co-authored-by: Isaac --- bundle/generate/genie_space.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bundle/generate/genie_space.go b/bundle/generate/genie_space.go index f11092d5f81..bf00abbc66e 100644 --- a/bundle/generate/genie_space.go +++ b/bundle/generate/genie_space.go @@ -9,8 +9,10 @@ import ( ) func ConvertGenieSpaceToValue(genieSpace *dashboards.GenieSpace, filePath string) (dyn.Value, error) { - // The majority of fields of the genie space struct are read-only. - // We copy the relevant fields manually. + // Emit only the fields a user authors in a bundle. serialized_space is + // written to a separate file and referenced via file_path, and output-only + // fields (e.g. space_id, etag) must not appear in the generated config, so + // we build the value field by field rather than marshaling the struct. dv := map[string]dyn.Value{ "title": dyn.NewValue(genieSpace.Title, []dyn.Location{{Line: 1}}), "warehouse_id": dyn.NewValue(genieSpace.WarehouseId, []dyn.Location{{Line: 2}}), From 55f1ed269784c427c9b46cbf63ecc9bba566d741 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 4 Jun 2026 11:02:28 +0200 Subject: [PATCH 30/42] Add comment on IncludeSerializedSpace --- bundle/direct/dresources/genie_space.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go index 7eeeb3b01ad..84aedf5059f 100644 --- a/bundle/direct/dresources/genie_space.go +++ b/bundle/direct/dresources/genie_space.go @@ -70,7 +70,7 @@ func (r *ResourceGenieSpace) RemapState(state *resources.GenieSpaceConfig) *reso func (r *ResourceGenieSpace) DoRead(ctx context.Context, id string) (*resources.GenieSpaceConfig, error) { space, err := r.client.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ SpaceId: id, - IncludeSerializedSpace: true, + IncludeSerializedSpace: true, // otherwise etag isn't returned ForceSendFields: nil, }) if err != nil { From 6566ff10f4744145fa429e0378bcf87a5ed018eb Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 4 Jun 2026 11:47:18 +0200 Subject: [PATCH 31/42] bundle: acceptance test for genie serialized_space payload Record the Genie create requests and assert the serialized_space we send: - from a file_path .json file: sent verbatim (formatting preserved). - from inline YAML: rendered as valid JSON (parsed with fromjson, which fails the test if the payload is not valid JSON). Co-authored-by: Isaac --- .../serialized_space/databricks.yml.tmpl | 19 ++++++++++++ .../serialized_space/out.create_requests.json | 30 +++++++++++++++++++ .../serialized_space/out.test.toml | 3 ++ .../genie_spaces/serialized_space/output.txt | 16 ++++++++++ .../genie_spaces/serialized_space/script | 21 +++++++++++++ .../serialized_space/space.geniespace.json | 1 + .../genie_spaces/serialized_space/test.toml | 6 ++++ 7 files changed, 96 insertions(+) create mode 100644 acceptance/bundle/resources/genie_spaces/serialized_space/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/genie_spaces/serialized_space/out.create_requests.json create mode 100644 acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml create mode 100644 acceptance/bundle/resources/genie_spaces/serialized_space/output.txt create mode 100644 acceptance/bundle/resources/genie_spaces/serialized_space/script create mode 100644 acceptance/bundle/resources/genie_spaces/serialized_space/space.geniespace.json create mode 100644 acceptance/bundle/resources/genie_spaces/serialized_space/test.toml diff --git a/acceptance/bundle/resources/genie_spaces/serialized_space/databricks.yml.tmpl b/acceptance/bundle/resources/genie_spaces/serialized_space/databricks.yml.tmpl new file mode 100644 index 00000000000..6b426d5c117 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/serialized_space/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: genie-serialized-space-$UNIQUE_NAME + +resources: + genie_spaces: + from_file: + title: "From File" + warehouse_id: "test-warehouse-id" + parent_path: /Users/$CURRENT_USER_NAME + file_path: "space.geniespace.json" + from_inline: + title: "From Inline" + warehouse_id: "test-warehouse-id" + parent_path: /Users/$CURRENT_USER_NAME + serialized_space: + version: 1 + data_sources: + tables: + - identifier: "main.sales.orders" diff --git a/acceptance/bundle/resources/genie_spaces/serialized_space/out.create_requests.json b/acceptance/bundle/resources/genie_spaces/serialized_space/out.create_requests.json new file mode 100644 index 00000000000..1ed6f9baf50 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/serialized_space/out.create_requests.json @@ -0,0 +1,30 @@ +[ + { + "title": "From File", + "serialized_space": "{\"version\": 1, \"data_sources\": {\"tables\": [{\"identifier\": \"main.sales.customers\"}]}}\n", + "parsed": { + "version": 1, + "data_sources": { + "tables": [ + { + "identifier": "main.sales.customers" + } + ] + } + } + }, + { + "title": "From Inline", + "serialized_space": "{\"data_sources\":{\"tables\":[{\"identifier\":\"main.sales.orders\"}]},\"version\":1}", + "parsed": { + "data_sources": { + "tables": [ + { + "identifier": "main.sales.orders" + } + ] + }, + "version": 1 + } + } +] diff --git a/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml b/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/serialized_space/output.txt b/acceptance/bundle/resources/genie_spaces/serialized_space/output.txt new file mode 100644 index 00000000000..ff660039377 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/serialized_space/output.txt @@ -0,0 +1,16 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/genie-serialized-space-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.genie_spaces.from_file + delete resources.genie_spaces.from_inline + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/genie-serialized-space-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/genie_spaces/serialized_space/script b/acceptance/bundle/resources/genie_spaces/serialized_space/script new file mode 100644 index 00000000000..bf5aea4c21d --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/serialized_space/script @@ -0,0 +1,21 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +rm -f out.requests.txt +trace $CLI bundle deploy + +# Capture the serialized_space sent to the Genie create API: +# - from_file must be sent verbatim from space.geniespace.json. +# - from_inline must be the inline YAML rendered as valid JSON (the fromjson +# below fails the test if the payload is not valid JSON). +jq -s '[.[] | select(.method == "POST" and .path == "/api/2.0/genie/spaces") + | {title: .body.title, + serialized_space: .body.serialized_space, + parsed: (.body.serialized_space | fromjson)}] + | sort_by(.title)' out.requests.txt > out.create_requests.json +rm -f out.requests.txt diff --git a/acceptance/bundle/resources/genie_spaces/serialized_space/space.geniespace.json b/acceptance/bundle/resources/genie_spaces/serialized_space/space.geniespace.json new file mode 100644 index 00000000000..ef4a1508015 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/serialized_space/space.geniespace.json @@ -0,0 +1 @@ +{"version": 1, "data_sources": {"tables": [{"identifier": "main.sales.customers"}]}} diff --git a/acceptance/bundle/resources/genie_spaces/serialized_space/test.toml b/acceptance/bundle/resources/genie_spaces/serialized_space/test.toml new file mode 100644 index 00000000000..a4821187360 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/serialized_space/test.toml @@ -0,0 +1,6 @@ +Local = true +RecordRequests = true + +Ignore = [ + "databricks.yml", +] From 4be04ae63625f307ff5de0a7705633214fc4a205 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 4 Jun 2026 12:01:52 +0200 Subject: [PATCH 32/42] bundle: don't send etag as If-Match on genie space update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review: the Genie backend bumps a space's etag when it migrates serialized_space to a newer schema version. Sending the last-observed etag as an If-Match guard would then fail a legitimate bundle update with 409 after such a migration. Stop sending it (Etag left empty). Drift is still detected on read via OverrideChangeDesc, which compares the stored and remote etags — this diverges from dashboards, which do send the etag. Co-authored-by: Isaac --- bundle/direct/dresources/genie_space.go | 14 ++++++++++---- bundle/direct/dresources/genie_space_test.go | 4 +++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go index 84aedf5059f..dcc03a8af33 100644 --- a/bundle/direct/dresources/genie_space.go +++ b/bundle/direct/dresources/genie_space.go @@ -29,6 +29,10 @@ var pathSerializedSpace = structpath.MustParsePath("serialized_space") // ignore_remote_changes (see resources.yml), so a UI edit produces no plan // entry. Sending the local body anyway would clobber the UI edit on every // unrelated update. +// - DoUpdate omits the etag (dashboard sends it as an If-Match guard): the +// backend bumps the etag when it migrates serialized_space to a newer +// schema version, so sending a stale etag would 409 the update after a +// migration. Drift is still detected on read via OverrideChangeDesc. // - DoCreate has expanded missing-parent-path detection: see // isMissingGenieParentPathError below. // @@ -225,10 +229,12 @@ func (r *ResourceGenieSpace) DoUpdate(ctx context.Context, id string, config *re WarehouseId: config.WarehouseId, ParentPath: config.ParentPath, SerializedSpace: serializedSpace, - // Send the etag we last observed. The backend uses it as an If-Match - // guard against concurrent writes, and OverrideChangeDesc uses the - // post-update etag to detect drift on subsequent plans. - Etag: config.Etag, + // Intentionally empty: we do not send an If-Match guard. The backend + // bumps the etag when it migrates serialized_space to a newer schema + // version, so sending the last-observed etag would fail the update with + // 409 after such a migration. Drift is still detected on read via + // OverrideChangeDesc, which compares the stored and remote etags. + Etag: "", ForceSendFields: utils.FilterFields[dashboards.GenieUpdateSpaceRequest](config.ForceSendFields, excludeForceSend...), }) diff --git a/bundle/direct/dresources/genie_space_test.go b/bundle/direct/dresources/genie_space_test.go index a72e50d9fe7..deb476b0204 100644 --- a/bundle/direct/dresources/genie_space_test.go +++ b/bundle/direct/dresources/genie_space_test.go @@ -181,11 +181,13 @@ func TestGenieSpaceDoUpdateRoundTripsEtag(t *testing.T) { }, } + // The stored etag (etag-7) must NOT be sent as an If-Match guard — it would + // 409 after a backend serialized_space schema migration. Only the etag from + // the response is persisted, for drift detection on the next plan. m.GetMockGenieAPI().EXPECT(). UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ SpaceId: "space-id", Title: "new", - Etag: "etag-7", }). Return(&dashboards.GenieSpace{ SpaceId: "space-id", From cdf670c1962b798cb1e0d3f79622f0368015bca7 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 4 Jun 2026 13:03:27 +0200 Subject: [PATCH 33/42] Regenerate direct --- bundle/direct/dresources/apitypes.generated.yml | 2 ++ bundle/direct/dresources/apitypes.yml | 2 -- bundle/direct/dresources/resources.generated.yml | 6 ++++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index b65d9ac41d4..069f8dbbe5f 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -18,6 +18,8 @@ experiments: ml.CreateExperiment external_locations: catalog.CreateExternalLocation +genie_spaces: dashboards.GenieSpace + jobs: jobs.JobSettings model_serving_endpoints: serving.CreateServingEndpoint diff --git a/bundle/direct/dresources/apitypes.yml b/bundle/direct/dresources/apitypes.yml index 11cba516a4d..7d478be47f7 100644 --- a/bundle/direct/dresources/apitypes.yml +++ b/bundle/direct/dresources/apitypes.yml @@ -4,8 +4,6 @@ # Set a value to null to remove a type: # jobs: null -genie_spaces: dashboards.GenieSpace - postgres_branches: postgres.BranchSpec postgres_endpoints: postgres.EndpointSpec diff --git a/bundle/direct/dresources/resources.generated.yml b/bundle/direct/dresources/resources.generated.yml index b73eb8931b2..61994d371a9 100644 --- a/bundle/direct/dresources/resources.generated.yml +++ b/bundle/direct/dresources/resources.generated.yml @@ -175,6 +175,12 @@ resources: - field: effective_file_event_queue reason: spec:output_only + genie_spaces: + + ignore_remote_changes: + - field: etag + reason: spec:output_only + # jobs: no api field behaviors # model_serving_endpoints: no api field behaviors From d1487eaa2445e06c81859f553a264de9d436fc5b Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 4 Jun 2026 14:55:02 +0200 Subject: [PATCH 34/42] bundle: error when a genie space sets both file_path and serialized_space A genie space body can be authored inline via serialized_space (written as YAML and marshalled to a JSON string) or loaded from a .geniespace.json file via file_path, and the resolved value is stored in state. Setting both is ambiguous, so reject it with an error. This intentionally departs from dashboards, which silently let file_path win. The mutator previously only warned on the both-set case and produced a confusing error when neither field was set; an absent serialized_space now passes through (the backend rejects an empty body). Add a table-driven unit test and a validate acceptance test for the both-set error. Co-authored-by: Isaac --- .../output.txt | 6 +- .../configure_genie_space_serialized_space.go | 12 +- ...igure_genie_space_serialized_space_test.go | 120 ++++++++++++++++++ .../mutator/translate_paths_genie_spaces.go | 2 +- 4 files changed, 134 insertions(+), 6 deletions(-) create mode 100644 bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space_test.go diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/output.txt b/acceptance/bundle/validate/genie_space_file_path_and_inline/output.txt index 589a8333c37..9402a3ff711 100644 --- a/acceptance/bundle/validate/genie_space_file_path_and_inline/output.txt +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/output.txt @@ -1,4 +1,4 @@ -Warning: both file_path and serialized_space are set; file_path will be used and serialized_space will be ignored +Error: both file_path and serialized_space are set; specify only one in databricks.yml:11:9 Name: genie-space-file-path-and-inline @@ -7,4 +7,6 @@ Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/genie-space-file-path-and-inline/default -Found 1 warning +Found 1 error + +Exit code: 1 diff --git a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go index a9327217c08..c51c97a51e4 100644 --- a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go +++ b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go @@ -37,12 +37,16 @@ func (c configureGenieSpaceSerializedSpace) Apply(_ context.Context, b *bundle.B ss := v.Get(serializedSpaceFieldName) if hasFilePath { + // file_path and serialized_space are two ways to provide the same + // content. Accepting both is ambiguous, so reject it instead of + // silently picking one. if ss.IsValid() && ss.Kind() != dyn.KindNil { diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: "both file_path and serialized_space are set; file_path will be used and serialized_space will be ignored", + Severity: diag.Error, + Summary: "both file_path and serialized_space are set; specify only one", Locations: ss.Locations(), }) + return v, nil } contents, err := b.SyncRoot.ReadFile(filePath) if err != nil { @@ -57,7 +61,9 @@ func (c configureGenieSpaceSerializedSpace) Apply(_ context.Context, b *bundle.B // round-trip decodes them as `float64`, and structdiff reports // false drift on every plan. switch ss.Kind() { - case dyn.KindNil, dyn.KindString: + case dyn.KindInvalid, dyn.KindNil, dyn.KindString: + // KindInvalid means serialized_space is absent (neither it nor + // file_path is set); leave it for backend validation to reject. return v, nil case dyn.KindMap, dyn.KindSequence: jsonBytes, err := json.Marshal(ss.AsAny()) diff --git a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space_test.go b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space_test.go new file mode 100644 index 00000000000..19c1b685f10 --- /dev/null +++ b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space_test.go @@ -0,0 +1,120 @@ +package resourcemutator_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator/resourcemutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/vfs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConfigureGenieSpaceSerializedSpace(t *testing.T) { + const fileName = "space.geniespace.json" + + tests := []struct { + name string + // filePath is set on the resource as-is (already sync-root-relative). + filePath string + // writeFile creates filePath with fileContents before the mutator runs. + writeFile bool + fileContents string + setSerialized bool + serializedSpace any + // wantSerialized is the expected serialized_space after a successful run. + wantSerialized any + // wantErr, when non-empty, is a substring expected in the diagnostics. + wantErr string + }{ + { + // The file is read verbatim, so formatting and the trailing newline + // are preserved (unlike the inline path, which re-marshals). + name: "file_path reads file contents verbatim", + filePath: fileName, + writeFile: true, + fileContents: `{"version": 1}` + "\n", + wantSerialized: `{"version": 1}` + "\n", + }, + { + // Inline maps are marshaled to a compact JSON string with sorted keys + // so config and state hold an identical string and don't drift. + name: "inline map is marshaled to a JSON string", + setSerialized: true, + serializedSpace: map[string]any{"version": 1}, + wantSerialized: `{"version":1}`, + }, + { + name: "inline string is left unchanged", + setSerialized: true, + serializedSpace: `{"version":1}`, + wantSerialized: `{"version":1}`, + }, + { + // Neither field set: the absent field must pass through, not error. + name: "neither file_path nor serialized_space passes through", + wantSerialized: nil, + }, + { + name: "both file_path and serialized_space is rejected", + filePath: fileName, + setSerialized: true, + serializedSpace: map[string]any{"version": 1}, + wantErr: "both file_path and serialized_space are set; specify only one", + }, + { + name: "non-structured serialized_space is rejected", + setSerialized: true, + serializedSpace: true, + wantErr: "serialized_space must be a string, map, or sequence, got bool", + }, + { + name: "unreadable file_path is an error", + filePath: "does_not_exist.json", + wantErr: "failed to read serialized genie space", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + if tt.writeFile { + require.NoError(t, os.WriteFile(filepath.Join(dir, tt.filePath), []byte(tt.fileContents), 0o600)) + } + + gs := &resources.GenieSpace{ + GenieSpaceConfig: resources.GenieSpaceConfig{Title: "My Genie Space"}, + FilePath: tt.filePath, + } + if tt.setSerialized { + gs.SerializedSpace = tt.serializedSpace + } + + b := &bundle.Bundle{ + SyncRootPath: dir, + BundleRootPath: dir, + SyncRoot: vfs.MustNew(dir), + Config: config.Root{ + Resources: config.Resources{ + GenieSpaces: map[string]*resources.GenieSpace{"my_space": gs}, + }, + }, + } + + diags := bundle.ApplySeq(t.Context(), b, resourcemutator.ConfigureGenieSpaceSerializedSpace()) + + if tt.wantErr != "" { + require.Error(t, diags.Error()) + assert.ErrorContains(t, diags.Error(), tt.wantErr) + return + } + + require.NoError(t, diags.Error()) + assert.Equal(t, tt.wantSerialized, b.Config.Resources.GenieSpaces["my_space"].SerializedSpace) + }) + } +} diff --git a/bundle/config/mutator/translate_paths_genie_spaces.go b/bundle/config/mutator/translate_paths_genie_spaces.go index 279d97be01c..4e6d41f1cec 100644 --- a/bundle/config/mutator/translate_paths_genie_spaces.go +++ b/bundle/config/mutator/translate_paths_genie_spaces.go @@ -8,7 +8,7 @@ import ( ) func (t *translateContext) applyGenieSpaceTranslations(ctx context.Context, v dyn.Value) (dyn.Value, error) { - // Convert the `file_path` field to a local absolute path. + // Rewrite the `file_path` field to a path relative to the bundle sync root. // We load the file at this path and use its contents for the genie space contents. return paths.VisitGenieSpacePaths(v, func(p dyn.Path, mode paths.TranslateMode, v dyn.Value) (dyn.Value, error) { From 7072a3989af4c05a762d1d713150550c842349ea Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Fri, 5 Jun 2026 09:53:17 +0200 Subject: [PATCH 35/42] bundle: remove unusable by-path flow from generate genie-space Genie spaces are not exposed as workspace files, so the Workspace API (GetStatusByPath) can never resolve one by path. The --existing-path and --existing-genie-space-path flags, resolveFromPath, and their help example were forked from the dashboard command but cannot succeed for genie spaces. Remove them; resolveFromID is now the only resolver. Document --key in the examples; it is supported via the parent generate command's persistent flag but was undocumented here. Add acceptance tests to match dashboard coverage: - genie_space_existing_id_not_found exercises the genie-specific 403-as-missing branch. - genie_space_inplace covers the deploy -> update -> generate --resource --force round trip, pinned to the direct engine (the only engine that deploys genie spaces). Co-authored-by: Isaac --- .../databricks.yml | 2 + .../out.test.toml | 3 + .../output.txt | 4 ++ .../genie_space_existing_id_not_found/script | 2 + .../genie_space_inplace/databricks.yml | 9 +++ .../genie_space_inplace/out.test.toml | 3 + .../generate/genie_space_inplace/output.txt | 30 ++++++++++ .../generate/genie_space_inplace/script | 13 +++++ .../genie_space_inplace/space.geniespace.json | 1 + .../generate/genie_space_inplace/test.toml | 6 ++ cmd/bundle/generate/genie_space.go | 58 +++---------------- 11 files changed, 80 insertions(+), 51 deletions(-) create mode 100644 acceptance/bundle/generate/genie_space_existing_id_not_found/databricks.yml create mode 100644 acceptance/bundle/generate/genie_space_existing_id_not_found/out.test.toml create mode 100644 acceptance/bundle/generate/genie_space_existing_id_not_found/output.txt create mode 100644 acceptance/bundle/generate/genie_space_existing_id_not_found/script create mode 100644 acceptance/bundle/generate/genie_space_inplace/databricks.yml create mode 100644 acceptance/bundle/generate/genie_space_inplace/out.test.toml create mode 100644 acceptance/bundle/generate/genie_space_inplace/output.txt create mode 100644 acceptance/bundle/generate/genie_space_inplace/script create mode 100644 acceptance/bundle/generate/genie_space_inplace/space.geniespace.json create mode 100644 acceptance/bundle/generate/genie_space_inplace/test.toml diff --git a/acceptance/bundle/generate/genie_space_existing_id_not_found/databricks.yml b/acceptance/bundle/generate/genie_space_existing_id_not_found/databricks.yml new file mode 100644 index 00000000000..576d7a9ef25 --- /dev/null +++ b/acceptance/bundle/generate/genie_space_existing_id_not_found/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: test-bundle diff --git a/acceptance/bundle/generate/genie_space_existing_id_not_found/out.test.toml b/acceptance/bundle/generate/genie_space_existing_id_not_found/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/generate/genie_space_existing_id_not_found/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/genie_space_existing_id_not_found/output.txt b/acceptance/bundle/generate/genie_space_existing_id_not_found/output.txt new file mode 100644 index 00000000000..7ccd778a6b1 --- /dev/null +++ b/acceptance/bundle/generate/genie_space_existing_id_not_found/output.txt @@ -0,0 +1,4 @@ +Error: genie space with ID f00dcafe not found + + +Exit code: 1 diff --git a/acceptance/bundle/generate/genie_space_existing_id_not_found/script b/acceptance/bundle/generate/genie_space_existing_id_not_found/script new file mode 100644 index 00000000000..94d8b8594bc --- /dev/null +++ b/acceptance/bundle/generate/genie_space_existing_id_not_found/script @@ -0,0 +1,2 @@ +# Test that bundle generate genie-space fails when the existing ID is not found +exec $CLI bundle generate genie-space --existing-id f00dcafe diff --git a/acceptance/bundle/generate/genie_space_inplace/databricks.yml b/acceptance/bundle/generate/genie_space_inplace/databricks.yml new file mode 100644 index 00000000000..e5a76861842 --- /dev/null +++ b/acceptance/bundle/generate/genie_space_inplace/databricks.yml @@ -0,0 +1,9 @@ +bundle: + name: genie space update inplace + +resources: + genie_spaces: + test_genie_space: + title: "test genie space" + warehouse_id: "my-warehouse-1234" + file_path: ./space.geniespace.json diff --git a/acceptance/bundle/generate/genie_space_inplace/out.test.toml b/acceptance/bundle/generate/genie_space_inplace/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/generate/genie_space_inplace/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/generate/genie_space_inplace/output.txt b/acceptance/bundle/generate/genie_space_inplace/output.txt new file mode 100644 index 00000000000..9beebf12605 --- /dev/null +++ b/acceptance/bundle/generate/genie_space_inplace/output.txt @@ -0,0 +1,30 @@ + +>>> cat space.geniespace.json +{} + +=== deploy initial genie space +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/genie space update inplace/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== update the genie space +>>> [CLI] genie update-space [GENIE_SPACE_ID] --serialized-space {"a":"b"} +{ + "etag": "2", + "parent_path": "/Users/[USERNAME]/.bundle/genie space update inplace/default/resources", + "serialized_space": "{\"a\":\"b\"}", + "space_id": "[GENIE_SPACE_ID]", + "title": "test genie space", + "warehouse_id": "my-warehouse-1234" +} + +=== update the genie space file using bundle generate +>>> [CLI] bundle generate genie-space --resource test_genie_space --force +Writing genie space to space.geniespace.json + +>>> cat space.geniespace.json +{ + "a": "b" +} diff --git a/acceptance/bundle/generate/genie_space_inplace/script b/acceptance/bundle/generate/genie_space_inplace/script new file mode 100644 index 00000000000..18ca58d2986 --- /dev/null +++ b/acceptance/bundle/generate/genie_space_inplace/script @@ -0,0 +1,13 @@ +trace cat space.geniespace.json + +title "deploy initial genie space" +trace $CLI bundle deploy +genie_space_id=$($CLI bundle summary --output json | jq -r '.resources.genie_spaces.test_genie_space.id') + +title "update the genie space" +trace $CLI genie update-space $genie_space_id --serialized-space '{"a":"b"}' + +title "update the genie space file using bundle generate" +trace $CLI bundle generate genie-space --resource test_genie_space --force + +trace cat space.geniespace.json diff --git a/acceptance/bundle/generate/genie_space_inplace/space.geniespace.json b/acceptance/bundle/generate/genie_space_inplace/space.geniespace.json new file mode 100644 index 00000000000..0967ef424bc --- /dev/null +++ b/acceptance/bundle/generate/genie_space_inplace/space.geniespace.json @@ -0,0 +1 @@ +{} diff --git a/acceptance/bundle/generate/genie_space_inplace/test.toml b/acceptance/bundle/generate/genie_space_inplace/test.toml new file mode 100644 index 00000000000..723c9907976 --- /dev/null +++ b/acceptance/bundle/generate/genie_space_inplace/test.toml @@ -0,0 +1,6 @@ +# Genie spaces are only deployed via the direct deployment engine. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +[[Repls]] +Old = "[0-9a-f]{32}" +New = "[GENIE_SPACE_ID]" diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 542dfd50b35..36ebdf54800 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -24,7 +24,6 @@ import ( "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/yamlsaver" "github.com/databricks/cli/libs/logdiag" @@ -38,9 +37,8 @@ import ( const genieSpaceWatchInterval = 1 * time.Second type genieSpace struct { - // Lookup flags for one-time generate. - existingPath string - existingID string + // Lookup flag for one-time generate. + existingID string // Lookup flag for existing bundle resource. resource string @@ -69,42 +67,6 @@ type genieSpace struct { err io.Writer } -func (g *genieSpace) resolveID(ctx context.Context, b *bundle.Bundle) string { - switch { - case g.existingPath != "": - return g.resolveFromPath(ctx, b) - case g.existingID != "": - return g.resolveFromID(ctx, b) - } - - logdiag.LogError(ctx, errors.New("expected one of --existing-path, --existing-id")) - return "" -} - -func (g *genieSpace) resolveFromPath(ctx context.Context, b *bundle.Bundle) string { - w := b.WorkspaceClient(ctx) - obj, err := w.Workspace.GetStatusByPath(ctx, g.existingPath) //nolint:staticcheck // Deprecated in SDK v0.127.0. Migration to WorkspaceHierarchyService tracked separately. - if err != nil { - if apierr.IsMissing(err) { - logdiag.LogError(ctx, fmt.Errorf("genie space %q not found", path.Base(g.existingPath))) - return "" - } - - logdiag.LogError(ctx, err) - return "" - } - - if obj.ResourceId == "" { - logdiag.LogDiag(ctx, diag.Diagnostic{ - Severity: diag.Error, - Summary: "expected a non-empty genie space resource ID", - }) - return "" - } - - return obj.ResourceId -} - func (g *genieSpace) resolveFromID(ctx context.Context, b *bundle.Bundle) string { w := b.WorkspaceClient(ctx) obj, err := w.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ @@ -390,7 +352,7 @@ func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { func (g *genieSpace) runForExisting(ctx context.Context, b *bundle.Bundle) { // Resolve the ID of the genie space to generate configuration for. - genieSpaceID := g.resolveID(ctx, b) + genieSpaceID := g.resolveFromID(ctx, b) if logdiag.HasError(ctx) { return } @@ -454,15 +416,14 @@ This command downloads an existing Genie space and creates bundle files that you can use to deploy the Genie space to other environments or manage it as code. Examples: - # Import Genie space by workspace path - databricks bundle generate genie-space --existing-path /Users/me/my-genie-space - # Import Genie space by ID - databricks bundle generate genie-space --existing-id abc123 + databricks bundle generate genie-space --existing-id abc123 --key my_genie_space # Watch for changes to keep bundle in sync with UI modifications databricks bundle generate genie-space --resource my_genie_space --watch --force +Use --key to set the resource name in the generated configuration. + What gets generated: - Genie space configuration YAML file with settings and a reference to the Genie space definition - Genie space definition (.geniespace.json) file with the serialized space content @@ -486,14 +447,11 @@ bundle files automatically, useful during active Genie space development.`, } // Lookup flags. - cmd.Flags().StringVar(&g.existingPath, "existing-path", "", `workspace path of the Genie space to generate configuration for`) cmd.Flags().StringVar(&g.existingID, "existing-id", "", `ID of the Genie space to generate configuration for`) cmd.Flags().StringVar(&g.resource, "resource", "", `resource key of Genie space to watch for changes`) - // Alias lookup flags that include the resource type name. - cmd.Flags().StringVar(&g.existingPath, "existing-genie-space-path", "", `workspace path of the Genie space to generate configuration for`) + // Alias lookup flag that includes the resource type name. cmd.Flags().StringVar(&g.existingID, "existing-genie-space-id", "", `ID of the Genie space to generate configuration for`) - cmd.Flags().MarkHidden("existing-genie-space-path") cmd.Flags().MarkHidden("existing-genie-space-id") // Output flags. @@ -506,7 +464,6 @@ bundle files automatically, useful during active Genie space development.`, // Exactly one of the lookup flags must be provided. cmd.MarkFlagsOneRequired( - "existing-path", "existing-id", "resource", ) @@ -515,7 +472,6 @@ bundle files automatically, useful during active Genie space development.`, cmd.Flags().BoolVar(&g.watch, "watch", false, `watch for changes to the Genie space and update the configuration`) // Make sure the watch flag is only used with the existing-resource flag. - cmd.MarkFlagsMutuallyExclusive("watch", "existing-path") cmd.MarkFlagsMutuallyExclusive("watch", "existing-id") // Make sure the bind flag is only used with the existing-resource flag. From 3911870570ffcfc6b48949cbef669184755d4cb9 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Fri, 5 Jun 2026 10:36:36 +0200 Subject: [PATCH 36/42] Autogenerate validation --- bundle/internal/validation/generated/enum_fields.go | 2 ++ bundle/internal/validation/generated/required_fields.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index 9e8f45fbd9a..8beb3e25a4a 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -62,6 +62,8 @@ var EnumFields = map[string][]string{ "resources.external_locations.*.encryption_details.sse_encryption_details.algorithm": {"AWS_SSE_KMS", "AWS_SSE_S3"}, "resources.external_locations.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FILES", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, + "resources.genie_spaces.*.permissions[*].level": {"CAN_ATTACH_TO", "CAN_BIND", "CAN_CREATE", "CAN_CREATE_APP", "CAN_EDIT", "CAN_EDIT_METADATA", "CAN_MANAGE", "CAN_MANAGE_PRODUCTION_VERSIONS", "CAN_MANAGE_RUN", "CAN_MANAGE_STAGING_VERSIONS", "CAN_MONITOR", "CAN_MONITOR_ONLY", "CAN_QUERY", "CAN_READ", "CAN_RESTART", "CAN_RUN", "CAN_USE", "CAN_VIEW", "CAN_VIEW_METADATA", "IS_OWNER"}, + "resources.jobs.*.continuous.pause_status": {"PAUSED", "UNPAUSED"}, "resources.jobs.*.continuous.task_retry_mode": {"NEVER", "ON_FAILURE"}, "resources.jobs.*.deployment.kind": {"BUNDLE", "SYSTEM_MANAGED"}, diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index ad6437f1887..3d47858587e 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -64,6 +64,8 @@ var RequiredFields = map[string][]string{ "resources.external_locations.*": {"credential_name", "name", "url"}, + "resources.genie_spaces.*.permissions[*]": {"level"}, + "resources.jobs.*.deployment": {"kind"}, "resources.jobs.*.environments[*]": {"environment_key"}, "resources.jobs.*.git_source": {"git_provider", "git_url"}, From 23b0f68c37fb49a5de4aabf2794c27ea9d3f0205 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Fri, 5 Jun 2026 10:43:35 +0200 Subject: [PATCH 37/42] Add gitattribute to force geniespace.json to eol=lf --- acceptance/.gitattributes | 1 + 1 file changed, 1 insertion(+) diff --git a/acceptance/.gitattributes b/acceptance/.gitattributes index 8d48122750e..8380bb6ae0b 100644 --- a/acceptance/.gitattributes +++ b/acceptance/.gitattributes @@ -4,6 +4,7 @@ # uploading the file's content to a workspace. *.txt text eol=lf *.lvdash.json text eol=lf +*.geniespace.json text eol=lf # The out.test.toml file is autogenerated based on the merged test.toml view. out.test.toml linguist-generated=true From cc53f0457aea0f35c7598a7c6c9a12355d1773c9 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Fri, 5 Jun 2026 11:02:11 +0200 Subject: [PATCH 38/42] bundle: add genie_space invariant and bind acceptance tests + changelog Adds the two test artifacts the dresources README requires for a new resource: - acceptance/bundle/invariant/configs/genie_space.yml.tmpl (no-drift), wired into the invariant matrix and excluded from cloud since the serialized_space schema is workspace-version-sensitive. - acceptance/bundle/deployment/bind/genie_space/ exercising the bind -> deploy -> unbind -> destroy lifecycle on the direct engine. Also adds the NEXT_CHANGELOG.md entry for the genie_spaces resource. Co-authored-by: Isaac --- NEXT_CHANGELOG.md | 1 + .../bind/genie_space/databricks.yml.tmpl | 10 ++++++ .../deployment/bind/genie_space/out.test.toml | 3 ++ .../deployment/bind/genie_space/output.txt | 34 +++++++++++++++++++ .../genie_space/sample-space.geniespace.json | 1 + .../bundle/deployment/bind/genie_space/script | 25 ++++++++++++++ .../deployment/bind/genie_space/test.toml | 13 +++++++ .../invariant/configs/genie_space.yml.tmpl | 14 ++++++++ .../bundle/invariant/no_drift/out.test.toml | 1 + acceptance/bundle/invariant/test.toml | 5 +++ 10 files changed, 107 insertions(+) create mode 100644 acceptance/bundle/deployment/bind/genie_space/databricks.yml.tmpl create mode 100644 acceptance/bundle/deployment/bind/genie_space/out.test.toml create mode 100644 acceptance/bundle/deployment/bind/genie_space/output.txt create mode 100644 acceptance/bundle/deployment/bind/genie_space/sample-space.geniespace.json create mode 100644 acceptance/bundle/deployment/bind/genie_space/script create mode 100644 acceptance/bundle/deployment/bind/genie_space/test.toml create mode 100644 acceptance/bundle/invariant/configs/genie_space.yml.tmpl diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 235bc55dfd7..1fce1cdab4c 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -13,6 +13,7 @@ * Fix script output dropping last line without trailing newline ([#4995](https://github.com/databricks/cli/pull/4995)). * Add `--select` flag to `bundle plan` and `bundle deploy` to plan/deploy a subset of resources (e.g. `--select my_job` or `--select jobs.my_job`); resources referenced by the selection are included transitively. Direct engine only ([#5413](https://github.com/databricks/cli/pull/5413)). * Support `purge_on_delete: true` on `postgres_projects` so bundles can hard-delete a Lakebase project on destroy (skipping the soft-delete retention window) ([#5414](https://github.com/databricks/cli/pull/5414)). +* Add the `genie_spaces` bundle resource for managing Databricks Genie spaces as code, plus `bundle generate genie-space` to import an existing space. Direct deployment engine only ([#5282](https://github.com/databricks/cli/pull/5282)). ### Dependency updates * Bump Go toolchain to 1.26.4 ([#5420](https://github.com/databricks/cli/pull/5420)). diff --git a/acceptance/bundle/deployment/bind/genie_space/databricks.yml.tmpl b/acceptance/bundle/deployment/bind/genie_space/databricks.yml.tmpl new file mode 100644 index 00000000000..3b8ccda5092 --- /dev/null +++ b/acceptance/bundle/deployment/bind/genie_space/databricks.yml.tmpl @@ -0,0 +1,10 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + genie_spaces: + genie_space1: + title: $GENIE_SPACE_TITLE + warehouse_id: "test-warehouse-id" + parent_path: /Users/$CURRENT_USER_NAME + file_path: "sample-space.geniespace.json" diff --git a/acceptance/bundle/deployment/bind/genie_space/out.test.toml b/acceptance/bundle/deployment/bind/genie_space/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/deployment/bind/genie_space/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/genie_space/output.txt b/acceptance/bundle/deployment/bind/genie_space/output.txt new file mode 100644 index 00000000000..46b67f277a9 --- /dev/null +++ b/acceptance/bundle/deployment/bind/genie_space/output.txt @@ -0,0 +1,34 @@ + +>>> [CLI] bundle deployment bind genie_space1 [GENIE_SPACE_ID] --auto-approve +Updating deployment state... +Successfully bound genie_space with an id '[GENIE_SPACE_ID]' +Run 'bundle deploy' to deploy changes to your workspace + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] genie get-space [GENIE_SPACE_ID] +{ + "parent_path": "/Users/[USERNAME]", + "title": "test genie space [UNIQUE_NAME]", + "warehouse_id": "test-warehouse-id" +} + +>>> [CLI] bundle deployment unbind genie_space1 +Updating deployment state... + +>>> [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! + +>>> [CLI] genie get-space [GENIE_SPACE_ID] +{ + "parent_path": "/Users/[USERNAME]", + "title": "test genie space [UNIQUE_NAME]", + "warehouse_id": "test-warehouse-id" +} diff --git a/acceptance/bundle/deployment/bind/genie_space/sample-space.geniespace.json b/acceptance/bundle/deployment/bind/genie_space/sample-space.geniespace.json new file mode 100644 index 00000000000..cc3e94c2d1d --- /dev/null +++ b/acceptance/bundle/deployment/bind/genie_space/sample-space.geniespace.json @@ -0,0 +1 @@ +{"version": 1, "display_name": "bind test space"} diff --git a/acceptance/bundle/deployment/bind/genie_space/script b/acceptance/bundle/deployment/bind/genie_space/script new file mode 100644 index 00000000000..be2b7aa1be8 --- /dev/null +++ b/acceptance/bundle/deployment/bind/genie_space/script @@ -0,0 +1,25 @@ +GENIE_SPACE_TITLE="test genie space $UNIQUE_NAME" + +export GENIE_SPACE_TITLE +envsubst < databricks.yml.tmpl > databricks.yml + +# Create a Genie space out of band, then bind the bundle resource to it. +GENIE_SPACE_ID=$($CLI genie create-space "test-warehouse-id" '{"version": 1, "display_name": "bind test space"}' --title "${GENIE_SPACE_TITLE}" | jq -r '.space_id') + +cleanupRemoveGenieSpace() { + $CLI genie trash-space "${GENIE_SPACE_ID}" +} +trap cleanupRemoveGenieSpace EXIT + +trace $CLI bundle deployment bind genie_space1 "${GENIE_SPACE_ID}" --auto-approve + +trace $CLI bundle deploy + +trace $CLI genie get-space "${GENIE_SPACE_ID}" | jq --sort-keys '{title, parent_path, warehouse_id}' + +trace $CLI bundle deployment unbind genie_space1 + +trace $CLI bundle destroy --auto-approve + +# Read the Genie space again (expecting it still exists and is not deleted): +trace $CLI genie get-space "${GENIE_SPACE_ID}" | jq --sort-keys '{title, parent_path, warehouse_id}' diff --git a/acceptance/bundle/deployment/bind/genie_space/test.toml b/acceptance/bundle/deployment/bind/genie_space/test.toml new file mode 100644 index 00000000000..f46c8906df6 --- /dev/null +++ b/acceptance/bundle/deployment/bind/genie_space/test.toml @@ -0,0 +1,13 @@ +Local = true + +# Genie spaces are only deployed via the direct deployment engine. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# Uses a literal warehouse id and a version-sensitive serialized_space, so this +# lifecycle test runs against the local mock server only (overrides the Cloud=true +# inherited from acceptance/bundle/deployment/test.toml). +Cloud = false + +[[Repls]] +Old = "[0-9a-f]{32}" +New = "[GENIE_SPACE_ID]" diff --git a/acceptance/bundle/invariant/configs/genie_space.yml.tmpl b/acceptance/bundle/invariant/configs/genie_space.yml.tmpl new file mode 100644 index 00000000000..fc4b6490263 --- /dev/null +++ b/acceptance/bundle/invariant/configs/genie_space.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + genie_spaces: + foo: + warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID + title: test-genie-space-$UNIQUE_NAME + # Inline (structured) serialized_space is marshalled to a JSON string by + # ConfigureGenieSpaceSerializedSpace; this config doubles as a regression + # guard that the normalization produces a drift-free deploy. + serialized_space: + version: 1 + display_name: test-genie-space-$UNIQUE_NAME diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 023feb47cdc..62de900a9ce 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -12,6 +12,7 @@ EnvMatrix.INPUT_CONFIG = [ "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", + "genie_space.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 6b0ee25fc61..345d18883a9 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -30,6 +30,7 @@ EnvMatrix.INPUT_CONFIG = [ "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", + "genie_space.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", @@ -80,6 +81,10 @@ no_external_location_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=external_loc # External volumes reference external locations; excluded from cloud for the same reason no_external_volume_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=volume_external.yml.tmpl"] +# Genie space serialized_space schema is workspace-version-sensitive; we only +# exercise it against the local mock server, not a real cloud workspace. +no_genie_space_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=genie_space.yml.tmpl"] + # Fake SQL endpoint for local tests [[Server]] Pattern = "POST /api/2.0/sql/statements/" From 4e18b60e03270af1f7d08bdb4ff7d6e032f75c71 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Fri, 5 Jun 2026 11:12:04 +0200 Subject: [PATCH 39/42] bundle: regenerate invariant out.test.toml for genie_space matrix entry The genie_space.yml.tmpl entry added to acceptance/bundle/invariant/test.toml propagates to the inherited config view dumped in the sibling invariant subdirectories. Regenerated via ./task test-update. Co-authored-by: Isaac --- acceptance/bundle/invariant/continue_293/out.test.toml | 1 + acceptance/bundle/invariant/migrate/out.test.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index 023feb47cdc..62de900a9ce 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -12,6 +12,7 @@ EnvMatrix.INPUT_CONFIG = [ "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", + "genie_space.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index 023feb47cdc..62de900a9ce 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -12,6 +12,7 @@ EnvMatrix.INPUT_CONFIG = [ "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", + "genie_space.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", From e90597e3cae759ba44262611b1b5bf568df0318f Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Fri, 5 Jun 2026 11:38:23 +0200 Subject: [PATCH 40/42] bundle: reject user-set etag on genie_space resources etag is internal drift-detection state persisted by the direct engine, never authored by the user, but it was exposed as a writable genie_space config field with no guard (unlike dashboards, which have ValidateDashboardEtags). A user-set etag was silently ignored, violating the repo rule to reject incompatible inputs with an actionable error. Add ValidateGenieSpaceEtags, mirroring ValidateDashboardEtags, wired into the Initialize phase, plus an acceptance test (pinned to the direct engine since genie spaces are direct-only). Co-authored-by: Isaac --- .../no_genie_space_etag/databricks.yml | 10 +++++ .../no_genie_space_etag/out.test.toml | 3 ++ .../validate/no_genie_space_etag/output.txt | 15 +++++++ .../validate/no_genie_space_etag/script | 1 + .../validate/no_genie_space_etag/test.toml | 3 ++ .../validate/validate_genie_space_etags.go | 39 +++++++++++++++++++ bundle/phases/initialize.go | 3 ++ 7 files changed, 74 insertions(+) create mode 100644 acceptance/bundle/validate/no_genie_space_etag/databricks.yml create mode 100644 acceptance/bundle/validate/no_genie_space_etag/out.test.toml create mode 100644 acceptance/bundle/validate/no_genie_space_etag/output.txt create mode 100644 acceptance/bundle/validate/no_genie_space_etag/script create mode 100644 acceptance/bundle/validate/no_genie_space_etag/test.toml create mode 100644 bundle/config/validate/validate_genie_space_etags.go diff --git a/acceptance/bundle/validate/no_genie_space_etag/databricks.yml b/acceptance/bundle/validate/no_genie_space_etag/databricks.yml new file mode 100644 index 00000000000..846608ec580 --- /dev/null +++ b/acceptance/bundle/validate/no_genie_space_etag/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: test-bundle + +resources: + genie_spaces: + foobar: + title: foobar + etag: "1234567890" + warehouse_id: "my-warehouse-1234" + serialized_space: "{}" diff --git a/acceptance/bundle/validate/no_genie_space_etag/out.test.toml b/acceptance/bundle/validate/no_genie_space_etag/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/validate/no_genie_space_etag/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/no_genie_space_etag/output.txt b/acceptance/bundle/validate/no_genie_space_etag/output.txt new file mode 100644 index 00000000000..58a7648f9aa --- /dev/null +++ b/acceptance/bundle/validate/no_genie_space_etag/output.txt @@ -0,0 +1,15 @@ + +>>> [CLI] bundle validate +Error: genie space "foobar" has an etag set. Etags must not be set in bundle configuration + at resources.genie_spaces.foobar + in databricks.yml:7:7 + +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/validate/no_genie_space_etag/script b/acceptance/bundle/validate/no_genie_space_etag/script new file mode 100644 index 00000000000..5350876150f --- /dev/null +++ b/acceptance/bundle/validate/no_genie_space_etag/script @@ -0,0 +1 @@ +trace $CLI bundle validate diff --git a/acceptance/bundle/validate/no_genie_space_etag/test.toml b/acceptance/bundle/validate/no_genie_space_etag/test.toml new file mode 100644 index 00000000000..d573410a054 --- /dev/null +++ b/acceptance/bundle/validate/no_genie_space_etag/test.toml @@ -0,0 +1,3 @@ +# Genie spaces are only supported by the direct deployment engine; under +# terraform the direct-only validation error would diverge the output. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/config/validate/validate_genie_space_etags.go b/bundle/config/validate/validate_genie_space_etags.go new file mode 100644 index 00000000000..84e9626c0f4 --- /dev/null +++ b/bundle/config/validate/validate_genie_space_etags.go @@ -0,0 +1,39 @@ +package validate + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +func ValidateGenieSpaceEtags() bundle.ReadOnlyMutator { + return &validateGenieSpaceEtags{} +} + +type validateGenieSpaceEtags struct{ bundle.RO } + +func (v *validateGenieSpaceEtags) Name() string { + return "validate:validate_genie_space_etags" +} + +func (v *validateGenieSpaceEtags) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + // No genie spaces should have etags set. They are purely internal state + // (persisted by the direct engine for drift detection), never authored by + // the user. Mirrors ValidateDashboardEtags. + for k, genieSpace := range b.Config.Resources.GenieSpaces { + if genieSpace.Etag != "" { + return diag.Diagnostics{ + { + Severity: diag.Error, + Summary: fmt.Sprintf("genie space %q has an etag set. Etags must not be set in bundle configuration", genieSpace.Title), + Paths: []dyn.Path{dyn.MustPathFromString("resources.genie_spaces." + k)}, + Locations: b.Config.GetLocations("resources.genie_spaces." + k), + }, + } + } + } + return nil +} diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index a40506ebb18..1b86e37dcee 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -158,6 +158,9 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // Validate that no dashboard etags are set. They are purely internal state and should not be set by the user. validate.ValidateDashboardEtags(), + // Validate that no genie space etags are set. They are purely internal state and should not be set by the user. + validate.ValidateGenieSpaceEtags(), + // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. From c22a3fa49203e89409a68bba4000d5b5c3bfec06 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Fri, 5 Jun 2026 11:38:32 +0200 Subject: [PATCH 41/42] acceptance: exclude genie_space from migrate and continue_293 invariant tests The genie_space invariant config (added when wiring up the resource) is incompatible with two invariant subdirs: migrate seeds the scenario with a terraform deploy (genie spaces are direct-only), and continue_293 deploys with v0.293.0 (which predates the resource). Exclude it from both, mirroring the existing direct-only (catalog, external_location) and not-in-old-version (vector_search) exclusions. no_drift still exercises it on the direct engine. Co-authored-by: Isaac --- acceptance/bundle/invariant/continue_293/test.toml | 3 +++ acceptance/bundle/invariant/migrate/test.toml | 2 ++ 2 files changed, 5 insertions(+) diff --git a/acceptance/bundle/invariant/continue_293/test.toml b/acceptance/bundle/invariant/continue_293/test.toml index 2afbcbf5e31..6887ada9a71 100644 --- a/acceptance/bundle/invariant/continue_293/test.toml +++ b/acceptance/bundle/invariant/continue_293/test.toml @@ -9,6 +9,9 @@ EnvMatrixExclude.no_model_with_permissions = ["INPUT_CONFIG=model_with_permissio # vector_search_endpoints resource is not supported on v0.293.0 EnvMatrixExclude.no_vector_search_endpoint = ["INPUT_CONFIG=vector_search_endpoint.yml.tmpl"] +# genie_spaces resource is not supported on v0.293.0 +EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"] + # Dotted pipeline configuration keys are not supported on v0.293.0 EnvMatrixExclude.no_pipeline_config_dots = ["INPUT_CONFIG=pipeline_config_dots.yml.tmpl"] diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index 5fa381832e7..9492aff72cd 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -5,6 +5,8 @@ EnvMatrixExclude.no_vector_search_index = ["INPUT_CONFIG=vector_search_index.yml # Error: Catalog resources are only supported with direct deployment mode EnvMatrixExclude.no_catalog = ["INPUT_CONFIG=catalog.yml.tmpl"] EnvMatrixExclude.no_external_location = ["INPUT_CONFIG=external_location.yml.tmpl"] +# Genie spaces are direct-only too; the terraform deploy that seeds the migration fails for them. +EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"] # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level}) # don't work in terraform mode: the terraform interpolator converts the path to From df2e06b266fd59318da9fa21b9991a3afec86921 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Fri, 5 Jun 2026 11:44:26 +0200 Subject: [PATCH 42/42] bundle/direct: copy remote etag into bind state for genie spaces Direct bind only copied the remote etag into temp bind state for dashboards. Genie spaces use the same etag-based drift signal, so binding a genie space (or 'generate genie-space --bind') finalized state without the etag, producing a bogus etag-driven update on the very next plan. Extend the condition to genie_spaces, matching dstate.ExportStateFromData which already handles both. The bind acceptance test now runs 'bundle plan' immediately after bind and asserts it is clean (0 to change); its serialized_space is an inline body that matches the out-of-band-created space byte-for-byte, so the etag is the only possible drift signal. Verified the test fails (1 to change) without the fix. Co-authored-by: Isaac --- .../deployment/bind/genie_space/databricks.yml.tmpl | 5 ++++- acceptance/bundle/deployment/bind/genie_space/output.txt | 3 +++ .../bind/genie_space/sample-space.geniespace.json | 1 - acceptance/bundle/deployment/bind/genie_space/script | 7 ++++++- bundle/direct/bind.go | 9 +++++---- 5 files changed, 18 insertions(+), 7 deletions(-) delete mode 100644 acceptance/bundle/deployment/bind/genie_space/sample-space.geniespace.json diff --git a/acceptance/bundle/deployment/bind/genie_space/databricks.yml.tmpl b/acceptance/bundle/deployment/bind/genie_space/databricks.yml.tmpl index 3b8ccda5092..a31d53c1785 100644 --- a/acceptance/bundle/deployment/bind/genie_space/databricks.yml.tmpl +++ b/acceptance/bundle/deployment/bind/genie_space/databricks.yml.tmpl @@ -7,4 +7,7 @@ resources: title: $GENIE_SPACE_TITLE warehouse_id: "test-warehouse-id" parent_path: /Users/$CURRENT_USER_NAME - file_path: "sample-space.geniespace.json" + # Inline body matches the out-of-band-created space byte-for-byte, so the + # only possible post-bind drift signal is the etag. + serialized_space: + version: 1 diff --git a/acceptance/bundle/deployment/bind/genie_space/output.txt b/acceptance/bundle/deployment/bind/genie_space/output.txt index 46b67f277a9..34c6295bb51 100644 --- a/acceptance/bundle/deployment/bind/genie_space/output.txt +++ b/acceptance/bundle/deployment/bind/genie_space/output.txt @@ -4,6 +4,9 @@ Updating deployment state... Successfully bound genie_space with an id '[GENIE_SPACE_ID]' Run 'bundle deploy' to deploy changes to your workspace +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Deploying resources... diff --git a/acceptance/bundle/deployment/bind/genie_space/sample-space.geniespace.json b/acceptance/bundle/deployment/bind/genie_space/sample-space.geniespace.json deleted file mode 100644 index cc3e94c2d1d..00000000000 --- a/acceptance/bundle/deployment/bind/genie_space/sample-space.geniespace.json +++ /dev/null @@ -1 +0,0 @@ -{"version": 1, "display_name": "bind test space"} diff --git a/acceptance/bundle/deployment/bind/genie_space/script b/acceptance/bundle/deployment/bind/genie_space/script index be2b7aa1be8..896685a2311 100644 --- a/acceptance/bundle/deployment/bind/genie_space/script +++ b/acceptance/bundle/deployment/bind/genie_space/script @@ -4,7 +4,7 @@ export GENIE_SPACE_TITLE envsubst < databricks.yml.tmpl > databricks.yml # Create a Genie space out of band, then bind the bundle resource to it. -GENIE_SPACE_ID=$($CLI genie create-space "test-warehouse-id" '{"version": 1, "display_name": "bind test space"}' --title "${GENIE_SPACE_TITLE}" | jq -r '.space_id') +GENIE_SPACE_ID=$($CLI genie create-space "test-warehouse-id" '{"version":1}' --title "${GENIE_SPACE_TITLE}" | jq -r '.space_id') cleanupRemoveGenieSpace() { $CLI genie trash-space "${GENIE_SPACE_ID}" @@ -13,6 +13,11 @@ trap cleanupRemoveGenieSpace EXIT trace $CLI bundle deployment bind genie_space1 "${GENIE_SPACE_ID}" --auto-approve +# Bind must copy the remote etag into state, so the first plan after bind is +# clean. Without that, the etag drift signal (empty stored vs remote) would +# produce a bogus update here. +trace $CLI bundle plan + trace $CLI bundle deploy trace $CLI genie get-space "${GENIE_SPACE_ID}" | jq --sort-keys '{title, parent_path, warehouse_id}' diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index f1c534bea9d..9760ce95666 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -132,10 +132,11 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac dependsOn = entry.DependsOn } - // Copy etag from remote state for dashboards. - // Dashboards store "etag" in state which is not provided by user but comes from remote. - // If we don't store "etag" in state, we won't detect remote drift correctly. - if strings.Contains(resourceKey, ".dashboards.") && entry != nil && entry.RemoteState != nil { + // Copy etag from remote state for resources that use etag-based drift + // detection (dashboards and genie spaces). The etag is not provided by the + // user; it comes from remote. If we don't store it in state, we won't + // detect remote drift correctly and the next plan shows a bogus update. + if (strings.Contains(resourceKey, ".dashboards.") || strings.Contains(resourceKey, ".genie_spaces.")) && entry != nil && entry.RemoteState != nil { etag, err := structaccess.Get(entry.RemoteState, structpath.NewStringKey(nil, "etag")) if err == nil && etag != nil { if etagStr, ok := etag.(string); ok && etagStr != "" {