Replace regex-based interpolation with character scanner

NEXT_CHANGELOG.md

@@ -12,6 +12,8 @@
 * Deduplicate grant entries with duplicate principals or privileges during initialization ([#4801](https://github.com/databricks/cli/pull/4801))
 * engine/direct: Fix unwanted recreation of secret scopes when scope_backend_type is not set ([#4834](https://github.com/databricks/cli/pull/4834))
 
+* Replace regex-based variable interpolation with a character scanner ([#4747](https://github.com/databricks/cli/pull/4747)).
+
 ### Dependency updates
 
 ### API Changes

acceptance/bundle/resource_deps/bad_syntax/out.deploy.direct.txt

@@ -1,3 +1,7 @@
+Warning: invalid variable reference ${resources.volumes.bar.bad..syntax}: invalid path
+  at resources.volumes.foo.schema_name
+  in databricks.yml:11:21
+
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
 Deploying resources...
 Updating deployment state...

acceptance/bundle/resource_deps/bad_syntax/out.plan.direct.txt

@@ -1,3 +1,7 @@
+Warning: invalid variable reference ${resources.volumes.bar.bad..syntax}: invalid path
+  at resources.volumes.foo.schema_name
+  in databricks.yml:11:21
+
 create volumes.bar
 create volumes.foo
 

acceptance/bundle/resource_deps/bad_syntax/out.plan.terraform.txt

@@ -1,3 +1,7 @@
+Warning: invalid variable reference ${resources.volumes.bar.bad..syntax}: invalid path
+  at resources.volumes.foo.schema_name
+  in databricks.yml:11:21
+
 Error: exit status 1
 
 Error: Invalid attribute name

acceptance/bundle/resource_deps/bad_syntax/output.txt

@@ -1,5 +1,9 @@
 
 >>> [CLI] bundle validate -o json
+Warning: invalid variable reference ${resources.volumes.bar.bad..syntax}: invalid path
+  at resources.volumes.foo.schema_name
+  in databricks.yml:11:21
+
 {
   "volumes": {
     "bar": {

acceptance/bundle/variables/dollar_before_non_brace/databricks.yml

@@ -0,0 +1,6 @@
+bundle:
+  name: '$foo is not a reference'
+
+variables:
+  a:
+    default: hello

acceptance/bundle/variables/dollar_before_non_brace/output.txt

@@ -0,0 +1,9 @@
+
+>>> [CLI] bundle validate
+Name: $foo is not a reference
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/$foo is not a reference/default
+
+Validation OK!

acceptance/bundle/variables/dollar_before_non_brace/script

@@ -0,0 +1 @@
+trace $CLI bundle validate

acceptance/bundle/variables/empty_ref/databricks.yml

@@ -0,0 +1,6 @@
+bundle:
+  name: '${}'
+
+variables:
+  a:
+    default: hello

acceptance/bundle/variables/empty_ref/output.txt

@@ -0,0 +1,13 @@
+
+>>> [CLI] bundle validate
+Warning: empty variable reference
+  at bundle.name
+  in databricks.yml:2:9
+
+Name: ${}
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/${}/default
+
+Found 1 warning

acceptance/bundle/variables/empty_ref/script

@@ -0,0 +1 @@
+trace $CLI bundle validate

acceptance/bundle/variables/leading_digit_ref/databricks.yml

@@ -0,0 +1,6 @@
+bundle:
+  name: '${0foo}'
+
+variables:
+  a:
+    default: hello

acceptance/bundle/variables/leading_digit_ref/output.txt

@@ -0,0 +1,13 @@
+
+>>> [CLI] bundle validate
+Warning: invalid variable reference ${0foo}: invalid key "0foo"
+  at bundle.name
+  in databricks.yml:2:9
+
+Name: ${0foo}
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/${0foo}/default
+
+Found 1 warning

acceptance/bundle/variables/leading_digit_ref/script

@@ -0,0 +1 @@
+trace $CLI bundle validate

acceptance/bundle/variables/malformed_reference/databricks.yml

@@ -0,0 +1,6 @@
+bundle:
+  name: "${foo.bar-}"
+
+variables:
+  a:
+    default: hello

acceptance/bundle/variables/malformed_reference/output.txt

@@ -0,0 +1,13 @@
+
+>>> [CLI] bundle validate
+Warning: invalid variable reference ${foo.bar-}: invalid key "bar-"
+  at bundle.name
+  in databricks.yml:2:9
+
+Name: ${foo.bar-}
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/${foo.bar-}/default
+
+Found 1 warning

acceptance/bundle/variables/malformed_reference/script

@@ -0,0 +1 @@
+trace $CLI bundle validate

acceptance/bundle/variables/unterminated_ref/databricks.yml

@@ -0,0 +1,6 @@
+bundle:
+  name: '${foo.bar'
+
+variables:
+  a:
+    default: hello

acceptance/bundle/variables/unterminated_ref/output.txt

@@ -0,0 +1,13 @@
+
+>>> [CLI] bundle validate
+Warning: unterminated variable reference
+  at bundle.name
+  in databricks.yml:2:9
+
+Name: ${foo.bar
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/${foo.bar/default
+
+Found 1 warning

acceptance/bundle/variables/unterminated_ref/script

@@ -0,0 +1 @@
+trace $CLI bundle validate

acceptance/bundle/variables/var_in_var_3level/databricks.yml

@@ -0,0 +1,12 @@
+bundle:
+  name: test-bundle
+
+variables:
+  env:
+    default: prod
+  region_prod:
+    default: us
+  endpoint_us:
+    default: https://us.example.com
+  result:
+    default: ${var.endpoint_${var.region_${var.env}}}

acceptance/bundle/variables/var_in_var_3level/output.txt

@@ -0,0 +1,20 @@
+
+>>> [CLI] bundle validate -o json
+{
+  "endpoint_us": {
+    "default": "https://us.example.com",
+    "value": "https://us.example.com"
+  },
+  "env": {
+    "default": "prod",
+    "value": "prod"
+  },
+  "region_prod": {
+    "default": "us",
+    "value": "us"
+  },
+  "result": {
+    "default": "https://us.example.com",
+    "value": "https://us.example.com"
+  }
+}

acceptance/bundle/variables/var_in_var_3level/script

@@ -0,0 +1 @@
+trace $CLI bundle validate -o json | jq .variables

bundle/config/mutator/warn_malformed_references.go

@@ -0,0 +1,72 @@
+package mutator
+
+import (
+	"context"
+	"errors"
+	"slices"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/interpolation"
+)
+
+type warnMalformedReferences struct{}
+
+// WarnMalformedReferences returns a mutator that emits warnings for strings
+// containing malformed variable references (e.g. "${foo.bar-}").
+func WarnMalformedReferences() bundle.Mutator {
+	return &warnMalformedReferences{}
+}
+
+func (*warnMalformedReferences) Name() string {
+	return "WarnMalformedReferences"
+}
+
+func (*warnMalformedReferences) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	var diags diag.Diagnostics
+	err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) {
+		_, err := dyn.Walk(root, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
+			// Only check values with source locations to avoid false positives
+			// from synthesized/computed values.
+			if len(v.Locations()) == 0 {
+				return v, nil
+			}
+
+			s, ok := v.AsString()
+			if !ok {
+				return v, nil
+			}
+
+			_, parseErr := interpolation.Parse(s)
+			if parseErr == nil {
+				return v, nil
+			}
+
+			var pe *interpolation.ParseError
+			if !errors.As(parseErr, &pe) {
+				return v, nil
+			}
+
+			// Clone locations and adjust column with the position offset
+			// so the diagnostic points to the problematic reference.
+			locs := slices.Clone(v.Locations())
+			if len(locs) > 0 {
+				locs[0].Column += pe.Pos
+			}
+
+			diags = append(diags, diag.Diagnostic{
+				Severity:  diag.Warning,
+				Summary:   pe.Msg,
+				Locations: locs,
+				Paths:     []dyn.Path{p},
+			})
+			return v, nil
+		})
+		return root, err
+	})
+	if err != nil {
+		diags = diags.Extend(diag.FromErr(err))
+	}
+	return diags
+}

bundle/phases/initialize.go

@@ -95,6 +95,10 @@ func Initialize(ctx context.Context, b *bundle.Bundle) {
 		// searches for strings with variable references in them.
 		mutator.RewriteWorkspacePrefix(),
 
+		// Walks the config tree and emits warnings for malformed variable references
+		// (e.g. "${foo.bar-}") before variable resolution occurs.
+		mutator.WarnMalformedReferences(),
+
 		// Reads (dynamic): variables.* (checks if there's a value assigned to variable already or if it has lookup reference)
 		// Updates (dynamic): variables.*.value (sets values from environment variables, variable files, or defaults)
 		// Resolves and sets values for bundle variables in the following order: from environment variables, from variable files and then defaults