fix: Imported variables are not used in other variables (#2432)

* fix: Imported variables are not used in other variables

* test: add test for the fix

* test: fix language handler test

* fix: import variable resolving

Co-authored-by: Russell Centanni <russell.centanni@gmail.com>
Co-authored-by: Fabian Kramm <fab.kramm@googlemail.com>

Author: 3 people

cmd/root.go

@@ -402,7 +402,7 @@ func (r *RawConfig) GetEnv(name string) string {
 		}
 
 		varName := "${" + name + "}"
-		out, err := r.Resolver.FillVariables(r.Ctx, varName)
+		out, err := r.Resolver.FillVariables(r.Ctx, varName, true)
 		if err == nil {
 			value := fmt.Sprintf("%v", out)
 			if value != varName && value != "" {

e2e/tests/imports/imports.go

@@ -1,14 +1,15 @@
 package imports
 
 import (
+	"os"
+	"strings"
+
 	"github.com/loft-sh/devspace/cmd"
 	"github.com/loft-sh/devspace/cmd/flags"
 	"github.com/loft-sh/devspace/e2e/framework"
 	"github.com/loft-sh/devspace/e2e/kube"
 	"github.com/loft-sh/devspace/pkg/util/factory"
 	"github.com/onsi/ginkgo"
-	"os"
-	"strings"
 )
 
 var _ = DevSpaceDescribe("imports", func() {
@@ -82,6 +83,8 @@ var _ = DevSpaceDescribe("imports", func() {
 
 		// read temp folder
 		framework.ExpectLocalFileContentsWithoutSpaces("import1.txt", "import2")
+		framework.ExpectLocalFileContentsWithoutSpaces("message.txt", "Life is Good!")
+		framework.ExpectLocalFileContentsWithoutSpaces("hello.txt", "")
 	})
 
 	ginkgo.It("should import correctly", func() {

e2e/tests/imports/testdata/conditional/devspace.yaml

@@ -3,13 +3,20 @@ name: base
 
 imports:
   - path: ./${IMPORT1_PATH}
+  - path: ./import.yaml
 
 vars:
   IMPORT1_PATH:
     source: env
     default: import1.yaml
+  TODAYS_MESSAGE:
+    source: command
+    command: echo ${NEW_MESSAGE}
+  HELLO_WORLD: ""
 
 pipelines:
   deploy:
     run: |-
       echo ${IMPORT1} > import1.txt
+      echo ${TODAYS_MESSAGE} > message.txt
+      echo ${HELLO_WORLD} > hello.txt

e2e/tests/imports/testdata/conditional/import.yaml

@@ -0,0 +1,7 @@
+version: v2beta1
+
+vars:
+  NEW_MESSAGE:
+    value: Life is Good!
+  HELLO_WORLD:
+    value: Hello World

pkg/devspace/config/loader/imports.go

@@ -46,7 +46,7 @@ func ResolveImports(ctx context.Context, resolver variable.Resolver, basePath st
 		return nil, errors.Errorf("Version is missing in devspace.yaml")
 	}
 
-	rawImportsInterface, err := resolver.FillVariablesInclude(ctx, rawImports, []string{"/imports/**"})
+	rawImportsInterface, err := resolver.FillVariablesInclude(ctx, rawImports, true, []string{"/imports/**"})
 	if err != nil {
 		return nil, err
 	}

pkg/devspace/config/loader/loader.go

@@ -506,15 +506,6 @@ func prepareProfiles(ctx context.Context, config map[string]interface{}, resolve
 			profileMap["replace"] = replace
 		}
 
-		// Resolve strategicMerge field
-		if profileMap["strategicMerge"] != nil {
-			strategicMerge, err := resolve(ctx, profileMap["strategicMerge"], resolver)
-			if err != nil {
-				return nil, err
-			}
-			profileMap["strategicMerge"] = strategicMerge
-		}
-
 		// Validate that the profile doesn't use forbidden expressions
 		err = validateProfile(profileMap)
 		if err != nil {
@@ -536,7 +527,7 @@ func resolve(ctx context.Context, data interface{}, resolver variable.Resolver)
 	}
 
 	// find and fill variables
-	return resolver.FillVariables(ctx, data)
+	return resolver.FillVariables(ctx, data, true)
 }
 
 func (l *configLoader) applyProfiles(ctx context.Context, data map[string]interface{}, options *ConfigOptions, resolver variable.Resolver, log log.Logger) (map[string]interface{}, error) {

pkg/devspace/config/loader/parser.go

@@ -126,7 +126,7 @@ func fillVariablesExcludeAndParse(ctx context.Context, resolver variable.Resolve
 
 func fillVariablesAndParse(ctx context.Context, resolver variable.Resolver, preparedConfig map[string]interface{}, log log.Logger, excludedPaths ...string) (*latest.Config, map[string]interface{}, error) {
 	// fill in variables and expressions
-	preparedConfigInterface, err := resolver.FillVariablesExclude(ctx, preparedConfig, excludedPaths)
+	preparedConfigInterface, err := resolver.FillVariablesExclude(ctx, preparedConfig, false, excludedPaths)
 	if err != nil {
 		return nil, nil, err
 	}

pkg/devspace/config/loader/variable/resolver.go

@@ -117,7 +117,7 @@ func (r *resolver) replaceString(ctx context.Context, str string) (interface{},
 	})
 }
 
-func (r *resolver) findVariablesInclude(haystack interface{}, include []*regexp.Regexp) ([]*latest.Variable, error) {
+func (r *resolver) findVariables(haystack interface{}, skipUnused bool, include []*regexp.Regexp) ([]*latest.Variable, error) {
 	// find out what vars are really used
 	varsUsed := map[string]bool{}
 	switch t := haystack.(type) {
@@ -144,10 +144,13 @@ func (r *resolver) findVariablesInclude(haystack interface{}, include []*regexp.
 		}
 	}
 
-	// add always resolve variables
-	for _, v := range r.vars {
-		if v.AlwaysResolve == nil || *v.AlwaysResolve {
-			varsUsed[v.Name] = true
+	// add always resolve variables. If skip unused is true, we don't
+	// resolve by default and instead only resolve the actually found variables
+	if !skipUnused {
+		for _, v := range r.vars {
+			if v.AlwaysResolve == nil || *v.AlwaysResolve {
+				varsUsed[v.Name] = true
+			}
 		}
 	}
 
@@ -162,7 +165,7 @@ func (r *resolver) findVariablesInclude(haystack interface{}, include []*regexp.
 }
 
 func (r *resolver) FindVariables(haystack interface{}) ([]*latest.Variable, error) {
-	return r.findVariablesInclude(haystack, nil)
+	return r.findVariables(haystack, false, nil)
 }
 
 func (r *resolver) getVariableDefinition(name string) *latest.Variable {
@@ -223,6 +226,7 @@ func (r *resolver) orderVariables(vars map[string]bool) ([]*latest.Variable, err
 	for i, j := 0, len(retVars)-1; i < j; i, j = i+1, j-1 {
 		retVars[i], retVars[j] = retVars[j], retVars[i]
 	}
+
 	return retVars, nil
 }
 
@@ -257,7 +261,7 @@ func (r *resolver) insertVariableGraph(g *graph.Graph, node *latest.Variable) er
 	return nil
 }
 
-func (r *resolver) FillVariablesInclude(ctx context.Context, haystack interface{}, includedPaths []string) (interface{}, error) {
+func (r *resolver) FillVariablesInclude(ctx context.Context, haystack interface{}, skipUnused bool, includedPaths []string) (interface{}, error) {
 	paths := []*regexp.Regexp{}
 	for _, path := range includedPaths {
 		path = strings.ReplaceAll(path, "**", ".+")
@@ -272,7 +276,7 @@ func (r *resolver) FillVariablesInclude(ctx context.Context, haystack interface{
 	}
 
 	// fill variables
-	preparedConfigInterface, err := r.findAndFillVariables(ctx, haystack, nil, paths)
+	preparedConfigInterface, err := r.findAndFillVariables(ctx, haystack, skipUnused, nil, paths)
 	if err != nil {
 		return nil, err
 	}
@@ -284,10 +288,10 @@ func (r *resolver) FillVariablesInclude(ctx context.Context, haystack interface{
 	}
 
 	// fill in variables again
-	return r.findAndFillVariables(ctx, preparedConfigInterface, nil, paths)
+	return r.findAndFillVariables(ctx, preparedConfigInterface, skipUnused, nil, paths)
 }
 
-func (r *resolver) FillVariablesExclude(ctx context.Context, haystack interface{}, excludedPaths []string) (interface{}, error) {
+func (r *resolver) FillVariablesExclude(ctx context.Context, haystack interface{}, skipUnused bool, excludedPaths []string) (interface{}, error) {
 	paths := []*regexp.Regexp{}
 	for _, path := range excludedPaths {
 		path = strings.ReplaceAll(path, "**", ".+")
@@ -302,7 +306,7 @@ func (r *resolver) FillVariablesExclude(ctx context.Context, haystack interface{
 	}
 
 	// fill variables
-	preparedConfigInterface, err := r.findAndFillVariables(ctx, haystack, paths, nil)
+	preparedConfigInterface, err := r.findAndFillVariables(ctx, haystack, skipUnused, paths, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -314,15 +318,15 @@ func (r *resolver) FillVariablesExclude(ctx context.Context, haystack interface{
 	}
 
 	// fill in variables again
-	return r.findAndFillVariables(ctx, preparedConfigInterface, paths, nil)
+	return r.findAndFillVariables(ctx, preparedConfigInterface, skipUnused, paths, nil)
 }
 
-func (r *resolver) FillVariables(ctx context.Context, haystack interface{}) (interface{}, error) {
-	return r.FillVariablesExclude(ctx, haystack, nil)
+func (r *resolver) FillVariables(ctx context.Context, haystack interface{}, skipUnused bool) (interface{}, error) {
+	return r.FillVariablesExclude(ctx, haystack, skipUnused, nil)
 }
 
-func (r *resolver) findAndFillVariables(ctx context.Context, haystack interface{}, exclude, include []*regexp.Regexp) (interface{}, error) {
-	varsUsed, err := r.findVariablesInclude(haystack, include)
+func (r *resolver) findAndFillVariables(ctx context.Context, haystack interface{}, skipUnused bool, exclude, include []*regexp.Regexp) (interface{}, error) {
+	varsUsed, err := r.findVariables(haystack, skipUnused, include)
 	if err != nil {
 		return nil, err
 	}

pkg/devspace/config/loader/variable/types.go

@@ -33,13 +33,13 @@ type Resolver interface {
 	FindVariables(haystack interface{}) ([]*latest.Variable, error)
 
 	// FillVariables finds the used variables first and then fills in those in the haystack
-	FillVariables(ctx context.Context, haystack interface{}) (interface{}, error)
+	FillVariables(ctx context.Context, haystack interface{}, skipUnused bool) (interface{}, error)
 
 	// FillVariablesExclude finds the used variables first and then fills in those that do not match the excluded paths in the haystack
-	FillVariablesExclude(ctx context.Context, haystack interface{}, excluded []string) (interface{}, error)
+	FillVariablesExclude(ctx context.Context, haystack interface{}, skipUnused bool, excluded []string) (interface{}, error)
 
 	// FillVariablesInclude finds the used variables first and then fills in those that match the included paths in the haystack
-	FillVariablesInclude(ctx context.Context, haystack interface{}, included []string) (interface{}, error)
+	FillVariablesInclude(ctx context.Context, haystack interface{}, skipUnused bool, included []string) (interface{}, error)
 
 	// ResolvedVariables returns the internal memory cache of the resolver with all resolved variables
 	ResolvedVariables() map[string]interface{}

pkg/devspace/config/versions/versions.go

@@ -349,7 +349,7 @@ func sanitizeMatchExpression(expression string) string {
 }
 
 func resolveVariableValue(ctx context.Context, name string, resolver variable.Resolver) (string, error) {
-	val, err := resolver.FillVariables(ctx, "${"+name+"}")
+	val, err := resolver.FillVariables(ctx, "${"+name+"}", true)
 	if err != nil {
 		return "", err
 	}