diff --git a/devspace-schema.json b/devspace-schema.json index 3b09756001..e8f212146f 100644 --- a/devspace-schema.json +++ b/devspace-schema.json @@ -2021,6 +2021,14 @@ ], "description": "Enabled specifies if the given import should be enabled" }, + "mergeStrategy": { + "type": "string", + "enum": [ + "shallowMerge", + "deepMerge" + ], + "description": "MergeStrategy specifies how the imported config should be merged (e.g. shallowMerge or deepMerge)" + }, "path": { "type": "string", "description": "Path is the local path where DevSpace can find the artifact.\nThis option is mutually exclusive with the git option.", diff --git a/docs/schemas/config-openapi.json b/docs/schemas/config-openapi.json index 9fcc8015c6..fa930e2907 100755 --- a/docs/schemas/config-openapi.json +++ b/docs/schemas/config-openapi.json @@ -1045,6 +1045,14 @@ "type": "boolean", "description": "Enabled specifies if the given import should be enabled" }, + "mergeStrategy": { + "type": "string", + "enum": [ + "shallowMerge", + "deepMerge" + ], + "description": "MergeStrategy specifies how the imported config should be merged (e.g. shallowMerge or deepMerge)" + }, "path": { "type": "string", "description": "Path is the local path where DevSpace can find the artifact.\nThis option is mutually exclusive with the git option.", diff --git a/pkg/devspace/config/loader/imports.go b/pkg/devspace/config/loader/imports.go index decc43f183..9542f57722 100644 --- a/pkg/devspace/config/loader/imports.go +++ b/pkg/devspace/config/loader/imports.go @@ -8,6 +8,7 @@ import ( "github.com/loft-sh/devspace/pkg/devspace/config/loader/variable" "github.com/loft-sh/devspace/pkg/devspace/config/versions" + "github.com/loft-sh/devspace/pkg/devspace/config/versions/latest" "github.com/loft-sh/devspace/pkg/devspace/config/versions/util" dependencyutil "github.com/loft-sh/devspace/pkg/devspace/dependency/util" "github.com/loft-sh/devspace/pkg/util/log" @@ -119,11 +120,13 @@ func ResolveImports(ctx context.Context, resolver variable.Resolver, basePath st mergedMap[section] = map[string]interface{}{} } - for key, value := range sectionMap { - _, ok := mergedMap[section].(map[string]interface{})[key] - if !ok { - mergedMap[section].(map[string]interface{})[key] = value - } + switch i.MergeStrategy { + case latest.MergeStrategyDeep: + deepMerge(mergedMap[section].(map[string]interface{}), sectionMap) + case "", latest.MergeStrategyShallow: + shallowMerge(mergedMap[section].(map[string]interface{}), sectionMap) + default: + return nil, fmt.Errorf("invalid mergeStrategy %q in import %s", i.MergeStrategy, configPath) } } @@ -143,3 +146,49 @@ func ResolveImports(ctx context.Context, resolver variable.Resolver, basePath st return mergedMap, nil } + +// deepMerge recursively merges src into dst. +// Maps are deep merged, other types (arrays, scalars) in dst take precedence. +func deepMerge(dst, src map[string]interface{}) { + for key, srcVal := range src { + // Skip nil source values + if srcVal == nil { + continue + } + + dstVal, exists := dst[key] + + // Key doesn't exist in dst - add from src + if !exists { + dst[key] = srcVal + continue + } + + // Dst value is nil - use src value + if dstVal == nil { + dst[key] = srcVal + continue + } + + // Both exist and are non-nil - check if both are maps + srcMap, srcIsMap := srcVal.(map[string]interface{}) + dstMap, dstIsMap := dstVal.(map[string]interface{}) + + if srcIsMap && dstIsMap { + // Both are maps - merge recursively + deepMerge(dstMap, srcMap) + } + + // For other types (arrays, scalars), dst takes precedence (no action needed) + } +} + +// shallowMerge merges src into dst only at the top level. +// Existing keys in dst take precedence. +func shallowMerge(dst, src map[string]interface{}) { + for key, value := range src { + if _, ok := dst[key]; !ok { + dst[key] = value + } + } +} diff --git a/pkg/devspace/config/loader/loader_test.go b/pkg/devspace/config/loader/loader_test.go index a497f2b483..0cc3446a7d 100644 --- a/pkg/devspace/config/loader/loader_test.go +++ b/pkg/devspace/config/loader/loader_test.go @@ -1964,6 +1964,595 @@ deployments: } } +// TestImportsDeepMerge validates that imports are deep merged correctly. +// Maps should be recursively merged while arrays and scalars are replaced. +func TestImportsDeepMerge(t *testing.T) { + dir := t.TempDir() + + wdBackup, err := os.Getwd() + if err != nil { + t.Fatalf("Error getting current working directory: %v", err) + } + err = os.Chdir(dir) + if err != nil { + t.Fatalf("Error changing working directory: %v", err) + } + defer func() { + err = os.Chdir(wdBackup) + if err != nil { + t.Fatalf("Error changing dir back: %v", err) + } + }() + + testCases := []struct { + name string + catalogYaml string + mainYaml string + expectedErr string + verify func(t *testing.T, config *latest.Config) + }{ + { + name: "Catalog provides base config, project adds minimal customization", + catalogYaml: ` +version: v2beta1 +name: catalog +dev: + api: + command: ["/bin/bash"] + ports: + - port: "8080:8080" + sync: + - path: ./src:/app/src + excludePaths: + - "**/__pycache__/" + - "**/*.pyc" + onUpload: + restartContainer: true + logs: + enabled: true +`, + mainYaml: ` +version: v2beta1 +name: my-project +imports: + - path: catalog.yaml + mergeStrategy: deepMerge +dev: + api: + labelSelector: + app.kubernetes.io/name: my-app +`, + verify: func(t *testing.T, config *latest.Config) { + dev := config.Dev + assert.Assert(t, dev["api"] != nil, "api dev config should exist") + + // Project customization + assert.Equal(t, dev["api"].LabelSelector["app.kubernetes.io/name"], "my-app", "labelSelector from project") + + // Catalog base config preserved + assert.Equal(t, len(dev["api"].Command), 1, "command from catalog") + assert.Equal(t, dev["api"].Command[0], "/bin/bash", "command value from catalog") + assert.Assert(t, len(dev["api"].Ports) >= 1, "ports from catalog") + assert.Assert(t, len(dev["api"].Sync) >= 1, "sync from catalog") + }, + }, + { + name: "Project replaces arrays but deep merges maps", + catalogYaml: ` +version: v2beta1 +name: catalog +dev: + api: + labelSelector: + app: catalog-app + version: v1 + command: ["/bin/bash"] + ports: + - port: "8080:8080" + - port: "8443:8443" +`, + mainYaml: ` +version: v2beta1 +name: my-project +imports: + - path: catalog.yaml + mergeStrategy: deepMerge +dev: + api: + labelSelector: + app: project-app + tier: frontend + ports: + - port: "3000:3000" +`, + verify: func(t *testing.T, config *latest.Config) { + dev := config.Dev + assert.Assert(t, dev["api"] != nil, "api dev config should exist") + + // Maps are deep merged + assert.Equal(t, dev["api"].LabelSelector["app"], "project-app", "project overrides 'app' in map") + assert.Equal(t, dev["api"].LabelSelector["version"], "v1", "catalog 'version' preserved in map") + assert.Equal(t, dev["api"].LabelSelector["tier"], "frontend", "project adds 'tier' in map") + + // Arrays are replaced entirely + assert.Equal(t, len(dev["api"].Ports), 1, "project ports replace catalog ports (array replaced)") + + // Catalog scalar values preserved + assert.Equal(t, len(dev["api"].Command), 1, "command from catalog") + assert.Equal(t, dev["api"].Command[0], "/bin/bash", "command value") + }, + }, + { + name: "Catalog with nested labelSelector, project adds and overrides", + catalogYaml: ` +version: v2beta1 +name: catalog +dev: + api: + labelSelector: + app: base-app + version: v1 + environment: production + command: ["/bin/bash"] + ports: + - port: "8080:8080" +`, + mainYaml: ` +version: v2beta1 +name: my-project +imports: + - path: catalog.yaml + mergeStrategy: deepMerge +dev: + api: + labelSelector: + app: my-custom-app + tier: frontend +`, + verify: func(t *testing.T, config *latest.Config) { + dev := config.Dev + assert.Assert(t, dev["api"] != nil, "api dev config should exist") + + // Project overrides specific label + assert.Equal(t, dev["api"].LabelSelector["app"], "my-custom-app", "project overrides 'app' label") + + // Project adds new label + assert.Equal(t, dev["api"].LabelSelector["tier"], "frontend", "project adds 'tier' label") + + // Catalog labels preserved when not overridden + assert.Equal(t, dev["api"].LabelSelector["version"], "v1", "catalog 'version' label preserved") + assert.Equal(t, dev["api"].LabelSelector["environment"], "production", "catalog 'environment' label preserved") + + // Catalog command/ports preserved + assert.Equal(t, len(dev["api"].Command), 1, "command from catalog") + assert.Assert(t, len(dev["api"].Ports) >= 1, "ports from catalog") + }, + }, + { + name: "Multiple services with independent merges", + catalogYaml: ` +version: v2beta1 +name: catalog +dev: + api: + command: ["/bin/bash"] + ports: + - port: "8080:8080" + sync: + - path: ./api:/app + worker: + command: ["/bin/sh"] + ports: + - port: "9090:9090" + sync: + - path: ./worker:/app +`, + mainYaml: ` +version: v2beta1 +name: my-project +imports: + - path: catalog.yaml + mergeStrategy: deepMerge +dev: + api: + labelSelector: + app.kubernetes.io/name: my-api + tier: backend + worker: + labelSelector: + app.kubernetes.io/name: my-worker + tier: jobs +`, + verify: func(t *testing.T, config *latest.Config) { + dev := config.Dev + // Verify api service + assert.Assert(t, dev["api"] != nil, "api dev config should exist") + assert.Equal(t, dev["api"].LabelSelector["app.kubernetes.io/name"], "my-api", "api labelSelector") + assert.Equal(t, dev["api"].LabelSelector["tier"], "backend", "api tier label") + assert.Equal(t, len(dev["api"].Command), 1, "api command from catalog") + assert.Equal(t, dev["api"].Command[0], "/bin/bash", "api command value") + assert.Assert(t, len(dev["api"].Ports) >= 1, "api ports from catalog") + assert.Assert(t, len(dev["api"].Sync) >= 1, "api sync from catalog") + + // Verify worker service + assert.Assert(t, dev["worker"] != nil, "worker dev config should exist") + assert.Equal(t, dev["worker"].LabelSelector["app.kubernetes.io/name"], "my-worker", "worker labelSelector") + assert.Equal(t, dev["worker"].LabelSelector["tier"], "jobs", "worker tier label") + assert.Equal(t, len(dev["worker"].Command), 1, "worker command from catalog") + assert.Equal(t, dev["worker"].Command[0], "/bin/sh", "worker command value") + assert.Assert(t, len(dev["worker"].Ports) >= 1, "worker ports from catalog") + assert.Assert(t, len(dev["worker"].Sync) >= 1, "worker sync from catalog") + }, + }, + { + name: "Project only defines labelSelector, everything else from catalog", + catalogYaml: ` +version: v2beta1 +name: catalog +dev: + api: + command: ["/bin/bash"] + ports: + - port: "8080:8080" + - port: "8443:8443" + sync: + - path: ./src:/usr/src/app/src + excludePaths: + - "**/__pycache__/" + - "**/*.pyc" + - "**/node_modules/" + onUpload: + restartContainer: true + logs: + enabled: true +`, + mainYaml: ` +version: v2beta1 +name: my-project +imports: + - path: catalog.yaml + mergeStrategy: deepMerge +dev: + api: + labelSelector: + app.kubernetes.io/name: my-minimal-app +`, + verify: func(t *testing.T, config *latest.Config) { + dev := config.Dev + assert.Assert(t, dev["api"] != nil, "api dev config should exist") + + // Only labelSelector from project + assert.Equal(t, dev["api"].LabelSelector["app.kubernetes.io/name"], "my-minimal-app", "labelSelector from project") + + // Everything else from catalog + assert.Equal(t, len(dev["api"].Command), 1, "command from catalog") + assert.Equal(t, dev["api"].Command[0], "/bin/bash", "command value") + assert.Equal(t, len(dev["api"].Ports), 2, "all ports from catalog") + assert.Assert(t, len(dev["api"].Sync) >= 1, "sync config from catalog") + }, + }, + { + name: "Default merge strategy is shallow merge", + catalogYaml: ` +version: v2beta1 +name: catalog +dev: + api: + labelSelector: + app: catalog-app + version: v1 +`, + mainYaml: ` +version: v2beta1 +name: my-project +imports: + - path: catalog.yaml +dev: + api: + labelSelector: + app: project-app + tier: frontend +`, + verify: func(t *testing.T, config *latest.Config) { + dev := config.Dev + assert.Assert(t, dev["api"] != nil, "api dev config should exist") + + // Maps are shallow merged + assert.Equal(t, dev["api"].LabelSelector["app"], "project-app", "project overrides 'app' in map") + assert.Equal(t, dev["api"].LabelSelector["tier"], "frontend", "project adds 'tier' in map") + assert.Equal(t, dev["api"].LabelSelector["version"], "", "catalog 'version' is NOT preserved in map because of shallow merge") + }, + }, + { + name: "Explicit shallowMerge strategy", + catalogYaml: ` +version: v2beta1 +name: catalog +dev: + api: + labelSelector: + app: catalog-app + version: v1 +`, + mainYaml: ` +version: v2beta1 +name: my-project +imports: + - path: catalog.yaml + mergeStrategy: shallowMerge +dev: + api: + labelSelector: + app: project-app + tier: frontend +`, + verify: func(t *testing.T, config *latest.Config) { + dev := config.Dev + assert.Assert(t, dev["api"] != nil, "api dev config should exist") + + // Maps are shallow merged + assert.Equal(t, dev["api"].LabelSelector["app"], "project-app", "project overrides 'app' in map") + assert.Equal(t, dev["api"].LabelSelector["tier"], "frontend", "project adds 'tier' in map") + assert.Equal(t, dev["api"].LabelSelector["version"], "", "catalog 'version' is NOT preserved in map because of shallow merge") + }, + }, + { + name: "mergeStrategy deepMerge on deployments with mutually exclusive fields", + catalogYaml: ` +version: v2beta1 +name: catalog +deployments: + my-app: + helm: + chart: + name: my-chart +`, + mainYaml: ` +version: v2beta1 +name: my-project +imports: + - path: catalog.yaml + mergeStrategy: deepMerge +deployments: + my-app: + kubectl: + manifests: + - my-manifest.yaml +`, + expectedErr: "deployments[my-app].kubectl and deployments[my-app].helm cannot be used together", + verify: func(t *testing.T, config *latest.Config) { + }, + }, + { + name: "mergeStrategy deepMerge on dev with mutually exclusive fields", + catalogYaml: ` +version: v2beta1 +name: catalog +dev: + my-app: + imageSelector: my-image +`, + mainYaml: ` +version: v2beta1 +name: my-project +imports: + - path: catalog.yaml + mergeStrategy: deepMerge +dev: + my-app: + labelSelector: + app: my-app +`, + expectedErr: "dev.my-app: image selector and label selector cannot be used together", + verify: func(t *testing.T, config *latest.Config) { + }, + }, + { + name: "mergeStrategy shallowMerge on dev with mutually exclusive fields (no regression)", + catalogYaml: ` +version: v2beta1 +name: catalog +dev: + my-app: + imageSelector: my-image +`, + mainYaml: ` +version: v2beta1 +name: my-project +imports: + - path: catalog.yaml + mergeStrategy: shallowMerge +dev: + my-app: + labelSelector: + app: my-app +`, + verify: func(t *testing.T, config *latest.Config) { + dev := config.Dev + assert.Assert(t, dev["my-app"] != nil, "my-app dev config should exist") + + // Only labelSelector is kept because my-app exists in dst, so src's my-app (imageSelector) is completely ignored + assert.Equal(t, dev["my-app"].ImageSelector, "", "imageSelector should NOT exist because of shallow merge") + assert.Equal(t, dev["my-app"].LabelSelector["app"], "my-app", "labelSelector should exist from project") + }, + }, + { + name: "mergeStrategy shallowMerge on deployments with mutually exclusive fields (no regression)", + catalogYaml: ` +version: v2beta1 +name: catalog +deployments: + my-app: + helm: + chart: + name: my-chart +`, + mainYaml: ` +version: v2beta1 +name: my-project +imports: + - path: catalog.yaml + mergeStrategy: shallowMerge +deployments: + my-app: + kubectl: + manifests: + - my-manifest.yaml +`, + verify: func(t *testing.T, config *latest.Config) { + deps := config.Deployments + assert.Assert(t, deps["my-app"] != nil, "my-app deployment should exist") + + // Only kubectl is kept because my-app exists in dst, so src's my-app (helm) is completely ignored + assert.Assert(t, deps["my-app"].Helm == nil, "helm should NOT exist because of shallow merge") + assert.Assert(t, deps["my-app"].Kubectl != nil, "kubectl should exist from project") + assert.Equal(t, len(deps["my-app"].Kubectl.Manifests), 1, "kubectl manifests should have 1 item") + assert.Equal(t, deps["my-app"].Kubectl.Manifests[0], "my-manifest.yaml", "kubectl manifest should be my-manifest.yaml") + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Create catalog file + err := fsutil.WriteToFile([]byte(tc.catalogYaml), filepath.Join(dir, "catalog.yaml")) + assert.NilError(t, err, "Error writing catalog.yaml") + + // Create main config file + err = fsutil.WriteToFile([]byte(tc.mainYaml), filepath.Join(dir, "devspace.yaml")) + assert.NilError(t, err, "Error writing devspace.yaml") + + // Load config + loader, err := NewConfigLoader(filepath.Join(dir, "devspace.yaml")) + assert.NilError(t, err, "Error creating config loader") + + config, err := loader.Load(context.TODO(), nil, &ConfigOptions{Dry: true}, log.Discard) + if tc.expectedErr != "" { + assert.ErrorContains(t, err, tc.expectedErr, "Error loading config in test case %s", tc.name) + return + } + assert.NilError(t, err, "Error loading config in test case %s", tc.name) + + // Verify using custom verification function + tc.verify(t, config.Config()) + }) + } +} + +func TestDeepMerge(t *testing.T) { + testCases := []struct { + name string + dst map[string]interface{} + src map[string]interface{} + expected map[string]interface{} + }{ + { + name: "Empty src map", + dst: map[string]interface{}{"a": 1}, + src: map[string]interface{}{}, + expected: map[string]interface{}{"a": 1}, + }, + { + name: "Empty dst map", + dst: map[string]interface{}{}, + src: map[string]interface{}{"a": 1}, + expected: map[string]interface{}{"a": 1}, + }, + { + name: "3 levels deep", + dst: map[string]interface{}{"a": map[string]interface{}{"b": map[string]interface{}{"c": 1}}}, + src: map[string]interface{}{"a": map[string]interface{}{"b": map[string]interface{}{"d": 2}}}, + expected: map[string]interface{}{ + "a": map[string]interface{}{ + "b": map[string]interface{}{ + "c": 1, + "d": 2, + }, + }, + }, + }, + { + name: "src has map, dst has scalar for same key", + dst: map[string]interface{}{"a": "string"}, + src: map[string]interface{}{"a": map[string]interface{}{"nested": true}}, + expected: map[string]interface{}{"a": "string"}, + }, + { + name: "src has scalar, dst has map for same key", + dst: map[string]interface{}{"a": map[string]interface{}{"nested": true}}, + src: map[string]interface{}{"a": "string"}, + expected: map[string]interface{}{"a": map[string]interface{}{"nested": true}}, + }, + { + name: "nil value in src map", + dst: map[string]interface{}{"a": 1}, + src: map[string]interface{}{"b": nil}, + expected: map[string]interface{}{"a": 1}, + }, + { + name: "nil value in dst map", + dst: map[string]interface{}{"a": nil}, + src: map[string]interface{}{"a": 1}, + expected: map[string]interface{}{"a": 1}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + deepMerge(tc.dst, tc.src) + + // Simple comparison for the test cases + dstYaml, _ := yaml.Marshal(tc.dst) + expectedYaml, _ := yaml.Marshal(tc.expected) + assert.Equal(t, string(dstYaml), string(expectedYaml), "Unexpected deepMerge result") + }) + } +} + +func TestShallowMerge(t *testing.T) { + testCases := []struct { + name string + dst map[string]interface{} + src map[string]interface{} + expected map[string]interface{} + }{ + { + name: "Empty src map", + dst: map[string]interface{}{"a": 1}, + src: map[string]interface{}{}, + expected: map[string]interface{}{"a": 1}, + }, + { + name: "Empty dst map", + dst: map[string]interface{}{}, + src: map[string]interface{}{"a": 1}, + expected: map[string]interface{}{"a": 1}, + }, + { + name: "Key exists in both (dst wins, no deep merge)", + dst: map[string]interface{}{"a": map[string]interface{}{"b": 1}}, + src: map[string]interface{}{"a": map[string]interface{}{"c": 2}}, + expected: map[string]interface{}{"a": map[string]interface{}{"b": 1}}, // "c" is completely ignored + }, + { + name: "Keys only in src are added", + dst: map[string]interface{}{"a": 1}, + src: map[string]interface{}{"b": 2}, + expected: map[string]interface{}{"a": 1, "b": 2}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + shallowMerge(tc.dst, tc.src) + + dstYaml, _ := yaml.Marshal(tc.dst) + expectedYaml, _ := yaml.Marshal(tc.expected) + assert.Equal(t, string(dstYaml), string(expectedYaml), "Unexpected shallowMerge result") + }) + } +} + func stripNames(config *latest.Config) { for k := range config.Images { config.Images[k].Name = "" diff --git a/pkg/devspace/config/versions/latest/schema.go b/pkg/devspace/config/versions/latest/schema.go index 7856c6be79..d77d9fe090 100644 --- a/pkg/devspace/config/versions/latest/schema.go +++ b/pkg/devspace/config/versions/latest/schema.go @@ -129,10 +129,23 @@ type Import struct { // Enabled specifies if the given import should be enabled Enabled *bool `yaml:"enabled,omitempty" json:"enabled,omitempty" jsonschema:"required"` + // MergeStrategy specifies how the imported config should be merged (e.g. shallowMerge or deepMerge) + MergeStrategy string `yaml:"mergeStrategy,omitempty" json:"mergeStrategy,omitempty" jsonschema:"enum=shallowMerge,enum=deepMerge"` + // SourceConfig defines the source for this import SourceConfig `yaml:",inline" json:",inline"` } +const ( + // MergeStrategyShallow performs a shallow merge where only top-level keys + // within each config section are merged. This is the default behavior. + MergeStrategyShallow = "shallowMerge" + + // MergeStrategyDeep performs a recursive deep merge where nested maps are + // merged key by key. Arrays and scalars in the local config take precedence. + MergeStrategyDeep = "deepMerge" +) + // Pipeline defines what DevSpace should do. A pipeline consists of one or more // jobs that are run in parallel and can depend on each other. Each job consists // of one or more conditional steps that are executed in order.