From c43622b86e8060d8a59189828fe8b37ee9b7866d Mon Sep 17 00:00:00 2001 From: Itxaka Date: Tue, 9 May 2023 14:56:12 +0200 Subject: [PATCH] Add missing patch (#17) --- go.mod | 3 +- go.sum | 1 + pkg/config/collector/collector.go | 139 +++++++++++- pkg/config/collector/collector_test.go | 288 ++++++++++++++++++++++++- 4 files changed, 425 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 9b867c1..5d41a6c 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,6 @@ require ( github.com/erikgeiser/promptkit v0.8.0 github.com/google/go-github/v40 v40.0.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/imdario/mergo v0.3.15 github.com/itchyny/gojq v0.12.12 github.com/jaypipes/ghw v0.10.0 github.com/joho/godotenv v1.5.1 @@ -40,6 +39,7 @@ require ( github.com/twpayne/go-vfs v1.7.2 github.com/urfave/cli/v2 v2.25.1 github.com/zloylos/grsync v1.7.0 + golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b golang.org/x/net v0.9.0 golang.org/x/oauth2 v0.7.0 gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 @@ -121,6 +121,7 @@ require ( github.com/hashicorp/go-version v1.3.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/huandu/xstrings v1.3.2 // indirect + github.com/imdario/mergo v0.3.15 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/ipfs/go-log/v2 v2.5.1 // indirect github.com/ishidawataru/sctp v0.0.0-20210707070123-9a39160e9062 // indirect diff --git a/go.sum b/go.sum index 00c4889..e5cfc71 100644 --- a/go.sum +++ b/go.sum @@ -1915,6 +1915,7 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b h1:SCE/18RnFsLrjydh/R/s5EVvHoZprqEQUuoxK8q2Pc4= +golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= diff --git a/pkg/config/collector/collector.go b/pkg/config/collector/collector.go index beb1f6d..27ea3c2 100644 --- a/pkg/config/collector/collector.go +++ b/pkg/config/collector/collector.go @@ -9,14 +9,16 @@ import ( "net/http" "os" "path/filepath" + "reflect" "strings" "time" "unicode" + "golang.org/x/exp/slices" + "github.com/kairos-io/kairos-sdk/machine" "github.com/avast/retry-go" - "github.com/imdario/mergo" "github.com/itchyny/gojq" "gopkg.in/yaml.v3" ) @@ -63,9 +65,142 @@ func (c *Config) MergeConfigURL() error { return c.MergeConfig(remoteConfig) } +func (c *Config) toMap() (map[string]interface{}, error) { + var result map[string]interface{} + data, err := yaml.Marshal(c) + if err != nil { + return result, err + } + + err = yaml.Unmarshal(data, &result) + return result, err +} + +func (c *Config) applyMap(i interface{}) error { + data, err := yaml.Marshal(i) + if err != nil { + return err + } + + err = yaml.Unmarshal(data, c) + return err +} + // MergeConfig merges the config passed as parameter back to the receiver Config. func (c *Config) MergeConfig(newConfig *Config) error { - return mergo.Merge(c, newConfig, func(c *mergo.Config) { c.Overwrite = true }) + var err error + + // convert the two configs into maps + aMap, err := c.toMap() + if err != nil { + return err + } + bMap, err := newConfig.toMap() + if err != nil { + return err + } + + // deep merge the two maps + cMap, err := DeepMerge(aMap, bMap) + if err != nil { + return err + } + + // apply the result of the deepmerge into the base config + return c.applyMap(cMap) +} + +func deepMergeSlices(sliceA, sliceB []interface{}) ([]interface{}, error) { + // We use the first item in the slice to determine if there are maps present. + // Do we need to do the same for other types? + firstItem := sliceA[0] + if reflect.ValueOf(firstItem).Kind() == reflect.Map { + temp := make(map[string]interface{}) + + // first we put in temp all the keys present in a, and assign them their existing values + for _, item := range sliceA { + for k, v := range item.(map[string]interface{}) { + temp[k] = v + } + } + + // then we go through b to merge each of its keys + for _, item := range sliceB { + for k, v := range item.(map[string]interface{}) { + current, ok := temp[k] + if ok { + // if the key exists, we deep merge it + dm, err := DeepMerge(current, v) + if err != nil { + return []interface{}{}, fmt.Errorf("cannot merge %s with %s", current, v) + } + temp[k] = dm + } else { + // otherwise we just set it + temp[k] = v + } + } + } + + return []interface{}{temp}, nil + } + + // for simple slices + for _, v := range sliceB { + i := slices.Index(sliceA, v) + if i < 0 { + sliceA = append(sliceA, v) + } + } + + return sliceA, nil +} + +func deepMergeMaps(a, b map[string]interface{}) (map[string]interface{}, error) { + // go through all items in b and merge them to a + for k, v := range b { + current, ok := a[k] + if ok { + // when the key is already set, we don't know what type it has, so we deep merge them in case they are maps + // or slices + res, err := DeepMerge(current, v) + if err != nil { + return a, err + } + a[k] = res + } else { + a[k] = v + } + } + + return a, nil +} + +// DeepMerge takes two data structures and merges them together deeply. The results can vary depending on how the +// arguments are passed since structure B will always overwrite what's on A. +func DeepMerge(a, b interface{}) (interface{}, error) { + if a == nil && b != nil { + return b, nil + } + + typeA := reflect.TypeOf(a) + typeB := reflect.TypeOf(b) + + // We don't support merging different data structures + if typeA.Kind() != typeB.Kind() { + return map[string]interface{}{}, fmt.Errorf("cannot merge %s with %s", typeA.String(), typeB.String()) + } + + if typeA.Kind() == reflect.Slice { + return deepMergeSlices(a.([]interface{}), b.([]interface{})) + } + + if typeA.Kind() == reflect.Map { + return deepMergeMaps(a.(map[string]interface{}), b.(map[string]interface{})) + } + + // for any other type, b should take precedence + return b, nil } // String returns a string which is a Yaml representation of the Config. diff --git a/pkg/config/collector/collector_test.go b/pkg/config/collector/collector_test.go index 5ed6e32..17264f7 100644 --- a/pkg/config/collector/collector_test.go +++ b/pkg/config/collector/collector_test.go @@ -7,7 +7,6 @@ import ( "path/filepath" "github.com/kairos-io/kairos/v2/pkg/config" - . "github.com/kairos-io/kairos/v2/pkg/config/collector" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -77,7 +76,7 @@ info: }) It("merges the keys", func() { Expect(originalConfig.MergeConfig(newConfig)).ToNot(HaveOccurred()) - info, isMap := (*originalConfig)["info"].(map[interface{}]interface{}) + info, isMap := (*originalConfig)["info"].(Config) Expect(isMap).To(BeTrue()) Expect(info["name"]).To(Equal("Mario")) Expect(info["surname"]).To(Equal("Bros")) @@ -180,7 +179,7 @@ info: Expect(ok).To(BeTrue()) Expect(surname).To(Equal("Bras")) - info, ok := (*originalConfig)["info"].(map[interface{}]interface{}) + info, ok := (*originalConfig)["info"].(Config) Expect(ok).To(BeTrue()) Expect(info["job"]).To(Equal("plumber")) Expect(info["girlfriend"]).To(Equal("princess")) @@ -190,7 +189,290 @@ info: }) }) + Describe("deepMerge", func() { + Context("different types", func() { + a := map[string]interface{}{} + b := []string{} + + It("merges", func() { + _, err := DeepMerge(a, b) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("cannot merge map[string]interface {} with []string")) + + _, err = DeepMerge(b, a) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("cannot merge []string with map[string]interface {}")) + }) + }) + + Context("simple slices", func() { + a := []interface{}{"one", "three"} + b := []interface{}{"two", 4} + + It("merges", func() { + c, err := DeepMerge(a, b) + Expect(err).ToNot(HaveOccurred()) + Expect(c).To(Equal([]interface{}{"one", "three", "two", 4})) + }) + }) + + Context("slices containing maps", func() { + a := []interface{}{ + map[string]interface{}{ + "users": []interface{}{ + map[string]interface{}{ + "kairos": map[string]interface{}{ + "passwd": "kairos", + }, + }, + }, + }, + } + b := []interface{}{ + map[string]interface{}{ + "users": []interface{}{ + map[string]interface{}{ + "foo": map[string]interface{}{ + "passwd": "bar", + }, + }, + }, + }, + } + + It("merges", func() { + c, err := DeepMerge(a, b) + Expect(err).ToNot(HaveOccurred()) + users := c.([]interface{})[0].(map[string]interface{})["users"] + Expect(users).To(HaveLen(1)) + Expect(users).To(Equal([]interface{}{ + map[string]interface{}{ + "kairos": map[string]interface{}{ + "passwd": "kairos", + }, + "foo": map[string]interface{}{ + "passwd": "bar", + }, + }, + })) + }) + }) + + Context("empty map", func() { + a := map[string]interface{}{} + b := map[string]interface{}{ + "foo": "bar", + } + + It("merges", func() { + c, err := DeepMerge(a, b) + Expect(err).ToNot(HaveOccurred()) + Expect(c).To(Equal(map[string]interface{}{ + "foo": "bar", + })) + }) + }) + + Context("simple map", func() { + a := map[string]interface{}{ + "es": "uno", + "nl": "een", + "#": 0, + } + b := map[string]interface{}{ + "en": "one", + "nl": "één", + "de": "Eins", + "#": 1, + } + + It("merges", func() { + c, err := DeepMerge(a, b) + Expect(err).ToNot(HaveOccurred()) + Expect(c).To(Equal(map[string]interface{}{ + "#": 1, + "de": "Eins", + "en": "one", + "es": "uno", + "nl": "één", + })) + }) + }) + }) + Describe("Scan", func() { + Context("duplicated configs", func() { + var cmdLinePath, tmpDir1 string + var err error + + BeforeEach(func() { + tmpDir1, err = os.MkdirTemp("", "config1") + Expect(err).ToNot(HaveOccurred()) + err := os.WriteFile(path.Join(tmpDir1, "local_config_1.yaml"), []byte(`#cloud-config + +stages: + initramfs: + - name: "Set user and password" + users: + kairos: + passwd: "kairos" + hostname: kairos-{{ trunc 4 .Random }} + +install: + auto: true + reboot: true + device: auto + grub_options: + extra_cmdline: foobarzz + bundles: + - rootfs_path: /usr/local/lib/extensions/kubo + targets: + - container://ttl.sh/97d4530c-df80-4eb4-9ae7-39f8f90c26e5:8h +`), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + err = os.WriteFile(path.Join(tmpDir1, "local_config_2.yaml"), []byte(`#cloud-config + +stages: + initramfs: + - name: "Set user and password" + users: + kairos: + passwd: "kairos" + hostname: kairos-{{ trunc 4 .Random }} + +install: + auto: true + reboot: true + device: auto + grub_options: + extra_cmdline: foobarzz + bundles: + - rootfs_path: /usr/local/lib/extensions/kubo + targets: + - container://ttl.sh/97d4530c-df80-4eb4-9ae7-39f8f90c26e5:8h +`), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + err = os.RemoveAll(tmpDir1) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should be the same as just one of them", func() { + o := &Options{} + err := o.Apply( + MergeBootLine, + WithBootCMDLineFile(cmdLinePath), + Directories(tmpDir1), + ) + Expect(err).ToNot(HaveOccurred()) + + c, err := Scan(o, config.FilterKeys) + Expect(err).ToNot(HaveOccurred()) + + fmt.Println(c.String()) + Expect(c.String()).To(Equal(`#cloud-config + +install: + auto: true + bundles: + - rootfs_path: /usr/local/lib/extensions/kubo + targets: + - container://ttl.sh/97d4530c-df80-4eb4-9ae7-39f8f90c26e5:8h + device: auto + grub_options: + extra_cmdline: foobarzz + reboot: true +stages: + initramfs: + - hostname: kairos-{{ trunc 4 .Random }} + name: Set user and password + users: + kairos: + passwd: kairos +`)) + }) + }) + Context("Deep merge maps within arrays", func() { + var cmdLinePath, tmpDir1 string + var err error + + BeforeEach(func() { + tmpDir1, err = os.MkdirTemp("", "config1") + Expect(err).ToNot(HaveOccurred()) + err := os.WriteFile(path.Join(tmpDir1, "local_config_1.yaml"), []byte(`#cloud-config +install: + auto: true + reboot: false + poweroff: false + grub_options: + extra_cmdline: "console=tty0" +options: + device: /dev/sda +stages: + initramfs: + - users: + kairos: + groups: + - sudo + passwd: kairos +`), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + err = os.WriteFile(path.Join(tmpDir1, "local_config_2.yaml"), []byte(`#cloud-config +stages: + initramfs: + - users: + foo: + groups: + - sudo + passwd: bar +`), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + err = os.RemoveAll(tmpDir1) + Expect(err).ToNot(HaveOccurred()) + }) + + It("merges all the sources accordingly", func() { + o := &Options{} + err := o.Apply( + MergeBootLine, + WithBootCMDLineFile(cmdLinePath), + Directories(tmpDir1), + ) + Expect(err).ToNot(HaveOccurred()) + + c, err := Scan(o, config.FilterKeys) + Expect(err).ToNot(HaveOccurred()) + + Expect(c.String()).To(Equal(`#cloud-config + +install: + auto: true + grub_options: + extra_cmdline: console=tty0 + poweroff: false + reboot: false +options: + device: /dev/sda +stages: + initramfs: + - users: + foo: + groups: + - sudo + passwd: bar + kairos: + groups: + - sudo + passwd: kairos +`)) + }) + }) + Context("multiple sources are defined", func() { var cmdLinePath, serverDir, tmpDir, tmpDir1, tmpDir2 string var err error