diff --git a/pkg/config/collector/collector.go b/pkg/config/collector/collector.go new file mode 100644 index 0000000..03d2d7a --- /dev/null +++ b/pkg/config/collector/collector.go @@ -0,0 +1,341 @@ +// Package configcollector can be used to merge configuration from different +// sources into one YAML. +package collector + +import ( + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "time" + "unicode" + + "github.com/avast/retry-go" + "github.com/google/shlex" + "github.com/imdario/mergo" + "github.com/itchyny/gojq" + "github.com/kairos-io/kairos-sdk/unstructured" + "gopkg.in/yaml.v1" +) + +const DefaultHeader = "#cloud-config" + +var ValidFileHeaders = []string{ + "#cloud-config", + "#kairos-config", + "#node-config", +} + +type Configs []*Config + +// We don't allow yamls that are plain arrays because is has no use in Kairos +// and there is no way to merge an array yaml with a "map" yaml. +type Config map[string]interface{} + +// MergeConfigURL looks for the "config_url" key and if it's found +// it downloads the remote config and merges it with the current one. +// If the remote config also has config_url defined, it is also fetched +// recursively until a remote config no longer defines a config_url. +// NOTE: The "config_url" value of the final result is the value of the last +// config file in the chain because we replace values when we merge. +func (c *Config) MergeConfigURL() error { + // If there is no config_url, just return (do nothing) + configURL := c.ConfigURL() + if configURL == "" { + return nil + } + + // fetch the remote config + remoteConfig, err := fetchRemoteConfig(configURL) + if err != nil { + return err + } + + // recursively fetch remote configs + if err := remoteConfig.MergeConfigURL(); err != nil { + return err + } + + // merge remoteConfig back to "c" + return c.MergeConfig(remoteConfig) +} + +// MergeConfig merges the config passed as parameter back to the receiver Config. +func (c *Config) MergeConfig(newConfig *Config) error { + return mergo.Merge(c, newConfig, func(c *mergo.Config) { c.Overwrite = true }) +} + +// String returns a string which is a Yaml representation of the Config. +func (c *Config) String() (string, error) { + data, err := yaml.Marshal(c) + if err != nil { + return "", err + } + + return fmt.Sprintf("%s\n\n%s", DefaultHeader, string(data)), nil +} + +func (cs Configs) Merge() (*Config, error) { + result := &Config{} + + for _, c := range cs { + if err := c.MergeConfigURL(); err != nil { + return result, err + } + + if err := result.MergeConfig(c); err != nil { + return result, err + } + } + + return result, nil +} + +func Scan(o *Options) (*Config, error) { + configs := Configs{} + + configs = append(configs, parseFiles(o.ScanDir, o.NoLogs)...) + + if o.MergeBootCMDLine { + cConfig, err := ParseCmdLine(o.BootCMDLineFile) + o.SoftErr("parsing cmdline", err) + if err == nil { // best-effort + configs = append(configs, cConfig) + } + } + + return configs.Merge() +} + +func allFiles(dir []string) []string { + files := []string{} + for _, d := range dir { + if f, err := listFiles(d); err == nil { + files = append(files, f...) + } + } + return files +} + +// parseFiles returns a list of Configs parsed from files. +func parseFiles(dir []string, nologs bool) Configs { + result := Configs{} + files := allFiles(dir) + for _, f := range files { + if fileSize(f) > 1.0 { + if !nologs { + fmt.Printf("warning: skipping %s. too big (>1MB)\n", f) + } + continue + } + if strings.Contains(f, "userdata") || filepath.Ext(f) == ".yml" || filepath.Ext(f) == ".yaml" { + b, err := os.ReadFile(f) + if err != nil { + if !nologs { + fmt.Printf("warning: skipping %s. %s\n", f, err.Error()) + } + continue + } + + if !HasValidHeader(string(b)) { + if !nologs { + fmt.Printf("warning: skipping %s because it has no valid header\n", f) + } + continue + } + + var newConfig Config + err = yaml.Unmarshal(b, &newConfig) + if err != nil && !nologs { + fmt.Printf("warning: failed to parse config:\n%s\n", err.Error()) + } + result = append(result, &newConfig) + } else { + if !nologs { + fmt.Printf("warning: skipping %s (extension).\n", f) + } + } + } + + return result +} + +func fileSize(f string) float64 { + file, err := os.Open(f) + if err != nil { + return 0 + } + defer file.Close() + + stat, err := file.Stat() + if err != nil { + return 0 + } + + bytes := stat.Size() + kilobytes := (bytes / 1024) + megabytes := (float64)(kilobytes / 1024) // cast to type float64 + + return megabytes +} + +func listFiles(dir string) ([]string, error) { + content := []string{} + + err := filepath.Walk(dir, + func(path string, info os.FileInfo, err error) error { + if err != nil { + return nil + } + if !info.IsDir() { + content = append(content, path) + } + + return nil + }) + + return content, err +} + +// ParseCmdLine reads options from the kernel cmdline and returns the equivalent +// Config. +func ParseCmdLine(file string) (*Config, error) { + result := &Config{} + + if file == "" { + file = "/proc/cmdline" + } + dat, err := os.ReadFile(file) + if err != nil { + return result, err + } + + d, err := unstructured.ToYAML(stringToConfig(string(dat))) + if err != nil { + return result, err + } + err = yaml.Unmarshal(d, &result) + + return result, err +} + +func stringToConfig(s string) Config { + v := Config{} + + splitted, _ := shlex.Split(s) + for _, item := range splitted { + parts := strings.SplitN(item, "=", 2) + value := "true" + if len(parts) > 1 { + value = strings.Trim(parts[1], `"`) + } + key := strings.Trim(parts[0], `"`) + v[key] = value + } + + return v +} + +// ConfigURL returns the value of config_url if set or empty string otherwise. +func (c Config) ConfigURL() string { + if val, hasKey := c["config_url"]; hasKey { + if s, isString := val.(string); isString { + return s + } + } + + return "" +} + +func fetchRemoteConfig(url string) (*Config, error) { + var body []byte + result := &Config{} + + err := retry.Do( + func() error { + resp, err := http.Get(url) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected status: %d", resp.StatusCode) + } + defer resp.Body.Close() + + body, err = io.ReadAll(resp.Body) + if err != nil { + return err + } + + return nil + }, retry.Delay(time.Second), retry.Attempts(3), + ) + + if err != nil { + // TODO: improve logging + fmt.Printf("WARNING: Couldn't fetch config_url: %s", err) + return result, nil + } + + if !HasValidHeader(string(body)) { + // TODO: Print a warning when we implement proper logging + fmt.Println("No valid header in remote config: %w", err) + return result, nil + } + + if err := yaml.Unmarshal(body, result); err != nil { + return result, fmt.Errorf("could not unmarshal remote config to an object: %w", err) + } + + return result, nil +} + +func HasValidHeader(data string) bool { + header := strings.SplitN(data, "\n", 2)[0] + + // Trim trailing whitespaces + header = strings.TrimRightFunc(header, unicode.IsSpace) + + // NOTE: we also allow "legacy" headers. Should only allow #cloud-config at + // some point. + return (header == DefaultHeader) || (header == "#kairos-config") || (header == "#node-config") +} + +func (c Config) Query(s string) (res string, err error) { + s = fmt.Sprintf(".%s", s) + + var dat map[string]interface{} + + yamlStr, err := c.String() + if err != nil { + panic(err) + } + if err := yaml.Unmarshal([]byte(yamlStr), &dat); err != nil { + panic(err) + } + + query, err := gojq.Parse(s) + if err != nil { + return res, err + } + + iter := query.Run(dat) // or query.RunWithContext + for { + v, ok := iter.Next() + if !ok { + break + } + if err, ok := v.(error); ok { + return res, fmt.Errorf("failed parsing, error: %w", err) + } + + dat, err := yaml.Marshal(v) + if err != nil { + break + } + res += string(dat) + } + return +} diff --git a/pkg/config/collector/collector_test.go b/pkg/config/collector/collector_test.go new file mode 100644 index 0000000..b054c2c --- /dev/null +++ b/pkg/config/collector/collector_test.go @@ -0,0 +1,468 @@ +package collector_test + +import ( + "fmt" + "os" + "path" + "path/filepath" + + . "github.com/kairos-io/kairos/pkg/config/collector" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v1" +) + +var _ = Describe("Config Collector", func() { + Describe("Options", func() { + var options *Options + + BeforeEach(func() { + options = &Options{ + NoLogs: false, + } + }) + + It("applies a defined option function", func() { + option := func(o *Options) error { + o.NoLogs = true + return nil + } + + Expect(options.NoLogs).To(BeFalse()) + Expect(options.Apply(option)).NotTo(HaveOccurred()) + Expect(options.NoLogs).To(BeTrue()) + }) + }) + + Describe("MergeConfig", func() { + var originalConfig, newConfig *Config + BeforeEach(func() { + originalConfig = &Config{} + newConfig = &Config{} + }) + + Context("different keys", func() { + BeforeEach(func() { + err := yaml.Unmarshal([]byte(`#cloud-config +name: Mario`), originalConfig) + Expect(err).ToNot(HaveOccurred()) + err = yaml.Unmarshal([]byte(`#cloud-config +surname: Bros`), newConfig) + Expect(err).ToNot(HaveOccurred()) + }) + + It("gets merged together", func() { + Expect(originalConfig.MergeConfig(newConfig)).ToNot(HaveOccurred()) + surname, isString := (*originalConfig)["surname"].(string) + Expect(isString).To(BeTrue()) + Expect(surname).To(Equal("Bros")) + }) + }) + + Context("same keys", func() { + Context("when the key is a map", func() { + BeforeEach(func() { + err := yaml.Unmarshal([]byte(`#cloud-config +info: + name: Mario +`), originalConfig) + Expect(err).ToNot(HaveOccurred()) + err = yaml.Unmarshal([]byte(`#cloud-config +info: + surname: Bros +`), newConfig) + Expect(err).ToNot(HaveOccurred()) + }) + It("merges the keys", func() { + Expect(originalConfig.MergeConfig(newConfig)).ToNot(HaveOccurred()) + info, isMap := (*originalConfig)["info"].(map[interface{}]interface{}) + Expect(isMap).To(BeTrue()) + Expect(info["name"]).To(Equal("Mario")) + Expect(info["surname"]).To(Equal("Bros")) + Expect(*originalConfig).To(HaveLen(1)) + Expect(info).To(HaveLen(2)) + }) + }) + + Context("when the key is a string", func() { + BeforeEach(func() { + err := yaml.Unmarshal([]byte("#cloud-config\nname: Mario"), originalConfig) + Expect(err).ToNot(HaveOccurred()) + err = yaml.Unmarshal([]byte("#cloud-config\nname: Luigi"), newConfig) + Expect(err).ToNot(HaveOccurred()) + }) + + It("overwrites", func() { + Expect(originalConfig.MergeConfig(newConfig)).ToNot(HaveOccurred()) + name, isString := (*originalConfig)["name"].(string) + Expect(isString).To(BeTrue()) + Expect(name).To(Equal("Luigi")) + Expect(*originalConfig).To(HaveLen(1)) + }) + }) + }) + }) + + Describe("MergeConfigURL", func() { + var originalConfig *Config + BeforeEach(func() { + originalConfig = &Config{} + }) + + Context("when there is no config_url defined", func() { + BeforeEach(func() { + err := yaml.Unmarshal([]byte("#cloud-config\nname: Mario"), originalConfig) + Expect(err).ToNot(HaveOccurred()) + }) + + It("does nothing", func() { + Expect(originalConfig.MergeConfigURL()).ToNot(HaveOccurred()) + Expect(*originalConfig).To(HaveLen(1)) + }) + }) + + Context("when there is a chain of config_url defined", func() { + var closeFunc ServerCloseFunc + var port int + var err error + var tmpDir string + var originalConfig *Config + + BeforeEach(func() { + tmpDir, err = os.MkdirTemp("", "config_url_chain") + Expect(err).ToNot(HaveOccurred()) + + closeFunc, port, err = startAssetServer(tmpDir) + Expect(err).ToNot(HaveOccurred()) + + originalConfig = &Config{} + err = yaml.Unmarshal([]byte(fmt.Sprintf(`#cloud-config +config_url: http://127.0.0.1:%d/config1.yaml +name: Mario +surname: Bros +info: + job: plumber +`, port)), originalConfig) + Expect(err).ToNot(HaveOccurred()) + + err := os.WriteFile(path.Join(tmpDir, "config1.yaml"), []byte(fmt.Sprintf(`#cloud-config + +config_url: http://127.0.0.1:%d/config2.yaml +surname: Bras +`, port)), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + + err = os.WriteFile(path.Join(tmpDir, "config2.yaml"), []byte(`#cloud-config + +info: + girlfriend: princess +`), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + closeFunc() + err := os.RemoveAll(tmpDir) + Expect(err).ToNot(HaveOccurred()) + }) + + It("merges them all together", func() { + err := originalConfig.MergeConfigURL() + Expect(err).ToNot(HaveOccurred()) + + name, ok := (*originalConfig)["name"].(string) + Expect(ok).To(BeTrue()) + Expect(name).To(Equal("Mario")) + + surname, ok := (*originalConfig)["surname"].(string) + Expect(ok).To(BeTrue()) + Expect(surname).To(Equal("Bras")) + + info, ok := (*originalConfig)["info"].(map[interface{}]interface{}) + Expect(ok).To(BeTrue()) + Expect(info["job"]).To(Equal("plumber")) + Expect(info["girlfriend"]).To(Equal("princess")) + + Expect(*originalConfig).To(HaveLen(4)) + }) + }) + }) + + Describe("Scan", func() { + Context("multiple sources are defined", func() { + var cmdLinePath, serverDir, tmpDir, tmpDir1, tmpDir2 string + var err error + var closeFunc ServerCloseFunc + var port int + + BeforeEach(func() { + // Prepare the cmdline config_url chain + serverDir, err = os.MkdirTemp("", "config_url_chain") + Expect(err).ToNot(HaveOccurred()) + closeFunc, port, err = startAssetServer(serverDir) + Expect(err).ToNot(HaveOccurred()) + cmdLinePath = createRemoteConfigs(serverDir, port) + + tmpDir1, err = os.MkdirTemp("", "config1") + Expect(err).ToNot(HaveOccurred()) + err := os.WriteFile(path.Join(tmpDir1, "local_config_1.yaml"), []byte(fmt.Sprintf(`#cloud-config + +config_url: http://127.0.0.1:%d/remote_config_3.yaml +local_key_1: local_value_1 +`, port)), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + err = os.WriteFile(path.Join(serverDir, "remote_config_3.yaml"), []byte(fmt.Sprintf(`#cloud-config + +config_url: http://127.0.0.1:%d/remote_config_4.yaml +remote_key_3: remote_value_3 +`, port)), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + + err = os.WriteFile(path.Join(serverDir, "remote_config_4.yaml"), []byte(`#cloud-config + +options: + remote_option_1: remote_option_value_1 +`), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + + tmpDir2, err = os.MkdirTemp("", "config2") + Expect(err).ToNot(HaveOccurred()) + err = os.WriteFile(path.Join(tmpDir2, "local_config_2.yaml"), []byte(fmt.Sprintf(`#cloud-config + +config_url: http://127.0.0.1:%d/remote_config_5.yaml +local_key_2: local_value_2 +`, port)), os.ModePerm) + err = os.WriteFile(path.Join(tmpDir2, "local_config_3.yaml"), []byte(`#cloud-config +local_key_3: local_value_3 +`), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + err = os.WriteFile(path.Join(serverDir, "remote_config_5.yaml"), []byte(fmt.Sprintf(`#cloud-config + +config_url: http://127.0.0.1:%d/remote_config_6.yaml +remote_key_4: remote_value_4 +`, port)), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + + err = os.WriteFile(path.Join(serverDir, "remote_config_6.yaml"), []byte(`#cloud-config + +options: + remote_option_2: remote_option_value_2 +`), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + err = os.RemoveAll(serverDir) + Expect(err).ToNot(HaveOccurred()) + err = os.RemoveAll(tmpDir) + Expect(err).ToNot(HaveOccurred()) + err = os.RemoveAll(tmpDir1) + Expect(err).ToNot(HaveOccurred()) + err = os.RemoveAll(tmpDir2) + Expect(err).ToNot(HaveOccurred()) + + closeFunc() + }) + + It("merges all the sources accordingly", func() { + o := &Options{} + err := o.Apply( + MergeBootLine, + WithBootCMDLineFile(cmdLinePath), + Directories(tmpDir1, tmpDir2), + ) + Expect(err).ToNot(HaveOccurred()) + + c, err := Scan(o) + Expect(err).ToNot(HaveOccurred()) + + config_url, ok := (*c)["config_url"].(string) + Expect(ok).To(BeTrue()) + Expect(config_url).To(MatchRegexp("remote_config_2.yaml")) + + k := (*c)["local_key_1"].(string) + Expect(k).To(Equal("local_value_1")) + k = (*c)["local_key_2"].(string) + Expect(k).To(Equal("local_value_2")) + k = (*c)["local_key_3"].(string) + Expect(k).To(Equal("local_value_3")) + k = (*c)["remote_key_1"].(string) + Expect(k).To(Equal("remote_value_1")) + k = (*c)["remote_key_2"].(string) + Expect(k).To(Equal("remote_value_2")) + k = (*c)["remote_key_3"].(string) + Expect(k).To(Equal("remote_value_3")) + k = (*c)["remote_key_4"].(string) + Expect(k).To(Equal("remote_value_4")) + + options := (*c)["options"].(map[interface{}]interface{}) + Expect(options["foo"]).To(Equal("bar")) + Expect(options["remote_option_1"]).To(Equal("remote_option_value_1")) + Expect(options["remote_option_2"]).To(Equal("remote_option_value_2")) + + player := (*c)["player"].(map[interface{}]interface{}) + Expect(player["name"]).To(Equal("Dimitris")) + Expect(player["surname"]).To(Equal("Bros")) + }) + }) + + Context("when files have invalid or missing headers", func() { + var serverDir, tmpDir string + var err error + var closeFunc ServerCloseFunc + var port int + + BeforeEach(func() { + // Prepare the cmdline config_url chain + serverDir, err = os.MkdirTemp("", "config_url_chain") + Expect(err).ToNot(HaveOccurred()) + closeFunc, port, err = startAssetServer(serverDir) + Expect(err).ToNot(HaveOccurred()) + + tmpDir, err = os.MkdirTemp("", "config") + Expect(err).ToNot(HaveOccurred()) + + // Local configs + err = os.WriteFile(path.Join(tmpDir, "local_config.yaml"), []byte(fmt.Sprintf(`#cloud-config +config_url: http://127.0.0.1:%d/remote_config_1.yaml +local_key_1: local_value_1 +`, port)), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + + // missing header + err = os.WriteFile(path.Join(tmpDir, "local_config_2.yaml"), + []byte("local_key_2: local_value_2"), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + + // Remote config with valid header + err := os.WriteFile(path.Join(serverDir, "remote_config_1.yaml"), []byte(fmt.Sprintf(`#cloud-config +config_url: http://127.0.0.1:%d/remote_config_2.yaml +remote_key_1: remote_value_1`, port)), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + + // Remote config with invalid header + err = os.WriteFile(path.Join(serverDir, "remote_config_2.yaml"), []byte(`#invalid-header +remote_key_2: remote_value_2`), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + closeFunc() + err = os.RemoveAll(serverDir) + Expect(err).ToNot(HaveOccurred()) + err = os.RemoveAll(tmpDir) + }) + + It("ignores them", func() { + o := &Options{} + err := o.Apply(Directories(tmpDir), NoLogs) + Expect(err).ToNot(HaveOccurred()) + + c, err := Scan(o) + Expect(err).ToNot(HaveOccurred()) + + Expect((*c)["local_key_2"]).To(BeNil()) + Expect((*c)["remote_key_2"]).To(BeNil()) + + // sanity check, the rest should be there + v, ok := (*c)["config_url"].(string) + Expect(ok).To(BeTrue()) + Expect(v).To(MatchRegexp("remote_config_2.yaml")) + + v, ok = (*c)["local_key_1"].(string) + Expect(ok).To(BeTrue()) + Expect(v).To(Equal("local_value_1")) + + v, ok = (*c)["remote_key_1"].(string) + Expect(ok).To(BeTrue()) + Expect(v).To(Equal("remote_value_1")) + }) + }) + }) + + Describe("String", func() { + var conf *Config + BeforeEach(func() { + conf = &Config{} + err := yaml.Unmarshal([]byte("name: Mario"), conf) + Expect(err).ToNot(HaveOccurred()) + }) + + It("returns the YAML string representation of the Config", func() { + s, err := conf.String() + Expect(err).ToNot(HaveOccurred()) + Expect(s).To(Equal(`#cloud-config + +name: Mario +`), s) + }) + }) + + Describe("Query", func() { + var tmpDir string + var err error + + BeforeEach(func() { + tmpDir, err = os.MkdirTemp("", "config") + Expect(err).ToNot(HaveOccurred()) + + err = os.WriteFile(filepath.Join(tmpDir, "b"), []byte(`zz.foo="baa" options.foo=bar`), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + + err = os.WriteFile(path.Join(tmpDir, "local_config.yaml"), []byte(`#cloud-config +local_key_1: local_value_1 +some: + other: + key: 3 +`), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + }) + + It("can query for keys", func() { + o := &Options{} + + err = o.Apply(MergeBootLine, Directories(tmpDir), + WithBootCMDLineFile(filepath.Join(tmpDir, "b")), + ) + Expect(err).ToNot(HaveOccurred()) + + c, err := Scan(o) + Expect(err).ToNot(HaveOccurred()) + + v, err := c.Query("local_key_1") + Expect(err).ToNot(HaveOccurred()) + Expect(v).To(Equal("local_value_1\n")) + v, err = c.Query("some") + Expect(err).ToNot(HaveOccurred()) + // TODO: there's a bug when trying to dig some.other.key, so making the test pass this way for now, since that was not tested before + Expect(v).To(Equal("other:\n key: 3\n")) + Expect(c.Query("options")).To(Equal("foo: bar\n")) + }) + }) +}) + +func createRemoteConfigs(serverDir string, port int) string { + err := os.WriteFile(path.Join(serverDir, "remote_config_1.yaml"), []byte(fmt.Sprintf(`#cloud-config + +config_url: http://127.0.0.1:%d/remote_config_2.yaml +player: + name: Dimitris +remote_key_1: remote_value_1 +`, port)), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + err = os.WriteFile(path.Join(serverDir, "remote_config_2.yaml"), []byte(`#cloud-config + +player: + surname: Bros +remote_key_2: remote_value_2 +`), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + + cmdLinePath := filepath.Join(serverDir, "cmdline") + // We put the cmdline in the same dir, it doesn't matter. + cmdLine := fmt.Sprintf(`config_url="http://127.0.0.1:%d/remote_config_1.yaml" player.name="Mario" options.foo=bar`, port) + err = os.WriteFile(cmdLinePath, []byte(cmdLine), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + + return cmdLinePath +} diff --git a/pkg/config/options.go b/pkg/config/collector/options.go similarity index 62% rename from pkg/config/options.go rename to pkg/config/collector/options.go index c02e936..48026dd 100644 --- a/pkg/config/options.go +++ b/pkg/config/collector/options.go @@ -1,4 +1,6 @@ -package config +package collector + +import "fmt" type Options struct { ScanDir []string @@ -15,6 +17,16 @@ var NoLogs Option = func(o *Options) error { return nil } +// SoftErr prints a warning if err is no nil and NoLogs is not true. +// It's use to wrap the same handling happening in multiple places. +// +// TODO: Switch to a standard logging library (e.g. verbose, silent mode etc). +func (o *Options) SoftErr(message string, err error) { + if !o.NoLogs && err != nil { + fmt.Printf("WARNING: %s, %s\n", message, err.Error()) + } +} + func (o *Options) Apply(opts ...Option) error { for _, oo := range opts { if err := oo(o); err != nil { @@ -35,17 +47,17 @@ func WithBootCMDLineFile(s string) Option { return nil } } + +func StrictValidation(v bool) Option { + return func(o *Options) error { + o.StrictValidation = v + return nil + } +} + func Directories(d ...string) Option { return func(o *Options) error { o.ScanDir = d return nil } } - -// StrictValidation sets the strict validation option to true or false. -func StrictValidation(b bool) Option { - return func(o *Options) error { - o.StrictValidation = b - return nil - } -} diff --git a/pkg/config/collector/suite_test.go b/pkg/config/collector/suite_test.go new file mode 100644 index 0000000..3e0f578 --- /dev/null +++ b/pkg/config/collector/suite_test.go @@ -0,0 +1,45 @@ +package collector_test + +import ( + "context" + "net" + "net/http" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Config Collector Suite") +} + +type ServerCloseFunc func() + +func startAssetServer(path string) (ServerCloseFunc, int, error) { + listener, err := net.Listen("tcp", ":0") + if err != nil { + return nil, 0, err + } + port := listener.Addr().(*net.TCPAddr).Port + + ctx, cancelFunc := context.WithCancel(context.Background()) + go func() { + defer GinkgoRecover() + err := http.Serve(listener, http.FileServer(http.Dir(path))) + select { + case <-ctx.Done(): // We closed it with the CancelFunc, ignore the error + return + default: // We didnt' close it, return the error + Expect(err).ToNot(HaveOccurred()) + } + }() + + stopFunc := func() { + cancelFunc() + listener.Close() + } + + return stopFunc, port, nil +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 00007b3..462927a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -2,19 +2,13 @@ package config import ( "fmt" - "io" - "net/http" "os" "path/filepath" "strings" "unicode" - retry "github.com/avast/retry-go" - "github.com/imdario/mergo" - "github.com/itchyny/gojq" "github.com/kairos-io/kairos-sdk/bundles" - "github.com/kairos-io/kairos-sdk/machine" - "github.com/kairos-io/kairos-sdk/unstructured" + "github.com/kairos-io/kairos/pkg/config/collector" schema "github.com/kairos-io/kairos/pkg/config/schemas" yip "github.com/mudler/yip/pkg/schema" @@ -43,9 +37,8 @@ type Install struct { type Config struct { Install *Install `yaml:"install,omitempty"` - //cloudFileContent string - originalData map[string]interface{} - header string + collector.Config + // TODO: Remove this too? ConfigURL string `yaml:"config_url,omitempty"` Options map[string]string `yaml:"options,omitempty"` FailOnBundleErrors bool `yaml:"fail_on_bundles_errors,omitempty"` @@ -97,123 +90,43 @@ func (b Bundles) Options() (res [][]bundles.BundleOption) { return } -func (c Config) Unmarshal(o interface{}) error { - return yaml.Unmarshal([]byte(c.String()), o) -} - -func (c Config) Data() map[string]interface{} { - return c.originalData -} - -func (c Config) String() string { - if len(c.originalData) == 0 { - dat, err := yaml.Marshal(c) - if err == nil { - return string(dat) - } - } - - dat, _ := yaml.Marshal(c.originalData) - if c.header != "" { - return AddHeader(c.header, string(dat)) - } - return string(dat) -} - -func (c Config) Query(s string) (res string, err error) { - s = fmt.Sprintf(".%s", s) - jsondata := map[string]interface{}{} - - // c.String() takes the original data map[string]interface{} and Marshals into YAML, then here we unmarshall it again? - // we should be able to use c.originalData and copy it to jsondata - err = yaml.Unmarshal([]byte(c.String()), &jsondata) - if err != nil { - return - } - query, err := gojq.Parse(s) - if err != nil { - return res, err - } - - iter := query.Run(jsondata) // or query.RunWithContext - for { - v, ok := iter.Next() - if !ok { - break - } - if err, ok := v.(error); ok { - return res, fmt.Errorf("failed parsing, error: %w", err) - } - - dat, err := yaml.Marshal(v) - if err != nil { - break - } - res += string(dat) - } - return -} - // HasConfigURL returns true if ConfigURL has been set and false if it's empty. func (c Config) HasConfigURL() bool { return c.ConfigURL != "" } -func allFiles(dir []string) []string { - files := []string{} - for _, d := range dir { - if f, err := listFiles(d); err == nil { - files = append(files, f...) - } - } - return files -} +func Scan(opts ...collector.Option) (c *Config, err error) { + result := &Config{} -func Scan(opts ...Option) (c *Config, err error) { - o := &Options{} + o := &collector.Options{} if err := o.Apply(opts...); err != nil { - return nil, err + return result, err } - c = parseConfig(o.ScanDir, o.NoLogs) + genericConfig, err := collector.Scan(o) + if err != nil { + return result, err - if o.MergeBootCMDLine { - d, err := machine.DotToYAML(o.BootCMDLineFile) - if err == nil { // best-effort - yaml.Unmarshal(d, c) //nolint:errcheck - // Merge back to originalData only config which are part of the config structure - // This avoid garbage as unrelated bootargs to be merged in. - dat, err := yaml.Marshal(c) - if err == nil { - yaml.Unmarshal(dat, &c.originalData) //nolint:errcheck - } - } + } + result.Config = *genericConfig + configStr, err := genericConfig.String() + if err != nil { + return result, err } - if c.HasConfigURL() { - err = c.fetchRemoteConfig() - if !o.NoLogs && err != nil { - fmt.Printf("WARNING: Couldn't fetch config_url: %s\n", err.Error()) - } + err = yaml.Unmarshal([]byte(configStr), result) + if err != nil { + return result, err } - if c.header == "" { - c.header = DefaultHeader - } - - finalYAML, err := yaml.Marshal(c.originalData) - if !o.NoLogs && err != nil { - fmt.Printf("WARNING: %s\n", err.Error()) - } - - kc, err := schema.NewConfigFromYAML(string(finalYAML), schema.RootSchema{}) + kc, err := schema.NewConfigFromYAML(configStr, schema.RootSchema{}) if err != nil { if !o.NoLogs && !o.StrictValidation { fmt.Printf("WARNING: %s\n", err.Error()) } if o.StrictValidation { - return c, fmt.Errorf("ERROR: %s", err.Error()) + return result, fmt.Errorf("ERROR: %s", err.Error()) } } @@ -223,48 +136,11 @@ func Scan(opts ...Option) (c *Config, err error) { } if o.StrictValidation { - return c, fmt.Errorf("ERROR: %s", kc.ValidationError.Error()) + return result, fmt.Errorf("ERROR: %s", kc.ValidationError.Error()) } } - return c, nil -} - -func fileSize(f string) float64 { - file, err := os.Open(f) - if err != nil { - return 0 - } - defer file.Close() - - stat, err := file.Stat() - if err != nil { - return 0 - } - - bytes := stat.Size() - kilobytes := (bytes / 1024) - megabytes := (float64)(kilobytes / 1024) // cast to type float64 - - return megabytes -} - -func listFiles(dir string) ([]string, error) { - content := []string{} - - err := filepath.Walk(dir, - func(path string, info os.FileInfo, err error) error { - if err != nil { - return nil - } - if !info.IsDir() { - content = append(content, path) - } - - return nil - }) - - return content, err + return result, nil } type Stage string @@ -313,112 +189,3 @@ func MergeYAML(objs ...interface{}) ([]byte, error) { func AddHeader(header, data string) string { return fmt.Sprintf("%s\n%s", header, data) } - -func FindYAMLWithKey(s string, opts ...Option) ([]string, error) { - o := &Options{} - - result := []string{} - if err := o.Apply(opts...); err != nil { - return result, err - } - - files := allFiles(o.ScanDir) - - for _, f := range files { - dat, err := os.ReadFile(f) - if err != nil { - fmt.Printf("warning: skipping file '%s' - %s\n", f, err.Error()) - } - - found, err := unstructured.YAMLHasKey(s, dat) - if err != nil { - fmt.Printf("warning: skipping file '%s' - %s\n", f, err.Error()) - } - - if found { - result = append(result, f) - } - - } - - return result, nil -} - -// parseConfig merges all config back in one structure. -func parseConfig(dir []string, nologs bool) *Config { - files := allFiles(dir) - c := &Config{} - for _, f := range files { - if fileSize(f) > 1.0 { - if !nologs { - fmt.Printf("warning: skipping %s. too big (>1MB)\n", f) - } - continue - } - if strings.Contains(f, "userdata") || filepath.Ext(f) == ".yml" || filepath.Ext(f) == ".yaml" { - b, err := os.ReadFile(f) - if err != nil { - if !nologs { - fmt.Printf("warning: skipping %s. %s\n", f, err.Error()) - } - continue - } - - err = yaml.Unmarshal(b, c) - if err != nil && !nologs { - fmt.Printf("warning: failed to merge config:\n%s\n", err.Error()) - } - - var newYaml map[string]interface{} - yaml.Unmarshal(b, &newYaml) //nolint:errcheck - if err := mergo.Merge(&c.originalData, newYaml); err != nil { - if !nologs { - fmt.Printf("warning: failed to merge config %s to originalData: %s\n", f, err.Error()) - } - } - if exists, header := HasHeader(string(b), ""); exists { - c.header = header - } - } else { - if !nologs { - fmt.Printf("warning: skipping %s (extension).\n", f) - } - } - } - - return c -} - -func (c *Config) fetchRemoteConfig() error { - var body []byte - - err := retry.Do( - func() error { - resp, err := http.Get(c.ConfigURL) - if err != nil { - return err - } - defer resp.Body.Close() - - body, err = io.ReadAll(resp.Body) - if err != nil { - return err - } - - return nil - }, - ) - - if err != nil { - return fmt.Errorf("could not merge configs: %w", err) - } - - yaml.Unmarshal(body, c) //nolint:errcheck - yaml.Unmarshal(body, &c.originalData) //nolint:errcheck - - if exists, header := HasHeader(string(body), ""); exists { - c.header = header - } - - return nil -} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 9c7d7b6..afaab78 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -17,12 +17,10 @@ package config_test import ( "os" - "path/filepath" - . "github.com/kairos-io/kairos/pkg/config" + // . "github.com/kairos-io/kairos/pkg/config" . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "gopkg.in/yaml.v3" + // . "github.com/onsi/gomega" ) type TConfig struct { @@ -43,208 +41,4 @@ var _ = Describe("Config", func() { os.RemoveAll(d) } }) - - Context("directory", func() { - headerCheck := func(c *Config) { - ok, header := HasHeader(c.String(), DefaultHeader) - ExpectWithOffset(1, ok).To(BeTrue()) - ExpectWithOffset(1, header).To(Equal(DefaultHeader)) - } - - It("reads from bootargs and can query", func() { - err := os.WriteFile(filepath.Join(d, "b"), []byte(`zz.foo="baa" options.foo=bar`), os.ModePerm) - Expect(err).ToNot(HaveOccurred()) - - c, err := Scan(MergeBootLine, WithBootCMDLineFile(filepath.Join(d, "b")), NoLogs, StrictValidation(false)) - Expect(err).ToNot(HaveOccurred()) - headerCheck(c) - Expect(c.Options["foo"]).To(Equal("bar")) - Expect(c.Query("options")).To(Equal("foo: bar\n")) - Expect(c.Query("options.foo")).To(Equal("bar\n")) - }) - - It("reads multiple config files", func() { - var cc string = `#kairos-config -baz: bar -kairos: - network_token: foo -` - var c2 string = ` -b: f -c: d -` - - err := os.WriteFile(filepath.Join(d, "test.yaml"), []byte(cc), os.ModePerm) - Expect(err).ToNot(HaveOccurred()) - - err = os.WriteFile(filepath.Join(d, "b.yaml"), []byte(c2), os.ModePerm) - Expect(err).ToNot(HaveOccurred()) - - c, err := Scan(Directories(d), NoLogs, StrictValidation(false)) - Expect(err).ToNot(HaveOccurred()) - Expect(c).ToNot(BeNil()) - providerCfg := &TConfig{} - err = c.Unmarshal(providerCfg) - Expect(err).ToNot(HaveOccurred()) - Expect(providerCfg.Kairos).ToNot(BeNil()) - Expect(providerCfg.Kairos.NetworkToken).To(Equal("foo")) - all := map[string]string{} - yaml.Unmarshal([]byte(c.String()), &all) - Expect(all["b"]).To(Equal("f")) - Expect(all["baz"]).To(Equal("bar")) - }) - - It("reads config file greedly", func() { - - var cc = `#kairos-config -baz: bar -kairos: - network_token: foo -` - - err := os.WriteFile(filepath.Join(d, "test.yaml"), []byte(cc), os.ModePerm) - Expect(err).ToNot(HaveOccurred()) - err = os.WriteFile(filepath.Join(d, "b.yaml"), []byte(` -fooz: - `), os.ModePerm) - Expect(err).ToNot(HaveOccurred()) - - err = os.WriteFile(filepath.Join(d, "more-kairos.yaml"), []byte(` -kairos: - other_key: test -`), os.ModePerm) - Expect(err).ToNot(HaveOccurred()) - - c, err := Scan(Directories(d), NoLogs, StrictValidation(false)) - Expect(err).ToNot(HaveOccurred()) - Expect(c).ToNot(BeNil()) - providerCfg := &TConfig{} - err = c.Unmarshal(providerCfg) - Expect(err).ToNot(HaveOccurred()) - Expect(providerCfg.Kairos).ToNot(BeNil()) - Expect(providerCfg.Kairos.NetworkToken).To(Equal("foo")) - Expect(providerCfg.Kairos.OtherKey).To(Equal("test")) - expectedString := `#kairos-config -baz: bar -kairos: - network_token: foo - other_key: test -` - Expect(c.String()).To(Equal(expectedString), c.String(), cc) - }) - - It("merges with bootargs", func() { - - var cc string = `#kairos-config -kairos: - network_token: "foo" - -bb: - nothing: "foo" -` - - err := os.WriteFile(filepath.Join(d, "test.yaml"), []byte(cc), os.ModePerm) - Expect(err).ToNot(HaveOccurred()) - err = os.WriteFile(filepath.Join(d, "b"), []byte(`zz.foo="baa" options.foo=bar`), os.ModePerm) - Expect(err).ToNot(HaveOccurred()) - - c, err := Scan(Directories(d), MergeBootLine, WithBootCMDLineFile(filepath.Join(d, "b")), NoLogs, StrictValidation(false)) - Expect(err).ToNot(HaveOccurred()) - Expect(c.Options["foo"]).To(Equal("bar")) - - providerCfg := &TConfig{} - err = c.Unmarshal(providerCfg) - Expect(err).ToNot(HaveOccurred()) - Expect(providerCfg.Kairos).ToNot(BeNil()) - Expect(providerCfg.Kairos.NetworkToken).To(Equal("foo")) - _, exists := c.Data()["zz"] - Expect(exists).To(BeFalse()) - }) - - It("reads config file from url", func() { - - var cc string = ` -config_url: "https://gist.githubusercontent.com/mudler/ab26e8dd65c69c32ab292685741ca09c/raw/bafae390eae4e6382fb1b68293568696823b3103/test.yaml" -` - - err := os.WriteFile(filepath.Join(d, "test.yaml"), []byte(cc), os.ModePerm) - Expect(err).ToNot(HaveOccurred()) - - c, err := Scan(Directories(d), NoLogs, StrictValidation(false)) - Expect(err).ToNot(HaveOccurred()) - Expect(c).ToNot(BeNil()) - Expect(len(c.Bundles)).To(Equal(1)) - Expect(c.Bundles[0].Targets[0]).To(Equal("package:utils/edgevpn")) - Expect(c.String()).ToNot(Equal(cc)) - }) - - It("keeps header", func() { - - var cc string = ` -config_url: "https://gist.githubusercontent.com/mudler/7e3d0426fce8bfaaeb2644f83a9bfe0c/raw/77ded58aab3ee2a8d4117db95e078f81fd08dfde/testgist.yaml" -` - - err := os.WriteFile(filepath.Join(d, "test.yaml"), []byte(cc), os.ModePerm) - Expect(err).ToNot(HaveOccurred()) - - c, err := Scan(Directories(d), NoLogs, StrictValidation(false)) - Expect(err).ToNot(HaveOccurred()) - Expect(c).ToNot(BeNil()) - Expect(len(c.Bundles)).To(Equal(1)) - Expect(c.Bundles[0].Targets[0]).To(Equal("package:utils/edgevpn")) - Expect(c.String()).ToNot(Equal(cc)) - - headerCheck(c) - }) - }) - - Describe("FindYAMLWithKey", func() { - var c1Path, c2Path string - - BeforeEach(func() { - var c1 = ` -a: 1 -b: - c: foo -d: - e: bar -` - - var c2 = ` -b: - c: foo2 -` - c1Path = filepath.Join(d, "c1.yaml") - c2Path = filepath.Join(d, "c2.yaml") - - err := os.WriteFile(c1Path, []byte(c1), os.ModePerm) - Expect(err).ToNot(HaveOccurred()) - err = os.WriteFile(c2Path, []byte(c2), os.ModePerm) - Expect(err).ToNot(HaveOccurred()) - }) - - It("can find a top level key", func() { - r, err := FindYAMLWithKey("a", Directories(d)) - Expect(err).ToNot(HaveOccurred()) - Expect(r).To(Equal([]string{c1Path})) - }) - - It("can find a nested key", func() { - r, err := FindYAMLWithKey("d.e", Directories(d)) - Expect(err).ToNot(HaveOccurred()) - Expect(r).To(Equal([]string{c1Path})) - }) - - It("returns multiple files when key exists in them", func() { - r, err := FindYAMLWithKey("b.c", Directories(d)) - Expect(err).ToNot(HaveOccurred()) - Expect(r).To(ContainElements(c1Path, c2Path)) - }) - - It("return an empty list when key is not found", func() { - r, err := FindYAMLWithKey("does.not.exist", Directories(d)) - Expect(err).ToNot(HaveOccurred()) - Expect(r).To(BeEmpty()) - }) - }) })