From 1073b07f6a20a236857a42e4358c3e809a30ba33 Mon Sep 17 00:00:00 2001 From: Mauro Morales Date: Wed, 29 Mar 2023 16:25:38 +0200 Subject: [PATCH] sparkles: Custom partitioning refactor config (#1180) * Introduce config/collector package to split the collection of config sources out of the config package. Each consumer of the new package will take care of unmarshalling the yaml to a specific Config struct, do validations etc. * Add tests and remove garbage * Follow all config_url chains and test it * Add missing options file and refactor cmdline code * Consolidate the way we merge configs no matter where they come from * Allow and use only files with valid headers Config is specific to Kairos while Collector is generic. This will allow us to do validations which are just related to Kairos at the config level, while including every type of key and querying of the full yaml at the Collector level splitting the responsibilities of each package. --------- Signed-off-by: Mauro Morales Signed-off-by: Dimitris Karakasilis --- internal/agent/agent.go | 12 ++++++++---- internal/agent/agent_test.go | 3 +-- internal/agent/install.go | 33 +++++++++++++++++++++++---------- internal/agent/notify.go | 9 +++++++-- internal/agent/reset.go | 3 ++- internal/agent/upgrade.go | 3 ++- 6 files changed, 43 insertions(+), 20 deletions(-) diff --git a/internal/agent/agent.go b/internal/agent/agent.go index fd2e1d3..32f10a5 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -11,6 +11,7 @@ import ( hook "github.com/kairos-io/kairos/internal/agent/hooks" "github.com/kairos-io/kairos/internal/bus" config "github.com/kairos-io/kairos/pkg/config" + "github.com/kairos-io/kairos/pkg/config/collector" "github.com/nxadm/tail" ) @@ -24,7 +25,7 @@ func Run(opts ...Option) error { os.MkdirAll("/usr/local/.kairos", 0600) //nolint:errcheck // Reads config - c, err := config.Scan(config.Directories(o.Dir...)) + c, err := config.Scan(collector.Directories(o.Dir...)) if err != nil { return err } @@ -75,13 +76,16 @@ func Run(opts ...Option) error { } // Re-read config files - c, err = config.Scan(config.Directories(o.Dir...)) + c, err = config.Scan(collector.Directories(o.Dir...)) if err != nil { return err } } - - _, err = bus.Manager.Publish(events.EventBootstrap, events.BootstrapPayload{APIAddress: o.APIAddress, Config: c.String(), Logfile: fileName}) + configStr, err := c.Config.String() + if err != nil { + panic(err) + } + _, err = bus.Manager.Publish(events.EventBootstrap, events.BootstrapPayload{APIAddress: o.APIAddress, Config: configStr, Logfile: fileName}) if o.Restart && err != nil { fmt.Println("Warning: Agent failed, restarting: ", err.Error()) diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index b04a04e..9ee6815 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -28,12 +28,11 @@ var _ = Describe("Bootstrap provider", func() { defer os.RemoveAll(f) wd, _ := os.Getwd() - fmt.Println(wd) os.WriteFile(filepath.Join(wd, "agent-provider-test"), []byte(testProvider), 0655) defer os.RemoveAll(filepath.Join(wd, "agent-provider-test")) - err = os.WriteFile(filepath.Join(f, "test.config.yaml"), []byte(` + err = os.WriteFile(filepath.Join(f, "test.config.yaml"), []byte(`#cloud-config doo: bar`), 0655) Expect(err).ToNot(HaveOccurred()) diff --git a/internal/agent/install.go b/internal/agent/install.go index 92adf93..9cac6e6 100644 --- a/internal/agent/install.go +++ b/internal/agent/install.go @@ -19,6 +19,7 @@ import ( "github.com/kairos-io/kairos/internal/bus" "github.com/kairos-io/kairos/internal/cmd" config "github.com/kairos-io/kairos/pkg/config" + "github.com/kairos-io/kairos/pkg/config/collector" qr "github.com/mudler/go-nodepair/qrcode" "github.com/mudler/go-pluggable" "github.com/pterm/pterm" @@ -79,15 +80,19 @@ func ManualInstall(c string, options map[string]string, strictValidations bool) return err } - cc, err := config.Scan(config.Directories(source), config.MergeBootLine, config.StrictValidation(strictValidations)) + cc, err := config.Scan(collector.Directories(source), collector.MergeBootLine, collector.StrictValidation(strictValidations)) if err != nil { return err } - options["cc"] = cc.String() + configStr, err := cc.String() + if err != nil { + return err + } + options["cc"] = configStr // unlike Install device is already set // options["device"] = cc.Install.Device - mergeOption(cc.String(), options) + mergeOption(configStr, options) if options["device"] == "" { options["device"] = cc.Install.Device @@ -122,11 +127,15 @@ func Install(dir ...string) error { // Reads config, and if present and offline is defined, // runs the installation - cc, err := config.Scan(config.Directories(dir...), config.MergeBootLine, config.NoLogs) + cc, err := config.Scan(collector.Directories(dir...), collector.MergeBootLine, collector.NoLogs) if err == nil && cc.Install != nil && cc.Install.Auto { - r["cc"] = cc.String() + configStr, err := cc.String() + if err != nil { + return err + } + r["cc"] = configStr r["device"] = cc.Install.Device - mergeOption(cc.String(), r) + mergeOption(configStr, r) err = RunInstall(r) if err != nil { @@ -160,7 +169,11 @@ func Install(dir ...string) error { return utils.Shell().Run() } - _, err = bus.Manager.Publish(events.EventChallenge, events.EventPayload{Config: cc.String()}) + configStr, err := cc.String() + if err != nil { + return err + } + _, err = bus.Manager.Publish(events.EventChallenge, events.EventPayload{Config: configStr}) if err != nil { return err } @@ -175,7 +188,7 @@ func Install(dir ...string) error { qr.Print(tk) } - if _, err := bus.Manager.Publish(events.EventInstall, events.InstallPayload{Token: tk, Config: cc.String()}); err != nil { + if _, err := bus.Manager.Publish(events.EventInstall, events.InstallPayload{Token: tk, Config: configStr}); err != nil { return err } @@ -196,12 +209,12 @@ func Install(dir ...string) error { // make sure the config we write has at least the #cloud-config header, // if any other was defined beforeahead header := "#cloud-config" - if hasHeader, head := config.HasHeader(cc.String(), ""); hasHeader { + if hasHeader, head := config.HasHeader(configStr, ""); hasHeader { header = head } // What we receive take precedence over the one in the system. best-effort - yaml.Unmarshal([]byte(cc.String()), &ccData) //nolint:errcheck + yaml.Unmarshal([]byte(configStr), &ccData) //nolint:errcheck if exists { yaml.Unmarshal([]byte(cloudConfig), &ccData) //nolint:errcheck if hasHeader, head := config.HasHeader(cloudConfig, ""); hasHeader { diff --git a/internal/agent/notify.go b/internal/agent/notify.go index f5acf54..ce91705 100644 --- a/internal/agent/notify.go +++ b/internal/agent/notify.go @@ -6,13 +6,14 @@ import ( events "github.com/kairos-io/kairos-sdk/bus" "github.com/kairos-io/kairos/internal/bus" "github.com/kairos-io/kairos/pkg/config" + "github.com/kairos-io/kairos/pkg/config/collector" "github.com/mudler/go-pluggable" ) func Notify(event string, dirs []string) error { bus.Manager.Initialize() - c, err := config.Scan(config.Directories(dirs...)) + c, err := config.Scan(collector.Directories(dirs...)) if err != nil { return err } @@ -21,8 +22,12 @@ func Notify(event string, dirs []string) error { return fmt.Errorf("event '%s' not defined", event) } + configStr, err := c.String() + if err != nil { + return err + } _, err = bus.Manager.Publish(pluggable.EventType(event), events.EventPayload{ - Config: c.String(), + Config: configStr, }) return err diff --git a/internal/agent/reset.go b/internal/agent/reset.go index a3278ac..440d106 100644 --- a/internal/agent/reset.go +++ b/internal/agent/reset.go @@ -15,6 +15,7 @@ import ( "github.com/kairos-io/kairos/internal/bus" "github.com/kairos-io/kairos/internal/cmd" "github.com/kairos-io/kairos/pkg/config" + "github.com/kairos-io/kairos/pkg/config/collector" "github.com/mudler/go-pluggable" "github.com/pterm/pterm" @@ -72,7 +73,7 @@ func Reset(dir ...string) error { args = append(args, "--reset-persistent") } - c, err := config.Scan(config.Directories(dir...)) + c, err := config.Scan(collector.Directories(dir...)) if err != nil { return err } diff --git a/internal/agent/upgrade.go b/internal/agent/upgrade.go index 21ff40f..8f31c1e 100644 --- a/internal/agent/upgrade.go +++ b/internal/agent/upgrade.go @@ -10,6 +10,7 @@ import ( events "github.com/kairos-io/kairos-sdk/bus" "github.com/kairos-io/kairos/pkg/config" + "github.com/kairos-io/kairos/pkg/config/collector" "github.com/kairos-io/kairos-sdk/utils" "github.com/kairos-io/kairos/internal/bus" @@ -99,7 +100,7 @@ func Upgrade( fmt.Printf("Upgrading to image: '%s'\n", img) } - c, err := config.Scan(config.Directories(dirs...), config.StrictValidation(strictValidations)) + c, err := config.Scan(collector.Directories(dirs...), collector.StrictValidation(strictValidations)) if err != nil { return err }