sparkles: Custom partitioning refactor config (#1180)

* Introduce config/collector package

to split the collection of config sources out of the config package.

Each consumer of the new package will take care of unmarshalling the
yaml to a specific Config struct, do validations etc.

* Add tests and remove garbage
* Follow all config_url chains and test it
* Add missing options file and refactor cmdline code
* Consolidate the way we merge configs no matter where they come from
* Allow and use only files with valid headers

Config is  specific to Kairos while Collector is generic. This
will allow us to do validations which are just related to Kairos at the
config level, while including every type of key and querying of the full
yaml at the Collector level splitting the responsibilities of each
package.

---------

Signed-off-by: Mauro Morales <mauro.morales@spectrocloud.com>
Signed-off-by: Dimitris Karakasilis <dimitris@karakasilis.me>

internal/agent/agent.go

@@ -11,6 +11,7 @@ import (
 	hook "github.com/kairos-io/kairos/internal/agent/hooks"
 	"github.com/kairos-io/kairos/internal/bus"
 	config "github.com/kairos-io/kairos/pkg/config"
+	"github.com/kairos-io/kairos/pkg/config/collector"
 	"github.com/nxadm/tail"
 )
 
@@ -24,7 +25,7 @@ func Run(opts ...Option) error {
 	os.MkdirAll("/usr/local/.kairos", 0600) //nolint:errcheck
 
 	// Reads config
-	c, err := config.Scan(config.Directories(o.Dir...))
+	c, err := config.Scan(collector.Directories(o.Dir...))
 	if err != nil {
 		return err
 	}
@@ -75,13 +76,16 @@ func Run(opts ...Option) error {
 		}
 
 		// Re-read config files
-		c, err = config.Scan(config.Directories(o.Dir...))
+		c, err = config.Scan(collector.Directories(o.Dir...))
 		if err != nil {
 			return err
 		}
 	}
-
+	configStr, err := c.Config.String()
-	_, err = bus.Manager.Publish(events.EventBootstrap, events.BootstrapPayload{APIAddress: o.APIAddress, Config: c.String(), Logfile: fileName})
+	if err != nil {
+		panic(err)
+	}
+	_, err = bus.Manager.Publish(events.EventBootstrap, events.BootstrapPayload{APIAddress: o.APIAddress, Config: configStr, Logfile: fileName})
 
 	if o.Restart && err != nil {
 		fmt.Println("Warning: Agent failed, restarting: ", err.Error())

internal/agent/agent_test.go

@@ -28,12 +28,11 @@ var _ = Describe("Bootstrap provider", func() {
 			defer os.RemoveAll(f)
 
 			wd, _ := os.Getwd()
-			fmt.Println(wd)
 			os.WriteFile(filepath.Join(wd, "agent-provider-test"), []byte(testProvider), 0655)
 
 			defer os.RemoveAll(filepath.Join(wd, "agent-provider-test"))
 
-			err = os.WriteFile(filepath.Join(f, "test.config.yaml"), []byte(`
+			err = os.WriteFile(filepath.Join(f, "test.config.yaml"), []byte(`#cloud-config
 doo: bar`), 0655)
 			Expect(err).ToNot(HaveOccurred())
 

internal/agent/install.go

@@ -19,6 +19,7 @@ import (
 	"github.com/kairos-io/kairos/internal/bus"
 	"github.com/kairos-io/kairos/internal/cmd"
 	config "github.com/kairos-io/kairos/pkg/config"
+	"github.com/kairos-io/kairos/pkg/config/collector"
 	qr "github.com/mudler/go-nodepair/qrcode"
 	"github.com/mudler/go-pluggable"
 	"github.com/pterm/pterm"
@@ -79,15 +80,19 @@ func ManualInstall(c string, options map[string]string, strictValidations bool)
 		return err
 	}
 
-	cc, err := config.Scan(config.Directories(source), config.MergeBootLine, config.StrictValidation(strictValidations))
+	cc, err := config.Scan(collector.Directories(source), collector.MergeBootLine, collector.StrictValidation(strictValidations))
 	if err != nil {
 		return err
 	}
-	options["cc"] = cc.String()
+	configStr, err := cc.String()
+	if err != nil {
+		return err
+	}
+	options["cc"] = configStr
 	// unlike Install device is already set
 	// options["device"] = cc.Install.Device
 
-	mergeOption(cc.String(), options)
+	mergeOption(configStr, options)
 
 	if options["device"] == "" {
 		options["device"] = cc.Install.Device
@@ -122,11 +127,15 @@ func Install(dir ...string) error {
 
 	// Reads config, and if present and offline is defined,
 	// runs the installation
-	cc, err := config.Scan(config.Directories(dir...), config.MergeBootLine, config.NoLogs)
+	cc, err := config.Scan(collector.Directories(dir...), collector.MergeBootLine, collector.NoLogs)
 	if err == nil && cc.Install != nil && cc.Install.Auto {
-		r["cc"] = cc.String()
+		configStr, err := cc.String()
+		if err != nil {
+			return err
+		}
+		r["cc"] = configStr
 		r["device"] = cc.Install.Device
-		mergeOption(cc.String(), r)
+		mergeOption(configStr, r)
 
 		err = RunInstall(r)
 		if err != nil {
@@ -160,7 +169,11 @@ func Install(dir ...string) error {
 		return utils.Shell().Run()
 	}
 
-	_, err = bus.Manager.Publish(events.EventChallenge, events.EventPayload{Config: cc.String()})
+	configStr, err := cc.String()
+	if err != nil {
+		return err
+	}
+	_, err = bus.Manager.Publish(events.EventChallenge, events.EventPayload{Config: configStr})
 	if err != nil {
 		return err
 	}
@@ -175,7 +188,7 @@ func Install(dir ...string) error {
 		qr.Print(tk)
 	}
 
-	if _, err := bus.Manager.Publish(events.EventInstall, events.InstallPayload{Token: tk, Config: cc.String()}); err != nil {
+	if _, err := bus.Manager.Publish(events.EventInstall, events.InstallPayload{Token: tk, Config: configStr}); err != nil {
 		return err
 	}
 
@@ -196,12 +209,12 @@ func Install(dir ...string) error {
 	// make sure the config we write has at least the #cloud-config header,
 	// if any other was defined beforeahead
 	header := "#cloud-config"
-	if hasHeader, head := config.HasHeader(cc.String(), ""); hasHeader {
+	if hasHeader, head := config.HasHeader(configStr, ""); hasHeader {
 		header = head
 	}
 
 	// What we receive take precedence over the one in the system. best-effort
-	yaml.Unmarshal([]byte(cc.String()), &ccData) //nolint:errcheck
+	yaml.Unmarshal([]byte(configStr), &ccData) //nolint:errcheck
 	if exists {
 		yaml.Unmarshal([]byte(cloudConfig), &ccData) //nolint:errcheck
 		if hasHeader, head := config.HasHeader(cloudConfig, ""); hasHeader {

internal/agent/notify.go

@@ -6,13 +6,14 @@ import (
 	events "github.com/kairos-io/kairos-sdk/bus"
 	"github.com/kairos-io/kairos/internal/bus"
 	"github.com/kairos-io/kairos/pkg/config"
+	"github.com/kairos-io/kairos/pkg/config/collector"
 	"github.com/mudler/go-pluggable"
 )
 
 func Notify(event string, dirs []string) error {
 	bus.Manager.Initialize()
 
-	c, err := config.Scan(config.Directories(dirs...))
+	c, err := config.Scan(collector.Directories(dirs...))
 	if err != nil {
 		return err
 	}
@@ -21,8 +22,12 @@ func Notify(event string, dirs []string) error {
 		return fmt.Errorf("event '%s' not defined", event)
 	}
 
+	configStr, err := c.String()
+	if err != nil {
+		return err
+	}
 	_, err = bus.Manager.Publish(pluggable.EventType(event), events.EventPayload{
-		Config: c.String(),
+		Config: configStr,
 	})
 
 	return err

internal/agent/reset.go

@@ -15,6 +15,7 @@ import (
 	"github.com/kairos-io/kairos/internal/bus"
 	"github.com/kairos-io/kairos/internal/cmd"
 	"github.com/kairos-io/kairos/pkg/config"
+	"github.com/kairos-io/kairos/pkg/config/collector"
 
 	"github.com/mudler/go-pluggable"
 	"github.com/pterm/pterm"
@@ -72,7 +73,7 @@ func Reset(dir ...string) error {
 		args = append(args, "--reset-persistent")
 	}
 
-	c, err := config.Scan(config.Directories(dir...))
+	c, err := config.Scan(collector.Directories(dir...))
 	if err != nil {
 		return err
 	}

internal/agent/upgrade.go

@@ -10,6 +10,7 @@ import (
 
 	events "github.com/kairos-io/kairos-sdk/bus"
 	"github.com/kairos-io/kairos/pkg/config"
+	"github.com/kairos-io/kairos/pkg/config/collector"
 
 	"github.com/kairos-io/kairos-sdk/utils"
 	"github.com/kairos-io/kairos/internal/bus"
@@ -99,7 +100,7 @@ func Upgrade(
 		fmt.Printf("Upgrading to image: '%s'\n", img)
 	}
 
-	c, err := config.Scan(config.Directories(dirs...), config.StrictValidation(strictValidations))
+	c, err := config.Scan(collector.Directories(dirs...), collector.StrictValidation(strictValidations))
 	if err != nil {
 		return err
 	}