Merge pull request #922 from joshwget/simplify-configuration

Simplify configuration

cmd/control/config.go

@@ -28,17 +28,6 @@ func configSubcommands() []cli.Command {
 			Usage:  "set a value",
 			Action: configSet,
 		},
-		{
-			Name:   "import",
-			Usage:  "import configuration from standard in or a file",
-			Action: runImport,
-			Flags: []cli.Flag{
-				cli.StringFlag{
-					Name:  "input, i",
-					Usage: "File from which to read",
-				},
-			},
-		},
 		{
 			Name:   "images",
 			Usage:  "List Docker images for a configuration from a file",
@@ -142,42 +131,6 @@ func env2map(env []string) map[string]string {
 	return m
 }
 
-func runImport(c *cli.Context) error {
-	var input io.ReadCloser
-	var err error
-	input = os.Stdin
-	cfg, err := config.LoadConfig()
-
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	inputFile := c.String("input")
-	if inputFile != "" {
-		input, err = os.Open(inputFile)
-		if err != nil {
-			log.Fatal(err)
-		}
-		defer input.Close()
-	}
-
-	bytes, err := ioutil.ReadAll(input)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	cfg, err = cfg.Import(bytes)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	if err := cfg.Save(); err != nil {
-		log.Fatal(err)
-	}
-
-	return nil
-}
-
 func configSet(c *cli.Context) error {
 	key := c.Args().Get(0)
 	value := c.Args().Get(1)
@@ -185,20 +138,11 @@ func configSet(c *cli.Context) error {
 		return nil
 	}
 
-	cfg, err := config.LoadConfig()
+	err := config.Set(key, value)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	cfgDiff, err := cfg.Set(key, value)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	if err := cfg.Save(cfgDiff); err != nil {
-		log.Fatal(err)
-	}
-
 	return nil
 }
 
@@ -208,14 +152,9 @@ func configGet(c *cli.Context) error {
 		return nil
 	}
 
-	cfg, err := config.LoadConfig()
+	val, err := config.Get(arg)
 	if err != nil {
-		log.WithFields(log.Fields{"err": err}).Fatal("config get: failed to load config")
-	}
-
-	val, err := cfg.GetIgnoreOmitEmpty(arg)
-	if err != nil {
-		log.WithFields(log.Fields{"cfg": cfg, "key": arg, "val": val, "err": err}).Fatal("config get: failed to retrieve value")
+		log.WithFields(log.Fields{"key": arg, "val": val, "err": err}).Fatal("config get: failed to retrieve value")
 	}
 
 	printYaml := false
@@ -245,25 +184,16 @@ func merge(c *cli.Context) error {
 		log.Fatal(err)
 	}
 
-	cfg, err := config.LoadConfig()
+	err = config.Merge(bytes)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	cfg, err = cfg.MergeBytes(bytes)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	if err := cfg.Save(); err != nil {
-		log.Fatal(err)
-	}
-
 	return nil
 }
 
 func export(c *cli.Context) error {
-	content, err := config.Dump(c.Bool("private"), c.Bool("full"))
+	content, err := config.Export(c.Bool("private"), c.Bool("full"))
 	if err != nil {
 		log.Fatal(err)
 	}

cmd/control/os.go

@@ -205,13 +205,7 @@ func startUpgradeContainer(image string, stage, force, reboot, kexec bool, upgra
 	}
 
 	if upgradeConsole {
-		cfg, err := config.LoadConfig()
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		cfg.Rancher.ForceConsoleRebuild = true
-		if err := cfg.Save(); err != nil {
+		if err := config.Set("rancher.force_console_rebuild", true); err != nil {
 			log.Fatal(err)
 		}
 	}

cmd/control/service.go

@@ -86,6 +86,10 @@ func serviceSubCommands() []cli.Command {
 	}
 }
 
+func updateIncludedServices(cfg *config.CloudConfig) error {
+	return config.Set("rancher.services_include", cfg.Rancher.ServicesInclude)
+}
+
 func disable(c *cli.Context) error {
 	changed := false
 	cfg, err := config.LoadConfig()
@@ -103,7 +107,7 @@ func disable(c *cli.Context) error {
 	}
 
 	if changed {
-		if err = cfg.Save(); err != nil {
+		if err = updateIncludedServices(cfg); err != nil {
 			logrus.Fatal(err)
 		}
 	}
@@ -127,7 +131,7 @@ func del(c *cli.Context) error {
 	}
 
 	if changed {
-		if err = cfg.Save(); err != nil {
+		if err = updateIncludedServices(cfg); err != nil {
 			logrus.Fatal(err)
 		}
 	}
@@ -159,7 +163,7 @@ func enable(c *cli.Context) error {
 			logrus.Fatal(err)
 		}
 
-		if err := cfg.Save(); err != nil {
+		if err = updateIncludedServices(cfg); err != nil {
 			logrus.Fatal(err)
 		}
 	}

cmd/control/tlsconf.go

@@ -65,19 +65,18 @@ func writeCerts(generateServer bool, hostname []string, cfg *config.CloudConfig,
 			return err
 		}
 
-		cfg, err = cfg.Merge(map[interface{}]interface{}{
-			"rancher": map[interface{}]interface{}{
-				"docker": map[interface{}]interface{}{
-					"server_cert": string(cert),
-					"server_key":  string(key),
-				},
-			},
-		})
-		if err != nil {
+		// certPath, keyPath are already written to by machineUtil.GenerateCert()
+		if err := config.Set("rancher.docker.server_cert", string(cert)); err != nil {
 			return err
 		}
+		if err := config.Set("rancher.docker.server_key", string(key)); err != nil {
+			return err
+		}
+	}
 
-		return cfg.Save() // certPath, keyPath are already written to by machineUtil.GenerateCert()
+	cfg, err := config.LoadConfig()
+	if err != nil {
+		return err
 	}
 
 	if err := ioutil.WriteFile(certPath, []byte(cfg.Rancher.Docker.ServerCert), 0400); err != nil {
@@ -88,50 +87,45 @@ func writeCerts(generateServer bool, hostname []string, cfg *config.CloudConfig,
 
 }
 
-func writeCaCerts(cfg *config.CloudConfig, caCertPath, caKeyPath string) (*config.CloudConfig, error) {
+func writeCaCerts(cfg *config.CloudConfig, caCertPath, caKeyPath string) error {
 	if cfg.Rancher.Docker.CACert == "" {
 		if err := machineUtil.GenerateCACertificate(caCertPath, caKeyPath, NAME, BITS); err != nil {
-			return nil, err
+			return err
 		}
 
 		caCert, err := ioutil.ReadFile(caCertPath)
 		if err != nil {
-			return nil, err
+			return err
 		}
 
 		caKey, err := ioutil.ReadFile(caKeyPath)
 		if err != nil {
-			return nil, err
+			return err
 		}
 
-		cfg, err = cfg.Merge(map[interface{}]interface{}{
-			"rancher": map[interface{}]interface{}{
-				"docker": map[interface{}]interface{}{
-					"ca_key":  string(caKey),
-					"ca_cert": string(caCert),
-				},
-			},
-		})
+		// caCertPath, caKeyPath are already written to by machineUtil.GenerateCACertificate()
+		if err := config.Set("rancher.docker.ca_cert", string(caCert)); err != nil {
+			return err
+		}
+		if err := config.Set("rancher.docker.ca_key", string(caKey)); err != nil {
+			return err
+		}
+	}
+
+	cfg, err := config.LoadConfig()
 	if err != nil {
-			return nil, err
-		}
-
-		if err = cfg.Save(); err != nil {
-			return nil, err
-		}
-
-		return cfg, nil // caCertPath, caKeyPath are already written to by machineUtil.GenerateCACertificate()
+		return err
 	}
 
 	if err := ioutil.WriteFile(caCertPath, []byte(cfg.Rancher.Docker.CACert), 0400); err != nil {
-		return nil, err
+		return err
 	}
 
 	if err := ioutil.WriteFile(caKeyPath, []byte(cfg.Rancher.Docker.CAKey), 0400); err != nil {
-		return nil, err
+		return err
 	}
 
-	return cfg, nil
+	return nil
 }
 
 func tlsConfCreate(c *cli.Context) error {
@@ -152,11 +146,6 @@ func generate(c *cli.Context) error {
 }
 
 func Generate(generateServer bool, outDir string, hostnames []string) error {
-	cfg, err := config.LoadConfig()
-	if err != nil {
-		return err
-	}
-
 	if outDir == "" {
 		if generateServer {
 			outDir = "/etc/docker/tls"
@@ -181,7 +170,11 @@ func Generate(generateServer bool, outDir string, hostnames []string) error {
 		}
 	}
 
-	cfg, err = writeCaCerts(cfg, caCertPath, caKeyPath)
+	cfg, err := config.LoadConfig()
+	if err != nil {
+		return err
+	}
+	err = writeCaCerts(cfg, caCertPath, caKeyPath)
 	if err != nil {
 		return err
 	}

config/config.go

@@ -3,28 +3,10 @@ package config
 import (
 	"fmt"
 
-	log "github.com/Sirupsen/logrus"
 	yaml "github.com/cloudfoundry-incubator/candiedyaml"
 	"github.com/rancher/os/util"
 )
 
-func (c *CloudConfig) Import(bytes []byte) (*CloudConfig, error) {
-	data, err := readConfig(bytes, false, CloudConfigPrivateFile)
-	if err != nil {
-		return c, err
-	}
-
-	return NewConfig().Merge(data)
-}
-
-func (c *CloudConfig) MergeBytes(bytes []byte) (*CloudConfig, error) {
-	data, err := readConfig(bytes, false)
-	if err != nil {
-		return c, err
-	}
-	return c.Merge(data)
-}
-
 var keysToStringify = []string{
 	"command",
 	"dns",
@@ -89,47 +71,36 @@ func StringifyValues(data map[interface{}]interface{}) map[interface{}]interface
 	return stringifyValue(data, nil).(map[interface{}]interface{})
 }
 
-func (c *CloudConfig) Merge(values map[interface{}]interface{}) (*CloudConfig, error) {
-	d := map[interface{}]interface{}{}
-	if err := util.Convert(c, &d); err != nil {
-		return c, err
+func Merge(bytes []byte) error {
+	data, err := readConfig(bytes, false)
+	if err != nil {
+		return err
 	}
-	r := util.MapsUnion(d, StringifyValues(values))
-	t := &CloudConfig{}
-	if err := util.Convert(r, t); err != nil {
-		return c, err
+	existing, err := readConfig(nil, false, CloudConfigFile)
+	if err != nil {
+		return err
 	}
-	return t, nil
+	return WriteToFile(util.Merge(existing, data), CloudConfigFile)
 }
 
-func Dump(private, full bool) (string, error) {
-	var cfg *CloudConfig
-	var err error
-
-	if full {
-		cfg, err = LoadConfig()
-	} else {
-		files := []string{CloudConfigBootFile, CloudConfigPrivateFile, CloudConfigFile}
-		if !private {
-			files = util.FilterStrings(files, func(x string) bool { return x != CloudConfigPrivateFile })
-		}
-		cfg, err = ChainCfgFuncs(nil,
-			func(_ *CloudConfig) (*CloudConfig, error) { return ReadConfig(nil, true, files...) },
-			amendNils,
-		)
+func Export(private, full bool) (string, error) {
+	rawCfg, err := LoadRawConfig(full)
+	if private {
+		rawCfg = filterPrivateKeys(rawCfg)
 	}
 
-	if err != nil {
-		return "", err
-	}
-
-	bytes, err := yaml.Marshal(*cfg)
+	bytes, err := yaml.Marshal(rawCfg)
 	return string(bytes), err
 }
 
-func (c *CloudConfig) Get(key string) (interface{}, error) {
+func Get(key string) (interface{}, error) {
+	cfg, err := LoadConfig()
+	if err != nil {
+		return nil, err
+	}
+
 	data := map[interface{}]interface{}{}
-	if err := util.Convert(c, &data); err != nil {
+	if err := util.ConvertIgnoreOmitEmpty(cfg, &data); err != nil {
 		return nil, err
 	}
 
@@ -137,58 +108,14 @@ func (c *CloudConfig) Get(key string) (interface{}, error) {
 	return v, nil
 }
 
-func (c *CloudConfig) GetIgnoreOmitEmpty(key string) (interface{}, error) {
-	data := map[interface{}]interface{}{}
-	if err := util.ConvertIgnoreOmitEmpty(c, &data); err != nil {
-		return nil, err
-	}
-
-	v, _ := getOrSetVal(key, data, nil)
-	return v, nil
-}
-
-func (c *CloudConfig) Set(key string, value interface{}) (map[interface{}]interface{}, error) {
+func Set(key string, value interface{}) error {
 	data := map[interface{}]interface{}{}
 	_, data = getOrSetVal(key, data, value)
 
-	return data, nil
-}
-
-func (c *CloudConfig) Save(cfgDiffs ...map[interface{}]interface{}) error {
-	files := append([]string{OsConfigFile, OemConfigFile}, CloudConfigDirFiles()...)
-	files = util.FilterStrings(files, func(x string) bool { return x != CloudConfigPrivateFile })
-	exCfg, err := ChainCfgFuncs(nil,
-		func(_ *CloudConfig) (*CloudConfig, error) {
-			return ReadConfig(nil, true, files...)
-		},
-		readCmdline,
-		amendNils)
+	existing, err := readConfig(nil, false, CloudConfigFile)
 	if err != nil {
 		return err
 	}
 
-	exCfg = mergeMetadata(exCfg, readMetadata())
-
-	exData := map[interface{}]interface{}{}
-	if err := util.Convert(exCfg, &exData); err != nil {
-		return err
-	}
-
-	data := map[interface{}]interface{}{}
-	if err := util.Convert(c, &data); err != nil {
-		return err
-	}
-
-	data = util.MapsDifference(data, exData)
-
-	// Apply any additional config diffs
-	for _, diff := range cfgDiffs {
-		data = util.MapsUnion(data, diff)
-	}
-
-	log.WithFields(log.Fields{"diff": data}).Debug("The diff we're about to save")
-	if err := saveToDisk(data); err != nil {
-		return err
-	}
-	return nil
+	return WriteToFile(util.Merge(existing, data), CloudConfigFile)
 }

config/config_test.go

@@ -1,13 +1,11 @@
 package config
 
 import (
-	"fmt"
 	yaml "github.com/cloudfoundry-incubator/candiedyaml"
 	"testing"
 
 	"github.com/rancher/os/util"
 	"github.com/stretchr/testify/require"
-	"strings"
 )
 
 func TestFilterKey(t *testing.T) {
@@ -102,57 +100,6 @@ func TestStringifyValues(t *testing.T) {
 	assert.Equal(expected, StringifyValues(data))
 }
 
-func TestFilterDottedKeys(t *testing.T) {
-	assert := require.New(t)
-
-	data := map[interface{}]interface{}{
-		"ssh_authorized_keys": []string{"pubk1", "pubk2"},
-		"hostname":            "ros-test",
-		"rancher": map[interface{}]interface{}{
-			"ssh": map[interface{}]interface{}{
-				"keys": map[interface{}]interface{}{
-					"dsa":     "dsa-test1",
-					"dsa-pub": "dsa-test2",
-				},
-			},
-			"docker": map[interface{}]interface{}{
-				"ca_key":  "ca_key-test3",
-				"ca_cert": "ca_cert-test4",
-				"args":    []string{"args_test5"},
-			},
-		},
-	}
-	expectedFiltered := map[interface{}]interface{}{
-		"ssh_authorized_keys": []string{"pubk1", "pubk2"},
-		"rancher": map[interface{}]interface{}{
-			"ssh": map[interface{}]interface{}{
-				"keys": map[interface{}]interface{}{
-					"dsa":     "dsa-test1",
-					"dsa-pub": "dsa-test2",
-				},
-			},
-		},
-	}
-	expectedRest := map[interface{}]interface{}{
-		"hostname": "ros-test",
-		"rancher": map[interface{}]interface{}{
-			"docker": map[interface{}]interface{}{
-				"ca_key":  "ca_key-test3",
-				"ca_cert": "ca_cert-test4",
-				"args":    []string{"args_test5"},
-			},
-		},
-	}
-
-	assert.Equal([]string{"rancher", "ssh"}, strings.Split("rancher.ssh", "."))
-	assert.Equal([]string{"ssh_authorized_keys"}, strings.Split("ssh_authorized_keys", "."))
-
-	filtered, rest := filterDottedKeys(data, []string{"ssh_authorized_keys", "rancher.ssh"})
-
-	assert.Equal(expectedFiltered, filtered)
-	assert.Equal(expectedRest, rest)
-}
-
 func TestUnmarshalOrReturnString(t *testing.T) {
 	assert := require.New(t)
 
@@ -358,8 +305,6 @@ func TestUserDocker(t *testing.T) {
 	err = util.Convert(config, &data)
 	assert.Nil(err)
 
-	fmt.Println(data)
-
 	val, ok := data["rancher"].(map[interface{}]interface{})["docker"]
 	assert.True(ok)
 

config/data_funcs.go

@@ -4,8 +4,9 @@ import (
 	log "github.com/Sirupsen/logrus"
 	yaml "github.com/cloudfoundry-incubator/candiedyaml"
 
-	"github.com/rancher/os/util"
 	"strings"
+
+	"github.com/rancher/os/util"
 )
 
 type CfgFunc func(*CloudConfig) (*CloudConfig, error)
@@ -58,17 +59,12 @@ func filterKey(data map[interface{}]interface{}, key []string) (filtered, rest m
 	return
 }
 
-func filterDottedKeys(data map[interface{}]interface{}, keys []string) (filtered, rest map[interface{}]interface{}) {
-	filtered = map[interface{}]interface{}{}
-	rest = util.MapCopy(data)
-
-	for _, key := range keys {
-		f, r := filterKey(data, strings.Split(key, "."))
-		filtered = util.MapsUnion(filtered, f)
-		rest = util.MapsIntersection(rest, r)
+func filterPrivateKeys(data map[interface{}]interface{}) map[interface{}]interface{} {
+	for _, privateKey := range PrivateKeys {
+		_, data = filterKey(data, strings.Split(privateKey, "."))
 	}
 
-	return
+	return data
 }
 
 func getOrSetVal(args string, data map[interface{}]interface{}, value interface{}) (interface{}, map[interface{}]interface{}) {

config/disk.go

@@ -15,14 +15,9 @@ import (
 	"github.com/rancher/os/util"
 )
 
-var osConfig *CloudConfig
-
-func NewConfig() *CloudConfig {
-	if osConfig == nil {
-		osConfig, _ = ReadConfig(nil, true, OsConfigFile, OemConfigFile)
-	}
-	newCfg := *osConfig
-	return &newCfg
+func NewConfig() map[interface{}]interface{} {
+	osConfig, _ := readConfig(nil, true, OsConfigFile, OemConfigFile)
+	return osConfig
 }
 
 func ReadConfig(bytes []byte, substituteMetadataVars bool, files ...string) (*CloudConfig, error) {
@@ -39,42 +34,50 @@ func ReadConfig(bytes []byte, substituteMetadataVars bool, files ...string) (*Cl
 	}
 }
 
-func LoadConfig() (*CloudConfig, error) {
-	cfg, err := ChainCfgFuncs(NewConfig(),
-		readFilesAndMetadata,
-		readCmdline,
-		amendNils,
-		amendContainerNames)
+func LoadRawConfig(full bool) (map[interface{}]interface{}, error) {
+	var base map[interface{}]interface{}
+	if full {
+		base = NewConfig()
+	}
+	user, err := readConfigs()
+	if err != nil {
+		return nil, err
+	}
+	cmdline, err := readCmdline()
+	if err != nil {
+		return nil, err
+	}
+	merged := util.Merge(base, util.Merge(user, cmdline))
+	merged, err = applyDebugFlags(merged)
+	if err != nil {
+		return nil, err
+	}
+	return mergeMetadata(merged, readMetadata()), nil
+}
+
+func LoadConfig() (*CloudConfig, error) {
+	rawCfg, err := LoadRawConfig(true)
 	if err != nil {
-		log.WithFields(log.Fields{"cfg": cfg, "err": err}).Error("Failed to load config")
 		return nil, err
 	}
 
-	log.Debug("Merging cloud-config from meta-data and user-data")
-	cfg = mergeMetadata(cfg, readMetadata())
-
-	if cfg.Rancher.Debug {
-		log.SetLevel(log.DebugLevel)
-		if !util.Contains(cfg.Rancher.Docker.Args, "-D") {
-			cfg.Rancher.Docker.Args = append(cfg.Rancher.Docker.Args, "-D")
+	cfg := &CloudConfig{}
+	if err := util.Convert(rawCfg, cfg); err != nil {
+		return nil, err
 	}
-		if !util.Contains(cfg.Rancher.SystemDocker.Args, "-D") {
-			cfg.Rancher.SystemDocker.Args = append(cfg.Rancher.SystemDocker.Args, "-D")
+	cfg, err = amendNils(cfg)
+	if err != nil {
+		return nil, err
 	}
-	} else {
-		if util.Contains(cfg.Rancher.Docker.Args, "-D") {
-			cfg.Rancher.Docker.Args = util.FilterStrings(cfg.Rancher.Docker.Args, func(x string) bool { return x != "-D" })
+	cfg, err = amendContainerNames(cfg)
+	if err != nil {
+		return nil, err
 	}
-		if util.Contains(cfg.Rancher.SystemDocker.Args, "-D") {
-			cfg.Rancher.SystemDocker.Args = util.FilterStrings(cfg.Rancher.SystemDocker.Args, func(x string) bool { return x != "-D" })
-		}
-	}
-
 	return cfg, nil
 }
 
 func CloudConfigDirFiles() []string {
-	files, err := util.DirLs(CloudConfigDir)
+	files, err := ioutil.ReadDir(CloudConfigDir)
 	if err != nil {
 		if os.IsNotExist(err) {
 			// do nothing
@@ -85,36 +88,56 @@ func CloudConfigDirFiles() []string {
 		return []string{}
 	}
 
-	files = util.Filter(files, func(x interface{}) bool {
-		f := x.(os.FileInfo)
-		if f.IsDir() || strings.HasPrefix(f.Name(), ".") {
-			return false
+	var finalFiles []string
+	for _, file := range files {
+		if !file.IsDir() && !strings.HasPrefix(file.Name(), ".") {
+			finalFiles = append(finalFiles, path.Join(CloudConfigDir, file.Name()))
+		}
 	}
-		return true
-	})
 
-	return util.ToStrings(util.Map(files, func(x interface{}) interface{} {
-		return path.Join(CloudConfigDir, x.(os.FileInfo).Name())
-	}))
+	return finalFiles
+}
+
+func applyDebugFlags(rawCfg map[interface{}]interface{}) (map[interface{}]interface{}, error) {
+	cfg := &CloudConfig{}
+	if err := util.Convert(rawCfg, cfg); err != nil {
+		return nil, err
+	}
+
+	if cfg.Rancher.Debug {
+		log.SetLevel(log.DebugLevel)
+		if !util.Contains(cfg.Rancher.Docker.Args, "-D") {
+			cfg.Rancher.Docker.Args = append(cfg.Rancher.Docker.Args, "-D")
+		}
+		if !util.Contains(cfg.Rancher.SystemDocker.Args, "-D") {
+			cfg.Rancher.SystemDocker.Args = append(cfg.Rancher.SystemDocker.Args, "-D")
+		}
+	}
+
+	_, rawCfg = getOrSetVal("rancher.docker.args", rawCfg, cfg.Rancher.Docker.Args)
+	_, rawCfg = getOrSetVal("rancher.system_docker.args", rawCfg, cfg.Rancher.SystemDocker.Args)
+	return rawCfg, nil
 }
 
 // mergeMetadata merges certain options from md (meta-data from the datasource)
 // onto cc (a CloudConfig derived from user-data), if they are not already set
 // on cc (i.e. user-data always takes precedence)
-func mergeMetadata(cc *CloudConfig, md datasource.Metadata) *CloudConfig {
-	if cc == nil {
-		return cc
+func mergeMetadata(rawCfg map[interface{}]interface{}, md datasource.Metadata) map[interface{}]interface{} {
+	if rawCfg == nil {
+		return nil
+	}
+	out := util.MapCopy(rawCfg)
+
+	outHostname, ok := out["hostname"]
+	if !ok {
+		outHostname = ""
 	}
-	out := cc
-	dirty := false
 
 	if md.Hostname != "" {
-		if out.Hostname != "" {
-			log.Debugf("Warning: user-data hostname (%s) overrides metadata hostname (%s)\n", out.Hostname, md.Hostname)
+		if outHostname != "" {
+			log.Debugf("Warning: user-data hostname (%s) overrides metadata hostname (%s)\n", outHostname, md.Hostname)
 		} else {
-			out = &(*cc)
-			dirty = true
-			out.Hostname = md.Hostname
+			out["hostname"] = md.Hostname
 		}
 	}
 
@@ -126,14 +149,18 @@ func mergeMetadata(cc *CloudConfig, md datasource.Metadata) *CloudConfig {
 
 	sort.Sort(sort.StringSlice(keys))
 
+	currentKeys, ok := out["ssh_authorized_keys"]
+	if !ok {
+		return out
+	}
+
+	finalKeys := currentKeys.([]interface{})
 	for _, k := range keys {
-		if !dirty {
-			out = &(*cc)
-			dirty = true
-		}
-		out.SSHAuthorizedKeys = append(out.SSHAuthorizedKeys, md.SSHPublicKeys[k])
+		finalKeys = append(finalKeys, md.SSHPublicKeys[k])
 	}
 
+	out["ssh_authorized_keys"] = finalKeys
+
 	return out
 }
 
@@ -145,44 +172,32 @@ func readMetadata() datasource.Metadata {
 	return metadata
 }
 
-func readFilesAndMetadata(c *CloudConfig) (*CloudConfig, error) {
+func readConfigs() (map[interface{}]interface{}, error) {
 	files := append(CloudConfigDirFiles(), CloudConfigFile)
 	data, err := readConfig(nil, true, files...)
 	if err != nil {
-		log.WithFields(log.Fields{"err": err, "files": files}).Error("Error reading config files")
-		return c, err
+		return nil, err
 	}
-
-	t, err := c.Merge(data)
-	if err != nil {
-		log.WithFields(log.Fields{"cfg": c, "data": data, "err": err}).Error("Error merging config data")
-		return c, err
-	}
-
-	return t, nil
+	return data, nil
 }
 
-func readCmdline(c *CloudConfig) (*CloudConfig, error) {
+func readCmdline() (map[interface{}]interface{}, error) {
 	log.Debug("Reading config cmdline")
 	cmdLine, err := ioutil.ReadFile("/proc/cmdline")
 	if err != nil {
 		log.WithFields(log.Fields{"err": err}).Error("Failed to read kernel params")
-		return c, err
+		return nil, err
 	}
 
 	if len(cmdLine) == 0 {
-		return c, nil
+		return nil, nil
 	}
 
 	log.Debugf("Config cmdline %s", cmdLine)
 
 	cmdLineObj := parseCmdline(strings.TrimSpace(string(cmdLine)))
 
-	t, err := c.Merge(cmdLineObj)
-	if err != nil {
-		log.WithFields(log.Fields{"cfg": c, "cmdLine": cmdLine, "data": cmdLineObj, "err": err}).Warn("Error adding kernel params to config")
-	}
-	return t, nil
+	return cmdLineObj, nil
 }
 
 func amendNils(c *CloudConfig) (*CloudConfig, error) {
@@ -227,23 +242,6 @@ func WriteToFile(data interface{}, filename string) error {
 	return ioutil.WriteFile(filename, content, 400)
 }
 
-func saveToDisk(data map[interface{}]interface{}) error {
-	private, config := filterDottedKeys(data, []string{
-		"rancher.ssh",
-		"rancher.docker.ca_key",
-		"rancher.docker.ca_cert",
-		"rancher.docker.server_key",
-		"rancher.docker.server_cert",
-	})
-
-	err := WriteToFile(config, CloudConfigFile)
-	if err != nil {
-		return err
-	}
-
-	return WriteToFile(private, CloudConfigPrivateFile)
-}
-
 func readConfig(bytes []byte, substituteMetadataVars bool, files ...string) (map[interface{}]interface{}, error) {
 	// You can't just overlay yaml bytes on to maps, it won't merge, but instead
 	// just override the keys and not merge the map values.
@@ -267,7 +265,7 @@ func readConfig(bytes []byte, substituteMetadataVars bool, files ...string) (map
 			return nil, err
 		}
 
-		left = util.MapsUnion(left, right)
+		left = util.Merge(left, right)
 	}
 
 	if bytes != nil && len(bytes) > 0 {
@@ -279,7 +277,7 @@ func readConfig(bytes []byte, substituteMetadataVars bool, files ...string) (map
 			return nil, err
 		}
 
-		left = util.MapsUnion(left, right)
+		left = util.Merge(left, right)
 	}
 
 	return left, nil

config/types.go

@@ -36,7 +36,6 @@ const (
 	OsConfigFile           = "/usr/share/ros/os-config.yml"
 	CloudConfigDir         = "/var/lib/rancher/conf/cloud-config.d"
 	CloudConfigBootFile    = "/var/lib/rancher/conf/cloud-config.d/boot.yml"
-	CloudConfigPrivateFile = "/var/lib/rancher/conf/cloud-config.d/private.yml"
 	CloudConfigNetworkFile = "/var/lib/rancher/conf/cloud-config.d/network.yml"
 	CloudConfigScriptFile  = "/var/lib/rancher/conf/cloud-config-script"
 	MetaDataFile           = "/var/lib/rancher/conf/metadata"
@@ -48,6 +47,13 @@ var (
 	VERSION       string
 	ARCH          string
 	SUFFIX        string
+	PrivateKeys   = []string{
+		"rancher.ssh",
+		"rancher.docker.ca_key",
+		"rancher.docker.ca_cert",
+		"rancher.docker.server_key",
+		"rancher.docker.server_cert",
+	}
 )
 
 func init() {

docker/service.go

@@ -108,8 +108,7 @@ func (s *Service) shouldRebuild(ctx context.Context) (bool, error) {
 		rebuilding := false
 		if outOfSync {
 			if cfg.Rancher.ForceConsoleRebuild && s.Name() == "console" {
-				cfg.Rancher.ForceConsoleRebuild = false
-				if err := cfg.Save(); err != nil {
+				if err := config.Set("rancher.force_console_rebuild", false); err != nil {
 					return false, err
 				}
 				rebuilding = true

init/init.go

@@ -204,7 +204,7 @@ func RunInit() error {
 			}
 
 			if cfg.Rancher.Debug {
-				cfgString, err := config.Dump(false, true)
+				cfgString, err := config.Export(false, true)
 				if err != nil {
 					log.WithFields(log.Fields{"err": err}).Error("Error serializing config")
 				} else {

util/util.go

@@ -3,15 +3,12 @@ package util
 import (
 	"bytes"
 	"io"
-	"io/ioutil"
 	"os"
 	"strings"
 
 	yaml "github.com/cloudfoundry-incubator/candiedyaml"
 
 	log "github.com/Sirupsen/logrus"
-
-	"reflect"
 )
 
 type AnyMap map[interface{}]interface{}
@@ -91,56 +88,7 @@ func Copy(d interface{}) interface{} {
 	}
 }
 
-func Replace(l, r interface{}) interface{} {
-	return r
-}
-
-func Equal(l, r interface{}) interface{} {
-	if reflect.DeepEqual(l, r) {
-		return l
-	}
-	return nil
-}
-
-func Filter(xs []interface{}, p func(x interface{}) bool) []interface{} {
-	return FlatMap(xs, func(x interface{}) []interface{} {
-		if p(x) {
-			return []interface{}{x}
-		}
-		return []interface{}{}
-	})
-}
-
-func FilterStrings(xs []string, p func(x string) bool) []string {
-	return FlatMapStrings(xs, func(x string) []string {
-		if p(x) {
-			return []string{x}
-		}
-		return []string{}
-	})
-}
-
-func Map(xs []interface{}, f func(x interface{}) interface{}) []interface{} {
-	return FlatMap(xs, func(x interface{}) []interface{} { return []interface{}{f(x)} })
-}
-
-func FlatMap(xs []interface{}, f func(x interface{}) []interface{}) []interface{} {
-	result := []interface{}{}
-	for _, x := range xs {
-		result = append(result, f(x)...)
-	}
-	return result
-}
-
-func FlatMapStrings(xs []string, f func(x string) []string) []string {
-	result := []string{}
-	for _, x := range xs {
-		result = append(result, f(x)...)
-	}
-	return result
-}
-
-func MapsUnion(left, right map[interface{}]interface{}) map[interface{}]interface{} {
+func Merge(left, right map[interface{}]interface{}) map[interface{}]interface{} {
 	result := MapCopy(left)
 
 	for k, r := range right {
@@ -149,12 +97,12 @@ func MapsUnion(left, right map[interface{}]interface{}) map[interface{}]interfac
 			case map[interface{}]interface{}:
 				switch r := r.(type) {
 				case map[interface{}]interface{}:
-					result[k] = MapsUnion(l, r)
+					result[k] = Merge(l, r)
 				default:
-					result[k] = Replace(l, r)
+					result[k] = r
 				}
 			default:
-				result[k] = Replace(l, r)
+				result[k] = r
 			}
 		} else {
 			result[k] = Copy(r)
@@ -164,66 +112,6 @@ func MapsUnion(left, right map[interface{}]interface{}) map[interface{}]interfac
 	return result
 }
 
-func MapsDifference(left, right map[interface{}]interface{}) map[interface{}]interface{} {
-	result := map[interface{}]interface{}{}
-
-	for k, l := range left {
-		if r, ok := right[k]; ok {
-			switch l := l.(type) {
-			case map[interface{}]interface{}:
-				switch r := r.(type) {
-				case map[interface{}]interface{}:
-					if len(l) == 0 && len(r) == 0 {
-						continue
-					} else if len(l) == 0 {
-						result[k] = l
-					} else if v := MapsDifference(l, r); len(v) > 0 {
-						result[k] = v
-					}
-				default:
-					if v := Equal(l, r); v == nil {
-						result[k] = l
-					}
-				}
-			default:
-				if v := Equal(l, r); v == nil {
-					result[k] = l
-				}
-			}
-		} else {
-			result[k] = l
-		}
-	}
-
-	return result
-}
-
-func MapsIntersection(left, right map[interface{}]interface{}) map[interface{}]interface{} {
-	result := map[interface{}]interface{}{}
-
-	for k, l := range left {
-		if r, ok := right[k]; ok {
-			switch l := l.(type) {
-			case map[interface{}]interface{}:
-				switch r := r.(type) {
-				case map[interface{}]interface{}:
-					result[k] = MapsIntersection(l, r)
-				default:
-					if v := Equal(l, r); v != nil {
-						result[k] = v
-					}
-				}
-			default:
-				if v := Equal(l, r); v != nil {
-					result[k] = v
-				}
-			}
-		}
-	}
-
-	return result
-}
-
 func MapCopy(data map[interface{}]interface{}) map[interface{}]interface{} {
 	result := map[interface{}]interface{}{}
 	for k, v := range data {
@@ -240,6 +128,16 @@ func SliceCopy(data []interface{}) []interface{} {
 	return result
 }
 
+func RemoveString(slice []string, s string) []string {
+	result := []string{}
+	for _, elem := range slice {
+		if elem != s {
+			result = append(result, elem)
+		}
+	}
+	return result
+}
+
 func ToStrings(data []interface{}) []string {
 	result := make([]string, len(data), len(data))
 	for k, v := range data {
@@ -248,18 +146,6 @@ func ToStrings(data []interface{}) []string {
 	return result
 }
 
-func DirLs(dir string) ([]interface{}, error) {
-	result := []interface{}{}
-	files, err := ioutil.ReadDir(dir)
-	if err != nil {
-		return result, err
-	}
-	for _, f := range files {
-		result = append(result, f)
-	}
-	return result, nil
-}
-
 func Map2KVPairs(m map[string]string) []string {
 	r := make([]string, 0, len(m))
 	for k, v := range m {

util/util_test.go

@@ -12,15 +12,6 @@ type testCloudConfig struct {
 	Key2     string `yaml:"key2,omitempty"`
 }
 
-func TestPassByValue(t *testing.T) {
-	assert := require.New(t)
-	cc0ptr := &testCloudConfig{}
-	cc0ptr.Hostname = "test0"
-	cc1 := *cc0ptr
-	cc1.Hostname = "test1"
-	assert.NotEqual(cc0ptr.Hostname, cc1.Hostname)
-}
-
 func TestConvertMergesLeftIntoRight(t *testing.T) {
 	assert := require.New(t)
 	cc0 := testCloudConfig{Key1: "k1v0", Key2: "k2v0"}
@@ -30,13 +21,6 @@ func TestConvertMergesLeftIntoRight(t *testing.T) {
 	assert.Equal(expected, cc0)
 }
 
-func TestNilMap(t *testing.T) {
-	assert := require.New(t)
-	var m map[string]interface{} = nil
-	assert.True(m == nil)
-	assert.True(len(m) == 0)
-}
-
 func NoTestCopyPointer(t *testing.T) {
 	assert := require.New(t)
 	testCCpt := &testCloudConfig{}
@@ -48,35 +32,6 @@ func NoTestCopyPointer(t *testing.T) {
 	assert.Equal("", m1["b"].(*testCloudConfig).Hostname)
 }
 
-func TestEmptyMap(t *testing.T) {
-	assert := require.New(t)
-	m := map[interface{}]interface{}{}
-	assert.True(len(m) == 0)
-}
-
-func tryMutateArg(p *string) *string {
-	s := "test"
-	p = &s
-	return p
-}
-
-func TestMutableArg(t *testing.T) {
-	assert := require.New(t)
-	s := "somestring"
-	p := &s
-	assert.NotEqual(tryMutateArg(p), p)
-}
-
-func TestFilter(t *testing.T) {
-	assert := require.New(t)
-	ss := []interface{}{"1", "2", "3", "4"}
-	assert.Equal([]interface{}{"1", "2", "4"}, Filter(ss, func(x interface{}) bool { return x != "3" }))
-
-	ss1 := append([]interface{}{}, "qqq")
-	assert.Equal([]interface{}{"qqq"}, ss1)
-
-}
-
 func TestMapCopy(t *testing.T) {
 	assert := require.New(t)
 	m0 := map[interface{}]interface{}{"a": 1, "b": map[interface{}]interface{}{"c": 3}, "d": "4"}
@@ -109,52 +64,7 @@ func TestSliceCopy(t *testing.T) {
 	assert.Equal(len(b1), len(b0)+1)
 }
 
-func TestMapsIntersection(t *testing.T) {
-	assert := require.New(t)
-
-	m0 := map[interface{}]interface{}{
-		"a": 1,
-		"b": map[interface{}]interface{}{"c": 3},
-		"d": "4",
-		"e": []interface{}{1, 2, 3},
-	}
-	m1 := MapCopy(m0)
-
-	delete(m0, "a")
-	b1 := m1["b"].(map[interface{}]interface{})
-	delete(b1, "c")
-	m1["e"] = []interface{}{2, 3, 4}
-	expected := map[interface{}]interface{}{"b": map[interface{}]interface{}{}, "d": "4"}
-	assert.Equal(expected, MapsIntersection(m0, m1))
-}
-
-func TestMapsDifference(t *testing.T) {
-	assert := require.New(t)
-
-	m0 := map[interface{}]interface{}{
-		"a": 1,
-		"b": map[interface{}]interface{}{"c": 3},
-		"d": "4",
-		"e": []interface{}{1, 2, 3},
-	}
-	m1 := MapCopy(m0)
-
-	assert.Equal(map[interface{}]interface{}{}, MapsDifference(m0, m0))
-	assert.Equal(map[interface{}]interface{}{}, MapsDifference(m0, m1))
-
-	delete(m1, "a")
-	b1 := m1["b"].(map[interface{}]interface{})
-	delete(b1, "c")
-	m1["e"] = []interface{}{2, 3, 4}
-
-	expectedM1M0 := map[interface{}]interface{}{"b": map[interface{}]interface{}{}, "e": []interface{}{2, 3, 4}}
-	assert.Equal(expectedM1M0, MapsDifference(m1, m0))
-
-	expectedM0M1 := map[interface{}]interface{}{"a": 1, "b": map[interface{}]interface{}{"c": 3}, "e": []interface{}{1, 2, 3}}
-	assert.Equal(expectedM0M1, MapsDifference(m0, m1))
-}
-
-func TestMapsUnion(t *testing.T) {
+func TestMerge(t *testing.T) {
 	assert := require.New(t)
 
 	m0 := map[interface{}]interface{}{
@@ -178,5 +88,5 @@ func TestMapsUnion(t *testing.T) {
 		"e": "added",
 		"f": []interface{}{2, 3, 4},
 	}
-	assert.Equal(expected, MapsUnion(m0, m1))
+	assert.Equal(expected, Merge(m0, m1))
 }