make cloud-config the configuration mechanism for RancherOS

.dockerignore

@@ -1,6 +1,8 @@
 .DS_Store
 .git
 .idea
+bin
+gopath
 tmp
 state
 build

cmd/cloudinit/cloudinit.go

@@ -48,86 +48,51 @@ const (
 	datasourceInterval    = 100 * time.Millisecond
 	datasourceMaxInterval = 30 * time.Second
 	datasourceTimeout     = 5 * time.Minute
+	sshKeyName            = "rancheros-cloud-config"
+	baseConfigDir         = "/var/lib/rancher/conf/cloud-config.d"
 )
 
 var (
-	baseConfigDir string
+	save    bool
-	outputDir     string
+	execute bool
-	outputFile    string
+	network bool
-	metaDataFile  string
+	flags   *flag.FlagSet
-	scriptFile    string
-	rancherYml    string
-	save          bool
-	execute       bool
-	network       bool
-	sshKeyName    string
-	flags         *flag.FlagSet
 )
 
 func init() {
 	flags = flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
-	flags.StringVar(&baseConfigDir, "base-config-dir", "/var/lib/rancher/conf/cloud-config.d", "base cloud config")
-	flags.StringVar(&outputDir, "dir", "/var/lib/rancher/conf", "working directory")
-	flags.StringVar(&outputFile, "file", "cloud-config-processed.yml", "output cloud config file name")
-	flags.StringVar(&metaDataFile, "metadata", "metadata", "output metdata file name")
-	flags.StringVar(&scriptFile, "script-file", "cloud-config-script", "output cloud config script file name")
-	flags.StringVar(&rancherYml, "rancher", "cloud-config-rancher.yml", "output cloud config rancher file name")
-	flags.StringVar(&sshKeyName, "ssh-key-name", "rancheros-cloud-config", "SSH key name")
 	flags.BoolVar(&network, "network", true, "use network based datasources")
 	flags.BoolVar(&save, "save", false, "save cloud config and exit")
 	flags.BoolVar(&execute, "execute", false, "execute saved cloud config")
 }
 
 func saveFiles(cloudConfigBytes, scriptBytes []byte, metadata datasource.Metadata) error {
-	scriptOutput := path.Join(outputDir, scriptFile)
+	os.Remove(rancherConfig.CloudConfigScriptFile)
-	cloudConfigOutput := path.Join(outputDir, outputFile)
+	os.Remove(rancherConfig.CloudConfigFile)
-	rancherYmlOutput := path.Join(outputDir, rancherYml)
+	os.Remove(rancherConfig.MetaDataFile)
-	metaDataOutput := path.Join(outputDir, metaDataFile)
-
-	os.Remove(scriptOutput)
-	os.Remove(cloudConfigOutput)
-	os.Remove(rancherYmlOutput)
-	os.Remove(metaDataOutput)
 
 	if len(scriptBytes) > 0 {
-		log.Infof("Writing to %s", scriptOutput)
+		log.Infof("Writing to %s", rancherConfig.CloudConfigScriptFile)
-		if err := ioutil.WriteFile(scriptOutput, scriptBytes, 500); err != nil {
+		if err := ioutil.WriteFile(rancherConfig.CloudConfigScriptFile, scriptBytes, 500); err != nil {
-			log.Errorf("Error while writing file %s: %v", scriptOutput, err)
+			log.Errorf("Error while writing file %s: %v", rancherConfig.CloudConfigScriptFile, err)
 			return err
 		}
 	}
 
-	cloudConfigBytes = append([]byte("#cloud-config\n"), cloudConfigBytes...)
+	if err := ioutil.WriteFile(rancherConfig.CloudConfigFile, cloudConfigBytes, 400); err != nil {
-	log.Infof("Writing to %s", cloudConfigOutput)
-	if err := ioutil.WriteFile(cloudConfigOutput, cloudConfigBytes, 500); err != nil {
-		log.Errorf("Error while writing file %s: %v", cloudConfigOutput, err)
 		return err
 	}
-
+	log.Infof("Written to %s:\n%s", rancherConfig.CloudConfigFile, string(cloudConfigBytes))
-	ccData := make(map[string]interface{})
-	if err := yaml.Unmarshal(cloudConfigBytes, ccData); err != nil {
-		return err
-	}
-
-	if rancher, ok := ccData["rancher"]; ok {
-		bytes, err := yaml.Marshal(rancher)
-		if err != nil {
-			return err
-		}
-
-		if err = ioutil.WriteFile(rancherYmlOutput, bytes, 400); err != nil {
-			return err
-		}
-	}
 
 	metaDataBytes, err := yaml.Marshal(metadata)
 	if err != nil {
 		return err
 	}
 
-	if err = ioutil.WriteFile(metaDataOutput, metaDataBytes, 400); err != nil {
+	if err = ioutil.WriteFile(rancherConfig.MetaDataFile, metaDataBytes, 400); err != nil {
 		return err
 	}
+	log.Infof("Written to %s:\n%s", rancherConfig.MetaDataFile, string(metaDataBytes))
 
 	return nil
 }
@@ -258,24 +223,22 @@ func saveCloudConfig() error {
 }
 
 func getSaveCloudConfig() (*config.CloudConfig, error) {
-	cloudConfig := path.Join(outputDir, outputFile)
+	ds := file.NewDatasource(rancherConfig.CloudConfigFile)
-
-	ds := file.NewDatasource(cloudConfig)
 	if !ds.IsAvailable() {
-		log.Infof("%s does not exist", cloudConfig)
+		log.Infof("%s does not exist", rancherConfig.CloudConfigFile)
 		return nil, nil
 	}
 
 	ccBytes, err := ds.FetchUserdata()
 	if err != nil {
-		log.Errorf("Failed to read user-data from %s: %v", cloudConfig, err)
+		log.Errorf("Failed to read user-data from %s: %v", rancherConfig.CloudConfigFile, err)
 		return nil, err
 	}
 
 	var cc config.CloudConfig
 	err = yaml.Unmarshal(ccBytes, &cc)
 	if err != nil {
-		log.Errorf("Failed to unmarshall user-data from %s: %v", cloudConfig, err)
+		log.Errorf("Failed to unmarshall user-data from %s: %v", rancherConfig.CloudConfigFile, err)
 		return nil, err
 	}
 
@@ -290,7 +253,7 @@ func executeCloudConfig() error {
 
 	var metadata datasource.Metadata
 
-	metaDataBytes, err := ioutil.ReadFile(path.Join(outputDir, metaDataFile))
+	metaDataBytes, err := ioutil.ReadFile(rancherConfig.MetaDataFile)
 	if err != nil {
 		return err
 	}
@@ -338,6 +301,8 @@ func executeCloudConfig() error {
 func Main() {
 	flags.Parse(rancherConfig.FilterGlobalConfig(os.Args[1:]))
 
+	log.Infof("Running cloud-init: save=%v, execute=%v", save, execute)
+
 	if save {
 		err := saveCloudConfig()
 		if err != nil {
@@ -376,10 +341,10 @@ func mergeConfigs(cc *config.CloudConfig, md datasource.Metadata) (out config.Cl
 
 // getDatasources creates a slice of possible Datasources for cloudinit based
 // on the different source command-line flags.
-func getDatasources(cfg *rancherConfig.Config) []datasource.Datasource {
+func getDatasources(cfg *rancherConfig.CloudConfig) []datasource.Datasource {
 	dss := make([]datasource.Datasource, 0, 5)
 
-	for _, ds := range cfg.CloudInit.Datasources {
+	for _, ds := range cfg.Rancher.CloudInit.Datasources {
 		parts := strings.SplitN(ds, ":", 2)
 
 		switch parts[0] {

cmd/control/config.go

@@ -76,16 +76,16 @@ func configSubcommands() []cli.Command {
 	}
 }
 
-func imagesFromConfig(cfg *config.Config) []string {
+func imagesFromConfig(cfg *config.CloudConfig) []string {
 	imagesMap := map[string]int{}
 
-	for _, service := range cfg.BootstrapContainers {
+	for _, service := range cfg.Rancher.BootstrapContainers {
 		imagesMap[service.Image] = 1
 	}
-	for _, service := range cfg.Autoformat {
+	for _, service := range cfg.Rancher.Autoformat {
 		imagesMap[service.Image] = 1
 	}
-	for _, service := range cfg.SystemContainers {
+	for _, service := range cfg.Rancher.Services {
 		imagesMap[service.Image] = 1
 	}
 
@@ -165,12 +165,12 @@ func configGet(c *cli.Context) {
 
 	cfg, err := config.LoadConfig()
 	if err != nil {
-		log.Fatal(err)
+		log.Panicln(err)
 	}
 
 	val, err := cfg.Get(arg)
 	if err != nil {
-		log.Fatal(err)
+		log.WithFields(log.Fields{"cfg": cfg, "arg": arg, "val": val}).Panicln(err)
 	}
 
 	printYaml := false
@@ -192,50 +192,6 @@ func configGet(c *cli.Context) {
 	}
 }
 
-func getOrSetVal(args string, data map[interface{}]interface{}, value interface{}) interface{} {
-	parts := strings.Split(args, ".")
-
-	for i, part := range parts {
-		val, ok := data[part]
-		last := i+1 == len(parts)
-
-		// Reached end, set the value
-		if last && value != nil {
-			if s, ok := value.(string); ok {
-				value = config.DummyMarshall(s)
-			}
-
-			data[part] = value
-			return value
-		}
-
-		// Missing intermediate key, create key
-		if !last && value != nil && !ok {
-			newData := map[interface{}]interface{}{}
-			data[part] = newData
-			data = newData
-			continue
-		}
-
-		if !ok {
-			break
-		}
-
-		if last {
-			return val
-		}
-
-		newData, ok := val.(map[interface{}]interface{})
-		if !ok {
-			break
-		}
-
-		data = newData
-	}
-
-	return ""
-}
-
 func merge(c *cli.Context) {
 	bytes, err := ioutil.ReadAll(os.Stdin)
 	if err != nil {

cmd/control/env.go

@@ -20,8 +20,8 @@ func envAction(c *cli.Context) {
 	args := c.Args()
 	osEnv := os.Environ()
 
-	envMap := make(map[string]string, len(cfg.Environment)+len(osEnv))
+	envMap := make(map[string]string, len(cfg.Rancher.Environment)+len(osEnv))
-	for k, v := range cfg.Environment {
+	for k, v := range cfg.Rancher.Environment {
 		envMap[k] = v
 	}
 	for k, v := range util.KVPairs2Map(osEnv) {

cmd/control/os.go

@@ -249,5 +249,5 @@ func getUpgradeUrl() (string, error) {
 		return "", err
 	}
 
-	return cfg.Upgrade.Url, nil
+	return cfg.Rancher.Upgrade.Url, nil
 }

cmd/control/service.go

@@ -2,7 +2,7 @@ package control
 
 import (
 	"fmt"
-	"log"
+	log "github.com/Sirupsen/logrus"
 	"strings"
 
 	"github.com/codegangsta/cli"
@@ -44,16 +44,16 @@ func disable(c *cli.Context) {
 	}
 
 	for _, service := range c.Args() {
-		if _, ok := cfg.ServicesInclude[service]; !ok {
+		if _, ok := cfg.Rancher.ServicesInclude[service]; !ok {
 			continue
 		}
 
-		cfg.ServicesInclude[service] = false
+		cfg.Rancher.ServicesInclude[service] = false
 		changed = true
 	}
 
 	if changed {
-		if err = cfg.Set("services_include", cfg.ServicesInclude); err != nil {
+		if err = cfg.Set("rancher.services_include", cfg.Rancher.ServicesInclude); err != nil {
 			log.Fatal(err)
 		}
 	}
@@ -67,15 +67,15 @@ func del(c *cli.Context) {
 	}
 
 	for _, service := range c.Args() {
-		if _, ok := cfg.ServicesInclude[service]; !ok {
+		if _, ok := cfg.Rancher.ServicesInclude[service]; !ok {
 			continue
 		}
-		delete(cfg.ServicesInclude, service)
+		delete(cfg.Rancher.ServicesInclude, service)
 		changed = true
 	}
 
 	if changed {
-		if err = cfg.Set("services_include", cfg.ServicesInclude); err != nil {
+		if err = cfg.Set("rancher.services_include", cfg.Rancher.ServicesInclude); err != nil {
 			log.Fatal(err)
 		}
 	}
@@ -89,20 +89,20 @@ func enable(c *cli.Context) {
 	}
 
 	for _, service := range c.Args() {
-		if val, ok := cfg.ServicesInclude[service]; !ok || !val {
+		if val, ok := cfg.Rancher.ServicesInclude[service]; !ok || !val {
 			if strings.HasPrefix(service, "/") && !strings.HasPrefix(service, "/var/lib/rancher/conf") {
 				log.Fatalf("ERROR: Service should be in path /var/lib/rancher/conf")
 			}
 			if _, err := docker.LoadServiceResource(service, true, cfg); err != nil {
 				log.Fatalf("could not load service %s", service)
 			}
-			cfg.ServicesInclude[service] = true
+			cfg.Rancher.ServicesInclude[service] = true
 			changed = true
 		}
 	}
 
 	if changed {
-		if err = cfg.Set("services_include", cfg.ServicesInclude); err != nil {
+		if err := cfg.Set("rancher.services_include", cfg.Rancher.ServicesInclude); err != nil {
 			log.Fatal(err)
 		}
 	}
@@ -115,11 +115,11 @@ func list(c *cli.Context) {
 	}
 
 	clone := make(map[string]bool)
-	for service, enabled := range cfg.ServicesInclude {
+	for service, enabled := range cfg.Rancher.ServicesInclude {
 		clone[service] = enabled
 	}
 
-	services, err := util.GetServices(cfg.Repositories.ToArray())
+	services, err := util.GetServices(cfg.Rancher.Repositories.ToArray())
 	if err != nil {
 		log.Fatalf("Failed to get services: %v", err)
 	}

cmd/control/tlsconf.go

@@ -44,12 +44,12 @@ func tlsConfCommands() []cli.Command {
 	}
 }
 
-func writeCerts(generateServer bool, hostname []string, cfg *config.Config, certPath, keyPath, caCertPath, caKeyPath string) error {
+func writeCerts(generateServer bool, hostname []string, cfg *config.CloudConfig, certPath, keyPath, caCertPath, caKeyPath string) error {
 	if !generateServer {
 		return machineUtil.GenerateCert([]string{""}, certPath, keyPath, caCertPath, caKeyPath, NAME, BITS)
 	}
 
-	if cfg.UserDocker.ServerKey == "" || cfg.UserDocker.ServerCert == "" {
+	if cfg.Rancher.UserDocker.ServerKey == "" || cfg.Rancher.UserDocker.ServerCert == "" {
 		err := machineUtil.GenerateCert(hostname, certPath, keyPath, caCertPath, caKeyPath, NAME, BITS)
 		if err != nil {
 			return err
@@ -65,26 +65,28 @@ func writeCerts(generateServer bool, hostname []string, cfg *config.Config, cert
 			return err
 		}
 
-		return cfg.SetConfig(&config.Config{
+		return cfg.SetConfig(&config.CloudConfig{
-			UserDocker: config.DockerConfig{
+			Rancher: config.RancherConfig{
-				CAKey:      cfg.UserDocker.CAKey,
+				UserDocker: config.DockerConfig{
-				CACert:     cfg.UserDocker.CACert,
+					CAKey:      cfg.Rancher.UserDocker.CAKey,
-				ServerCert: string(cert),
+					CACert:     cfg.Rancher.UserDocker.CACert,
-				ServerKey:  string(key),
+					ServerCert: string(cert),
+					ServerKey:  string(key),
+				},
 			},
 		})
 	}
 
-	if err := ioutil.WriteFile(certPath, []byte(cfg.UserDocker.ServerCert), 0400); err != nil {
+	if err := ioutil.WriteFile(certPath, []byte(cfg.Rancher.UserDocker.ServerCert), 0400); err != nil {
 		return err
 	}
 
-	return ioutil.WriteFile(keyPath, []byte(cfg.UserDocker.ServerKey), 0400)
+	return ioutil.WriteFile(keyPath, []byte(cfg.Rancher.UserDocker.ServerKey), 0400)
 
 }
 
-func writeCaCerts(cfg *config.Config, caCertPath, caKeyPath string) error {
+func writeCaCerts(cfg *config.CloudConfig, caCertPath, caKeyPath string) error {
-	if cfg.UserDocker.CACert == "" {
+	if cfg.Rancher.UserDocker.CACert == "" {
 		if err := machineUtil.GenerateCACertificate(caCertPath, caKeyPath, NAME, BITS); err != nil {
 			return err
 		}
@@ -99,10 +101,12 @@ func writeCaCerts(cfg *config.Config, caCertPath, caKeyPath string) error {
 			return err
 		}
 
-		err = cfg.SetConfig(&config.Config{
+		err = cfg.SetConfig(&config.CloudConfig{
-			UserDocker: config.DockerConfig{
+			Rancher: config.RancherConfig{
-				CAKey:  string(caKey),
+				UserDocker: config.DockerConfig{
-				CACert: string(caCert),
+					CAKey:  string(caKey),
+					CACert: string(caCert),
+				},
 			},
 		})
 		if err != nil {
@@ -112,11 +116,11 @@ func writeCaCerts(cfg *config.Config, caCertPath, caKeyPath string) error {
 		return nil
 	}
 
-	if err := ioutil.WriteFile(caCertPath, []byte(cfg.UserDocker.CACert), 0400); err != nil {
+	if err := ioutil.WriteFile(caCertPath, []byte(cfg.Rancher.UserDocker.CACert), 0400); err != nil {
 		return err
 	}
 
-	return ioutil.WriteFile(caKeyPath, []byte(cfg.UserDocker.CAKey), 0400)
+	return ioutil.WriteFile(caKeyPath, []byte(cfg.Rancher.UserDocker.CAKey), 0400)
 }
 
 func tlsConfCreate(c *cli.Context) {

cmd/network/network.go

@@ -20,14 +20,14 @@ import (
 func Main() {
 	args := os.Args
 	if len(args) > 1 {
-		fmt.Println("call " + args[0] + " to load network config from rancher.yml config file")
+		fmt.Println("call " + args[0] + " to load network config from cloud-config.yml")
 		return
 	}
 	cfg, err := config.LoadConfig()
 	if err != nil {
 		log.Fatal(err)
 	}
-	ApplyNetworkConfigs(&cfg.Network)
+	ApplyNetworkConfigs(&cfg.Rancher.Network)
 }
 
 func createInterfaces(netCfg *config.NetworkConfig) error {

cmd/respawn/respawn.go

@@ -68,7 +68,7 @@ func run(c *cli.Context) {
 	var wg sync.WaitGroup
 
 	for _, line := range strings.Split(string(input), "\n") {
-		if strings.TrimSpace(line) == "" {
+		if strings.TrimSpace(line) == "" || strings.Index(strings.TrimSpace(line), "#") == 0 {
 			continue
 		}
 		wg.Add(1)

config/config.go

@@ -2,58 +2,21 @@ package config
 
 import (
 	"io/ioutil"
-	"os"
 	"strings"
 
-	"github.com/rancherio/rancher-compose/librcompose/project"
-
 	log "github.com/Sirupsen/logrus"
 	"github.com/rancherio/os/util"
+	"github.com/rancherio/rancher-compose/librcompose/project"
 	"gopkg.in/yaml.v2"
 )
 
-func (c *Config) privilegedMerge(newConfig Config) error {
+func (c *CloudConfig) Import(bytes []byte) error {
-	err := c.overlay(newConfig)
-	if err != nil {
-		return err
-	}
-
-	for k, v := range newConfig.SystemContainers {
-		c.SystemContainers[k] = v
-	}
-
-	return nil
-}
-
-func (c *Config) overlay(newConfig Config) error {
-	newConfig.clearReadOnly()
-	return util.Convert(&newConfig, c)
-}
-
-func (c *Config) clearReadOnly() {
-	c.BootstrapContainers = make(map[string]*project.ServiceConfig, 0)
-	c.SystemContainers = make(map[string]*project.ServiceConfig, 0)
-}
-
-func clearReadOnly(data map[interface{}]interface{}) map[interface{}]interface{} {
-	newData := make(map[interface{}]interface{})
-	for k, v := range data {
-		newData[k] = v
-	}
-
-	delete(newData, "system_container")
-	delete(newData, "bootstrap_container")
-
-	return newData
-}
-
-func (c *Config) Import(bytes []byte) error {
 	data, err := readConfig(bytes, PrivateConfigFile)
 	if err != nil {
 		return err
 	}
 
-	if err = saveToDisk(data); err != nil {
+	if err := saveToDisk(data); err != nil {
 		return err
 	}
 
@@ -61,7 +24,7 @@ func (c *Config) Import(bytes []byte) error {
 }
 
 // This function only sets "non-empty" values
-func (c *Config) SetConfig(newConfig *Config) error {
+func (c *CloudConfig) SetConfig(newConfig *CloudConfig) error {
 	bytes, err := yaml.Marshal(newConfig)
 	if err != nil {
 		return err
@@ -70,83 +33,63 @@ func (c *Config) SetConfig(newConfig *Config) error {
 	return c.Merge(bytes)
 }
 
-func (c *Config) Merge(bytes []byte) error {
+func (c *CloudConfig) Merge(bytes []byte) error {
-	data, err := readSavedConfig(bytes)
+	data, err := readConfig(bytes, LocalConfigFile, PrivateConfigFile)
 	if err != nil {
 		return err
 	}
 
-	err = saveToDisk(data)
+	if err := saveToDisk(data); err != nil {
-	if err != nil {
 		return err
 	}
 
 	return c.Reload()
 }
 
-func LoadConfig() (*Config, error) {
+func LoadConfig() (*CloudConfig, error) {
 	cfg := NewConfig()
 	if err := cfg.Reload(); err != nil {
+		log.WithFields(log.Fields{"cfg": cfg}).Panicln(err)
 		return nil, err
 	}
 
-	if cfg.Debug {
+	if cfg.Rancher.Debug {
 		log.SetLevel(log.DebugLevel)
-		if !util.Contains(cfg.UserDocker.Args, "-D") {
+		if !util.Contains(cfg.Rancher.UserDocker.Args, "-D") {
-			cfg.UserDocker.Args = append(cfg.UserDocker.Args, "-D")
+			cfg.Rancher.UserDocker.Args = append(cfg.Rancher.UserDocker.Args, "-D")
 		}
-		if !util.Contains(cfg.SystemDocker.Args, "-D") {
+		if !util.Contains(cfg.Rancher.SystemDocker.Args, "-D") {
-			cfg.SystemDocker.Args = append(cfg.SystemDocker.Args, "-D")
+			cfg.Rancher.SystemDocker.Args = append(cfg.Rancher.SystemDocker.Args, "-D")
 		}
 	}
 
 	return cfg, nil
 }
 
-func (c *Config) readArgs() error {
+func (c *CloudConfig) merge(values map[interface{}]interface{}) error {
-	log.Debug("Reading config args")
-	parts := make([]string, len(os.Args))
-
-	for _, arg := range os.Args[1:] {
-		if strings.HasPrefix(arg, "--") {
-			arg = arg[2:]
-		}
-
-		kv := strings.SplitN(arg, "=", 2)
-		kv[0] = strings.Replace(kv[0], "-", ".", -1)
-		parts = append(parts, strings.Join(kv, "="))
-	}
-
-	cmdLine := strings.Join(parts, " ")
-	if len(cmdLine) == 0 {
-		return nil
-	}
-
-	log.Debugf("Config Args %s", cmdLine)
-
-	cmdLineObj := parseCmdline(strings.TrimSpace(cmdLine))
-
-	return c.merge(cmdLineObj)
-}
-
-func (c *Config) merge(values map[interface{}]interface{}) error {
-	values = clearReadOnly(values)
 	return util.Convert(values, c)
 }
 
-func (c *Config) readFiles() error {
+func (c *CloudConfig) readFiles() error {
-	data, err := readSavedConfig(nil)
+	data, err := readConfig(nil, CloudConfigFile, LocalConfigFile, PrivateConfigFile)
 	if err != nil {
+		log.Panicln(err)
 		return err
 	}
 
-	return c.merge(data)
+	if err := c.merge(data); err != nil {
+		log.WithFields(log.Fields{"cfg": c, "data": data}).Panicln(err)
+		return err
+	}
+
+	return nil
 }
 
-func (c *Config) readCmdline() error {
+func (c *CloudConfig) readCmdline() error {
 	log.Debug("Reading config cmdline")
 	cmdLine, err := ioutil.ReadFile("/proc/cmdline")
 	if err != nil {
+		log.Panicln(err)
 		return err
 	}
 
@@ -157,16 +100,21 @@ func (c *Config) readCmdline() error {
 	log.Debugf("Config cmdline %s", cmdLine)
 
 	cmdLineObj := parseCmdline(strings.TrimSpace(string(cmdLine)))
-	return c.merge(cmdLineObj)
+
+	if err := c.merge(cmdLineObj); err != nil {
+		log.WithFields(log.Fields{"cfg": c, "cmdLine": cmdLine, "data": cmdLineObj}).Panicln(err)
+		return err
+	}
+	return nil
 }
 
 func Dump(private, full bool) (string, error) {
-	files := []string{CloudConfigFile, ConfigFile}
+	files := []string{CloudConfigFile, LocalConfigFile}
 	if private {
 		files = append(files, PrivateConfigFile)
 	}
 
-	c := &Config{}
+	c := &CloudConfig{}
 
 	if full {
 		c = NewConfig()
@@ -177,23 +125,23 @@ func Dump(private, full bool) (string, error) {
 		return "", err
 	}
 
-	err = c.merge(data)
+	if err := c.merge(data); err != nil {
-	if err != nil {
 		return "", err
 	}
 
-	err = c.readGlobals()
+	if err := c.readGlobals(); err != nil {
-	if err != nil {
 		return "", err
 	}
 
+	c.amendNils()
+
 	bytes, err := yaml.Marshal(c)
 	return string(bytes), err
 }
 
-func (c *Config) configureConsole() error {
+func (c *CloudConfig) configureConsole() error {
-	if console, ok := c.SystemContainers[CONSOLE_CONTAINER]; ok {
+	if console, ok := c.Rancher.Services[CONSOLE_CONTAINER]; ok {
-		if c.Console.Persistent {
+		if c.Rancher.Console.Persistent {
 			console.Labels.MapParts()[REMOVE] = "false"
 		} else {
 			console.Labels.MapParts()[REMOVE] = "true"
@@ -203,22 +151,41 @@ func (c *Config) configureConsole() error {
 	return nil
 }
 
-func (c *Config) readGlobals() error {
+func (c *CloudConfig) amendNils() error {
+	if c.Rancher.Environment == nil {
+		c.Rancher.Environment = map[string]string{}
+	}
+	if c.Rancher.Autoformat == nil {
+		c.Rancher.Autoformat = map[string]*project.ServiceConfig{}
+	}
+	if c.Rancher.BootstrapContainers == nil {
+		c.Rancher.BootstrapContainers = map[string]*project.ServiceConfig{}
+	}
+	if c.Rancher.Services == nil {
+		c.Rancher.Services = map[string]*project.ServiceConfig{}
+	}
+	if c.Rancher.ServicesInclude == nil {
+		c.Rancher.ServicesInclude = map[string]bool{}
+	}
+	return nil
+}
+
+func (c *CloudConfig) readGlobals() error {
 	return util.ShortCircuit(
 		c.readCmdline,
-		c.readArgs,
+		c.configureConsole, // TODO: this smells (it is a write hidden inside a read)
-		c.configureConsole,
 	)
 }
 
-func (c *Config) Reload() error {
+func (c *CloudConfig) Reload() error {
 	return util.ShortCircuit(
 		c.readFiles,
 		c.readGlobals,
+		c.amendNils,
 	)
 }
 
-func (c *Config) Get(key string) (interface{}, error) {
+func (c *CloudConfig) Get(key string) (interface{}, error) {
 	data := make(map[interface{}]interface{})
 	err := util.Convert(c, &data)
 	if err != nil {
@@ -228,8 +195,8 @@ func (c *Config) Get(key string) (interface{}, error) {
 	return getOrSetVal(key, data, nil), nil
 }
 
-func (c *Config) Set(key string, value interface{}) error {
+func (c *CloudConfig) Set(key string, value interface{}) error {
-	data, err := readSavedConfig(nil)
+	data, err := readConfig(nil, LocalConfigFile, PrivateConfigFile)
 	if err != nil {
 		return err
 	}
@@ -242,8 +209,7 @@ func (c *Config) Set(key string, value interface{}) error {
 		return err
 	}
 
-	err = saveToDisk(data)
+	if err := saveToDisk(data); err != nil {
-	if err != nil {
 		return err
 	}
 

config/config_test.go

@@ -3,102 +3,252 @@ package config
 import (
 	"fmt"
 	"gopkg.in/yaml.v2"
-	"log"
 	"testing"
 
 	"github.com/rancherio/os/util"
+	"github.com/stretchr/testify/require"
+	"strings"
 )
-import "reflect"
 
-func testParseCmdline(t *testing.T) {
+func TestNilMap(t *testing.T) {
-	expected := map[string]interface{}{
+	assert := require.New(t)
-		"rescue":   true,
+	var m map[string]interface{} = nil
-		"key1":     "value1",
+	assert.True(m == nil)
-		"key2":     "value2",
+}
-		"keyArray": []string{"1", "2"},
+
-		"obj1": map[string]interface{}{
+func TestMapCopy(t *testing.T) {
-			"key3": "3value",
+	assert := require.New(t)
-			"obj2": map[string]interface{}{
+	m0 := map[interface{}]interface{}{"a": 1, "b": map[interface{}]interface{}{"c": 3}, "d": "4"}
-				"key4": true,
+	m1 := util.MapCopy(m0)
+
+	delete(m0, "a")
+	assert.Equal(len(m1), len(m0)+1)
+
+	b0 := m0["b"].(map[interface{}]interface{})
+	b1 := m1["b"].(map[interface{}]interface{})
+	b1["e"] = "queer"
+
+	assert.Equal(len(b1), len(b0)+1)
+}
+
+func TestMapsIntersection(t *testing.T) {
+	assert := require.New(t)
+
+	m0 := map[interface{}]interface{}{"a": 1, "b": map[interface{}]interface{}{"c": 3}, "d": "4"}
+	m1 := util.MapCopy(m0)
+
+	delete(m0, "a")
+	b1 := m1["b"].(map[interface{}]interface{})
+	delete(b1, "c")
+	expected := map[interface{}]interface{}{"b": map[interface{}]interface{}{}, "d": "4"}
+	assert.Equal(expected, util.MapsIntersection(m0, m1, util.Equal))
+}
+
+func TestMapsUnion(t *testing.T) {
+	assert := require.New(t)
+
+	m0 := map[interface{}]interface{}{"a": 1, "b": map[interface{}]interface{}{"c": 3}, "d": "4"}
+	m1 := util.MapCopy(m0)
+	m1["e"] = "added"
+	m1["d"] = "replaced"
+
+	delete(m0, "a")
+	b1 := m1["b"].(map[interface{}]interface{})
+	delete(b1, "c")
+	expected := map[interface{}]interface{}{"a": 1, "b": map[interface{}]interface{}{"c": 3}, "d": "replaced", "e": "added"}
+	assert.Equal(expected, util.MapsUnion(m0, m1, util.Replace))
+}
+
+func TestFilterKey(t *testing.T) {
+	assert := require.New(t)
+	data := map[interface{}]interface{}{
+		"ssh_authorized_keys": []string{"pubk1", "pubk2"},
+		"hostname":            "ros-test",
+		"rancher": map[interface{}]interface{}{
+			"ssh": map[interface{}]interface{}{
+				"keys": map[interface{}]interface{}{
+					"dsa":     "dsa-test1",
+					"dsa-pub": "dsa-test2",
+				},
+			},
+			"user_docker": map[interface{}]interface{}{
+				"ca_key":  "ca_key-test3",
+				"ca_cert": "ca_cert-test4",
+				"args":    []string{"args_test5"},
 			},
 		},
-		"key5": 5,
+	}
+	expectedFiltered := map[interface{}]interface{}{
+		"rancher": map[interface{}]interface{}{
+			"ssh": map[interface{}]interface{}{
+				"keys": map[interface{}]interface{}{
+					"dsa":     "dsa-test1",
+					"dsa-pub": "dsa-test2",
+				},
+			},
+		},
+	}
+	expectedRest := map[interface{}]interface{}{
+		"ssh_authorized_keys": []string{"pubk1", "pubk2"},
+		"hostname":            "ros-test",
+		"rancher": map[interface{}]interface{}{
+			"user_docker": map[interface{}]interface{}{
+				"ca_key":  "ca_key-test3",
+				"ca_cert": "ca_cert-test4",
+				"args":    []string{"args_test5"},
+			},
+		},
+	}
+	filtered, rest := filterKey(data, []string{"rancher", "ssh"})
+	assert.Equal(expectedFiltered, filtered)
+	assert.Equal(expectedRest, rest)
+}
+
+func TestFilterDottedKeys(t *testing.T) {
+	assert := require.New(t)
+
+	data := map[interface{}]interface{}{
+		"ssh_authorized_keys": []string{"pubk1", "pubk2"},
+		"hostname":            "ros-test",
+		"rancher": map[interface{}]interface{}{
+			"ssh": map[interface{}]interface{}{
+				"keys": map[interface{}]interface{}{
+					"dsa":     "dsa-test1",
+					"dsa-pub": "dsa-test2",
+				},
+			},
+			"user_docker": map[interface{}]interface{}{
+				"ca_key":  "ca_key-test3",
+				"ca_cert": "ca_cert-test4",
+				"args":    []string{"args_test5"},
+			},
+		},
+	}
+	expectedFiltered := map[interface{}]interface{}{
+		"ssh_authorized_keys": []string{"pubk1", "pubk2"},
+		"rancher": map[interface{}]interface{}{
+			"ssh": map[interface{}]interface{}{
+				"keys": map[interface{}]interface{}{
+					"dsa":     "dsa-test1",
+					"dsa-pub": "dsa-test2",
+				},
+			},
+		},
+	}
+	expectedRest := map[interface{}]interface{}{
+		"hostname": "ros-test",
+		"rancher": map[interface{}]interface{}{
+			"user_docker": map[interface{}]interface{}{
+				"ca_key":  "ca_key-test3",
+				"ca_cert": "ca_cert-test4",
+				"args":    []string{"args_test5"},
+			},
+		},
+	}
+
+	assert.Equal([]string{"rancher", "ssh"}, strings.Split("rancher.ssh", "."))
+	assert.Equal([]string{"ssh_authorized_keys"}, strings.Split("ssh_authorized_keys", "."))
+
+	filtered, rest := filterDottedKeys(data, []string{"ssh_authorized_keys", "rancher.ssh"})
+
+	assert.Equal(expectedFiltered, filtered)
+	assert.Equal(expectedRest, rest)
+}
+
+func TestParseCmdline(t *testing.T) {
+	assert := require.New(t)
+
+	expected := map[interface{}]interface{}{
+		"rancher": map[interface{}]interface{}{
+			"rescue":   true,
+			"key1":     "value1",
+			"key2":     "value2",
+			"keyArray": []string{"1", "2"},
+			"obj1": map[interface{}]interface{}{
+				"key3": "3value",
+				"obj2": map[interface{}]interface{}{
+					"key4": true,
+				},
+			},
+			"key5": 5,
+		},
 	}
 
 	actual := parseCmdline("a b rancher.rescue rancher.keyArray=[1,2] rancher.key1=value1 c rancher.key2=value2 rancher.obj1.key3=3value rancher.obj1.obj2.key4 rancher.key5=5")
 
-	ok := reflect.DeepEqual(actual, expected)
+	assert.Equal(expected, actual)
-	if !ok {
-		t.Fatalf("%v != %v", actual, expected)
-	}
 }
 
 func TestGet(t *testing.T) {
+	assert := require.New(t)
+
 	data := map[interface{}]interface{}{
 		"key": "value",
-		"key2": map[interface{}]interface{}{
+		"rancher": map[interface{}]interface{}{
-			"subkey": "subvalue",
+			"key2": map[interface{}]interface{}{
-			"subnum": 42,
+				"subkey": "subvalue",
+				"subnum": 42,
+			},
 		},
 	}
 
 	tests := map[string]interface{}{
-		"key":          "value",
+		"key": "value",
-		"key2.subkey":  "subvalue",
+		"rancher.key2.subkey":  "subvalue",
-		"key2.subnum":  42,
+		"rancher.key2.subnum":  42,
-		"key2.subkey2": "",
+		"rancher.key2.subkey2": "",
-		"foo":          "",
+		"foo": "",
 	}
 
 	for k, v := range tests {
-		if getOrSetVal(k, data, nil) != v {
+		assert.Equal(v, getOrSetVal(k, data, nil))
-			t.Fatalf("Expected %v, got %v, for key %s", v, getOrSetVal(k, data, nil), k)
-		}
 	}
 }
 
 func TestSet(t *testing.T) {
+	assert := require.New(t)
+
 	data := map[interface{}]interface{}{
 		"key": "value",
-		"key2": map[interface{}]interface{}{
+		"rancher": map[interface{}]interface{}{
-			"subkey": "subvalue",
+			"key2": map[interface{}]interface{}{
-			"subnum": 42,
+				"subkey": "subvalue",
+				"subnum": 42,
+			},
 		},
 	}
 
 	expected := map[interface{}]interface{}{
 		"key": "value2",
-		"key2": map[interface{}]interface{}{
+		"rancher": map[interface{}]interface{}{
-			"subkey":  "subvalue2",
+			"key2": map[interface{}]interface{}{
-			"subkey2": "value",
+				"subkey":  "subvalue2",
-			"subkey3": 43,
+				"subkey2": "value",
-			"subnum":  42,
+				"subkey3": 43,
-		},
+				"subnum":  42,
-		"key3": map[interface{}]interface{}{
+			},
-			"subkey3": 44,
+			"key3": map[interface{}]interface{}{
+				"subkey3": 44,
+			},
 		},
 		"key4": "value4",
 	}
 
 	tests := map[string]interface{}{
-		"key":          "value2",
+		"key": "value2",
-		"key2.subkey":  "subvalue2",
+		"rancher.key2.subkey":  "subvalue2",
-		"key2.subkey2": "value",
+		"rancher.key2.subkey2": "value",
-		"key2.subkey3": 43,
+		"rancher.key2.subkey3": 43,
-		"key3.subkey3": 44,
+		"rancher.key3.subkey3": 44,
-		"key4":         "value4",
+		"key4":                 "value4",
 	}
 
 	for k, v := range tests {
 		getOrSetVal(k, data, v)
-		if getOrSetVal(k, data, nil) != v {
+		assert.Equal(v, getOrSetVal(k, data, nil))
-			t.Fatalf("Expected %v, got %v, for key %s", v, getOrSetVal(k, data, nil), k)
-		}
 	}
 
-	if !reflect.DeepEqual(data, expected) {
+	assert.Equal(expected, data)
-		t.Fatalf("Expected %v, got %v", expected, data)
-	}
 }
 
 type OuterData struct {
@@ -149,43 +299,35 @@ one:
 }
 
 func TestUserDocker(t *testing.T) {
-	config := &Config{
+	assert := require.New(t)
-		UserDocker: DockerConfig{
+
-			TLS: true,
+	config := &CloudConfig{
+		Rancher: RancherConfig{
+			UserDocker: DockerConfig{
+				TLS: true,
+			},
 		},
 	}
 
 	bytes, err := yaml.Marshal(config)
-	if err != nil {
+	assert.Nil(err)
-		log.Fatal(err)
-	}
 
 	config = NewConfig()
 	err = yaml.Unmarshal(bytes, config)
-	if err != nil {
+	assert.Nil(err)
-		log.Fatal(err)
-	}
 
-	data := make(map[interface{}]interface{})
+	data := make(map[interface{}]map[interface{}]interface{})
 	util.Convert(config, data)
 
 	fmt.Println(data)
 
-	val, ok := data["user_docker"]
+	val, ok := data["rancher"]["user_docker"]
-	if !ok {
+	assert.True(ok)
-		t.Fatal("Failed to find user_docker")
-	}
 
-	if m, ok := val.(map[interface{}]interface{}); ok {
+	m, ok := val.(map[interface{}]interface{})
-		if v, ok := m["tls"]; ok {
+	assert.True(ok)
-			if v != true {
+	v, ok := m["tls"]
-				t.Fatal("user_docker.tls is not true")
+	assert.True(ok)
-			}
+	assert.True(v.(bool))
-		} else {
-			t.Fatal("user_docker.tls is not found")
-		}
-	} else {
-		t.Fatal("Bad data")
-	}
 
 }

config/data_funcs.go

@@ -3,11 +3,60 @@ package config
 import (
 	log "github.com/Sirupsen/logrus"
 
+	"github.com/rancherio/os/util"
 	"regexp"
 	"strconv"
 	"strings"
 )
 
+func filterKey(data map[interface{}]interface{}, key []string) (filtered, rest map[interface{}]interface{}) {
+	if len(key) == 0 {
+		return data, map[interface{}]interface{}{}
+	}
+
+	filtered = map[interface{}]interface{}{}
+	rest = util.MapCopy(data)
+
+	k := key[0]
+	if d, ok := data[k]; ok {
+		switch d := d.(type) {
+
+		case map[interface{}]interface{}:
+			f, r := filterKey(d, key[1:])
+
+			if len(f) != 0 {
+				filtered[k] = f
+			}
+
+			if len(r) != 0 {
+				rest[k] = r
+			} else {
+				delete(rest, k)
+			}
+
+		default:
+			filtered[k] = d
+			delete(rest, k)
+		}
+
+	}
+
+	return
+}
+
+func filterDottedKeys(data map[interface{}]interface{}, keys []string) (filtered, rest map[interface{}]interface{}) {
+	filtered = map[interface{}]interface{}{}
+	rest = util.MapCopy(data)
+
+	for _, key := range keys {
+		f, r := filterKey(data, strings.Split(key, "."))
+		filtered = util.MapsUnion(filtered, f, util.Replace)
+		rest = util.MapsIntersection(rest, r, util.Equal)
+	}
+
+	return
+}
+
 func getOrSetVal(args string, data map[interface{}]interface{}, value interface{}) interface{} {
 	parts := strings.Split(args, ".")
 
@@ -95,7 +144,7 @@ outer:
 		}
 
 		current := result
-		keys := strings.Split(kv[0], ".")[1:]
+		keys := strings.Split(kv[0], ".")
 		for i, key := range keys {
 			if i == len(keys)-1 {
 				current[key] = DummyMarshall(value)

config/default.go

@@ -1,13 +1,14 @@
 package config
 
-func NewConfig() *Config {
+func NewConfig() *CloudConfig {
 	return ReadConfig(OsConfigFile)
 }
 
-func ReadConfig(file string) *Config {
+func ReadConfig(file string) *CloudConfig {
 	if data, err := readConfig(nil, file); err == nil {
-		c := &Config{}
+		c := &CloudConfig{}
 		c.merge(data)
+		c.amendNils()
 		return c
 	} else {
 		return nil

config/disk.go

@@ -18,38 +18,15 @@ func writeToFile(data interface{}, filename string) error {
 }
 
 func saveToDisk(data map[interface{}]interface{}) error {
-	config := make(map[interface{}]interface{})
+	private, config := filterDottedKeys(data, []string{
-	private := make(map[interface{}]interface{})
+		"rancher.ssh",
+		"rancher.user_docker.ca_key",
+		"rancher.user_docker.ca_cert",
+		"rancher.user_docker.server_key",
+		"rancher.user_docker.server_cert",
+	})
 
-	for k, v := range data {
+	err := writeToFile(config, LocalConfigFile)
-		if k == "ssh" {
-			private[k] = v
-		} else if k == "user_docker" {
-			var userDockerConfig DockerConfig
-			var userDockerConfigPrivate DockerConfig
-			err := util.Convert(v, &userDockerConfig)
-			if err != nil {
-				return err
-			}
-
-			userDockerConfigPrivate.CAKey = userDockerConfig.CAKey
-			userDockerConfigPrivate.CACert = userDockerConfig.CACert
-			userDockerConfigPrivate.ServerKey = userDockerConfig.ServerKey
-			userDockerConfigPrivate.ServerCert = userDockerConfig.ServerCert
-
-			userDockerConfig.CAKey = ""
-			userDockerConfig.CACert = ""
-			userDockerConfig.ServerKey = ""
-			userDockerConfig.ServerCert = ""
-
-			config[k] = userDockerConfig
-			private[k] = userDockerConfigPrivate
-		} else {
-			config[k] = v
-		}
-	}
-
-	err := writeToFile(config, ConfigFile)
 	if err != nil {
 		return err
 	}
@@ -57,10 +34,6 @@ func saveToDisk(data map[interface{}]interface{}) error {
 	return writeToFile(private, PrivateConfigFile)
 }
 
-func readSavedConfig(bytes []byte) (map[interface{}]interface{}, error) {
-	return readConfig(bytes, CloudConfigFile, ConfigFile, PrivateConfigFile)
-}
-
 func readConfig(bytes []byte, files ...string) (map[interface{}]interface{}, error) {
 	// You can't just overlay yaml bytes on to maps, it won't merge, but instead
 	// just override the keys and not merge the map values.
@@ -77,7 +50,7 @@ func readConfig(bytes []byte, files ...string) (map[interface{}]interface{}, err
 			return nil, err
 		}
 
-		util.MergeMaps(left, right)
+		left = util.MapsUnion(left, right, util.Replace)
 	}
 
 	if bytes != nil && len(bytes) > 0 {
@@ -86,7 +59,7 @@ func readConfig(bytes []byte, files ...string) (map[interface{}]interface{}, err
 			return nil, err
 		}
 
-		util.MergeMaps(left, right)
+		left = util.MapsUnion(left, right, util.Replace)
 	}
 
 	return left, nil

config/init.go

@@ -6,9 +6,9 @@ import (
 	log "github.com/Sirupsen/logrus"
 )
 
-type InitFunc func(*Config) error
+type InitFunc func(*CloudConfig) error
 
-func RunInitFuncs(cfg *Config, initFuncs []InitFunc) error {
+func RunInitFuncs(cfg *CloudConfig, initFuncs []InitFunc) error {
 	for i, initFunc := range initFuncs {
 		log.Debugf("[%d/%d] Starting", i+1, len(initFuncs))
 		if err := initFunc(cfg); err != nil {

config/types.go

@@ -1,6 +1,9 @@
 package config
 
-import "github.com/rancherio/rancher-compose/librcompose/project"
+import (
+	"github.com/coreos/coreos-cloudinit/config"
+	"github.com/rancherio/rancher-compose/librcompose/project"
+)
 
 const (
 	CONSOLE_CONTAINER  = "console"
@@ -24,14 +27,17 @@ const (
 	RELOAD_CONFIG = "io.rancher.os.reloadconfig"
 	SCOPE         = "io.rancher.os.scope"
 	SYSTEM        = "system"
+
+	OsConfigFile          = "/os-config.yml"
+	CloudConfigFile       = "/var/lib/rancher/conf/cloud-config.yml"
+	CloudConfigScriptFile = "/var/lib/rancher/conf/cloud-config-script"
+	MetaDataFile          = "/var/lib/rancher/conf/metadata"
+	LocalConfigFile       = "/var/lib/rancher/conf/cloud-config-local.yml"
+	PrivateConfigFile     = "/var/lib/rancher/conf/cloud-config-private.yml"
 )
 
 var (
-	VERSION           string
+	VERSION string
-	OsConfigFile      = "/os-config.yml"
-	CloudConfigFile   = "/var/lib/rancher/conf/cloud-config-rancher.yml"
-	ConfigFile        = "/var/lib/rancher/conf/rancher.yml"
-	PrivateConfigFile = "/var/lib/rancher/conf/rancher-private.yml"
 )
 
 type ContainerConfig struct {
@@ -49,7 +55,16 @@ type Repository struct {
 
 type Repositories map[string]Repository
 
-type Config struct {
+type CloudConfig struct {
+	SSHAuthorizedKeys []string      `yaml:"ssh_authorized_keys"`
+	WriteFiles        []config.File `yaml:"write_files"`
+	Hostname          string        `yaml:"hostname"`
+	Users             []config.User `yaml:"users"`
+
+	Rancher RancherConfig `yaml:"rancher,omitempty"`
+}
+
+type RancherConfig struct {
 	Environment         map[string]string                 `yaml:"environment,omitempty"`
 	Services            map[string]*project.ServiceConfig `yaml:"services,omitempty"`
 	BootstrapContainers map[string]*project.ServiceConfig `yaml:"bootstrap,omitempty"`
@@ -65,7 +80,6 @@ type Config struct {
 	Repositories        Repositories                      `yaml:"repositories,omitempty"`
 	Ssh                 SshConfig                         `yaml:"ssh,omitempty"`
 	State               StateConfig                       `yaml:"state,omitempty"`
-	SystemContainers    map[string]*project.ServiceConfig `yaml:"system_containers,omitempty"`
 	SystemDocker        DockerConfig                      `yaml:"system_docker,omitempty"`
 	Upgrade             UpgradeConfig                     `yaml:"upgrade,omitempty"`
 	UserContainers      []ContainerConfig                 `yaml:"user_containers,omitempty"`

docker/container.go

@@ -282,12 +282,10 @@ func (c *Container) addLink(link string) {
 func (c *Container) parseService() {
 	if c.requiresSyslog() {
 		c.addLink("syslog")
-		log.Infof("[%v]: Implicitly linked to 'syslog'", c.Name)
 	}
 
 	if c.requiresUserDocker() {
 		c.addLink("dockerwait")
-		log.Infof("[%v]: Implicitly linked to 'dockerwait'", c.Name)
 	} else if c.ContainerCfg.Service.Image != "" {
 		client, err := NewClient(c.dockerHost)
 		if err != nil {
@@ -298,7 +296,6 @@ func (c *Container) parseService() {
 		i, _ := client.InspectImage(c.ContainerCfg.Service.Image)
 		if i == nil {
 			c.addLink("network")
-			log.Infof("[%v]: Implicitly linked to 'network'", c.Name)
 		}
 	}
 
@@ -538,6 +535,7 @@ func appendVolumesFrom(client *dockerClient.Client, containerCfg *config.Contain
 }
 
 func (c *Container) start(createOnly, wait bool) *Container {
+	log.Debugf("Container: STARTING '%v', createOnly: %v, !detach: %v, wait: %v", c.Name, createOnly, !c.detach, wait)
 	c.Lookup()
 	c.Stage()
 
@@ -619,9 +617,11 @@ func (c *Container) start(createOnly, wait bool) *Container {
 		}
 	}
 
+	log.Debugf("Container: WAIT? '%v' !c.detach && wait: %v", c.Name, !c.detach && wait)
 	if !c.detach && wait {
 		var exitCode int
 		exitCode, c.Err = client.WaitContainer(c.Container.ID)
+		log.Debugf("Container: FINISHED '%v', exitCode: %v", c.Name, exitCode)
 		if exitCode != 0 {
 			c.Err = errors.New(fmt.Sprintf("Container %s exited with code %d", c.Name, exitCode))
 		}

docker/factory.go

@@ -9,7 +9,7 @@ import (
 )
 
 type ContainerFactory struct {
-	cfg *config.Config
+	cfg *config.CloudConfig
 }
 
 type containerBasedService struct {
@@ -18,10 +18,10 @@ type containerBasedService struct {
 	project       *project.Project
 	container     *Container
 	serviceConfig *project.ServiceConfig
-	cfg           *config.Config
+	cfg           *config.CloudConfig
 }
 
-func NewContainerFactory(cfg *config.Config) *ContainerFactory {
+func NewContainerFactory(cfg *config.CloudConfig) *ContainerFactory {
 	return &ContainerFactory{
 		cfg: cfg,
 	}
@@ -34,7 +34,7 @@ func (c *containerBasedService) Up() error {
 	fakeCreate := false
 	create := containerCfg.CreateOnly
 
-	if util.Contains(c.cfg.Disable, c.name) {
+	if util.Contains(c.cfg.Rancher.Disable, c.name) {
 		fakeCreate = true
 	}
 

docker/services.go

@@ -11,7 +11,7 @@ import (
 )
 
 type configEnvironment struct {
-	cfg *config.Config
+	cfg *config.CloudConfig
 }
 
 func appendEnv(array []string, key, value string) []string {
@@ -23,11 +23,11 @@ func appendEnv(array []string, key, value string) []string {
 	return append(array, fmt.Sprintf("%s=%s", key, value))
 }
 
-func lookupKeys(cfg *config.Config, keys ...string) []string {
+func lookupKeys(cfg *config.CloudConfig, keys ...string) []string {
 	for _, key := range keys {
 		if strings.HasSuffix(key, "*") {
 			result := []string{}
-			for envKey, envValue := range cfg.Environment {
+			for envKey, envValue := range cfg.Rancher.Environment {
 				keyPrefix := key[:len(key)-1]
 				if strings.HasPrefix(envKey, keyPrefix) {
 					result = appendEnv(result, envKey, envValue)
@@ -37,7 +37,7 @@ func lookupKeys(cfg *config.Config, keys ...string) []string {
 			if len(result) > 0 {
 				return result
 			}
-		} else if value, ok := cfg.Environment[key]; ok {
+		} else if value, ok := cfg.Rancher.Environment[key]; ok {
 			return appendEnv([]string{}, key, value)
 		}
 	}
@@ -50,7 +50,7 @@ func (c *configEnvironment) Lookup(key, serviceName string, serviceConfig *proje
 	return lookupKeys(c.cfg, fullKey, key)
 }
 
-func RunServices(name string, cfg *config.Config, configs map[string]*project.ServiceConfig) error {
+func RunServices(name string, cfg *config.CloudConfig, configs map[string]*project.ServiceConfig) error {
 	network := false
 	projectEvents := make(chan project.ProjectEvent)
 	p := project.NewProject(name, NewContainerFactory(cfg))
@@ -61,21 +61,26 @@ func RunServices(name string, cfg *config.Config, configs map[string]*project.Se
 	for name, serviceConfig := range configs {
 		if err := p.AddConfig(name, serviceConfig); err != nil {
 			log.Infof("Failed loading service %s", name)
+			continue
 		}
+		enabled[name] = true
 	}
 
 	p.ReloadCallback = func() error {
-		err := cfg.Reload()
+		if p.Name != "system-init" {
-		if err != nil {
+			return nil
+		}
+
+		if err := cfg.Reload(); err != nil {
 			return err
 		}
 
-		for service, serviceEnabled := range cfg.ServicesInclude {
+		for service, serviceEnabled := range cfg.Rancher.ServicesInclude {
 			if !serviceEnabled {
 				continue
 			}
 
-			if _, ok := enabled[service]; ok {
+			if en, ok := enabled[service]; ok && en {
 				continue
 			}
 
@@ -89,8 +94,7 @@ func RunServices(name string, cfg *config.Config, configs map[string]*project.Se
 				continue
 			}
 
-			err = p.Load(bytes)
+			if err := p.Load(bytes); err != nil {
-			if err != nil {
 				log.Errorf("Failed to load %s : %v", service, err)
 				continue
 			}
@@ -98,17 +102,15 @@ func RunServices(name string, cfg *config.Config, configs map[string]*project.Se
 			enabled[service] = true
 		}
 
-		for service, config := range cfg.Services {
+		for service, config := range cfg.Rancher.Services {
-			if _, ok := enabled[service]; ok {
+			if en, ok := enabled[service]; ok && en {
 				continue
 			}
 
-			err = p.AddConfig(service, config)
+			if err := p.AddConfig(service, config); err != nil {
-			if err != nil {
 				log.Errorf("Failed to load %s : %v", service, err)
 				continue
 			}
-
 			enabled[service] = true
 		}
 
@@ -123,14 +125,13 @@ func RunServices(name string, cfg *config.Config, configs map[string]*project.Se
 		}
 	}()
 
-	err := p.ReloadCallback()
+	if err := p.ReloadCallback(); err != nil {
-	if err != nil {
 		log.Errorf("Failed to reload %s : %v", name, err)
 		return err
 	}
 	return p.Up()
 }
 
-func LoadServiceResource(name string, network bool, cfg *config.Config) ([]byte, error) {
+func LoadServiceResource(name string, network bool, cfg *config.CloudConfig) ([]byte, error) {
-	return util.LoadResource(name, network, cfg.Repositories.ToArray())
+	return util.LoadResource(name, network, cfg.Rancher.Repositories.ToArray())
 }

init/bootstrap.go

@@ -14,24 +14,24 @@ import (
 	"strings"
 )
 
-func autoformat(cfg *config.Config) error {
+func autoformat(cfg *config.CloudConfig) error {
-	if len(cfg.State.Autoformat) == 0 || util.ResolveDevice(cfg.State.Dev) != "" {
+	if len(cfg.Rancher.State.Autoformat) == 0 || util.ResolveDevice(cfg.Rancher.State.Dev) != "" {
 		return nil
 	}
-	AUTOFORMAT := "AUTOFORMAT=" + strings.Join(cfg.State.Autoformat, " ")
+	AUTOFORMAT := "AUTOFORMAT=" + strings.Join(cfg.Rancher.State.Autoformat, " ")
-	FORMATZERO := "FORMATZERO=" + fmt.Sprint(cfg.State.FormatZero)
+	FORMATZERO := "FORMATZERO=" + fmt.Sprint(cfg.Rancher.State.FormatZero)
-	cfg.Autoformat["autoformat"].Environment = project.NewMaporEqualSlice([]string{AUTOFORMAT, FORMATZERO})
+	cfg.Rancher.Autoformat["autoformat"].Environment = project.NewMaporEqualSlice([]string{AUTOFORMAT, FORMATZERO})
 	log.Info("Running Autoformat services")
-	err := docker.RunServices("autoformat", cfg, cfg.Autoformat)
+	err := docker.RunServices("autoformat", cfg, cfg.Rancher.Autoformat)
 	return err
 }
 
-func runBootstrapContainers(cfg *config.Config) error {
+func runBootstrapContainers(cfg *config.CloudConfig) error {
 	log.Info("Running Bootstrap services")
-	return docker.RunServices("bootstrap", cfg, cfg.BootstrapContainers)
+	return docker.RunServices("bootstrap", cfg, cfg.Rancher.BootstrapContainers)
 }
 
-func startDocker(cfg *config.Config) (chan interface{}, error) {
+func startDocker(cfg *config.CloudConfig) (chan interface{}, error) {
 	for _, d := range []string{config.DOCKER_SYSTEM_HOST, "/var/run"} {
 		err := os.MkdirAll(d, 0700)
 		if err != nil {
@@ -39,8 +39,8 @@ func startDocker(cfg *config.Config) (chan interface{}, error) {
 		}
 	}
 
-	cmd := exec.Command(cfg.BootstrapDocker.Args[0], cfg.BootstrapDocker.Args[1:]...)
+	cmd := exec.Command(cfg.Rancher.BootstrapDocker.Args[0], cfg.Rancher.BootstrapDocker.Args[1:]...)
-	if cfg.Debug {
+	if cfg.Rancher.Debug {
 		cmd.Stdout = os.Stdout
 		cmd.Stderr = os.Stderr
 	}
@@ -67,7 +67,7 @@ func stopDocker(c chan interface{}) error {
 	return os.RemoveAll(config.DOCKER_SYSTEM_HOME)
 }
 
-func bootstrap(cfg *config.Config) error {
+func bootstrap(cfg *config.CloudConfig) error {
 	log.Info("Launching Bootstrap Docker")
 	c, err := startDocker(cfg)
 	if err != nil {

init/init.go

@@ -72,7 +72,7 @@ var (
 	}
 )
 
-func createSymlinks(cfg *config.Config, symlinks map[string]string) error {
+func createSymlinks(cfg *config.CloudConfig, symlinks map[string]string) error {
 	log.Debug("Creating symlinking")
 	for dest, src := range symlinks {
 		if _, err := os.Stat(dest); os.IsNotExist(err) {
@@ -112,12 +112,12 @@ func createMounts(mounts ...[]string) error {
 	return nil
 }
 
-func remountRo(cfg *config.Config) error {
+func remountRo(cfg *config.CloudConfig) error {
 	log.Info("Remouting root read only")
 	return util.Remount("/", "ro")
 }
 
-func mountCgroups(cfg *config.Config) error {
+func mountCgroups(cfg *config.CloudConfig) error {
 	for _, cgroup := range cgroups {
 		err := createDirs("/sys/fs/cgroup/" + cgroup)
 		if err != nil {
@@ -137,7 +137,7 @@ func mountCgroups(cfg *config.Config) error {
 	return nil
 }
 
-func extractModules(cfg *config.Config) error {
+func extractModules(cfg *config.CloudConfig) error {
 	if _, err := os.Stat(config.MODULES_ARCHIVE); os.IsNotExist(err) {
 		log.Debug("Modules do not exist")
 		return nil
@@ -147,7 +147,7 @@ func extractModules(cfg *config.Config) error {
 	return util.ExtractTar(config.MODULES_ARCHIVE, "/")
 }
 
-func setResolvConf(cfg *config.Config) error {
+func setResolvConf(cfg *config.CloudConfig) error {
 	log.Debug("Creating /etc/resolv.conf")
 	//f, err := os.OpenFile("/etc/resolv.conf", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
 	f, err := os.Create("/etc/resolv.conf")
@@ -157,14 +157,14 @@ func setResolvConf(cfg *config.Config) error {
 
 	defer f.Close()
 
-	for _, dns := range cfg.Network.Dns.Nameservers {
+	for _, dns := range cfg.Rancher.Network.Dns.Nameservers {
 		content := fmt.Sprintf("nameserver %s\n", dns)
 		if _, err = f.Write([]byte(content)); err != nil {
 			return err
 		}
 	}
 
-	search := strings.Join(cfg.Network.Dns.Search, " ")
+	search := strings.Join(cfg.Rancher.Network.Dns.Search, " ")
 	if search != "" {
 		content := fmt.Sprintf("search %s\n", search)
 		if _, err = f.Write([]byte(content)); err != nil {
@@ -172,8 +172,8 @@ func setResolvConf(cfg *config.Config) error {
 		}
 	}
 
-	if cfg.Network.Dns.Domain != "" {
+	if cfg.Rancher.Network.Dns.Domain != "" {
-		content := fmt.Sprintf("domain %s\n", cfg.Network.Dns.Domain)
+		content := fmt.Sprintf("domain %s\n", cfg.Rancher.Network.Dns.Domain)
 		if _, err = f.Write([]byte(content)); err != nil {
 			return err
 		}
@@ -182,7 +182,7 @@ func setResolvConf(cfg *config.Config) error {
 	return nil
 }
 
-func loadModules(cfg *config.Config) error {
+func loadModules(cfg *config.CloudConfig) error {
 	filesystems, err := ioutil.ReadFile("/proc/filesystems")
 	if err != nil {
 		return err
@@ -196,7 +196,7 @@ func loadModules(cfg *config.Config) error {
 		}
 	}
 
-	for _, module := range cfg.Modules {
+	for _, module := range cfg.Rancher.Modules {
 		log.Debugf("Loading module %s", module)
 		err = exec.Command("/sbin/modprobe", module).Run()
 		if err != nil {
@@ -207,7 +207,7 @@ func loadModules(cfg *config.Config) error {
 	return nil
 }
 
-func sysInit(cfg *config.Config) error {
+func sysInit(cfg *config.CloudConfig) error {
 	args := append([]string{SYSINIT}, os.Args[1:]...)
 
 	var cmd *exec.Cmd
@@ -227,9 +227,9 @@ func sysInit(cfg *config.Config) error {
 	return os.Stdin.Close()
 }
 
-func execDocker(cfg *config.Config) error {
+func execDocker(cfg *config.CloudConfig) error {
 	log.Info("Launching System Docker")
-	if !cfg.Debug {
+	if !cfg.Rancher.Debug {
 		output, err := os.Create("/var/log/system-docker.log")
 		if err != nil {
 			return err
@@ -240,7 +240,7 @@ func execDocker(cfg *config.Config) error {
 	}
 
 	os.Stdin.Close()
-	return syscall.Exec(SYSTEM_DOCKER, cfg.SystemDocker.Args, os.Environ())
+	return syscall.Exec(SYSTEM_DOCKER, cfg.Rancher.SystemDocker.Args, os.Environ())
 }
 
 func MainInit() {
@@ -249,24 +249,24 @@ func MainInit() {
 	}
 }
 
-func mountStateTmpfs(cfg *config.Config) error {
+func mountStateTmpfs(cfg *config.CloudConfig) error {
 	log.Debugf("State will not be persisted")
 	return util.Mount("none", STATE, "tmpfs", "")
 }
 
-func mountState(cfg *config.Config) error {
+func mountState(cfg *config.CloudConfig) error {
 	var err error
 
-	if cfg.State.Dev != "" {
+	if cfg.Rancher.State.Dev != "" {
-		dev := util.ResolveDevice(cfg.State.Dev)
+		dev := util.ResolveDevice(cfg.Rancher.State.Dev)
 		if dev == "" {
-			msg := fmt.Sprintf("Could not resolve device %q", cfg.State.Dev)
+			msg := fmt.Sprintf("Could not resolve device %q", cfg.Rancher.State.Dev)
 			log.Infof(msg)
 			return fmt.Errorf(msg)
 		}
 		log.Infof("Mounting state device %s to %s", dev, STATE)
 
-		fsType := cfg.State.FsType
+		fsType := cfg.Rancher.State.FsType
 		if fsType == "auto" {
 			fsType, err = util.GetFsType(dev)
 		}
@@ -282,16 +282,16 @@ func mountState(cfg *config.Config) error {
 	return err
 }
 
-func tryMountAndBootstrap(cfg *config.Config) error {
+func tryMountAndBootstrap(cfg *config.CloudConfig) error {
 	if err := mountState(cfg); err != nil {
 		if err := bootstrap(cfg); err != nil {
-			if cfg.State.Required {
+			if cfg.Rancher.State.Required {
 				return err
 			}
 			return mountStateTmpfs(cfg)
 		}
 		if err := mountState(cfg); err != nil {
-			if cfg.State.Required {
+			if cfg.Rancher.State.Required {
 				return err
 			}
 			return mountStateTmpfs(cfg)
@@ -300,11 +300,11 @@ func tryMountAndBootstrap(cfg *config.Config) error {
 	return nil
 }
 
-func createGroups(cfg *config.Config) error {
+func createGroups(cfg *config.CloudConfig) error {
 	return ioutil.WriteFile("/etc/group", []byte("root:x:0:\n"), 0644)
 }
 
-func touchSocket(cfg *config.Config) error {
+func touchSocket(cfg *config.CloudConfig) error {
 	for _, path := range []string{"/var/run/docker.sock", "/var/run/system-docker.sock"} {
 		if err := syscall.Unlink(path); err != nil && !os.IsNotExist(err) {
 			return err
@@ -318,8 +318,8 @@ func touchSocket(cfg *config.Config) error {
 	return nil
 }
 
-func setupSystemBridge(cfg *config.Config) error {
+func setupSystemBridge(cfg *config.CloudConfig) error {
-	bridge, cidr := cfg.SystemDocker.BridgeConfig()
+	bridge, cidr := cfg.Rancher.SystemDocker.BridgeConfig()
 	if bridge == "" {
 		return nil
 	}
@@ -335,20 +335,20 @@ func setupSystemBridge(cfg *config.Config) error {
 }
 
 func RunInit() error {
-	var cfg config.Config
+	var cfg config.CloudConfig
 
 	os.Setenv("PATH", "/sbin:/usr/sbin:/usr/bin")
 	os.Setenv("DOCKER_RAMDISK", "true")
 
 	initFuncs := []config.InitFunc{
-		func(cfg *config.Config) error {
+		func(cfg *config.CloudConfig) error {
 			return createDirs(dirs...)
 		},
-		func(cfg *config.Config) error {
+		func(cfg *config.CloudConfig) error {
 			log.Info("Setting up mounts")
 			return createMounts(mounts...)
 		},
-		func(cfg *config.Config) error {
+		func(cfg *config.CloudConfig) error {
 			newCfg, err := config.LoadConfig()
 			if err == nil {
 				newCfg, err = config.LoadConfig()
@@ -357,17 +357,15 @@ func RunInit() error {
 				*cfg = *newCfg
 			}
 
-			if cfg.Debug {
+			if cfg.Rancher.Debug {
 				cfgString, _ := config.Dump(false, true)
-				if cfgString != "" {
+				log.Debugf("os-config dump: \n%s", cfgString)
-					log.Debugf("Config: %s", cfgString)
-				}
 			}
 
 			return err
 		},
 		mountCgroups,
-		func(cfg *config.Config) error {
+		func(cfg *config.CloudConfig) error {
 			return createSymlinks(cfg, symlinks)
 		},
 		createGroups,
@@ -376,15 +374,15 @@ func RunInit() error {
 		setResolvConf,
 		setupSystemBridge,
 		tryMountAndBootstrap,
-		func(cfg *config.Config) error {
+		func(cfg *config.CloudConfig) error {
 			return cfg.Reload()
 		},
 		loadModules,
 		setResolvConf,
-		func(cfg *config.Config) error {
+		func(cfg *config.CloudConfig) error {
 			return createDirs(postDirs...)
 		},
-		func(cfg *config.Config) error {
+		func(cfg *config.CloudConfig) error {
 			return createMounts(postMounts...)
 		},
 		touchSocket,

init/sysinit.go

@@ -11,24 +11,6 @@ import (
 	"github.com/rancherio/os/docker"
 )
 
-func importImage(client *dockerClient.Client, name, fileName string) error {
-	file, err := os.Open(fileName)
-	if err != nil {
-		return err
-	}
-
-	defer file.Close()
-
-	log.Debugf("Importing image for %s", fileName)
-	repo, tag := dockerClient.ParseRepositoryTag(name)
-	return client.ImportImage(dockerClient.ImportImageOptions{
-		Source:      "-",
-		Repository:  repo,
-		Tag:         tag,
-		InputStream: file,
-	})
-}
-
 func hasImage(name string) bool {
 	stamp := path.Join(STATE, name)
 	if _, err := os.Stat(stamp); os.IsNotExist(err) {
@@ -37,7 +19,7 @@ func hasImage(name string) bool {
 	return true
 }
 
-func findImages(cfg *config.Config) ([]string, error) {
+func findImages(cfg *config.CloudConfig) ([]string, error) {
 	log.Debugf("Looking for images at %s", config.IMAGES_PATH)
 
 	result := []string{}
@@ -68,7 +50,7 @@ func findImages(cfg *config.Config) ([]string, error) {
 	return result, nil
 }
 
-func loadImages(cfg *config.Config) error {
+func loadImages(cfg *config.CloudConfig) error {
 	images, err := findImages(cfg)
 	if err != nil || len(images) == 0 {
 		return err
@@ -106,12 +88,12 @@ func loadImages(cfg *config.Config) error {
 	return nil
 }
 
-func runContainers(cfg *config.Config) error {
+func runContainers(cfg *config.CloudConfig) error {
-	return docker.RunServices("system-init", cfg, cfg.SystemContainers)
+	return docker.RunServices("system-init", cfg, cfg.Rancher.Services)
 }
 
-func tailConsole(cfg *config.Config) error {
+func tailConsole(cfg *config.CloudConfig) error {
-	if !cfg.Console.Tail {
+	if !cfg.Rancher.Console.Tail {
 		return nil
 	}
 
@@ -120,7 +102,7 @@ func tailConsole(cfg *config.Config) error {
 		return err
 	}
 
-	console, ok := cfg.SystemContainers[config.CONSOLE_CONTAINER]
+	console, ok := cfg.Rancher.Services[config.CONSOLE_CONTAINER]
 	if !ok {
 		log.Error("Console not found")
 		return nil
@@ -151,11 +133,11 @@ func SysInit() error {
 	initFuncs := []config.InitFunc{
 		loadImages,
 		runContainers,
-		func(cfg *config.Config) error {
+		func(cfg *config.CloudConfig) error {
 			syscall.Sync()
 			return nil
 		},
-		func(cfg *config.Config) error {
+		func(cfg *config.CloudConfig) error {
 			log.Infof("RancherOS %s started", config.VERSION)
 			return nil
 		},

os-config.yml

@@ -1,419 +1,310 @@
-bootstrap:
+rancher:
-  udev:
+  bootstrap:
-    image: rancher/os-udev:v0.4.0-dev
+    udev:
-    command: []
+      image: rancher/os-udev:v0.4.0-dev
-    dns: []
+      labels:
-    dns_search: []
+        io.rancher.os.detach: false
-    env_file: []
+        io.rancher.os.scope: system
-    environment: []
+      log_driver: json-file
-    labels:
+      net: host
-      io.rancher.os.detach: false
+      uts: host
-      io.rancher.os.scope: system
+      privileged: true
-    links: []
+      volumes:
-    log_driver: json-file
+      - /dev:/host/dev
-    net: host
+      - /lib/modules:/lib/modules
-    uts: host
+      - /lib/firmware:/lib/firmware
-    privileged: true
-    volumes:
-    - /dev:/host/dev
-    - /lib/modules:/lib/modules
-    - /lib/firmware:/lib/firmware
-autoformat:
   autoformat:
-    image: rancher/os-autoformat:v0.4.0-dev
+    autoformat:
-    command: []
+      image: rancher/os-autoformat:v0.4.0-dev
-    dns: []
+      labels:
-    dns_search: []
+        io.rancher.os.detach: false
-    env_file: []
+        io.rancher.os.scope: system
-    environment: []
+      log_driver: json-file
-    labels:
+      net: none
-      io.rancher.os.detach: false
+      privileged: true
-      io.rancher.os.scope: system
+    udev:
-    links: []
+      image: rancher/os-udev:v0.4.0-dev
-    log_driver: json-file
+      labels:
-    net: none
+        io.rancher.os.detach: false
-    privileged: true
+        io.rancher.os.scope: system
-    volumes: []
+      links:
-  udev:
+      - autoformat
-    image: rancher/os-udev:v0.4.0-dev
+      log_driver: json-file
-    command: []
+      net: host
-    dns: []
+      uts: host
-    dns_search: []
+      privileged: true
-    env_file: []
+      volumes:
-    environment: []
+      - /dev:/host/dev
-    labels:
+      - /lib/modules:/lib/modules
-      io.rancher.os.detach: false
+      - /lib/firmware:/lib/firmware
-      io.rancher.os.scope: system
+  bootstrap_docker:
-    links:
+    args: [docker, -d, -s, overlay, -b, none, --restart=false, -g, /var/lib/system-docker,
-    - autoformat
+      -G, root, -H, 'unix:///var/run/system-docker.sock']
-    log_driver: json-file
+  cloud_init:
-    net: host
+    datasources:
-    uts: host
+    - configdrive:/media/config-2
-    privileged: true
+  services_include: {}
-    volumes:
-    - /dev:/host/dev
-    - /lib/modules:/lib/modules
-    - /lib/firmware:/lib/firmware
-bootstrap_docker:
-  args: [docker, -d, -s, overlay, -b, none, --restart=false, -g, /var/lib/system-docker,
-    -G, root, -H, 'unix:///var/run/system-docker.sock']
-cloud_init:
-  datasources:
-  - configdrive:/media/config-2
-services_include:
-  ubuntu-console: false
-network:
-  dns:
-    nameservers: [8.8.8.8, 8.8.4.4]
-  interfaces:
-    eth*:
-      dhcp: true
-    lo:
-      address: 127.0.0.1/8
-repositories:
-  core:
-    url: https://raw.githubusercontent.com/rancherio/os-services/v0.4.0
-state:
-  fstype: auto
-  dev: LABEL=RANCHER_STATE
-system_containers:
-  acpid:
-    image: rancher/os-acpid:v0.4.0-dev
-    command: []
-    dns: []
-    dns_search: []
-    env_file: []
-    environment: []
-    labels:
-      io.rancher.os.scope: system
-    links: []
-    net: host
-    uts: host
-    privileged: true
-    volumes_from:
-    - command-volumes
-    - system-volumes
-  all-volumes:
-    image: rancher/os-state:v0.4.0-dev
-    command: []
-    dns: []
-    dns_search: []
-    env_file: []
-    environment: []
-    labels:
-      io.rancher.os.createonly: true
-      io.rancher.os.scope: system
-    links: []
-    log_driver: json-file
-    net: none
-    privileged: true
-    read_only: true
-    volumes_from:
-    - docker-volumes
-    - command-volumes
-    - user-volumes
-    - system-volumes
-  cloud-init:
-    image: rancher/os-cloudinit:v0.4.0-dev
-    command: []
-    dns: []
-    dns_search: []
-    env_file: []
-    environment: []
-    labels:
-      io.rancher.os.detach: false
-      io.rancher.os.reloadconfig: true
-      io.rancher.os.scope: system
-    links:
-    - preload-user-images
-    - cloud-init-pre
-    - network
-    net: host
-    uts: host
-    privileged: true
-    volumes_from:
-    - command-volumes
-    - system-volumes
-  cloud-init-pre:
-    image: rancher/os-cloudinit:v0.4.0-dev
-    command: []
-    dns: []
-    dns_search: []
-    env_file: []
-    environment:
-    - CLOUD_INIT_NETWORK=false
-    labels:
-      io.rancher.os.detach: false
-      io.rancher.os.reloadconfig: true
-      io.rancher.os.scope: system
-    links:
-    - preload-system-images
-    net: host
-    uts: host
-    privileged: true
-    volumes_from:
-    - command-volumes
-    - system-volumes
-  command-volumes:
-    image: rancher/os-state:v0.4.0-dev
-    command: []
-    dns: []
-    dns_search: []
-    env_file: []
-    environment: []
-    labels:
-      io.rancher.os.createonly: true
-      io.rancher.os.scope: system
-    links: []
-    log_driver: json-file
-    net: none
-    privileged: true
-    read_only: true
-    volumes:
-    - /init:/sbin/halt:ro
-    - /init:/sbin/poweroff:ro
-    - /init:/sbin/reboot:ro
-    - /init:/sbin/shutdown:ro
-    - /init:/sbin/netconf:ro
-    - /init:/usr/bin/cloud-init:ro
-    - /init:/usr/bin/rancherctl:ro
-    - /init:/usr/bin/ros:ro
-    - /init:/usr/bin/respawn:ro
-    - /init:/usr/bin/system-docker:ro
-    - /init:/usr/sbin/wait-for-docker:ro
-    - /lib/modules:/lib/modules
-    - /usr/bin/docker:/usr/bin/docker:ro
-  console:
-    image: rancher/os-console:v0.4.0-dev
-    command: []
-    dns: []
-    dns_search: []
-    env_file: []
-    environment: []
-    labels:
-      io.rancher.os.remove: true
-      io.rancher.os.scope: system
-    links:
-    - cloud-init
-    net: host
-    uts: host
-    pid: host
-    ipc: host
-    privileged: true
-    restart: always
-    volumes_from:
-    - all-volumes
-  docker:
-    image: rancher/os-docker:v0.4.0-dev
-    command: []
-    dns: []
-    dns_search: []
-    env_file: []
-    environment: []
-    labels:
-      io.rancher.os.scope: system
-    links:
-    - network
-    net: host
-    uts: host
-    pid: host
-    ipc: host
-    privileged: true
-    restart: always
-    volumes_from:
-    - all-volumes
-  docker-volumes:
-    image: rancher/os-state:v0.4.0-dev
-    command: []
-    dns: []
-    dns_search: []
-    env_file: []
-    environment: []
-    labels:
-      io.rancher.os.createonly: true
-      io.rancher.os.scope: system
-    links: []
-    log_driver: json-file
-    net: none
-    privileged: true
-    read_only: true
-    volumes:
-    - /var/lib/rancher/conf:/var/lib/rancher/conf
-    - /var/lib/docker:/var/lib/docker
-    - /var/lib/system-docker:/var/lib/system-docker
-  dockerwait:
-    image: rancher/os-dockerwait:v0.4.0-dev
-    command: []
-    dns: []
-    dns_search: []
-    env_file: []
-    environment: []
-    labels:
-      io.rancher.os.detach: false
-      io.rancher.os.scope: system
-    links:
-    - docker
-    net: host
-    uts: host
-    volumes_from:
-    - all-volumes
   network:
-    image: rancher/os-network:v0.4.0-dev
+    dns:
-    command: []
+      nameservers: [8.8.8.8, 8.8.4.4]
-    dns: []
+    interfaces:
-    dns_search: []
+      eth*:
-    env_file: []
+        dhcp: true
-    environment: []
+      lo:
-    labels:
+        address: 127.0.0.1/8
-      io.rancher.os.detach: false
+  repositories:
-      io.rancher.os.scope: system
+    core:
-    links:
+      url: https://raw.githubusercontent.com/rancherio/os-services/v0.4.0
-    - cloud-init-pre
+  state:
-    net: host
+    fstype: auto
-    uts: host
+    dev: LABEL=RANCHER_STATE
-    privileged: true
+  services:
-    volumes_from:
+    acpid:
-    - command-volumes
+      image: rancher/os-acpid:v0.4.0-dev
-    - system-volumes
+      labels:
-  ntp:
+        io.rancher.os.scope: system
-    image: rancher/os-ntp:v0.4.0-dev
+      net: host
-    command: []
+      uts: host
-    dns: []
+      privileged: true
-    dns_search: []
+      volumes_from:
-    env_file: []
+      - command-volumes
-    environment: []
+      - system-volumes
-    labels:
+    all-volumes:
-      io.rancher.os.scope: system
+      image: rancher/os-state:v0.4.0-dev
-    links:
+      labels:
-    - cloud-init
+        io.rancher.os.createonly: true
-    - network
+        io.rancher.os.scope: system
-    net: host
+      log_driver: json-file
-    uts: host
+      net: none
-    privileged: true
+      privileged: true
-    restart: always
+      read_only: true
-  preload-system-images:
+      volumes_from:
-    image: rancher/os-preload:v0.4.0-dev
+      - docker-volumes
-    command: []
+      - command-volumes
-    dns: []
+      - user-volumes
-    dns_search: []
+      - system-volumes
-    env_file: []
+    cloud-init:
-    environment: []
+      image: rancher/os-cloudinit:v0.4.0-dev
-    labels:
+      labels:
-      io.rancher.os.detach: false
+        io.rancher.os.detach: false
-      io.rancher.os.scope: system
+        io.rancher.os.reloadconfig: true
-    links: []
+        io.rancher.os.scope: system
-    privileged: true
+      links:
-    volumes:
+      - cloud-init-pre
-    - /var/run/system-docker.sock:/var/run/docker.sock
+      - network
-    - /var/lib/system-docker/preload:/mnt/preload
+      net: host
-    volumes_from:
+      uts: host
-    - command-volumes
+      privileged: true
-    - system-volumes
+      volumes_from:
-  preload-user-images:
+      - command-volumes
-    image: rancher/os-preload:v0.4.0-dev
+      - system-volumes
-    command: []
+    cloud-init-pre:
-    dns: []
+      image: rancher/os-cloudinit:v0.4.0-dev
-    dns_search: []
+      environment:
-    env_file: []
+      - CLOUD_INIT_NETWORK=false
-    environment: []
+      labels:
-    labels:
+        io.rancher.os.detach: false
-      io.rancher.os.detach: false
+        io.rancher.os.reloadconfig: true
-      io.rancher.os.scope: system
+        io.rancher.os.scope: system
-    links:
+      links:
-    - dockerwait
+      - preload-system-images
-    privileged: true
+      net: host
-    volumes:
+      uts: host
-    - /var/run/docker.sock:/var/run/docker.sock
+      privileged: true
-    - /var/lib/docker/preload:/mnt/preload
+      volumes_from:
-    volumes_from:
+      - command-volumes
-    - command-volumes
+      - system-volumes
-    - system-volumes
+    command-volumes:
-  syslog:
+      image: rancher/os-state:v0.4.0-dev
-    image: rancher/os-syslog:v0.4.0-dev
+      labels:
-    command: []
+        io.rancher.os.createonly: true
-    dns: []
+        io.rancher.os.scope: system
-    dns_search: []
+      log_driver: json-file
-    env_file: []
+      net: none
-    environment: []
+      privileged: true
-    labels:
+      read_only: true
-      io.rancher.os.scope: system
+      volumes:
-    links: []
+      - /init:/sbin/halt:ro
-    log_driver: json-file
+      - /init:/sbin/poweroff:ro
-    net: host
+      - /init:/sbin/reboot:ro
-    uts: host
+      - /init:/sbin/shutdown:ro
-    privileged: true
+      - /init:/sbin/netconf:ro
-    restart: always
+      - /init:/usr/bin/cloud-init:ro
-    volumes_from:
+      - /init:/usr/bin/rancherctl:ro
-    - system-volumes
+      - /init:/usr/bin/ros:ro
-  system-volumes:
+      - /init:/usr/bin/respawn:ro
-    image: rancher/os-state:v0.4.0-dev
+      - /init:/usr/bin/system-docker:ro
-    command: []
+      - /init:/usr/sbin/wait-for-docker:ro
-    dns: []
+      - /lib/modules:/lib/modules
-    dns_search: []
+      - /usr/bin/docker:/usr/bin/docker:ro
-    env_file: []
+    console:
-    environment: []
+      image: rancher/os-console:v0.4.0-dev
-    labels:
+      labels:
-      io.rancher.os.createonly: true
+        io.rancher.os.remove: true
-      io.rancher.os.scope: system
+        io.rancher.os.scope: system
-    links: []
+      links:
-    log_driver: json-file
+      - cloud-init
-    net: none
+      - dockerwait  # because console runs `loud-init -execute`, which may need docker
-    privileged: true
+      net: host
-    read_only: true
+      uts: host
-    volumes:
+      pid: host
-    - /dev:/host/dev
+      ipc: host
-    - /os-config.yml:/os-config.yml
+      privileged: true
-    - /var/lib/rancher:/var/lib/rancher
+      restart: always
-    - /var/lib/rancher/conf:/var/lib/rancher/conf
+      volumes_from:
-    - /etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt.rancher
+      - all-volumes
-    - /lib/modules:/lib/modules
+    docker:
-    - /lib/firmware:/lib/firmware
+      image: rancher/os-docker:v0.4.0-dev
-    - /var/run:/var/run
+      labels:
-    - /var/log:/var/log
+        io.rancher.os.scope: system
-  udev:
+      links:
-    image: rancher/os-udev:v0.4.0-dev
+      - cloud-init
-    command: []
+      - network
-    dns: []
+      net: host
-    dns_search: []
+      uts: host
-    env_file: []
+      pid: host
-    environment:
+      ipc: host
-    - DAEMON=true
+      privileged: true
-    labels:
+      restart: always
-      io.rancher.os.detach: true
+      volumes_from:
-      io.rancher.os.scope: system
+      - all-volumes
-    links: []
+    docker-volumes:
-    net: host
+      image: rancher/os-state:v0.4.0-dev
-    uts: host
+      labels:
-    privileged: true
+        io.rancher.os.createonly: true
-    restart: always
+        io.rancher.os.scope: system
-    volumes_from:
+      log_driver: json-file
-    - system-volumes
+      net: none
-  user-volumes:
+      privileged: true
-    image: rancher/os-state:v0.4.0-dev
+      read_only: true
-    command: []
+      volumes:
-    dns: []
+      - /var/lib/rancher/conf:/var/lib/rancher/conf
-    dns_search: []
+      - /var/lib/docker:/var/lib/docker
-    env_file: []
+      - /var/lib/system-docker:/var/lib/system-docker
-    environment: []
+    dockerwait:
-    labels:
+      image: rancher/os-dockerwait:v0.4.0-dev
-      io.rancher.os.createonly: true
+      labels:
-      io.rancher.os.scope: system
+        io.rancher.os.detach: false
-    links: []
+        io.rancher.os.scope: system
-    log_driver: json-file
+      links:
-    net: none
+      - docker
-    privileged: true
+      net: host
-    read_only: true
+      uts: host
-    volumes:
+      volumes_from:
-    - /home:/home
+      - all-volumes
-    - /opt:/opt
+    network:
-system_docker:
+      image: rancher/os-network:v0.4.0-dev
-  args: [docker, -d, --log-driver, syslog, -s, overlay, -b, docker-sys, --fixed-cidr,
+      labels:
-    172.18.42.1/16, --restart=false, -g, /var/lib/system-docker, -G, root,
+        io.rancher.os.detach: false
-    -H, 'unix:///var/run/system-docker.sock', --userland-proxy=false]
+        io.rancher.os.scope: system
-upgrade:
+      links:
-  url: https://releases.rancher.com/os/versions.yml
+      - cloud-init-pre
-  image: rancher/os
+      net: host
-user_docker:
+      uts: host
-  tls_args: [--tlsverify, --tlscacert=ca.pem, --tlscert=server-cert.pem, --tlskey=server-key.pem,
+      privileged: true
-    '-H=0.0.0.0:2376']
+      volumes_from:
-  args: [docker, -d, -s, overlay, -G, docker, -H, 'unix:///var/run/docker.sock', --userland-proxy=false]
+      - command-volumes
+      - system-volumes
+    ntp:
+      image: rancher/os-ntp:v0.4.0-dev
+      labels:
+        io.rancher.os.scope: system
+      links:
+      - cloud-init
+      - network
+      net: host
+      uts: host
+      privileged: true
+      restart: always
+    preload-system-images:
+      image: rancher/os-preload:v0.4.0-dev
+      labels:
+        io.rancher.os.detach: false
+        io.rancher.os.scope: system
+      privileged: true
+      volumes:
+      - /var/run/system-docker.sock:/var/run/docker.sock
+      - /var/lib/system-docker/preload:/mnt/preload
+      volumes_from:
+      - command-volumes
+      - system-volumes
+    preload-user-images:
+      image: rancher/os-preload:v0.4.0-dev
+      labels:
+        io.rancher.os.detach: false
+        io.rancher.os.scope: system
+      links:
+      - dockerwait
+      privileged: true
+      volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /var/lib/docker/preload:/mnt/preload
+      volumes_from:
+      - command-volumes
+      - system-volumes
+    syslog:
+      image: rancher/os-syslog:v0.4.0-dev
+      labels:
+        io.rancher.os.scope: system
+      log_driver: json-file
+      net: host
+      uts: host
+      privileged: true
+      restart: always
+      volumes_from:
+      - system-volumes
+    system-volumes:
+      image: rancher/os-state:v0.4.0-dev
+      labels:
+        io.rancher.os.createonly: true
+        io.rancher.os.scope: system
+      log_driver: json-file
+      net: none
+      privileged: true
+      read_only: true
+      volumes:
+      - /dev:/host/dev
+      - /os-config.yml:/os-config.yml
+      - /var/lib/rancher:/var/lib/rancher
+      - /var/lib/rancher/conf:/var/lib/rancher/conf
+      - /etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt.rancher
+      - /lib/modules:/lib/modules
+      - /lib/firmware:/lib/firmware
+      - /var/run:/var/run
+      - /var/log:/var/log
+    udev:
+      image: rancher/os-udev:v0.4.0-dev
+      environment:
+      - DAEMON=true
+      labels:
+        io.rancher.os.detach: true
+        io.rancher.os.scope: system
+      net: host
+      uts: host
+      privileged: true
+      restart: always
+      volumes_from:
+      - system-volumes
+    user-volumes:
+      image: rancher/os-state:v0.4.0-dev
+      labels:
+        io.rancher.os.createonly: true
+        io.rancher.os.scope: system
+      log_driver: json-file
+      net: none
+      privileged: true
+      read_only: true
+      volumes:
+      - /home:/home
+      - /opt:/opt
+  system_docker:
+    args: [docker, -d, --log-driver, syslog, -s, overlay, -b, docker-sys, --fixed-cidr,
+      172.18.42.1/16, --restart=false, -g, /var/lib/system-docker, -G, root,
+      -H, 'unix:///var/run/system-docker.sock', --userland-proxy=false]
+  upgrade:
+    url: https://releases.rancher.com/os/versions.yml
+    image: rancher/os
+  user_docker:
+    tls_args: [--tlsverify, --tlscacert=ca.pem, --tlscert=server-cert.pem, --tlskey=server-key.pem,
+      '-H=0.0.0.0:2376']
+    args: [docker, -d, -s, overlay, -G, docker, -H, 'unix:///var/run/docker.sock', --userland-proxy=false]

util/util.go

@@ -18,6 +18,7 @@ import (
 	log "github.com/Sirupsen/logrus"
 
 	"github.com/docker/docker/pkg/mount"
+	"reflect"
 )
 
 var (
@@ -154,6 +155,7 @@ func RandSeq(n int) string {
 func Convert(from, to interface{}) error {
 	bytes, err := yaml.Marshal(from)
 	if err != nil {
+		log.WithFields(log.Fields{"from": from}).Panicln(err)
 		return err
 	}
 
@@ -164,37 +166,93 @@ func MergeBytes(left, right []byte) ([]byte, error) {
 	leftMap := make(map[interface{}]interface{})
 	rightMap := make(map[interface{}]interface{})
 
-	err := yaml.Unmarshal(left, &leftMap)
+	if err := yaml.Unmarshal(left, &leftMap); err != nil {
-	if err != nil {
 		return nil, err
 	}
 
-	err = yaml.Unmarshal(right, &rightMap)
+	if err := yaml.Unmarshal(right, &rightMap); err != nil {
-	if err != nil {
 		return nil, err
 	}
 
-	MergeMaps(leftMap, rightMap)
+	return yaml.Marshal(MapsUnion(leftMap, rightMap, Replace))
-
-	return yaml.Marshal(leftMap)
 }
 
-func MergeMaps(left, right map[interface{}]interface{}) {
+func Copy(d interface{}) interface{} {
-	for k, v := range right {
+	switch d := d.(type) {
-		merged := false
+	case map[interface{}]interface{}:
-		if existing, ok := left[k]; ok {
+		return MapCopy(d)
-			if rightMap, ok := v.(map[interface{}]interface{}); ok {
+	default:
-				if leftMap, ok := existing.(map[interface{}]interface{}); ok {
+		return d
-					merged = true
+	}
-					MergeMaps(leftMap, rightMap)
+}
+
+func Replace(l, r interface{}) interface{} {
+	return r
+}
+
+func Equal(l, r interface{}) interface{} {
+	if reflect.DeepEqual(l, r) {
+		return l
+	}
+	return nil
+}
+
+func MapsUnion(left, right map[interface{}]interface{}, op func(interface{}, interface{}) interface{}) map[interface{}]interface{} {
+	result := MapCopy(left)
+
+	for k, r := range right {
+		if l, ok := result[k]; ok {
+			switch l := l.(type) {
+			case map[interface{}]interface{}:
+				switch r := r.(type) {
+				case map[interface{}]interface{}:
+					result[k] = MapsUnion(l, r, op)
+				default:
+					result[k] = op(l, r)
+				}
+			default:
+				result[k] = op(l, r)
+			}
+		} else {
+			result[k] = Copy(r)
+		}
+	}
+
+	return result
+}
+
+func MapsIntersection(left, right map[interface{}]interface{}, op func(interface{}, interface{}) interface{}) map[interface{}]interface{} {
+	result := map[interface{}]interface{}{}
+
+	for k, l := range left {
+		if r, ok := right[k]; ok {
+			switch r := r.(type) {
+			case map[interface{}]interface{}:
+				switch l := l.(type) {
+				case map[interface{}]interface{}:
+					result[k] = MapsIntersection(l, r, op)
+				default:
+					if i := op(l, r); i != nil {
+						result[k] = i
+					}
+				}
+			default:
+				if i := op(l, r); i != nil {
+					result[k] = i
 				}
 			}
 		}
-
-		if !merged {
-			left[k] = v
-		}
 	}
+
+	return result
+}
+
+func MapCopy(data map[interface{}]interface{}) map[interface{}]interface{} {
+	result := map[interface{}]interface{}{}
+	for k, v := range data {
+		result[k] = Copy(v)
+	}
+	return result
 }
 
 func GetServices(urls []string) ([]string, error) {