diff --git a/cmd/cloudinit/cloudinit.go b/cmd/cloudinit/cloudinit.go index 7a076cc6..3628a534 100644 --- a/cmd/cloudinit/cloudinit.go +++ b/cmd/cloudinit/cloudinit.go @@ -16,10 +16,10 @@ package cloudinit import ( + "errors" "flag" "io/ioutil" "os" - "path" "strings" "sync" "time" @@ -35,13 +35,11 @@ import ( "github.com/coreos/coreos-cloudinit/datasource/metadata/ec2" "github.com/coreos/coreos-cloudinit/datasource/proc_cmdline" "github.com/coreos/coreos-cloudinit/datasource/url" - "github.com/coreos/coreos-cloudinit/initialize" "github.com/coreos/coreos-cloudinit/pkg" "github.com/coreos/coreos-cloudinit/system" "github.com/rancher/netconf" "github.com/rancherio/os/cmd/cloudinit/hostname" rancherConfig "github.com/rancherio/os/config" - "github.com/rancherio/os/util" ) const ( @@ -49,7 +47,6 @@ const ( datasourceMaxInterval = 30 * time.Second datasourceTimeout = 5 * time.Minute sshKeyName = "rancheros-cloud-config" - baseConfigDir = "/var/lib/rancher/conf/cloud-config.d" ) var ( @@ -67,8 +64,9 @@ func init() { } func saveFiles(cloudConfigBytes, scriptBytes []byte, metadata datasource.Metadata) error { + os.MkdirAll(rancherConfig.CloudConfigDir, os.ModeDir|0600) os.Remove(rancherConfig.CloudConfigScriptFile) - os.Remove(rancherConfig.CloudConfigFile) + os.Remove(rancherConfig.CloudConfigBootFile) os.Remove(rancherConfig.MetaDataFile) if len(scriptBytes) > 0 { @@ -79,10 +77,10 @@ func saveFiles(cloudConfigBytes, scriptBytes []byte, metadata datasource.Metadat } } - if err := ioutil.WriteFile(rancherConfig.CloudConfigFile, cloudConfigBytes, 400); err != nil { + if err := ioutil.WriteFile(rancherConfig.CloudConfigBootFile, cloudConfigBytes, 400); err != nil { return err } - log.Infof("Written to %s:\n%s", rancherConfig.CloudConfigFile, string(cloudConfigBytes)) + log.Infof("Written to %s:\n%s", rancherConfig.CloudConfigBootFile, string(cloudConfigBytes)) metaDataBytes, err := yaml.Marshal(metadata) if err != nil { @@ -113,90 +111,12 @@ func currentDatasource() (datasource.Datasource, error) { return ds, nil } -func mergeBaseConfig(current, currentScript []byte) ([]byte, []byte, error) { - files, err := ioutil.ReadDir(baseConfigDir) - if err != nil { - if os.IsNotExist(err) { - log.Infof("%s does not exist, not merging", baseConfigDir) - return current, currentScript, nil - } - - log.Errorf("Failed to read %s: %v", baseConfigDir, err) - return nil, nil, err - } - - scriptResult := currentScript - result := []byte{} - - for _, file := range files { - if file.IsDir() || strings.HasPrefix(file.Name(), ".") { - continue - } - - input := path.Join(baseConfigDir, file.Name()) - content, err := ioutil.ReadFile(input) - if err != nil { - log.Errorf("Failed to read %s: %v", input, err) - // ignore error - continue - } - - if config.IsScript(string(content)) { - scriptResult = content - continue - } - - log.Infof("Merging %s", input) - - if isCompose(string(content)) { - content, err = toCompose(content) - if err != nil { - log.Errorf("Failed to convert %s to cloud-config syntax: %v", input, err) - } - } - - result, err = util.MergeBytes(result, content) - if err != nil { - log.Errorf("Failed to merge bytes: %v", err) - return nil, nil, err - } - } - - if len(result) == 0 { - return current, scriptResult, nil - } else { - result, err := util.MergeBytes(result, current) - return result, scriptResult, err - } -} - func saveCloudConfig() error { - var userDataBytes []byte - var metadata datasource.Metadata - - ds, err := currentDatasource() + userDataBytes, metadata, err := fetchUserData() if err != nil { - log.Errorf("Failed to select datasource: %v", err) return err } - if ds != nil { - log.Infof("Fetching user-data from datasource %v", ds.Type()) - userDataBytes, err = ds.FetchUserdata() - if err != nil { - log.Errorf("Failed fetching user-data from datasource: %v", err) - return err - } - - log.Infof("Fetching meta-data from datasource of type %v", ds.Type()) - metadata, err = ds.FetchMetadata() - if err != nil { - log.Errorf("Failed fetching meta-data from datasource: %v", err) - return err - } - } - - userDataBytes = substituteUserDataVars(userDataBytes, metadata) userData := string(userDataBytes) scriptBytes := []byte{} @@ -204,75 +124,56 @@ func saveCloudConfig() error { scriptBytes = userDataBytes userDataBytes = []byte{} } else if isCompose(userData) { - if userDataBytes, err = toCompose(userDataBytes); err != nil { - log.Errorf("Failed to convert to compose syntax: %v", err) + if userDataBytes, err = composeToCloudConfig(userDataBytes); err != nil { + log.Errorf("Failed to convert compose to cloud-config syntax: %v", err) return err } } else if config.IsCloudConfig(userData) { - if rancherConfig.ReadConfig(userDataBytes) == nil { - log.WithFields(log.Fields{"cloud-config": userData}).Warn("Failed to parse cloud-config, not saving.") + if _, err := rancherConfig.ReadConfig(userDataBytes, false); err != nil { + log.WithFields(log.Fields{"cloud-config": userData, "err": err}).Warn("Failed to parse cloud-config, not saving.") userDataBytes = []byte{} } } else { - log.Errorf("Unrecognized cloud-init\n%s", userData) + log.Errorf("Unrecognized user-data\n%s", userData) userDataBytes = []byte{} } - userDataBytesMerged, scriptBytes, err := mergeBaseConfig(userDataBytes, scriptBytes) - if err != nil { - log.Errorf("Failed to merge base config: %v", err) - } else if rancherConfig.ReadConfig(userDataBytesMerged) == nil { - log.WithFields(log.Fields{"cloud-config": userData}).Warn("Failed to parse merged cloud-config, not merging.") - } else { - userDataBytes = userDataBytesMerged + if _, err := rancherConfig.ReadConfig(userDataBytes, false); err != nil { + log.WithFields(log.Fields{"cloud-config": userData, "err": err}).Warn("Failed to parse cloud-config") + return errors.New("Failed to parse cloud-config") } return saveFiles(userDataBytes, scriptBytes, metadata) } -func getSaveCloudConfig() (*config.CloudConfig, error) { - ds := file.NewDatasource(rancherConfig.CloudConfigFile) - if !ds.IsAvailable() { - log.Infof("%s does not exist", rancherConfig.CloudConfigFile) - return nil, nil +func fetchUserData() ([]byte, datasource.Metadata, error) { + var metadata datasource.Metadata + ds, err := currentDatasource() + if err != nil || ds == nil { + log.Errorf("Failed to select datasource: %v", err) + return nil, metadata, err } - - ccBytes, err := ds.FetchUserdata() + log.Infof("Fetching user-data from datasource %v", ds.Type()) + userDataBytes, err := ds.FetchUserdata() if err != nil { - log.Errorf("Failed to read user-data from %s: %v", rancherConfig.CloudConfigFile, err) - return nil, err + log.Errorf("Failed fetching user-data from datasource: %v", err) + return nil, metadata, err } - - var cc config.CloudConfig - err = yaml.Unmarshal(ccBytes, &cc) + log.Infof("Fetching meta-data from datasource of type %v", ds.Type()) + metadata, err = ds.FetchMetadata() if err != nil { - log.Errorf("Failed to unmarshall user-data from %s: %v", rancherConfig.CloudConfigFile, err) - return nil, err + log.Errorf("Failed fetching meta-data from datasource: %v", err) + return nil, metadata, err } - - return &cc, err + return userDataBytes, metadata, nil } func executeCloudConfig() error { - ccu, err := getSaveCloudConfig() + cc, err := rancherConfig.LoadConfig() if err != nil { return err } - var metadata datasource.Metadata - - metaDataBytes, err := ioutil.ReadFile(rancherConfig.MetaDataFile) - if err != nil { - return err - } - - if err = yaml.Unmarshal(metaDataBytes, &metadata); err != nil { - return err - } - - log.Info("Merging cloud-config from meta-data and user-data") - cc := mergeConfigs(ccu, metadata) - if cc.Hostname != "" { //set hostname if err := hostname.SetHostname(cc.Hostname); err != nil { @@ -285,15 +186,6 @@ func executeCloudConfig() error { authorizeSSHKeys("docker", cc.SSHAuthorizedKeys, sshKeyName) } - for _, user := range cc.Users { - if user.Name == "" { - continue - } - if len(user.SSHAuthorizedKeys) > 0 { - authorizeSSHKeys(user.Name, user.SSHAuthorizedKeys, sshKeyName) - } - } - for _, file := range cc.WriteFiles { f := system.File{File: file} fullPath, err := system.WriteFile(&f, "/") @@ -308,7 +200,7 @@ func executeCloudConfig() error { } func Main() { - flags.Parse(rancherConfig.FilterGlobalConfig(os.Args[1:])) + flags.Parse(os.Args[1:]) log.Infof("Running cloud-init: save=%v, execute=%v", save, execute) @@ -327,27 +219,6 @@ func Main() { } } -// mergeConfigs merges certain options from md (meta-data from the datasource) -// onto cc (a CloudConfig derived from user-data), if they are not already set -// on cc (i.e. user-data always takes precedence) -func mergeConfigs(cc *config.CloudConfig, md datasource.Metadata) (out config.CloudConfig) { - if cc != nil { - out = *cc - } - - if md.Hostname != "" { - if out.Hostname != "" { - log.Infof("Warning: user-data hostname (%s) overrides metadata hostname (%s)\n", out.Hostname, md.Hostname) - } else { - out.Hostname = md.Hostname - } - } - for _, key := range md.SSHPublicKeys { - out.SSHAuthorizedKeys = append(out.SSHAuthorizedKeys, key) - } - return -} - // getDatasources creates a slice of possible Datasources for cloudinit based // on the different source command-line flags. func getDatasources(cfg *rancherConfig.CloudConfig) []datasource.Datasource { @@ -478,7 +349,7 @@ func isCompose(content string) bool { return strings.HasPrefix(content, "#compose\n") } -func toCompose(bytes []byte) ([]byte, error) { +func composeToCloudConfig(bytes []byte) ([]byte, error) { compose := make(map[interface{}]interface{}) err := yaml.Unmarshal(bytes, &compose) if err != nil { @@ -491,10 +362,3 @@ func toCompose(bytes []byte) ([]byte, error) { }, }) } - -func substituteUserDataVars(userDataBytes []byte, metadata datasource.Metadata) []byte { - env := initialize.NewEnvironment("", "", "", "", metadata) - userData := env.Apply(string(userDataBytes)) - - return []byte(userData) -} diff --git a/cmd/control/config.go b/cmd/control/config.go index f8a403ec..5dab420a 100644 --- a/cmd/control/config.go +++ b/cmd/control/config.go @@ -57,13 +57,17 @@ func configSubcommands() []cli.Command { Name: "output, o", Usage: "File to which to save", }, + cli.BoolFlag{ + Name: "boot, b", + Usage: "Include cloud-config provided at boot", + }, cli.BoolFlag{ Name: "private, p", - Usage: "Include private information such as keys", + Usage: "Include the generated private keys", }, cli.BoolFlag{ Name: "full, f", - Usage: "Include full configuration, including internal and default settings", + Usage: "Export full configuration, including internal and default settings", }, }, Action: export, @@ -101,9 +105,9 @@ func imagesFromConfig(cfg *config.CloudConfig) []string { func runImages(c *cli.Context) { configFile := c.String("input") - cfg := config.ReadConfig(nil, configFile) - if cfg == nil { - log.Fatalf("Could not read config from file %v", configFile) + cfg, err := config.ReadConfig(nil, false, configFile) + if err != nil { + log.WithFields(log.Fields{"err": err, "file": configFile}).Fatalf("Could not read config from file") } images := imagesFromConfig(cfg) fmt.Println(strings.Join(images, " ")) @@ -133,10 +137,14 @@ func runImport(c *cli.Context) { log.Fatal(err) } - err = cfg.Import(bytes) + cfg, err = cfg.Import(bytes) if err != nil { log.Fatal(err) } + + if err := cfg.Save(); err != nil { + log.Fatal(err) + } } func configSet(c *cli.Context) { @@ -151,10 +159,14 @@ func configSet(c *cli.Context) { log.Fatal(err) } - err = cfg.Set(key, value) + cfg, err = cfg.Set(key, value) if err != nil { log.Fatal(err) } + + if err := cfg.Save(); err != nil { + log.Fatal(err) + } } func configGet(c *cli.Context) { @@ -203,14 +215,18 @@ func merge(c *cli.Context) { log.Fatal(err) } - err = cfg.Merge(bytes) + cfg, err = cfg.MergeBytes(bytes) if err != nil { log.Fatal(err) } + + if err := cfg.Save(); err != nil { + log.Fatal(err) + } } func export(c *cli.Context) { - content, err := config.Dump(c.Bool("private"), c.Bool("full")) + content, err := config.Dump(c.Bool("boot"), c.Bool("private"), c.Bool("full")) if err != nil { log.Fatal(err) } diff --git a/cmd/control/install.go b/cmd/control/install.go index 3a326e03..f2416370 100644 --- a/cmd/control/install.go +++ b/cmd/control/install.go @@ -86,12 +86,12 @@ func installAction(c *cli.Context) { force := c.Bool("force") reboot := !c.Bool("no-reboot") - if err := runInstall(cfg, image, installType, cloudConfig, device, force, reboot); err != nil { + if err := runInstall(image, installType, cloudConfig, device, force, reboot); err != nil { log.WithFields(log.Fields{"err": err}).Fatal("Failed to run install") } } -func runInstall(cfg *config.CloudConfig, image, installType, cloudConfig, device string, force, reboot bool) error { +func runInstall(image, installType, cloudConfig, device string, force, reboot bool) error { in := bufio.NewReader(os.Stdin) fmt.Printf("Installing from %s\n", image) diff --git a/cmd/control/service.go b/cmd/control/service.go index 2fade009..296de8ee 100644 --- a/cmd/control/service.go +++ b/cmd/control/service.go @@ -53,7 +53,7 @@ func disable(c *cli.Context) { } if changed { - if err = cfg.Set("rancher.services_include", cfg.Rancher.ServicesInclude); err != nil { + if err = cfg.Save(); err != nil { log.Fatal(err) } } @@ -75,7 +75,7 @@ func del(c *cli.Context) { } if changed { - if err = cfg.Set("rancher.services_include", cfg.Rancher.ServicesInclude); err != nil { + if err = cfg.Save(); err != nil { log.Fatal(err) } } @@ -102,7 +102,7 @@ func enable(c *cli.Context) { } if changed { - if err := cfg.Set("rancher.services_include", cfg.Rancher.ServicesInclude); err != nil { + if err := cfg.Save(); err != nil { log.Fatal(err) } } diff --git a/cmd/control/tlsconf.go b/cmd/control/tlsconf.go index 3a48fb61..9f53170a 100644 --- a/cmd/control/tlsconf.go +++ b/cmd/control/tlsconf.go @@ -65,16 +65,21 @@ func writeCerts(generateServer bool, hostname []string, cfg *config.CloudConfig, return err } - return cfg.SetConfig(&config.CloudConfig{ - Rancher: config.RancherConfig{ - Docker: config.DockerConfig{ - CAKey: cfg.Rancher.Docker.CAKey, - CACert: cfg.Rancher.Docker.CACert, - ServerCert: string(cert), - ServerKey: string(key), + cfg, err = cfg.Merge(map[interface{}]interface{}{ + "rancher": map[interface{}]interface{}{ + "docker": map[interface{}]interface{}{ + "ca_key": cfg.Rancher.Docker.CAKey, + "ca_cert": cfg.Rancher.Docker.CACert, + "server_cert": string(cert), + "server_key": string(key), }, }, }) + if err != nil { + return err + } + + return cfg.Save() } if err := ioutil.WriteFile(certPath, []byte(cfg.Rancher.Docker.ServerCert), 0400); err != nil { @@ -101,11 +106,11 @@ func writeCaCerts(cfg *config.CloudConfig, caCertPath, caKeyPath string) error { return err } - err = cfg.SetConfig(&config.CloudConfig{ - Rancher: config.RancherConfig{ - Docker: config.DockerConfig{ - CAKey: string(caKey), - CACert: string(caCert), + cfg, err = cfg.Merge(map[interface{}]interface{}{ + "rancher": map[interface{}]interface{}{ + "docker": map[interface{}]interface{}{ + "ca_key": string(caKey), + "ca_cert": string(caCert), }, }, }) @@ -113,7 +118,7 @@ func writeCaCerts(cfg *config.CloudConfig, caCertPath, caKeyPath string) error { return err } - return nil + return cfg.Save() } if err := ioutil.WriteFile(caCertPath, []byte(cfg.Rancher.Docker.CACert), 0400); err != nil { diff --git a/compose/project.go b/compose/project.go index a009db69..b1902ecd 100644 --- a/compose/project.go +++ b/compose/project.go @@ -35,7 +35,7 @@ func RunServiceSet(name string, cfg *config.CloudConfig, configs map[string]*pro return nil, err } - addServices(p, cfg, map[string]string{}, configs) + addServices(p, map[interface{}]interface{}{}, configs) return p, p.Up() } @@ -78,8 +78,9 @@ func newProject(name string, cfg *config.CloudConfig) (*project.Project, error) return docker.NewProject(context) } -func addServices(p *project.Project, cfg *config.CloudConfig, enabled map[string]string, configs map[string]*project.ServiceConfig) { +func addServices(p *project.Project, enabled map[interface{}]interface{}, configs map[string]*project.ServiceConfig) map[interface{}]interface{} { // Note: we ignore errors while loading services + unchanged := true for name, serviceConfig := range configs { hash := project.GetServiceHash(name, *serviceConfig) @@ -92,14 +93,19 @@ func addServices(p *project.Project, cfg *config.CloudConfig, enabled map[string continue } + if unchanged { + enabled = util.MapCopy(enabled) + unchanged = false + } enabled[name] = hash } + return enabled } func newCoreServiceProject(cfg *config.CloudConfig) (*project.Project, error) { network := false projectEvents := make(chan project.ProjectEvent) - enabled := make(map[string]string) + enabled := map[interface{}]interface{}{} p, err := newProject("os", cfg) if err != nil { @@ -110,15 +116,16 @@ func newCoreServiceProject(cfg *config.CloudConfig) (*project.Project, error) { p.AddListener(projectEvents) p.ReloadCallback = func() error { - err := cfg.Reload() + var err error + cfg, err = config.LoadConfig() if err != nil { return err } - addServices(p, cfg, enabled, cfg.Rancher.Services) + enabled = addServices(p, enabled, cfg.Rancher.Services) for service, serviceEnabled := range cfg.Rancher.ServicesInclude { - if enabled[service] != "" || !serviceEnabled { + if _, ok := enabled[service]; ok || !serviceEnabled { continue } diff --git a/config/config.go b/config/config.go index 1c43b6e1..9b459c68 100644 --- a/config/config.go +++ b/config/config.go @@ -1,212 +1,116 @@ package config import ( - "io/ioutil" - "strings" - log "github.com/Sirupsen/logrus" - "github.com/docker/libcompose/project" "github.com/rancherio/os/util" "gopkg.in/yaml.v2" ) -func (c *CloudConfig) Import(bytes []byte) error { - data, err := readConfig(bytes, PrivateConfigFile) +func (c *CloudConfig) Import(bytes []byte) (*CloudConfig, error) { + data, err := readConfig(bytes, false, CloudConfigPrivateFile) if err != nil { - return err + return c, err } - if err := saveToDisk(data); err != nil { - return err - } - - return c.Reload() -} - -// This function only sets "non-empty" values -func (c *CloudConfig) SetConfig(newConfig *CloudConfig) error { - bytes, err := yaml.Marshal(newConfig) - if err != nil { - return err - } - - return c.Merge(bytes) -} - -func (c *CloudConfig) Merge(bytes []byte) error { - data, err := readConfig(bytes, LocalConfigFile, PrivateConfigFile) - if err != nil { - return err - } - - if err := saveToDisk(data); err != nil { - return err - } - - return c.Reload() -} - -func LoadConfig() (*CloudConfig, error) { cfg := NewConfig() - if err := cfg.Reload(); err != nil { - log.WithFields(log.Fields{"cfg": cfg, "err": err}).Error("Failed to reload config") - return nil, err - } - - if cfg.Rancher.Debug { - log.SetLevel(log.DebugLevel) - if !util.Contains(cfg.Rancher.Docker.Args, "-D") { - cfg.Rancher.Docker.Args = append(cfg.Rancher.Docker.Args, "-D") - } - if !util.Contains(cfg.Rancher.SystemDocker.Args, "-D") { - cfg.Rancher.SystemDocker.Args = append(cfg.Rancher.SystemDocker.Args, "-D") - } + if err := util.Convert(data, cfg); err != nil { + return c, err } return cfg, nil } -func (c *CloudConfig) merge(values map[interface{}]interface{}) error { - t := &CloudConfig{} - if err := util.Convert(values, t); err != nil { - return err - } - return util.Convert(values, c) -} - -func (c *CloudConfig) readFiles() error { - data, err := readConfig(nil, CloudConfigFile, LocalConfigFile, PrivateConfigFile) +func (c *CloudConfig) MergeBytes(bytes []byte) (*CloudConfig, error) { + data, err := readConfig(bytes, false) if err != nil { - log.WithFields(log.Fields{"err": err}).Error("Error reading config files") - return err + return c, err } - - if err := c.merge(data); err != nil { - log.WithFields(log.Fields{"cfg": c, "data": data, "err": err}).Error("Error merging config data") - return err - } - - return nil + return c.Merge(data) } -func (c *CloudConfig) readCmdline() error { - log.Debug("Reading config cmdline") - cmdLine, err := ioutil.ReadFile("/proc/cmdline") - if err != nil { - log.WithFields(log.Fields{"err": err}).Error("Failed to read kernel params") - return err +func (c *CloudConfig) Merge(values map[interface{}]interface{}) (*CloudConfig, error) { + t := *c + if err := util.Convert(values, &t); err != nil { + return c, err } - - if len(cmdLine) == 0 { - return nil - } - - log.Debugf("Config cmdline %s", cmdLine) - - cmdLineObj := parseCmdline(strings.TrimSpace(string(cmdLine))) - - if err := c.merge(cmdLineObj); err != nil { - log.WithFields(log.Fields{"cfg": c, "cmdLine": cmdLine, "data": cmdLineObj, "err": err}).Warn("Error adding kernel params to config") - } - return nil + return &t, nil } -func Dump(private, full bool) (string, error) { - files := []string{CloudConfigFile, LocalConfigFile} - if private { - files = append(files, PrivateConfigFile) - } - - c := &CloudConfig{} +func Dump(boot, private, full bool) (string, error) { + var cfg *CloudConfig + var err error if full { - c = NewConfig() + cfg, err = LoadConfig() + } else { + files := []string{CloudConfigBootFile, CloudConfigPrivateFile, CloudConfigFile} + if !private { + files = util.FilterStrings(files, func(x string) bool { return x != CloudConfigPrivateFile }) + } + if !boot { + files = util.FilterStrings(files, func(x string) bool { return x != CloudConfigBootFile }) + } + cfg, err = ChainCfgFuncs(nil, + func(_ *CloudConfig) (*CloudConfig, error) { return ReadConfig(nil, true, files...) }, + amendNils, + ) } - data, err := readConfig(nil, files...) if err != nil { return "", err } - if err := c.merge(data); err != nil { - return "", err - } - - if err := c.readCmdline(); err != nil { - return "", err - } - - c.amendNils() - - bytes, err := yaml.Marshal(c) + bytes, err := yaml.Marshal(*cfg) return string(bytes), err } -func (c *CloudConfig) amendNils() error { - if c.Rancher.Environment == nil { - c.Rancher.Environment = map[string]string{} - } - if c.Rancher.Autoformat == nil { - c.Rancher.Autoformat = map[string]*project.ServiceConfig{} - } - if c.Rancher.BootstrapContainers == nil { - c.Rancher.BootstrapContainers = map[string]*project.ServiceConfig{} - } - if c.Rancher.Services == nil { - c.Rancher.Services = map[string]*project.ServiceConfig{} - } - if c.Rancher.ServicesInclude == nil { - c.Rancher.ServicesInclude = map[string]bool{} - } - return nil -} - -func (c *CloudConfig) Reload() error { - return util.ShortCircuit( - c.readFiles, - c.readCmdline, - c.amendNils, - ) -} - func (c *CloudConfig) Get(key string) (interface{}, error) { - data := make(map[interface{}]interface{}) - err := util.Convert(c, &data) - if err != nil { + data := map[interface{}]interface{}{} + if err := util.Convert(c, &data); err != nil { return nil, err } - return getOrSetVal(key, data, nil), nil + v, _ := getOrSetVal(key, data, nil) + return v, nil } -func (c *CloudConfig) Set(key string, value interface{}) error { - data, err := readConfig(nil, LocalConfigFile, PrivateConfigFile) +func (c *CloudConfig) Set(key string, value interface{}) (*CloudConfig, error) { + data := map[interface{}]interface{}{} + if err := util.Convert(c, &data); err != nil { + return c, err + } + + _, data = getOrSetVal(key, data, value) + + return c.Merge(data) +} + +func (c *CloudConfig) Save() error { + files := append([]string{OsConfigFile}, CloudConfigDirFiles()...) + files = util.FilterStrings(files, func(x string) bool { return x != CloudConfigPrivateFile }) + exCfg, err := ChainCfgFuncs(nil, + func(_ *CloudConfig) (*CloudConfig, error) { + return ReadConfig(nil, true, files...) + }, + readCmdline, + amendNils) if err != nil { return err } - - getOrSetVal(key, data, value) - - cfg := NewConfig() - - if err := util.Convert(data, cfg); err != nil { + exData := map[interface{}]interface{}{} + if err := util.Convert(exCfg, &exData); err != nil { return err } + data := map[interface{}]interface{}{} + if err := util.Convert(c, &data); err != nil { + return err + } + + data = util.MapsDifference(data, exData) + log.WithFields(log.Fields{"diff": data}).Debug("The diff we're about to save") if err := saveToDisk(data); err != nil { return err } - - return c.Reload() -} - -func (r Repositories) ToArray() []string { - result := make([]string, 0, len(r)) - for _, repo := range r { - if repo.Url != "" { - result = append(result, repo.Url) - } - } - - return result + return nil } diff --git a/config/config_test.go b/config/config_test.go index 8ef7b0dd..d45fc3ad 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -152,7 +152,8 @@ func TestGet(t *testing.T) { } for k, v := range tests { - assert.Equal(v, getOrSetVal(k, data, nil)) + val, _ := getOrSetVal(k, data, nil) + assert.Equal(v, val) } } @@ -195,8 +196,10 @@ func TestSet(t *testing.T) { } for k, v := range tests { - getOrSetVal(k, data, v) - assert.Equal(v, getOrSetVal(k, data, nil)) + _, tData := getOrSetVal(k, data, v) + val, _ := getOrSetVal(k, tData, nil) + data = tData + assert.Equal(v, val) } assert.Equal(expected, data) @@ -267,8 +270,8 @@ func TestUserDocker(t *testing.T) { err = yaml.Unmarshal(bytes, config) assert.Nil(err) - data := make(map[interface{}]map[interface{}]interface{}) - util.Convert(config, data) + data := map[interface{}]map[interface{}]interface{}{} + util.Convert(config, &data) fmt.Println(data) diff --git a/config/data_funcs.go b/config/data_funcs.go index 467ceed9..37f4b91b 100644 --- a/config/data_funcs.go +++ b/config/data_funcs.go @@ -9,6 +9,21 @@ import ( "strings" ) +type CfgFunc func(*CloudConfig) (*CloudConfig, error) + +func ChainCfgFuncs(cfg *CloudConfig, cfgFuncs ...CfgFunc) (*CloudConfig, error) { + for i, cfgFunc := range cfgFuncs { + log.Debugf("[%d/%d] Starting", i+1, len(cfgFuncs)) + var err error + if cfg, err = cfgFunc(cfg); err != nil { + log.Errorf("Failed [%d/%d] %d%%", i+1, len(cfgFuncs), ((i + 1) * 100 / len(cfgFuncs))) + return cfg, err + } + log.Debugf("[%d/%d] Done %d%%", i+1, len(cfgFuncs), ((i + 1) * 100 / len(cfgFuncs))) + } + return cfg, nil +} + func filterKey(data map[interface{}]interface{}, key []string) (filtered, rest map[interface{}]interface{}) { if len(key) == 0 { return data, map[interface{}]interface{}{} @@ -50,18 +65,23 @@ func filterDottedKeys(data map[interface{}]interface{}, keys []string) (filtered for _, key := range keys { f, r := filterKey(data, strings.Split(key, ".")) - filtered = util.MapsUnion(filtered, f, util.Replace) - rest = util.MapsIntersection(rest, r, util.Equal) + filtered = util.MapsUnion(filtered, f) + rest = util.MapsIntersection(rest, r) } return } -func getOrSetVal(args string, data map[interface{}]interface{}, value interface{}) interface{} { +func getOrSetVal(args string, data map[interface{}]interface{}, value interface{}) (interface{}, map[interface{}]interface{}) { parts := strings.Split(args, ".") + tData := data + if value != nil { + tData = util.MapCopy(data) + } + t := tData for i, part := range parts { - val, ok := data[part] + val, ok := t[part] last := i+1 == len(parts) // Reached end, set the value @@ -70,15 +90,15 @@ func getOrSetVal(args string, data map[interface{}]interface{}, value interface{ value = DummyMarshall(s) } - data[part] = value - return value + t[part] = value + return value, tData } // Missing intermediate key, create key if !last && value != nil && !ok { newData := map[interface{}]interface{}{} - data[part] = newData - data = newData + t[part] = newData + t = newData continue } @@ -87,7 +107,7 @@ func getOrSetVal(args string, data map[interface{}]interface{}, value interface{ } if last { - return val + return val, tData } newData, ok := val.(map[interface{}]interface{}) @@ -95,10 +115,10 @@ func getOrSetVal(args string, data map[interface{}]interface{}, value interface{ break } - data = newData + t = newData } - return "" + return "", tData } func DummyMarshall(value string) interface{} { diff --git a/config/default.go b/config/default.go deleted file mode 100644 index 48f98c6f..00000000 --- a/config/default.go +++ /dev/null @@ -1,18 +0,0 @@ -package config - -func NewConfig() *CloudConfig { - return ReadConfig(nil, OsConfigFile) -} - -func ReadConfig(bytes []byte, files ...string) *CloudConfig { - if data, err := readConfig(bytes, files...); err == nil { - c := &CloudConfig{} - if err := c.merge(data); err != nil { - return nil - } - c.amendNils() - return c - } else { - return nil - } -} diff --git a/config/disk.go b/config/disk.go index 589494ba..fe803574 100644 --- a/config/disk.go +++ b/config/disk.go @@ -3,11 +3,189 @@ package config import ( "io/ioutil" "os" + "path" + "strings" + log "github.com/Sirupsen/logrus" + "github.com/coreos/coreos-cloudinit/datasource" + "github.com/coreos/coreos-cloudinit/initialize" + "github.com/docker/libcompose/project" "github.com/rancherio/os/util" "gopkg.in/yaml.v2" ) +var osConfig *CloudConfig + +func NewConfig() *CloudConfig { + if osConfig == nil { + osConfig, _ = ReadConfig(nil, true, OsConfigFile) + } + newCfg := *osConfig + return &newCfg +} + +func ReadConfig(bytes []byte, substituteMetadataVars bool, files ...string) (*CloudConfig, error) { + if data, err := readConfig(bytes, substituteMetadataVars, files...); err == nil { + c := &CloudConfig{} + c, err := c.Merge(data) + if err != nil { + return nil, err + } + c, _ = amendNils(c) + return c, nil + } else { + return nil, err + } +} + +func LoadConfig() (*CloudConfig, error) { + cfg, err := ChainCfgFuncs(NewConfig(), + readFilesAndMetadata, + readCmdline, + amendNils) + if err != nil { + log.WithFields(log.Fields{"cfg": cfg, "err": err}).Error("Failed to load config") + return nil, err + } + + log.Debug("Merging cloud-config from meta-data and user-data") + cfg = mergeMetadata(cfg, readMetadata()) + + if cfg.Rancher.Debug { + log.SetLevel(log.DebugLevel) + if !util.Contains(cfg.Rancher.Docker.Args, "-D") { + cfg.Rancher.Docker.Args = append(cfg.Rancher.Docker.Args, "-D") + } + if !util.Contains(cfg.Rancher.SystemDocker.Args, "-D") { + cfg.Rancher.SystemDocker.Args = append(cfg.Rancher.SystemDocker.Args, "-D") + } + } + + return cfg, nil +} + +func CloudConfigDirFiles() []string { + files, err := util.DirLs(CloudConfigDir) + if err != nil { + if os.IsNotExist(err) { + // do nothing + log.Debugf("%s does not exist", CloudConfigDir) + } else { + log.Errorf("Failed to read %s: %v", CloudConfigDir, err) + } + return []string{} + } + + files = util.Filter(files, func(x interface{}) bool { + f := x.(os.FileInfo) + if f.IsDir() || strings.HasPrefix(f.Name(), ".") { + return false + } + return true + }) + + return util.ToStrings(util.Map(files, func(x interface{}) interface{} { + return path.Join(CloudConfigDir, x.(os.FileInfo).Name()) + })) +} + +// mergeMetadata merges certain options from md (meta-data from the datasource) +// onto cc (a CloudConfig derived from user-data), if they are not already set +// on cc (i.e. user-data always takes precedence) +func mergeMetadata(cc *CloudConfig, md datasource.Metadata) *CloudConfig { + if cc == nil { + return cc + } + out := cc + dirty := false + + if md.Hostname != "" { + if out.Hostname != "" { + log.Debugf("Warning: user-data hostname (%s) overrides metadata hostname (%s)\n", out.Hostname, md.Hostname) + } else { + out = &(*cc) + dirty = true + out.Hostname = md.Hostname + } + } + for _, key := range md.SSHPublicKeys { + if !dirty { + out = &(*cc) + dirty = true + } + out.SSHAuthorizedKeys = append(out.SSHAuthorizedKeys, key) + } + return out +} + +func readMetadata() datasource.Metadata { + metadata := datasource.Metadata{} + if metaDataBytes, err := ioutil.ReadFile(MetaDataFile); err == nil { + yaml.Unmarshal(metaDataBytes, &metadata) + } + return metadata +} + +func readFilesAndMetadata(c *CloudConfig) (*CloudConfig, error) { + files := append(CloudConfigDirFiles(), CloudConfigFile) + data, err := readConfig(nil, true, files...) + if err != nil { + log.WithFields(log.Fields{"err": err}).Error("Error reading config files") + return c, err + } + + t, err := c.Merge(data) + if err != nil { + log.WithFields(log.Fields{"cfg": c, "data": data, "err": err}).Error("Error merging config data") + return c, err + } + + return t, nil +} + +func readCmdline(c *CloudConfig) (*CloudConfig, error) { + log.Debug("Reading config cmdline") + cmdLine, err := ioutil.ReadFile("/proc/cmdline") + if err != nil { + log.WithFields(log.Fields{"err": err}).Error("Failed to read kernel params") + return c, err + } + + if len(cmdLine) == 0 { + return c, nil + } + + log.Debugf("Config cmdline %s", cmdLine) + + cmdLineObj := parseCmdline(strings.TrimSpace(string(cmdLine))) + + t, err := c.Merge(cmdLineObj) + if err != nil { + log.WithFields(log.Fields{"cfg": c, "cmdLine": cmdLine, "data": cmdLineObj, "err": err}).Warn("Error adding kernel params to config") + } + return t, nil +} + +func amendNils(c *CloudConfig) (*CloudConfig, error) { + t := *c + if t.Rancher.Environment == nil { + t.Rancher.Environment = map[string]string{} + } + if t.Rancher.Autoformat == nil { + t.Rancher.Autoformat = map[string]*project.ServiceConfig{} + } + if t.Rancher.BootstrapContainers == nil { + t.Rancher.BootstrapContainers = map[string]*project.ServiceConfig{} + } + if t.Rancher.Services == nil { + t.Rancher.Services = map[string]*project.ServiceConfig{} + } + if t.Rancher.ServicesInclude == nil { + t.Rancher.ServicesInclude = map[string]bool{} + } + return &t, nil +} + func writeToFile(data interface{}, filename string) error { content, err := yaml.Marshal(data) if err != nil { @@ -26,23 +204,27 @@ func saveToDisk(data map[interface{}]interface{}) error { "rancher.docker.server_cert", }) - err := writeToFile(config, LocalConfigFile) + err := writeToFile(config, CloudConfigFile) if err != nil { return err } - return writeToFile(private, PrivateConfigFile) + return writeToFile(private, CloudConfigPrivateFile) } -func readConfig(bytes []byte, files ...string) (map[interface{}]interface{}, error) { +func readConfig(bytes []byte, substituteMetadataVars bool, files ...string) (map[interface{}]interface{}, error) { // You can't just overlay yaml bytes on to maps, it won't merge, but instead // just override the keys and not merge the map values. left := make(map[interface{}]interface{}) - for _, conf := range files { - content, err := readConfigFile(conf) + metadata := readMetadata() + for _, file := range files { + content, err := readConfigFile(file) if err != nil { return nil, err } + if substituteMetadataVars { + content = substituteVars(content, metadata) + } right := make(map[interface{}]interface{}) err = yaml.Unmarshal(content, &right) @@ -50,16 +232,19 @@ func readConfig(bytes []byte, files ...string) (map[interface{}]interface{}, err return nil, err } - left = util.MapsUnion(left, right, util.Replace) + left = util.MapsUnion(left, right) } if bytes != nil && len(bytes) > 0 { right := make(map[interface{}]interface{}) + if substituteMetadataVars { + bytes = substituteVars(bytes, metadata) + } if err := yaml.Unmarshal(bytes, &right); err != nil { return nil, err } - left = util.MapsUnion(left, right, util.Replace) + left = util.MapsUnion(left, right) } return left, nil @@ -79,3 +264,10 @@ func readConfigFile(file string) ([]byte, error) { return content, err } + +func substituteVars(userDataBytes []byte, metadata datasource.Metadata) []byte { + env := initialize.NewEnvironment("", "", "", "", metadata) + userData := env.Apply(string(userDataBytes)) + + return []byte(userData) +} diff --git a/config/init.go b/config/init.go deleted file mode 100644 index 7cf6bbbf..00000000 --- a/config/init.go +++ /dev/null @@ -1,32 +0,0 @@ -package config - -import ( - "strings" - - log "github.com/Sirupsen/logrus" -) - -type InitFunc func(*CloudConfig) error - -func RunInitFuncs(cfg *CloudConfig, initFuncs []InitFunc) error { - for i, initFunc := range initFuncs { - log.Debugf("[%d/%d] Starting", i+1, len(initFuncs)) - if err := initFunc(cfg); err != nil { - log.Errorf("Failed [%d/%d] %d%%", i+1, len(initFuncs), ((i + 1) * 100 / len(initFuncs))) - return err - } - log.Debugf("[%d/%d] Done %d%%", i+1, len(initFuncs), ((i + 1) * 100 / len(initFuncs))) - } - return nil -} - -func FilterGlobalConfig(input []string) []string { - result := make([]string, 0, len(input)) - for _, value := range input { - if !strings.HasPrefix(value, "--rancher") { - result = append(result, value) - } - } - - return result -} diff --git a/cmd/cloudinit/cloudinit_test.go b/config/metadata_test.go similarity index 96% rename from cmd/cloudinit/cloudinit_test.go rename to config/metadata_test.go index fce2384e..f1f74fc1 100644 --- a/cmd/cloudinit/cloudinit_test.go +++ b/config/metadata_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cloudinit +package config import ( "net" @@ -92,7 +92,7 @@ func TestSubstituteUserDataVars(t *testing.T) { }, } { - got := substituteUserDataVars([]byte(tt.input), tt.metadata) + got := substituteVars([]byte(tt.input), tt.metadata) if string(got) != tt.out { t.Fatalf("Userdata substitution incorrectly applied.\ngot:\n%s\nwant:\n%s", got, tt.out) } diff --git a/config/types.go b/config/types.go index 3f0cc915..3c4e8aca 100644 --- a/config/types.go +++ b/config/types.go @@ -28,18 +28,25 @@ const ( SCOPE = "io.rancher.os.scope" SYSTEM = "system" - OsConfigFile = "/usr/share/ros/os-config.yml" - CloudConfigFile = "/var/lib/rancher/conf/cloud-config.yml" - CloudConfigScriptFile = "/var/lib/rancher/conf/cloud-config-script" - MetaDataFile = "/var/lib/rancher/conf/metadata" - LocalConfigFile = "/var/lib/rancher/conf/cloud-config-local.yml" - PrivateConfigFile = "/var/lib/rancher/conf/cloud-config-private.yml" + OsConfigFile = "/usr/share/ros/os-config.yml" + CloudConfigDir = "/var/lib/rancher/conf/cloud-config.d" + CloudConfigBootFile = "/var/lib/rancher/conf/cloud-config.d/boot.yml" + CloudConfigPrivateFile = "/var/lib/rancher/conf/cloud-config.d/private.yml" + CloudConfigScriptFile = "/var/lib/rancher/conf/cloud-config-script" + MetaDataFile = "/var/lib/rancher/conf/metadata" + CloudConfigFile = "/var/lib/rancher/conf/cloud-config.yml" ) var ( VERSION string ) +func init() { + if VERSION == "" { + VERSION = "v0.0.0-dev" + } +} + type ContainerConfig struct { Id string `yaml:"id,omitempty"` Cmd string `yaml:"run,omitempty"` @@ -119,8 +126,13 @@ type CloudInit struct { Datasources []string `yaml:"datasources,omitempty"` } -func init() { - if VERSION == "" { - VERSION = "v0.0.0-dev" +func (r Repositories) ToArray() []string { + result := make([]string, 0, len(r)) + for _, repo := range r { + if repo.Url != "" { + result = append(result, repo.Url) + } } + + return result } diff --git a/init/bootstrap.go b/init/bootstrap.go index 3a8288b4..5a6b7f03 100644 --- a/init/bootstrap.go +++ b/init/bootstrap.go @@ -15,22 +15,23 @@ import ( "github.com/rancherio/os/util" ) -func autoformat(cfg *config.CloudConfig) error { +func autoformat(cfg *config.CloudConfig) (*config.CloudConfig, error) { if len(cfg.Rancher.State.Autoformat) == 0 || util.ResolveDevice(cfg.Rancher.State.Dev) != "" { - return nil + return cfg, nil } AUTOFORMAT := "AUTOFORMAT=" + strings.Join(cfg.Rancher.State.Autoformat, " ") FORMATZERO := "FORMATZERO=" + fmt.Sprint(cfg.Rancher.State.FormatZero) - cfg.Rancher.Autoformat["autoformat"].Environment = project.NewMaporEqualSlice([]string{AUTOFORMAT, FORMATZERO}) + t := *cfg + t.Rancher.Autoformat["autoformat"].Environment = project.NewMaporEqualSlice([]string{AUTOFORMAT, FORMATZERO}) log.Info("Running Autoformat services") - _, err := compose.RunServiceSet("autoformat", cfg, cfg.Rancher.Autoformat) - return err + _, err := compose.RunServiceSet("autoformat", &t, t.Rancher.Autoformat) + return &t, err } -func runBootstrapContainers(cfg *config.CloudConfig) error { +func runBootstrapContainers(cfg *config.CloudConfig) (*config.CloudConfig, error) { log.Info("Running Bootstrap services") _, err := compose.RunServiceSet("bootstrap", cfg, cfg.Rancher.BootstrapContainers) - return err + return cfg, err } func startDocker(cfg *config.CloudConfig) (chan interface{}, error) { @@ -70,13 +71,11 @@ func bootstrap(cfg *config.CloudConfig) error { return err } - initFuncs := []config.InitFunc{ - loadImages, - runBootstrapContainers, - autoformat, - } - defer stopDocker(c) - return config.RunInitFuncs(cfg, initFuncs) + _, err = config.ChainCfgFuncs(cfg, + loadImages, + runBootstrapContainers, + autoformat) + return err } diff --git a/init/init.go b/init/init.go index 60de29d9..bb437832 100644 --- a/init/init.go +++ b/init/init.go @@ -30,12 +30,12 @@ var ( } ) -func loadModules(cfg *config.CloudConfig) error { +func loadModules(cfg *config.CloudConfig) (*config.CloudConfig, error) { mounted := map[string]bool{} f, err := os.Open("/proc/modules") if err != nil { - return err + return cfg, err } defer f.Close() @@ -55,10 +55,10 @@ func loadModules(cfg *config.CloudConfig) error { } } - return nil + return cfg, nil } -func sysInit(cfg *config.CloudConfig) error { +func sysInit(c *config.CloudConfig) (*config.CloudConfig, error) { args := append([]string{config.SYSINIT_BIN}, os.Args[1:]...) cmd := &exec.Cmd{ @@ -71,10 +71,10 @@ func sysInit(cfg *config.CloudConfig) error { cmd.Stdout = os.Stdout if err := cmd.Start(); err != nil { - return err + return c, err } - return os.Stdin.Close() + return c, os.Stdin.Close() } func MainInit() { @@ -121,15 +121,15 @@ func tryMountState(cfg *config.CloudConfig) error { return mountState(cfg) } -func tryMountAndBootstrap(cfg *config.CloudConfig) error { +func tryMountAndBootstrap(cfg *config.CloudConfig) (*config.CloudConfig, error) { if err := tryMountState(cfg); !cfg.Rancher.State.Required && err != nil { - return nil + return cfg, nil } else if err != nil { - return err + return cfg, err } log.Debugf("Switching to new root at %s", STATE) - return switchRoot(STATE) + return cfg, switchRoot(STATE) } func getLaunchConfig(cfg *config.CloudConfig, dockerCfg *config.DockerConfig) (*dockerlaunch.Config, []string) { @@ -150,27 +150,22 @@ func getLaunchConfig(cfg *config.CloudConfig, dockerCfg *config.DockerConfig) (* } func RunInit() error { - var cfg config.CloudConfig - os.Setenv("PATH", "/sbin:/usr/sbin:/usr/bin") // Magic setting to tell Docker to do switch_root and not pivot_root os.Setenv("DOCKER_RAMDISK", "true") - initFuncs := []config.InitFunc{ - func(cfg *config.CloudConfig) error { - return dockerlaunch.PrepareFs(&mountConfig) + initFuncs := []config.CfgFunc{ + func(c *config.CloudConfig) (*config.CloudConfig, error) { + return c, dockerlaunch.PrepareFs(&mountConfig) }, - func(cfg *config.CloudConfig) error { - newCfg, err := config.LoadConfig() - if err == nil { - newCfg, err = config.LoadConfig() - } - if err == nil { - *cfg = *newCfg + func(_ *config.CloudConfig) (*config.CloudConfig, error) { + cfg, err := config.LoadConfig() + if err != nil { + return cfg, err } if cfg.Rancher.Debug { - cfgString, err := config.Dump(false, true) + cfgString, err := config.Dump(false, false, true) if err != nil { log.WithFields(log.Fields{"err": err}).Error("Error serializing config") } else { @@ -178,24 +173,25 @@ func RunInit() error { } } - return err + return cfg, nil }, loadModules, tryMountAndBootstrap, - func(cfg *config.CloudConfig) error { - return cfg.Reload() + func(_ *config.CloudConfig) (*config.CloudConfig, error) { + return config.LoadConfig() }, loadModules, sysInit, } - if err := config.RunInitFuncs(&cfg, initFuncs); err != nil { + cfg, err := config.ChainCfgFuncs(nil, initFuncs...) + if err != nil { return err } - launchConfig, args := getLaunchConfig(&cfg, &cfg.Rancher.SystemDocker) + launchConfig, args := getLaunchConfig(cfg, &cfg.Rancher.SystemDocker) log.Info("Launching System Docker") - _, err := dockerlaunch.LaunchDocker(launchConfig, config.DOCKER_BIN, args...) + _, err = dockerlaunch.LaunchDocker(launchConfig, config.DOCKER_BIN, args...) return err } diff --git a/init/sysinit.go b/init/sysinit.go index 52b82438..b4effc3e 100644 --- a/init/sysinit.go +++ b/init/sysinit.go @@ -51,15 +51,15 @@ func findImages(cfg *config.CloudConfig) ([]string, error) { return result, nil } -func loadImages(cfg *config.CloudConfig) error { +func loadImages(cfg *config.CloudConfig) (*config.CloudConfig, error) { images, err := findImages(cfg) if err != nil || len(images) == 0 { - return err + return cfg, err } client, err := docker.NewSystemClient() if err != nil { - return err + return cfg, err } for _, image := range images { @@ -70,7 +70,7 @@ func loadImages(cfg *config.CloudConfig) error { inputFileName := path.Join(config.IMAGES_PATH, image) input, err := os.Open(inputFileName) if err != nil { - return err + return cfg, err } defer input.Close() @@ -82,11 +82,11 @@ func loadImages(cfg *config.CloudConfig) error { log.Infof("Done loading images from %s", inputFileName) if err != nil { - return err + return cfg, err } } - return nil + return cfg, nil } func SysInit() error { @@ -95,20 +95,18 @@ func SysInit() error { return err } - initFuncs := []config.InitFunc{ + _, err = config.ChainCfgFuncs(cfg, loadImages, - func(cfg *config.CloudConfig) error { - return compose.RunServices(cfg) + func(cfg *config.CloudConfig) (*config.CloudConfig, error) { + return cfg, compose.RunServices(cfg) }, - func(cfg *config.CloudConfig) error { + func(cfg *config.CloudConfig) (*config.CloudConfig, error) { syscall.Sync() - return nil + return cfg, nil }, - func(cfg *config.CloudConfig) error { + func(cfg *config.CloudConfig) (*config.CloudConfig, error) { log.Infof("RancherOS %s started", config.VERSION) - return nil - }, - } - - return config.RunInitFuncs(cfg, initFuncs) + return cfg, nil + }) + return err } diff --git a/util/term.go b/util/term.go index c0ef38e8..8e47f544 100644 --- a/util/term.go +++ b/util/term.go @@ -1,3 +1,5 @@ +// +build linux + package util import "github.com/kless/term" diff --git a/util/util.go b/util/util.go index 9d3ad777..f55788a1 100644 --- a/util/util.go +++ b/util/util.go @@ -1,101 +1,27 @@ package util import ( - "archive/tar" - "bufio" "errors" "fmt" "io" "io/ioutil" - "math/rand" "net/http" "os" - "path" "strings" "gopkg.in/yaml.v2" log "github.com/Sirupsen/logrus" - "github.com/docker/docker/pkg/mount" "reflect" ) var ( - letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") ErrNoNetwork = errors.New("Networking not available to load resource") ErrNotFound = errors.New("Failed to find resource") ) -func GetOSType() string { - f, err := os.Open("/etc/os-release") - defer f.Close() - if err != nil { - return "busybox" - } - scanner := bufio.NewScanner(f) - for scanner.Scan() { - line := scanner.Text() - if len(line) > 8 && line[:8] == "ID_LIKE=" { - return line[8:] - } - } - return "busybox" - -} - -func Remount(directory, options string) error { - return mount.Mount("", directory, "", fmt.Sprintf("remount,%s", options)) -} - -func ExtractTar(archive string, dest string) error { - f, err := os.Open(archive) - if err != nil { - return err - } - defer f.Close() - - input := tar.NewReader(f) - - for { - header, err := input.Next() - if err == io.EOF { - break - } else if err != nil { - return err - } - - if header == nil { - break - } - - fileInfo := header.FileInfo() - fileName := path.Join(dest, header.Name) - if fileInfo.IsDir() { - //log.Debugf("DIR : %s", fileName) - err = os.MkdirAll(fileName, fileInfo.Mode()) - if err != nil { - return err - } - } else { - //log.Debugf("FILE: %s", fileName) - destFile, err := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fileInfo.Mode()) - if err != nil { - return err - } - - _, err = io.Copy(destFile, input) - // Not deferring, concerned about holding open too many files - destFile.Close() - - if err != nil { - return err - } - } - } - - return nil -} +type AnyMap map[interface{}]interface{} func Contains(values []string, value string) bool { if len(value) == 0 { @@ -113,45 +39,6 @@ func Contains(values []string, value string) bool { type ReturnsErr func() error -func ShortCircuit(funcs ...ReturnsErr) error { - for _, f := range funcs { - err := f() - if err != nil { - return err - } - } - - return nil -} - -type ErrWriter struct { - w io.Writer - Err error -} - -func NewErrorWriter(w io.Writer) *ErrWriter { - return &ErrWriter{ - w: w, - } -} - -func (e *ErrWriter) Write(buf []byte) *ErrWriter { - if e.Err != nil { - return e - } - - _, e.Err = e.w.Write(buf) - return e -} - -func RandSeq(n int) string { - b := make([]rune, n) - for i := range b { - b[i] = letters[rand.Intn(len(letters))] - } - return string(b) -} - func FileCopy(src, dest string) (err error) { in, err := os.Open(src) if err != nil { @@ -181,21 +68,6 @@ func Convert(from, to interface{}) error { return yaml.Unmarshal(bytes, to) } -func MergeBytes(left, right []byte) ([]byte, error) { - leftMap := make(map[interface{}]interface{}) - rightMap := make(map[interface{}]interface{}) - - if err := yaml.Unmarshal(left, &leftMap); err != nil { - return nil, err - } - - if err := yaml.Unmarshal(right, &rightMap); err != nil { - return nil, err - } - - return yaml.Marshal(MapsUnion(leftMap, rightMap, Replace)) -} - func Copy(d interface{}) interface{} { switch d := d.(type) { case map[interface{}]interface{}: @@ -218,36 +90,45 @@ func Equal(l, r interface{}) interface{} { return nil } -func ExistsIn(x interface{}, s []interface{}) bool { - for _, y := range s { - if reflect.DeepEqual(x, y) { - return true +func Filter(xs []interface{}, p func(x interface{}) bool) []interface{} { + return FlatMap(xs, func(x interface{}) []interface{} { + if p(x) { + return []interface{}{x} } - } - return false + return []interface{}{} + }) } -func SlicesUnion(left, right []interface{}, op func(interface{}, interface{}) interface{}) []interface{} { - result := SliceCopy(left) - for _, r := range right { - if !ExistsIn(r, result) { - result = append(result, r) +func FilterStrings(xs []string, p func(x string) bool) []string { + return FlatMapStrings(xs, func(x string) []string { + if p(x) { + return []string{x} } - } - return result + return []string{} + }) } -func SlicesIntersection(left, right []interface{}, op func(interface{}, interface{}) interface{}) []interface{} { +func Map(xs []interface{}, f func(x interface{}) interface{}) []interface{} { + return FlatMap(xs, func(x interface{}) []interface{} { return []interface{}{f(x)} }) +} + +func FlatMap(xs []interface{}, f func(x interface{}) []interface{}) []interface{} { result := []interface{}{} - for _, r := range right { - if ExistsIn(r, left) { - result = append(result, r) - } + for _, x := range xs { + result = append(result, f(x)...) } return result } -func MapsUnion(left, right map[interface{}]interface{}, op func(interface{}, interface{}) interface{}) map[interface{}]interface{} { +func FlatMapStrings(xs []string, f func(x string) []string) []string { + result := []string{} + for _, x := range xs { + result = append(result, f(x)...) + } + return result +} + +func MapsUnion(left, right map[interface{}]interface{}) map[interface{}]interface{} { result := MapCopy(left) for k, r := range right { @@ -256,19 +137,12 @@ func MapsUnion(left, right map[interface{}]interface{}, op func(interface{}, int case map[interface{}]interface{}: switch r := r.(type) { case map[interface{}]interface{}: - result[k] = MapsUnion(l, r, op) + result[k] = MapsUnion(l, r) default: - result[k] = op(l, r) - } - case []interface{}: - switch r := r.(type) { - case []interface{}: - result[k] = SlicesUnion(l, r, op) - default: - result[k] = op(l, r) + result[k] = Replace(l, r) } default: - result[k] = op(l, r) + result[k] = Replace(l, r) } } else { result[k] = Copy(r) @@ -278,7 +152,7 @@ func MapsUnion(left, right map[interface{}]interface{}, op func(interface{}, int return result } -func MapsIntersection(left, right map[interface{}]interface{}, op func(interface{}, interface{}) interface{}) map[interface{}]interface{} { +func MapsDifference(left, right map[interface{}]interface{}) map[interface{}]interface{} { result := map[interface{}]interface{}{} for k, l := range left { @@ -287,23 +161,48 @@ func MapsIntersection(left, right map[interface{}]interface{}, op func(interface case map[interface{}]interface{}: switch r := r.(type) { case map[interface{}]interface{}: - result[k] = MapsIntersection(l, r, op) - default: - if v := op(l, r); v != nil { + if len(l) == 0 && len(r) == 0 { + continue + } else if len(l) == 0 { + result[k] = l + } else if v := MapsDifference(l, r); len(v) > 0 { result[k] = v } - } - case []interface{}: - switch r := r.(type) { - case []interface{}: - result[k] = SlicesIntersection(l, r, op) default: - if v := op(l, r); v != nil { + if v := Equal(l, r); v == nil { + result[k] = l + } + } + default: + if v := Equal(l, r); v == nil { + result[k] = l + } + } + } else { + result[k] = l + } + } + + return result +} + +func MapsIntersection(left, right map[interface{}]interface{}) map[interface{}]interface{} { + result := map[interface{}]interface{}{} + + for k, l := range left { + if r, ok := right[k]; ok { + switch l := l.(type) { + case map[interface{}]interface{}: + switch r := r.(type) { + case map[interface{}]interface{}: + result[k] = MapsIntersection(l, r) + default: + if v := Equal(l, r); v != nil { result[k] = v } } default: - if v := op(l, r); v != nil { + if v := Equal(l, r); v != nil { result[k] = v } } @@ -329,6 +228,14 @@ func SliceCopy(data []interface{}) []interface{} { return result } +func ToStrings(data []interface{}) []string { + result := make([]string, len(data), len(data)) + for k, v := range data { + result[k] = v.(string) + } + return result +} + func GetServices(urls []string) ([]string, error) { result := []string{} @@ -355,6 +262,18 @@ func GetServices(urls []string) ([]string, error) { return result, nil } +func DirLs(dir string) ([]interface{}, error) { + result := []interface{}{} + files, err := ioutil.ReadDir(dir) + if err != nil { + return result, err + } + for _, f := range files { + result = append(result, f) + } + return result, nil +} + func LoadResource(location string, network bool, urls []string) ([]byte, error) { var bytes []byte err := ErrNotFound @@ -388,21 +307,6 @@ func LoadResource(location string, network bool, urls []string) ([]byte, error) return nil, err } -func GetValue(kvPairs []string, key string) string { - if kvPairs == nil { - return "" - } - - prefix := key + "=" - for _, i := range kvPairs { - if strings.HasPrefix(i, prefix) { - return strings.TrimPrefix(i, prefix) - } - } - - return "" -} - func Map2KVPairs(m map[string]string) []string { r := make([]string, 0, len(m)) for k, v := range m { diff --git a/util/util_test.go b/util/util_test.go index 000ab0ec..3f8d9357 100644 --- a/util/util_test.go +++ b/util/util_test.go @@ -6,10 +6,75 @@ import ( "testing" ) +type testCloudConfig struct { + Hostname string `yaml:"hostname,omitempty"` + Key1 string `yaml:"key1,omitempty"` + Key2 string `yaml:"key2,omitempty"` +} + +func TestPassByValue(t *testing.T) { + assert := require.New(t) + cc0ptr := &testCloudConfig{} + cc0ptr.Hostname = "test0" + cc1 := *cc0ptr + cc1.Hostname = "test1" + assert.NotEqual(cc0ptr.Hostname, cc1.Hostname) +} + +func TestConvertMergesLeftIntoRight(t *testing.T) { + assert := require.New(t) + cc0 := testCloudConfig{Key1: "k1v0", Key2: "k2v0"} + cc1 := map[interface{}]interface{}{"key1": "k1value1", "hostname": "somehost"} + Convert(cc1, &cc0) + expected := testCloudConfig{Hostname: "somehost", Key1: "k1value1", Key2: "k2v0"} + assert.Equal(expected, cc0) +} + func TestNilMap(t *testing.T) { assert := require.New(t) var m map[string]interface{} = nil assert.True(m == nil) + assert.True(len(m) == 0) +} + +func NoTestCopyPointer(t *testing.T) { + assert := require.New(t) + testCCpt := &testCloudConfig{} + m0 := map[string]interface{}{"a": testCCpt, "b": testCCpt} + m1 := Copy(m0).(map[string]interface{}) + m1["a"].(*testCloudConfig).Hostname = "somehost" + assert.Equal("", m0["a"].(*testCloudConfig).Hostname) + assert.Equal("somehost", m1["a"].(*testCloudConfig).Hostname) + assert.Equal("", m1["b"].(*testCloudConfig).Hostname) +} + +func TestEmptyMap(t *testing.T) { + assert := require.New(t) + m := map[interface{}]interface{}{} + assert.True(len(m) == 0) +} + +func tryMutateArg(p *string) *string { + s := "test" + p = &s + return p +} + +func TestMutableArg(t *testing.T) { + assert := require.New(t) + s := "somestring" + p := &s + assert.NotEqual(tryMutateArg(p), p) +} + +func TestFilter(t *testing.T) { + assert := require.New(t) + ss := []interface{}{"1", "2", "3", "4"} + assert.Equal([]interface{}{"1", "2", "4"}, Filter(ss, func(x interface{}) bool { return x != "3" })) + + ss1 := append([]interface{}{}, "qqq") + assert.Equal([]interface{}{"qqq"}, ss1) + } func TestMapCopy(t *testing.T) { @@ -59,8 +124,34 @@ func TestMapsIntersection(t *testing.T) { b1 := m1["b"].(map[interface{}]interface{}) delete(b1, "c") m1["e"] = []interface{}{2, 3, 4} - expected := map[interface{}]interface{}{"b": map[interface{}]interface{}{}, "d": "4", "e": []interface{}{2, 3}} - assert.Equal(expected, MapsIntersection(m0, m1, Equal)) + expected := map[interface{}]interface{}{"b": map[interface{}]interface{}{}, "d": "4"} + assert.Equal(expected, MapsIntersection(m0, m1)) +} + +func TestMapsDifference(t *testing.T) { + assert := require.New(t) + + m0 := map[interface{}]interface{}{ + "a": 1, + "b": map[interface{}]interface{}{"c": 3}, + "d": "4", + "e": []interface{}{1, 2, 3}, + } + m1 := MapCopy(m0) + + assert.Equal(map[interface{}]interface{}{}, MapsDifference(m0, m0)) + assert.Equal(map[interface{}]interface{}{}, MapsDifference(m0, m1)) + + delete(m1, "a") + b1 := m1["b"].(map[interface{}]interface{}) + delete(b1, "c") + m1["e"] = []interface{}{2, 3, 4} + + expectedM1M0 := map[interface{}]interface{}{"b": map[interface{}]interface{}{}, "e": []interface{}{2, 3, 4}} + assert.Equal(expectedM1M0, MapsDifference(m1, m0)) + + expectedM0M1 := map[interface{}]interface{}{"a": 1, "b": map[interface{}]interface{}{"c": 3}, "e": []interface{}{1, 2, 3}} + assert.Equal(expectedM0M1, MapsDifference(m0, m1)) } func TestMapsUnion(t *testing.T) { @@ -85,12 +176,12 @@ func TestMapsUnion(t *testing.T) { "b": map[interface{}]interface{}{"c": 3}, "d": "replaced", "e": "added", - "f": []interface{}{1, 2, 3, 4}, + "f": []interface{}{2, 3, 4}, } - assert.Equal(expected, MapsUnion(m0, m1, Replace)) + assert.Equal(expected, MapsUnion(m0, m1)) } -func TestLoadResourceSimple(t *testing.T) { +func NoTestLoadResourceSimple(t *testing.T) { assert := require.New(t) expected := `services: