rancher/os
1
0
Fork 1
mirror of https://github.com/rancher/os.git synced 2025-07-13 14:44:03 +00:00
Code Issues Releases Wiki Activity

Merge pull request #580 from imikushin/revamp-config

Browse Source 
reshuffle cloud-config
This commit is contained in:
Ivan Mikushin 2015-10-01 00:09:00 +05:00
commit 510a07cd62
20 changed files with 658 additions and 695 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

200
cmd/cloudinit/cloudinit.go
View File

@ -16,10 +16,10 @@
package cloudinit package cloudinit
import ( import (
"errors"
"flag" "flag"
"io/ioutil" "io/ioutil"
"os" "os"
"path"
"strings" "strings"
"sync" "sync"
"time" "time"
@ -35,13 +35,11 @@ import (
"github.com/coreos/coreos-cloudinit/datasource/metadata/ec2" "github.com/coreos/coreos-cloudinit/datasource/metadata/ec2"
"github.com/coreos/coreos-cloudinit/datasource/proc_cmdline" "github.com/coreos/coreos-cloudinit/datasource/proc_cmdline"
"github.com/coreos/coreos-cloudinit/datasource/url" "github.com/coreos/coreos-cloudinit/datasource/url"
"github.com/coreos/coreos-cloudinit/initialize"
"github.com/coreos/coreos-cloudinit/pkg" "github.com/coreos/coreos-cloudinit/pkg"
"github.com/coreos/coreos-cloudinit/system" "github.com/coreos/coreos-cloudinit/system"
"github.com/rancher/netconf" "github.com/rancher/netconf"
"github.com/rancherio/os/cmd/cloudinit/hostname" "github.com/rancherio/os/cmd/cloudinit/hostname"
rancherConfig "github.com/rancherio/os/config" rancherConfig "github.com/rancherio/os/config"
"github.com/rancherio/os/util"
) )
const ( const (
@ -49,7 +47,6 @@ const (
datasourceMaxInterval = 30 * time.Second datasourceMaxInterval = 30 * time.Second
datasourceTimeout = 5 * time.Minute datasourceTimeout = 5 * time.Minute
sshKeyName = "rancheros-cloud-config" sshKeyName = "rancheros-cloud-config"
baseConfigDir = "/var/lib/rancher/conf/cloud-config.d"
) )
var ( var (
@ -67,8 +64,9 @@ func init() {
} }
func saveFiles(cloudConfigBytes, scriptBytes []byte, metadata datasource.Metadata) error { func saveFiles(cloudConfigBytes, scriptBytes []byte, metadata datasource.Metadata) error {
os.MkdirAll(rancherConfig.CloudConfigDir, os.ModeDir|0600)
os.Remove(rancherConfig.CloudConfigScriptFile) os.Remove(rancherConfig.CloudConfigScriptFile)
os.Remove(rancherConfig.CloudConfigFile) os.Remove(rancherConfig.CloudConfigBootFile)
os.Remove(rancherConfig.MetaDataFile) os.Remove(rancherConfig.MetaDataFile)
if len(scriptBytes) > 0 { if len(scriptBytes) > 0 {
@ -79,10 +77,10 @@ func saveFiles(cloudConfigBytes, scriptBytes []byte, metadata datasource.Metadat
} }
} }
if err := ioutil.WriteFile(rancherConfig.CloudConfigFile, cloudConfigBytes, 400); err != nil { if err := ioutil.WriteFile(rancherConfig.CloudConfigBootFile, cloudConfigBytes, 400); err != nil {
return err return err
} }
log.Infof("Written to %s:\n%s", rancherConfig.CloudConfigFile, string(cloudConfigBytes)) log.Infof("Written to %s:\n%s", rancherConfig.CloudConfigBootFile, string(cloudConfigBytes))
metaDataBytes, err := yaml.Marshal(metadata) metaDataBytes, err := yaml.Marshal(metadata)
if err != nil { if err != nil {
@ -113,90 +111,12 @@ func currentDatasource() (datasource.Datasource, error) {
return ds, nil return ds, nil
} }
func mergeBaseConfig(current, currentScript []byte) ([]byte, []byte, error) {
files, err := ioutil.ReadDir(baseConfigDir)
if err != nil {
if os.IsNotExist(err) {
log.Infof("%s does not exist, not merging", baseConfigDir)
return current, currentScript, nil
}
log.Errorf("Failed to read %s: %v", baseConfigDir, err)
return nil, nil, err
}
scriptResult := currentScript
result := []byte{}
for _, file := range files {
if file.IsDir() || strings.HasPrefix(file.Name(), ".") {
continue
}
input := path.Join(baseConfigDir, file.Name())
content, err := ioutil.ReadFile(input)
if err != nil {
log.Errorf("Failed to read %s: %v", input, err)
// ignore error
continue
}
if config.IsScript(string(content)) {
scriptResult = content
continue
}
log.Infof("Merging %s", input)
if isCompose(string(content)) {
content, err = toCompose(content)
if err != nil {
log.Errorf("Failed to convert %s to cloud-config syntax: %v", input, err)
}
}
result, err = util.MergeBytes(result, content)
if err != nil {
log.Errorf("Failed to merge bytes: %v", err)
return nil, nil, err
}
}
if len(result) == 0 {
return current, scriptResult, nil
} else {
result, err := util.MergeBytes(result, current)
return result, scriptResult, err
}
}
func saveCloudConfig() error { func saveCloudConfig() error {
var userDataBytes []byte userDataBytes, metadata, err := fetchUserData()
var metadata datasource.Metadata
ds, err := currentDatasource()
if err != nil { if err != nil {
log.Errorf("Failed to select datasource: %v", err)
return err return err
} }
if ds != nil {
log.Infof("Fetching user-data from datasource %v", ds.Type())
userDataBytes, err = ds.FetchUserdata()
if err != nil {
log.Errorf("Failed fetching user-data from datasource: %v", err)
return err
}
log.Infof("Fetching meta-data from datasource of type %v", ds.Type())
metadata, err = ds.FetchMetadata()
if err != nil {
log.Errorf("Failed fetching meta-data from datasource: %v", err)
return err
}
}
userDataBytes = substituteUserDataVars(userDataBytes, metadata)
userData := string(userDataBytes) userData := string(userDataBytes)
scriptBytes := []byte{} scriptBytes := []byte{}
@ -204,75 +124,56 @@ func saveCloudConfig() error {
scriptBytes = userDataBytes scriptBytes = userDataBytes
userDataBytes = []byte{} userDataBytes = []byte{}
} else if isCompose(userData) { } else if isCompose(userData) {
if userDataBytes, err = toCompose(userDataBytes); err != nil { if userDataBytes, err = composeToCloudConfig(userDataBytes); err != nil {
log.Errorf("Failed to convert to compose syntax: %v", err) log.Errorf("Failed to convert compose to cloud-config syntax: %v", err)
return err return err
} }
} else if config.IsCloudConfig(userData) { } else if config.IsCloudConfig(userData) {
if rancherConfig.ReadConfig(userDataBytes) == nil { if _, err := rancherConfig.ReadConfig(userDataBytes, false); err != nil {
log.WithFields(log.Fields{"cloud-config": userData}).Warn("Failed to parse cloud-config, not saving.") log.WithFields(log.Fields{"cloud-config": userData, "err": err}).Warn("Failed to parse cloud-config, not saving.")
userDataBytes = []byte{} userDataBytes = []byte{}
} }
} else { } else {
log.Errorf("Unrecognized cloud-init\n%s", userData) log.Errorf("Unrecognized user-data\n%s", userData)
userDataBytes = []byte{} userDataBytes = []byte{}
} }
userDataBytesMerged, scriptBytes, err := mergeBaseConfig(userDataBytes, scriptBytes) if _, err := rancherConfig.ReadConfig(userDataBytes, false); err != nil {
if err != nil { log.WithFields(log.Fields{"cloud-config": userData, "err": err}).Warn("Failed to parse cloud-config")
log.Errorf("Failed to merge base config: %v", err) return errors.New("Failed to parse cloud-config")
} else if rancherConfig.ReadConfig(userDataBytesMerged) == nil {
log.WithFields(log.Fields{"cloud-config": userData}).Warn("Failed to parse merged cloud-config, not merging.")
} else {
userDataBytes = userDataBytesMerged
} }
return saveFiles(userDataBytes, scriptBytes, metadata) return saveFiles(userDataBytes, scriptBytes, metadata)
} }
func getSaveCloudConfig() (*config.CloudConfig, error) { func fetchUserData() ([]byte, datasource.Metadata, error) {
ds := file.NewDatasource(rancherConfig.CloudConfigFile) var metadata datasource.Metadata
if !ds.IsAvailable() { ds, err := currentDatasource()
log.Infof("%s does not exist", rancherConfig.CloudConfigFile) if err != nil || ds == nil {
return nil, nil log.Errorf("Failed to select datasource: %v", err)
return nil, metadata, err
} }
log.Infof("Fetching user-data from datasource %v", ds.Type())
ccBytes, err := ds.FetchUserdata() userDataBytes, err := ds.FetchUserdata()
if err != nil { if err != nil {
log.Errorf("Failed to read user-data from %s: %v", rancherConfig.CloudConfigFile, err) log.Errorf("Failed fetching user-data from datasource: %v", err)
return nil, err return nil, metadata, err
} }
log.Infof("Fetching meta-data from datasource of type %v", ds.Type())
var cc config.CloudConfig metadata, err = ds.FetchMetadata()
err = yaml.Unmarshal(ccBytes, &cc)
if err != nil { if err != nil {
log.Errorf("Failed to unmarshall user-data from %s: %v", rancherConfig.CloudConfigFile, err) log.Errorf("Failed fetching meta-data from datasource: %v", err)
return nil, err return nil, metadata, err
} }
return userDataBytes, metadata, nil
return &cc, err
} }
func executeCloudConfig() error { func executeCloudConfig() error {
ccu, err := getSaveCloudConfig() cc, err := rancherConfig.LoadConfig()
if err != nil { if err != nil {
return err return err
} }
var metadata datasource.Metadata
metaDataBytes, err := ioutil.ReadFile(rancherConfig.MetaDataFile)
if err != nil {
return err
}
if err = yaml.Unmarshal(metaDataBytes, &metadata); err != nil {
return err
}
log.Info("Merging cloud-config from meta-data and user-data")
cc := mergeConfigs(ccu, metadata)
if cc.Hostname != "" { if cc.Hostname != "" {
//set hostname //set hostname
if err := hostname.SetHostname(cc.Hostname); err != nil { if err := hostname.SetHostname(cc.Hostname); err != nil {
@ -285,15 +186,6 @@ func executeCloudConfig() error {
authorizeSSHKeys("docker", cc.SSHAuthorizedKeys, sshKeyName) authorizeSSHKeys("docker", cc.SSHAuthorizedKeys, sshKeyName)
} }
for _, user := range cc.Users {
if user.Name == "" {
continue
}
if len(user.SSHAuthorizedKeys) > 0 {
authorizeSSHKeys(user.Name, user.SSHAuthorizedKeys, sshKeyName)
}
}
for _, file := range cc.WriteFiles { for _, file := range cc.WriteFiles {
f := system.File{File: file} f := system.File{File: file}
fullPath, err := system.WriteFile(&f, "/") fullPath, err := system.WriteFile(&f, "/")
@ -308,7 +200,7 @@ func executeCloudConfig() error {
} }
func Main() { func Main() {
flags.Parse(rancherConfig.FilterGlobalConfig(os.Args[1:])) flags.Parse(os.Args[1:])
log.Infof("Running cloud-init: save=%v, execute=%v", save, execute) log.Infof("Running cloud-init: save=%v, execute=%v", save, execute)
@ -327,27 +219,6 @@ func Main() {
} }
} }
// mergeConfigs merges certain options from md (meta-data from the datasource)
// onto cc (a CloudConfig derived from user-data), if they are not already set
// on cc (i.e. user-data always takes precedence)
func mergeConfigs(cc *config.CloudConfig, md datasource.Metadata) (out config.CloudConfig) {
if cc != nil {
out = *cc
}
if md.Hostname != "" {
if out.Hostname != "" {
log.Infof("Warning: user-data hostname (%s) overrides metadata hostname (%s)\n", out.Hostname, md.Hostname)
} else {
out.Hostname = md.Hostname
}
}
for _, key := range md.SSHPublicKeys {
out.SSHAuthorizedKeys = append(out.SSHAuthorizedKeys, key)
}
return
}
// getDatasources creates a slice of possible Datasources for cloudinit based // getDatasources creates a slice of possible Datasources for cloudinit based
// on the different source command-line flags. // on the different source command-line flags.
func getDatasources(cfg *rancherConfig.CloudConfig) []datasource.Datasource { func getDatasources(cfg *rancherConfig.CloudConfig) []datasource.Datasource {
@ -478,7 +349,7 @@ func isCompose(content string) bool {
return strings.HasPrefix(content, "#compose\n") return strings.HasPrefix(content, "#compose\n")
} }
func toCompose(bytes []byte) ([]byte, error) { func composeToCloudConfig(bytes []byte) ([]byte, error) {
compose := make(map[interface{}]interface{}) compose := make(map[interface{}]interface{})
err := yaml.Unmarshal(bytes, &compose) err := yaml.Unmarshal(bytes, &compose)
if err != nil { if err != nil {
@ -491,10 +362,3 @@ func toCompose(bytes []byte) ([]byte, error) {
}, },
}) })
} }
func substituteUserDataVars(userDataBytes []byte, metadata datasource.Metadata) []byte {
env := initialize.NewEnvironment("", "", "", "", metadata)
userData := env.Apply(string(userDataBytes))
return []byte(userData)
}

34
cmd/control/config.go
View File

@ -57,13 +57,17 @@ func configSubcommands() []cli.Command {
Name: "output, o", Name: "output, o",
Usage: "File to which to save", Usage: "File to which to save",
}, },
cli.BoolFlag{
Name: "boot, b",
Usage: "Include cloud-config provided at boot",
},
cli.BoolFlag{ cli.BoolFlag{
Name: "private, p", Name: "private, p",
Usage: "Include private information such as keys", Usage: "Include the generated private keys",
}, },
cli.BoolFlag{ cli.BoolFlag{
Name: "full, f", Name: "full, f",
Usage: "Include full configuration, including internal and default settings", Usage: "Export full configuration, including internal and default settings",
}, },
}, },
Action: export, Action: export,
@ -101,9 +105,9 @@ func imagesFromConfig(cfg *config.CloudConfig) []string {
func runImages(c *cli.Context) { func runImages(c *cli.Context) {
configFile := c.String("input") configFile := c.String("input")
cfg := config.ReadConfig(nil, configFile) cfg, err := config.ReadConfig(nil, false, configFile)
if cfg == nil { if err != nil {
log.Fatalf("Could not read config from file %v", configFile) log.WithFields(log.Fields{"err": err, "file": configFile}).Fatalf("Could not read config from file")
} }
images := imagesFromConfig(cfg) images := imagesFromConfig(cfg)
fmt.Println(strings.Join(images, " ")) fmt.Println(strings.Join(images, " "))
@ -133,10 +137,14 @@ func runImport(c *cli.Context) {
log.Fatal(err) log.Fatal(err)
} }
err = cfg.Import(bytes) cfg, err = cfg.Import(bytes)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
if err := cfg.Save(); err != nil {
log.Fatal(err)
}
} }
func configSet(c *cli.Context) { func configSet(c *cli.Context) {
@ -151,10 +159,14 @@ func configSet(c *cli.Context) {
log.Fatal(err) log.Fatal(err)
} }
err = cfg.Set(key, value) cfg, err = cfg.Set(key, value)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
if err := cfg.Save(); err != nil {
log.Fatal(err)
}
} }
func configGet(c *cli.Context) { func configGet(c *cli.Context) {
@ -203,14 +215,18 @@ func merge(c *cli.Context) {
log.Fatal(err) log.Fatal(err)
} }
err = cfg.Merge(bytes) cfg, err = cfg.MergeBytes(bytes)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
if err := cfg.Save(); err != nil {
log.Fatal(err)
}
} }
func export(c *cli.Context) { func export(c *cli.Context) {
content, err := config.Dump(c.Bool("private"), c.Bool("full")) content, err := config.Dump(c.Bool("boot"), c.Bool("private"), c.Bool("full"))
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }

4
cmd/control/install.go
View File

@ -86,12 +86,12 @@ func installAction(c *cli.Context) {
force := c.Bool("force") force := c.Bool("force")
reboot := !c.Bool("no-reboot") reboot := !c.Bool("no-reboot")
if err := runInstall(cfg, image, installType, cloudConfig, device, force, reboot); err != nil { if err := runInstall(image, installType, cloudConfig, device, force, reboot); err != nil {
log.WithFields(log.Fields{"err": err}).Fatal("Failed to run install") log.WithFields(log.Fields{"err": err}).Fatal("Failed to run install")
} }
} }
func runInstall(cfg *config.CloudConfig, image, installType, cloudConfig, device string, force, reboot bool) error { func runInstall(image, installType, cloudConfig, device string, force, reboot bool) error {
in := bufio.NewReader(os.Stdin) in := bufio.NewReader(os.Stdin)
fmt.Printf("Installing from %s\n", image) fmt.Printf("Installing from %s\n", image)

6
cmd/control/service.go
View File

@ -53,7 +53,7 @@ func disable(c *cli.Context) {
} }
if changed { if changed {
if err = cfg.Set("rancher.services_include", cfg.Rancher.ServicesInclude); err != nil { if err = cfg.Save(); err != nil {
log.Fatal(err) log.Fatal(err)
} }
} }
@ -75,7 +75,7 @@ func del(c *cli.Context) {
} }
if changed { if changed {
if err = cfg.Set("rancher.services_include", cfg.Rancher.ServicesInclude); err != nil { if err = cfg.Save(); err != nil {
log.Fatal(err) log.Fatal(err)
} }
} }
@ -102,7 +102,7 @@ func enable(c *cli.Context) {
} }
if changed { if changed {
if err := cfg.Set("rancher.services_include", cfg.Rancher.ServicesInclude); err != nil { if err := cfg.Save(); err != nil {
log.Fatal(err) log.Fatal(err)
} }
} }

31
cmd/control/tlsconf.go
View File

@ -65,16 +65,21 @@ func writeCerts(generateServer bool, hostname []string, cfg *config.CloudConfig,
return err return err
} }
return cfg.SetConfig(&config.CloudConfig{ cfg, err = cfg.Merge(map[interface{}]interface{}{
Rancher: config.RancherConfig{ "rancher": map[interface{}]interface{}{
Docker: config.DockerConfig{ "docker": map[interface{}]interface{}{
CAKey: cfg.Rancher.Docker.CAKey, "ca_key": cfg.Rancher.Docker.CAKey,
CACert: cfg.Rancher.Docker.CACert, "ca_cert": cfg.Rancher.Docker.CACert,
ServerCert: string(cert), "server_cert": string(cert),
ServerKey: string(key), "server_key": string(key),
}, },
}, },
}) })
if err != nil {
return err
}
return cfg.Save()
} }
if err := ioutil.WriteFile(certPath, []byte(cfg.Rancher.Docker.ServerCert), 0400); err != nil { if err := ioutil.WriteFile(certPath, []byte(cfg.Rancher.Docker.ServerCert), 0400); err != nil {
@ -101,11 +106,11 @@ func writeCaCerts(cfg *config.CloudConfig, caCertPath, caKeyPath string) error {
return err return err
} }
err = cfg.SetConfig(&config.CloudConfig{ cfg, err = cfg.Merge(map[interface{}]interface{}{
Rancher: config.RancherConfig{ "rancher": map[interface{}]interface{}{
Docker: config.DockerConfig{ "docker": map[interface{}]interface{}{
CAKey: string(caKey), "ca_key": string(caKey),
CACert: string(caCert), "ca_cert": string(caCert),
}, },
}, },
}) })
@ -113,7 +118,7 @@ func writeCaCerts(cfg *config.CloudConfig, caCertPath, caKeyPath string) error {
return err return err
} }
return nil return cfg.Save()
} }
if err := ioutil.WriteFile(caCertPath, []byte(cfg.Rancher.Docker.CACert), 0400); err != nil { if err := ioutil.WriteFile(caCertPath, []byte(cfg.Rancher.Docker.CACert), 0400); err != nil {

19
compose/project.go
View File

@ -35,7 +35,7 @@ func RunServiceSet(name string, cfg *config.CloudConfig, configs map[string]*pro
return nil, err return nil, err
} }
addServices(p, cfg, map[string]string{}, configs) addServices(p, map[interface{}]interface{}{}, configs)
return p, p.Up() return p, p.Up()
} }
@ -78,8 +78,9 @@ func newProject(name string, cfg *config.CloudConfig) (*project.Project, error)
return docker.NewProject(context) return docker.NewProject(context)
} }
func addServices(p *project.Project, cfg *config.CloudConfig, enabled map[string]string, configs map[string]*project.ServiceConfig) { func addServices(p *project.Project, enabled map[interface{}]interface{}, configs map[string]*project.ServiceConfig) map[interface{}]interface{} {
// Note: we ignore errors while loading services // Note: we ignore errors while loading services
unchanged := true
for name, serviceConfig := range configs { for name, serviceConfig := range configs {
hash := project.GetServiceHash(name, *serviceConfig) hash := project.GetServiceHash(name, *serviceConfig)
@ -92,14 +93,19 @@ func addServices(p *project.Project, cfg *config.CloudConfig, enabled map[string
continue continue
} }
if unchanged {
enabled = util.MapCopy(enabled)
unchanged = false
}
enabled[name] = hash enabled[name] = hash
} }
return enabled
} }
func newCoreServiceProject(cfg *config.CloudConfig) (*project.Project, error) { func newCoreServiceProject(cfg *config.CloudConfig) (*project.Project, error) {
network := false network := false
projectEvents := make(chan project.ProjectEvent) projectEvents := make(chan project.ProjectEvent)
enabled := make(map[string]string) enabled := map[interface{}]interface{}{}
p, err := newProject("os", cfg) p, err := newProject("os", cfg)
if err != nil { if err != nil {
@ -110,15 +116,16 @@ func newCoreServiceProject(cfg *config.CloudConfig) (*project.Project, error) {
p.AddListener(projectEvents) p.AddListener(projectEvents)
p.ReloadCallback = func() error { p.ReloadCallback = func() error {
err := cfg.Reload() var err error
cfg, err = config.LoadConfig()
if err != nil { if err != nil {
return err return err
} }
addServices(p, cfg, enabled, cfg.Rancher.Services) enabled = addServices(p, enabled, cfg.Rancher.Services)
for service, serviceEnabled := range cfg.Rancher.ServicesInclude { for service, serviceEnabled := range cfg.Rancher.ServicesInclude {
if enabled[service] != "" || !serviceEnabled { if _, ok := enabled[service]; ok || !serviceEnabled {
continue continue
} }

226
config/config.go
View File

@ -1,212 +1,116 @@
package config package config
import ( import (
"io/ioutil"
"strings"
log "github.com/Sirupsen/logrus" log "github.com/Sirupsen/logrus"
"github.com/docker/libcompose/project"
"github.com/rancherio/os/util" "github.com/rancherio/os/util"
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
) )
func (c *CloudConfig) Import(bytes []byte) error { func (c *CloudConfig) Import(bytes []byte) (*CloudConfig, error) {
data, err := readConfig(bytes, PrivateConfigFile) data, err := readConfig(bytes, false, CloudConfigPrivateFile)
if err != nil { if err != nil {
return err return c, err
} }
if err := saveToDisk(data); err != nil {
return err
}
return c.Reload()
}
// This function only sets "non-empty" values
func (c *CloudConfig) SetConfig(newConfig *CloudConfig) error {
bytes, err := yaml.Marshal(newConfig)
if err != nil {
return err
}
return c.Merge(bytes)
}
func (c *CloudConfig) Merge(bytes []byte) error {
data, err := readConfig(bytes, LocalConfigFile, PrivateConfigFile)
if err != nil {
return err
}
if err := saveToDisk(data); err != nil {
return err
}
return c.Reload()
}
func LoadConfig() (*CloudConfig, error) {
cfg := NewConfig() cfg := NewConfig()
if err := cfg.Reload(); err != nil { if err := util.Convert(data, cfg); err != nil {
log.WithFields(log.Fields{"cfg": cfg, "err": err}).Error("Failed to reload config") return c, err
return nil, err
}
if cfg.Rancher.Debug {
log.SetLevel(log.DebugLevel)
if !util.Contains(cfg.Rancher.Docker.Args, "-D") {
cfg.Rancher.Docker.Args = append(cfg.Rancher.Docker.Args, "-D")
}
if !util.Contains(cfg.Rancher.SystemDocker.Args, "-D") {
cfg.Rancher.SystemDocker.Args = append(cfg.Rancher.SystemDocker.Args, "-D")
}
} }
return cfg, nil return cfg, nil
} }
func (c *CloudConfig) merge(values map[interface{}]interface{}) error { func (c *CloudConfig) MergeBytes(bytes []byte) (*CloudConfig, error) {
t := &CloudConfig{} data, err := readConfig(bytes, false)
if err := util.Convert(values, t); err != nil {
return err
}
return util.Convert(values, c)
}
func (c *CloudConfig) readFiles() error {
data, err := readConfig(nil, CloudConfigFile, LocalConfigFile, PrivateConfigFile)
if err != nil { if err != nil {
log.WithFields(log.Fields{"err": err}).Error("Error reading config files") return c, err
return err
} }
return c.Merge(data)
if err := c.merge(data); err != nil {
log.WithFields(log.Fields{"cfg": c, "data": data, "err": err}).Error("Error merging config data")
return err
}
return nil
} }
func (c *CloudConfig) readCmdline() error { func (c *CloudConfig) Merge(values map[interface{}]interface{}) (*CloudConfig, error) {
log.Debug("Reading config cmdline") t := *c
cmdLine, err := ioutil.ReadFile("/proc/cmdline") if err := util.Convert(values, &t); err != nil {
if err != nil { return c, err
log.WithFields(log.Fields{"err": err}).Error("Failed to read kernel params")
return err
} }
return &t, nil
if len(cmdLine) == 0 {
return nil
}
log.Debugf("Config cmdline %s", cmdLine)
cmdLineObj := parseCmdline(strings.TrimSpace(string(cmdLine)))
if err := c.merge(cmdLineObj); err != nil {
log.WithFields(log.Fields{"cfg": c, "cmdLine": cmdLine, "data": cmdLineObj, "err": err}).Warn("Error adding kernel params to config")
}
return nil
} }
func Dump(private, full bool) (string, error) { func Dump(boot, private, full bool) (string, error) {
files := []string{CloudConfigFile, LocalConfigFile} var cfg *CloudConfig
if private { var err error
files = append(files, PrivateConfigFile)
}
c := &CloudConfig{}
if full { if full {
c = NewConfig() cfg, err = LoadConfig()
} else {
files := []string{CloudConfigBootFile, CloudConfigPrivateFile, CloudConfigFile}
if !private {
files = util.FilterStrings(files, func(x string) bool { return x != CloudConfigPrivateFile })
}
if !boot {
files = util.FilterStrings(files, func(x string) bool { return x != CloudConfigBootFile })
}
cfg, err = ChainCfgFuncs(nil,
func(_ *CloudConfig) (*CloudConfig, error) { return ReadConfig(nil, true, files...) },
amendNils,
)
} }
data, err := readConfig(nil, files...)
if err != nil { if err != nil {
return "", err return "", err
} }
if err := c.merge(data); err != nil { bytes, err := yaml.Marshal(*cfg)
return "", err
}
if err := c.readCmdline(); err != nil {
return "", err
}
c.amendNils()
bytes, err := yaml.Marshal(c)
return string(bytes), err return string(bytes), err
} }
func (c *CloudConfig) amendNils() error {
if c.Rancher.Environment == nil {
c.Rancher.Environment = map[string]string{}
}
if c.Rancher.Autoformat == nil {
c.Rancher.Autoformat = map[string]*project.ServiceConfig{}
}
if c.Rancher.BootstrapContainers == nil {
c.Rancher.BootstrapContainers = map[string]*project.ServiceConfig{}
}
if c.Rancher.Services == nil {
c.Rancher.Services = map[string]*project.ServiceConfig{}
}
if c.Rancher.ServicesInclude == nil {
c.Rancher.ServicesInclude = map[string]bool{}
}
return nil
}
func (c *CloudConfig) Reload() error {
return util.ShortCircuit(
c.readFiles,
c.readCmdline,
c.amendNils,
)
}
func (c *CloudConfig) Get(key string) (interface{}, error) { func (c *CloudConfig) Get(key string) (interface{}, error) {
data := make(map[interface{}]interface{}) data := map[interface{}]interface{}{}
err := util.Convert(c, &data) if err := util.Convert(c, &data); err != nil {
if err != nil {
return nil, err return nil, err
} }
return getOrSetVal(key, data, nil), nil v, _ := getOrSetVal(key, data, nil)
return v, nil
} }
func (c *CloudConfig) Set(key string, value interface{}) error { func (c *CloudConfig) Set(key string, value interface{}) (*CloudConfig, error) {
data, err := readConfig(nil, LocalConfigFile, PrivateConfigFile) data := map[interface{}]interface{}{}
if err := util.Convert(c, &data); err != nil {
return c, err
}
_, data = getOrSetVal(key, data, value)
return c.Merge(data)
}
func (c *CloudConfig) Save() error {
files := append([]string{OsConfigFile}, CloudConfigDirFiles()...)
files = util.FilterStrings(files, func(x string) bool { return x != CloudConfigPrivateFile })
exCfg, err := ChainCfgFuncs(nil,
func(_ *CloudConfig) (*CloudConfig, error) {
return ReadConfig(nil, true, files...)
},
readCmdline,
amendNils)
if err != nil { if err != nil {
return err return err
} }
exData := map[interface{}]interface{}{}
getOrSetVal(key, data, value) if err := util.Convert(exCfg, &exData); err != nil {
cfg := NewConfig()
if err := util.Convert(data, cfg); err != nil {
return err return err
} }
data := map[interface{}]interface{}{}
if err := util.Convert(c, &data); err != nil {
return err
}
data = util.MapsDifference(data, exData)
log.WithFields(log.Fields{"diff": data}).Debug("The diff we're about to save")
if err := saveToDisk(data); err != nil { if err := saveToDisk(data); err != nil {
return err return err
} }
return nil
return c.Reload()
}
func (r Repositories) ToArray() []string {
result := make([]string, 0, len(r))
for _, repo := range r {
if repo.Url != "" {
result = append(result, repo.Url)
}
}
return result
} }

13
config/config_test.go
View File

@ -152,7 +152,8 @@ func TestGet(t *testing.T) {
} }
for k, v := range tests { for k, v := range tests {
assert.Equal(v, getOrSetVal(k, data, nil)) val, _ := getOrSetVal(k, data, nil)
assert.Equal(v, val)
} }
} }
@ -195,8 +196,10 @@ func TestSet(t *testing.T) {
} }
for k, v := range tests { for k, v := range tests {
getOrSetVal(k, data, v) _, tData := getOrSetVal(k, data, v)
assert.Equal(v, getOrSetVal(k, data, nil)) val, _ := getOrSetVal(k, tData, nil)
data = tData
assert.Equal(v, val)
} }
assert.Equal(expected, data) assert.Equal(expected, data)
@ -267,8 +270,8 @@ func TestUserDocker(t *testing.T) {
err = yaml.Unmarshal(bytes, config) err = yaml.Unmarshal(bytes, config)
assert.Nil(err) assert.Nil(err)
data := make(map[interface{}]map[interface{}]interface{}) data := map[interface{}]map[interface{}]interface{}{}
util.Convert(config, data) util.Convert(config, &data)
fmt.Println(data) fmt.Println(data)

42
config/data_funcs.go
View File

@ -9,6 +9,21 @@ import (
"strings" "strings"
) )
type CfgFunc func(*CloudConfig) (*CloudConfig, error)
func ChainCfgFuncs(cfg *CloudConfig, cfgFuncs ...CfgFunc) (*CloudConfig, error) {
for i, cfgFunc := range cfgFuncs {
log.Debugf("[%d/%d] Starting", i+1, len(cfgFuncs))
var err error
if cfg, err = cfgFunc(cfg); err != nil {
log.Errorf("Failed [%d/%d] %d%%", i+1, len(cfgFuncs), ((i + 1) * 100 / len(cfgFuncs)))
return cfg, err
}
log.Debugf("[%d/%d] Done %d%%", i+1, len(cfgFuncs), ((i + 1) * 100 / len(cfgFuncs)))
}
return cfg, nil
}
func filterKey(data map[interface{}]interface{}, key []string) (filtered, rest map[interface{}]interface{}) { func filterKey(data map[interface{}]interface{}, key []string) (filtered, rest map[interface{}]interface{}) {
if len(key) == 0 { if len(key) == 0 {
return data, map[interface{}]interface{}{} return data, map[interface{}]interface{}{}
@ -50,18 +65,23 @@ func filterDottedKeys(data map[interface{}]interface{}, keys []string) (filtered
for _, key := range keys { for _, key := range keys {
f, r := filterKey(data, strings.Split(key, ".")) f, r := filterKey(data, strings.Split(key, "."))
filtered = util.MapsUnion(filtered, f, util.Replace) filtered = util.MapsUnion(filtered, f)
rest = util.MapsIntersection(rest, r, util.Equal) rest = util.MapsIntersection(rest, r)
} }
return return
} }
func getOrSetVal(args string, data map[interface{}]interface{}, value interface{}) interface{} { func getOrSetVal(args string, data map[interface{}]interface{}, value interface{}) (interface{}, map[interface{}]interface{}) {
parts := strings.Split(args, ".") parts := strings.Split(args, ".")
tData := data
if value != nil {
tData = util.MapCopy(data)
}
t := tData
for i, part := range parts { for i, part := range parts {
val, ok := data[part] val, ok := t[part]
last := i+1 == len(parts) last := i+1 == len(parts)
// Reached end, set the value // Reached end, set the value
@ -70,15 +90,15 @@ func getOrSetVal(args string, data map[interface{}]interface{}, value interface{
value = DummyMarshall(s) value = DummyMarshall(s)
} }
data[part] = value t[part] = value
return value return value, tData
} }
// Missing intermediate key, create key // Missing intermediate key, create key
if !last && value != nil && !ok { if !last && value != nil && !ok {
newData := map[interface{}]interface{}{} newData := map[interface{}]interface{}{}
data[part] = newData t[part] = newData
data = newData t = newData
continue continue
} }
@ -87,7 +107,7 @@ func getOrSetVal(args string, data map[interface{}]interface{}, value interface{
} }
if last { if last {
return val return val, tData
} }
newData, ok := val.(map[interface{}]interface{}) newData, ok := val.(map[interface{}]interface{})
@ -95,10 +115,10 @@ func getOrSetVal(args string, data map[interface{}]interface{}, value interface{
break break
} }
data = newData t = newData
} }
return "" return "", tData
} }
func DummyMarshall(value string) interface{} { func DummyMarshall(value string) interface{} {

18
config/default.go
View File

@ -1,18 +0,0 @@
package config
func NewConfig() *CloudConfig {
return ReadConfig(nil, OsConfigFile)
}
func ReadConfig(bytes []byte, files ...string) *CloudConfig {
if data, err := readConfig(bytes, files...); err == nil {
c := &CloudConfig{}
if err := c.merge(data); err != nil {
return nil
}
c.amendNils()
return c
} else {
return nil
}
}

206
config/disk.go
View File

@ -3,11 +3,189 @@ package config
import ( import (
"io/ioutil" "io/ioutil"
"os" "os"
"path"
"strings"
log "github.com/Sirupsen/logrus"
"github.com/coreos/coreos-cloudinit/datasource"
"github.com/coreos/coreos-cloudinit/initialize"
"github.com/docker/libcompose/project"
"github.com/rancherio/os/util" "github.com/rancherio/os/util"
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
) )
var osConfig *CloudConfig
func NewConfig() *CloudConfig {
if osConfig == nil {
osConfig, _ = ReadConfig(nil, true, OsConfigFile)
}
newCfg := *osConfig
return &newCfg
}
func ReadConfig(bytes []byte, substituteMetadataVars bool, files ...string) (*CloudConfig, error) {
if data, err := readConfig(bytes, substituteMetadataVars, files...); err == nil {
c := &CloudConfig{}
c, err := c.Merge(data)
if err != nil {
return nil, err
}
c, _ = amendNils(c)
return c, nil
} else {
return nil, err
}
}
func LoadConfig() (*CloudConfig, error) {
cfg, err := ChainCfgFuncs(NewConfig(),
readFilesAndMetadata,
readCmdline,
amendNils)
if err != nil {
log.WithFields(log.Fields{"cfg": cfg, "err": err}).Error("Failed to load config")
return nil, err
}
log.Debug("Merging cloud-config from meta-data and user-data")
cfg = mergeMetadata(cfg, readMetadata())
if cfg.Rancher.Debug {
log.SetLevel(log.DebugLevel)
if !util.Contains(cfg.Rancher.Docker.Args, "-D") {
cfg.Rancher.Docker.Args = append(cfg.Rancher.Docker.Args, "-D")
}
if !util.Contains(cfg.Rancher.SystemDocker.Args, "-D") {
cfg.Rancher.SystemDocker.Args = append(cfg.Rancher.SystemDocker.Args, "-D")
}
}
return cfg, nil
}
func CloudConfigDirFiles() []string {
files, err := util.DirLs(CloudConfigDir)
if err != nil {
if os.IsNotExist(err) {
// do nothing
log.Debugf("%s does not exist", CloudConfigDir)
} else {
log.Errorf("Failed to read %s: %v", CloudConfigDir, err)
}
return []string{}
}
files = util.Filter(files, func(x interface{}) bool {
f := x.(os.FileInfo)
if f.IsDir() || strings.HasPrefix(f.Name(), ".") {
return false
}
return true
})
return util.ToStrings(util.Map(files, func(x interface{}) interface{} {
return path.Join(CloudConfigDir, x.(os.FileInfo).Name())
}))
}
// mergeMetadata merges certain options from md (meta-data from the datasource)
// onto cc (a CloudConfig derived from user-data), if they are not already set
// on cc (i.e. user-data always takes precedence)
func mergeMetadata(cc *CloudConfig, md datasource.Metadata) *CloudConfig {
if cc == nil {
return cc
}
out := cc
dirty := false
if md.Hostname != "" {
if out.Hostname != "" {
log.Debugf("Warning: user-data hostname (%s) overrides metadata hostname (%s)\n", out.Hostname, md.Hostname)
} else {
out = &(*cc)
dirty = true
out.Hostname = md.Hostname
}
}
for _, key := range md.SSHPublicKeys {
if !dirty {
out = &(*cc)
dirty = true
}
out.SSHAuthorizedKeys = append(out.SSHAuthorizedKeys, key)
}
return out
}
func readMetadata() datasource.Metadata {
metadata := datasource.Metadata{}
if metaDataBytes, err := ioutil.ReadFile(MetaDataFile); err == nil {
yaml.Unmarshal(metaDataBytes, &metadata)
}
return metadata
}
func readFilesAndMetadata(c *CloudConfig) (*CloudConfig, error) {
files := append(CloudConfigDirFiles(), CloudConfigFile)
data, err := readConfig(nil, true, files...)
if err != nil {
log.WithFields(log.Fields{"err": err}).Error("Error reading config files")
return c, err
}
t, err := c.Merge(data)
if err != nil {
log.WithFields(log.Fields{"cfg": c, "data": data, "err": err}).Error("Error merging config data")
return c, err
}
return t, nil
}
func readCmdline(c *CloudConfig) (*CloudConfig, error) {
log.Debug("Reading config cmdline")
cmdLine, err := ioutil.ReadFile("/proc/cmdline")
if err != nil {
log.WithFields(log.Fields{"err": err}).Error("Failed to read kernel params")
return c, err
}
if len(cmdLine) == 0 {
return c, nil
}
log.Debugf("Config cmdline %s", cmdLine)
cmdLineObj := parseCmdline(strings.TrimSpace(string(cmdLine)))
t, err := c.Merge(cmdLineObj)
if err != nil {
log.WithFields(log.Fields{"cfg": c, "cmdLine": cmdLine, "data": cmdLineObj, "err": err}).Warn("Error adding kernel params to config")
}
return t, nil
}
func amendNils(c *CloudConfig) (*CloudConfig, error) {
t := *c
if t.Rancher.Environment == nil {
t.Rancher.Environment = map[string]string{}
}
if t.Rancher.Autoformat == nil {
t.Rancher.Autoformat = map[string]*project.ServiceConfig{}
}
if t.Rancher.BootstrapContainers == nil {
t.Rancher.BootstrapContainers = map[string]*project.ServiceConfig{}
}
if t.Rancher.Services == nil {
t.Rancher.Services = map[string]*project.ServiceConfig{}
}
if t.Rancher.ServicesInclude == nil {
t.Rancher.ServicesInclude = map[string]bool{}
}
return &t, nil
}
func writeToFile(data interface{}, filename string) error { func writeToFile(data interface{}, filename string) error {
content, err := yaml.Marshal(data) content, err := yaml.Marshal(data)
if err != nil { if err != nil {
@ -26,23 +204,27 @@ func saveToDisk(data map[interface{}]interface{}) error {
"rancher.docker.server_cert", "rancher.docker.server_cert",
}) })
err := writeToFile(config, LocalConfigFile) err := writeToFile(config, CloudConfigFile)
if err != nil { if err != nil {
return err return err
} }
return writeToFile(private, PrivateConfigFile) return writeToFile(private, CloudConfigPrivateFile)
} }
func readConfig(bytes []byte, files ...string) (map[interface{}]interface{}, error) { func readConfig(bytes []byte, substituteMetadataVars bool, files ...string) (map[interface{}]interface{}, error) {
// You can't just overlay yaml bytes on to maps, it won't merge, but instead // You can't just overlay yaml bytes on to maps, it won't merge, but instead
// just override the keys and not merge the map values. // just override the keys and not merge the map values.
left := make(map[interface{}]interface{}) left := make(map[interface{}]interface{})
for _, conf := range files { metadata := readMetadata()
content, err := readConfigFile(conf) for _, file := range files {
content, err := readConfigFile(file)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if substituteMetadataVars {
content = substituteVars(content, metadata)
}
right := make(map[interface{}]interface{}) right := make(map[interface{}]interface{})
err = yaml.Unmarshal(content, &right) err = yaml.Unmarshal(content, &right)
@ -50,16 +232,19 @@ func readConfig(bytes []byte, files ...string) (map[interface{}]interface{}, err
return nil, err return nil, err
} }
left = util.MapsUnion(left, right, util.Replace) left = util.MapsUnion(left, right)
} }
if bytes != nil && len(bytes) > 0 { if bytes != nil && len(bytes) > 0 {
right := make(map[interface{}]interface{}) right := make(map[interface{}]interface{})
if substituteMetadataVars {
bytes = substituteVars(bytes, metadata)
}
if err := yaml.Unmarshal(bytes, &right); err != nil { if err := yaml.Unmarshal(bytes, &right); err != nil {
return nil, err return nil, err
} }
left = util.MapsUnion(left, right, util.Replace) left = util.MapsUnion(left, right)
} }
return left, nil return left, nil
@ -79,3 +264,10 @@ func readConfigFile(file string) ([]byte, error) {
return content, err return content, err
} }
func substituteVars(userDataBytes []byte, metadata datasource.Metadata) []byte {
env := initialize.NewEnvironment("", "", "", "", metadata)
userData := env.Apply(string(userDataBytes))
return []byte(userData)
}

32
config/init.go
View File

@ -1,32 +0,0 @@
package config
import (
"strings"
log "github.com/Sirupsen/logrus"
)
type InitFunc func(*CloudConfig) error
func RunInitFuncs(cfg *CloudConfig, initFuncs []InitFunc) error {
for i, initFunc := range initFuncs {
log.Debugf("[%d/%d] Starting", i+1, len(initFuncs))
if err := initFunc(cfg); err != nil {
log.Errorf("Failed [%d/%d] %d%%", i+1, len(initFuncs), ((i + 1) * 100 / len(initFuncs)))
return err
}
log.Debugf("[%d/%d] Done %d%%", i+1, len(initFuncs), ((i + 1) * 100 / len(initFuncs)))
}
return nil
}
func FilterGlobalConfig(input []string) []string {
result := make([]string, 0, len(input))
for _, value := range input {
if !strings.HasPrefix(value, "--rancher") {
result = append(result, value)
}
}
return result
}

4
cmd/cloudinit/cloudinit_test.go → config/metadata_test.go
View File

@ -13,7 +13,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package cloudinit package config
import ( import (
"net" "net"
@ -92,7 +92,7 @@ func TestSubstituteUserDataVars(t *testing.T) {
}, },
} { } {
got := substituteUserDataVars([]byte(tt.input), tt.metadata) got := substituteVars([]byte(tt.input), tt.metadata)
if string(got) != tt.out { if string(got) != tt.out {
t.Fatalf("Userdata substitution incorrectly applied.\ngot:\n%s\nwant:\n%s", got, tt.out) t.Fatalf("Userdata substitution incorrectly applied.\ngot:\n%s\nwant:\n%s", got, tt.out)
} }

24
config/types.go
View File

@ -29,17 +29,24 @@ const (
SYSTEM = "system" SYSTEM = "system"
OsConfigFile = "/usr/share/ros/os-config.yml" OsConfigFile = "/usr/share/ros/os-config.yml"
CloudConfigFile = "/var/lib/rancher/conf/cloud-config.yml" CloudConfigDir = "/var/lib/rancher/conf/cloud-config.d"
CloudConfigBootFile = "/var/lib/rancher/conf/cloud-config.d/boot.yml"
CloudConfigPrivateFile = "/var/lib/rancher/conf/cloud-config.d/private.yml"
CloudConfigScriptFile = "/var/lib/rancher/conf/cloud-config-script" CloudConfigScriptFile = "/var/lib/rancher/conf/cloud-config-script"
MetaDataFile = "/var/lib/rancher/conf/metadata" MetaDataFile = "/var/lib/rancher/conf/metadata"
LocalConfigFile = "/var/lib/rancher/conf/cloud-config-local.yml" CloudConfigFile = "/var/lib/rancher/conf/cloud-config.yml"
PrivateConfigFile = "/var/lib/rancher/conf/cloud-config-private.yml"
) )
var ( var (
VERSION string VERSION string
) )
func init() {
if VERSION == "" {
VERSION = "v0.0.0-dev"
}
}
type ContainerConfig struct { type ContainerConfig struct {
Id string `yaml:"id,omitempty"` Id string `yaml:"id,omitempty"`
Cmd string `yaml:"run,omitempty"` Cmd string `yaml:"run,omitempty"`
@ -119,8 +126,13 @@ type CloudInit struct {
Datasources []string `yaml:"datasources,omitempty"` Datasources []string `yaml:"datasources,omitempty"`
} }
func init() { func (r Repositories) ToArray() []string {
if VERSION == "" { result := make([]string, 0, len(r))
VERSION = "v0.0.0-dev" for _, repo := range r {
if repo.Url != "" {
result = append(result, repo.Url)
} }
}
return result
} }

27
init/bootstrap.go
View File

@ -15,22 +15,23 @@ import (
"github.com/rancherio/os/util" "github.com/rancherio/os/util"
) )
func autoformat(cfg *config.CloudConfig) error { func autoformat(cfg *config.CloudConfig) (*config.CloudConfig, error) {
if len(cfg.Rancher.State.Autoformat) == 0 || util.ResolveDevice(cfg.Rancher.State.Dev) != "" { if len(cfg.Rancher.State.Autoformat) == 0 || util.ResolveDevice(cfg.Rancher.State.Dev) != "" {
return nil return cfg, nil
} }
AUTOFORMAT := "AUTOFORMAT=" + strings.Join(cfg.Rancher.State.Autoformat, " ") AUTOFORMAT := "AUTOFORMAT=" + strings.Join(cfg.Rancher.State.Autoformat, " ")
FORMATZERO := "FORMATZERO=" + fmt.Sprint(cfg.Rancher.State.FormatZero) FORMATZERO := "FORMATZERO=" + fmt.Sprint(cfg.Rancher.State.FormatZero)
cfg.Rancher.Autoformat["autoformat"].Environment = project.NewMaporEqualSlice([]string{AUTOFORMAT, FORMATZERO}) t := *cfg
t.Rancher.Autoformat["autoformat"].Environment = project.NewMaporEqualSlice([]string{AUTOFORMAT, FORMATZERO})
log.Info("Running Autoformat services") log.Info("Running Autoformat services")
_, err := compose.RunServiceSet("autoformat", cfg, cfg.Rancher.Autoformat) _, err := compose.RunServiceSet("autoformat", &t, t.Rancher.Autoformat)
return err return &t, err
} }
func runBootstrapContainers(cfg *config.CloudConfig) error { func runBootstrapContainers(cfg *config.CloudConfig) (*config.CloudConfig, error) {
log.Info("Running Bootstrap services") log.Info("Running Bootstrap services")
_, err := compose.RunServiceSet("bootstrap", cfg, cfg.Rancher.BootstrapContainers) _, err := compose.RunServiceSet("bootstrap", cfg, cfg.Rancher.BootstrapContainers)
return err return cfg, err
} }
func startDocker(cfg *config.CloudConfig) (chan interface{}, error) { func startDocker(cfg *config.CloudConfig) (chan interface{}, error) {
@ -70,13 +71,11 @@ func bootstrap(cfg *config.CloudConfig) error {
return err return err
} }
initFuncs := []config.InitFunc{
loadImages,
runBootstrapContainers,
autoformat,
}
defer stopDocker(c) defer stopDocker(c)
return config.RunInitFuncs(cfg, initFuncs) _, err = config.ChainCfgFuncs(cfg,
loadImages,
runBootstrapContainers,
autoformat)
return err
} }

54
init/init.go
View File

@ -30,12 +30,12 @@ var (
} }
) )
func loadModules(cfg *config.CloudConfig) error { func loadModules(cfg *config.CloudConfig) (*config.CloudConfig, error) {
mounted := map[string]bool{} mounted := map[string]bool{}
f, err := os.Open("/proc/modules") f, err := os.Open("/proc/modules")
if err != nil { if err != nil {
return err return cfg, err
} }
defer f.Close() defer f.Close()
@ -55,10 +55,10 @@ func loadModules(cfg *config.CloudConfig) error {
} }
} }
return nil return cfg, nil
} }
func sysInit(cfg *config.CloudConfig) error { func sysInit(c *config.CloudConfig) (*config.CloudConfig, error) {
args := append([]string{config.SYSINIT_BIN}, os.Args[1:]...) args := append([]string{config.SYSINIT_BIN}, os.Args[1:]...)
cmd := &exec.Cmd{ cmd := &exec.Cmd{
@ -71,10 +71,10 @@ func sysInit(cfg *config.CloudConfig) error {
cmd.Stdout = os.Stdout cmd.Stdout = os.Stdout
if err := cmd.Start(); err != nil { if err := cmd.Start(); err != nil {
return err return c, err
} }
return os.Stdin.Close() return c, os.Stdin.Close()
} }
func MainInit() { func MainInit() {
@ -121,15 +121,15 @@ func tryMountState(cfg *config.CloudConfig) error {
return mountState(cfg) return mountState(cfg)
} }
func tryMountAndBootstrap(cfg *config.CloudConfig) error { func tryMountAndBootstrap(cfg *config.CloudConfig) (*config.CloudConfig, error) {
if err := tryMountState(cfg); !cfg.Rancher.State.Required && err != nil { if err := tryMountState(cfg); !cfg.Rancher.State.Required && err != nil {
return nil return cfg, nil
} else if err != nil { } else if err != nil {
return err return cfg, err
} }
log.Debugf("Switching to new root at %s", STATE) log.Debugf("Switching to new root at %s", STATE)
return switchRoot(STATE) return cfg, switchRoot(STATE)
} }
func getLaunchConfig(cfg *config.CloudConfig, dockerCfg *config.DockerConfig) (*dockerlaunch.Config, []string) { func getLaunchConfig(cfg *config.CloudConfig, dockerCfg *config.DockerConfig) (*dockerlaunch.Config, []string) {
@ -150,27 +150,22 @@ func getLaunchConfig(cfg *config.CloudConfig, dockerCfg *config.DockerConfig) (*
} }
func RunInit() error { func RunInit() error {
var cfg config.CloudConfig
os.Setenv("PATH", "/sbin:/usr/sbin:/usr/bin") os.Setenv("PATH", "/sbin:/usr/sbin:/usr/bin")
// Magic setting to tell Docker to do switch_root and not pivot_root // Magic setting to tell Docker to do switch_root and not pivot_root
os.Setenv("DOCKER_RAMDISK", "true") os.Setenv("DOCKER_RAMDISK", "true")
initFuncs := []config.InitFunc{ initFuncs := []config.CfgFunc{
func(cfg *config.CloudConfig) error { func(c *config.CloudConfig) (*config.CloudConfig, error) {
return dockerlaunch.PrepareFs(&mountConfig) return c, dockerlaunch.PrepareFs(&mountConfig)
}, },
func(cfg *config.CloudConfig) error { func(_ *config.CloudConfig) (*config.CloudConfig, error) {
newCfg, err := config.LoadConfig() cfg, err := config.LoadConfig()
if err == nil { if err != nil {
newCfg, err = config.LoadConfig() return cfg, err
}
if err == nil {
*cfg = *newCfg
} }
if cfg.Rancher.Debug { if cfg.Rancher.Debug {
cfgString, err := config.Dump(false, true) cfgString, err := config.Dump(false, false, true)
if err != nil { if err != nil {
log.WithFields(log.Fields{"err": err}).Error("Error serializing config") log.WithFields(log.Fields{"err": err}).Error("Error serializing config")
} else { } else {
@ -178,24 +173,25 @@ func RunInit() error {
} }
} }
return err return cfg, nil
}, },
loadModules, loadModules,
tryMountAndBootstrap, tryMountAndBootstrap,
func(cfg *config.CloudConfig) error { func(_ *config.CloudConfig) (*config.CloudConfig, error) {
return cfg.Reload() return config.LoadConfig()
}, },
loadModules, loadModules,
sysInit, sysInit,
} }
if err := config.RunInitFuncs(&cfg, initFuncs); err != nil { cfg, err := config.ChainCfgFuncs(nil, initFuncs...)
if err != nil {
return err return err
} }
launchConfig, args := getLaunchConfig(&cfg, &cfg.Rancher.SystemDocker) launchConfig, args := getLaunchConfig(cfg, &cfg.Rancher.SystemDocker)
log.Info("Launching System Docker") log.Info("Launching System Docker")
_, err := dockerlaunch.LaunchDocker(launchConfig, config.DOCKER_BIN, args...) _, err = dockerlaunch.LaunchDocker(launchConfig, config.DOCKER_BIN, args...)
return err return err
} }

32
init/sysinit.go
View File

@ -51,15 +51,15 @@ func findImages(cfg *config.CloudConfig) ([]string, error) {
return result, nil return result, nil
} }
func loadImages(cfg *config.CloudConfig) error { func loadImages(cfg *config.CloudConfig) (*config.CloudConfig, error) {
images, err := findImages(cfg) images, err := findImages(cfg)
if err != nil || len(images) == 0 { if err != nil || len(images) == 0 {
return err return cfg, err
} }
client, err := docker.NewSystemClient() client, err := docker.NewSystemClient()
if err != nil { if err != nil {
return err return cfg, err
} }
for _, image := range images { for _, image := range images {
@ -70,7 +70,7 @@ func loadImages(cfg *config.CloudConfig) error {
inputFileName := path.Join(config.IMAGES_PATH, image) inputFileName := path.Join(config.IMAGES_PATH, image)
input, err := os.Open(inputFileName) input, err := os.Open(inputFileName)
if err != nil { if err != nil {
return err return cfg, err
} }
defer input.Close() defer input.Close()
@ -82,11 +82,11 @@ func loadImages(cfg *config.CloudConfig) error {
log.Infof("Done loading images from %s", inputFileName) log.Infof("Done loading images from %s", inputFileName)
if err != nil { if err != nil {
return err return cfg, err
} }
} }
return nil return cfg, nil
} }
func SysInit() error { func SysInit() error {
@ -95,20 +95,18 @@ func SysInit() error {
return err return err
} }
initFuncs := []config.InitFunc{ _, err = config.ChainCfgFuncs(cfg,
loadImages, loadImages,
func(cfg *config.CloudConfig) error { func(cfg *config.CloudConfig) (*config.CloudConfig, error) {
return compose.RunServices(cfg) return cfg, compose.RunServices(cfg)
}, },
func(cfg *config.CloudConfig) error { func(cfg *config.CloudConfig) (*config.CloudConfig, error) {
syscall.Sync() syscall.Sync()
return nil return cfg, nil
}, },
func(cfg *config.CloudConfig) error { func(cfg *config.CloudConfig) (*config.CloudConfig, error) {
log.Infof("RancherOS %s started", config.VERSION) log.Infof("RancherOS %s started", config.VERSION)
return nil return cfg, nil
}, })
} return err
return config.RunInitFuncs(cfg, initFuncs)
} }

2
util/term.go
View File

@ -1,3 +1,5 @@
// +build linux
package util package util
import "github.com/kless/term" import "github.com/kless/term"

268
util/util.go
View File

@ -1,101 +1,27 @@
package util package util
import ( import (
"archive/tar"
"bufio"
"errors" "errors"
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
"math/rand"
"net/http" "net/http"
"os" "os"
"path"
"strings" "strings"
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
log "github.com/Sirupsen/logrus" log "github.com/Sirupsen/logrus"
"github.com/docker/docker/pkg/mount"
"reflect" "reflect"
) )
var ( var (
letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
ErrNoNetwork = errors.New("Networking not available to load resource") ErrNoNetwork = errors.New("Networking not available to load resource")
ErrNotFound = errors.New("Failed to find resource") ErrNotFound = errors.New("Failed to find resource")
) )
func GetOSType() string { type AnyMap map[interface{}]interface{}
f, err := os.Open("/etc/os-release")
defer f.Close()
if err != nil {
return "busybox"
}
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := scanner.Text()
if len(line) > 8 && line[:8] == "ID_LIKE=" {
return line[8:]
}
}
return "busybox"
}
func Remount(directory, options string) error {
return mount.Mount("", directory, "", fmt.Sprintf("remount,%s", options))
}
func ExtractTar(archive string, dest string) error {
f, err := os.Open(archive)
if err != nil {
return err
}
defer f.Close()
input := tar.NewReader(f)
for {
header, err := input.Next()
if err == io.EOF {
break
} else if err != nil {
return err
}
if header == nil {
break
}
fileInfo := header.FileInfo()
fileName := path.Join(dest, header.Name)
if fileInfo.IsDir() {
//log.Debugf("DIR : %s", fileName)
err = os.MkdirAll(fileName, fileInfo.Mode())
if err != nil {
return err
}
} else {
//log.Debugf("FILE: %s", fileName)
destFile, err := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fileInfo.Mode())
if err != nil {
return err
}
_, err = io.Copy(destFile, input)
// Not deferring, concerned about holding open too many files
destFile.Close()
if err != nil {
return err
}
}
}
return nil
}
func Contains(values []string, value string) bool { func Contains(values []string, value string) bool {
if len(value) == 0 { if len(value) == 0 {
@ -113,45 +39,6 @@ func Contains(values []string, value string) bool {
type ReturnsErr func() error type ReturnsErr func() error
func ShortCircuit(funcs ...ReturnsErr) error {
for _, f := range funcs {
err := f()
if err != nil {
return err
}
}
return nil
}
type ErrWriter struct {
w io.Writer
Err error
}
func NewErrorWriter(w io.Writer) *ErrWriter {
return &ErrWriter{
w: w,
}
}
func (e *ErrWriter) Write(buf []byte) *ErrWriter {
if e.Err != nil {
return e
}
_, e.Err = e.w.Write(buf)
return e
}
func RandSeq(n int) string {
b := make([]rune, n)
for i := range b {
b[i] = letters[rand.Intn(len(letters))]
}
return string(b)
}
func FileCopy(src, dest string) (err error) { func FileCopy(src, dest string) (err error) {
in, err := os.Open(src) in, err := os.Open(src)
if err != nil { if err != nil {
@ -181,21 +68,6 @@ func Convert(from, to interface{}) error {
return yaml.Unmarshal(bytes, to) return yaml.Unmarshal(bytes, to)
} }
func MergeBytes(left, right []byte) ([]byte, error) {
leftMap := make(map[interface{}]interface{})
rightMap := make(map[interface{}]interface{})
if err := yaml.Unmarshal(left, &leftMap); err != nil {
return nil, err
}
if err := yaml.Unmarshal(right, &rightMap); err != nil {
return nil, err
}
return yaml.Marshal(MapsUnion(leftMap, rightMap, Replace))
}
func Copy(d interface{}) interface{} { func Copy(d interface{}) interface{} {
switch d := d.(type) { switch d := d.(type) {
case map[interface{}]interface{}: case map[interface{}]interface{}:
@ -218,36 +90,45 @@ func Equal(l, r interface{}) interface{} {
return nil return nil
} }
func ExistsIn(x interface{}, s []interface{}) bool { func Filter(xs []interface{}, p func(x interface{}) bool) []interface{} {
for _, y := range s { return FlatMap(xs, func(x interface{}) []interface{} {
if reflect.DeepEqual(x, y) { if p(x) {
return true return []interface{}{x}
} }
} return []interface{}{}
return false })
} }
func SlicesUnion(left, right []interface{}, op func(interface{}, interface{}) interface{}) []interface{} { func FilterStrings(xs []string, p func(x string) bool) []string {
result := SliceCopy(left) return FlatMapStrings(xs, func(x string) []string {
for _, r := range right { if p(x) {
if !ExistsIn(r, result) { return []string{x}
result = append(result, r)
} }
} return []string{}
return result })
} }
func SlicesIntersection(left, right []interface{}, op func(interface{}, interface{}) interface{}) []interface{} { func Map(xs []interface{}, f func(x interface{}) interface{}) []interface{} {
return FlatMap(xs, func(x interface{}) []interface{} { return []interface{}{f(x)} })
}
func FlatMap(xs []interface{}, f func(x interface{}) []interface{}) []interface{} {
result := []interface{}{} result := []interface{}{}
for _, r := range right { for _, x := range xs {
if ExistsIn(r, left) { result = append(result, f(x)...)
result = append(result, r)
}
} }
return result return result
} }
func MapsUnion(left, right map[interface{}]interface{}, op func(interface{}, interface{}) interface{}) map[interface{}]interface{} { func FlatMapStrings(xs []string, f func(x string) []string) []string {
result := []string{}
for _, x := range xs {
result = append(result, f(x)...)
}
return result
}
func MapsUnion(left, right map[interface{}]interface{}) map[interface{}]interface{} {
result := MapCopy(left) result := MapCopy(left)
for k, r := range right { for k, r := range right {
@ -256,19 +137,12 @@ func MapsUnion(left, right map[interface{}]interface{}, op func(interface{}, int
case map[interface{}]interface{}: case map[interface{}]interface{}:
switch r := r.(type) { switch r := r.(type) {
case map[interface{}]interface{}: case map[interface{}]interface{}:
result[k] = MapsUnion(l, r, op) result[k] = MapsUnion(l, r)
default: default:
result[k] = op(l, r) result[k] = Replace(l, r)
}
case []interface{}:
switch r := r.(type) {
case []interface{}:
result[k] = SlicesUnion(l, r, op)
default:
result[k] = op(l, r)
} }
default: default:
result[k] = op(l, r) result[k] = Replace(l, r)
} }
} else { } else {
result[k] = Copy(r) result[k] = Copy(r)
@ -278,7 +152,7 @@ func MapsUnion(left, right map[interface{}]interface{}, op func(interface{}, int
return result return result
} }
func MapsIntersection(left, right map[interface{}]interface{}, op func(interface{}, interface{}) interface{}) map[interface{}]interface{} { func MapsDifference(left, right map[interface{}]interface{}) map[interface{}]interface{} {
result := map[interface{}]interface{}{} result := map[interface{}]interface{}{}
for k, l := range left { for k, l := range left {
@ -287,23 +161,48 @@ func MapsIntersection(left, right map[interface{}]interface{}, op func(interface
case map[interface{}]interface{}: case map[interface{}]interface{}:
switch r := r.(type) { switch r := r.(type) {
case map[interface{}]interface{}: case map[interface{}]interface{}:
result[k] = MapsIntersection(l, r, op) if len(l) == 0 && len(r) == 0 {
default: continue
if v := op(l, r); v != nil { } else if len(l) == 0 {
result[k] = l
} else if v := MapsDifference(l, r); len(v) > 0 {
result[k] = v result[k] = v
} }
default:
if v := Equal(l, r); v == nil {
result[k] = l
} }
case []interface{}: }
default:
if v := Equal(l, r); v == nil {
result[k] = l
}
}
} else {
result[k] = l
}
}
return result
}
func MapsIntersection(left, right map[interface{}]interface{}) map[interface{}]interface{} {
result := map[interface{}]interface{}{}
for k, l := range left {
if r, ok := right[k]; ok {
switch l := l.(type) {
case map[interface{}]interface{}:
switch r := r.(type) { switch r := r.(type) {
case []interface{}: case map[interface{}]interface{}:
result[k] = SlicesIntersection(l, r, op) result[k] = MapsIntersection(l, r)
default: default:
if v := op(l, r); v != nil { if v := Equal(l, r); v != nil {
result[k] = v result[k] = v
} }
} }
default: default:
if v := op(l, r); v != nil { if v := Equal(l, r); v != nil {
result[k] = v result[k] = v
} }
} }
@ -329,6 +228,14 @@ func SliceCopy(data []interface{}) []interface{} {
return result return result
} }
func ToStrings(data []interface{}) []string {
result := make([]string, len(data), len(data))
for k, v := range data {
result[k] = v.(string)
}
return result
}
func GetServices(urls []string) ([]string, error) { func GetServices(urls []string) ([]string, error) {
result := []string{} result := []string{}
@ -355,6 +262,18 @@ func GetServices(urls []string) ([]string, error) {
return result, nil return result, nil
} }
func DirLs(dir string) ([]interface{}, error) {
result := []interface{}{}
files, err := ioutil.ReadDir(dir)
if err != nil {
return result, err
}
for _, f := range files {
result = append(result, f)
}
return result, nil
}
func LoadResource(location string, network bool, urls []string) ([]byte, error) { func LoadResource(location string, network bool, urls []string) ([]byte, error) {
var bytes []byte var bytes []byte
err := ErrNotFound err := ErrNotFound
@ -388,21 +307,6 @@ func LoadResource(location string, network bool, urls []string) ([]byte, error)
return nil, err return nil, err
} }
func GetValue(kvPairs []string, key string) string {
if kvPairs == nil {
return ""
}
prefix := key + "="
for _, i := range kvPairs {
if strings.HasPrefix(i, prefix) {
return strings.TrimPrefix(i, prefix)
}
}
return ""
}
func Map2KVPairs(m map[string]string) []string { func Map2KVPairs(m map[string]string) []string {
r := make([]string, 0, len(m)) r := make([]string, 0, len(m))
for k, v := range m { for k, v := range m {

101
util/util_test.go
View File

@ -6,10 +6,75 @@ import (
"testing" "testing"
) )
type testCloudConfig struct {
Hostname string `yaml:"hostname,omitempty"`
Key1 string `yaml:"key1,omitempty"`
Key2 string `yaml:"key2,omitempty"`
}
func TestPassByValue(t *testing.T) {
assert := require.New(t)
cc0ptr := &testCloudConfig{}
cc0ptr.Hostname = "test0"
cc1 := *cc0ptr
cc1.Hostname = "test1"
assert.NotEqual(cc0ptr.Hostname, cc1.Hostname)
}
func TestConvertMergesLeftIntoRight(t *testing.T) {
assert := require.New(t)
cc0 := testCloudConfig{Key1: "k1v0", Key2: "k2v0"}
cc1 := map[interface{}]interface{}{"key1": "k1value1", "hostname": "somehost"}
Convert(cc1, &cc0)
expected := testCloudConfig{Hostname: "somehost", Key1: "k1value1", Key2: "k2v0"}
assert.Equal(expected, cc0)
}
func TestNilMap(t *testing.T) { func TestNilMap(t *testing.T) {
assert := require.New(t) assert := require.New(t)
var m map[string]interface{} = nil var m map[string]interface{} = nil
assert.True(m == nil) assert.True(m == nil)
assert.True(len(m) == 0)
}
func NoTestCopyPointer(t *testing.T) {
assert := require.New(t)
testCCpt := &testCloudConfig{}
m0 := map[string]interface{}{"a": testCCpt, "b": testCCpt}
m1 := Copy(m0).(map[string]interface{})
m1["a"].(*testCloudConfig).Hostname = "somehost"
assert.Equal("", m0["a"].(*testCloudConfig).Hostname)
assert.Equal("somehost", m1["a"].(*testCloudConfig).Hostname)
assert.Equal("", m1["b"].(*testCloudConfig).Hostname)
}
func TestEmptyMap(t *testing.T) {
assert := require.New(t)
m := map[interface{}]interface{}{}
assert.True(len(m) == 0)
}
func tryMutateArg(p *string) *string {
s := "test"
p = &s
return p
}
func TestMutableArg(t *testing.T) {
assert := require.New(t)
s := "somestring"
p := &s
assert.NotEqual(tryMutateArg(p), p)
}
func TestFilter(t *testing.T) {
assert := require.New(t)
ss := []interface{}{"1", "2", "3", "4"}
assert.Equal([]interface{}{"1", "2", "4"}, Filter(ss, func(x interface{}) bool { return x != "3" }))
ss1 := append([]interface{}{}, "qqq")
assert.Equal([]interface{}{"qqq"}, ss1)
} }
func TestMapCopy(t *testing.T) { func TestMapCopy(t *testing.T) {
@ -59,8 +124,34 @@ func TestMapsIntersection(t *testing.T) {
b1 := m1["b"].(map[interface{}]interface{}) b1 := m1["b"].(map[interface{}]interface{})
delete(b1, "c") delete(b1, "c")
m1["e"] = []interface{}{2, 3, 4} m1["e"] = []interface{}{2, 3, 4}
expected := map[interface{}]interface{}{"b": map[interface{}]interface{}{}, "d": "4", "e": []interface{}{2, 3}} expected := map[interface{}]interface{}{"b": map[interface{}]interface{}{}, "d": "4"}
assert.Equal(expected, MapsIntersection(m0, m1, Equal)) assert.Equal(expected, MapsIntersection(m0, m1))
}
func TestMapsDifference(t *testing.T) {
assert := require.New(t)
m0 := map[interface{}]interface{}{
"a": 1,
"b": map[interface{}]interface{}{"c": 3},
"d": "4",
"e": []interface{}{1, 2, 3},
}
m1 := MapCopy(m0)
assert.Equal(map[interface{}]interface{}{}, MapsDifference(m0, m0))
assert.Equal(map[interface{}]interface{}{}, MapsDifference(m0, m1))
delete(m1, "a")
b1 := m1["b"].(map[interface{}]interface{})
delete(b1, "c")
m1["e"] = []interface{}{2, 3, 4}
expectedM1M0 := map[interface{}]interface{}{"b": map[interface{}]interface{}{}, "e": []interface{}{2, 3, 4}}
assert.Equal(expectedM1M0, MapsDifference(m1, m0))
expectedM0M1 := map[interface{}]interface{}{"a": 1, "b": map[interface{}]interface{}{"c": 3}, "e": []interface{}{1, 2, 3}}
assert.Equal(expectedM0M1, MapsDifference(m0, m1))
} }
func TestMapsUnion(t *testing.T) { func TestMapsUnion(t *testing.T) {
@ -85,12 +176,12 @@ func TestMapsUnion(t *testing.T) {
"b": map[interface{}]interface{}{"c": 3}, "b": map[interface{}]interface{}{"c": 3},
"d": "replaced", "d": "replaced",
"e": "added", "e": "added",
"f": []interface{}{1, 2, 3, 4}, "f": []interface{}{2, 3, 4},
} }
assert.Equal(expected, MapsUnion(m0, m1, Replace)) assert.Equal(expected, MapsUnion(m0, m1))
} }
func TestLoadResourceSimple(t *testing.T) { func NoTestLoadResourceSimple(t *testing.T) {
assert := require.New(t) assert := require.New(t)
expected := `services: expected := `services: