rancher/os
1
0
Fork 1
mirror of https://github.com/rancher/os.git synced 2025-07-12 22:27:59 +00:00
Code Issues Releases Wiki Activity

Merge pull request #580 from imikushin/revamp-config

Browse Source 
reshuffle cloud-config
This commit is contained in:
Ivan Mikushin 2015-10-01 00:09:00 +05:00
commit 510a07cd62
20 changed files with 658 additions and 695 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

200
cmd/cloudinit/cloudinit.go
View File

@ -16,10 +16,10 @@
package cloudinit
import (
"errors"
"flag"
"io/ioutil"
"os"
"path"
"strings"
"sync"
"time"
@ -35,13 +35,11 @@ import (
"github.com/coreos/coreos-cloudinit/datasource/metadata/ec2"
"github.com/coreos/coreos-cloudinit/datasource/proc_cmdline"
"github.com/coreos/coreos-cloudinit/datasource/url"
"github.com/coreos/coreos-cloudinit/initialize"
"github.com/coreos/coreos-cloudinit/pkg"
"github.com/coreos/coreos-cloudinit/system"
"github.com/rancher/netconf"
"github.com/rancherio/os/cmd/cloudinit/hostname"
rancherConfig "github.com/rancherio/os/config"
"github.com/rancherio/os/util"
)
const (
@ -49,7 +47,6 @@ const (
datasourceMaxInterval = 30 * time.Second
datasourceTimeout = 5 * time.Minute
sshKeyName = "rancheros-cloud-config"
baseConfigDir = "/var/lib/rancher/conf/cloud-config.d"
)
var (
@ -67,8 +64,9 @@ func init() {
}
func saveFiles(cloudConfigBytes, scriptBytes []byte, metadata datasource.Metadata) error {
os.MkdirAll(rancherConfig.CloudConfigDir, os.ModeDir|0600)
os.Remove(rancherConfig.CloudConfigScriptFile)
os.Remove(rancherConfig.CloudConfigFile)
os.Remove(rancherConfig.CloudConfigBootFile)
os.Remove(rancherConfig.MetaDataFile)
if len(scriptBytes) > 0 {
@ -79,10 +77,10 @@ func saveFiles(cloudConfigBytes, scriptBytes []byte, metadata datasource.Metadat
}
}
if err := ioutil.WriteFile(rancherConfig.CloudConfigFile, cloudConfigBytes, 400); err != nil {
if err := ioutil.WriteFile(rancherConfig.CloudConfigBootFile, cloudConfigBytes, 400); err != nil {
return err
}
log.Infof("Written to %s:\n%s", rancherConfig.CloudConfigFile, string(cloudConfigBytes))
log.Infof("Written to %s:\n%s", rancherConfig.CloudConfigBootFile, string(cloudConfigBytes))
metaDataBytes, err := yaml.Marshal(metadata)
if err != nil {
@ -113,90 +111,12 @@ func currentDatasource() (datasource.Datasource, error) {
return ds, nil
}
func mergeBaseConfig(current, currentScript []byte) ([]byte, []byte, error) {
files, err := ioutil.ReadDir(baseConfigDir)
if err != nil {
if os.IsNotExist(err) {
log.Infof("%s does not exist, not merging", baseConfigDir)
return current, currentScript, nil
}
log.Errorf("Failed to read %s: %v", baseConfigDir, err)
return nil, nil, err
}
scriptResult := currentScript
result := []byte{}
for _, file := range files {
if file.IsDir() || strings.HasPrefix(file.Name(), ".") {
continue
}
input := path.Join(baseConfigDir, file.Name())
content, err := ioutil.ReadFile(input)
if err != nil {
log.Errorf("Failed to read %s: %v", input, err)
// ignore error
continue
}
if config.IsScript(string(content)) {
scriptResult = content
continue
}
log.Infof("Merging %s", input)
if isCompose(string(content)) {
content, err = toCompose(content)
if err != nil {
log.Errorf("Failed to convert %s to cloud-config syntax: %v", input, err)
}
}
result, err = util.MergeBytes(result, content)
if err != nil {
log.Errorf("Failed to merge bytes: %v", err)
return nil, nil, err
}
}
if len(result) == 0 {
return current, scriptResult, nil
} else {
result, err := util.MergeBytes(result, current)
return result, scriptResult, err
}
}
func saveCloudConfig() error {
var userDataBytes []byte
var metadata datasource.Metadata
ds, err := currentDatasource()
userDataBytes, metadata, err := fetchUserData()
if err != nil {
log.Errorf("Failed to select datasource: %v", err)
return err
}
if ds != nil {
log.Infof("Fetching user-data from datasource %v", ds.Type())
userDataBytes, err = ds.FetchUserdata()
if err != nil {
log.Errorf("Failed fetching user-data from datasource: %v", err)
return err
}
log.Infof("Fetching meta-data from datasource of type %v", ds.Type())
metadata, err = ds.FetchMetadata()
if err != nil {
log.Errorf("Failed fetching meta-data from datasource: %v", err)
return err
}
}
userDataBytes = substituteUserDataVars(userDataBytes, metadata)
userData := string(userDataBytes)
scriptBytes := []byte{}
@ -204,75 +124,56 @@ func saveCloudConfig() error {
scriptBytes = userDataBytes
userDataBytes = []byte{}
} else if isCompose(userData) {
if userDataBytes, err = toCompose(userDataBytes); err != nil {
log.Errorf("Failed to convert to compose syntax: %v", err)
if userDataBytes, err = composeToCloudConfig(userDataBytes); err != nil {
log.Errorf("Failed to convert compose to cloud-config syntax: %v", err)
return err
}
} else if config.IsCloudConfig(userData) {
if rancherConfig.ReadConfig(userDataBytes) == nil {
log.WithFields(log.Fields{"cloud-config": userData}).Warn("Failed to parse cloud-config, not saving.")
if _, err := rancherConfig.ReadConfig(userDataBytes, false); err != nil {
log.WithFields(log.Fields{"cloud-config": userData, "err": err}).Warn("Failed to parse cloud-config, not saving.")
userDataBytes = []byte{}
}
} else {
log.Errorf("Unrecognized cloud-init\n%s", userData)
log.Errorf("Unrecognized user-data\n%s", userData)
userDataBytes = []byte{}
}
userDataBytesMerged, scriptBytes, err := mergeBaseConfig(userDataBytes, scriptBytes)
if err != nil {
log.Errorf("Failed to merge base config: %v", err)
} else if rancherConfig.ReadConfig(userDataBytesMerged) == nil {
log.WithFields(log.Fields{"cloud-config": userData}).Warn("Failed to parse merged cloud-config, not merging.")
} else {
userDataBytes = userDataBytesMerged
if _, err := rancherConfig.ReadConfig(userDataBytes, false); err != nil {
log.WithFields(log.Fields{"cloud-config": userData, "err": err}).Warn("Failed to parse cloud-config")
return errors.New("Failed to parse cloud-config")
}
return saveFiles(userDataBytes, scriptBytes, metadata)
}
func getSaveCloudConfig() (*config.CloudConfig, error) {
ds := file.NewDatasource(rancherConfig.CloudConfigFile)
if !ds.IsAvailable() {
log.Infof("%s does not exist", rancherConfig.CloudConfigFile)
return nil, nil
func fetchUserData() ([]byte, datasource.Metadata, error) {
var metadata datasource.Metadata
ds, err := currentDatasource()
if err != nil || ds == nil {
log.Errorf("Failed to select datasource: %v", err)
return nil, metadata, err
}
ccBytes, err := ds.FetchUserdata()
log.Infof("Fetching user-data from datasource %v", ds.Type())
userDataBytes, err := ds.FetchUserdata()
if err != nil {
log.Errorf("Failed to read user-data from %s: %v", rancherConfig.CloudConfigFile, err)
return nil, err
log.Errorf("Failed fetching user-data from datasource: %v", err)
return nil, metadata, err
}
var cc config.CloudConfig
err = yaml.Unmarshal(ccBytes, &cc)
log.Infof("Fetching meta-data from datasource of type %v", ds.Type())
metadata, err = ds.FetchMetadata()
if err != nil {
log.Errorf("Failed to unmarshall user-data from %s: %v", rancherConfig.CloudConfigFile, err)
return nil, err
log.Errorf("Failed fetching meta-data from datasource: %v", err)
return nil, metadata, err
}
return &cc, err
return userDataBytes, metadata, nil
}
func executeCloudConfig() error {
ccu, err := getSaveCloudConfig()
cc, err := rancherConfig.LoadConfig()
if err != nil {
return err
}
var metadata datasource.Metadata
metaDataBytes, err := ioutil.ReadFile(rancherConfig.MetaDataFile)
if err != nil {
return err
}
if err = yaml.Unmarshal(metaDataBytes, &metadata); err != nil {
return err
}
log.Info("Merging cloud-config from meta-data and user-data")
cc := mergeConfigs(ccu, metadata)
if cc.Hostname != "" {
//set hostname
if err := hostname.SetHostname(cc.Hostname); err != nil {
@ -285,15 +186,6 @@ func executeCloudConfig() error {
authorizeSSHKeys("docker", cc.SSHAuthorizedKeys, sshKeyName)
}
for _, user := range cc.Users {
if user.Name == "" {
continue
}
if len(user.SSHAuthorizedKeys) > 0 {
authorizeSSHKeys(user.Name, user.SSHAuthorizedKeys, sshKeyName)
}
}
for _, file := range cc.WriteFiles {
f := system.File{File: file}
fullPath, err := system.WriteFile(&f, "/")
@ -308,7 +200,7 @@ func executeCloudConfig() error {
}
func Main() {
flags.Parse(rancherConfig.FilterGlobalConfig(os.Args[1:]))
flags.Parse(os.Args[1:])
log.Infof("Running cloud-init: save=%v, execute=%v", save, execute)
@ -327,27 +219,6 @@ func Main() {
}
}
// mergeConfigs merges certain options from md (meta-data from the datasource)
// onto cc (a CloudConfig derived from user-data), if they are not already set
// on cc (i.e. user-data always takes precedence)
func mergeConfigs(cc *config.CloudConfig, md datasource.Metadata) (out config.CloudConfig) {
if cc != nil {
out = *cc
}
if md.Hostname != "" {
if out.Hostname != "" {
log.Infof("Warning: user-data hostname (%s) overrides metadata hostname (%s)\n", out.Hostname, md.Hostname)
} else {
out.Hostname = md.Hostname
}
}
for _, key := range md.SSHPublicKeys {
out.SSHAuthorizedKeys = append(out.SSHAuthorizedKeys, key)
}
return
}
// getDatasources creates a slice of possible Datasources for cloudinit based
// on the different source command-line flags.
func getDatasources(cfg *rancherConfig.CloudConfig) []datasource.Datasource {
@ -478,7 +349,7 @@ func isCompose(content string) bool {
return strings.HasPrefix(content, "#compose\n")
}
func toCompose(bytes []byte) ([]byte, error) {
func composeToCloudConfig(bytes []byte) ([]byte, error) {
compose := make(map[interface{}]interface{})
err := yaml.Unmarshal(bytes, &compose)
if err != nil {
@ -491,10 +362,3 @@ func toCompose(bytes []byte) ([]byte, error) {
},
})
}
func substituteUserDataVars(userDataBytes []byte, metadata datasource.Metadata) []byte {
env := initialize.NewEnvironment("", "", "", "", metadata)
userData := env.Apply(string(userDataBytes))
return []byte(userData)
}

34
cmd/control/config.go
View File

@ -57,13 +57,17 @@ func configSubcommands() []cli.Command {
Name: "output, o",
Usage: "File to which to save",
},
cli.BoolFlag{
Name: "boot, b",
Usage: "Include cloud-config provided at boot",
},
cli.BoolFlag{
Name: "private, p",
Usage: "Include private information such as keys",
Usage: "Include the generated private keys",
},
cli.BoolFlag{
Name: "full, f",
Usage: "Include full configuration, including internal and default settings",
Usage: "Export full configuration, including internal and default settings",
},
},
Action: export,
@ -101,9 +105,9 @@ func imagesFromConfig(cfg *config.CloudConfig) []string {
func runImages(c *cli.Context) {
configFile := c.String("input")
cfg := config.ReadConfig(nil, configFile)
if cfg == nil {
log.Fatalf("Could not read config from file %v", configFile)
cfg, err := config.ReadConfig(nil, false, configFile)
if err != nil {
log.WithFields(log.Fields{"err": err, "file": configFile}).Fatalf("Could not read config from file")
}
images := imagesFromConfig(cfg)
fmt.Println(strings.Join(images, " "))
@ -133,10 +137,14 @@ func runImport(c *cli.Context) {
log.Fatal(err)
}
err = cfg.Import(bytes)
cfg, err = cfg.Import(bytes)
if err != nil {
log.Fatal(err)
}
if err := cfg.Save(); err != nil {
log.Fatal(err)
}
}
func configSet(c *cli.Context) {
@ -151,10 +159,14 @@ func configSet(c *cli.Context) {
log.Fatal(err)
}
err = cfg.Set(key, value)
cfg, err = cfg.Set(key, value)
if err != nil {
log.Fatal(err)
}
if err := cfg.Save(); err != nil {
log.Fatal(err)
}
}
func configGet(c *cli.Context) {
@ -203,14 +215,18 @@ func merge(c *cli.Context) {
log.Fatal(err)
}
err = cfg.Merge(bytes)
cfg, err = cfg.MergeBytes(bytes)
if err != nil {
log.Fatal(err)
}
if err := cfg.Save(); err != nil {
log.Fatal(err)
}
}
func export(c *cli.Context) {
content, err := config.Dump(c.Bool("private"), c.Bool("full"))
content, err := config.Dump(c.Bool("boot"), c.Bool("private"), c.Bool("full"))
if err != nil {
log.Fatal(err)
}

4
cmd/control/install.go
View File

@ -86,12 +86,12 @@ func installAction(c *cli.Context) {
force := c.Bool("force")
reboot := !c.Bool("no-reboot")
if err := runInstall(cfg, image, installType, cloudConfig, device, force, reboot); err != nil {
if err := runInstall(image, installType, cloudConfig, device, force, reboot); err != nil {
log.WithFields(log.Fields{"err": err}).Fatal("Failed to run install")
}
}
func runInstall(cfg *config.CloudConfig, image, installType, cloudConfig, device string, force, reboot bool) error {
func runInstall(image, installType, cloudConfig, device string, force, reboot bool) error {
in := bufio.NewReader(os.Stdin)
fmt.Printf("Installing from %s\n", image)

6
cmd/control/service.go
View File

@ -53,7 +53,7 @@ func disable(c *cli.Context) {
}
if changed {
if err = cfg.Set("rancher.services_include", cfg.Rancher.ServicesInclude); err != nil {
if err = cfg.Save(); err != nil {
log.Fatal(err)
}
}
@ -75,7 +75,7 @@ func del(c *cli.Context) {
}
if changed {
if err = cfg.Set("rancher.services_include", cfg.Rancher.ServicesInclude); err != nil {
if err = cfg.Save(); err != nil {
log.Fatal(err)
}
}
@ -102,7 +102,7 @@ func enable(c *cli.Context) {
}
if changed {
if err := cfg.Set("rancher.services_include", cfg.Rancher.ServicesInclude); err != nil {
if err := cfg.Save(); err != nil {
log.Fatal(err)
}
}

31
cmd/control/tlsconf.go
View File

@ -65,16 +65,21 @@ func writeCerts(generateServer bool, hostname []string, cfg *config.CloudConfig,
return err
}
return cfg.SetConfig(&config.CloudConfig{
Rancher: config.RancherConfig{
Docker: config.DockerConfig{
CAKey: cfg.Rancher.Docker.CAKey,
CACert: cfg.Rancher.Docker.CACert,
ServerCert: string(cert),
ServerKey: string(key),
cfg, err = cfg.Merge(map[interface{}]interface{}{
"rancher": map[interface{}]interface{}{
"docker": map[interface{}]interface{}{
"ca_key": cfg.Rancher.Docker.CAKey,
"ca_cert": cfg.Rancher.Docker.CACert,
"server_cert": string(cert),
"server_key": string(key),
},
},
})
if err != nil {
return err
}
return cfg.Save()
}
if err := ioutil.WriteFile(certPath, []byte(cfg.Rancher.Docker.ServerCert), 0400); err != nil {
@ -101,11 +106,11 @@ func writeCaCerts(cfg *config.CloudConfig, caCertPath, caKeyPath string) error {
return err
}
err = cfg.SetConfig(&config.CloudConfig{
Rancher: config.RancherConfig{
Docker: config.DockerConfig{
CAKey: string(caKey),
CACert: string(caCert),
cfg, err = cfg.Merge(map[interface{}]interface{}{
"rancher": map[interface{}]interface{}{
"docker": map[interface{}]interface{}{
"ca_key": string(caKey),
"ca_cert": string(caCert),
},
},
})
@ -113,7 +118,7 @@ func writeCaCerts(cfg *config.CloudConfig, caCertPath, caKeyPath string) error {
return err
}
return nil
return cfg.Save()
}
if err := ioutil.WriteFile(caCertPath, []byte(cfg.Rancher.Docker.CACert), 0400); err != nil {

19
compose/project.go
View File

@ -35,7 +35,7 @@ func RunServiceSet(name string, cfg *config.CloudConfig, configs map[string]*pro
return nil, err
}
addServices(p, cfg, map[string]string{}, configs)
addServices(p, map[interface{}]interface{}{}, configs)
return p, p.Up()
}
@ -78,8 +78,9 @@ func newProject(name string, cfg *config.CloudConfig) (*project.Project, error)
return docker.NewProject(context)
}
func addServices(p *project.Project, cfg *config.CloudConfig, enabled map[string]string, configs map[string]*project.ServiceConfig) {
func addServices(p *project.Project, enabled map[interface{}]interface{}, configs map[string]*project.ServiceConfig) map[interface{}]interface{} {
// Note: we ignore errors while loading services
unchanged := true
for name, serviceConfig := range configs {
hash := project.GetServiceHash(name, *serviceConfig)
@ -92,14 +93,19 @@ func addServices(p *project.Project, cfg *config.CloudConfig, enabled map[string
continue
}
if unchanged {
enabled = util.MapCopy(enabled)
unchanged = false
}
enabled[name] = hash
}
return enabled
}
func newCoreServiceProject(cfg *config.CloudConfig) (*project.Project, error) {
network := false
projectEvents := make(chan project.ProjectEvent)
enabled := make(map[string]string)
enabled := map[interface{}]interface{}{}
p, err := newProject("os", cfg)
if err != nil {
@ -110,15 +116,16 @@ func newCoreServiceProject(cfg *config.CloudConfig) (*project.Project, error) {
p.AddListener(projectEvents)
p.ReloadCallback = func() error {
err := cfg.Reload()
var err error
cfg, err = config.LoadConfig()
if err != nil {
return err
}
addServices(p, cfg, enabled, cfg.Rancher.Services)
enabled = addServices(p, enabled, cfg.Rancher.Services)
for service, serviceEnabled := range cfg.Rancher.ServicesInclude {
if enabled[service] != "" || !serviceEnabled {
if _, ok := enabled[service]; ok || !serviceEnabled {
continue
}

226
config/config.go
View File

@ -1,212 +1,116 @@
package config
import (
"io/ioutil"
"strings"
log "github.com/Sirupsen/logrus"
"github.com/docker/libcompose/project"
"github.com/rancherio/os/util"
"gopkg.in/yaml.v2"
)
func (c *CloudConfig) Import(bytes []byte) error {
data, err := readConfig(bytes, PrivateConfigFile)
func (c *CloudConfig) Import(bytes []byte) (*CloudConfig, error) {
data, err := readConfig(bytes, false, CloudConfigPrivateFile)
if err != nil {
return err
return c, err
}
if err := saveToDisk(data); err != nil {
return err
}
return c.Reload()
}
// This function only sets "non-empty" values
func (c *CloudConfig) SetConfig(newConfig *CloudConfig) error {
bytes, err := yaml.Marshal(newConfig)
if err != nil {
return err
}
return c.Merge(bytes)
}
func (c *CloudConfig) Merge(bytes []byte) error {
data, err := readConfig(bytes, LocalConfigFile, PrivateConfigFile)
if err != nil {
return err
}
if err := saveToDisk(data); err != nil {
return err
}
return c.Reload()
}
func LoadConfig() (*CloudConfig, error) {
cfg := NewConfig()
if err := cfg.Reload(); err != nil {
log.WithFields(log.Fields{"cfg": cfg, "err": err}).Error("Failed to reload config")
return nil, err
}
if cfg.Rancher.Debug {
log.SetLevel(log.DebugLevel)
if !util.Contains(cfg.Rancher.Docker.Args, "-D") {
cfg.Rancher.Docker.Args = append(cfg.Rancher.Docker.Args, "-D")
}
if !util.Contains(cfg.Rancher.SystemDocker.Args, "-D") {
cfg.Rancher.SystemDocker.Args = append(cfg.Rancher.SystemDocker.Args, "-D")
}
if err := util.Convert(data, cfg); err != nil {
return c, err
}
return cfg, nil
}
func (c *CloudConfig) merge(values map[interface{}]interface{}) error {
t := &CloudConfig{}
if err := util.Convert(values, t); err != nil {
return err
}
return util.Convert(values, c)
}
func (c *CloudConfig) readFiles() error {
data, err := readConfig(nil, CloudConfigFile, LocalConfigFile, PrivateConfigFile)
func (c *CloudConfig) MergeBytes(bytes []byte) (*CloudConfig, error) {
data, err := readConfig(bytes, false)
if err != nil {
log.WithFields(log.Fields{"err": err}).Error("Error reading config files")
return err
return c, err
}
if err := c.merge(data); err != nil {
log.WithFields(log.Fields{"cfg": c, "data": data, "err": err}).Error("Error merging config data")
return err
}
return nil
return c.Merge(data)
}
func (c *CloudConfig) readCmdline() error {
log.Debug("Reading config cmdline")
cmdLine, err := ioutil.ReadFile("/proc/cmdline")
if err != nil {
log.WithFields(log.Fields{"err": err}).Error("Failed to read kernel params")
return err
func (c *CloudConfig) Merge(values map[interface{}]interface{}) (*CloudConfig, error) {
t := *c
if err := util.Convert(values, &t); err != nil {
return c, err
}
if len(cmdLine) == 0 {
return nil
}
log.Debugf("Config cmdline %s", cmdLine)
cmdLineObj := parseCmdline(strings.TrimSpace(string(cmdLine)))
if err := c.merge(cmdLineObj); err != nil {
log.WithFields(log.Fields{"cfg": c, "cmdLine": cmdLine, "data": cmdLineObj, "err": err}).Warn("Error adding kernel params to config")
}
return nil
return &t, nil
}
func Dump(private, full bool) (string, error) {
files := []string{CloudConfigFile, LocalConfigFile}
if private {
files = append(files, PrivateConfigFile)
}
c := &CloudConfig{}
func Dump(boot, private, full bool) (string, error) {
var cfg *CloudConfig
var err error
if full {
c = NewConfig()
cfg, err = LoadConfig()
} else {
files := []string{CloudConfigBootFile, CloudConfigPrivateFile, CloudConfigFile}
if !private {
files = util.FilterStrings(files, func(x string) bool { return x != CloudConfigPrivateFile })
}
if !boot {
files = util.FilterStrings(files, func(x string) bool { return x != CloudConfigBootFile })
}
cfg, err = ChainCfgFuncs(nil,
func(_ *CloudConfig) (*CloudConfig, error) { return ReadConfig(nil, true, files...) },
amendNils,
)
}
data, err := readConfig(nil, files...)
if err != nil {
return "", err
}
if err := c.merge(data); err != nil {
return "", err
}
if err := c.readCmdline(); err != nil {
return "", err
}
c.amendNils()
bytes, err := yaml.Marshal(c)
bytes, err := yaml.Marshal(*cfg)
return string(bytes), err
}
func (c *CloudConfig) amendNils() error {
if c.Rancher.Environment == nil {
c.Rancher.Environment = map[string]string{}
}
if c.Rancher.Autoformat == nil {
c.Rancher.Autoformat = map[string]*project.ServiceConfig{}
}
if c.Rancher.BootstrapContainers == nil {
c.Rancher.BootstrapContainers = map[string]*project.ServiceConfig{}
}
if c.Rancher.Services == nil {
c.Rancher.Services = map[string]*project.ServiceConfig{}
}
if c.Rancher.ServicesInclude == nil {
c.Rancher.ServicesInclude = map[string]bool{}
}
return nil
}
func (c *CloudConfig) Reload() error {
return util.ShortCircuit(
c.readFiles,
c.readCmdline,
c.amendNils,
)
}
func (c *CloudConfig) Get(key string) (interface{}, error) {
data := make(map[interface{}]interface{})
err := util.Convert(c, &data)
if err != nil {
data := map[interface{}]interface{}{}
if err := util.Convert(c, &data); err != nil {
return nil, err
}
return getOrSetVal(key, data, nil), nil
v, _ := getOrSetVal(key, data, nil)
return v, nil
}
func (c *CloudConfig) Set(key string, value interface{}) error {
data, err := readConfig(nil, LocalConfigFile, PrivateConfigFile)
func (c *CloudConfig) Set(key string, value interface{}) (*CloudConfig, error) {
data := map[interface{}]interface{}{}
if err := util.Convert(c, &data); err != nil {
return c, err
}
_, data = getOrSetVal(key, data, value)
return c.Merge(data)
}
func (c *CloudConfig) Save() error {
files := append([]string{OsConfigFile}, CloudConfigDirFiles()...)
files = util.FilterStrings(files, func(x string) bool { return x != CloudConfigPrivateFile })
exCfg, err := ChainCfgFuncs(nil,
func(_ *CloudConfig) (*CloudConfig, error) {
return ReadConfig(nil, true, files...)
},
readCmdline,
amendNils)
if err != nil {
return err
}
getOrSetVal(key, data, value)
cfg := NewConfig()
if err := util.Convert(data, cfg); err != nil {
exData := map[interface{}]interface{}{}
if err := util.Convert(exCfg, &exData); err != nil {
return err
}
data := map[interface{}]interface{}{}
if err := util.Convert(c, &data); err != nil {
return err
}
data = util.MapsDifference(data, exData)
log.WithFields(log.Fields{"diff": data}).Debug("The diff we're about to save")
if err := saveToDisk(data); err != nil {
return err
}
return c.Reload()
}
func (r Repositories) ToArray() []string {
result := make([]string, 0, len(r))
for _, repo := range r {
if repo.Url != "" {
result = append(result, repo.Url)
}
}
return result
return nil
}

13
config/config_test.go
View File

@ -152,7 +152,8 @@ func TestGet(t *testing.T) {
}
for k, v := range tests {
assert.Equal(v, getOrSetVal(k, data, nil))
val, _ := getOrSetVal(k, data, nil)
assert.Equal(v, val)
}
}
@ -195,8 +196,10 @@ func TestSet(t *testing.T) {
}
for k, v := range tests {
getOrSetVal(k, data, v)
assert.Equal(v, getOrSetVal(k, data, nil))
_, tData := getOrSetVal(k, data, v)
val, _ := getOrSetVal(k, tData, nil)
data = tData
assert.Equal(v, val)
}
assert.Equal(expected, data)
@ -267,8 +270,8 @@ func TestUserDocker(t *testing.T) {
err = yaml.Unmarshal(bytes, config)
assert.Nil(err)
data := make(map[interface{}]map[interface{}]interface{})
util.Convert(config, data)
data := map[interface{}]map[interface{}]interface{}{}
util.Convert(config, &data)
fmt.Println(data)

42
config/data_funcs.go
View File

@ -9,6 +9,21 @@ import (
"strings"
)
type CfgFunc func(*CloudConfig) (*CloudConfig, error)
func ChainCfgFuncs(cfg *CloudConfig, cfgFuncs ...CfgFunc) (*CloudConfig, error) {
for i, cfgFunc := range cfgFuncs {
log.Debugf("[%d/%d] Starting", i+1, len(cfgFuncs))
var err error
if cfg, err = cfgFunc(cfg); err != nil {
log.Errorf("Failed [%d/%d] %d%%", i+1, len(cfgFuncs), ((i + 1) * 100 / len(cfgFuncs)))
return cfg, err
}
log.Debugf("[%d/%d] Done %d%%", i+1, len(cfgFuncs), ((i + 1) * 100 / len(cfgFuncs)))
}
return cfg, nil
}
func filterKey(data map[interface{}]interface{}, key []string) (filtered, rest map[interface{}]interface{}) {
if len(key) == 0 {
return data, map[interface{}]interface{}{}
@ -50,18 +65,23 @@ func filterDottedKeys(data map[interface{}]interface{}, keys []string) (filtered
for _, key := range keys {
f, r := filterKey(data, strings.Split(key, "."))
filtered = util.MapsUnion(filtered, f, util.Replace)
rest = util.MapsIntersection(rest, r, util.Equal)
filtered = util.MapsUnion(filtered, f)
rest = util.MapsIntersection(rest, r)
}
return
}
func getOrSetVal(args string, data map[interface{}]interface{}, value interface{}) interface{} {
func getOrSetVal(args string, data map[interface{}]interface{}, value interface{}) (interface{}, map[interface{}]interface{}) {
parts := strings.Split(args, ".")
tData := data
if value != nil {
tData = util.MapCopy(data)
}
t := tData
for i, part := range parts {
val, ok := data[part]
val, ok := t[part]
last := i+1 == len(parts)
// Reached end, set the value
@ -70,15 +90,15 @@ func getOrSetVal(args string, data map[interface{}]interface{}, value interface{
value = DummyMarshall(s)
}
data[part] = value
return value
t[part] = value
return value, tData
}
// Missing intermediate key, create key
if !last && value != nil && !ok {
newData := map[interface{}]interface{}{}
data[part] = newData
data = newData
t[part] = newData
t = newData
continue
}
@ -87,7 +107,7 @@ func getOrSetVal(args string, data map[interface{}]interface{}, value interface{
}
if last {
return val
return val, tData
}
newData, ok := val.(map[interface{}]interface{})
@ -95,10 +115,10 @@ func getOrSetVal(args string, data map[interface{}]interface{}, value interface{
break
}
data = newData
t = newData
}
return ""
return "", tData
}
func DummyMarshall(value string) interface{} {

18
config/default.go
View File

@ -1,18 +0,0 @@
package config
func NewConfig() *CloudConfig {
return ReadConfig(nil, OsConfigFile)
}
func ReadConfig(bytes []byte, files ...string) *CloudConfig {
if data, err := readConfig(bytes, files...); err == nil {
c := &CloudConfig{}
if err := c.merge(data); err != nil {
return nil
}
c.amendNils()
return c
} else {
return nil
}
}

206
config/disk.go
View File

@ -3,11 +3,189 @@ package config
import (
"io/ioutil"
"os"
"path"
"strings"
log "github.com/Sirupsen/logrus"
"github.com/coreos/coreos-cloudinit/datasource"
"github.com/coreos/coreos-cloudinit/initialize"
"github.com/docker/libcompose/project"
"github.com/rancherio/os/util"
"gopkg.in/yaml.v2"
)
var osConfig *CloudConfig
func NewConfig() *CloudConfig {
if osConfig == nil {
osConfig, _ = ReadConfig(nil, true, OsConfigFile)
}
newCfg := *osConfig
return &newCfg
}
func ReadConfig(bytes []byte, substituteMetadataVars bool, files ...string) (*CloudConfig, error) {
if data, err := readConfig(bytes, substituteMetadataVars, files...); err == nil {
c := &CloudConfig{}
c, err := c.Merge(data)
if err != nil {
return nil, err
}
c, _ = amendNils(c)
return c, nil
} else {
return nil, err
}
}
func LoadConfig() (*CloudConfig, error) {
cfg, err := ChainCfgFuncs(NewConfig(),
readFilesAndMetadata,
readCmdline,
amendNils)
if err != nil {
log.WithFields(log.Fields{"cfg": cfg, "err": err}).Error("Failed to load config")
return nil, err
}
log.Debug("Merging cloud-config from meta-data and user-data")
cfg = mergeMetadata(cfg, readMetadata())
if cfg.Rancher.Debug {
log.SetLevel(log.DebugLevel)
if !util.Contains(cfg.Rancher.Docker.Args, "-D") {
cfg.Rancher.Docker.Args = append(cfg.Rancher.Docker.Args, "-D")
}
if !util.Contains(cfg.Rancher.SystemDocker.Args, "-D") {
cfg.Rancher.SystemDocker.Args = append(cfg.Rancher.SystemDocker.Args, "-D")
}
}
return cfg, nil
}
func CloudConfigDirFiles() []string {
files, err := util.DirLs(CloudConfigDir)
if err != nil {
if os.IsNotExist(err) {
// do nothing
log.Debugf("%s does not exist", CloudConfigDir)
} else {
log.Errorf("Failed to read %s: %v", CloudConfigDir, err)
}
return []string{}
}
files = util.Filter(files, func(x interface{}) bool {
f := x.(os.FileInfo)
if f.IsDir() || strings.HasPrefix(f.Name(), ".") {
return false
}
return true
})
return util.ToStrings(util.Map(files, func(x interface{}) interface{} {
return path.Join(CloudConfigDir, x.(os.FileInfo).Name())
}))
}
// mergeMetadata merges certain options from md (meta-data from the datasource)
// onto cc (a CloudConfig derived from user-data), if they are not already set
// on cc (i.e. user-data always takes precedence)
func mergeMetadata(cc *CloudConfig, md datasource.Metadata) *CloudConfig {
if cc == nil {
return cc
}
out := cc
dirty := false
if md.Hostname != "" {
if out.Hostname != "" {
log.Debugf("Warning: user-data hostname (%s) overrides metadata hostname (%s)\n", out.Hostname, md.Hostname)
} else {
out = &(*cc)
dirty = true
out.Hostname = md.Hostname
}
}
for _, key := range md.SSHPublicKeys {
if !dirty {
out = &(*cc)
dirty = true
}
out.SSHAuthorizedKeys = append(out.SSHAuthorizedKeys, key)
}
return out
}
func readMetadata() datasource.Metadata {
metadata := datasource.Metadata{}
if metaDataBytes, err := ioutil.ReadFile(MetaDataFile); err == nil {
yaml.Unmarshal(metaDataBytes, &metadata)
}
return metadata
}
func readFilesAndMetadata(c *CloudConfig) (*CloudConfig, error) {
files := append(CloudConfigDirFiles(), CloudConfigFile)
data, err := readConfig(nil, true, files...)
if err != nil {
log.WithFields(log.Fields{"err": err}).Error("Error reading config files")
return c, err
}
t, err := c.Merge(data)
if err != nil {
log.WithFields(log.Fields{"cfg": c, "data": data, "err": err}).Error("Error merging config data")
return c, err
}
return t, nil
}
func readCmdline(c *CloudConfig) (*CloudConfig, error) {
log.Debug("Reading config cmdline")
cmdLine, err := ioutil.ReadFile("/proc/cmdline")
if err != nil {
log.WithFields(log.Fields{"err": err}).Error("Failed to read kernel params")
return c, err
}
if len(cmdLine) == 0 {
return c, nil
}
log.Debugf("Config cmdline %s", cmdLine)
cmdLineObj := parseCmdline(strings.TrimSpace(string(cmdLine)))
t, err := c.Merge(cmdLineObj)
if err != nil {
log.WithFields(log.Fields{"cfg": c, "cmdLine": cmdLine, "data": cmdLineObj, "err": err}).Warn("Error adding kernel params to config")
}
return t, nil
}
func amendNils(c *CloudConfig) (*CloudConfig, error) {
t := *c
if t.Rancher.Environment == nil {
t.Rancher.Environment = map[string]string{}
}
if t.Rancher.Autoformat == nil {
t.Rancher.Autoformat = map[string]*project.ServiceConfig{}
}
if t.Rancher.BootstrapContainers == nil {
t.Rancher.BootstrapContainers = map[string]*project.ServiceConfig{}
}
if t.Rancher.Services == nil {
t.Rancher.Services = map[string]*project.ServiceConfig{}
}
if t.Rancher.ServicesInclude == nil {
t.Rancher.ServicesInclude = map[string]bool{}
}
return &t, nil
}
func writeToFile(data interface{}, filename string) error {
content, err := yaml.Marshal(data)
if err != nil {
@ -26,23 +204,27 @@ func saveToDisk(data map[interface{}]interface{}) error {
"rancher.docker.server_cert",
})
err := writeToFile(config, LocalConfigFile)
err := writeToFile(config, CloudConfigFile)
if err != nil {
return err
}
return writeToFile(private, PrivateConfigFile)
return writeToFile(private, CloudConfigPrivateFile)
}
func readConfig(bytes []byte, files ...string) (map[interface{}]interface{}, error) {
func readConfig(bytes []byte, substituteMetadataVars bool, files ...string) (map[interface{}]interface{}, error) {
// You can't just overlay yaml bytes on to maps, it won't merge, but instead
// just override the keys and not merge the map values.
left := make(map[interface{}]interface{})
for _, conf := range files {
content, err := readConfigFile(conf)
metadata := readMetadata()
for _, file := range files {
content, err := readConfigFile(file)
if err != nil {
return nil, err
}
if substituteMetadataVars {
content = substituteVars(content, metadata)
}
right := make(map[interface{}]interface{})
err = yaml.Unmarshal(content, &right)
@ -50,16 +232,19 @@ func readConfig(bytes []byte, files ...string) (map[interface{}]interface{}, err
return nil, err
}
left = util.MapsUnion(left, right, util.Replace)
left = util.MapsUnion(left, right)
}
if bytes != nil && len(bytes) > 0 {
right := make(map[interface{}]interface{})
if substituteMetadataVars {
bytes = substituteVars(bytes, metadata)
}
if err := yaml.Unmarshal(bytes, &right); err != nil {
return nil, err
}
left = util.MapsUnion(left, right, util.Replace)
left = util.MapsUnion(left, right)
}
return left, nil
@ -79,3 +264,10 @@ func readConfigFile(file string) ([]byte, error) {
return content, err
}
func substituteVars(userDataBytes []byte, metadata datasource.Metadata) []byte {
env := initialize.NewEnvironment("", "", "", "", metadata)
userData := env.Apply(string(userDataBytes))
return []byte(userData)
}

32
config/init.go
View File

@ -1,32 +0,0 @@
package config
import (
"strings"
log "github.com/Sirupsen/logrus"
)
type InitFunc func(*CloudConfig) error
func RunInitFuncs(cfg *CloudConfig, initFuncs []InitFunc) error {
for i, initFunc := range initFuncs {
log.Debugf("[%d/%d] Starting", i+1, len(initFuncs))
if err := initFunc(cfg); err != nil {
log.Errorf("Failed [%d/%d] %d%%", i+1, len(initFuncs), ((i + 1) * 100 / len(initFuncs)))
return err
}
log.Debugf("[%d/%d] Done %d%%", i+1, len(initFuncs), ((i + 1) * 100 / len(initFuncs)))
}
return nil
}
func FilterGlobalConfig(input []string) []string {
result := make([]string, 0, len(input))
for _, value := range input {
if !strings.HasPrefix(value, "--rancher") {
result = append(result, value)
}
}
return result
}

4
cmd/cloudinit/cloudinit_test.go → config/metadata_test.go
View File

@ -13,7 +13,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package cloudinit
package config
import (
"net"
@ -92,7 +92,7 @@ func TestSubstituteUserDataVars(t *testing.T) {
},
} {
got := substituteUserDataVars([]byte(tt.input), tt.metadata)
got := substituteVars([]byte(tt.input), tt.metadata)
if string(got) != tt.out {
t.Fatalf("Userdata substitution incorrectly applied.\ngot:\n%s\nwant:\n%s", got, tt.out)
}

30
config/types.go
View File

@ -28,18 +28,25 @@ const (
SCOPE = "io.rancher.os.scope"
SYSTEM = "system"
OsConfigFile = "/usr/share/ros/os-config.yml"
CloudConfigFile = "/var/lib/rancher/conf/cloud-config.yml"
CloudConfigScriptFile = "/var/lib/rancher/conf/cloud-config-script"
MetaDataFile = "/var/lib/rancher/conf/metadata"
LocalConfigFile = "/var/lib/rancher/conf/cloud-config-local.yml"
PrivateConfigFile = "/var/lib/rancher/conf/cloud-config-private.yml"
OsConfigFile = "/usr/share/ros/os-config.yml"
CloudConfigDir = "/var/lib/rancher/conf/cloud-config.d"
CloudConfigBootFile = "/var/lib/rancher/conf/cloud-config.d/boot.yml"
CloudConfigPrivateFile = "/var/lib/rancher/conf/cloud-config.d/private.yml"
CloudConfigScriptFile = "/var/lib/rancher/conf/cloud-config-script"
MetaDataFile = "/var/lib/rancher/conf/metadata"
CloudConfigFile = "/var/lib/rancher/conf/cloud-config.yml"
)
var (
VERSION string
)
func init() {
if VERSION == "" {
VERSION = "v0.0.0-dev"
}
}
type ContainerConfig struct {
Id string `yaml:"id,omitempty"`
Cmd string `yaml:"run,omitempty"`
@ -119,8 +126,13 @@ type CloudInit struct {
Datasources []string `yaml:"datasources,omitempty"`
}
func init() {
if VERSION == "" {
VERSION = "v0.0.0-dev"
func (r Repositories) ToArray() []string {
result := make([]string, 0, len(r))
for _, repo := range r {
if repo.Url != "" {
result = append(result, repo.Url)
}
}
return result
}

27
init/bootstrap.go
View File

@ -15,22 +15,23 @@ import (
"github.com/rancherio/os/util"
)
func autoformat(cfg *config.CloudConfig) error {
func autoformat(cfg *config.CloudConfig) (*config.CloudConfig, error) {
if len(cfg.Rancher.State.Autoformat) == 0 || util.ResolveDevice(cfg.Rancher.State.Dev) != "" {
return nil
return cfg, nil
}
AUTOFORMAT := "AUTOFORMAT=" + strings.Join(cfg.Rancher.State.Autoformat, " ")
FORMATZERO := "FORMATZERO=" + fmt.Sprint(cfg.Rancher.State.FormatZero)
cfg.Rancher.Autoformat["autoformat"].Environment = project.NewMaporEqualSlice([]string{AUTOFORMAT, FORMATZERO})
t := *cfg
t.Rancher.Autoformat["autoformat"].Environment = project.NewMaporEqualSlice([]string{AUTOFORMAT, FORMATZERO})
log.Info("Running Autoformat services")
_, err := compose.RunServiceSet("autoformat", cfg, cfg.Rancher.Autoformat)
return err
_, err := compose.RunServiceSet("autoformat", &t, t.Rancher.Autoformat)
return &t, err
}
func runBootstrapContainers(cfg *config.CloudConfig) error {
func runBootstrapContainers(cfg *config.CloudConfig) (*config.CloudConfig, error) {
log.Info("Running Bootstrap services")
_, err := compose.RunServiceSet("bootstrap", cfg, cfg.Rancher.BootstrapContainers)
return err
return cfg, err
}
func startDocker(cfg *config.CloudConfig) (chan interface{}, error) {
@ -70,13 +71,11 @@ func bootstrap(cfg *config.CloudConfig) error {
return err
}
initFuncs := []config.InitFunc{
loadImages,
runBootstrapContainers,
autoformat,
}
defer stopDocker(c)
return config.RunInitFuncs(cfg, initFuncs)
_, err = config.ChainCfgFuncs(cfg,
loadImages,
runBootstrapContainers,
autoformat)
return err
}

54
init/init.go
View File

@ -30,12 +30,12 @@ var (
}
)
func loadModules(cfg *config.CloudConfig) error {
func loadModules(cfg *config.CloudConfig) (*config.CloudConfig, error) {
mounted := map[string]bool{}
f, err := os.Open("/proc/modules")
if err != nil {
return err
return cfg, err
}
defer f.Close()
@ -55,10 +55,10 @@ func loadModules(cfg *config.CloudConfig) error {
}
}
return nil
return cfg, nil
}
func sysInit(cfg *config.CloudConfig) error {
func sysInit(c *config.CloudConfig) (*config.CloudConfig, error) {
args := append([]string{config.SYSINIT_BIN}, os.Args[1:]...)
cmd := &exec.Cmd{
@ -71,10 +71,10 @@ func sysInit(cfg *config.CloudConfig) error {
cmd.Stdout = os.Stdout
if err := cmd.Start(); err != nil {
return err
return c, err
}
return os.Stdin.Close()
return c, os.Stdin.Close()
}
func MainInit() {
@ -121,15 +121,15 @@ func tryMountState(cfg *config.CloudConfig) error {
return mountState(cfg)
}
func tryMountAndBootstrap(cfg *config.CloudConfig) error {
func tryMountAndBootstrap(cfg *config.CloudConfig) (*config.CloudConfig, error) {
if err := tryMountState(cfg); !cfg.Rancher.State.Required && err != nil {
return nil
return cfg, nil
} else if err != nil {
return err
return cfg, err
}
log.Debugf("Switching to new root at %s", STATE)
return switchRoot(STATE)
return cfg, switchRoot(STATE)
}
func getLaunchConfig(cfg *config.CloudConfig, dockerCfg *config.DockerConfig) (*dockerlaunch.Config, []string) {
@ -150,27 +150,22 @@ func getLaunchConfig(cfg *config.CloudConfig, dockerCfg *config.DockerConfig) (*
}
func RunInit() error {
var cfg config.CloudConfig
os.Setenv("PATH", "/sbin:/usr/sbin:/usr/bin")
// Magic setting to tell Docker to do switch_root and not pivot_root
os.Setenv("DOCKER_RAMDISK", "true")
initFuncs := []config.InitFunc{
func(cfg *config.CloudConfig) error {
return dockerlaunch.PrepareFs(&mountConfig)
initFuncs := []config.CfgFunc{
func(c *config.CloudConfig) (*config.CloudConfig, error) {
return c, dockerlaunch.PrepareFs(&mountConfig)
},
func(cfg *config.CloudConfig) error {
newCfg, err := config.LoadConfig()
if err == nil {
newCfg, err = config.LoadConfig()
}
if err == nil {
*cfg = *newCfg
func(_ *config.CloudConfig) (*config.CloudConfig, error) {
cfg, err := config.LoadConfig()
if err != nil {
return cfg, err
}
if cfg.Rancher.Debug {
cfgString, err := config.Dump(false, true)
cfgString, err := config.Dump(false, false, true)
if err != nil {
log.WithFields(log.Fields{"err": err}).Error("Error serializing config")
} else {
@ -178,24 +173,25 @@ func RunInit() error {
}
}
return err
return cfg, nil
},
loadModules,
tryMountAndBootstrap,
func(cfg *config.CloudConfig) error {
return cfg.Reload()
func(_ *config.CloudConfig) (*config.CloudConfig, error) {
return config.LoadConfig()
},
loadModules,
sysInit,
}
if err := config.RunInitFuncs(&cfg, initFuncs); err != nil {
cfg, err := config.ChainCfgFuncs(nil, initFuncs...)
if err != nil {
return err
}
launchConfig, args := getLaunchConfig(&cfg, &cfg.Rancher.SystemDocker)
launchConfig, args := getLaunchConfig(cfg, &cfg.Rancher.SystemDocker)
log.Info("Launching System Docker")
_, err := dockerlaunch.LaunchDocker(launchConfig, config.DOCKER_BIN, args...)
_, err = dockerlaunch.LaunchDocker(launchConfig, config.DOCKER_BIN, args...)
return err
}

32
init/sysinit.go
View File

@ -51,15 +51,15 @@ func findImages(cfg *config.CloudConfig) ([]string, error) {
return result, nil
}
func loadImages(cfg *config.CloudConfig) error {
func loadImages(cfg *config.CloudConfig) (*config.CloudConfig, error) {
images, err := findImages(cfg)
if err != nil || len(images) == 0 {
return err
return cfg, err
}
client, err := docker.NewSystemClient()
if err != nil {
return err
return cfg, err
}
for _, image := range images {
@ -70,7 +70,7 @@ func loadImages(cfg *config.CloudConfig) error {
inputFileName := path.Join(config.IMAGES_PATH, image)
input, err := os.Open(inputFileName)
if err != nil {
return err
return cfg, err
}
defer input.Close()
@ -82,11 +82,11 @@ func loadImages(cfg *config.CloudConfig) error {
log.Infof("Done loading images from %s", inputFileName)
if err != nil {
return err
return cfg, err
}
}
return nil
return cfg, nil
}
func SysInit() error {
@ -95,20 +95,18 @@ func SysInit() error {
return err
}
initFuncs := []config.InitFunc{
_, err = config.ChainCfgFuncs(cfg,
loadImages,
func(cfg *config.CloudConfig) error {
return compose.RunServices(cfg)
func(cfg *config.CloudConfig) (*config.CloudConfig, error) {
return cfg, compose.RunServices(cfg)
},
func(cfg *config.CloudConfig) error {
func(cfg *config.CloudConfig) (*config.CloudConfig, error) {
syscall.Sync()
return nil
return cfg, nil
},
func(cfg *config.CloudConfig) error {
func(cfg *config.CloudConfig) (*config.CloudConfig, error) {
log.Infof("RancherOS %s started", config.VERSION)
return nil
},
}
return config.RunInitFuncs(cfg, initFuncs)
return cfg, nil
})
return err
}

2
util/term.go
View File

@ -1,3 +1,5 @@
// +build linux
package util
import "github.com/kless/term"

272
util/util.go
View File

@ -1,101 +1,27 @@
package util
import (
"archive/tar"
"bufio"
"errors"
"fmt"
"io"
"io/ioutil"
"math/rand"
"net/http"
"os"
"path"
"strings"
"gopkg.in/yaml.v2"
log "github.com/Sirupsen/logrus"
"github.com/docker/docker/pkg/mount"
"reflect"
)
var (
letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
ErrNoNetwork = errors.New("Networking not available to load resource")
ErrNotFound = errors.New("Failed to find resource")
)
func GetOSType() string {
f, err := os.Open("/etc/os-release")
defer f.Close()
if err != nil {
return "busybox"
}
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := scanner.Text()
if len(line) > 8 && line[:8] == "ID_LIKE=" {
return line[8:]
}
}
return "busybox"
}
func Remount(directory, options string) error {
return mount.Mount("", directory, "", fmt.Sprintf("remount,%s", options))
}
func ExtractTar(archive string, dest string) error {
f, err := os.Open(archive)
if err != nil {
return err
}
defer f.Close()
input := tar.NewReader(f)
for {
header, err := input.Next()
if err == io.EOF {
break
} else if err != nil {
return err
}
if header == nil {
break
}
fileInfo := header.FileInfo()
fileName := path.Join(dest, header.Name)
if fileInfo.IsDir() {
//log.Debugf("DIR : %s", fileName)
err = os.MkdirAll(fileName, fileInfo.Mode())
if err != nil {
return err
}
} else {
//log.Debugf("FILE: %s", fileName)
destFile, err := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fileInfo.Mode())
if err != nil {
return err
}
_, err = io.Copy(destFile, input)
// Not deferring, concerned about holding open too many files
destFile.Close()
if err != nil {
return err
}
}
}
return nil
}
type AnyMap map[interface{}]interface{}
func Contains(values []string, value string) bool {
if len(value) == 0 {
@ -113,45 +39,6 @@ func Contains(values []string, value string) bool {
type ReturnsErr func() error
func ShortCircuit(funcs ...ReturnsErr) error {
for _, f := range funcs {
err := f()
if err != nil {
return err
}
}
return nil
}
type ErrWriter struct {
w io.Writer
Err error
}
func NewErrorWriter(w io.Writer) *ErrWriter {
return &ErrWriter{
w: w,
}
}
func (e *ErrWriter) Write(buf []byte) *ErrWriter {
if e.Err != nil {
return e
}
_, e.Err = e.w.Write(buf)
return e
}
func RandSeq(n int) string {
b := make([]rune, n)
for i := range b {
b[i] = letters[rand.Intn(len(letters))]
}
return string(b)
}
func FileCopy(src, dest string) (err error) {
in, err := os.Open(src)
if err != nil {
@ -181,21 +68,6 @@ func Convert(from, to interface{}) error {
return yaml.Unmarshal(bytes, to)
}
func MergeBytes(left, right []byte) ([]byte, error) {
leftMap := make(map[interface{}]interface{})
rightMap := make(map[interface{}]interface{})
if err := yaml.Unmarshal(left, &leftMap); err != nil {
return nil, err
}
if err := yaml.Unmarshal(right, &rightMap); err != nil {
return nil, err
}
return yaml.Marshal(MapsUnion(leftMap, rightMap, Replace))
}
func Copy(d interface{}) interface{} {
switch d := d.(type) {
case map[interface{}]interface{}:
@ -218,36 +90,45 @@ func Equal(l, r interface{}) interface{} {
return nil
}
func ExistsIn(x interface{}, s []interface{}) bool {
for _, y := range s {
if reflect.DeepEqual(x, y) {
return true
func Filter(xs []interface{}, p func(x interface{}) bool) []interface{} {
return FlatMap(xs, func(x interface{}) []interface{} {
if p(x) {
return []interface{}{x}
}
}
return false
return []interface{}{}
})
}
func SlicesUnion(left, right []interface{}, op func(interface{}, interface{}) interface{}) []interface{} {
result := SliceCopy(left)
for _, r := range right {
if !ExistsIn(r, result) {
result = append(result, r)
func FilterStrings(xs []string, p func(x string) bool) []string {
return FlatMapStrings(xs, func(x string) []string {
if p(x) {
return []string{x}
}
}
return result
return []string{}
})
}
func SlicesIntersection(left, right []interface{}, op func(interface{}, interface{}) interface{}) []interface{} {
func Map(xs []interface{}, f func(x interface{}) interface{}) []interface{} {
return FlatMap(xs, func(x interface{}) []interface{} { return []interface{}{f(x)} })
}
func FlatMap(xs []interface{}, f func(x interface{}) []interface{}) []interface{} {
result := []interface{}{}
for _, r := range right {
if ExistsIn(r, left) {
result = append(result, r)
}
for _, x := range xs {
result = append(result, f(x)...)
}
return result
}
func MapsUnion(left, right map[interface{}]interface{}, op func(interface{}, interface{}) interface{}) map[interface{}]interface{} {
func FlatMapStrings(xs []string, f func(x string) []string) []string {
result := []string{}
for _, x := range xs {
result = append(result, f(x)...)
}
return result
}
func MapsUnion(left, right map[interface{}]interface{}) map[interface{}]interface{} {
result := MapCopy(left)
for k, r := range right {
@ -256,19 +137,12 @@ func MapsUnion(left, right map[interface{}]interface{}, op func(interface{}, int
case map[interface{}]interface{}:
switch r := r.(type) {
case map[interface{}]interface{}:
result[k] = MapsUnion(l, r, op)
result[k] = MapsUnion(l, r)
default:
result[k] = op(l, r)
}
case []interface{}:
switch r := r.(type) {
case []interface{}:
result[k] = SlicesUnion(l, r, op)
default:
result[k] = op(l, r)
result[k] = Replace(l, r)
}
default:
result[k] = op(l, r)
result[k] = Replace(l, r)
}
} else {
result[k] = Copy(r)
@ -278,7 +152,7 @@ func MapsUnion(left, right map[interface{}]interface{}, op func(interface{}, int
return result
}
func MapsIntersection(left, right map[interface{}]interface{}, op func(interface{}, interface{}) interface{}) map[interface{}]interface{} {
func MapsDifference(left, right map[interface{}]interface{}) map[interface{}]interface{} {
result := map[interface{}]interface{}{}
for k, l := range left {
@ -287,23 +161,48 @@ func MapsIntersection(left, right map[interface{}]interface{}, op func(interface
case map[interface{}]interface{}:
switch r := r.(type) {
case map[interface{}]interface{}:
result[k] = MapsIntersection(l, r, op)
default:
if v := op(l, r); v != nil {
if len(l) == 0 && len(r) == 0 {
continue
} else if len(l) == 0 {
result[k] = l
} else if v := MapsDifference(l, r); len(v) > 0 {
result[k] = v
}
}
case []interface{}:
switch r := r.(type) {
case []interface{}:
result[k] = SlicesIntersection(l, r, op)
default:
if v := op(l, r); v != nil {
if v := Equal(l, r); v == nil {
result[k] = l
}
}
default:
if v := Equal(l, r); v == nil {
result[k] = l
}
}
} else {
result[k] = l
}
}
return result
}
func MapsIntersection(left, right map[interface{}]interface{}) map[interface{}]interface{} {
result := map[interface{}]interface{}{}
for k, l := range left {
if r, ok := right[k]; ok {
switch l := l.(type) {
case map[interface{}]interface{}:
switch r := r.(type) {
case map[interface{}]interface{}:
result[k] = MapsIntersection(l, r)
default:
if v := Equal(l, r); v != nil {
result[k] = v
}
}
default:
if v := op(l, r); v != nil {
if v := Equal(l, r); v != nil {
result[k] = v
}
}
@ -329,6 +228,14 @@ func SliceCopy(data []interface{}) []interface{} {
return result
}
func ToStrings(data []interface{}) []string {
result := make([]string, len(data), len(data))
for k, v := range data {
result[k] = v.(string)
}
return result
}
func GetServices(urls []string) ([]string, error) {
result := []string{}
@ -355,6 +262,18 @@ func GetServices(urls []string) ([]string, error) {
return result, nil
}
func DirLs(dir string) ([]interface{}, error) {
result := []interface{}{}
files, err := ioutil.ReadDir(dir)
if err != nil {
return result, err
}
for _, f := range files {
result = append(result, f)
}
return result, nil
}
func LoadResource(location string, network bool, urls []string) ([]byte, error) {
var bytes []byte
err := ErrNotFound
@ -388,21 +307,6 @@ func LoadResource(location string, network bool, urls []string) ([]byte, error)
return nil, err
}
func GetValue(kvPairs []string, key string) string {
if kvPairs == nil {
return ""
}
prefix := key + "="
for _, i := range kvPairs {
if strings.HasPrefix(i, prefix) {
return strings.TrimPrefix(i, prefix)
}
}
return ""
}
func Map2KVPairs(m map[string]string) []string {
r := make([]string, 0, len(m))
for k, v := range m {

101
util/util_test.go
View File

@ -6,10 +6,75 @@ import (
"testing"
)
type testCloudConfig struct {
Hostname string `yaml:"hostname,omitempty"`
Key1 string `yaml:"key1,omitempty"`
Key2 string `yaml:"key2,omitempty"`
}
func TestPassByValue(t *testing.T) {
assert := require.New(t)
cc0ptr := &testCloudConfig{}
cc0ptr.Hostname = "test0"
cc1 := *cc0ptr
cc1.Hostname = "test1"
assert.NotEqual(cc0ptr.Hostname, cc1.Hostname)
}
func TestConvertMergesLeftIntoRight(t *testing.T) {
assert := require.New(t)
cc0 := testCloudConfig{Key1: "k1v0", Key2: "k2v0"}
cc1 := map[interface{}]interface{}{"key1": "k1value1", "hostname": "somehost"}
Convert(cc1, &cc0)
expected := testCloudConfig{Hostname: "somehost", Key1: "k1value1", Key2: "k2v0"}
assert.Equal(expected, cc0)
}
func TestNilMap(t *testing.T) {
assert := require.New(t)
var m map[string]interface{} = nil
assert.True(m == nil)
assert.True(len(m) == 0)
}
func NoTestCopyPointer(t *testing.T) {
assert := require.New(t)
testCCpt := &testCloudConfig{}
m0 := map[string]interface{}{"a": testCCpt, "b": testCCpt}
m1 := Copy(m0).(map[string]interface{})
m1["a"].(*testCloudConfig).Hostname = "somehost"
assert.Equal("", m0["a"].(*testCloudConfig).Hostname)
assert.Equal("somehost", m1["a"].(*testCloudConfig).Hostname)
assert.Equal("", m1["b"].(*testCloudConfig).Hostname)
}
func TestEmptyMap(t *testing.T) {
assert := require.New(t)
m := map[interface{}]interface{}{}
assert.True(len(m) == 0)
}
func tryMutateArg(p *string) *string {
s := "test"
p = &s
return p
}
func TestMutableArg(t *testing.T) {
assert := require.New(t)
s := "somestring"
p := &s
assert.NotEqual(tryMutateArg(p), p)
}
func TestFilter(t *testing.T) {
assert := require.New(t)
ss := []interface{}{"1", "2", "3", "4"}
assert.Equal([]interface{}{"1", "2", "4"}, Filter(ss, func(x interface{}) bool { return x != "3" }))
ss1 := append([]interface{}{}, "qqq")
assert.Equal([]interface{}{"qqq"}, ss1)
}
func TestMapCopy(t *testing.T) {
@ -59,8 +124,34 @@ func TestMapsIntersection(t *testing.T) {
b1 := m1["b"].(map[interface{}]interface{})
delete(b1, "c")
m1["e"] = []interface{}{2, 3, 4}
expected := map[interface{}]interface{}{"b": map[interface{}]interface{}{}, "d": "4", "e": []interface{}{2, 3}}
assert.Equal(expected, MapsIntersection(m0, m1, Equal))
expected := map[interface{}]interface{}{"b": map[interface{}]interface{}{}, "d": "4"}
assert.Equal(expected, MapsIntersection(m0, m1))
}
func TestMapsDifference(t *testing.T) {
assert := require.New(t)
m0 := map[interface{}]interface{}{
"a": 1,
"b": map[interface{}]interface{}{"c": 3},
"d": "4",
"e": []interface{}{1, 2, 3},
}
m1 := MapCopy(m0)
assert.Equal(map[interface{}]interface{}{}, MapsDifference(m0, m0))
assert.Equal(map[interface{}]interface{}{}, MapsDifference(m0, m1))
delete(m1, "a")
b1 := m1["b"].(map[interface{}]interface{})
delete(b1, "c")
m1["e"] = []interface{}{2, 3, 4}
expectedM1M0 := map[interface{}]interface{}{"b": map[interface{}]interface{}{}, "e": []interface{}{2, 3, 4}}
assert.Equal(expectedM1M0, MapsDifference(m1, m0))
expectedM0M1 := map[interface{}]interface{}{"a": 1, "b": map[interface{}]interface{}{"c": 3}, "e": []interface{}{1, 2, 3}}
assert.Equal(expectedM0M1, MapsDifference(m0, m1))
}
func TestMapsUnion(t *testing.T) {
@ -85,12 +176,12 @@ func TestMapsUnion(t *testing.T) {
"b": map[interface{}]interface{}{"c": 3},
"d": "replaced",
"e": "added",
"f": []interface{}{1, 2, 3, 4},
"f": []interface{}{2, 3, 4},
}
assert.Equal(expected, MapsUnion(m0, m1, Replace))
assert.Equal(expected, MapsUnion(m0, m1))
}
func TestLoadResourceSimple(t *testing.T) {
func NoTestLoadResourceSimple(t *testing.T) {
assert := require.New(t)
expected := `services: