From e0820d240f6aad1d4ed2d19057df69f6249d0fee Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Fri, 10 Apr 2015 09:32:14 -0700 Subject: [PATCH 01/14] Rename files --- cmd/cloudinit/{authorizeSSHKeys.go => authorize_ssh_keys.go} | 0 cmd/cloudinit/{gcecloudinit.go => gce.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename cmd/cloudinit/{authorizeSSHKeys.go => authorize_ssh_keys.go} (100%) rename cmd/cloudinit/{gcecloudinit.go => gce.go} (100%) diff --git a/cmd/cloudinit/authorizeSSHKeys.go b/cmd/cloudinit/authorize_ssh_keys.go similarity index 100% rename from cmd/cloudinit/authorizeSSHKeys.go rename to cmd/cloudinit/authorize_ssh_keys.go diff --git a/cmd/cloudinit/gcecloudinit.go b/cmd/cloudinit/gce.go similarity index 100% rename from cmd/cloudinit/gcecloudinit.go rename to cmd/cloudinit/gce.go From 6bb6fb0eec25b80de86cc8f524aaba0fc6d8f531 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Wed, 15 Apr 2015 23:03:27 -0700 Subject: [PATCH 02/14] Support cloud-config.d --- cmd/cloudinit/cloudinit.go | 317 ++++++++++++++++++++++++++----------- 1 file changed, 224 insertions(+), 93 deletions(-) diff --git a/cmd/cloudinit/cloudinit.go b/cmd/cloudinit/cloudinit.go index 2100a376..aa72655d 100644 --- a/cmd/cloudinit/cloudinit.go +++ b/cmd/cloudinit/cloudinit.go @@ -33,11 +33,11 @@ import ( "github.com/coreos/coreos-cloudinit/datasource/metadata/ec2" "github.com/coreos/coreos-cloudinit/datasource/proc_cmdline" "github.com/coreos/coreos-cloudinit/datasource/url" - "github.com/coreos/coreos-cloudinit/initialize" "github.com/coreos/coreos-cloudinit/pkg" "github.com/coreos/coreos-cloudinit/system" rancherNetwork "github.com/rancherio/os/cmd/network" rancherConfig "github.com/rancherio/os/config" + "github.com/rancherio/os/util" "gopkg.in/yaml.v2" ) @@ -48,21 +48,25 @@ const ( ) var ( - outputDir string - outputFile string - scriptFile string - rancherYml string - save bool - execute bool - network bool - sshKeyName string - flags *flag.FlagSet + baseConfigDir string + outputDir string + outputFile string + metaDataFile string + scriptFile string + rancherYml string + save bool + execute bool + network bool + sshKeyName string + flags *flag.FlagSet ) func init() { flags = flag.NewFlagSet(os.Args[0], flag.ContinueOnError) + flags.StringVar(&baseConfigDir, "base-config-dir", "/var/lib/rancher/conf/cloud-config.d", "base cloud config") flags.StringVar(&outputDir, "dir", "/var/lib/rancher/conf", "working directory") flags.StringVar(&outputFile, "file", "cloud-config-processed.yml", "output cloud config file name") + flags.StringVar(&metaDataFile, "metadata", "metadata", "output metdata file name") flags.StringVar(&scriptFile, "script-file", "cloud-config-script", "output cloud config script file name") flags.StringVar(&rancherYml, "rancher", "cloud-config-rancher.yml", "output cloud config rancher file name") flags.StringVar(&sshKeyName, "ssh-key-name", "rancheros-cloud-config", "SSH key name") @@ -71,61 +75,61 @@ func init() { flags.BoolVar(&execute, "execute", false, "execute saved cloud config") } -func saveFiles(script *config.Script, userdataBytes []byte, cc *config.CloudConfig) error { - var fileData []byte +func saveFiles(cloudConfigBytes, scriptBytes []byte, metadata datasource.Metadata) error { + scriptOutput := path.Join(outputDir, scriptFile) + cloudConfigOutput := path.Join(outputDir, outputFile) + rancherYmlOutput := path.Join(outputDir, rancherYml) + metaDataOutput := path.Join(outputDir, metaDataFile) - os.Remove(path.Join(outputDir, scriptFile)) - os.Remove(path.Join(outputDir, outputFile)) - os.Remove(path.Join(outputDir, rancherYml)) + os.Remove(scriptOutput) + os.Remove(cloudConfigOutput) + os.Remove(rancherYmlOutput) + os.Remove(metaDataOutput) - if script != nil { - fileData = userdataBytes - output := path.Join(outputDir, scriptFile) - log.Infof("Writing cloud-config script to %s", output) - if err := ioutil.WriteFile(output, fileData, 500); err != nil { - log.Errorf("Error while writing file %v", err) + if len(scriptBytes) > 0 { + log.Infof("Writing to %s", scriptOutput) + if err := ioutil.WriteFile(scriptOutput, scriptBytes, 500); err != nil { + log.Errorf("Error while writing file %s: %v", scriptOutput, err) return err } } - if data, err := yaml.Marshal(cc); err != nil { - log.Errorf("Error while marshalling cloud config %v", err) - return err - } else { - fileData = append([]byte("#cloud-config\n"), data...) - } - - output := path.Join(outputDir, outputFile) - log.Infof("Writing merged cloud-config to %s", output) - if err := ioutil.WriteFile(output, fileData, 400); err != nil { - log.Errorf("Error while writing file %v", err) + cloudConfigBytes = append([]byte("#cloud-config\n"), cloudConfigBytes...) + log.Infof("Writing to %s", cloudConfigOutput) + if err := ioutil.WriteFile(cloudConfigOutput, cloudConfigBytes, 500); err != nil { + log.Errorf("Error while writing file %s: %v", cloudConfigOutput, err) return err } - if script == nil { - ccData := make(map[string]interface{}) - if err := yaml.Unmarshal(userdataBytes, ccData); err != nil { + ccData := make(map[string]interface{}) + if err := yaml.Unmarshal(cloudConfigBytes, ccData); err != nil { + return err + } + + if rancher, ok := ccData["rancher"]; ok { + bytes, err := yaml.Marshal(rancher) + if err != nil { return err } - if rancher, ok := ccData["rancher"]; ok { - bytes, err := yaml.Marshal(rancher) - if err != nil { - return err - } - - if err = ioutil.WriteFile(path.Join(outputDir, rancherYml), bytes, 400); err != nil { - return err - } + if err = ioutil.WriteFile(rancherYmlOutput, bytes, 400); err != nil { + return err } } + metaDataBytes, err := yaml.Marshal(metadata) + if err != nil { + return err + } + + if err = ioutil.WriteFile(metaDataOutput, metaDataBytes, 400); err != nil { + return err + } + return nil } -func Main() { - flags.Parse(rancherConfig.FilterGlobalConfig(os.Args[1:])) - +func currentDatasource() (datasource.Datasource, error) { cfg, err := rancherConfig.LoadConfig() if err != nil { log.Fatalf("Failed to read rancher config %v", err) @@ -133,53 +137,152 @@ func Main() { dss := getDatasources(cfg) if len(dss) == 0 { - log.Infof("No datasources available %v", cfg.CloudInit.Datasources) - os.Exit(0) + return nil, nil } ds := selectDatasource(dss) - if ds == nil { - log.Info("No datasources found") - os.Exit(0) - } + return ds, nil +} - log.Infof("Fetching user-data from datasource %v", ds.Type()) - userdataBytes, err := ds.FetchUserdata() +func mergeBaseConfig(current []byte) ([]byte, error) { + files, err := ioutil.ReadDir(baseConfigDir) if err != nil { - log.Fatalf("Failed fetching user-data from datasource: %v", err) - } - - log.Infof("Fetching meta-data from datasource of type %v", ds.Type()) - metadata, err := ds.FetchMetadata() - if err != nil { - log.Infof("Failed fetching meta-data from datasource: %v", err) - os.Exit(1) - } - - var ccu *config.CloudConfig - var script *config.Script - if ud, err := initialize.ParseUserData(string(userdataBytes)); err != nil { - log.Fatalf("Failed to parse user-data: %v\n", err) - } else { - switch t := ud.(type) { - case *config.CloudConfig: - ccu = t - case *config.Script: - script = t + if os.IsNotExist(err) { + log.Infof("%s does not exist, not merging", baseConfigDir) + return current, nil } + + log.Errorf("Failed to read %s: %v", baseConfigDir, err) + return nil, err + } + + result := []byte{} + + for _, file := range files { + if file.IsDir() || strings.HasPrefix(file.Name(), ".") { + continue + } + + input := path.Join(baseConfigDir, file.Name()) + content, err := ioutil.ReadFile(input) + if err != nil { + log.Errorf("Failed to read %s: %v", input, err) + // ignore error + continue + } + + log.Infof("Merging %s", input) + result, err = util.MergeBytes(result, content) + if err != nil { + log.Errorf("Failed to merge bytes: %v", err) + return nil, err + } + } + + if len(result) == 0 { + return current, nil + } else { + return util.MergeBytes(result, current) + } +} + +func saveCloudConfig() error { + var userDataBytes []byte + var metadata datasource.Metadata + + ds, err := currentDatasource() + if err != nil { + log.Errorf("Failed to select datasource: %v", err) + return err + } + + if ds != nil { + log.Infof("Fetching user-data from datasource %v", ds.Type()) + userDataBytes, err = ds.FetchUserdata() + if err != nil { + log.Errorf("Failed fetching user-data from datasource: %v", err) + return err + } + + log.Infof("Fetching meta-data from datasource of type %v", ds.Type()) + metadata, err = ds.FetchMetadata() + if err != nil { + log.Errorf("Failed fetching meta-data from datasource: %v", err) + return err + } + } + + userData := string(userDataBytes) + scriptBytes := []byte{} + + if config.IsScript(userData) { + scriptBytes = userDataBytes + userDataBytes = []byte{} + } else if isCompose(userData) { + if userDataBytes, err = toCompose(userDataBytes); err != nil { + log.Errorf("Failed to convert to compose syntax: %v", err) + return err + } + } else if config.IsCloudConfig(userData) { + // nothing to do + } else { + log.Errorf("Unrecognized cloud-init\n%s", userData) + userDataBytes = []byte{} + } + + if userDataBytes, err = mergeBaseConfig(userDataBytes); err != nil { + log.Errorf("Failed to merge base config: %v", err) + return err + } + + return saveFiles(userDataBytes, scriptBytes, metadata) +} + +func getSaveCloudConfig() (*config.CloudConfig, error) { + cloudConfig := path.Join(outputDir, outputFile) + + ds := file.NewDatasource(cloudConfig) + if !ds.IsAvailable() { + log.Infof("%s does not exist", cloudConfig) + return nil, nil + } + + ccBytes, err := ds.FetchUserdata() + if err != nil { + log.Errorf("Failed to read user-data from %s: %v", cloudConfig, err) + return nil, err + } + + var cc config.CloudConfig + err = yaml.Unmarshal(ccBytes, &cc) + if err != nil { + log.Errorf("Failed to unmarshall user-data from %s: %v", cloudConfig, err) + return nil, err + } + + return &cc, err +} + +func executeCloudConfig() error { + ccu, err := getSaveCloudConfig() + if err != nil { + return err + } + + var metadata datasource.Metadata + + metaDataBytes, err := ioutil.ReadFile(path.Join(outputDir, metaDataFile)) + if err != nil { + return err + } + + if err = yaml.Unmarshal(metaDataBytes, &metadata); err != nil { + return err } log.Info("Merging cloud-config from meta-data and user-data") cc := mergeConfigs(ccu, metadata) - if save { - err = saveFiles(script, userdataBytes, &cc) - if err != nil { - log.Fatal(err) - } - os.Exit(0) - } - if len(cc.SSHAuthorizedKeys) > 0 { authorizeSSHKeys("rancher", cc.SSHAuthorizedKeys, sshKeyName) } @@ -201,6 +304,26 @@ func Main() { } log.Printf("Wrote file %s to filesystem", fullPath) } + + return nil +} + +func Main() { + flags.Parse(rancherConfig.FilterGlobalConfig(os.Args[1:])) + + if save { + err := saveCloudConfig() + if err != nil { + log.Fatalf("Failed to save cloud config: %v", err) + } + } + + if execute { + err := executeCloudConfig() + if err != nil { + log.Fatalf("Failed to save cloud config: %v", err) + } + } } // mergeConfigs merges certain options from md (meta-data from the datasource) @@ -229,16 +352,6 @@ func mergeConfigs(cc *config.CloudConfig, md datasource.Metadata) (out config.Cl func getDatasources(cfg *rancherConfig.Config) []datasource.Datasource { dss := make([]datasource.Datasource, 0, 5) - if execute { - cloudConfig := path.Join(outputDir, outputFile) - if _, err := os.Stat(cloudConfig); os.IsNotExist(err) { - return dss - } - - dss = append(dss, file.NewDatasource(cloudConfig)) - return dss - } - for _, ds := range cfg.CloudInit.Datasources { parts := strings.SplitN(ds, ":", 2) @@ -355,3 +468,21 @@ func selectDatasource(sources []datasource.Datasource) datasource.Datasource { close(stop) return s } + +func isCompose(content string) bool { + return strings.HasPrefix(content, "#compose\n") +} + +func toCompose(bytes []byte) ([]byte, error) { + result := make(map[interface{}]interface{}) + compose := make(map[interface{}]interface{}) + err := yaml.Unmarshal(bytes, &result) + if err != nil { + return nil, err + } + + result["services"] = map[interface{}]interface{}{ + "cloud-config": compose, + } + return yaml.Marshal(result) +} From 61b5525d5b794afd08b1c8b4eb378761dc29d864 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Wed, 15 Apr 2015 23:03:49 -0700 Subject: [PATCH 03/14] Check if os-release exists first --- scripts/dockerimages/scripts/console.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dockerimages/scripts/console.sh b/scripts/dockerimages/scripts/console.sh index 7806839d..a944c9f7 100755 --- a/scripts/dockerimages/scripts/console.sh +++ b/scripts/dockerimages/scripts/console.sh @@ -76,7 +76,7 @@ if ! grep -q '^UseDNS no' /etc/ssh/sshd_config; then fi ID_TYPE="busybox" -if grep -q 'ID_LIKE=' /etc/os-release; then +if [ -e /etc/os-release ] && grep -q 'ID_LIKE=' /etc/os-release; then ID_TYPE=$(grep 'ID_LIKE=' /etc/os-release | cut -d'=' -f2) fi From be8fa8e5b7d94575832f52d8ceedf1a253c8e9ed Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Wed, 15 Apr 2015 23:16:23 -0700 Subject: [PATCH 04/14] Services as compose format --- cmd/control/service.go | 37 ++++++++++--------------------------- config/default.go | 6 ++++-- config/types.go | 5 +++-- docker/container.go | 31 ++++++++++++++++++++++++++++--- init/sysinit.go | 41 ++++++++++++++++++++++++++++++----------- util/util.go | 28 ++++++++++++++++++++++++++-- 6 files changed, 101 insertions(+), 47 deletions(-) diff --git a/cmd/control/service.go b/cmd/control/service.go index 11c63a3f..d43bba3b 100644 --- a/cmd/control/service.go +++ b/cmd/control/service.go @@ -6,7 +6,6 @@ import ( "github.com/codegangsta/cli" "github.com/rancherio/os/config" - "github.com/rancherio/os/util" ) func serviceSubCommands() []cli.Command { @@ -37,21 +36,16 @@ func disable(c *cli.Context) { } for _, service := range c.Args() { - filtered := make([]string, 0, len(c.Args())) - for _, existing := range cfg.EnabledServices { - if existing != service { - filtered = append(filtered, existing) - } + if _, ok := cfg.Services[service]; !ok { + continue } - if len(filtered) != len(c.Args()) { - cfg.EnabledServices = filtered - changed = true - } + cfg.Services[service] = false + changed = true } if changed { - if err = cfg.Set("enabled_services", cfg.EnabledServices); err != nil { + if err = cfg.Set("services", cfg.Services); err != nil { log.Fatal(err) } } @@ -65,14 +59,14 @@ func enable(c *cli.Context) { } for _, service := range c.Args() { - if !util.Contains(cfg.EnabledServices, service) { - cfg.EnabledServices = append(cfg.EnabledServices, service) + if val, ok := cfg.Services[service]; !ok || !val { + cfg.Services[service] = true changed = true } } if changed { - if err = cfg.Set("enabled_services", cfg.EnabledServices); err != nil { + if err = cfg.Set("services", cfg.Services); err != nil { log.Fatal(err) } } @@ -84,22 +78,11 @@ func list(c *cli.Context) { log.Fatal(err) } - enabled := map[string]bool{} - - for _, service := range cfg.EnabledServices { - enabled[service] = true - } - - for service, _ := range cfg.Services { - if _, ok := enabled[service]; ok { - delete(enabled, service) + for service, enabled := range cfg.Services { + if enabled { fmt.Printf("enabled %s\n", service) } else { fmt.Printf("disabled %s\n", service) } } - - for service, _ := range enabled { - fmt.Printf("enabled %s\n", service) - } } diff --git a/config/default.go b/config/default.go index 403f9f45..4beb32a1 100644 --- a/config/default.go +++ b/config/default.go @@ -301,8 +301,10 @@ func NewConfig() *Config { Net: "host", }, }, - EnabledServices: []string{}, - Services: map[string]Config{ + Services: map[string]bool{ + "ubuntu-console": false, + }, + BundledServices: map[string]Config{ "ubuntu-console": { SystemContainers: map[string]*project.ServiceConfig{ "console": { diff --git a/config/types.go b/config/types.go index b775945a..0aee7d06 100644 --- a/config/types.go +++ b/config/types.go @@ -43,14 +43,15 @@ type ContainerConfig struct { } type Config struct { - Services map[string]Config `yaml:"services,omitempty"` + Environment map[string]string `yaml:"environment,omitempty"` + BundledServices map[string]Config `yaml:"bundled_services,omitempty"` BootstrapContainers map[string]*project.ServiceConfig `yaml:"bootstrap_containers,omitempty"` BootstrapDocker DockerConfig `yaml:"bootstrap_docker,omitempty"` CloudInit CloudInit `yaml:"cloud_init,omitempty"` Console ConsoleConfig `yaml:"console,omitempty"` Debug bool `yaml:"debug,omitempty"` Disable []string `yaml:"disable,omitempty"` - EnabledServices []string `yaml:"enabled_services,omitempty"` + Services map[string]bool `yaml:"services,omitempty"` Modules []string `yaml:"modules,omitempty"` Network NetworkConfig `yaml:"network,omitempty"` Ssh SshConfig `yaml:"ssh,omitempty"` diff --git a/docker/container.go b/docker/container.go index f4a5d427..f5690f8c 100644 --- a/docker/container.go +++ b/docker/container.go @@ -170,11 +170,36 @@ func (c *Container) Reset() *Container { return c } +func (c *Container) requiresSyslog() bool { + return (c.ContainerCfg.Service.LogDriver == "" || c.ContainerCfg.Service.LogDriver == "syslog") +} + +func (c *Container) hasLink(link string) bool { + return util.Contains(c.ContainerCfg.Service.Links, link) +} + +func (c *Container) addLink(link string) { + c.ContainerCfg.Service.Links = append(c.ContainerCfg.Service.Links, link) +} + func (c *Container) parseService() { - if (c.ContainerCfg.Service.LogDriver == "" || c.ContainerCfg.Service.LogDriver == "syslog") && - !util.Contains(c.ContainerCfg.Service.Links, "syslog") { + client, err := NewClient(c.dockerHost) + if err != nil { + c.Err = err + return + } + + if c.ContainerCfg.Service.Image != "" { + i, _ := client.InspectImage(c.ContainerCfg.Service.Image) + if i == nil && !c.hasLink("network") { + log.Debugf("Adding network link to %s", c.Name) + c.addLink("network") + } + } + + if c.requiresSyslog() && !c.hasLink("syslog") { log.Debugf("Adding syslog link to %s\n", c.Name) - c.ContainerCfg.Service.Links = append(c.ContainerCfg.Service.Links, "syslog") + c.addLink("syslog") } cfg, hostConfig, err := docker.Convert(c.ContainerCfg.Service) diff --git a/init/sysinit.go b/init/sysinit.go index 723b88d1..cca1b8a8 100644 --- a/init/sysinit.go +++ b/init/sysinit.go @@ -110,40 +110,51 @@ func loadImages(cfg *config.Config) error { } func runServices(name string, cfg *config.Config, configs map[string]*project.ServiceConfig) error { - project := project.NewProject(name, docker.NewContainerFactory(cfg)) + network := false + projectEvents := make(chan project.ProjectEvent) + p := project.NewProject(name, docker.NewContainerFactory(cfg)) + p.AddListener(projectEvents) enabled := make(map[string]bool) for name, serviceConfig := range configs { - if err := project.AddConfig(name, serviceConfig); err != nil { + if err := p.AddConfig(name, serviceConfig); err != nil { log.Infof("Failed loading service %s", name) } } - project.ReloadCallback = func() error { + p.ReloadCallback = func() error { err := cfg.Reload() if err != nil { return err } - for _, service := range cfg.EnabledServices { + for service, serviceEnabled := range cfg.Services { + if !serviceEnabled { + continue + } + if _, ok := enabled[service]; ok { continue } - if config, ok := cfg.Services[service]; ok { + if config, ok := cfg.BundledServices[service]; ok { for name, s := range config.SystemContainers { - if err := project.AddConfig(name, s); err != nil { + if err := p.AddConfig(name, s); err != nil { log.Errorf("Failed to load %s : %v", name, err) } } } else { - bytes, err := util.LoadResource(service) + bytes, err := util.LoadResource(service, network) if err != nil { - log.Errorf("Failed to load %s : %v", service, err) + if err == util.ErrNoNetwork { + log.Debugf("Can not load %s, networking not enabled", service) + } else { + log.Errorf("Failed to load %s : %v", service, err) + } continue } - err = project.Load(bytes) + err = p.Load(bytes) if err != nil { log.Errorf("Failed to load %s : %v", service, err) continue @@ -156,12 +167,20 @@ func runServices(name string, cfg *config.Config, configs map[string]*project.Se return nil } - err := project.ReloadCallback() + go func() { + for event := range projectEvents { + if event.Event == project.CONTAINER_STARTED && event.Service.Name() == "network" { + network = true + } + } + }() + + err := p.ReloadCallback() if err != nil { log.Errorf("Failed to reload %s : %v", name, err) return err } - return project.Up() + return p.Up() } func runContainers(cfg *config.Config) error { diff --git a/util/util.go b/util/util.go index b97b9aaa..162612e6 100644 --- a/util/util.go +++ b/util/util.go @@ -2,6 +2,7 @@ package util import ( "archive/tar" + "errors" "fmt" "io" "io/ioutil" @@ -17,7 +18,8 @@ import ( ) var ( - letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + ErrNoNetwork = errors.New("Networking not available to load resource") ) func mountProc() error { @@ -168,6 +170,25 @@ func Convert(from, to interface{}) error { return yaml.Unmarshal(bytes, to) } +func MergeBytes(left, right []byte) ([]byte, error) { + leftMap := make(map[interface{}]interface{}) + rightMap := make(map[interface{}]interface{}) + + err := yaml.Unmarshal(left, &leftMap) + if err != nil { + return nil, err + } + + err = yaml.Unmarshal(right, &rightMap) + if err != nil { + return nil, err + } + + MergeMaps(leftMap, rightMap) + + return yaml.Marshal(leftMap) +} + func MergeMaps(left, right map[interface{}]interface{}) { for k, v := range right { merged := false @@ -186,8 +207,11 @@ func MergeMaps(left, right map[interface{}]interface{}) { } } -func LoadResource(location string) ([]byte, error) { +func LoadResource(location string, network bool) ([]byte, error) { if strings.HasPrefix(location, "http:/") || strings.HasPrefix(location, "https:/") { + if !network { + return nil, ErrNoNetwork + } resp, err := http.Get(location) if err != nil { return nil, err From 4982e07fd09919ba71dd99f00c8c8e6f30c6d6ef Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Sun, 12 Apr 2015 23:06:42 -0700 Subject: [PATCH 05/14] Godep changes --- Godeps/Godeps.json | 13 +- .../coreos-cloudinit/initialize/config.go | 293 ----------------- .../initialize/config_test.go | 299 ------------------ .../coreos/coreos-cloudinit/initialize/env.go | 116 ------- .../coreos-cloudinit/initialize/env_test.go | 148 --------- .../coreos-cloudinit/initialize/github.go | 32 -- .../coreos-cloudinit/initialize/ssh_keys.go | 57 ---- .../initialize/ssh_keys_test.go | 56 ---- .../coreos-cloudinit/initialize/user_data.go | 39 --- .../initialize/user_data_test.go | 74 ----- .../coreos-cloudinit/initialize/workspace.go | 66 ---- .../rancher-compose/project/project.go | 29 ++ .../rancher-compose/project/types.go | 1 + 13 files changed, 34 insertions(+), 1189 deletions(-) delete mode 100644 Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/config.go delete mode 100644 Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/config_test.go delete mode 100644 Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/env.go delete mode 100644 Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/env_test.go delete mode 100644 Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/github.go delete mode 100644 Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/ssh_keys.go delete mode 100644 Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/ssh_keys_test.go delete mode 100644 Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/user_data.go delete mode 100644 Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/user_data_test.go delete mode 100644 Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/workspace.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 56102c98..434613bd 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -22,11 +22,6 @@ "Comment": "v1.3.2-6-g405c260", "Rev": "405c2600b19ae77516c967f8ee8ebde5624d3663" }, - { - "ImportPath": "github.com/coreos/coreos-cloudinit/initialize", - "Comment": "v1.3.2-6-g405c260", - "Rev": "405c2600b19ae77516c967f8ee8ebde5624d3663" - }, { "ImportPath": "github.com/coreos/coreos-cloudinit/network", "Comment": "v1.3.2-6-g405c260", @@ -226,13 +221,13 @@ }, { "ImportPath": "github.com/rancherio/rancher-compose/docker", - "Comment": "0.1.0-8-g45c7de3", - "Rev": "45c7de3d9b5b106475cf1461df7550bdaab0a9aa" + "Comment": "0.1.0-9-g7343438", + "Rev": "734343806223cab5de0532ea7add0beeb2cbea92" }, { "ImportPath": "github.com/rancherio/rancher-compose/project", - "Comment": "0.1.0-8-g45c7de3", - "Rev": "45c7de3d9b5b106475cf1461df7550bdaab0a9aa" + "Comment": "0.1.0-9-g7343438", + "Rev": "734343806223cab5de0532ea7add0beeb2cbea92" }, { "ImportPath": "github.com/ryanuber/go-glob", diff --git a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/config.go b/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/config.go deleted file mode 100644 index 3bf7f541..00000000 --- a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/config.go +++ /dev/null @@ -1,293 +0,0 @@ -// Copyright 2015 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package initialize - -import ( - "errors" - "fmt" - "log" - "path" - - "github.com/coreos/coreos-cloudinit/config" - "github.com/coreos/coreos-cloudinit/network" - "github.com/coreos/coreos-cloudinit/system" -) - -// CloudConfigFile represents a CoreOS specific configuration option that can generate -// an associated system.File to be written to disk -type CloudConfigFile interface { - // File should either return (*system.File, error), or (nil, nil) if nothing - // needs to be done for this configuration option. - File() (*system.File, error) -} - -// CloudConfigUnit represents a CoreOS specific configuration option that can generate -// associated system.Units to be created/enabled appropriately -type CloudConfigUnit interface { - Units() []system.Unit -} - -// Apply renders a CloudConfig to an Environment. This can involve things like -// configuring the hostname, adding new users, writing various configuration -// files to disk, and manipulating systemd services. -func Apply(cfg config.CloudConfig, ifaces []network.InterfaceGenerator, env *Environment) error { - if cfg.Hostname != "" { - if err := system.SetHostname(cfg.Hostname); err != nil { - return err - } - log.Printf("Set hostname to %s", cfg.Hostname) - } - - for _, user := range cfg.Users { - if user.Name == "" { - log.Printf("User object has no 'name' field, skipping") - continue - } - - if system.UserExists(&user) { - log.Printf("User '%s' exists, ignoring creation-time fields", user.Name) - if user.PasswordHash != "" { - log.Printf("Setting '%s' user's password", user.Name) - if err := system.SetUserPassword(user.Name, user.PasswordHash); err != nil { - log.Printf("Failed setting '%s' user's password: %v", user.Name, err) - return err - } - } - } else { - log.Printf("Creating user '%s'", user.Name) - if err := system.CreateUser(&user); err != nil { - log.Printf("Failed creating user '%s': %v", user.Name, err) - return err - } - } - - if len(user.SSHAuthorizedKeys) > 0 { - log.Printf("Authorizing %d SSH keys for user '%s'", len(user.SSHAuthorizedKeys), user.Name) - if err := system.AuthorizeSSHKeys(user.Name, env.SSHKeyName(), user.SSHAuthorizedKeys); err != nil { - return err - } - } - if user.SSHImportGithubUser != "" { - log.Printf("Authorizing github user %s SSH keys for CoreOS user '%s'", user.SSHImportGithubUser, user.Name) - if err := SSHImportGithubUser(user.Name, user.SSHImportGithubUser); err != nil { - return err - } - } - for _, u := range user.SSHImportGithubUsers { - log.Printf("Authorizing github user %s SSH keys for CoreOS user '%s'", u, user.Name) - if err := SSHImportGithubUser(user.Name, u); err != nil { - return err - } - } - if user.SSHImportURL != "" { - log.Printf("Authorizing SSH keys for CoreOS user '%s' from '%s'", user.Name, user.SSHImportURL) - if err := SSHImportKeysFromURL(user.Name, user.SSHImportURL); err != nil { - return err - } - } - } - - if len(cfg.SSHAuthorizedKeys) > 0 { - err := system.AuthorizeSSHKeys("core", env.SSHKeyName(), cfg.SSHAuthorizedKeys) - if err == nil { - log.Printf("Authorized SSH keys for core user") - } else { - return err - } - } - - var writeFiles []system.File - for _, file := range cfg.WriteFiles { - writeFiles = append(writeFiles, system.File{File: file}) - } - - for _, ccf := range []CloudConfigFile{ - system.OEM{OEM: cfg.CoreOS.OEM}, - system.Update{Update: cfg.CoreOS.Update, ReadConfig: system.DefaultReadConfig}, - system.EtcHosts{EtcHosts: cfg.ManageEtcHosts}, - system.Flannel{Flannel: cfg.CoreOS.Flannel}, - } { - f, err := ccf.File() - if err != nil { - return err - } - if f != nil { - writeFiles = append(writeFiles, *f) - } - } - - var units []system.Unit - for _, u := range cfg.CoreOS.Units { - units = append(units, system.Unit{Unit: u}) - } - - for _, ccu := range []CloudConfigUnit{ - system.Etcd{Etcd: cfg.CoreOS.Etcd}, - system.Fleet{Fleet: cfg.CoreOS.Fleet}, - system.Locksmith{Locksmith: cfg.CoreOS.Locksmith}, - system.Update{Update: cfg.CoreOS.Update, ReadConfig: system.DefaultReadConfig}, - } { - units = append(units, ccu.Units()...) - } - - wroteEnvironment := false - for _, file := range writeFiles { - fullPath, err := system.WriteFile(&file, env.Root()) - if err != nil { - return err - } - if path.Clean(file.Path) == "/etc/environment" { - wroteEnvironment = true - } - log.Printf("Wrote file %s to filesystem", fullPath) - } - - if !wroteEnvironment { - ef := env.DefaultEnvironmentFile() - if ef != nil { - err := system.WriteEnvFile(ef, env.Root()) - if err != nil { - return err - } - log.Printf("Updated /etc/environment") - } - } - - if len(ifaces) > 0 { - units = append(units, createNetworkingUnits(ifaces)...) - if err := system.RestartNetwork(ifaces); err != nil { - return err - } - } - - um := system.NewUnitManager(env.Root()) - return processUnits(units, env.Root(), um) -} - -func createNetworkingUnits(interfaces []network.InterfaceGenerator) (units []system.Unit) { - appendNewUnit := func(units []system.Unit, name, content string) []system.Unit { - if content == "" { - return units - } - return append(units, system.Unit{Unit: config.Unit{ - Name: name, - Runtime: true, - Content: content, - }}) - } - for _, i := range interfaces { - units = appendNewUnit(units, fmt.Sprintf("%s.netdev", i.Filename()), i.Netdev()) - units = appendNewUnit(units, fmt.Sprintf("%s.link", i.Filename()), i.Link()) - units = appendNewUnit(units, fmt.Sprintf("%s.network", i.Filename()), i.Network()) - } - return units -} - -// processUnits takes a set of Units and applies them to the given root using -// the given UnitManager. This can involve things like writing unit files to -// disk, masking/unmasking units, or invoking systemd -// commands against units. It returns any error encountered. -func processUnits(units []system.Unit, root string, um system.UnitManager) error { - type action struct { - unit system.Unit - command string - } - actions := make([]action, 0, len(units)) - reload := false - restartNetworkd := false - for _, unit := range units { - if unit.Name == "" { - log.Printf("Skipping unit without name") - continue - } - - if unit.Content != "" { - log.Printf("Writing unit %q to filesystem", unit.Name) - if err := um.PlaceUnit(unit); err != nil { - return err - } - log.Printf("Wrote unit %q", unit.Name) - reload = true - } - - for _, dropin := range unit.DropIns { - if dropin.Name != "" && dropin.Content != "" { - log.Printf("Writing drop-in unit %q to filesystem", dropin.Name) - if err := um.PlaceUnitDropIn(unit, dropin); err != nil { - return err - } - log.Printf("Wrote drop-in unit %q", dropin.Name) - reload = true - } - } - - if unit.Mask { - log.Printf("Masking unit file %q", unit.Name) - if err := um.MaskUnit(unit); err != nil { - return err - } - } else if unit.Runtime { - log.Printf("Ensuring runtime unit file %q is unmasked", unit.Name) - if err := um.UnmaskUnit(unit); err != nil { - return err - } - } - - if unit.Enable { - if unit.Group() != "network" { - log.Printf("Enabling unit file %q", unit.Name) - if err := um.EnableUnitFile(unit); err != nil { - return err - } - log.Printf("Enabled unit %q", unit.Name) - } else { - log.Printf("Skipping enable for network-like unit %q", unit.Name) - } - } - - if unit.Group() == "network" { - restartNetworkd = true - } else if unit.Command != "" { - actions = append(actions, action{unit, unit.Command}) - } - } - - if reload { - if err := um.DaemonReload(); err != nil { - return errors.New(fmt.Sprintf("failed systemd daemon-reload: %s", err)) - } - } - - if restartNetworkd { - log.Printf("Restarting systemd-networkd") - networkd := system.Unit{Unit: config.Unit{Name: "systemd-networkd.service"}} - res, err := um.RunUnitCommand(networkd, "restart") - if err != nil { - return err - } - log.Printf("Restarted systemd-networkd (%s)", res) - } - - for _, action := range actions { - log.Printf("Calling unit command %q on %q'", action.command, action.unit.Name) - res, err := um.RunUnitCommand(action.unit, action.command) - if err != nil { - return err - } - log.Printf("Result of %q on %q: %s", action.command, action.unit.Name, res) - } - - return nil -} diff --git a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/config_test.go b/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/config_test.go deleted file mode 100644 index 33be737e..00000000 --- a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/config_test.go +++ /dev/null @@ -1,299 +0,0 @@ -// Copyright 2015 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package initialize - -import ( - "reflect" - "testing" - - "github.com/coreos/coreos-cloudinit/config" - "github.com/coreos/coreos-cloudinit/network" - "github.com/coreos/coreos-cloudinit/system" -) - -type TestUnitManager struct { - placed []string - enabled []string - masked []string - unmasked []string - commands []UnitAction - reload bool -} - -type UnitAction struct { - unit string - command string -} - -func (tum *TestUnitManager) PlaceUnit(u system.Unit) error { - tum.placed = append(tum.placed, u.Name) - return nil -} -func (tum *TestUnitManager) PlaceUnitDropIn(u system.Unit, d config.UnitDropIn) error { - tum.placed = append(tum.placed, u.Name+".d/"+d.Name) - return nil -} -func (tum *TestUnitManager) EnableUnitFile(u system.Unit) error { - tum.enabled = append(tum.enabled, u.Name) - return nil -} -func (tum *TestUnitManager) RunUnitCommand(u system.Unit, c string) (string, error) { - tum.commands = append(tum.commands, UnitAction{u.Name, c}) - return "", nil -} -func (tum *TestUnitManager) DaemonReload() error { - tum.reload = true - return nil -} -func (tum *TestUnitManager) MaskUnit(u system.Unit) error { - tum.masked = append(tum.masked, u.Name) - return nil -} -func (tum *TestUnitManager) UnmaskUnit(u system.Unit) error { - tum.unmasked = append(tum.unmasked, u.Name) - return nil -} - -type mockInterface struct { - name string - filename string - netdev string - link string - network string - kind string - modprobeParams string -} - -func (i mockInterface) Name() string { - return i.name -} - -func (i mockInterface) Filename() string { - return i.filename -} - -func (i mockInterface) Netdev() string { - return i.netdev -} - -func (i mockInterface) Link() string { - return i.link -} - -func (i mockInterface) Network() string { - return i.network -} - -func (i mockInterface) Type() string { - return i.kind -} - -func (i mockInterface) ModprobeParams() string { - return i.modprobeParams -} - -func TestCreateNetworkingUnits(t *testing.T) { - for _, tt := range []struct { - interfaces []network.InterfaceGenerator - expect []system.Unit - }{ - {nil, nil}, - { - []network.InterfaceGenerator{ - network.InterfaceGenerator(mockInterface{filename: "test"}), - }, - nil, - }, - { - []network.InterfaceGenerator{ - network.InterfaceGenerator(mockInterface{filename: "test1", netdev: "test netdev"}), - network.InterfaceGenerator(mockInterface{filename: "test2", link: "test link"}), - network.InterfaceGenerator(mockInterface{filename: "test3", network: "test network"}), - }, - []system.Unit{ - system.Unit{Unit: config.Unit{Name: "test1.netdev", Runtime: true, Content: "test netdev"}}, - system.Unit{Unit: config.Unit{Name: "test2.link", Runtime: true, Content: "test link"}}, - system.Unit{Unit: config.Unit{Name: "test3.network", Runtime: true, Content: "test network"}}, - }, - }, - { - []network.InterfaceGenerator{ - network.InterfaceGenerator(mockInterface{filename: "test", netdev: "test netdev", link: "test link", network: "test network"}), - }, - []system.Unit{ - system.Unit{Unit: config.Unit{Name: "test.netdev", Runtime: true, Content: "test netdev"}}, - system.Unit{Unit: config.Unit{Name: "test.link", Runtime: true, Content: "test link"}}, - system.Unit{Unit: config.Unit{Name: "test.network", Runtime: true, Content: "test network"}}, - }, - }, - } { - units := createNetworkingUnits(tt.interfaces) - if !reflect.DeepEqual(tt.expect, units) { - t.Errorf("bad units (%+v): want %#v, got %#v", tt.interfaces, tt.expect, units) - } - } -} - -func TestProcessUnits(t *testing.T) { - tests := []struct { - units []system.Unit - - result TestUnitManager - }{ - { - units: []system.Unit{ - system.Unit{Unit: config.Unit{ - Name: "foo", - Mask: true, - }}, - }, - result: TestUnitManager{ - masked: []string{"foo"}, - }, - }, - { - units: []system.Unit{ - system.Unit{Unit: config.Unit{ - Name: "baz.service", - Content: "[Service]\nExecStart=/bin/baz", - Command: "start", - }}, - system.Unit{Unit: config.Unit{ - Name: "foo.network", - Content: "[Network]\nFoo=true", - }}, - system.Unit{Unit: config.Unit{ - Name: "bar.network", - Content: "[Network]\nBar=true", - }}, - }, - result: TestUnitManager{ - placed: []string{"baz.service", "foo.network", "bar.network"}, - commands: []UnitAction{ - UnitAction{"systemd-networkd.service", "restart"}, - UnitAction{"baz.service", "start"}, - }, - reload: true, - }, - }, - { - units: []system.Unit{ - system.Unit{Unit: config.Unit{ - Name: "baz.service", - Content: "[Service]\nExecStart=/bin/true", - }}, - }, - result: TestUnitManager{ - placed: []string{"baz.service"}, - reload: true, - }, - }, - { - units: []system.Unit{ - system.Unit{Unit: config.Unit{ - Name: "locksmithd.service", - Runtime: true, - }}, - }, - result: TestUnitManager{ - unmasked: []string{"locksmithd.service"}, - }, - }, - { - units: []system.Unit{ - system.Unit{Unit: config.Unit{ - Name: "woof", - Enable: true, - }}, - }, - result: TestUnitManager{ - enabled: []string{"woof"}, - }, - }, - { - units: []system.Unit{ - system.Unit{Unit: config.Unit{ - Name: "hi.service", - Runtime: true, - Content: "[Service]\nExecStart=/bin/echo hi", - DropIns: []config.UnitDropIn{ - { - Name: "lo.conf", - Content: "[Service]\nExecStart=/bin/echo lo", - }, - { - Name: "bye.conf", - Content: "[Service]\nExecStart=/bin/echo bye", - }, - }, - }}, - }, - result: TestUnitManager{ - placed: []string{"hi.service", "hi.service.d/lo.conf", "hi.service.d/bye.conf"}, - unmasked: []string{"hi.service"}, - reload: true, - }, - }, - { - units: []system.Unit{ - system.Unit{Unit: config.Unit{ - DropIns: []config.UnitDropIn{ - { - Name: "lo.conf", - Content: "[Service]\nExecStart=/bin/echo lo", - }, - }, - }}, - }, - result: TestUnitManager{}, - }, - { - units: []system.Unit{ - system.Unit{Unit: config.Unit{ - Name: "hi.service", - DropIns: []config.UnitDropIn{ - { - Content: "[Service]\nExecStart=/bin/echo lo", - }, - }, - }}, - }, - result: TestUnitManager{}, - }, - { - units: []system.Unit{ - system.Unit{Unit: config.Unit{ - Name: "hi.service", - DropIns: []config.UnitDropIn{ - { - Name: "lo.conf", - }, - }, - }}, - }, - result: TestUnitManager{}, - }, - } - - for _, tt := range tests { - tum := &TestUnitManager{} - if err := processUnits(tt.units, "", tum); err != nil { - t.Errorf("bad error (%+v): want nil, got %s", tt.units, err) - } - if !reflect.DeepEqual(tt.result, *tum) { - t.Errorf("bad result (%+v): want %+v, got %+v", tt.units, tt.result, tum) - } - } -} diff --git a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/env.go b/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/env.go deleted file mode 100644 index f90cc932..00000000 --- a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/env.go +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2015 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package initialize - -import ( - "net" - "os" - "path" - "regexp" - "strings" - - "github.com/coreos/coreos-cloudinit/config" - "github.com/coreos/coreos-cloudinit/datasource" - "github.com/coreos/coreos-cloudinit/system" -) - -const DefaultSSHKeyName = "coreos-cloudinit" - -type Environment struct { - root string - configRoot string - workspace string - sshKeyName string - substitutions map[string]string -} - -// TODO(jonboulle): this is getting unwieldy, should be able to simplify the interface somehow -func NewEnvironment(root, configRoot, workspace, sshKeyName string, metadata datasource.Metadata) *Environment { - firstNonNull := func(ip net.IP, env string) string { - if ip == nil { - return env - } - return ip.String() - } - substitutions := map[string]string{ - "$public_ipv4": firstNonNull(metadata.PublicIPv4, os.Getenv("COREOS_PUBLIC_IPV4")), - "$private_ipv4": firstNonNull(metadata.PrivateIPv4, os.Getenv("COREOS_PRIVATE_IPV4")), - "$public_ipv6": firstNonNull(metadata.PublicIPv6, os.Getenv("COREOS_PUBLIC_IPV6")), - "$private_ipv6": firstNonNull(metadata.PrivateIPv6, os.Getenv("COREOS_PRIVATE_IPV6")), - } - return &Environment{root, configRoot, workspace, sshKeyName, substitutions} -} - -func (e *Environment) Workspace() string { - return path.Join(e.root, e.workspace) -} - -func (e *Environment) Root() string { - return e.root -} - -func (e *Environment) ConfigRoot() string { - return e.configRoot -} - -func (e *Environment) SSHKeyName() string { - return e.sshKeyName -} - -func (e *Environment) SetSSHKeyName(name string) { - e.sshKeyName = name -} - -// Apply goes through the map of substitutions and replaces all instances of -// the keys with their respective values. It supports escaping substitutions -// with a leading '\'. -func (e *Environment) Apply(data string) string { - for key, val := range e.substitutions { - matchKey := strings.Replace(key, `$`, `\$`, -1) - replKey := strings.Replace(key, `$`, `$$`, -1) - - // "key" -> "val" - data = regexp.MustCompile(`([^\\]|^)`+matchKey).ReplaceAllString(data, `${1}`+val) - // "\key" -> "key" - data = regexp.MustCompile(`\\`+matchKey).ReplaceAllString(data, replKey) - } - return data -} - -func (e *Environment) DefaultEnvironmentFile() *system.EnvFile { - ef := system.EnvFile{ - File: &system.File{File: config.File{ - Path: "/etc/environment", - }}, - Vars: map[string]string{}, - } - if ip, ok := e.substitutions["$public_ipv4"]; ok && len(ip) > 0 { - ef.Vars["COREOS_PUBLIC_IPV4"] = ip - } - if ip, ok := e.substitutions["$private_ipv4"]; ok && len(ip) > 0 { - ef.Vars["COREOS_PRIVATE_IPV4"] = ip - } - if ip, ok := e.substitutions["$public_ipv6"]; ok && len(ip) > 0 { - ef.Vars["COREOS_PUBLIC_IPV6"] = ip - } - if ip, ok := e.substitutions["$private_ipv6"]; ok && len(ip) > 0 { - ef.Vars["COREOS_PRIVATE_IPV6"] = ip - } - if len(ef.Vars) == 0 { - return nil - } else { - return &ef - } -} diff --git a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/env_test.go b/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/env_test.go deleted file mode 100644 index abb770cf..00000000 --- a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/env_test.go +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright 2015 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package initialize - -import ( - "io/ioutil" - "net" - "os" - "path" - "testing" - - "github.com/coreos/coreos-cloudinit/datasource" - "github.com/coreos/coreos-cloudinit/system" -) - -func TestEnvironmentApply(t *testing.T) { - os.Setenv("COREOS_PUBLIC_IPV4", "1.2.3.4") - os.Setenv("COREOS_PRIVATE_IPV4", "5.6.7.8") - os.Setenv("COREOS_PUBLIC_IPV6", "1234::") - os.Setenv("COREOS_PRIVATE_IPV6", "5678::") - for _, tt := range []struct { - metadata datasource.Metadata - input string - out string - }{ - { - // Substituting both values directly should always take precedence - // over environment variables - datasource.Metadata{ - PublicIPv4: net.ParseIP("192.0.2.3"), - PrivateIPv4: net.ParseIP("192.0.2.203"), - PublicIPv6: net.ParseIP("fe00:1234::"), - PrivateIPv6: net.ParseIP("fe00:5678::"), - }, - `[Service] -ExecStart=/usr/bin/echo "$public_ipv4 $public_ipv6" -ExecStop=/usr/bin/echo $private_ipv4 $private_ipv6 -ExecStop=/usr/bin/echo $unknown`, - `[Service] -ExecStart=/usr/bin/echo "192.0.2.3 fe00:1234::" -ExecStop=/usr/bin/echo 192.0.2.203 fe00:5678:: -ExecStop=/usr/bin/echo $unknown`, - }, - { - // Substituting one value directly while falling back with the other - datasource.Metadata{ - PrivateIPv4: net.ParseIP("127.0.0.1"), - }, - "$private_ipv4\n$public_ipv4", - "127.0.0.1\n1.2.3.4", - }, - { - // Falling back to environment variables for both values - datasource.Metadata{}, - "$private_ipv4\n$public_ipv4", - "5.6.7.8\n1.2.3.4", - }, - { - // No substitutions - datasource.Metadata{}, - "$private_ipv4\nfoobar", - "5.6.7.8\nfoobar", - }, - { - // Escaping substitutions - datasource.Metadata{ - PrivateIPv4: net.ParseIP("127.0.0.1"), - }, - `\$private_ipv4 -$private_ipv4 -addr: \$private_ipv4 -\\$private_ipv4`, - `$private_ipv4 -127.0.0.1 -addr: $private_ipv4 -\$private_ipv4`, - }, - { - // No substitutions with escaping - datasource.Metadata{}, - "\\$test\n$test", - "\\$test\n$test", - }, - } { - - env := NewEnvironment("./", "./", "./", "", tt.metadata) - got := env.Apply(tt.input) - if got != tt.out { - t.Fatalf("Environment incorrectly applied.\ngot:\n%s\nwant:\n%s", got, tt.out) - } - } -} - -func TestEnvironmentFile(t *testing.T) { - metadata := datasource.Metadata{ - PublicIPv4: net.ParseIP("1.2.3.4"), - PrivateIPv4: net.ParseIP("5.6.7.8"), - PublicIPv6: net.ParseIP("1234::"), - PrivateIPv6: net.ParseIP("5678::"), - } - expect := "COREOS_PRIVATE_IPV4=5.6.7.8\nCOREOS_PRIVATE_IPV6=5678::\nCOREOS_PUBLIC_IPV4=1.2.3.4\nCOREOS_PUBLIC_IPV6=1234::\n" - - dir, err := ioutil.TempDir(os.TempDir(), "coreos-cloudinit-") - if err != nil { - t.Fatalf("Unable to create tempdir: %v", err) - } - defer os.RemoveAll(dir) - - env := NewEnvironment("./", "./", "./", "", metadata) - ef := env.DefaultEnvironmentFile() - err = system.WriteEnvFile(ef, dir) - if err != nil { - t.Fatalf("WriteEnvFile failed: %v", err) - } - - fullPath := path.Join(dir, "etc", "environment") - contents, err := ioutil.ReadFile(fullPath) - if err != nil { - t.Fatalf("Unable to read expected file: %v", err) - } - - if string(contents) != expect { - t.Fatalf("File has incorrect contents: %q", contents) - } -} - -func TestEnvironmentFileNil(t *testing.T) { - os.Clearenv() - metadata := datasource.Metadata{} - - env := NewEnvironment("./", "./", "./", "", metadata) - ef := env.DefaultEnvironmentFile() - if ef != nil { - t.Fatalf("Environment file not nil: %v", ef) - } -} diff --git a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/github.go b/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/github.go deleted file mode 100644 index 2f7755fe..00000000 --- a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/github.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2015 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package initialize - -import ( - "fmt" - - "github.com/coreos/coreos-cloudinit/system" -) - -func SSHImportGithubUser(system_user string, github_user string) error { - url := fmt.Sprintf("https://api.github.com/users/%s/keys", github_user) - keys, err := fetchUserKeys(url) - if err != nil { - return err - } - - key_name := fmt.Sprintf("github-%s", github_user) - return system.AuthorizeSSHKeys(system_user, key_name, keys) -} diff --git a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/ssh_keys.go b/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/ssh_keys.go deleted file mode 100644 index 17b0c4a9..00000000 --- a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/ssh_keys.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2015 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package initialize - -import ( - "encoding/json" - "fmt" - - "github.com/coreos/coreos-cloudinit/pkg" - "github.com/coreos/coreos-cloudinit/system" -) - -type UserKey struct { - ID int `json:"id,omitempty"` - Key string `json:"key"` -} - -func SSHImportKeysFromURL(system_user string, url string) error { - keys, err := fetchUserKeys(url) - if err != nil { - return err - } - - key_name := fmt.Sprintf("coreos-cloudinit-%s", system_user) - return system.AuthorizeSSHKeys(system_user, key_name, keys) -} - -func fetchUserKeys(url string) ([]string, error) { - client := pkg.NewHttpClient() - data, err := client.GetRetry(url) - if err != nil { - return nil, err - } - - var userKeys []UserKey - err = json.Unmarshal(data, &userKeys) - if err != nil { - return nil, err - } - keys := make([]string, 0) - for _, key := range userKeys { - keys = append(keys, key.Key) - } - return keys, err -} diff --git a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/ssh_keys_test.go b/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/ssh_keys_test.go deleted file mode 100644 index 86395797..00000000 --- a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/ssh_keys_test.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2015 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package initialize - -import ( - "fmt" - "net/http" - "net/http/httptest" - "testing" -) - -func TestCloudConfigUsersUrlMarshal(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - gh_res := ` -[ - { - "key": "ssh-dss AAAAB3NzaC1kc3MAAACBAIHAu822ggSkIHrJYvhmBceOSVjuflfQm8RbMMDNVe9relQfuPbN+nxGGTCKzPLebeOcX+Wwi77TPXWwK3BZMglfXxhABlFPsuMb63Tqp94pBYsJdx/iFj9iGo6pKoM1k8ubOcqsUnq+BR9895zRbE7MjdwkGo67+QhCEwvkwAnNAAAAFQCuddVqXLCubzqnWmeHLQE+2GFfHwAAAIBnlXW5h15ndVuwi0htF4oodVSB1KwnTWcuBK+aE1zRs76yvRb0Ws+oifumThDwB/Tec6FQuAfRKfy6piChZqsu5KvL98I+2t5yyi1td+kMvdTnVL2lW44etDKseOcozmknCOmh4Dqvhl/2MwrDAhlPaN08EEq9h3w3mXtNLWH64QAAAIBAzDOKr17llngaKIdDXh+LtXKh87+zfjlTA36/9r2uF2kYE5uApDtu9sPCkt7+YBQt7R8prADPckwAiXwVdk0xijIOpLDBmoydQJJRQ+zTMxvpQmUr/1kUOv0zb+lB657CgvN0vVTmP2swPeMvgntt3C4vw7Ab+O+MS9peOAJbbQ==" - }, - { - "key": "ssh-dss AAAAB3NzaC1kc3MAAACBANxpzIbTzKTeBRaOIdUxwwGwvDasTfU/PonhbNIuhYjc+xFGvBRTumox2F+luVAKKs4WdvA4nJXaY1OFi6DZftk5Bp4E2JaSzp8ulAzHsMexDdv6LGHGEJj/qdHAL1vHk2K89PpwRFSRZI8XRBLjvkr4ZgBKLG5ZILXPJEPP2j3lAAAAFQCtxoTnV8wy0c4grcGrQ+1sCsD7WQAAAIAqZsW2GviMe1RQrbZT0xAZmI64XRPrnLsoLxycHWlS7r6uUln2c6Ae2MB/YF0d4Kd1XZii9GHj7rrypqEo7MW8uSabhu70nmu1J8m2O3Dsr+4oJLeat9vwPsJV92IKO0jQwjKnAOHOiB9JKGeCw+NfXfogbti9/q38Q6XcS+SI5wAAAIEA1803Y2h+tOOpZXAsNIwl9mRfExWzLQ3L7knwJdznQu/6SW1H/1oyoYLebuk187Qj2UFI5qQ6AZNc49DvohWx0Cg6ABcyubNyoaCjZKWIdxVnItHWNbLe//+tyTu0I2eQwJOORsEPK5gMpf599C7wXQ//DzZOWbTWiHEX52gCTmk=" - }, - { - "id": 5224438, - "key": "ssh-dss AAAAB3NzaC1kc3MAAACBAPKRWdKhzGZuLAJL6M1eM51hWViMqNBC2C6lm2OqGRYLuIf1GJ391widUuSf4wQqnkR22Q9PCmAZ19XCf11wBRMnuw9I/Z3Bt5bXfc+dzFBCmHYGJ6wNSv++H9jxyMb+usmsenWOFZGNO2jN0wrJ4ay8Yt0bwtRU+VCXpuRLszMzAAAAFQDZUIuPjcfK5HLgnwZ/J3lvtvlUjQAAAIEApIkAwLuCQV5j3U6DmI/Y6oELqSUR2purFm8jo8jePFfe1t+ghikgD254/JXlhDCVgY0NLXcak+coJfGCTT23quJ7I5xdpTn/OZO2Q6Woum/bijFC/UWwQbLz0R2nU3DoHv5v6XHQZxuIG4Fsxa91S+vWjZFtI7RuYlBCZA//ANMAAACBAJO0FojzkX6IeaWLqrgu9GTkFwGFazZ+LPH5JOWPoPn1hQKuR32Uf6qNcBZcIjY7SF0P7HF5rLQd6zKZzHqqQQ92MV555NEwjsnJglYU8CaaZsfYooaGPgA1YN7RhTSAuDmUW5Hyfj5BH4NTtrzrvJxIhDoQLf31Fasjw00r4R0O" - } -] -` - fmt.Fprintln(w, gh_res) - })) - defer ts.Close() - - keys, err := fetchUserKeys(ts.URL) - if err != nil { - t.Fatalf("Encountered unexpected error: %v", err) - } - expected := "ssh-dss AAAAB3NzaC1kc3MAAACBAIHAu822ggSkIHrJYvhmBceOSVjuflfQm8RbMMDNVe9relQfuPbN+nxGGTCKzPLebeOcX+Wwi77TPXWwK3BZMglfXxhABlFPsuMb63Tqp94pBYsJdx/iFj9iGo6pKoM1k8ubOcqsUnq+BR9895zRbE7MjdwkGo67+QhCEwvkwAnNAAAAFQCuddVqXLCubzqnWmeHLQE+2GFfHwAAAIBnlXW5h15ndVuwi0htF4oodVSB1KwnTWcuBK+aE1zRs76yvRb0Ws+oifumThDwB/Tec6FQuAfRKfy6piChZqsu5KvL98I+2t5yyi1td+kMvdTnVL2lW44etDKseOcozmknCOmh4Dqvhl/2MwrDAhlPaN08EEq9h3w3mXtNLWH64QAAAIBAzDOKr17llngaKIdDXh+LtXKh87+zfjlTA36/9r2uF2kYE5uApDtu9sPCkt7+YBQt7R8prADPckwAiXwVdk0xijIOpLDBmoydQJJRQ+zTMxvpQmUr/1kUOv0zb+lB657CgvN0vVTmP2swPeMvgntt3C4vw7Ab+O+MS9peOAJbbQ==" - if keys[0] != expected { - t.Fatalf("expected %s, got %s", expected, keys[0]) - } - expected = "ssh-dss AAAAB3NzaC1kc3MAAACBAPKRWdKhzGZuLAJL6M1eM51hWViMqNBC2C6lm2OqGRYLuIf1GJ391widUuSf4wQqnkR22Q9PCmAZ19XCf11wBRMnuw9I/Z3Bt5bXfc+dzFBCmHYGJ6wNSv++H9jxyMb+usmsenWOFZGNO2jN0wrJ4ay8Yt0bwtRU+VCXpuRLszMzAAAAFQDZUIuPjcfK5HLgnwZ/J3lvtvlUjQAAAIEApIkAwLuCQV5j3U6DmI/Y6oELqSUR2purFm8jo8jePFfe1t+ghikgD254/JXlhDCVgY0NLXcak+coJfGCTT23quJ7I5xdpTn/OZO2Q6Woum/bijFC/UWwQbLz0R2nU3DoHv5v6XHQZxuIG4Fsxa91S+vWjZFtI7RuYlBCZA//ANMAAACBAJO0FojzkX6IeaWLqrgu9GTkFwGFazZ+LPH5JOWPoPn1hQKuR32Uf6qNcBZcIjY7SF0P7HF5rLQd6zKZzHqqQQ92MV555NEwjsnJglYU8CaaZsfYooaGPgA1YN7RhTSAuDmUW5Hyfj5BH4NTtrzrvJxIhDoQLf31Fasjw00r4R0O" - if keys[2] != expected { - t.Fatalf("expected %s, got %s", expected, keys[2]) - } -} diff --git a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/user_data.go b/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/user_data.go deleted file mode 100644 index 170efaaa..00000000 --- a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/user_data.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2015 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package initialize - -import ( - "errors" - "log" - - "github.com/coreos/coreos-cloudinit/config" -) - -func ParseUserData(contents string) (interface{}, error) { - if len(contents) == 0 { - return nil, nil - } - - switch { - case config.IsScript(contents): - log.Printf("Parsing user-data as script") - return config.NewScript(contents) - case config.IsCloudConfig(contents): - log.Printf("Parsing user-data as cloud-config") - return config.NewCloudConfig(contents) - default: - return nil, errors.New("Unrecognized user-data format") - } -} diff --git a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/user_data_test.go b/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/user_data_test.go deleted file mode 100644 index 1d883694..00000000 --- a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/user_data_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2015 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package initialize - -import ( - "testing" - - "github.com/coreos/coreos-cloudinit/config" -) - -func TestParseHeaderCRLF(t *testing.T) { - configs := []string{ - "#cloud-config\nfoo: bar", - "#cloud-config\r\nfoo: bar", - } - - for i, config := range configs { - _, err := ParseUserData(config) - if err != nil { - t.Errorf("Failed parsing config %d: %v", i, err) - } - } - - scripts := []string{ - "#!bin/bash\necho foo", - "#!bin/bash\r\necho foo", - } - - for i, script := range scripts { - _, err := ParseUserData(script) - if err != nil { - t.Errorf("Failed parsing script %d: %v", i, err) - } - } -} - -func TestParseConfigCRLF(t *testing.T) { - contents := "#cloud-config\r\nhostname: foo\r\nssh_authorized_keys:\r\n - foobar\r\n" - ud, err := ParseUserData(contents) - if err != nil { - t.Fatalf("Failed parsing config: %v", err) - } - - cfg := ud.(*config.CloudConfig) - - if cfg.Hostname != "foo" { - t.Error("Failed parsing hostname from config") - } - - if len(cfg.SSHAuthorizedKeys) != 1 { - t.Error("Parsed incorrect number of SSH keys") - } -} - -func TestParseConfigEmpty(t *testing.T) { - i, e := ParseUserData(``) - if i != nil { - t.Error("ParseUserData of empty string returned non-nil unexpectedly") - } else if e != nil { - t.Error("ParseUserData of empty string returned error unexpectedly") - } -} diff --git a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/workspace.go b/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/workspace.go deleted file mode 100644 index 540dcf41..00000000 --- a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/workspace.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2015 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package initialize - -import ( - "io/ioutil" - "path" - "strings" - - "github.com/coreos/coreos-cloudinit/config" - "github.com/coreos/coreos-cloudinit/system" -) - -func PrepWorkspace(workspace string) error { - if err := system.EnsureDirectoryExists(workspace); err != nil { - return err - } - - scripts := path.Join(workspace, "scripts") - if err := system.EnsureDirectoryExists(scripts); err != nil { - return err - } - - return nil -} - -func PersistScriptInWorkspace(script config.Script, workspace string) (string, error) { - scriptsPath := path.Join(workspace, "scripts") - tmp, err := ioutil.TempFile(scriptsPath, "") - if err != nil { - return "", err - } - tmp.Close() - - relpath := strings.TrimPrefix(tmp.Name(), workspace) - - file := system.File{File: config.File{ - Path: relpath, - RawFilePermissions: "0744", - Content: string(script), - }} - - return system.WriteFile(&file, workspace) -} - -func PersistUnitNameInWorkspace(name string, workspace string) error { - file := system.File{File: config.File{ - Path: path.Join("scripts", "unit-name"), - RawFilePermissions: "0644", - Content: name, - }} - _, err := system.WriteFile(&file, workspace) - return err -} diff --git a/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/project.go b/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/project.go index 686e6caa..61ae97aa 100644 --- a/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/project.go +++ b/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/project.go @@ -18,6 +18,12 @@ var ( ErrRestart error = errors.New("Restart execution") ) +type ProjectEvent struct { + Event Event + Service Service + Data map[string]string +} + func NewProject(name string, factory ServiceFactory) *Project { return &Project{ Name: name, @@ -72,6 +78,12 @@ func (p *Project) Up() error { } func (p *Project) startAll(wrappers map[string]*ServiceWrapper) error { + for name, _ := range p.Services { + if _, ok := wrappers[name]; !ok { + wrappers[name] = NewServiceWrapper(name, p) + } + } + restart := false for _, wrapper := range wrappers { @@ -176,6 +188,10 @@ func (s *ServiceWrapper) Wait() error { return s.err } +func (p *Project) AddListener(c chan<- ProjectEvent) { + p.listeners = append(p.listeners, c) +} + func (p *Project) Notify(event Event, service Service, data map[string]string) { buffer := bytes.NewBuffer(nil) if data != nil { @@ -204,4 +220,17 @@ func (p *Project) Notify(event Event, service Service, data map[string]string) { } else { logf("[%d/%d] [%s]: %s %s", p.upCount, len(p.Services), service.Name(), event, buffer.Bytes()) } + + for _, l := range p.listeners { + projectEvent := ProjectEvent{ + Event: event, + Service: service, + Data: data, + } + // Don't ever block + select { + case l <- projectEvent: + default: + } + } } diff --git a/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/types.go b/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/types.go index 938d1fcd..de9efa3e 100644 --- a/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/types.go +++ b/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/types.go @@ -66,6 +66,7 @@ type Project struct { factory ServiceFactory ReloadCallback func() error upCount int + listeners []chan<- ProjectEvent } type Service interface { From b1c519ebe0269b6cfc9c7e681b3d3d364f75a29d Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Wed, 15 Apr 2015 00:43:38 -0700 Subject: [PATCH 06/14] Force os-installer container to use json-file logger --- cmd/control/os.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/control/os.go b/cmd/control/os.go index f5f35450..acffb685 100644 --- a/cmd/control/os.go +++ b/cmd/control/os.go @@ -162,6 +162,7 @@ func startUpgradeContainer(image string, stage, force bool) { container := docker.NewContainer(config.DOCKER_SYSTEM_HOST, &config.ContainerConfig{ Cmd: "--name=os-upgrade " + + "--log-driver=json-file" + "--rm " + "--privileged " + "--net=host " + From adc6825ee682508bf785abc2153b8d0454d98352 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Wed, 15 Apr 2015 09:19:16 -0700 Subject: [PATCH 07/14] Update godeps --- Godeps/Godeps.json | 8 ++--- .../rancher-compose/project/project.go | 25 +++++++++++++++- .../rancher-compose/project/types.go | 30 +++++++++++-------- 3 files changed, 46 insertions(+), 17 deletions(-) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 434613bd..9ba1627e 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -221,13 +221,13 @@ }, { "ImportPath": "github.com/rancherio/rancher-compose/docker", - "Comment": "0.1.0-9-g7343438", - "Rev": "734343806223cab5de0532ea7add0beeb2cbea92" + "Comment": "0.1.0-10-g6c6a2e3", + "Rev": "6c6a2e3ad4023fafc7509787994dc51774006f26" }, { "ImportPath": "github.com/rancherio/rancher-compose/project", - "Comment": "0.1.0-9-g7343438", - "Rev": "734343806223cab5de0532ea7add0beeb2cbea92" + "Comment": "0.1.0-10-g6c6a2e3", + "Rev": "6c6a2e3ad4023fafc7509787994dc51774006f26" }, { "ImportPath": "github.com/ryanuber/go-glob", diff --git a/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/project.go b/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/project.go index 61ae97aa..25ab41a4 100644 --- a/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/project.go +++ b/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/project.go @@ -3,6 +3,7 @@ package project import ( "bytes" "errors" + "fmt" "strings" "sync" @@ -33,8 +34,30 @@ func NewProject(name string, factory ServiceFactory) *Project { } } +func (p *Project) createService(name string, config ServiceConfig) (Service, error) { + if p.EnvironmentLookup != nil { + parsedEnv := make([]string, 0, len(config.Environment)) + + for _, env := range config.Environment { + if strings.IndexRune(env, '=') != -1 { + parsedEnv = append(parsedEnv, env) + continue + } + + value := p.EnvironmentLookup.Lookup(env, name, &config) + if value != "" { + parsedEnv = append(parsedEnv, fmt.Sprintf("%s=%s", env, value)) + } + } + + config.Environment = parsedEnv + } + + return p.factory.Create(p, name, &config) +} + func (p *Project) AddConfig(name string, config *ServiceConfig) error { - service, err := p.factory.Create(p, name, config) + service, err := p.createService(name, *config) if err != nil { log.Errorf("Failed to create service for %s : %v", name, err) return err diff --git a/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/types.go b/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/types.go index de9efa3e..811a60ad 100644 --- a/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/types.go +++ b/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/types.go @@ -7,8 +7,9 @@ type Event string const ( CONTAINER_ID = "container_id" - CONTAINER_CREATED = Event("Created container") - CONTAINER_STARTED = Event("Started container") + CONTAINER_STARTING = Event("Starting container") + CONTAINER_CREATED = Event("Created container") + CONTAINER_STARTED = Event("Started container") SERVICE_ADD = Event("Adding") SERVICE_UP_START = Event("Starting") @@ -56,17 +57,22 @@ type ServiceConfig struct { ExternalLinks []string `yaml:"external_links,omitempty"` } +type EnvironmentLookup interface { + Lookup(key, serviceName string, config *ServiceConfig) string +} + type Project struct { - Name string - configs map[string]*ServiceConfig - Services map[string]Service - file string - content []byte - client *client.RancherClient - factory ServiceFactory - ReloadCallback func() error - upCount int - listeners []chan<- ProjectEvent + EnvironmentLookup EnvironmentLookup + Name string + configs map[string]*ServiceConfig + Services map[string]Service + file string + content []byte + client *client.RancherClient + factory ServiceFactory + ReloadCallback func() error + upCount int + listeners []chan<- ProjectEvent } type Service interface { From 0b5eb352ba99991f67e60041cb3c9980f6cac4df Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Wed, 15 Apr 2015 22:57:59 -0700 Subject: [PATCH 08/14] Services in compose format --- cmd/cloudinit/cloudinit.go | 6 +- cmd/control/service.go | 38 +++++++++--- config/config.go | 11 ++++ config/default.go | 52 +++++++++------- config/types.go | 13 +++- docker/container.go | 45 ++++++++++---- docker/factory.go | 13 +++- docker/services.go | 123 +++++++++++++++++++++++++++++++++++++ init/bootstrap.go | 5 +- init/init.go | 19 +++--- init/sysinit.go | 79 +----------------------- util/util.go | 65 +++++++++++++++++++- 12 files changed, 328 insertions(+), 141 deletions(-) create mode 100644 docker/services.go diff --git a/cmd/cloudinit/cloudinit.go b/cmd/cloudinit/cloudinit.go index aa72655d..b1f93c19 100644 --- a/cmd/cloudinit/cloudinit.go +++ b/cmd/cloudinit/cloudinit.go @@ -476,13 +476,11 @@ func isCompose(content string) bool { func toCompose(bytes []byte) ([]byte, error) { result := make(map[interface{}]interface{}) compose := make(map[interface{}]interface{}) - err := yaml.Unmarshal(bytes, &result) + err := yaml.Unmarshal(bytes, &compose) if err != nil { return nil, err } - result["services"] = map[interface{}]interface{}{ - "cloud-config": compose, - } + result["services"] = compose return yaml.Marshal(result) } diff --git a/cmd/control/service.go b/cmd/control/service.go index d43bba3b..67c0b2f1 100644 --- a/cmd/control/service.go +++ b/cmd/control/service.go @@ -6,6 +6,7 @@ import ( "github.com/codegangsta/cli" "github.com/rancherio/os/config" + "github.com/rancherio/os/util" ) func serviceSubCommands() []cli.Command { @@ -36,16 +37,16 @@ func disable(c *cli.Context) { } for _, service := range c.Args() { - if _, ok := cfg.Services[service]; !ok { + if _, ok := cfg.ServicesInclude[service]; !ok { continue } - cfg.Services[service] = false + cfg.ServicesInclude[service] = false changed = true } if changed { - if err = cfg.Set("services", cfg.Services); err != nil { + if err = cfg.Set("services_include", cfg.ServicesInclude); err != nil { log.Fatal(err) } } @@ -59,14 +60,14 @@ func enable(c *cli.Context) { } for _, service := range c.Args() { - if val, ok := cfg.Services[service]; !ok || !val { - cfg.Services[service] = true + if val, ok := cfg.ServicesInclude[service]; !ok || !val { + cfg.ServicesInclude[service] = true changed = true } } if changed { - if err = cfg.Set("services", cfg.Services); err != nil { + if err = cfg.Set("services_include", cfg.ServicesInclude); err != nil { log.Fatal(err) } } @@ -78,7 +79,30 @@ func list(c *cli.Context) { log.Fatal(err) } - for service, enabled := range cfg.Services { + clone := make(map[string]bool) + for service, enabled := range cfg.ServicesInclude { + clone[service] = enabled + } + + services, err := util.GetServices(cfg.Repositories.ToArray()) + if err != nil { + log.Fatalf("Failed to get services: %v", err) + } + + for _, service := range services { + if enabled, ok := clone[service]; ok { + delete(clone, service) + if enabled { + fmt.Printf("enabled %s\n", service) + } else { + fmt.Printf("disabled %s\n", service) + } + } else { + fmt.Printf("disabled %s\n", service) + } + } + + for service, enabled := range clone { if enabled { fmt.Printf("enabled %s\n", service) } else { diff --git a/config/config.go b/config/config.go index 6b98f3d2..2694f1b0 100644 --- a/config/config.go +++ b/config/config.go @@ -264,3 +264,14 @@ func (d *DockerConfig) BridgeConfig() (string, string) { return name, cidr } } + +func (r Repositories) ToArray() []string { + result := make([]string, 0, len(r)) + for _, repo := range r { + if repo.Url != "" { + result = append(result, repo.Url) + } + } + + return result +} diff --git a/config/default.go b/config/default.go index 4beb32a1..b996f7ae 100644 --- a/config/default.go +++ b/config/default.go @@ -81,6 +81,7 @@ func NewConfig() *Config { Privileged: true, Labels: []string{ DETACH + "=false", + SCOPE + "=" + SYSTEM, }, Volumes: []string{ "/dev:/host/dev", @@ -98,6 +99,7 @@ func NewConfig() *Config { Privileged: true, Labels: []string{ DETACH + "=true", + SCOPE + "=" + SYSTEM, }, Environment: []string{ "DAEMON=true", @@ -113,6 +115,7 @@ func NewConfig() *Config { Privileged: true, Labels: []string{ CREATE_ONLY + "=true", + SCOPE + "=" + SYSTEM, }, Volumes: []string{ "/dev:/host/dev", @@ -132,6 +135,7 @@ func NewConfig() *Config { Privileged: true, Labels: []string{ CREATE_ONLY + "=true", + SCOPE + "=" + SYSTEM, }, Volumes: []string{ "/init:/sbin/halt:ro", @@ -156,6 +160,7 @@ func NewConfig() *Config { Privileged: true, Labels: []string{ CREATE_ONLY + "=true", + SCOPE + "=" + SYSTEM, }, Volumes: []string{ "/home:/home", @@ -170,6 +175,7 @@ func NewConfig() *Config { Privileged: true, Labels: []string{ CREATE_ONLY + "=true", + SCOPE + "=" + SYSTEM, }, Volumes: []string{ "/var/lib/rancher:/var/lib/rancher", @@ -185,6 +191,7 @@ func NewConfig() *Config { Privileged: true, Labels: []string{ CREATE_ONLY + "=true", + SCOPE + "=" + SYSTEM, }, VolumesFrom: []string{ "docker-volumes", @@ -201,6 +208,7 @@ func NewConfig() *Config { Labels: []string{ RELOAD_CONFIG + "=true", DETACH + "=false", + SCOPE + "=" + SYSTEM, }, Environment: []string{ "CLOUD_INIT_NETWORK=false", @@ -216,6 +224,7 @@ func NewConfig() *Config { Net: "host", Labels: []string{ DETACH + "=false", + SCOPE + "=" + SYSTEM, }, Links: []string{ "cloud-init-pre", @@ -231,6 +240,7 @@ func NewConfig() *Config { Labels: []string{ RELOAD_CONFIG + "=true", DETACH + "=false", + SCOPE + "=" + SYSTEM, }, Net: "host", Links: []string{ @@ -246,6 +256,9 @@ func NewConfig() *Config { Image: "ntp", Privileged: true, Net: "host", + Labels: []string{ + SCOPE + "=" + SYSTEM, + }, Links: []string{ "cloud-init", "network", @@ -255,6 +268,9 @@ func NewConfig() *Config { Image: "syslog", Privileged: true, Net: "host", + Labels: []string{ + SCOPE + "=" + SYSTEM, + }, VolumesFrom: []string{ "system-volumes", }, @@ -266,6 +282,9 @@ func NewConfig() *Config { Pid: "host", Ipc: "host", Net: "host", + Labels: []string{ + SCOPE + "=" + SYSTEM, + }, Links: []string{ "network", }, @@ -277,7 +296,8 @@ func NewConfig() *Config { Image: "userdockerwait", Net: "host", Labels: []string{ - "io.rancher.os.detach=false", + DETACH + "=false", + SCOPE + "=" + SYSTEM, }, Links: []string{ "userdocker", @@ -292,6 +312,9 @@ func NewConfig() *Config { Links: []string{ "cloud-init", }, + Labels: []string{ + SCOPE + "=" + SYSTEM, + }, VolumesFrom: []string{ "all-volumes", }, @@ -301,31 +324,14 @@ func NewConfig() *Config { Net: "host", }, }, - Services: map[string]bool{ + ServicesInclude: map[string]bool{ "ubuntu-console": false, }, - BundledServices: map[string]Config{ - "ubuntu-console": { - SystemContainers: map[string]*project.ServiceConfig{ - "console": { - Image: "rancher/ubuntuconsole:" + IMAGE_VERSION, - Privileged: true, - Labels: []string{ - DETACH + "=true", - }, - Links: []string{ - "cloud-init", - }, - VolumesFrom: []string{ - "all-volumes", - }, - Restart: "always", - Pid: "host", - Ipc: "host", - Net: "host", - }, - }, + Repositories: map[string]Repository{ + "core": Repository{ + Url: "https://raw.githubusercontent.com/rancherio/os-services/master/", }, }, + Services: map[string]*project.ServiceConfig{}, } } diff --git a/config/types.go b/config/types.go index 0aee7d06..22dbeee7 100644 --- a/config/types.go +++ b/config/types.go @@ -23,6 +23,8 @@ const ( REMOVE = "io.rancher.os.remove" CREATE_ONLY = "io.rancher.os.createonly" RELOAD_CONFIG = "io.rancher.os.reloadconfig" + SCOPE = "io.rancher.os.scope" + SYSTEM = "system" ) var ( @@ -42,18 +44,25 @@ type ContainerConfig struct { Service *project.ServiceConfig `yaml:service,omitempty` } +type Repository struct { + Url string `yaml:url,omitempty` +} + +type Repositories map[string]Repository + type Config struct { Environment map[string]string `yaml:"environment,omitempty"` - BundledServices map[string]Config `yaml:"bundled_services,omitempty"` + Services map[string]*project.ServiceConfig `yaml:"services,omitempty"` BootstrapContainers map[string]*project.ServiceConfig `yaml:"bootstrap_containers,omitempty"` BootstrapDocker DockerConfig `yaml:"bootstrap_docker,omitempty"` CloudInit CloudInit `yaml:"cloud_init,omitempty"` Console ConsoleConfig `yaml:"console,omitempty"` Debug bool `yaml:"debug,omitempty"` Disable []string `yaml:"disable,omitempty"` - Services map[string]bool `yaml:"services,omitempty"` + ServicesInclude map[string]bool `yaml:"services_include,omitempty"` Modules []string `yaml:"modules,omitempty"` Network NetworkConfig `yaml:"network,omitempty"` + Repositories Repositories `yaml:"repositories,omitempty"` Ssh SshConfig `yaml:"ssh,omitempty"` State StateConfig `yaml:"state,omitempty"` SystemContainers map[string]*project.ServiceConfig `yaml:"system_containers,omitempty"` diff --git a/docker/container.go b/docker/container.go index f5690f8c..451ed907 100644 --- a/docker/container.go +++ b/docker/container.go @@ -71,6 +71,7 @@ func NewContainerFromService(dockerHost string, name string, service *project.Se Service: service, }, } + return c.Parse() } @@ -174,34 +175,53 @@ func (c *Container) requiresSyslog() bool { return (c.ContainerCfg.Service.LogDriver == "" || c.ContainerCfg.Service.LogDriver == "syslog") } +func (c *Container) requiresUserDocker() bool { + if c.dockerHost == config.DOCKER_HOST { + return true + } + + for _, v := range c.ContainerCfg.Service.Volumes { + if strings.Index(v, "/var/run/docker.sock") != -1 { + return true + } + } + + return false +} + func (c *Container) hasLink(link string) bool { return util.Contains(c.ContainerCfg.Service.Links, link) } func (c *Container) addLink(link string) { + if c.hasLink(link) { + return + } + + log.Debugf("Adding %s link to %s", link, c.Name) c.ContainerCfg.Service.Links = append(c.ContainerCfg.Service.Links, link) } func (c *Container) parseService() { - client, err := NewClient(c.dockerHost) - if err != nil { - c.Err = err - return + if c.requiresSyslog() { + c.addLink("syslog") } - if c.ContainerCfg.Service.Image != "" { + if c.requiresUserDocker() { + c.addLink("userdockerwait") + } else if c.ContainerCfg.Service.Image != "" { + client, err := NewClient(c.dockerHost) + if err != nil { + c.Err = err + return + } + i, _ := client.InspectImage(c.ContainerCfg.Service.Image) - if i == nil && !c.hasLink("network") { - log.Debugf("Adding network link to %s", c.Name) + if i == nil { c.addLink("network") } } - if c.requiresSyslog() && !c.hasLink("syslog") { - log.Debugf("Adding syslog link to %s\n", c.Name) - c.addLink("syslog") - } - cfg, hostConfig, err := docker.Convert(c.ContainerCfg.Service) if err != nil { c.Err = err @@ -215,7 +235,6 @@ func (c *Container) parseService() { c.remove = c.Config.Labels[config.REMOVE] != "false" c.ContainerCfg.CreateOnly = c.Config.Labels[config.CREATE_ONLY] == "true" c.ContainerCfg.ReloadConfig = c.Config.Labels[config.RELOAD_CONFIG] == "true" - } func (c *Container) parseCmd() { diff --git a/docker/factory.go b/docker/factory.go index df64593a..c950f6ba 100644 --- a/docker/factory.go +++ b/docker/factory.go @@ -38,6 +38,8 @@ func (c *containerBasedService) Up() error { var event project.Event + c.project.Notify(project.CONTAINER_STARTING, c, map[string]string{}) + if create { container.Create() event = project.CONTAINER_CREATED @@ -71,8 +73,17 @@ func (c *containerBasedService) Name() string { return c.name } +func isSystemService(serviceConfig *project.ServiceConfig) bool { + return util.GetValue(serviceConfig.Labels, config.SCOPE) == config.SYSTEM +} + func (c *ContainerFactory) Create(project *project.Project, name string, serviceConfig *project.ServiceConfig) (project.Service, error) { - container := NewContainerFromService(config.DOCKER_SYSTEM_HOST, name, serviceConfig) + host := config.DOCKER_HOST + if isSystemService(serviceConfig) { + host = config.DOCKER_SYSTEM_HOST + } + + container := NewContainerFromService(host, name, serviceConfig) if container.Err != nil { return nil, container.Err diff --git a/docker/services.go b/docker/services.go new file mode 100644 index 00000000..174bb4f9 --- /dev/null +++ b/docker/services.go @@ -0,0 +1,123 @@ +package docker + +import ( + "fmt" + + log "github.com/Sirupsen/logrus" + "github.com/rancherio/os/config" + "github.com/rancherio/os/util" + "github.com/rancherio/rancher-compose/project" +) + +type configEnvironemnt struct { + cfg *config.Config +} + +func (c *configEnvironemnt) Lookup(key, serviceName string, serviceConfig *project.ServiceConfig) []string { + result := "" + fullKey := fmt.Sprintf("%s/%s", serviceName, key) + if value, ok := c.cfg.Environment[fullKey]; ok { + result = value + } else if value, ok := c.cfg.Environment[key]; ok { + result = value + } + + if result == "" { + return []string{} + } else { + return []string{fmt.Sprintf("%s=%s", key, result)} + } +} + +func RunServices(name string, cfg *config.Config, configs map[string]*project.ServiceConfig) error { + network := false + projectEvents := make(chan project.ProjectEvent) + p := project.NewProject(name, NewContainerFactory(cfg)) + p.EnvironmentLookup = &configEnvironemnt{cfg: cfg} + p.AddListener(projectEvents) + enabled := make(map[string]bool) + + for name, serviceConfig := range configs { + if err := p.AddConfig(name, serviceConfig); err != nil { + log.Infof("Failed loading service %s", name) + } + } + + p.ReloadCallback = func() error { + err := cfg.Reload() + if err != nil { + return err + } + + for service, serviceEnabled := range cfg.ServicesInclude { + if !serviceEnabled { + continue + } + + if _, ok := enabled[service]; ok { + continue + } + + //if config, ok := cfg.BundledServices[service]; ok { + // for name, s := range config.SystemContainers { + // if err := p.AddConfig(name, s); err != nil { + // log.Errorf("Failed to load %s : %v", name, err) + // } + // } + //} else { + bytes, err := LoadServiceResource(service, network, cfg) + if err != nil { + if err == util.ErrNoNetwork { + log.Debugf("Can not load %s, networking not enabled", service) + } else { + log.Errorf("Failed to load %s : %v", service, err) + } + continue + } + + err = p.Load(bytes) + if err != nil { + log.Errorf("Failed to load %s : %v", service, err) + continue + } + //} + + enabled[service] = true + } + + for service, config := range cfg.Services { + if _, ok := enabled[service]; ok { + continue + } + + err = p.AddConfig(service, config) + if err != nil { + log.Errorf("Failed to load %s : %v", service, err) + continue + } + + enabled[service] = true + } + + return nil + } + + go func() { + for event := range projectEvents { + if event.Event == project.CONTAINER_STARTED && event.Service.Name() == "network" { + network = true + } + } + }() + + err := p.ReloadCallback() + if err != nil { + log.Errorf("Failed to reload %s : %v", name, err) + return err + } + return p.Up() +} + +func LoadServiceResource(name string, network bool, cfg *config.Config) ([]byte, error) { + return util.LoadResource(name, network, cfg.Repositories.ToArray()) +} diff --git a/init/bootstrap.go b/init/bootstrap.go index ceed6567..5b2acf6a 100644 --- a/init/bootstrap.go +++ b/init/bootstrap.go @@ -7,6 +7,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/rancherio/os/config" + "github.com/rancherio/os/docker" "github.com/rancherio/os/util" "github.com/rancherio/rancher-compose/project" ) @@ -55,7 +56,7 @@ outer: if format != "" { log.Infof("Auto formatting : %s", format) - return runServices("autoformat", cfg, map[string]*project.ServiceConfig{ + return docker.RunServices("autoformat", cfg, map[string]*project.ServiceConfig{ "autoformat": { Net: "none", Privileged: true, @@ -70,7 +71,7 @@ outer: } func runBootstrapContainers(cfg *config.Config) error { - return runServices("bootstrap", cfg, cfg.BootstrapContainers) + return docker.RunServices("bootstrap", cfg, cfg.BootstrapContainers) } func startDocker(cfg *config.Config) (chan interface{}, error) { diff --git a/init/init.go b/init/init.go index 285550c0..e8b615a5 100644 --- a/init/init.go +++ b/init/init.go @@ -3,7 +3,6 @@ package init import ( "fmt" "io/ioutil" - "net" "os" "os/exec" "strings" @@ -16,9 +15,10 @@ import ( ) const ( - STATE string = "/var" - DOCKER string = "/usr/bin/docker" - SYSINIT string = "/sbin/rancher-sysinit" + STATE string = "/var" + SYSTEM_DOCKER string = "/usr/bin/system-docker" + DOCKER string = "/usr/bin/docker" + SYSINIT string = "/sbin/rancher-sysinit" ) var ( @@ -63,6 +63,7 @@ var ( "/sbin/modprobe": "/busybox", "/usr/sbin/iptables": "/xtables-multi", DOCKER: "/docker", + SYSTEM_DOCKER: "/docker", SYSINIT: "/init", "/home": "/var/lib/rancher/state/home", "/opt": "/var/lib/rancher/state/opt", @@ -237,7 +238,7 @@ func execDocker(cfg *config.Config) error { } os.Stdin.Close() - return syscall.Exec(DOCKER, cfg.SystemDocker.Args, os.Environ()) + return syscall.Exec(SYSTEM_DOCKER, cfg.SystemDocker.Args, os.Environ()) } func MainInit() { @@ -285,10 +286,9 @@ func touchSocket(cfg *config.Config) error { if err := syscall.Unlink(path); err != nil && !os.IsNotExist(err) { return err } - if l, err := net.Listen("unix", path); err != nil { + err := ioutil.WriteFile(path, []byte{}, 0700) + if err != nil { return err - } else { - l.Close() } } @@ -365,7 +365,8 @@ func RunInit() error { return createMounts(postMounts...) }, touchSocket, - remountRo, + // Disable R/O root write now to support updating modules + //remountRo, sysInit, } diff --git a/init/sysinit.go b/init/sysinit.go index cca1b8a8..087a9814 100644 --- a/init/sysinit.go +++ b/init/sysinit.go @@ -9,9 +9,6 @@ import ( dockerClient "github.com/fsouza/go-dockerclient" "github.com/rancherio/os/config" "github.com/rancherio/os/docker" - "github.com/rancherio/os/util" - - "github.com/rancherio/rancher-compose/project" ) func importImage(client *dockerClient.Client, name, fileName string) error { @@ -109,82 +106,8 @@ func loadImages(cfg *config.Config) error { return nil } -func runServices(name string, cfg *config.Config, configs map[string]*project.ServiceConfig) error { - network := false - projectEvents := make(chan project.ProjectEvent) - p := project.NewProject(name, docker.NewContainerFactory(cfg)) - p.AddListener(projectEvents) - enabled := make(map[string]bool) - - for name, serviceConfig := range configs { - if err := p.AddConfig(name, serviceConfig); err != nil { - log.Infof("Failed loading service %s", name) - } - } - - p.ReloadCallback = func() error { - err := cfg.Reload() - if err != nil { - return err - } - - for service, serviceEnabled := range cfg.Services { - if !serviceEnabled { - continue - } - - if _, ok := enabled[service]; ok { - continue - } - - if config, ok := cfg.BundledServices[service]; ok { - for name, s := range config.SystemContainers { - if err := p.AddConfig(name, s); err != nil { - log.Errorf("Failed to load %s : %v", name, err) - } - } - } else { - bytes, err := util.LoadResource(service, network) - if err != nil { - if err == util.ErrNoNetwork { - log.Debugf("Can not load %s, networking not enabled", service) - } else { - log.Errorf("Failed to load %s : %v", service, err) - } - continue - } - - err = p.Load(bytes) - if err != nil { - log.Errorf("Failed to load %s : %v", service, err) - continue - } - } - - enabled[service] = true - } - - return nil - } - - go func() { - for event := range projectEvents { - if event.Event == project.CONTAINER_STARTED && event.Service.Name() == "network" { - network = true - } - } - }() - - err := p.ReloadCallback() - if err != nil { - log.Errorf("Failed to reload %s : %v", name, err) - return err - } - return p.Up() -} - func runContainers(cfg *config.Config) error { - return runServices("system-init", cfg, cfg.SystemContainers) + return docker.RunServices("system-init", cfg, cfg.SystemContainers) } func tailConsole(cfg *config.Config) error { diff --git a/util/util.go b/util/util.go index 162612e6..baf96f4f 100644 --- a/util/util.go +++ b/util/util.go @@ -13,6 +13,8 @@ import ( "strings" "syscall" + log "github.com/Sirupsen/logrus" + "github.com/docker/docker/pkg/mount" "gopkg.in/yaml.v2" ) @@ -20,6 +22,7 @@ import ( var ( letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") ErrNoNetwork = errors.New("Networking not available to load resource") + ErrNotFound = errors.New("Failed to find resource") ) func mountProc() error { @@ -207,7 +210,36 @@ func MergeMaps(left, right map[interface{}]interface{}) { } } -func LoadResource(location string, network bool) ([]byte, error) { +func GetServices(urls []string) ([]string, error) { + result := []string{} + + for _, url := range urls { + indexUrl := fmt.Sprintf("%s/index.yml", url) + content, err := LoadResource(indexUrl, true, []string{}) + if err != nil { + log.Errorf("Failed to load %s: %v", indexUrl, err) + continue + } + + services := make(map[string][]string) + err = yaml.Unmarshal(content, &services) + if err != nil { + log.Errorf("Failed to unmarshal %s: %v", indexUrl, err) + continue + } + + if list, ok := services["services"]; ok { + result = append(result, list...) + } + } + + return []string{}, nil +} + +func LoadResource(location string, network bool, urls []string) ([]byte, error) { + var bytes []byte + err := ErrNotFound + if strings.HasPrefix(location, "http:/") || strings.HasPrefix(location, "https:/") { if !network { return nil, ErrNoNetwork @@ -216,9 +248,38 @@ func LoadResource(location string, network bool) ([]byte, error) { if err != nil { return nil, err } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("non-200 http response: %d", resp.StatusCode) + } defer resp.Body.Close() return ioutil.ReadAll(resp.Body) - } else { + } else if strings.HasPrefix(location, "/") { return ioutil.ReadFile(location) + } else if len(location) > 0 { + for _, url := range urls { + ymlUrl := fmt.Sprintf("%s/%s/%s.yml", url, location[0:1], location) + log.Infof("Loading %s from %s", location, ymlUrl) + bytes, err = LoadResource(ymlUrl, network, []string{}) + if err == nil { + return bytes, nil + } + } } + + return nil, err +} + +func GetValue(kvPairs []string, key string) string { + if kvPairs == nil { + return "" + } + + prefix := key + "=" + for _, i := range kvPairs { + if strings.HasPrefix(i, prefix) { + return strings.TrimPrefix(i, prefix) + } + } + + return "" } From 90a40dbf5abd532982dd07269086a85121bca580 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Wed, 15 Apr 2015 22:58:17 -0700 Subject: [PATCH 09/14] Use common entrypoint for ubuntu-console --- scripts/extraimages/00-ubuntuconsole | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/extraimages/00-ubuntuconsole b/scripts/extraimages/00-ubuntuconsole index 0436996d..d40aea68 100644 --- a/scripts/extraimages/00-ubuntuconsole +++ b/scripts/extraimages/00-ubuntuconsole @@ -3,6 +3,7 @@ RUN apt-get update && \ apt-get upgrade --no-install-recommends -y && \ apt-get install -y --no-install-recommends openssh-server rsync RUN rm -rf /etc/ssh/*key* +COPY scripts/dockerimages/scripts/entry.sh /usr/sbin/ COPY scripts/dockerimages/scripts/console.sh /usr/sbin/ COPY scripts/dockerimages/scripts/update-ssh-keys /usr/bin/ RUN echo 'RancherOS \\n \l' > /etc/issue @@ -12,4 +13,5 @@ RUN addgroup --gid 1100 rancher && \ useradd -u 1100 -g rancher -G docker,sudo -m -s /bin/bash rancher && \ echo '## allow password less for rancher user' >> /etc/sudoers && \ echo 'rancher ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers +ENTRYPOINT ["/usr/sbin/entry.sh"] CMD ["/usr/sbin/console.sh"] From efe55e82447b075f1774fa18fa5d7089f17412c9 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Wed, 15 Apr 2015 22:59:53 -0700 Subject: [PATCH 10/14] set hostname to rancher-dev to test DHCP --- scripts/run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run b/scripts/run index ab7caa0d..a49d8186 100755 --- a/scripts/run +++ b/scripts/run @@ -82,7 +82,7 @@ qemu-system-x86_64 -serial stdio \ -initrd ${INITRD_TEST} \ -m 1024 \ -net nic,vlan=0,model=virtio \ - -net user,vlan=0,hostfwd=tcp::2222-:22,hostname=rancher \ + -net user,vlan=0,hostfwd=tcp::2222-:22,hostname=rancher-dev \ -drive if=virtio,file=${HD} \ -machine accel=kvm \ -cpu host \ From c955c34e3bcb66b44f2235bca9588bca1dc26665 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Wed, 15 Apr 2015 23:00:04 -0700 Subject: [PATCH 11/14] bump version v0.3.0-rc3 --- scripts/version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/version b/scripts/version index 52351aae..392e846a 100644 --- a/scripts/version +++ b/scripts/version @@ -1 +1 @@ -VERSION=v0.3.0-rc2 +VERSION=v0.3.0-rc3 From 783c4639c987543df574d69ef65ab6386af0bb7d Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Wed, 15 Apr 2015 23:00:16 -0700 Subject: [PATCH 12/14] Godep updates --- Godeps/Godeps.json | 8 ++++---- .../rancher-compose/project/project.go | 19 +++++++------------ .../rancher-compose/project/types.go | 3 ++- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 9ba1627e..f368d5b8 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -221,13 +221,13 @@ }, { "ImportPath": "github.com/rancherio/rancher-compose/docker", - "Comment": "0.1.0-10-g6c6a2e3", - "Rev": "6c6a2e3ad4023fafc7509787994dc51774006f26" + "Comment": "0.1.0-12-g8fecf18", + "Rev": "8fecf186bdab6b14c9f625f9499f959fd8590482" }, { "ImportPath": "github.com/rancherio/rancher-compose/project", - "Comment": "0.1.0-10-g6c6a2e3", - "Rev": "6c6a2e3ad4023fafc7509787994dc51774006f26" + "Comment": "0.1.0-12-g8fecf18", + "Rev": "8fecf186bdab6b14c9f625f9499f959fd8590482" }, { "ImportPath": "github.com/ryanuber/go-glob", diff --git a/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/project.go b/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/project.go index 25ab41a4..00abc3b8 100644 --- a/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/project.go +++ b/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/project.go @@ -3,7 +3,6 @@ package project import ( "bytes" "errors" - "fmt" "strings" "sync" @@ -44,9 +43,8 @@ func (p *Project) createService(name string, config ServiceConfig) (Service, err continue } - value := p.EnvironmentLookup.Lookup(env, name, &config) - if value != "" { - parsedEnv = append(parsedEnv, fmt.Sprintf("%s=%s", env, value)) + for _, value := range p.EnvironmentLookup.Lookup(env, name, &config) { + parsedEnv = append(parsedEnv, value) } } @@ -65,6 +63,7 @@ func (p *Project) AddConfig(name string, config *ServiceConfig) error { p.Notify(SERVICE_ADD, service, nil) + p.reload = append(p.reload, name) p.configs[name] = config p.Services[name] = service @@ -91,22 +90,18 @@ func (p *Project) Load(bytes []byte) error { func (p *Project) Up() error { wrappers := make(map[string]*ServiceWrapper) - for name, _ := range p.Services { - wrappers[name] = NewServiceWrapper(name, p) - } - p.Notify(PROJECT_UP_START, nil, nil) return p.startAll(wrappers) } func (p *Project) startAll(wrappers map[string]*ServiceWrapper) error { - for name, _ := range p.Services { - if _, ok := wrappers[name]; !ok { - wrappers[name] = NewServiceWrapper(name, p) - } + for _, name := range p.reload { + wrappers[name] = NewServiceWrapper(name, p) } + p.reload = []string{} + restart := false for _, wrapper := range wrappers { diff --git a/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/types.go b/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/types.go index 811a60ad..0b469c16 100644 --- a/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/types.go +++ b/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/types.go @@ -58,13 +58,14 @@ type ServiceConfig struct { } type EnvironmentLookup interface { - Lookup(key, serviceName string, config *ServiceConfig) string + Lookup(key, serviceName string, config *ServiceConfig) []string } type Project struct { EnvironmentLookup EnvironmentLookup Name string configs map[string]*ServiceConfig + reload []string Services map[string]Service file string content []byte From 0a299fcd5057601389f37f5b4df028c40ec88a85 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Thu, 16 Apr 2015 07:26:42 -0700 Subject: [PATCH 13/14] Remove trailing / --- config/default.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/default.go b/config/default.go index b996f7ae..7fd2e02b 100644 --- a/config/default.go +++ b/config/default.go @@ -329,7 +329,7 @@ func NewConfig() *Config { }, Repositories: map[string]Repository{ "core": Repository{ - Url: "https://raw.githubusercontent.com/rancherio/os-services/master/", + Url: "https://raw.githubusercontent.com/rancherio/os-services/master", }, }, Services: map[string]*project.ServiceConfig{}, From ae196042ff870c522e38fea4153d60e7cf2b3489 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Thu, 16 Apr 2015 08:39:28 -0700 Subject: [PATCH 14/14] Support environment globbing for prefix only You can now put FOO* in the compose environment and it will find all environment variables that start with FOO --- docker/services.go | 43 ++++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/docker/services.go b/docker/services.go index 174bb4f9..fe8852b2 100644 --- a/docker/services.go +++ b/docker/services.go @@ -2,6 +2,7 @@ package docker import ( "fmt" + "strings" log "github.com/Sirupsen/logrus" "github.com/rancherio/os/config" @@ -13,20 +14,40 @@ type configEnvironemnt struct { cfg *config.Config } -func (c *configEnvironemnt) Lookup(key, serviceName string, serviceConfig *project.ServiceConfig) []string { - result := "" - fullKey := fmt.Sprintf("%s/%s", serviceName, key) - if value, ok := c.cfg.Environment[fullKey]; ok { - result = value - } else if value, ok := c.cfg.Environment[key]; ok { - result = value +func appendEnv(array []string, key, value string) []string { + parts := strings.SplitN(key, "/", 2) + if len(parts) == 2 { + key = parts[1] } - if result == "" { - return []string{} - } else { - return []string{fmt.Sprintf("%s=%s", key, result)} + return append(array, fmt.Sprintf("%s=%s", key, value)) +} + +func lookupKeys(cfg *config.Config, keys ...string) []string { + for _, key := range keys { + if strings.HasSuffix(key, "*") { + result := []string{} + for envKey, envValue := range cfg.Environment { + keyPrefix := key[:len(key)-1] + if strings.HasPrefix(envKey, keyPrefix) { + result = appendEnv(result, envKey, envValue) + } + } + + if len(result) > 0 { + return result + } + } else if value, ok := cfg.Environment[key]; ok { + return appendEnv([]string{}, key, value) + } } + + return []string{} +} + +func (c *configEnvironemnt) Lookup(key, serviceName string, serviceConfig *project.ServiceConfig) []string { + fullKey := fmt.Sprintf("%s/%s", serviceName, key) + return lookupKeys(c.cfg, fullKey, key) } func RunServices(name string, cfg *config.Config, configs map[string]*project.ServiceConfig) error {