diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 56102c98..f368d5b8 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -22,11 +22,6 @@ "Comment": "v1.3.2-6-g405c260", "Rev": "405c2600b19ae77516c967f8ee8ebde5624d3663" }, - { - "ImportPath": "github.com/coreos/coreos-cloudinit/initialize", - "Comment": "v1.3.2-6-g405c260", - "Rev": "405c2600b19ae77516c967f8ee8ebde5624d3663" - }, { "ImportPath": "github.com/coreos/coreos-cloudinit/network", "Comment": "v1.3.2-6-g405c260", @@ -226,13 +221,13 @@ }, { "ImportPath": "github.com/rancherio/rancher-compose/docker", - "Comment": "0.1.0-8-g45c7de3", - "Rev": "45c7de3d9b5b106475cf1461df7550bdaab0a9aa" + "Comment": "0.1.0-12-g8fecf18", + "Rev": "8fecf186bdab6b14c9f625f9499f959fd8590482" }, { "ImportPath": "github.com/rancherio/rancher-compose/project", - "Comment": "0.1.0-8-g45c7de3", - "Rev": "45c7de3d9b5b106475cf1461df7550bdaab0a9aa" + "Comment": "0.1.0-12-g8fecf18", + "Rev": "8fecf186bdab6b14c9f625f9499f959fd8590482" }, { "ImportPath": "github.com/ryanuber/go-glob", diff --git a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/config.go b/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/config.go deleted file mode 100644 index 3bf7f541..00000000 --- a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/config.go +++ /dev/null @@ -1,293 +0,0 @@ -// Copyright 2015 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package initialize - -import ( - "errors" - "fmt" - "log" - "path" - - "github.com/coreos/coreos-cloudinit/config" - "github.com/coreos/coreos-cloudinit/network" - "github.com/coreos/coreos-cloudinit/system" -) - -// CloudConfigFile represents a CoreOS specific configuration option that can generate -// an associated system.File to be written to disk -type CloudConfigFile interface { - // File should either return (*system.File, error), or (nil, nil) if nothing - // needs to be done for this configuration option. - File() (*system.File, error) -} - -// CloudConfigUnit represents a CoreOS specific configuration option that can generate -// associated system.Units to be created/enabled appropriately -type CloudConfigUnit interface { - Units() []system.Unit -} - -// Apply renders a CloudConfig to an Environment. This can involve things like -// configuring the hostname, adding new users, writing various configuration -// files to disk, and manipulating systemd services. -func Apply(cfg config.CloudConfig, ifaces []network.InterfaceGenerator, env *Environment) error { - if cfg.Hostname != "" { - if err := system.SetHostname(cfg.Hostname); err != nil { - return err - } - log.Printf("Set hostname to %s", cfg.Hostname) - } - - for _, user := range cfg.Users { - if user.Name == "" { - log.Printf("User object has no 'name' field, skipping") - continue - } - - if system.UserExists(&user) { - log.Printf("User '%s' exists, ignoring creation-time fields", user.Name) - if user.PasswordHash != "" { - log.Printf("Setting '%s' user's password", user.Name) - if err := system.SetUserPassword(user.Name, user.PasswordHash); err != nil { - log.Printf("Failed setting '%s' user's password: %v", user.Name, err) - return err - } - } - } else { - log.Printf("Creating user '%s'", user.Name) - if err := system.CreateUser(&user); err != nil { - log.Printf("Failed creating user '%s': %v", user.Name, err) - return err - } - } - - if len(user.SSHAuthorizedKeys) > 0 { - log.Printf("Authorizing %d SSH keys for user '%s'", len(user.SSHAuthorizedKeys), user.Name) - if err := system.AuthorizeSSHKeys(user.Name, env.SSHKeyName(), user.SSHAuthorizedKeys); err != nil { - return err - } - } - if user.SSHImportGithubUser != "" { - log.Printf("Authorizing github user %s SSH keys for CoreOS user '%s'", user.SSHImportGithubUser, user.Name) - if err := SSHImportGithubUser(user.Name, user.SSHImportGithubUser); err != nil { - return err - } - } - for _, u := range user.SSHImportGithubUsers { - log.Printf("Authorizing github user %s SSH keys for CoreOS user '%s'", u, user.Name) - if err := SSHImportGithubUser(user.Name, u); err != nil { - return err - } - } - if user.SSHImportURL != "" { - log.Printf("Authorizing SSH keys for CoreOS user '%s' from '%s'", user.Name, user.SSHImportURL) - if err := SSHImportKeysFromURL(user.Name, user.SSHImportURL); err != nil { - return err - } - } - } - - if len(cfg.SSHAuthorizedKeys) > 0 { - err := system.AuthorizeSSHKeys("core", env.SSHKeyName(), cfg.SSHAuthorizedKeys) - if err == nil { - log.Printf("Authorized SSH keys for core user") - } else { - return err - } - } - - var writeFiles []system.File - for _, file := range cfg.WriteFiles { - writeFiles = append(writeFiles, system.File{File: file}) - } - - for _, ccf := range []CloudConfigFile{ - system.OEM{OEM: cfg.CoreOS.OEM}, - system.Update{Update: cfg.CoreOS.Update, ReadConfig: system.DefaultReadConfig}, - system.EtcHosts{EtcHosts: cfg.ManageEtcHosts}, - system.Flannel{Flannel: cfg.CoreOS.Flannel}, - } { - f, err := ccf.File() - if err != nil { - return err - } - if f != nil { - writeFiles = append(writeFiles, *f) - } - } - - var units []system.Unit - for _, u := range cfg.CoreOS.Units { - units = append(units, system.Unit{Unit: u}) - } - - for _, ccu := range []CloudConfigUnit{ - system.Etcd{Etcd: cfg.CoreOS.Etcd}, - system.Fleet{Fleet: cfg.CoreOS.Fleet}, - system.Locksmith{Locksmith: cfg.CoreOS.Locksmith}, - system.Update{Update: cfg.CoreOS.Update, ReadConfig: system.DefaultReadConfig}, - } { - units = append(units, ccu.Units()...) - } - - wroteEnvironment := false - for _, file := range writeFiles { - fullPath, err := system.WriteFile(&file, env.Root()) - if err != nil { - return err - } - if path.Clean(file.Path) == "/etc/environment" { - wroteEnvironment = true - } - log.Printf("Wrote file %s to filesystem", fullPath) - } - - if !wroteEnvironment { - ef := env.DefaultEnvironmentFile() - if ef != nil { - err := system.WriteEnvFile(ef, env.Root()) - if err != nil { - return err - } - log.Printf("Updated /etc/environment") - } - } - - if len(ifaces) > 0 { - units = append(units, createNetworkingUnits(ifaces)...) - if err := system.RestartNetwork(ifaces); err != nil { - return err - } - } - - um := system.NewUnitManager(env.Root()) - return processUnits(units, env.Root(), um) -} - -func createNetworkingUnits(interfaces []network.InterfaceGenerator) (units []system.Unit) { - appendNewUnit := func(units []system.Unit, name, content string) []system.Unit { - if content == "" { - return units - } - return append(units, system.Unit{Unit: config.Unit{ - Name: name, - Runtime: true, - Content: content, - }}) - } - for _, i := range interfaces { - units = appendNewUnit(units, fmt.Sprintf("%s.netdev", i.Filename()), i.Netdev()) - units = appendNewUnit(units, fmt.Sprintf("%s.link", i.Filename()), i.Link()) - units = appendNewUnit(units, fmt.Sprintf("%s.network", i.Filename()), i.Network()) - } - return units -} - -// processUnits takes a set of Units and applies them to the given root using -// the given UnitManager. This can involve things like writing unit files to -// disk, masking/unmasking units, or invoking systemd -// commands against units. It returns any error encountered. -func processUnits(units []system.Unit, root string, um system.UnitManager) error { - type action struct { - unit system.Unit - command string - } - actions := make([]action, 0, len(units)) - reload := false - restartNetworkd := false - for _, unit := range units { - if unit.Name == "" { - log.Printf("Skipping unit without name") - continue - } - - if unit.Content != "" { - log.Printf("Writing unit %q to filesystem", unit.Name) - if err := um.PlaceUnit(unit); err != nil { - return err - } - log.Printf("Wrote unit %q", unit.Name) - reload = true - } - - for _, dropin := range unit.DropIns { - if dropin.Name != "" && dropin.Content != "" { - log.Printf("Writing drop-in unit %q to filesystem", dropin.Name) - if err := um.PlaceUnitDropIn(unit, dropin); err != nil { - return err - } - log.Printf("Wrote drop-in unit %q", dropin.Name) - reload = true - } - } - - if unit.Mask { - log.Printf("Masking unit file %q", unit.Name) - if err := um.MaskUnit(unit); err != nil { - return err - } - } else if unit.Runtime { - log.Printf("Ensuring runtime unit file %q is unmasked", unit.Name) - if err := um.UnmaskUnit(unit); err != nil { - return err - } - } - - if unit.Enable { - if unit.Group() != "network" { - log.Printf("Enabling unit file %q", unit.Name) - if err := um.EnableUnitFile(unit); err != nil { - return err - } - log.Printf("Enabled unit %q", unit.Name) - } else { - log.Printf("Skipping enable for network-like unit %q", unit.Name) - } - } - - if unit.Group() == "network" { - restartNetworkd = true - } else if unit.Command != "" { - actions = append(actions, action{unit, unit.Command}) - } - } - - if reload { - if err := um.DaemonReload(); err != nil { - return errors.New(fmt.Sprintf("failed systemd daemon-reload: %s", err)) - } - } - - if restartNetworkd { - log.Printf("Restarting systemd-networkd") - networkd := system.Unit{Unit: config.Unit{Name: "systemd-networkd.service"}} - res, err := um.RunUnitCommand(networkd, "restart") - if err != nil { - return err - } - log.Printf("Restarted systemd-networkd (%s)", res) - } - - for _, action := range actions { - log.Printf("Calling unit command %q on %q'", action.command, action.unit.Name) - res, err := um.RunUnitCommand(action.unit, action.command) - if err != nil { - return err - } - log.Printf("Result of %q on %q: %s", action.command, action.unit.Name, res) - } - - return nil -} diff --git a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/config_test.go b/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/config_test.go deleted file mode 100644 index 33be737e..00000000 --- a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/config_test.go +++ /dev/null @@ -1,299 +0,0 @@ -// Copyright 2015 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package initialize - -import ( - "reflect" - "testing" - - "github.com/coreos/coreos-cloudinit/config" - "github.com/coreos/coreos-cloudinit/network" - "github.com/coreos/coreos-cloudinit/system" -) - -type TestUnitManager struct { - placed []string - enabled []string - masked []string - unmasked []string - commands []UnitAction - reload bool -} - -type UnitAction struct { - unit string - command string -} - -func (tum *TestUnitManager) PlaceUnit(u system.Unit) error { - tum.placed = append(tum.placed, u.Name) - return nil -} -func (tum *TestUnitManager) PlaceUnitDropIn(u system.Unit, d config.UnitDropIn) error { - tum.placed = append(tum.placed, u.Name+".d/"+d.Name) - return nil -} -func (tum *TestUnitManager) EnableUnitFile(u system.Unit) error { - tum.enabled = append(tum.enabled, u.Name) - return nil -} -func (tum *TestUnitManager) RunUnitCommand(u system.Unit, c string) (string, error) { - tum.commands = append(tum.commands, UnitAction{u.Name, c}) - return "", nil -} -func (tum *TestUnitManager) DaemonReload() error { - tum.reload = true - return nil -} -func (tum *TestUnitManager) MaskUnit(u system.Unit) error { - tum.masked = append(tum.masked, u.Name) - return nil -} -func (tum *TestUnitManager) UnmaskUnit(u system.Unit) error { - tum.unmasked = append(tum.unmasked, u.Name) - return nil -} - -type mockInterface struct { - name string - filename string - netdev string - link string - network string - kind string - modprobeParams string -} - -func (i mockInterface) Name() string { - return i.name -} - -func (i mockInterface) Filename() string { - return i.filename -} - -func (i mockInterface) Netdev() string { - return i.netdev -} - -func (i mockInterface) Link() string { - return i.link -} - -func (i mockInterface) Network() string { - return i.network -} - -func (i mockInterface) Type() string { - return i.kind -} - -func (i mockInterface) ModprobeParams() string { - return i.modprobeParams -} - -func TestCreateNetworkingUnits(t *testing.T) { - for _, tt := range []struct { - interfaces []network.InterfaceGenerator - expect []system.Unit - }{ - {nil, nil}, - { - []network.InterfaceGenerator{ - network.InterfaceGenerator(mockInterface{filename: "test"}), - }, - nil, - }, - { - []network.InterfaceGenerator{ - network.InterfaceGenerator(mockInterface{filename: "test1", netdev: "test netdev"}), - network.InterfaceGenerator(mockInterface{filename: "test2", link: "test link"}), - network.InterfaceGenerator(mockInterface{filename: "test3", network: "test network"}), - }, - []system.Unit{ - system.Unit{Unit: config.Unit{Name: "test1.netdev", Runtime: true, Content: "test netdev"}}, - system.Unit{Unit: config.Unit{Name: "test2.link", Runtime: true, Content: "test link"}}, - system.Unit{Unit: config.Unit{Name: "test3.network", Runtime: true, Content: "test network"}}, - }, - }, - { - []network.InterfaceGenerator{ - network.InterfaceGenerator(mockInterface{filename: "test", netdev: "test netdev", link: "test link", network: "test network"}), - }, - []system.Unit{ - system.Unit{Unit: config.Unit{Name: "test.netdev", Runtime: true, Content: "test netdev"}}, - system.Unit{Unit: config.Unit{Name: "test.link", Runtime: true, Content: "test link"}}, - system.Unit{Unit: config.Unit{Name: "test.network", Runtime: true, Content: "test network"}}, - }, - }, - } { - units := createNetworkingUnits(tt.interfaces) - if !reflect.DeepEqual(tt.expect, units) { - t.Errorf("bad units (%+v): want %#v, got %#v", tt.interfaces, tt.expect, units) - } - } -} - -func TestProcessUnits(t *testing.T) { - tests := []struct { - units []system.Unit - - result TestUnitManager - }{ - { - units: []system.Unit{ - system.Unit{Unit: config.Unit{ - Name: "foo", - Mask: true, - }}, - }, - result: TestUnitManager{ - masked: []string{"foo"}, - }, - }, - { - units: []system.Unit{ - system.Unit{Unit: config.Unit{ - Name: "baz.service", - Content: "[Service]\nExecStart=/bin/baz", - Command: "start", - }}, - system.Unit{Unit: config.Unit{ - Name: "foo.network", - Content: "[Network]\nFoo=true", - }}, - system.Unit{Unit: config.Unit{ - Name: "bar.network", - Content: "[Network]\nBar=true", - }}, - }, - result: TestUnitManager{ - placed: []string{"baz.service", "foo.network", "bar.network"}, - commands: []UnitAction{ - UnitAction{"systemd-networkd.service", "restart"}, - UnitAction{"baz.service", "start"}, - }, - reload: true, - }, - }, - { - units: []system.Unit{ - system.Unit{Unit: config.Unit{ - Name: "baz.service", - Content: "[Service]\nExecStart=/bin/true", - }}, - }, - result: TestUnitManager{ - placed: []string{"baz.service"}, - reload: true, - }, - }, - { - units: []system.Unit{ - system.Unit{Unit: config.Unit{ - Name: "locksmithd.service", - Runtime: true, - }}, - }, - result: TestUnitManager{ - unmasked: []string{"locksmithd.service"}, - }, - }, - { - units: []system.Unit{ - system.Unit{Unit: config.Unit{ - Name: "woof", - Enable: true, - }}, - }, - result: TestUnitManager{ - enabled: []string{"woof"}, - }, - }, - { - units: []system.Unit{ - system.Unit{Unit: config.Unit{ - Name: "hi.service", - Runtime: true, - Content: "[Service]\nExecStart=/bin/echo hi", - DropIns: []config.UnitDropIn{ - { - Name: "lo.conf", - Content: "[Service]\nExecStart=/bin/echo lo", - }, - { - Name: "bye.conf", - Content: "[Service]\nExecStart=/bin/echo bye", - }, - }, - }}, - }, - result: TestUnitManager{ - placed: []string{"hi.service", "hi.service.d/lo.conf", "hi.service.d/bye.conf"}, - unmasked: []string{"hi.service"}, - reload: true, - }, - }, - { - units: []system.Unit{ - system.Unit{Unit: config.Unit{ - DropIns: []config.UnitDropIn{ - { - Name: "lo.conf", - Content: "[Service]\nExecStart=/bin/echo lo", - }, - }, - }}, - }, - result: TestUnitManager{}, - }, - { - units: []system.Unit{ - system.Unit{Unit: config.Unit{ - Name: "hi.service", - DropIns: []config.UnitDropIn{ - { - Content: "[Service]\nExecStart=/bin/echo lo", - }, - }, - }}, - }, - result: TestUnitManager{}, - }, - { - units: []system.Unit{ - system.Unit{Unit: config.Unit{ - Name: "hi.service", - DropIns: []config.UnitDropIn{ - { - Name: "lo.conf", - }, - }, - }}, - }, - result: TestUnitManager{}, - }, - } - - for _, tt := range tests { - tum := &TestUnitManager{} - if err := processUnits(tt.units, "", tum); err != nil { - t.Errorf("bad error (%+v): want nil, got %s", tt.units, err) - } - if !reflect.DeepEqual(tt.result, *tum) { - t.Errorf("bad result (%+v): want %+v, got %+v", tt.units, tt.result, tum) - } - } -} diff --git a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/env.go b/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/env.go deleted file mode 100644 index f90cc932..00000000 --- a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/env.go +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2015 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package initialize - -import ( - "net" - "os" - "path" - "regexp" - "strings" - - "github.com/coreos/coreos-cloudinit/config" - "github.com/coreos/coreos-cloudinit/datasource" - "github.com/coreos/coreos-cloudinit/system" -) - -const DefaultSSHKeyName = "coreos-cloudinit" - -type Environment struct { - root string - configRoot string - workspace string - sshKeyName string - substitutions map[string]string -} - -// TODO(jonboulle): this is getting unwieldy, should be able to simplify the interface somehow -func NewEnvironment(root, configRoot, workspace, sshKeyName string, metadata datasource.Metadata) *Environment { - firstNonNull := func(ip net.IP, env string) string { - if ip == nil { - return env - } - return ip.String() - } - substitutions := map[string]string{ - "$public_ipv4": firstNonNull(metadata.PublicIPv4, os.Getenv("COREOS_PUBLIC_IPV4")), - "$private_ipv4": firstNonNull(metadata.PrivateIPv4, os.Getenv("COREOS_PRIVATE_IPV4")), - "$public_ipv6": firstNonNull(metadata.PublicIPv6, os.Getenv("COREOS_PUBLIC_IPV6")), - "$private_ipv6": firstNonNull(metadata.PrivateIPv6, os.Getenv("COREOS_PRIVATE_IPV6")), - } - return &Environment{root, configRoot, workspace, sshKeyName, substitutions} -} - -func (e *Environment) Workspace() string { - return path.Join(e.root, e.workspace) -} - -func (e *Environment) Root() string { - return e.root -} - -func (e *Environment) ConfigRoot() string { - return e.configRoot -} - -func (e *Environment) SSHKeyName() string { - return e.sshKeyName -} - -func (e *Environment) SetSSHKeyName(name string) { - e.sshKeyName = name -} - -// Apply goes through the map of substitutions and replaces all instances of -// the keys with their respective values. It supports escaping substitutions -// with a leading '\'. -func (e *Environment) Apply(data string) string { - for key, val := range e.substitutions { - matchKey := strings.Replace(key, `$`, `\$`, -1) - replKey := strings.Replace(key, `$`, `$$`, -1) - - // "key" -> "val" - data = regexp.MustCompile(`([^\\]|^)`+matchKey).ReplaceAllString(data, `${1}`+val) - // "\key" -> "key" - data = regexp.MustCompile(`\\`+matchKey).ReplaceAllString(data, replKey) - } - return data -} - -func (e *Environment) DefaultEnvironmentFile() *system.EnvFile { - ef := system.EnvFile{ - File: &system.File{File: config.File{ - Path: "/etc/environment", - }}, - Vars: map[string]string{}, - } - if ip, ok := e.substitutions["$public_ipv4"]; ok && len(ip) > 0 { - ef.Vars["COREOS_PUBLIC_IPV4"] = ip - } - if ip, ok := e.substitutions["$private_ipv4"]; ok && len(ip) > 0 { - ef.Vars["COREOS_PRIVATE_IPV4"] = ip - } - if ip, ok := e.substitutions["$public_ipv6"]; ok && len(ip) > 0 { - ef.Vars["COREOS_PUBLIC_IPV6"] = ip - } - if ip, ok := e.substitutions["$private_ipv6"]; ok && len(ip) > 0 { - ef.Vars["COREOS_PRIVATE_IPV6"] = ip - } - if len(ef.Vars) == 0 { - return nil - } else { - return &ef - } -} diff --git a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/env_test.go b/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/env_test.go deleted file mode 100644 index abb770cf..00000000 --- a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/env_test.go +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright 2015 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package initialize - -import ( - "io/ioutil" - "net" - "os" - "path" - "testing" - - "github.com/coreos/coreos-cloudinit/datasource" - "github.com/coreos/coreos-cloudinit/system" -) - -func TestEnvironmentApply(t *testing.T) { - os.Setenv("COREOS_PUBLIC_IPV4", "1.2.3.4") - os.Setenv("COREOS_PRIVATE_IPV4", "5.6.7.8") - os.Setenv("COREOS_PUBLIC_IPV6", "1234::") - os.Setenv("COREOS_PRIVATE_IPV6", "5678::") - for _, tt := range []struct { - metadata datasource.Metadata - input string - out string - }{ - { - // Substituting both values directly should always take precedence - // over environment variables - datasource.Metadata{ - PublicIPv4: net.ParseIP("192.0.2.3"), - PrivateIPv4: net.ParseIP("192.0.2.203"), - PublicIPv6: net.ParseIP("fe00:1234::"), - PrivateIPv6: net.ParseIP("fe00:5678::"), - }, - `[Service] -ExecStart=/usr/bin/echo "$public_ipv4 $public_ipv6" -ExecStop=/usr/bin/echo $private_ipv4 $private_ipv6 -ExecStop=/usr/bin/echo $unknown`, - `[Service] -ExecStart=/usr/bin/echo "192.0.2.3 fe00:1234::" -ExecStop=/usr/bin/echo 192.0.2.203 fe00:5678:: -ExecStop=/usr/bin/echo $unknown`, - }, - { - // Substituting one value directly while falling back with the other - datasource.Metadata{ - PrivateIPv4: net.ParseIP("127.0.0.1"), - }, - "$private_ipv4\n$public_ipv4", - "127.0.0.1\n1.2.3.4", - }, - { - // Falling back to environment variables for both values - datasource.Metadata{}, - "$private_ipv4\n$public_ipv4", - "5.6.7.8\n1.2.3.4", - }, - { - // No substitutions - datasource.Metadata{}, - "$private_ipv4\nfoobar", - "5.6.7.8\nfoobar", - }, - { - // Escaping substitutions - datasource.Metadata{ - PrivateIPv4: net.ParseIP("127.0.0.1"), - }, - `\$private_ipv4 -$private_ipv4 -addr: \$private_ipv4 -\\$private_ipv4`, - `$private_ipv4 -127.0.0.1 -addr: $private_ipv4 -\$private_ipv4`, - }, - { - // No substitutions with escaping - datasource.Metadata{}, - "\\$test\n$test", - "\\$test\n$test", - }, - } { - - env := NewEnvironment("./", "./", "./", "", tt.metadata) - got := env.Apply(tt.input) - if got != tt.out { - t.Fatalf("Environment incorrectly applied.\ngot:\n%s\nwant:\n%s", got, tt.out) - } - } -} - -func TestEnvironmentFile(t *testing.T) { - metadata := datasource.Metadata{ - PublicIPv4: net.ParseIP("1.2.3.4"), - PrivateIPv4: net.ParseIP("5.6.7.8"), - PublicIPv6: net.ParseIP("1234::"), - PrivateIPv6: net.ParseIP("5678::"), - } - expect := "COREOS_PRIVATE_IPV4=5.6.7.8\nCOREOS_PRIVATE_IPV6=5678::\nCOREOS_PUBLIC_IPV4=1.2.3.4\nCOREOS_PUBLIC_IPV6=1234::\n" - - dir, err := ioutil.TempDir(os.TempDir(), "coreos-cloudinit-") - if err != nil { - t.Fatalf("Unable to create tempdir: %v", err) - } - defer os.RemoveAll(dir) - - env := NewEnvironment("./", "./", "./", "", metadata) - ef := env.DefaultEnvironmentFile() - err = system.WriteEnvFile(ef, dir) - if err != nil { - t.Fatalf("WriteEnvFile failed: %v", err) - } - - fullPath := path.Join(dir, "etc", "environment") - contents, err := ioutil.ReadFile(fullPath) - if err != nil { - t.Fatalf("Unable to read expected file: %v", err) - } - - if string(contents) != expect { - t.Fatalf("File has incorrect contents: %q", contents) - } -} - -func TestEnvironmentFileNil(t *testing.T) { - os.Clearenv() - metadata := datasource.Metadata{} - - env := NewEnvironment("./", "./", "./", "", metadata) - ef := env.DefaultEnvironmentFile() - if ef != nil { - t.Fatalf("Environment file not nil: %v", ef) - } -} diff --git a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/github.go b/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/github.go deleted file mode 100644 index 2f7755fe..00000000 --- a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/github.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2015 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package initialize - -import ( - "fmt" - - "github.com/coreos/coreos-cloudinit/system" -) - -func SSHImportGithubUser(system_user string, github_user string) error { - url := fmt.Sprintf("https://api.github.com/users/%s/keys", github_user) - keys, err := fetchUserKeys(url) - if err != nil { - return err - } - - key_name := fmt.Sprintf("github-%s", github_user) - return system.AuthorizeSSHKeys(system_user, key_name, keys) -} diff --git a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/ssh_keys.go b/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/ssh_keys.go deleted file mode 100644 index 17b0c4a9..00000000 --- a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/ssh_keys.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2015 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package initialize - -import ( - "encoding/json" - "fmt" - - "github.com/coreos/coreos-cloudinit/pkg" - "github.com/coreos/coreos-cloudinit/system" -) - -type UserKey struct { - ID int `json:"id,omitempty"` - Key string `json:"key"` -} - -func SSHImportKeysFromURL(system_user string, url string) error { - keys, err := fetchUserKeys(url) - if err != nil { - return err - } - - key_name := fmt.Sprintf("coreos-cloudinit-%s", system_user) - return system.AuthorizeSSHKeys(system_user, key_name, keys) -} - -func fetchUserKeys(url string) ([]string, error) { - client := pkg.NewHttpClient() - data, err := client.GetRetry(url) - if err != nil { - return nil, err - } - - var userKeys []UserKey - err = json.Unmarshal(data, &userKeys) - if err != nil { - return nil, err - } - keys := make([]string, 0) - for _, key := range userKeys { - keys = append(keys, key.Key) - } - return keys, err -} diff --git a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/ssh_keys_test.go b/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/ssh_keys_test.go deleted file mode 100644 index 86395797..00000000 --- a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/ssh_keys_test.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2015 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package initialize - -import ( - "fmt" - "net/http" - "net/http/httptest" - "testing" -) - -func TestCloudConfigUsersUrlMarshal(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - gh_res := ` -[ - { - "key": "ssh-dss AAAAB3NzaC1kc3MAAACBAIHAu822ggSkIHrJYvhmBceOSVjuflfQm8RbMMDNVe9relQfuPbN+nxGGTCKzPLebeOcX+Wwi77TPXWwK3BZMglfXxhABlFPsuMb63Tqp94pBYsJdx/iFj9iGo6pKoM1k8ubOcqsUnq+BR9895zRbE7MjdwkGo67+QhCEwvkwAnNAAAAFQCuddVqXLCubzqnWmeHLQE+2GFfHwAAAIBnlXW5h15ndVuwi0htF4oodVSB1KwnTWcuBK+aE1zRs76yvRb0Ws+oifumThDwB/Tec6FQuAfRKfy6piChZqsu5KvL98I+2t5yyi1td+kMvdTnVL2lW44etDKseOcozmknCOmh4Dqvhl/2MwrDAhlPaN08EEq9h3w3mXtNLWH64QAAAIBAzDOKr17llngaKIdDXh+LtXKh87+zfjlTA36/9r2uF2kYE5uApDtu9sPCkt7+YBQt7R8prADPckwAiXwVdk0xijIOpLDBmoydQJJRQ+zTMxvpQmUr/1kUOv0zb+lB657CgvN0vVTmP2swPeMvgntt3C4vw7Ab+O+MS9peOAJbbQ==" - }, - { - "key": "ssh-dss AAAAB3NzaC1kc3MAAACBANxpzIbTzKTeBRaOIdUxwwGwvDasTfU/PonhbNIuhYjc+xFGvBRTumox2F+luVAKKs4WdvA4nJXaY1OFi6DZftk5Bp4E2JaSzp8ulAzHsMexDdv6LGHGEJj/qdHAL1vHk2K89PpwRFSRZI8XRBLjvkr4ZgBKLG5ZILXPJEPP2j3lAAAAFQCtxoTnV8wy0c4grcGrQ+1sCsD7WQAAAIAqZsW2GviMe1RQrbZT0xAZmI64XRPrnLsoLxycHWlS7r6uUln2c6Ae2MB/YF0d4Kd1XZii9GHj7rrypqEo7MW8uSabhu70nmu1J8m2O3Dsr+4oJLeat9vwPsJV92IKO0jQwjKnAOHOiB9JKGeCw+NfXfogbti9/q38Q6XcS+SI5wAAAIEA1803Y2h+tOOpZXAsNIwl9mRfExWzLQ3L7knwJdznQu/6SW1H/1oyoYLebuk187Qj2UFI5qQ6AZNc49DvohWx0Cg6ABcyubNyoaCjZKWIdxVnItHWNbLe//+tyTu0I2eQwJOORsEPK5gMpf599C7wXQ//DzZOWbTWiHEX52gCTmk=" - }, - { - "id": 5224438, - "key": "ssh-dss AAAAB3NzaC1kc3MAAACBAPKRWdKhzGZuLAJL6M1eM51hWViMqNBC2C6lm2OqGRYLuIf1GJ391widUuSf4wQqnkR22Q9PCmAZ19XCf11wBRMnuw9I/Z3Bt5bXfc+dzFBCmHYGJ6wNSv++H9jxyMb+usmsenWOFZGNO2jN0wrJ4ay8Yt0bwtRU+VCXpuRLszMzAAAAFQDZUIuPjcfK5HLgnwZ/J3lvtvlUjQAAAIEApIkAwLuCQV5j3U6DmI/Y6oELqSUR2purFm8jo8jePFfe1t+ghikgD254/JXlhDCVgY0NLXcak+coJfGCTT23quJ7I5xdpTn/OZO2Q6Woum/bijFC/UWwQbLz0R2nU3DoHv5v6XHQZxuIG4Fsxa91S+vWjZFtI7RuYlBCZA//ANMAAACBAJO0FojzkX6IeaWLqrgu9GTkFwGFazZ+LPH5JOWPoPn1hQKuR32Uf6qNcBZcIjY7SF0P7HF5rLQd6zKZzHqqQQ92MV555NEwjsnJglYU8CaaZsfYooaGPgA1YN7RhTSAuDmUW5Hyfj5BH4NTtrzrvJxIhDoQLf31Fasjw00r4R0O" - } -] -` - fmt.Fprintln(w, gh_res) - })) - defer ts.Close() - - keys, err := fetchUserKeys(ts.URL) - if err != nil { - t.Fatalf("Encountered unexpected error: %v", err) - } - expected := "ssh-dss AAAAB3NzaC1kc3MAAACBAIHAu822ggSkIHrJYvhmBceOSVjuflfQm8RbMMDNVe9relQfuPbN+nxGGTCKzPLebeOcX+Wwi77TPXWwK3BZMglfXxhABlFPsuMb63Tqp94pBYsJdx/iFj9iGo6pKoM1k8ubOcqsUnq+BR9895zRbE7MjdwkGo67+QhCEwvkwAnNAAAAFQCuddVqXLCubzqnWmeHLQE+2GFfHwAAAIBnlXW5h15ndVuwi0htF4oodVSB1KwnTWcuBK+aE1zRs76yvRb0Ws+oifumThDwB/Tec6FQuAfRKfy6piChZqsu5KvL98I+2t5yyi1td+kMvdTnVL2lW44etDKseOcozmknCOmh4Dqvhl/2MwrDAhlPaN08EEq9h3w3mXtNLWH64QAAAIBAzDOKr17llngaKIdDXh+LtXKh87+zfjlTA36/9r2uF2kYE5uApDtu9sPCkt7+YBQt7R8prADPckwAiXwVdk0xijIOpLDBmoydQJJRQ+zTMxvpQmUr/1kUOv0zb+lB657CgvN0vVTmP2swPeMvgntt3C4vw7Ab+O+MS9peOAJbbQ==" - if keys[0] != expected { - t.Fatalf("expected %s, got %s", expected, keys[0]) - } - expected = "ssh-dss AAAAB3NzaC1kc3MAAACBAPKRWdKhzGZuLAJL6M1eM51hWViMqNBC2C6lm2OqGRYLuIf1GJ391widUuSf4wQqnkR22Q9PCmAZ19XCf11wBRMnuw9I/Z3Bt5bXfc+dzFBCmHYGJ6wNSv++H9jxyMb+usmsenWOFZGNO2jN0wrJ4ay8Yt0bwtRU+VCXpuRLszMzAAAAFQDZUIuPjcfK5HLgnwZ/J3lvtvlUjQAAAIEApIkAwLuCQV5j3U6DmI/Y6oELqSUR2purFm8jo8jePFfe1t+ghikgD254/JXlhDCVgY0NLXcak+coJfGCTT23quJ7I5xdpTn/OZO2Q6Woum/bijFC/UWwQbLz0R2nU3DoHv5v6XHQZxuIG4Fsxa91S+vWjZFtI7RuYlBCZA//ANMAAACBAJO0FojzkX6IeaWLqrgu9GTkFwGFazZ+LPH5JOWPoPn1hQKuR32Uf6qNcBZcIjY7SF0P7HF5rLQd6zKZzHqqQQ92MV555NEwjsnJglYU8CaaZsfYooaGPgA1YN7RhTSAuDmUW5Hyfj5BH4NTtrzrvJxIhDoQLf31Fasjw00r4R0O" - if keys[2] != expected { - t.Fatalf("expected %s, got %s", expected, keys[2]) - } -} diff --git a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/user_data.go b/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/user_data.go deleted file mode 100644 index 170efaaa..00000000 --- a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/user_data.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2015 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package initialize - -import ( - "errors" - "log" - - "github.com/coreos/coreos-cloudinit/config" -) - -func ParseUserData(contents string) (interface{}, error) { - if len(contents) == 0 { - return nil, nil - } - - switch { - case config.IsScript(contents): - log.Printf("Parsing user-data as script") - return config.NewScript(contents) - case config.IsCloudConfig(contents): - log.Printf("Parsing user-data as cloud-config") - return config.NewCloudConfig(contents) - default: - return nil, errors.New("Unrecognized user-data format") - } -} diff --git a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/user_data_test.go b/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/user_data_test.go deleted file mode 100644 index 1d883694..00000000 --- a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/user_data_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2015 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package initialize - -import ( - "testing" - - "github.com/coreos/coreos-cloudinit/config" -) - -func TestParseHeaderCRLF(t *testing.T) { - configs := []string{ - "#cloud-config\nfoo: bar", - "#cloud-config\r\nfoo: bar", - } - - for i, config := range configs { - _, err := ParseUserData(config) - if err != nil { - t.Errorf("Failed parsing config %d: %v", i, err) - } - } - - scripts := []string{ - "#!bin/bash\necho foo", - "#!bin/bash\r\necho foo", - } - - for i, script := range scripts { - _, err := ParseUserData(script) - if err != nil { - t.Errorf("Failed parsing script %d: %v", i, err) - } - } -} - -func TestParseConfigCRLF(t *testing.T) { - contents := "#cloud-config\r\nhostname: foo\r\nssh_authorized_keys:\r\n - foobar\r\n" - ud, err := ParseUserData(contents) - if err != nil { - t.Fatalf("Failed parsing config: %v", err) - } - - cfg := ud.(*config.CloudConfig) - - if cfg.Hostname != "foo" { - t.Error("Failed parsing hostname from config") - } - - if len(cfg.SSHAuthorizedKeys) != 1 { - t.Error("Parsed incorrect number of SSH keys") - } -} - -func TestParseConfigEmpty(t *testing.T) { - i, e := ParseUserData(``) - if i != nil { - t.Error("ParseUserData of empty string returned non-nil unexpectedly") - } else if e != nil { - t.Error("ParseUserData of empty string returned error unexpectedly") - } -} diff --git a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/workspace.go b/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/workspace.go deleted file mode 100644 index 540dcf41..00000000 --- a/Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/workspace.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2015 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package initialize - -import ( - "io/ioutil" - "path" - "strings" - - "github.com/coreos/coreos-cloudinit/config" - "github.com/coreos/coreos-cloudinit/system" -) - -func PrepWorkspace(workspace string) error { - if err := system.EnsureDirectoryExists(workspace); err != nil { - return err - } - - scripts := path.Join(workspace, "scripts") - if err := system.EnsureDirectoryExists(scripts); err != nil { - return err - } - - return nil -} - -func PersistScriptInWorkspace(script config.Script, workspace string) (string, error) { - scriptsPath := path.Join(workspace, "scripts") - tmp, err := ioutil.TempFile(scriptsPath, "") - if err != nil { - return "", err - } - tmp.Close() - - relpath := strings.TrimPrefix(tmp.Name(), workspace) - - file := system.File{File: config.File{ - Path: relpath, - RawFilePermissions: "0744", - Content: string(script), - }} - - return system.WriteFile(&file, workspace) -} - -func PersistUnitNameInWorkspace(name string, workspace string) error { - file := system.File{File: config.File{ - Path: path.Join("scripts", "unit-name"), - RawFilePermissions: "0644", - Content: name, - }} - _, err := system.WriteFile(&file, workspace) - return err -} diff --git a/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/project.go b/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/project.go index 686e6caa..00abc3b8 100644 --- a/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/project.go +++ b/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/project.go @@ -18,6 +18,12 @@ var ( ErrRestart error = errors.New("Restart execution") ) +type ProjectEvent struct { + Event Event + Service Service + Data map[string]string +} + func NewProject(name string, factory ServiceFactory) *Project { return &Project{ Name: name, @@ -27,8 +33,29 @@ func NewProject(name string, factory ServiceFactory) *Project { } } +func (p *Project) createService(name string, config ServiceConfig) (Service, error) { + if p.EnvironmentLookup != nil { + parsedEnv := make([]string, 0, len(config.Environment)) + + for _, env := range config.Environment { + if strings.IndexRune(env, '=') != -1 { + parsedEnv = append(parsedEnv, env) + continue + } + + for _, value := range p.EnvironmentLookup.Lookup(env, name, &config) { + parsedEnv = append(parsedEnv, value) + } + } + + config.Environment = parsedEnv + } + + return p.factory.Create(p, name, &config) +} + func (p *Project) AddConfig(name string, config *ServiceConfig) error { - service, err := p.factory.Create(p, name, config) + service, err := p.createService(name, *config) if err != nil { log.Errorf("Failed to create service for %s : %v", name, err) return err @@ -36,6 +63,7 @@ func (p *Project) AddConfig(name string, config *ServiceConfig) error { p.Notify(SERVICE_ADD, service, nil) + p.reload = append(p.reload, name) p.configs[name] = config p.Services[name] = service @@ -62,16 +90,18 @@ func (p *Project) Load(bytes []byte) error { func (p *Project) Up() error { wrappers := make(map[string]*ServiceWrapper) - for name, _ := range p.Services { - wrappers[name] = NewServiceWrapper(name, p) - } - p.Notify(PROJECT_UP_START, nil, nil) return p.startAll(wrappers) } func (p *Project) startAll(wrappers map[string]*ServiceWrapper) error { + for _, name := range p.reload { + wrappers[name] = NewServiceWrapper(name, p) + } + + p.reload = []string{} + restart := false for _, wrapper := range wrappers { @@ -176,6 +206,10 @@ func (s *ServiceWrapper) Wait() error { return s.err } +func (p *Project) AddListener(c chan<- ProjectEvent) { + p.listeners = append(p.listeners, c) +} + func (p *Project) Notify(event Event, service Service, data map[string]string) { buffer := bytes.NewBuffer(nil) if data != nil { @@ -204,4 +238,17 @@ func (p *Project) Notify(event Event, service Service, data map[string]string) { } else { logf("[%d/%d] [%s]: %s %s", p.upCount, len(p.Services), service.Name(), event, buffer.Bytes()) } + + for _, l := range p.listeners { + projectEvent := ProjectEvent{ + Event: event, + Service: service, + Data: data, + } + // Don't ever block + select { + case l <- projectEvent: + default: + } + } } diff --git a/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/types.go b/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/types.go index 938d1fcd..0b469c16 100644 --- a/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/types.go +++ b/Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/types.go @@ -7,8 +7,9 @@ type Event string const ( CONTAINER_ID = "container_id" - CONTAINER_CREATED = Event("Created container") - CONTAINER_STARTED = Event("Started container") + CONTAINER_STARTING = Event("Starting container") + CONTAINER_CREATED = Event("Created container") + CONTAINER_STARTED = Event("Started container") SERVICE_ADD = Event("Adding") SERVICE_UP_START = Event("Starting") @@ -56,16 +57,23 @@ type ServiceConfig struct { ExternalLinks []string `yaml:"external_links,omitempty"` } +type EnvironmentLookup interface { + Lookup(key, serviceName string, config *ServiceConfig) []string +} + type Project struct { - Name string - configs map[string]*ServiceConfig - Services map[string]Service - file string - content []byte - client *client.RancherClient - factory ServiceFactory - ReloadCallback func() error - upCount int + EnvironmentLookup EnvironmentLookup + Name string + configs map[string]*ServiceConfig + reload []string + Services map[string]Service + file string + content []byte + client *client.RancherClient + factory ServiceFactory + ReloadCallback func() error + upCount int + listeners []chan<- ProjectEvent } type Service interface { diff --git a/cmd/cloudinit/authorizeSSHKeys.go b/cmd/cloudinit/authorize_ssh_keys.go similarity index 100% rename from cmd/cloudinit/authorizeSSHKeys.go rename to cmd/cloudinit/authorize_ssh_keys.go diff --git a/cmd/cloudinit/cloudinit.go b/cmd/cloudinit/cloudinit.go index 2100a376..b1f93c19 100644 --- a/cmd/cloudinit/cloudinit.go +++ b/cmd/cloudinit/cloudinit.go @@ -33,11 +33,11 @@ import ( "github.com/coreos/coreos-cloudinit/datasource/metadata/ec2" "github.com/coreos/coreos-cloudinit/datasource/proc_cmdline" "github.com/coreos/coreos-cloudinit/datasource/url" - "github.com/coreos/coreos-cloudinit/initialize" "github.com/coreos/coreos-cloudinit/pkg" "github.com/coreos/coreos-cloudinit/system" rancherNetwork "github.com/rancherio/os/cmd/network" rancherConfig "github.com/rancherio/os/config" + "github.com/rancherio/os/util" "gopkg.in/yaml.v2" ) @@ -48,21 +48,25 @@ const ( ) var ( - outputDir string - outputFile string - scriptFile string - rancherYml string - save bool - execute bool - network bool - sshKeyName string - flags *flag.FlagSet + baseConfigDir string + outputDir string + outputFile string + metaDataFile string + scriptFile string + rancherYml string + save bool + execute bool + network bool + sshKeyName string + flags *flag.FlagSet ) func init() { flags = flag.NewFlagSet(os.Args[0], flag.ContinueOnError) + flags.StringVar(&baseConfigDir, "base-config-dir", "/var/lib/rancher/conf/cloud-config.d", "base cloud config") flags.StringVar(&outputDir, "dir", "/var/lib/rancher/conf", "working directory") flags.StringVar(&outputFile, "file", "cloud-config-processed.yml", "output cloud config file name") + flags.StringVar(&metaDataFile, "metadata", "metadata", "output metdata file name") flags.StringVar(&scriptFile, "script-file", "cloud-config-script", "output cloud config script file name") flags.StringVar(&rancherYml, "rancher", "cloud-config-rancher.yml", "output cloud config rancher file name") flags.StringVar(&sshKeyName, "ssh-key-name", "rancheros-cloud-config", "SSH key name") @@ -71,61 +75,61 @@ func init() { flags.BoolVar(&execute, "execute", false, "execute saved cloud config") } -func saveFiles(script *config.Script, userdataBytes []byte, cc *config.CloudConfig) error { - var fileData []byte +func saveFiles(cloudConfigBytes, scriptBytes []byte, metadata datasource.Metadata) error { + scriptOutput := path.Join(outputDir, scriptFile) + cloudConfigOutput := path.Join(outputDir, outputFile) + rancherYmlOutput := path.Join(outputDir, rancherYml) + metaDataOutput := path.Join(outputDir, metaDataFile) - os.Remove(path.Join(outputDir, scriptFile)) - os.Remove(path.Join(outputDir, outputFile)) - os.Remove(path.Join(outputDir, rancherYml)) + os.Remove(scriptOutput) + os.Remove(cloudConfigOutput) + os.Remove(rancherYmlOutput) + os.Remove(metaDataOutput) - if script != nil { - fileData = userdataBytes - output := path.Join(outputDir, scriptFile) - log.Infof("Writing cloud-config script to %s", output) - if err := ioutil.WriteFile(output, fileData, 500); err != nil { - log.Errorf("Error while writing file %v", err) + if len(scriptBytes) > 0 { + log.Infof("Writing to %s", scriptOutput) + if err := ioutil.WriteFile(scriptOutput, scriptBytes, 500); err != nil { + log.Errorf("Error while writing file %s: %v", scriptOutput, err) return err } } - if data, err := yaml.Marshal(cc); err != nil { - log.Errorf("Error while marshalling cloud config %v", err) - return err - } else { - fileData = append([]byte("#cloud-config\n"), data...) - } - - output := path.Join(outputDir, outputFile) - log.Infof("Writing merged cloud-config to %s", output) - if err := ioutil.WriteFile(output, fileData, 400); err != nil { - log.Errorf("Error while writing file %v", err) + cloudConfigBytes = append([]byte("#cloud-config\n"), cloudConfigBytes...) + log.Infof("Writing to %s", cloudConfigOutput) + if err := ioutil.WriteFile(cloudConfigOutput, cloudConfigBytes, 500); err != nil { + log.Errorf("Error while writing file %s: %v", cloudConfigOutput, err) return err } - if script == nil { - ccData := make(map[string]interface{}) - if err := yaml.Unmarshal(userdataBytes, ccData); err != nil { + ccData := make(map[string]interface{}) + if err := yaml.Unmarshal(cloudConfigBytes, ccData); err != nil { + return err + } + + if rancher, ok := ccData["rancher"]; ok { + bytes, err := yaml.Marshal(rancher) + if err != nil { return err } - if rancher, ok := ccData["rancher"]; ok { - bytes, err := yaml.Marshal(rancher) - if err != nil { - return err - } - - if err = ioutil.WriteFile(path.Join(outputDir, rancherYml), bytes, 400); err != nil { - return err - } + if err = ioutil.WriteFile(rancherYmlOutput, bytes, 400); err != nil { + return err } } + metaDataBytes, err := yaml.Marshal(metadata) + if err != nil { + return err + } + + if err = ioutil.WriteFile(metaDataOutput, metaDataBytes, 400); err != nil { + return err + } + return nil } -func Main() { - flags.Parse(rancherConfig.FilterGlobalConfig(os.Args[1:])) - +func currentDatasource() (datasource.Datasource, error) { cfg, err := rancherConfig.LoadConfig() if err != nil { log.Fatalf("Failed to read rancher config %v", err) @@ -133,53 +137,152 @@ func Main() { dss := getDatasources(cfg) if len(dss) == 0 { - log.Infof("No datasources available %v", cfg.CloudInit.Datasources) - os.Exit(0) + return nil, nil } ds := selectDatasource(dss) - if ds == nil { - log.Info("No datasources found") - os.Exit(0) - } + return ds, nil +} - log.Infof("Fetching user-data from datasource %v", ds.Type()) - userdataBytes, err := ds.FetchUserdata() +func mergeBaseConfig(current []byte) ([]byte, error) { + files, err := ioutil.ReadDir(baseConfigDir) if err != nil { - log.Fatalf("Failed fetching user-data from datasource: %v", err) - } - - log.Infof("Fetching meta-data from datasource of type %v", ds.Type()) - metadata, err := ds.FetchMetadata() - if err != nil { - log.Infof("Failed fetching meta-data from datasource: %v", err) - os.Exit(1) - } - - var ccu *config.CloudConfig - var script *config.Script - if ud, err := initialize.ParseUserData(string(userdataBytes)); err != nil { - log.Fatalf("Failed to parse user-data: %v\n", err) - } else { - switch t := ud.(type) { - case *config.CloudConfig: - ccu = t - case *config.Script: - script = t + if os.IsNotExist(err) { + log.Infof("%s does not exist, not merging", baseConfigDir) + return current, nil } + + log.Errorf("Failed to read %s: %v", baseConfigDir, err) + return nil, err + } + + result := []byte{} + + for _, file := range files { + if file.IsDir() || strings.HasPrefix(file.Name(), ".") { + continue + } + + input := path.Join(baseConfigDir, file.Name()) + content, err := ioutil.ReadFile(input) + if err != nil { + log.Errorf("Failed to read %s: %v", input, err) + // ignore error + continue + } + + log.Infof("Merging %s", input) + result, err = util.MergeBytes(result, content) + if err != nil { + log.Errorf("Failed to merge bytes: %v", err) + return nil, err + } + } + + if len(result) == 0 { + return current, nil + } else { + return util.MergeBytes(result, current) + } +} + +func saveCloudConfig() error { + var userDataBytes []byte + var metadata datasource.Metadata + + ds, err := currentDatasource() + if err != nil { + log.Errorf("Failed to select datasource: %v", err) + return err + } + + if ds != nil { + log.Infof("Fetching user-data from datasource %v", ds.Type()) + userDataBytes, err = ds.FetchUserdata() + if err != nil { + log.Errorf("Failed fetching user-data from datasource: %v", err) + return err + } + + log.Infof("Fetching meta-data from datasource of type %v", ds.Type()) + metadata, err = ds.FetchMetadata() + if err != nil { + log.Errorf("Failed fetching meta-data from datasource: %v", err) + return err + } + } + + userData := string(userDataBytes) + scriptBytes := []byte{} + + if config.IsScript(userData) { + scriptBytes = userDataBytes + userDataBytes = []byte{} + } else if isCompose(userData) { + if userDataBytes, err = toCompose(userDataBytes); err != nil { + log.Errorf("Failed to convert to compose syntax: %v", err) + return err + } + } else if config.IsCloudConfig(userData) { + // nothing to do + } else { + log.Errorf("Unrecognized cloud-init\n%s", userData) + userDataBytes = []byte{} + } + + if userDataBytes, err = mergeBaseConfig(userDataBytes); err != nil { + log.Errorf("Failed to merge base config: %v", err) + return err + } + + return saveFiles(userDataBytes, scriptBytes, metadata) +} + +func getSaveCloudConfig() (*config.CloudConfig, error) { + cloudConfig := path.Join(outputDir, outputFile) + + ds := file.NewDatasource(cloudConfig) + if !ds.IsAvailable() { + log.Infof("%s does not exist", cloudConfig) + return nil, nil + } + + ccBytes, err := ds.FetchUserdata() + if err != nil { + log.Errorf("Failed to read user-data from %s: %v", cloudConfig, err) + return nil, err + } + + var cc config.CloudConfig + err = yaml.Unmarshal(ccBytes, &cc) + if err != nil { + log.Errorf("Failed to unmarshall user-data from %s: %v", cloudConfig, err) + return nil, err + } + + return &cc, err +} + +func executeCloudConfig() error { + ccu, err := getSaveCloudConfig() + if err != nil { + return err + } + + var metadata datasource.Metadata + + metaDataBytes, err := ioutil.ReadFile(path.Join(outputDir, metaDataFile)) + if err != nil { + return err + } + + if err = yaml.Unmarshal(metaDataBytes, &metadata); err != nil { + return err } log.Info("Merging cloud-config from meta-data and user-data") cc := mergeConfigs(ccu, metadata) - if save { - err = saveFiles(script, userdataBytes, &cc) - if err != nil { - log.Fatal(err) - } - os.Exit(0) - } - if len(cc.SSHAuthorizedKeys) > 0 { authorizeSSHKeys("rancher", cc.SSHAuthorizedKeys, sshKeyName) } @@ -201,6 +304,26 @@ func Main() { } log.Printf("Wrote file %s to filesystem", fullPath) } + + return nil +} + +func Main() { + flags.Parse(rancherConfig.FilterGlobalConfig(os.Args[1:])) + + if save { + err := saveCloudConfig() + if err != nil { + log.Fatalf("Failed to save cloud config: %v", err) + } + } + + if execute { + err := executeCloudConfig() + if err != nil { + log.Fatalf("Failed to save cloud config: %v", err) + } + } } // mergeConfigs merges certain options from md (meta-data from the datasource) @@ -229,16 +352,6 @@ func mergeConfigs(cc *config.CloudConfig, md datasource.Metadata) (out config.Cl func getDatasources(cfg *rancherConfig.Config) []datasource.Datasource { dss := make([]datasource.Datasource, 0, 5) - if execute { - cloudConfig := path.Join(outputDir, outputFile) - if _, err := os.Stat(cloudConfig); os.IsNotExist(err) { - return dss - } - - dss = append(dss, file.NewDatasource(cloudConfig)) - return dss - } - for _, ds := range cfg.CloudInit.Datasources { parts := strings.SplitN(ds, ":", 2) @@ -355,3 +468,19 @@ func selectDatasource(sources []datasource.Datasource) datasource.Datasource { close(stop) return s } + +func isCompose(content string) bool { + return strings.HasPrefix(content, "#compose\n") +} + +func toCompose(bytes []byte) ([]byte, error) { + result := make(map[interface{}]interface{}) + compose := make(map[interface{}]interface{}) + err := yaml.Unmarshal(bytes, &compose) + if err != nil { + return nil, err + } + + result["services"] = compose + return yaml.Marshal(result) +} diff --git a/cmd/cloudinit/gcecloudinit.go b/cmd/cloudinit/gce.go similarity index 100% rename from cmd/cloudinit/gcecloudinit.go rename to cmd/cloudinit/gce.go diff --git a/cmd/control/os.go b/cmd/control/os.go index f5f35450..acffb685 100644 --- a/cmd/control/os.go +++ b/cmd/control/os.go @@ -162,6 +162,7 @@ func startUpgradeContainer(image string, stage, force bool) { container := docker.NewContainer(config.DOCKER_SYSTEM_HOST, &config.ContainerConfig{ Cmd: "--name=os-upgrade " + + "--log-driver=json-file" + "--rm " + "--privileged " + "--net=host " + diff --git a/cmd/control/service.go b/cmd/control/service.go index 11c63a3f..67c0b2f1 100644 --- a/cmd/control/service.go +++ b/cmd/control/service.go @@ -37,21 +37,16 @@ func disable(c *cli.Context) { } for _, service := range c.Args() { - filtered := make([]string, 0, len(c.Args())) - for _, existing := range cfg.EnabledServices { - if existing != service { - filtered = append(filtered, existing) - } + if _, ok := cfg.ServicesInclude[service]; !ok { + continue } - if len(filtered) != len(c.Args()) { - cfg.EnabledServices = filtered - changed = true - } + cfg.ServicesInclude[service] = false + changed = true } if changed { - if err = cfg.Set("enabled_services", cfg.EnabledServices); err != nil { + if err = cfg.Set("services_include", cfg.ServicesInclude); err != nil { log.Fatal(err) } } @@ -65,14 +60,14 @@ func enable(c *cli.Context) { } for _, service := range c.Args() { - if !util.Contains(cfg.EnabledServices, service) { - cfg.EnabledServices = append(cfg.EnabledServices, service) + if val, ok := cfg.ServicesInclude[service]; !ok || !val { + cfg.ServicesInclude[service] = true changed = true } } if changed { - if err = cfg.Set("enabled_services", cfg.EnabledServices); err != nil { + if err = cfg.Set("services_include", cfg.ServicesInclude); err != nil { log.Fatal(err) } } @@ -84,22 +79,34 @@ func list(c *cli.Context) { log.Fatal(err) } - enabled := map[string]bool{} - - for _, service := range cfg.EnabledServices { - enabled[service] = true + clone := make(map[string]bool) + for service, enabled := range cfg.ServicesInclude { + clone[service] = enabled } - for service, _ := range cfg.Services { - if _, ok := enabled[service]; ok { - delete(enabled, service) - fmt.Printf("enabled %s\n", service) + services, err := util.GetServices(cfg.Repositories.ToArray()) + if err != nil { + log.Fatalf("Failed to get services: %v", err) + } + + for _, service := range services { + if enabled, ok := clone[service]; ok { + delete(clone, service) + if enabled { + fmt.Printf("enabled %s\n", service) + } else { + fmt.Printf("disabled %s\n", service) + } } else { fmt.Printf("disabled %s\n", service) } } - for service, _ := range enabled { - fmt.Printf("enabled %s\n", service) + for service, enabled := range clone { + if enabled { + fmt.Printf("enabled %s\n", service) + } else { + fmt.Printf("disabled %s\n", service) + } } } diff --git a/config/config.go b/config/config.go index 6b98f3d2..2694f1b0 100644 --- a/config/config.go +++ b/config/config.go @@ -264,3 +264,14 @@ func (d *DockerConfig) BridgeConfig() (string, string) { return name, cidr } } + +func (r Repositories) ToArray() []string { + result := make([]string, 0, len(r)) + for _, repo := range r { + if repo.Url != "" { + result = append(result, repo.Url) + } + } + + return result +} diff --git a/config/default.go b/config/default.go index 403f9f45..7fd2e02b 100644 --- a/config/default.go +++ b/config/default.go @@ -81,6 +81,7 @@ func NewConfig() *Config { Privileged: true, Labels: []string{ DETACH + "=false", + SCOPE + "=" + SYSTEM, }, Volumes: []string{ "/dev:/host/dev", @@ -98,6 +99,7 @@ func NewConfig() *Config { Privileged: true, Labels: []string{ DETACH + "=true", + SCOPE + "=" + SYSTEM, }, Environment: []string{ "DAEMON=true", @@ -113,6 +115,7 @@ func NewConfig() *Config { Privileged: true, Labels: []string{ CREATE_ONLY + "=true", + SCOPE + "=" + SYSTEM, }, Volumes: []string{ "/dev:/host/dev", @@ -132,6 +135,7 @@ func NewConfig() *Config { Privileged: true, Labels: []string{ CREATE_ONLY + "=true", + SCOPE + "=" + SYSTEM, }, Volumes: []string{ "/init:/sbin/halt:ro", @@ -156,6 +160,7 @@ func NewConfig() *Config { Privileged: true, Labels: []string{ CREATE_ONLY + "=true", + SCOPE + "=" + SYSTEM, }, Volumes: []string{ "/home:/home", @@ -170,6 +175,7 @@ func NewConfig() *Config { Privileged: true, Labels: []string{ CREATE_ONLY + "=true", + SCOPE + "=" + SYSTEM, }, Volumes: []string{ "/var/lib/rancher:/var/lib/rancher", @@ -185,6 +191,7 @@ func NewConfig() *Config { Privileged: true, Labels: []string{ CREATE_ONLY + "=true", + SCOPE + "=" + SYSTEM, }, VolumesFrom: []string{ "docker-volumes", @@ -201,6 +208,7 @@ func NewConfig() *Config { Labels: []string{ RELOAD_CONFIG + "=true", DETACH + "=false", + SCOPE + "=" + SYSTEM, }, Environment: []string{ "CLOUD_INIT_NETWORK=false", @@ -216,6 +224,7 @@ func NewConfig() *Config { Net: "host", Labels: []string{ DETACH + "=false", + SCOPE + "=" + SYSTEM, }, Links: []string{ "cloud-init-pre", @@ -231,6 +240,7 @@ func NewConfig() *Config { Labels: []string{ RELOAD_CONFIG + "=true", DETACH + "=false", + SCOPE + "=" + SYSTEM, }, Net: "host", Links: []string{ @@ -246,6 +256,9 @@ func NewConfig() *Config { Image: "ntp", Privileged: true, Net: "host", + Labels: []string{ + SCOPE + "=" + SYSTEM, + }, Links: []string{ "cloud-init", "network", @@ -255,6 +268,9 @@ func NewConfig() *Config { Image: "syslog", Privileged: true, Net: "host", + Labels: []string{ + SCOPE + "=" + SYSTEM, + }, VolumesFrom: []string{ "system-volumes", }, @@ -266,6 +282,9 @@ func NewConfig() *Config { Pid: "host", Ipc: "host", Net: "host", + Labels: []string{ + SCOPE + "=" + SYSTEM, + }, Links: []string{ "network", }, @@ -277,7 +296,8 @@ func NewConfig() *Config { Image: "userdockerwait", Net: "host", Labels: []string{ - "io.rancher.os.detach=false", + DETACH + "=false", + SCOPE + "=" + SYSTEM, }, Links: []string{ "userdocker", @@ -292,6 +312,9 @@ func NewConfig() *Config { Links: []string{ "cloud-init", }, + Labels: []string{ + SCOPE + "=" + SYSTEM, + }, VolumesFrom: []string{ "all-volumes", }, @@ -301,29 +324,14 @@ func NewConfig() *Config { Net: "host", }, }, - EnabledServices: []string{}, - Services: map[string]Config{ - "ubuntu-console": { - SystemContainers: map[string]*project.ServiceConfig{ - "console": { - Image: "rancher/ubuntuconsole:" + IMAGE_VERSION, - Privileged: true, - Labels: []string{ - DETACH + "=true", - }, - Links: []string{ - "cloud-init", - }, - VolumesFrom: []string{ - "all-volumes", - }, - Restart: "always", - Pid: "host", - Ipc: "host", - Net: "host", - }, - }, + ServicesInclude: map[string]bool{ + "ubuntu-console": false, + }, + Repositories: map[string]Repository{ + "core": Repository{ + Url: "https://raw.githubusercontent.com/rancherio/os-services/master", }, }, + Services: map[string]*project.ServiceConfig{}, } } diff --git a/config/types.go b/config/types.go index b775945a..22dbeee7 100644 --- a/config/types.go +++ b/config/types.go @@ -23,6 +23,8 @@ const ( REMOVE = "io.rancher.os.remove" CREATE_ONLY = "io.rancher.os.createonly" RELOAD_CONFIG = "io.rancher.os.reloadconfig" + SCOPE = "io.rancher.os.scope" + SYSTEM = "system" ) var ( @@ -42,17 +44,25 @@ type ContainerConfig struct { Service *project.ServiceConfig `yaml:service,omitempty` } +type Repository struct { + Url string `yaml:url,omitempty` +} + +type Repositories map[string]Repository + type Config struct { - Services map[string]Config `yaml:"services,omitempty"` + Environment map[string]string `yaml:"environment,omitempty"` + Services map[string]*project.ServiceConfig `yaml:"services,omitempty"` BootstrapContainers map[string]*project.ServiceConfig `yaml:"bootstrap_containers,omitempty"` BootstrapDocker DockerConfig `yaml:"bootstrap_docker,omitempty"` CloudInit CloudInit `yaml:"cloud_init,omitempty"` Console ConsoleConfig `yaml:"console,omitempty"` Debug bool `yaml:"debug,omitempty"` Disable []string `yaml:"disable,omitempty"` - EnabledServices []string `yaml:"enabled_services,omitempty"` + ServicesInclude map[string]bool `yaml:"services_include,omitempty"` Modules []string `yaml:"modules,omitempty"` Network NetworkConfig `yaml:"network,omitempty"` + Repositories Repositories `yaml:"repositories,omitempty"` Ssh SshConfig `yaml:"ssh,omitempty"` State StateConfig `yaml:"state,omitempty"` SystemContainers map[string]*project.ServiceConfig `yaml:"system_containers,omitempty"` diff --git a/docker/container.go b/docker/container.go index f4a5d427..451ed907 100644 --- a/docker/container.go +++ b/docker/container.go @@ -71,6 +71,7 @@ func NewContainerFromService(dockerHost string, name string, service *project.Se Service: service, }, } + return c.Parse() } @@ -170,11 +171,55 @@ func (c *Container) Reset() *Container { return c } +func (c *Container) requiresSyslog() bool { + return (c.ContainerCfg.Service.LogDriver == "" || c.ContainerCfg.Service.LogDriver == "syslog") +} + +func (c *Container) requiresUserDocker() bool { + if c.dockerHost == config.DOCKER_HOST { + return true + } + + for _, v := range c.ContainerCfg.Service.Volumes { + if strings.Index(v, "/var/run/docker.sock") != -1 { + return true + } + } + + return false +} + +func (c *Container) hasLink(link string) bool { + return util.Contains(c.ContainerCfg.Service.Links, link) +} + +func (c *Container) addLink(link string) { + if c.hasLink(link) { + return + } + + log.Debugf("Adding %s link to %s", link, c.Name) + c.ContainerCfg.Service.Links = append(c.ContainerCfg.Service.Links, link) +} + func (c *Container) parseService() { - if (c.ContainerCfg.Service.LogDriver == "" || c.ContainerCfg.Service.LogDriver == "syslog") && - !util.Contains(c.ContainerCfg.Service.Links, "syslog") { - log.Debugf("Adding syslog link to %s\n", c.Name) - c.ContainerCfg.Service.Links = append(c.ContainerCfg.Service.Links, "syslog") + if c.requiresSyslog() { + c.addLink("syslog") + } + + if c.requiresUserDocker() { + c.addLink("userdockerwait") + } else if c.ContainerCfg.Service.Image != "" { + client, err := NewClient(c.dockerHost) + if err != nil { + c.Err = err + return + } + + i, _ := client.InspectImage(c.ContainerCfg.Service.Image) + if i == nil { + c.addLink("network") + } } cfg, hostConfig, err := docker.Convert(c.ContainerCfg.Service) @@ -190,7 +235,6 @@ func (c *Container) parseService() { c.remove = c.Config.Labels[config.REMOVE] != "false" c.ContainerCfg.CreateOnly = c.Config.Labels[config.CREATE_ONLY] == "true" c.ContainerCfg.ReloadConfig = c.Config.Labels[config.RELOAD_CONFIG] == "true" - } func (c *Container) parseCmd() { diff --git a/docker/factory.go b/docker/factory.go index df64593a..c950f6ba 100644 --- a/docker/factory.go +++ b/docker/factory.go @@ -38,6 +38,8 @@ func (c *containerBasedService) Up() error { var event project.Event + c.project.Notify(project.CONTAINER_STARTING, c, map[string]string{}) + if create { container.Create() event = project.CONTAINER_CREATED @@ -71,8 +73,17 @@ func (c *containerBasedService) Name() string { return c.name } +func isSystemService(serviceConfig *project.ServiceConfig) bool { + return util.GetValue(serviceConfig.Labels, config.SCOPE) == config.SYSTEM +} + func (c *ContainerFactory) Create(project *project.Project, name string, serviceConfig *project.ServiceConfig) (project.Service, error) { - container := NewContainerFromService(config.DOCKER_SYSTEM_HOST, name, serviceConfig) + host := config.DOCKER_HOST + if isSystemService(serviceConfig) { + host = config.DOCKER_SYSTEM_HOST + } + + container := NewContainerFromService(host, name, serviceConfig) if container.Err != nil { return nil, container.Err diff --git a/docker/services.go b/docker/services.go new file mode 100644 index 00000000..fe8852b2 --- /dev/null +++ b/docker/services.go @@ -0,0 +1,144 @@ +package docker + +import ( + "fmt" + "strings" + + log "github.com/Sirupsen/logrus" + "github.com/rancherio/os/config" + "github.com/rancherio/os/util" + "github.com/rancherio/rancher-compose/project" +) + +type configEnvironemnt struct { + cfg *config.Config +} + +func appendEnv(array []string, key, value string) []string { + parts := strings.SplitN(key, "/", 2) + if len(parts) == 2 { + key = parts[1] + } + + return append(array, fmt.Sprintf("%s=%s", key, value)) +} + +func lookupKeys(cfg *config.Config, keys ...string) []string { + for _, key := range keys { + if strings.HasSuffix(key, "*") { + result := []string{} + for envKey, envValue := range cfg.Environment { + keyPrefix := key[:len(key)-1] + if strings.HasPrefix(envKey, keyPrefix) { + result = appendEnv(result, envKey, envValue) + } + } + + if len(result) > 0 { + return result + } + } else if value, ok := cfg.Environment[key]; ok { + return appendEnv([]string{}, key, value) + } + } + + return []string{} +} + +func (c *configEnvironemnt) Lookup(key, serviceName string, serviceConfig *project.ServiceConfig) []string { + fullKey := fmt.Sprintf("%s/%s", serviceName, key) + return lookupKeys(c.cfg, fullKey, key) +} + +func RunServices(name string, cfg *config.Config, configs map[string]*project.ServiceConfig) error { + network := false + projectEvents := make(chan project.ProjectEvent) + p := project.NewProject(name, NewContainerFactory(cfg)) + p.EnvironmentLookup = &configEnvironemnt{cfg: cfg} + p.AddListener(projectEvents) + enabled := make(map[string]bool) + + for name, serviceConfig := range configs { + if err := p.AddConfig(name, serviceConfig); err != nil { + log.Infof("Failed loading service %s", name) + } + } + + p.ReloadCallback = func() error { + err := cfg.Reload() + if err != nil { + return err + } + + for service, serviceEnabled := range cfg.ServicesInclude { + if !serviceEnabled { + continue + } + + if _, ok := enabled[service]; ok { + continue + } + + //if config, ok := cfg.BundledServices[service]; ok { + // for name, s := range config.SystemContainers { + // if err := p.AddConfig(name, s); err != nil { + // log.Errorf("Failed to load %s : %v", name, err) + // } + // } + //} else { + bytes, err := LoadServiceResource(service, network, cfg) + if err != nil { + if err == util.ErrNoNetwork { + log.Debugf("Can not load %s, networking not enabled", service) + } else { + log.Errorf("Failed to load %s : %v", service, err) + } + continue + } + + err = p.Load(bytes) + if err != nil { + log.Errorf("Failed to load %s : %v", service, err) + continue + } + //} + + enabled[service] = true + } + + for service, config := range cfg.Services { + if _, ok := enabled[service]; ok { + continue + } + + err = p.AddConfig(service, config) + if err != nil { + log.Errorf("Failed to load %s : %v", service, err) + continue + } + + enabled[service] = true + } + + return nil + } + + go func() { + for event := range projectEvents { + if event.Event == project.CONTAINER_STARTED && event.Service.Name() == "network" { + network = true + } + } + }() + + err := p.ReloadCallback() + if err != nil { + log.Errorf("Failed to reload %s : %v", name, err) + return err + } + return p.Up() +} + +func LoadServiceResource(name string, network bool, cfg *config.Config) ([]byte, error) { + return util.LoadResource(name, network, cfg.Repositories.ToArray()) +} diff --git a/init/bootstrap.go b/init/bootstrap.go index ceed6567..5b2acf6a 100644 --- a/init/bootstrap.go +++ b/init/bootstrap.go @@ -7,6 +7,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/rancherio/os/config" + "github.com/rancherio/os/docker" "github.com/rancherio/os/util" "github.com/rancherio/rancher-compose/project" ) @@ -55,7 +56,7 @@ outer: if format != "" { log.Infof("Auto formatting : %s", format) - return runServices("autoformat", cfg, map[string]*project.ServiceConfig{ + return docker.RunServices("autoformat", cfg, map[string]*project.ServiceConfig{ "autoformat": { Net: "none", Privileged: true, @@ -70,7 +71,7 @@ outer: } func runBootstrapContainers(cfg *config.Config) error { - return runServices("bootstrap", cfg, cfg.BootstrapContainers) + return docker.RunServices("bootstrap", cfg, cfg.BootstrapContainers) } func startDocker(cfg *config.Config) (chan interface{}, error) { diff --git a/init/init.go b/init/init.go index 285550c0..e8b615a5 100644 --- a/init/init.go +++ b/init/init.go @@ -3,7 +3,6 @@ package init import ( "fmt" "io/ioutil" - "net" "os" "os/exec" "strings" @@ -16,9 +15,10 @@ import ( ) const ( - STATE string = "/var" - DOCKER string = "/usr/bin/docker" - SYSINIT string = "/sbin/rancher-sysinit" + STATE string = "/var" + SYSTEM_DOCKER string = "/usr/bin/system-docker" + DOCKER string = "/usr/bin/docker" + SYSINIT string = "/sbin/rancher-sysinit" ) var ( @@ -63,6 +63,7 @@ var ( "/sbin/modprobe": "/busybox", "/usr/sbin/iptables": "/xtables-multi", DOCKER: "/docker", + SYSTEM_DOCKER: "/docker", SYSINIT: "/init", "/home": "/var/lib/rancher/state/home", "/opt": "/var/lib/rancher/state/opt", @@ -237,7 +238,7 @@ func execDocker(cfg *config.Config) error { } os.Stdin.Close() - return syscall.Exec(DOCKER, cfg.SystemDocker.Args, os.Environ()) + return syscall.Exec(SYSTEM_DOCKER, cfg.SystemDocker.Args, os.Environ()) } func MainInit() { @@ -285,10 +286,9 @@ func touchSocket(cfg *config.Config) error { if err := syscall.Unlink(path); err != nil && !os.IsNotExist(err) { return err } - if l, err := net.Listen("unix", path); err != nil { + err := ioutil.WriteFile(path, []byte{}, 0700) + if err != nil { return err - } else { - l.Close() } } @@ -365,7 +365,8 @@ func RunInit() error { return createMounts(postMounts...) }, touchSocket, - remountRo, + // Disable R/O root write now to support updating modules + //remountRo, sysInit, } diff --git a/init/sysinit.go b/init/sysinit.go index 723b88d1..087a9814 100644 --- a/init/sysinit.go +++ b/init/sysinit.go @@ -9,9 +9,6 @@ import ( dockerClient "github.com/fsouza/go-dockerclient" "github.com/rancherio/os/config" "github.com/rancherio/os/docker" - "github.com/rancherio/os/util" - - "github.com/rancherio/rancher-compose/project" ) func importImage(client *dockerClient.Client, name, fileName string) error { @@ -109,63 +106,8 @@ func loadImages(cfg *config.Config) error { return nil } -func runServices(name string, cfg *config.Config, configs map[string]*project.ServiceConfig) error { - project := project.NewProject(name, docker.NewContainerFactory(cfg)) - enabled := make(map[string]bool) - - for name, serviceConfig := range configs { - if err := project.AddConfig(name, serviceConfig); err != nil { - log.Infof("Failed loading service %s", name) - } - } - - project.ReloadCallback = func() error { - err := cfg.Reload() - if err != nil { - return err - } - - for _, service := range cfg.EnabledServices { - if _, ok := enabled[service]; ok { - continue - } - - if config, ok := cfg.Services[service]; ok { - for name, s := range config.SystemContainers { - if err := project.AddConfig(name, s); err != nil { - log.Errorf("Failed to load %s : %v", name, err) - } - } - } else { - bytes, err := util.LoadResource(service) - if err != nil { - log.Errorf("Failed to load %s : %v", service, err) - continue - } - - err = project.Load(bytes) - if err != nil { - log.Errorf("Failed to load %s : %v", service, err) - continue - } - } - - enabled[service] = true - } - - return nil - } - - err := project.ReloadCallback() - if err != nil { - log.Errorf("Failed to reload %s : %v", name, err) - return err - } - return project.Up() -} - func runContainers(cfg *config.Config) error { - return runServices("system-init", cfg, cfg.SystemContainers) + return docker.RunServices("system-init", cfg, cfg.SystemContainers) } func tailConsole(cfg *config.Config) error { diff --git a/scripts/dockerimages/scripts/console.sh b/scripts/dockerimages/scripts/console.sh index 7806839d..a944c9f7 100755 --- a/scripts/dockerimages/scripts/console.sh +++ b/scripts/dockerimages/scripts/console.sh @@ -76,7 +76,7 @@ if ! grep -q '^UseDNS no' /etc/ssh/sshd_config; then fi ID_TYPE="busybox" -if grep -q 'ID_LIKE=' /etc/os-release; then +if [ -e /etc/os-release ] && grep -q 'ID_LIKE=' /etc/os-release; then ID_TYPE=$(grep 'ID_LIKE=' /etc/os-release | cut -d'=' -f2) fi diff --git a/scripts/extraimages/00-ubuntuconsole b/scripts/extraimages/00-ubuntuconsole index 0436996d..d40aea68 100644 --- a/scripts/extraimages/00-ubuntuconsole +++ b/scripts/extraimages/00-ubuntuconsole @@ -3,6 +3,7 @@ RUN apt-get update && \ apt-get upgrade --no-install-recommends -y && \ apt-get install -y --no-install-recommends openssh-server rsync RUN rm -rf /etc/ssh/*key* +COPY scripts/dockerimages/scripts/entry.sh /usr/sbin/ COPY scripts/dockerimages/scripts/console.sh /usr/sbin/ COPY scripts/dockerimages/scripts/update-ssh-keys /usr/bin/ RUN echo 'RancherOS \\n \l' > /etc/issue @@ -12,4 +13,5 @@ RUN addgroup --gid 1100 rancher && \ useradd -u 1100 -g rancher -G docker,sudo -m -s /bin/bash rancher && \ echo '## allow password less for rancher user' >> /etc/sudoers && \ echo 'rancher ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers +ENTRYPOINT ["/usr/sbin/entry.sh"] CMD ["/usr/sbin/console.sh"] diff --git a/scripts/run b/scripts/run index ab7caa0d..a49d8186 100755 --- a/scripts/run +++ b/scripts/run @@ -82,7 +82,7 @@ qemu-system-x86_64 -serial stdio \ -initrd ${INITRD_TEST} \ -m 1024 \ -net nic,vlan=0,model=virtio \ - -net user,vlan=0,hostfwd=tcp::2222-:22,hostname=rancher \ + -net user,vlan=0,hostfwd=tcp::2222-:22,hostname=rancher-dev \ -drive if=virtio,file=${HD} \ -machine accel=kvm \ -cpu host \ diff --git a/scripts/version b/scripts/version index 52351aae..392e846a 100644 --- a/scripts/version +++ b/scripts/version @@ -1 +1 @@ -VERSION=v0.3.0-rc2 +VERSION=v0.3.0-rc3 diff --git a/util/util.go b/util/util.go index b97b9aaa..baf96f4f 100644 --- a/util/util.go +++ b/util/util.go @@ -2,6 +2,7 @@ package util import ( "archive/tar" + "errors" "fmt" "io" "io/ioutil" @@ -12,12 +13,16 @@ import ( "strings" "syscall" + log "github.com/Sirupsen/logrus" + "github.com/docker/docker/pkg/mount" "gopkg.in/yaml.v2" ) var ( - letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + ErrNoNetwork = errors.New("Networking not available to load resource") + ErrNotFound = errors.New("Failed to find resource") ) func mountProc() error { @@ -168,6 +173,25 @@ func Convert(from, to interface{}) error { return yaml.Unmarshal(bytes, to) } +func MergeBytes(left, right []byte) ([]byte, error) { + leftMap := make(map[interface{}]interface{}) + rightMap := make(map[interface{}]interface{}) + + err := yaml.Unmarshal(left, &leftMap) + if err != nil { + return nil, err + } + + err = yaml.Unmarshal(right, &rightMap) + if err != nil { + return nil, err + } + + MergeMaps(leftMap, rightMap) + + return yaml.Marshal(leftMap) +} + func MergeMaps(left, right map[interface{}]interface{}) { for k, v := range right { merged := false @@ -186,15 +210,76 @@ func MergeMaps(left, right map[interface{}]interface{}) { } } -func LoadResource(location string) ([]byte, error) { +func GetServices(urls []string) ([]string, error) { + result := []string{} + + for _, url := range urls { + indexUrl := fmt.Sprintf("%s/index.yml", url) + content, err := LoadResource(indexUrl, true, []string{}) + if err != nil { + log.Errorf("Failed to load %s: %v", indexUrl, err) + continue + } + + services := make(map[string][]string) + err = yaml.Unmarshal(content, &services) + if err != nil { + log.Errorf("Failed to unmarshal %s: %v", indexUrl, err) + continue + } + + if list, ok := services["services"]; ok { + result = append(result, list...) + } + } + + return []string{}, nil +} + +func LoadResource(location string, network bool, urls []string) ([]byte, error) { + var bytes []byte + err := ErrNotFound + if strings.HasPrefix(location, "http:/") || strings.HasPrefix(location, "https:/") { + if !network { + return nil, ErrNoNetwork + } resp, err := http.Get(location) if err != nil { return nil, err } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("non-200 http response: %d", resp.StatusCode) + } defer resp.Body.Close() return ioutil.ReadAll(resp.Body) - } else { + } else if strings.HasPrefix(location, "/") { return ioutil.ReadFile(location) + } else if len(location) > 0 { + for _, url := range urls { + ymlUrl := fmt.Sprintf("%s/%s/%s.yml", url, location[0:1], location) + log.Infof("Loading %s from %s", location, ymlUrl) + bytes, err = LoadResource(ymlUrl, network, []string{}) + if err == nil { + return bytes, nil + } + } } + + return nil, err +} + +func GetValue(kvPairs []string, key string) string { + if kvPairs == nil { + return "" + } + + prefix := key + "=" + for _, i := range kvPairs { + if strings.HasPrefix(i, prefix) { + return strings.TrimPrefix(i, prefix) + } + } + + return "" }