Merge pull request #182 from ibuildthecloud/compose-services

Compose services

Godeps/Godeps.json

@@ -22,11 +22,6 @@
 			"Comment": "v1.3.2-6-g405c260",
 			"Rev": "405c2600b19ae77516c967f8ee8ebde5624d3663"
 		},
-		{
-			"ImportPath": "github.com/coreos/coreos-cloudinit/initialize",
-			"Comment": "v1.3.2-6-g405c260",
-			"Rev": "405c2600b19ae77516c967f8ee8ebde5624d3663"
-		},
 		{
 			"ImportPath": "github.com/coreos/coreos-cloudinit/network",
 			"Comment": "v1.3.2-6-g405c260",
@@ -226,13 +221,13 @@
 		},
 		{
 			"ImportPath": "github.com/rancherio/rancher-compose/docker",
-			"Comment": "0.1.0-8-g45c7de3",
+			"Comment": "0.1.0-12-g8fecf18",
-			"Rev": "45c7de3d9b5b106475cf1461df7550bdaab0a9aa"
+			"Rev": "8fecf186bdab6b14c9f625f9499f959fd8590482"
 		},
 		{
 			"ImportPath": "github.com/rancherio/rancher-compose/project",
-			"Comment": "0.1.0-8-g45c7de3",
+			"Comment": "0.1.0-12-g8fecf18",
-			"Rev": "45c7de3d9b5b106475cf1461df7550bdaab0a9aa"
+			"Rev": "8fecf186bdab6b14c9f625f9499f959fd8590482"
 		},
 		{
 			"ImportPath": "github.com/ryanuber/go-glob",

Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/config.go

@@ -1,293 +0,0 @@
-// Copyright 2015 CoreOS, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package initialize
-
-import (
-	"errors"
-	"fmt"
-	"log"
-	"path"
-
-	"github.com/coreos/coreos-cloudinit/config"
-	"github.com/coreos/coreos-cloudinit/network"
-	"github.com/coreos/coreos-cloudinit/system"
-)
-
-// CloudConfigFile represents a CoreOS specific configuration option that can generate
-// an associated system.File to be written to disk
-type CloudConfigFile interface {
-	// File should either return (*system.File, error), or (nil, nil) if nothing
-	// needs to be done for this configuration option.
-	File() (*system.File, error)
-}
-
-// CloudConfigUnit represents a CoreOS specific configuration option that can generate
-// associated system.Units to be created/enabled appropriately
-type CloudConfigUnit interface {
-	Units() []system.Unit
-}
-
-// Apply renders a CloudConfig to an Environment. This can involve things like
-// configuring the hostname, adding new users, writing various configuration
-// files to disk, and manipulating systemd services.
-func Apply(cfg config.CloudConfig, ifaces []network.InterfaceGenerator, env *Environment) error {
-	if cfg.Hostname != "" {
-		if err := system.SetHostname(cfg.Hostname); err != nil {
-			return err
-		}
-		log.Printf("Set hostname to %s", cfg.Hostname)
-	}
-
-	for _, user := range cfg.Users {
-		if user.Name == "" {
-			log.Printf("User object has no 'name' field, skipping")
-			continue
-		}
-
-		if system.UserExists(&user) {
-			log.Printf("User '%s' exists, ignoring creation-time fields", user.Name)
-			if user.PasswordHash != "" {
-				log.Printf("Setting '%s' user's password", user.Name)
-				if err := system.SetUserPassword(user.Name, user.PasswordHash); err != nil {
-					log.Printf("Failed setting '%s' user's password: %v", user.Name, err)
-					return err
-				}
-			}
-		} else {
-			log.Printf("Creating user '%s'", user.Name)
-			if err := system.CreateUser(&user); err != nil {
-				log.Printf("Failed creating user '%s': %v", user.Name, err)
-				return err
-			}
-		}
-
-		if len(user.SSHAuthorizedKeys) > 0 {
-			log.Printf("Authorizing %d SSH keys for user '%s'", len(user.SSHAuthorizedKeys), user.Name)
-			if err := system.AuthorizeSSHKeys(user.Name, env.SSHKeyName(), user.SSHAuthorizedKeys); err != nil {
-				return err
-			}
-		}
-		if user.SSHImportGithubUser != "" {
-			log.Printf("Authorizing github user %s SSH keys for CoreOS user '%s'", user.SSHImportGithubUser, user.Name)
-			if err := SSHImportGithubUser(user.Name, user.SSHImportGithubUser); err != nil {
-				return err
-			}
-		}
-		for _, u := range user.SSHImportGithubUsers {
-			log.Printf("Authorizing github user %s SSH keys for CoreOS user '%s'", u, user.Name)
-			if err := SSHImportGithubUser(user.Name, u); err != nil {
-				return err
-			}
-		}
-		if user.SSHImportURL != "" {
-			log.Printf("Authorizing SSH keys for CoreOS user '%s' from '%s'", user.Name, user.SSHImportURL)
-			if err := SSHImportKeysFromURL(user.Name, user.SSHImportURL); err != nil {
-				return err
-			}
-		}
-	}
-
-	if len(cfg.SSHAuthorizedKeys) > 0 {
-		err := system.AuthorizeSSHKeys("core", env.SSHKeyName(), cfg.SSHAuthorizedKeys)
-		if err == nil {
-			log.Printf("Authorized SSH keys for core user")
-		} else {
-			return err
-		}
-	}
-
-	var writeFiles []system.File
-	for _, file := range cfg.WriteFiles {
-		writeFiles = append(writeFiles, system.File{File: file})
-	}
-
-	for _, ccf := range []CloudConfigFile{
-		system.OEM{OEM: cfg.CoreOS.OEM},
-		system.Update{Update: cfg.CoreOS.Update, ReadConfig: system.DefaultReadConfig},
-		system.EtcHosts{EtcHosts: cfg.ManageEtcHosts},
-		system.Flannel{Flannel: cfg.CoreOS.Flannel},
-	} {
-		f, err := ccf.File()
-		if err != nil {
-			return err
-		}
-		if f != nil {
-			writeFiles = append(writeFiles, *f)
-		}
-	}
-
-	var units []system.Unit
-	for _, u := range cfg.CoreOS.Units {
-		units = append(units, system.Unit{Unit: u})
-	}
-
-	for _, ccu := range []CloudConfigUnit{
-		system.Etcd{Etcd: cfg.CoreOS.Etcd},
-		system.Fleet{Fleet: cfg.CoreOS.Fleet},
-		system.Locksmith{Locksmith: cfg.CoreOS.Locksmith},
-		system.Update{Update: cfg.CoreOS.Update, ReadConfig: system.DefaultReadConfig},
-	} {
-		units = append(units, ccu.Units()...)
-	}
-
-	wroteEnvironment := false
-	for _, file := range writeFiles {
-		fullPath, err := system.WriteFile(&file, env.Root())
-		if err != nil {
-			return err
-		}
-		if path.Clean(file.Path) == "/etc/environment" {
-			wroteEnvironment = true
-		}
-		log.Printf("Wrote file %s to filesystem", fullPath)
-	}
-
-	if !wroteEnvironment {
-		ef := env.DefaultEnvironmentFile()
-		if ef != nil {
-			err := system.WriteEnvFile(ef, env.Root())
-			if err != nil {
-				return err
-			}
-			log.Printf("Updated /etc/environment")
-		}
-	}
-
-	if len(ifaces) > 0 {
-		units = append(units, createNetworkingUnits(ifaces)...)
-		if err := system.RestartNetwork(ifaces); err != nil {
-			return err
-		}
-	}
-
-	um := system.NewUnitManager(env.Root())
-	return processUnits(units, env.Root(), um)
-}
-
-func createNetworkingUnits(interfaces []network.InterfaceGenerator) (units []system.Unit) {
-	appendNewUnit := func(units []system.Unit, name, content string) []system.Unit {
-		if content == "" {
-			return units
-		}
-		return append(units, system.Unit{Unit: config.Unit{
-			Name:    name,
-			Runtime: true,
-			Content: content,
-		}})
-	}
-	for _, i := range interfaces {
-		units = appendNewUnit(units, fmt.Sprintf("%s.netdev", i.Filename()), i.Netdev())
-		units = appendNewUnit(units, fmt.Sprintf("%s.link", i.Filename()), i.Link())
-		units = appendNewUnit(units, fmt.Sprintf("%s.network", i.Filename()), i.Network())
-	}
-	return units
-}
-
-// processUnits takes a set of Units and applies them to the given root using
-// the given UnitManager. This can involve things like writing unit files to
-// disk, masking/unmasking units, or invoking systemd
-// commands against units. It returns any error encountered.
-func processUnits(units []system.Unit, root string, um system.UnitManager) error {
-	type action struct {
-		unit    system.Unit
-		command string
-	}
-	actions := make([]action, 0, len(units))
-	reload := false
-	restartNetworkd := false
-	for _, unit := range units {
-		if unit.Name == "" {
-			log.Printf("Skipping unit without name")
-			continue
-		}
-
-		if unit.Content != "" {
-			log.Printf("Writing unit %q to filesystem", unit.Name)
-			if err := um.PlaceUnit(unit); err != nil {
-				return err
-			}
-			log.Printf("Wrote unit %q", unit.Name)
-			reload = true
-		}
-
-		for _, dropin := range unit.DropIns {
-			if dropin.Name != "" && dropin.Content != "" {
-				log.Printf("Writing drop-in unit %q to filesystem", dropin.Name)
-				if err := um.PlaceUnitDropIn(unit, dropin); err != nil {
-					return err
-				}
-				log.Printf("Wrote drop-in unit %q", dropin.Name)
-				reload = true
-			}
-		}
-
-		if unit.Mask {
-			log.Printf("Masking unit file %q", unit.Name)
-			if err := um.MaskUnit(unit); err != nil {
-				return err
-			}
-		} else if unit.Runtime {
-			log.Printf("Ensuring runtime unit file %q is unmasked", unit.Name)
-			if err := um.UnmaskUnit(unit); err != nil {
-				return err
-			}
-		}
-
-		if unit.Enable {
-			if unit.Group() != "network" {
-				log.Printf("Enabling unit file %q", unit.Name)
-				if err := um.EnableUnitFile(unit); err != nil {
-					return err
-				}
-				log.Printf("Enabled unit %q", unit.Name)
-			} else {
-				log.Printf("Skipping enable for network-like unit %q", unit.Name)
-			}
-		}
-
-		if unit.Group() == "network" {
-			restartNetworkd = true
-		} else if unit.Command != "" {
-			actions = append(actions, action{unit, unit.Command})
-		}
-	}
-
-	if reload {
-		if err := um.DaemonReload(); err != nil {
-			return errors.New(fmt.Sprintf("failed systemd daemon-reload: %s", err))
-		}
-	}
-
-	if restartNetworkd {
-		log.Printf("Restarting systemd-networkd")
-		networkd := system.Unit{Unit: config.Unit{Name: "systemd-networkd.service"}}
-		res, err := um.RunUnitCommand(networkd, "restart")
-		if err != nil {
-			return err
-		}
-		log.Printf("Restarted systemd-networkd (%s)", res)
-	}
-
-	for _, action := range actions {
-		log.Printf("Calling unit command %q on %q'", action.command, action.unit.Name)
-		res, err := um.RunUnitCommand(action.unit, action.command)
-		if err != nil {
-			return err
-		}
-		log.Printf("Result of %q on %q: %s", action.command, action.unit.Name, res)
-	}
-
-	return nil
-}

Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/config_test.go

@@ -1,299 +0,0 @@
-// Copyright 2015 CoreOS, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package initialize
-
-import (
-	"reflect"
-	"testing"
-
-	"github.com/coreos/coreos-cloudinit/config"
-	"github.com/coreos/coreos-cloudinit/network"
-	"github.com/coreos/coreos-cloudinit/system"
-)
-
-type TestUnitManager struct {
-	placed   []string
-	enabled  []string
-	masked   []string
-	unmasked []string
-	commands []UnitAction
-	reload   bool
-}
-
-type UnitAction struct {
-	unit    string
-	command string
-}
-
-func (tum *TestUnitManager) PlaceUnit(u system.Unit) error {
-	tum.placed = append(tum.placed, u.Name)
-	return nil
-}
-func (tum *TestUnitManager) PlaceUnitDropIn(u system.Unit, d config.UnitDropIn) error {
-	tum.placed = append(tum.placed, u.Name+".d/"+d.Name)
-	return nil
-}
-func (tum *TestUnitManager) EnableUnitFile(u system.Unit) error {
-	tum.enabled = append(tum.enabled, u.Name)
-	return nil
-}
-func (tum *TestUnitManager) RunUnitCommand(u system.Unit, c string) (string, error) {
-	tum.commands = append(tum.commands, UnitAction{u.Name, c})
-	return "", nil
-}
-func (tum *TestUnitManager) DaemonReload() error {
-	tum.reload = true
-	return nil
-}
-func (tum *TestUnitManager) MaskUnit(u system.Unit) error {
-	tum.masked = append(tum.masked, u.Name)
-	return nil
-}
-func (tum *TestUnitManager) UnmaskUnit(u system.Unit) error {
-	tum.unmasked = append(tum.unmasked, u.Name)
-	return nil
-}
-
-type mockInterface struct {
-	name           string
-	filename       string
-	netdev         string
-	link           string
-	network        string
-	kind           string
-	modprobeParams string
-}
-
-func (i mockInterface) Name() string {
-	return i.name
-}
-
-func (i mockInterface) Filename() string {
-	return i.filename
-}
-
-func (i mockInterface) Netdev() string {
-	return i.netdev
-}
-
-func (i mockInterface) Link() string {
-	return i.link
-}
-
-func (i mockInterface) Network() string {
-	return i.network
-}
-
-func (i mockInterface) Type() string {
-	return i.kind
-}
-
-func (i mockInterface) ModprobeParams() string {
-	return i.modprobeParams
-}
-
-func TestCreateNetworkingUnits(t *testing.T) {
-	for _, tt := range []struct {
-		interfaces []network.InterfaceGenerator
-		expect     []system.Unit
-	}{
-		{nil, nil},
-		{
-			[]network.InterfaceGenerator{
-				network.InterfaceGenerator(mockInterface{filename: "test"}),
-			},
-			nil,
-		},
-		{
-			[]network.InterfaceGenerator{
-				network.InterfaceGenerator(mockInterface{filename: "test1", netdev: "test netdev"}),
-				network.InterfaceGenerator(mockInterface{filename: "test2", link: "test link"}),
-				network.InterfaceGenerator(mockInterface{filename: "test3", network: "test network"}),
-			},
-			[]system.Unit{
-				system.Unit{Unit: config.Unit{Name: "test1.netdev", Runtime: true, Content: "test netdev"}},
-				system.Unit{Unit: config.Unit{Name: "test2.link", Runtime: true, Content: "test link"}},
-				system.Unit{Unit: config.Unit{Name: "test3.network", Runtime: true, Content: "test network"}},
-			},
-		},
-		{
-			[]network.InterfaceGenerator{
-				network.InterfaceGenerator(mockInterface{filename: "test", netdev: "test netdev", link: "test link", network: "test network"}),
-			},
-			[]system.Unit{
-				system.Unit{Unit: config.Unit{Name: "test.netdev", Runtime: true, Content: "test netdev"}},
-				system.Unit{Unit: config.Unit{Name: "test.link", Runtime: true, Content: "test link"}},
-				system.Unit{Unit: config.Unit{Name: "test.network", Runtime: true, Content: "test network"}},
-			},
-		},
-	} {
-		units := createNetworkingUnits(tt.interfaces)
-		if !reflect.DeepEqual(tt.expect, units) {
-			t.Errorf("bad units (%+v): want %#v, got %#v", tt.interfaces, tt.expect, units)
-		}
-	}
-}
-
-func TestProcessUnits(t *testing.T) {
-	tests := []struct {
-		units []system.Unit
-
-		result TestUnitManager
-	}{
-		{
-			units: []system.Unit{
-				system.Unit{Unit: config.Unit{
-					Name: "foo",
-					Mask: true,
-				}},
-			},
-			result: TestUnitManager{
-				masked: []string{"foo"},
-			},
-		},
-		{
-			units: []system.Unit{
-				system.Unit{Unit: config.Unit{
-					Name:    "baz.service",
-					Content: "[Service]\nExecStart=/bin/baz",
-					Command: "start",
-				}},
-				system.Unit{Unit: config.Unit{
-					Name:    "foo.network",
-					Content: "[Network]\nFoo=true",
-				}},
-				system.Unit{Unit: config.Unit{
-					Name:    "bar.network",
-					Content: "[Network]\nBar=true",
-				}},
-			},
-			result: TestUnitManager{
-				placed: []string{"baz.service", "foo.network", "bar.network"},
-				commands: []UnitAction{
-					UnitAction{"systemd-networkd.service", "restart"},
-					UnitAction{"baz.service", "start"},
-				},
-				reload: true,
-			},
-		},
-		{
-			units: []system.Unit{
-				system.Unit{Unit: config.Unit{
-					Name:    "baz.service",
-					Content: "[Service]\nExecStart=/bin/true",
-				}},
-			},
-			result: TestUnitManager{
-				placed: []string{"baz.service"},
-				reload: true,
-			},
-		},
-		{
-			units: []system.Unit{
-				system.Unit{Unit: config.Unit{
-					Name:    "locksmithd.service",
-					Runtime: true,
-				}},
-			},
-			result: TestUnitManager{
-				unmasked: []string{"locksmithd.service"},
-			},
-		},
-		{
-			units: []system.Unit{
-				system.Unit{Unit: config.Unit{
-					Name:   "woof",
-					Enable: true,
-				}},
-			},
-			result: TestUnitManager{
-				enabled: []string{"woof"},
-			},
-		},
-		{
-			units: []system.Unit{
-				system.Unit{Unit: config.Unit{
-					Name:    "hi.service",
-					Runtime: true,
-					Content: "[Service]\nExecStart=/bin/echo hi",
-					DropIns: []config.UnitDropIn{
-						{
-							Name:    "lo.conf",
-							Content: "[Service]\nExecStart=/bin/echo lo",
-						},
-						{
-							Name:    "bye.conf",
-							Content: "[Service]\nExecStart=/bin/echo bye",
-						},
-					},
-				}},
-			},
-			result: TestUnitManager{
-				placed:   []string{"hi.service", "hi.service.d/lo.conf", "hi.service.d/bye.conf"},
-				unmasked: []string{"hi.service"},
-				reload:   true,
-			},
-		},
-		{
-			units: []system.Unit{
-				system.Unit{Unit: config.Unit{
-					DropIns: []config.UnitDropIn{
-						{
-							Name:    "lo.conf",
-							Content: "[Service]\nExecStart=/bin/echo lo",
-						},
-					},
-				}},
-			},
-			result: TestUnitManager{},
-		},
-		{
-			units: []system.Unit{
-				system.Unit{Unit: config.Unit{
-					Name: "hi.service",
-					DropIns: []config.UnitDropIn{
-						{
-							Content: "[Service]\nExecStart=/bin/echo lo",
-						},
-					},
-				}},
-			},
-			result: TestUnitManager{},
-		},
-		{
-			units: []system.Unit{
-				system.Unit{Unit: config.Unit{
-					Name: "hi.service",
-					DropIns: []config.UnitDropIn{
-						{
-							Name: "lo.conf",
-						},
-					},
-				}},
-			},
-			result: TestUnitManager{},
-		},
-	}
-
-	for _, tt := range tests {
-		tum := &TestUnitManager{}
-		if err := processUnits(tt.units, "", tum); err != nil {
-			t.Errorf("bad error (%+v): want nil, got %s", tt.units, err)
-		}
-		if !reflect.DeepEqual(tt.result, *tum) {
-			t.Errorf("bad result (%+v): want %+v, got %+v", tt.units, tt.result, tum)
-		}
-	}
-}

Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/env.go

@@ -1,116 +0,0 @@
-// Copyright 2015 CoreOS, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package initialize
-
-import (
-	"net"
-	"os"
-	"path"
-	"regexp"
-	"strings"
-
-	"github.com/coreos/coreos-cloudinit/config"
-	"github.com/coreos/coreos-cloudinit/datasource"
-	"github.com/coreos/coreos-cloudinit/system"
-)
-
-const DefaultSSHKeyName = "coreos-cloudinit"
-
-type Environment struct {
-	root          string
-	configRoot    string
-	workspace     string
-	sshKeyName    string
-	substitutions map[string]string
-}
-
-// TODO(jonboulle): this is getting unwieldy, should be able to simplify the interface somehow
-func NewEnvironment(root, configRoot, workspace, sshKeyName string, metadata datasource.Metadata) *Environment {
-	firstNonNull := func(ip net.IP, env string) string {
-		if ip == nil {
-			return env
-		}
-		return ip.String()
-	}
-	substitutions := map[string]string{
-		"$public_ipv4":  firstNonNull(metadata.PublicIPv4, os.Getenv("COREOS_PUBLIC_IPV4")),
-		"$private_ipv4": firstNonNull(metadata.PrivateIPv4, os.Getenv("COREOS_PRIVATE_IPV4")),
-		"$public_ipv6":  firstNonNull(metadata.PublicIPv6, os.Getenv("COREOS_PUBLIC_IPV6")),
-		"$private_ipv6": firstNonNull(metadata.PrivateIPv6, os.Getenv("COREOS_PRIVATE_IPV6")),
-	}
-	return &Environment{root, configRoot, workspace, sshKeyName, substitutions}
-}
-
-func (e *Environment) Workspace() string {
-	return path.Join(e.root, e.workspace)
-}
-
-func (e *Environment) Root() string {
-	return e.root
-}
-
-func (e *Environment) ConfigRoot() string {
-	return e.configRoot
-}
-
-func (e *Environment) SSHKeyName() string {
-	return e.sshKeyName
-}
-
-func (e *Environment) SetSSHKeyName(name string) {
-	e.sshKeyName = name
-}
-
-// Apply goes through the map of substitutions and replaces all instances of
-// the keys with their respective values. It supports escaping substitutions
-// with a leading '\'.
-func (e *Environment) Apply(data string) string {
-	for key, val := range e.substitutions {
-		matchKey := strings.Replace(key, `$`, `\$`, -1)
-		replKey := strings.Replace(key, `$`, `$$`, -1)
-
-		// "key" -> "val"
-		data = regexp.MustCompile(`([^\\]|^)`+matchKey).ReplaceAllString(data, `${1}`+val)
-		// "\key" -> "key"
-		data = regexp.MustCompile(`\\`+matchKey).ReplaceAllString(data, replKey)
-	}
-	return data
-}
-
-func (e *Environment) DefaultEnvironmentFile() *system.EnvFile {
-	ef := system.EnvFile{
-		File: &system.File{File: config.File{
-			Path: "/etc/environment",
-		}},
-		Vars: map[string]string{},
-	}
-	if ip, ok := e.substitutions["$public_ipv4"]; ok && len(ip) > 0 {
-		ef.Vars["COREOS_PUBLIC_IPV4"] = ip
-	}
-	if ip, ok := e.substitutions["$private_ipv4"]; ok && len(ip) > 0 {
-		ef.Vars["COREOS_PRIVATE_IPV4"] = ip
-	}
-	if ip, ok := e.substitutions["$public_ipv6"]; ok && len(ip) > 0 {
-		ef.Vars["COREOS_PUBLIC_IPV6"] = ip
-	}
-	if ip, ok := e.substitutions["$private_ipv6"]; ok && len(ip) > 0 {
-		ef.Vars["COREOS_PRIVATE_IPV6"] = ip
-	}
-	if len(ef.Vars) == 0 {
-		return nil
-	} else {
-		return &ef
-	}
-}

Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/env_test.go

@@ -1,148 +0,0 @@
-// Copyright 2015 CoreOS, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package initialize
-
-import (
-	"io/ioutil"
-	"net"
-	"os"
-	"path"
-	"testing"
-
-	"github.com/coreos/coreos-cloudinit/datasource"
-	"github.com/coreos/coreos-cloudinit/system"
-)
-
-func TestEnvironmentApply(t *testing.T) {
-	os.Setenv("COREOS_PUBLIC_IPV4", "1.2.3.4")
-	os.Setenv("COREOS_PRIVATE_IPV4", "5.6.7.8")
-	os.Setenv("COREOS_PUBLIC_IPV6", "1234::")
-	os.Setenv("COREOS_PRIVATE_IPV6", "5678::")
-	for _, tt := range []struct {
-		metadata datasource.Metadata
-		input    string
-		out      string
-	}{
-		{
-			// Substituting both values directly should always take precedence
-			// over environment variables
-			datasource.Metadata{
-				PublicIPv4:  net.ParseIP("192.0.2.3"),
-				PrivateIPv4: net.ParseIP("192.0.2.203"),
-				PublicIPv6:  net.ParseIP("fe00:1234::"),
-				PrivateIPv6: net.ParseIP("fe00:5678::"),
-			},
-			`[Service]
-ExecStart=/usr/bin/echo "$public_ipv4 $public_ipv6"
-ExecStop=/usr/bin/echo $private_ipv4 $private_ipv6
-ExecStop=/usr/bin/echo $unknown`,
-			`[Service]
-ExecStart=/usr/bin/echo "192.0.2.3 fe00:1234::"
-ExecStop=/usr/bin/echo 192.0.2.203 fe00:5678::
-ExecStop=/usr/bin/echo $unknown`,
-		},
-		{
-			// Substituting one value directly while falling back with the other
-			datasource.Metadata{
-				PrivateIPv4: net.ParseIP("127.0.0.1"),
-			},
-			"$private_ipv4\n$public_ipv4",
-			"127.0.0.1\n1.2.3.4",
-		},
-		{
-			// Falling back to environment variables for both values
-			datasource.Metadata{},
-			"$private_ipv4\n$public_ipv4",
-			"5.6.7.8\n1.2.3.4",
-		},
-		{
-			// No substitutions
-			datasource.Metadata{},
-			"$private_ipv4\nfoobar",
-			"5.6.7.8\nfoobar",
-		},
-		{
-			// Escaping substitutions
-			datasource.Metadata{
-				PrivateIPv4: net.ParseIP("127.0.0.1"),
-			},
-			`\$private_ipv4
-$private_ipv4
-addr: \$private_ipv4
-\\$private_ipv4`,
-			`$private_ipv4
-127.0.0.1
-addr: $private_ipv4
-\$private_ipv4`,
-		},
-		{
-			// No substitutions with escaping
-			datasource.Metadata{},
-			"\\$test\n$test",
-			"\\$test\n$test",
-		},
-	} {
-
-		env := NewEnvironment("./", "./", "./", "", tt.metadata)
-		got := env.Apply(tt.input)
-		if got != tt.out {
-			t.Fatalf("Environment incorrectly applied.\ngot:\n%s\nwant:\n%s", got, tt.out)
-		}
-	}
-}
-
-func TestEnvironmentFile(t *testing.T) {
-	metadata := datasource.Metadata{
-		PublicIPv4:  net.ParseIP("1.2.3.4"),
-		PrivateIPv4: net.ParseIP("5.6.7.8"),
-		PublicIPv6:  net.ParseIP("1234::"),
-		PrivateIPv6: net.ParseIP("5678::"),
-	}
-	expect := "COREOS_PRIVATE_IPV4=5.6.7.8\nCOREOS_PRIVATE_IPV6=5678::\nCOREOS_PUBLIC_IPV4=1.2.3.4\nCOREOS_PUBLIC_IPV6=1234::\n"
-
-	dir, err := ioutil.TempDir(os.TempDir(), "coreos-cloudinit-")
-	if err != nil {
-		t.Fatalf("Unable to create tempdir: %v", err)
-	}
-	defer os.RemoveAll(dir)
-
-	env := NewEnvironment("./", "./", "./", "", metadata)
-	ef := env.DefaultEnvironmentFile()
-	err = system.WriteEnvFile(ef, dir)
-	if err != nil {
-		t.Fatalf("WriteEnvFile failed: %v", err)
-	}
-
-	fullPath := path.Join(dir, "etc", "environment")
-	contents, err := ioutil.ReadFile(fullPath)
-	if err != nil {
-		t.Fatalf("Unable to read expected file: %v", err)
-	}
-
-	if string(contents) != expect {
-		t.Fatalf("File has incorrect contents: %q", contents)
-	}
-}
-
-func TestEnvironmentFileNil(t *testing.T) {
-	os.Clearenv()
-	metadata := datasource.Metadata{}
-
-	env := NewEnvironment("./", "./", "./", "", metadata)
-	ef := env.DefaultEnvironmentFile()
-	if ef != nil {
-		t.Fatalf("Environment file not nil: %v", ef)
-	}
-}

Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/github.go

@@ -1,32 +0,0 @@
-// Copyright 2015 CoreOS, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package initialize
-
-import (
-	"fmt"
-
-	"github.com/coreos/coreos-cloudinit/system"
-)
-
-func SSHImportGithubUser(system_user string, github_user string) error {
-	url := fmt.Sprintf("https://api.github.com/users/%s/keys", github_user)
-	keys, err := fetchUserKeys(url)
-	if err != nil {
-		return err
-	}
-
-	key_name := fmt.Sprintf("github-%s", github_user)
-	return system.AuthorizeSSHKeys(system_user, key_name, keys)
-}

Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/ssh_keys.go

@@ -1,57 +0,0 @@
-// Copyright 2015 CoreOS, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package initialize
-
-import (
-	"encoding/json"
-	"fmt"
-
-	"github.com/coreos/coreos-cloudinit/pkg"
-	"github.com/coreos/coreos-cloudinit/system"
-)
-
-type UserKey struct {
-	ID  int    `json:"id,omitempty"`
-	Key string `json:"key"`
-}
-
-func SSHImportKeysFromURL(system_user string, url string) error {
-	keys, err := fetchUserKeys(url)
-	if err != nil {
-		return err
-	}
-
-	key_name := fmt.Sprintf("coreos-cloudinit-%s", system_user)
-	return system.AuthorizeSSHKeys(system_user, key_name, keys)
-}
-
-func fetchUserKeys(url string) ([]string, error) {
-	client := pkg.NewHttpClient()
-	data, err := client.GetRetry(url)
-	if err != nil {
-		return nil, err
-	}
-
-	var userKeys []UserKey
-	err = json.Unmarshal(data, &userKeys)
-	if err != nil {
-		return nil, err
-	}
-	keys := make([]string, 0)
-	for _, key := range userKeys {
-		keys = append(keys, key.Key)
-	}
-	return keys, err
-}

Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/ssh_keys_test.go

@@ -1,56 +0,0 @@
-// Copyright 2015 CoreOS, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package initialize
-
-import (
-	"fmt"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-)
-
-func TestCloudConfigUsersUrlMarshal(t *testing.T) {
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		gh_res := `
-[
-  {
-    "key": "ssh-dss AAAAB3NzaC1kc3MAAACBAIHAu822ggSkIHrJYvhmBceOSVjuflfQm8RbMMDNVe9relQfuPbN+nxGGTCKzPLebeOcX+Wwi77TPXWwK3BZMglfXxhABlFPsuMb63Tqp94pBYsJdx/iFj9iGo6pKoM1k8ubOcqsUnq+BR9895zRbE7MjdwkGo67+QhCEwvkwAnNAAAAFQCuddVqXLCubzqnWmeHLQE+2GFfHwAAAIBnlXW5h15ndVuwi0htF4oodVSB1KwnTWcuBK+aE1zRs76yvRb0Ws+oifumThDwB/Tec6FQuAfRKfy6piChZqsu5KvL98I+2t5yyi1td+kMvdTnVL2lW44etDKseOcozmknCOmh4Dqvhl/2MwrDAhlPaN08EEq9h3w3mXtNLWH64QAAAIBAzDOKr17llngaKIdDXh+LtXKh87+zfjlTA36/9r2uF2kYE5uApDtu9sPCkt7+YBQt7R8prADPckwAiXwVdk0xijIOpLDBmoydQJJRQ+zTMxvpQmUr/1kUOv0zb+lB657CgvN0vVTmP2swPeMvgntt3C4vw7Ab+O+MS9peOAJbbQ=="
-  },
-  {
-    "key": "ssh-dss AAAAB3NzaC1kc3MAAACBANxpzIbTzKTeBRaOIdUxwwGwvDasTfU/PonhbNIuhYjc+xFGvBRTumox2F+luVAKKs4WdvA4nJXaY1OFi6DZftk5Bp4E2JaSzp8ulAzHsMexDdv6LGHGEJj/qdHAL1vHk2K89PpwRFSRZI8XRBLjvkr4ZgBKLG5ZILXPJEPP2j3lAAAAFQCtxoTnV8wy0c4grcGrQ+1sCsD7WQAAAIAqZsW2GviMe1RQrbZT0xAZmI64XRPrnLsoLxycHWlS7r6uUln2c6Ae2MB/YF0d4Kd1XZii9GHj7rrypqEo7MW8uSabhu70nmu1J8m2O3Dsr+4oJLeat9vwPsJV92IKO0jQwjKnAOHOiB9JKGeCw+NfXfogbti9/q38Q6XcS+SI5wAAAIEA1803Y2h+tOOpZXAsNIwl9mRfExWzLQ3L7knwJdznQu/6SW1H/1oyoYLebuk187Qj2UFI5qQ6AZNc49DvohWx0Cg6ABcyubNyoaCjZKWIdxVnItHWNbLe//+tyTu0I2eQwJOORsEPK5gMpf599C7wXQ//DzZOWbTWiHEX52gCTmk="
-  },
-  {
-    "id": 5224438,
-    "key": "ssh-dss AAAAB3NzaC1kc3MAAACBAPKRWdKhzGZuLAJL6M1eM51hWViMqNBC2C6lm2OqGRYLuIf1GJ391widUuSf4wQqnkR22Q9PCmAZ19XCf11wBRMnuw9I/Z3Bt5bXfc+dzFBCmHYGJ6wNSv++H9jxyMb+usmsenWOFZGNO2jN0wrJ4ay8Yt0bwtRU+VCXpuRLszMzAAAAFQDZUIuPjcfK5HLgnwZ/J3lvtvlUjQAAAIEApIkAwLuCQV5j3U6DmI/Y6oELqSUR2purFm8jo8jePFfe1t+ghikgD254/JXlhDCVgY0NLXcak+coJfGCTT23quJ7I5xdpTn/OZO2Q6Woum/bijFC/UWwQbLz0R2nU3DoHv5v6XHQZxuIG4Fsxa91S+vWjZFtI7RuYlBCZA//ANMAAACBAJO0FojzkX6IeaWLqrgu9GTkFwGFazZ+LPH5JOWPoPn1hQKuR32Uf6qNcBZcIjY7SF0P7HF5rLQd6zKZzHqqQQ92MV555NEwjsnJglYU8CaaZsfYooaGPgA1YN7RhTSAuDmUW5Hyfj5BH4NTtrzrvJxIhDoQLf31Fasjw00r4R0O"
-  }
-]
-`
-		fmt.Fprintln(w, gh_res)
-	}))
-	defer ts.Close()
-
-	keys, err := fetchUserKeys(ts.URL)
-	if err != nil {
-		t.Fatalf("Encountered unexpected error: %v", err)
-	}
-	expected := "ssh-dss AAAAB3NzaC1kc3MAAACBAIHAu822ggSkIHrJYvhmBceOSVjuflfQm8RbMMDNVe9relQfuPbN+nxGGTCKzPLebeOcX+Wwi77TPXWwK3BZMglfXxhABlFPsuMb63Tqp94pBYsJdx/iFj9iGo6pKoM1k8ubOcqsUnq+BR9895zRbE7MjdwkGo67+QhCEwvkwAnNAAAAFQCuddVqXLCubzqnWmeHLQE+2GFfHwAAAIBnlXW5h15ndVuwi0htF4oodVSB1KwnTWcuBK+aE1zRs76yvRb0Ws+oifumThDwB/Tec6FQuAfRKfy6piChZqsu5KvL98I+2t5yyi1td+kMvdTnVL2lW44etDKseOcozmknCOmh4Dqvhl/2MwrDAhlPaN08EEq9h3w3mXtNLWH64QAAAIBAzDOKr17llngaKIdDXh+LtXKh87+zfjlTA36/9r2uF2kYE5uApDtu9sPCkt7+YBQt7R8prADPckwAiXwVdk0xijIOpLDBmoydQJJRQ+zTMxvpQmUr/1kUOv0zb+lB657CgvN0vVTmP2swPeMvgntt3C4vw7Ab+O+MS9peOAJbbQ=="
-	if keys[0] != expected {
-		t.Fatalf("expected %s, got %s", expected, keys[0])
-	}
-	expected = "ssh-dss AAAAB3NzaC1kc3MAAACBAPKRWdKhzGZuLAJL6M1eM51hWViMqNBC2C6lm2OqGRYLuIf1GJ391widUuSf4wQqnkR22Q9PCmAZ19XCf11wBRMnuw9I/Z3Bt5bXfc+dzFBCmHYGJ6wNSv++H9jxyMb+usmsenWOFZGNO2jN0wrJ4ay8Yt0bwtRU+VCXpuRLszMzAAAAFQDZUIuPjcfK5HLgnwZ/J3lvtvlUjQAAAIEApIkAwLuCQV5j3U6DmI/Y6oELqSUR2purFm8jo8jePFfe1t+ghikgD254/JXlhDCVgY0NLXcak+coJfGCTT23quJ7I5xdpTn/OZO2Q6Woum/bijFC/UWwQbLz0R2nU3DoHv5v6XHQZxuIG4Fsxa91S+vWjZFtI7RuYlBCZA//ANMAAACBAJO0FojzkX6IeaWLqrgu9GTkFwGFazZ+LPH5JOWPoPn1hQKuR32Uf6qNcBZcIjY7SF0P7HF5rLQd6zKZzHqqQQ92MV555NEwjsnJglYU8CaaZsfYooaGPgA1YN7RhTSAuDmUW5Hyfj5BH4NTtrzrvJxIhDoQLf31Fasjw00r4R0O"
-	if keys[2] != expected {
-		t.Fatalf("expected %s, got %s", expected, keys[2])
-	}
-}

Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/user_data.go

@@ -1,39 +0,0 @@
-// Copyright 2015 CoreOS, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package initialize
-
-import (
-	"errors"
-	"log"
-
-	"github.com/coreos/coreos-cloudinit/config"
-)
-
-func ParseUserData(contents string) (interface{}, error) {
-	if len(contents) == 0 {
-		return nil, nil
-	}
-
-	switch {
-	case config.IsScript(contents):
-		log.Printf("Parsing user-data as script")
-		return config.NewScript(contents)
-	case config.IsCloudConfig(contents):
-		log.Printf("Parsing user-data as cloud-config")
-		return config.NewCloudConfig(contents)
-	default:
-		return nil, errors.New("Unrecognized user-data format")
-	}
-}

Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/user_data_test.go

@@ -1,74 +0,0 @@
-// Copyright 2015 CoreOS, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package initialize
-
-import (
-	"testing"
-
-	"github.com/coreos/coreos-cloudinit/config"
-)
-
-func TestParseHeaderCRLF(t *testing.T) {
-	configs := []string{
-		"#cloud-config\nfoo: bar",
-		"#cloud-config\r\nfoo: bar",
-	}
-
-	for i, config := range configs {
-		_, err := ParseUserData(config)
-		if err != nil {
-			t.Errorf("Failed parsing config %d: %v", i, err)
-		}
-	}
-
-	scripts := []string{
-		"#!bin/bash\necho foo",
-		"#!bin/bash\r\necho foo",
-	}
-
-	for i, script := range scripts {
-		_, err := ParseUserData(script)
-		if err != nil {
-			t.Errorf("Failed parsing script %d: %v", i, err)
-		}
-	}
-}
-
-func TestParseConfigCRLF(t *testing.T) {
-	contents := "#cloud-config\r\nhostname: foo\r\nssh_authorized_keys:\r\n  - foobar\r\n"
-	ud, err := ParseUserData(contents)
-	if err != nil {
-		t.Fatalf("Failed parsing config: %v", err)
-	}
-
-	cfg := ud.(*config.CloudConfig)
-
-	if cfg.Hostname != "foo" {
-		t.Error("Failed parsing hostname from config")
-	}
-
-	if len(cfg.SSHAuthorizedKeys) != 1 {
-		t.Error("Parsed incorrect number of SSH keys")
-	}
-}
-
-func TestParseConfigEmpty(t *testing.T) {
-	i, e := ParseUserData(``)
-	if i != nil {
-		t.Error("ParseUserData of empty string returned non-nil unexpectedly")
-	} else if e != nil {
-		t.Error("ParseUserData of empty string returned error unexpectedly")
-	}
-}

Godeps/_workspace/src/github.com/coreos/coreos-cloudinit/initialize/workspace.go

@@ -1,66 +0,0 @@
-// Copyright 2015 CoreOS, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package initialize
-
-import (
-	"io/ioutil"
-	"path"
-	"strings"
-
-	"github.com/coreos/coreos-cloudinit/config"
-	"github.com/coreos/coreos-cloudinit/system"
-)
-
-func PrepWorkspace(workspace string) error {
-	if err := system.EnsureDirectoryExists(workspace); err != nil {
-		return err
-	}
-
-	scripts := path.Join(workspace, "scripts")
-	if err := system.EnsureDirectoryExists(scripts); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func PersistScriptInWorkspace(script config.Script, workspace string) (string, error) {
-	scriptsPath := path.Join(workspace, "scripts")
-	tmp, err := ioutil.TempFile(scriptsPath, "")
-	if err != nil {
-		return "", err
-	}
-	tmp.Close()
-
-	relpath := strings.TrimPrefix(tmp.Name(), workspace)
-
-	file := system.File{File: config.File{
-		Path:               relpath,
-		RawFilePermissions: "0744",
-		Content:            string(script),
-	}}
-
-	return system.WriteFile(&file, workspace)
-}
-
-func PersistUnitNameInWorkspace(name string, workspace string) error {
-	file := system.File{File: config.File{
-		Path:               path.Join("scripts", "unit-name"),
-		RawFilePermissions: "0644",
-		Content:            name,
-	}}
-	_, err := system.WriteFile(&file, workspace)
-	return err
-}

Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/project.go

@@ -18,6 +18,12 @@ var (
 	ErrRestart error        = errors.New("Restart execution")
 )
 
+type ProjectEvent struct {
+	Event   Event
+	Service Service
+	Data    map[string]string
+}
+
 func NewProject(name string, factory ServiceFactory) *Project {
 	return &Project{
 		Name:     name,
@@ -27,8 +33,29 @@ func NewProject(name string, factory ServiceFactory) *Project {
 	}
 }
 
+func (p *Project) createService(name string, config ServiceConfig) (Service, error) {
+	if p.EnvironmentLookup != nil {
+		parsedEnv := make([]string, 0, len(config.Environment))
+
+		for _, env := range config.Environment {
+			if strings.IndexRune(env, '=') != -1 {
+				parsedEnv = append(parsedEnv, env)
+				continue
+			}
+
+			for _, value := range p.EnvironmentLookup.Lookup(env, name, &config) {
+				parsedEnv = append(parsedEnv, value)
+			}
+		}
+
+		config.Environment = parsedEnv
+	}
+
+	return p.factory.Create(p, name, &config)
+}
+
 func (p *Project) AddConfig(name string, config *ServiceConfig) error {
-	service, err := p.factory.Create(p, name, config)
+	service, err := p.createService(name, *config)
 	if err != nil {
 		log.Errorf("Failed to create service for %s : %v", name, err)
 		return err
@@ -36,6 +63,7 @@ func (p *Project) AddConfig(name string, config *ServiceConfig) error {
 
 	p.Notify(SERVICE_ADD, service, nil)
 
+	p.reload = append(p.reload, name)
 	p.configs[name] = config
 	p.Services[name] = service
 
@@ -62,16 +90,18 @@ func (p *Project) Load(bytes []byte) error {
 func (p *Project) Up() error {
 	wrappers := make(map[string]*ServiceWrapper)
 
-	for name, _ := range p.Services {
-		wrappers[name] = NewServiceWrapper(name, p)
-	}
-
 	p.Notify(PROJECT_UP_START, nil, nil)
 
 	return p.startAll(wrappers)
 }
 
 func (p *Project) startAll(wrappers map[string]*ServiceWrapper) error {
+	for _, name := range p.reload {
+		wrappers[name] = NewServiceWrapper(name, p)
+	}
+
+	p.reload = []string{}
+
 	restart := false
 
 	for _, wrapper := range wrappers {
@@ -176,6 +206,10 @@ func (s *ServiceWrapper) Wait() error {
 	return s.err
 }
 
+func (p *Project) AddListener(c chan<- ProjectEvent) {
+	p.listeners = append(p.listeners, c)
+}
+
 func (p *Project) Notify(event Event, service Service, data map[string]string) {
 	buffer := bytes.NewBuffer(nil)
 	if data != nil {
@@ -204,4 +238,17 @@ func (p *Project) Notify(event Event, service Service, data map[string]string) {
 	} else {
 		logf("[%d/%d] [%s]: %s %s", p.upCount, len(p.Services), service.Name(), event, buffer.Bytes())
 	}
+
+	for _, l := range p.listeners {
+		projectEvent := ProjectEvent{
+			Event:   event,
+			Service: service,
+			Data:    data,
+		}
+		// Don't ever block
+		select {
+		case l <- projectEvent:
+		default:
+		}
+	}
 }

Godeps/_workspace/src/github.com/rancherio/rancher-compose/project/types.go

@@ -7,8 +7,9 @@ type Event string
 const (
 	CONTAINER_ID = "container_id"
 
-	CONTAINER_CREATED = Event("Created container")
+	CONTAINER_STARTING = Event("Starting container")
-	CONTAINER_STARTED = Event("Started container")
+	CONTAINER_CREATED  = Event("Created container")
+	CONTAINER_STARTED  = Event("Started container")
 
 	SERVICE_ADD      = Event("Adding")
 	SERVICE_UP_START = Event("Starting")
@@ -56,16 +57,23 @@ type ServiceConfig struct {
 	ExternalLinks []string `yaml:"external_links,omitempty"`
 }
 
+type EnvironmentLookup interface {
+	Lookup(key, serviceName string, config *ServiceConfig) []string
+}
+
 type Project struct {
-	Name           string
+	EnvironmentLookup EnvironmentLookup
-	configs        map[string]*ServiceConfig
+	Name              string
-	Services       map[string]Service
+	configs           map[string]*ServiceConfig
-	file           string
+	reload            []string
-	content        []byte
+	Services          map[string]Service
-	client         *client.RancherClient
+	file              string
-	factory        ServiceFactory
+	content           []byte
-	ReloadCallback func() error
+	client            *client.RancherClient
-	upCount        int
+	factory           ServiceFactory
+	ReloadCallback    func() error
+	upCount           int
+	listeners         []chan<- ProjectEvent
 }
 
 type Service interface {

cmd/cloudinit/cloudinit.go

@@ -33,11 +33,11 @@ import (
 	"github.com/coreos/coreos-cloudinit/datasource/metadata/ec2"
 	"github.com/coreos/coreos-cloudinit/datasource/proc_cmdline"
 	"github.com/coreos/coreos-cloudinit/datasource/url"
-	"github.com/coreos/coreos-cloudinit/initialize"
 	"github.com/coreos/coreos-cloudinit/pkg"
 	"github.com/coreos/coreos-cloudinit/system"
 	rancherNetwork "github.com/rancherio/os/cmd/network"
 	rancherConfig "github.com/rancherio/os/config"
+	"github.com/rancherio/os/util"
 	"gopkg.in/yaml.v2"
 )
 
@@ -48,21 +48,25 @@ const (
 )
 
 var (
-	outputDir  string
+	baseConfigDir string
-	outputFile string
+	outputDir     string
-	scriptFile string
+	outputFile    string
-	rancherYml string
+	metaDataFile  string
-	save       bool
+	scriptFile    string
-	execute    bool
+	rancherYml    string
-	network    bool
+	save          bool
-	sshKeyName string
+	execute       bool
-	flags      *flag.FlagSet
+	network       bool
+	sshKeyName    string
+	flags         *flag.FlagSet
 )
 
 func init() {
 	flags = flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
+	flags.StringVar(&baseConfigDir, "base-config-dir", "/var/lib/rancher/conf/cloud-config.d", "base cloud config")
 	flags.StringVar(&outputDir, "dir", "/var/lib/rancher/conf", "working directory")
 	flags.StringVar(&outputFile, "file", "cloud-config-processed.yml", "output cloud config file name")
+	flags.StringVar(&metaDataFile, "metadata", "metadata", "output metdata file name")
 	flags.StringVar(&scriptFile, "script-file", "cloud-config-script", "output cloud config script file name")
 	flags.StringVar(&rancherYml, "rancher", "cloud-config-rancher.yml", "output cloud config rancher file name")
 	flags.StringVar(&sshKeyName, "ssh-key-name", "rancheros-cloud-config", "SSH key name")
@@ -71,61 +75,61 @@ func init() {
 	flags.BoolVar(&execute, "execute", false, "execute saved cloud config")
 }
 
-func saveFiles(script *config.Script, userdataBytes []byte, cc *config.CloudConfig) error {
+func saveFiles(cloudConfigBytes, scriptBytes []byte, metadata datasource.Metadata) error {
-	var fileData []byte
+	scriptOutput := path.Join(outputDir, scriptFile)
+	cloudConfigOutput := path.Join(outputDir, outputFile)
+	rancherYmlOutput := path.Join(outputDir, rancherYml)
+	metaDataOutput := path.Join(outputDir, metaDataFile)
 
-	os.Remove(path.Join(outputDir, scriptFile))
+	os.Remove(scriptOutput)
-	os.Remove(path.Join(outputDir, outputFile))
+	os.Remove(cloudConfigOutput)
-	os.Remove(path.Join(outputDir, rancherYml))
+	os.Remove(rancherYmlOutput)
+	os.Remove(metaDataOutput)
 
-	if script != nil {
+	if len(scriptBytes) > 0 {
-		fileData = userdataBytes
+		log.Infof("Writing to %s", scriptOutput)
-		output := path.Join(outputDir, scriptFile)
+		if err := ioutil.WriteFile(scriptOutput, scriptBytes, 500); err != nil {
-		log.Infof("Writing cloud-config script to %s", output)
+			log.Errorf("Error while writing file %s: %v", scriptOutput, err)
-		if err := ioutil.WriteFile(output, fileData, 500); err != nil {
-			log.Errorf("Error while writing file %v", err)
 			return err
 		}
 	}
 
-	if data, err := yaml.Marshal(cc); err != nil {
+	cloudConfigBytes = append([]byte("#cloud-config\n"), cloudConfigBytes...)
-		log.Errorf("Error while marshalling cloud config %v", err)
+	log.Infof("Writing to %s", cloudConfigOutput)
-		return err
+	if err := ioutil.WriteFile(cloudConfigOutput, cloudConfigBytes, 500); err != nil {
-	} else {
+		log.Errorf("Error while writing file %s: %v", cloudConfigOutput, err)
-		fileData = append([]byte("#cloud-config\n"), data...)
-	}
-
-	output := path.Join(outputDir, outputFile)
-	log.Infof("Writing merged cloud-config to %s", output)
-	if err := ioutil.WriteFile(output, fileData, 400); err != nil {
-		log.Errorf("Error while writing file %v", err)
 		return err
 	}
 
-	if script == nil {
+	ccData := make(map[string]interface{})
-		ccData := make(map[string]interface{})
+	if err := yaml.Unmarshal(cloudConfigBytes, ccData); err != nil {
-		if err := yaml.Unmarshal(userdataBytes, ccData); err != nil {
+		return err
+	}
+
+	if rancher, ok := ccData["rancher"]; ok {
+		bytes, err := yaml.Marshal(rancher)
+		if err != nil {
 			return err
 		}
 
-		if rancher, ok := ccData["rancher"]; ok {
+		if err = ioutil.WriteFile(rancherYmlOutput, bytes, 400); err != nil {
-			bytes, err := yaml.Marshal(rancher)
+			return err
-			if err != nil {
-				return err
-			}
-
-			if err = ioutil.WriteFile(path.Join(outputDir, rancherYml), bytes, 400); err != nil {
-				return err
-			}
 		}
 	}
 
+	metaDataBytes, err := yaml.Marshal(metadata)
+	if err != nil {
+		return err
+	}
+
+	if err = ioutil.WriteFile(metaDataOutput, metaDataBytes, 400); err != nil {
+		return err
+	}
+
 	return nil
 }
 
-func Main() {
+func currentDatasource() (datasource.Datasource, error) {
-	flags.Parse(rancherConfig.FilterGlobalConfig(os.Args[1:]))
-
 	cfg, err := rancherConfig.LoadConfig()
 	if err != nil {
 		log.Fatalf("Failed to read rancher config %v", err)
@@ -133,53 +137,152 @@ func Main() {
 
 	dss := getDatasources(cfg)
 	if len(dss) == 0 {
-		log.Infof("No datasources available %v", cfg.CloudInit.Datasources)
+		return nil, nil
-		os.Exit(0)
 	}
 
 	ds := selectDatasource(dss)
-	if ds == nil {
+	return ds, nil
-		log.Info("No datasources found")
+}
-		os.Exit(0)
-	}
 
-	log.Infof("Fetching user-data from datasource %v", ds.Type())
+func mergeBaseConfig(current []byte) ([]byte, error) {
-	userdataBytes, err := ds.FetchUserdata()
+	files, err := ioutil.ReadDir(baseConfigDir)
 	if err != nil {
-		log.Fatalf("Failed fetching user-data from datasource: %v", err)
+		if os.IsNotExist(err) {
-	}
+			log.Infof("%s does not exist, not merging", baseConfigDir)
-
+			return current, nil
-	log.Infof("Fetching meta-data from datasource of type %v", ds.Type())
-	metadata, err := ds.FetchMetadata()
-	if err != nil {
-		log.Infof("Failed fetching meta-data from datasource: %v", err)
-		os.Exit(1)
-	}
-
-	var ccu *config.CloudConfig
-	var script *config.Script
-	if ud, err := initialize.ParseUserData(string(userdataBytes)); err != nil {
-		log.Fatalf("Failed to parse user-data: %v\n", err)
-	} else {
-		switch t := ud.(type) {
-		case *config.CloudConfig:
-			ccu = t
-		case *config.Script:
-			script = t
 		}
+
+		log.Errorf("Failed to read %s: %v", baseConfigDir, err)
+		return nil, err
+	}
+
+	result := []byte{}
+
+	for _, file := range files {
+		if file.IsDir() || strings.HasPrefix(file.Name(), ".") {
+			continue
+		}
+
+		input := path.Join(baseConfigDir, file.Name())
+		content, err := ioutil.ReadFile(input)
+		if err != nil {
+			log.Errorf("Failed to read %s: %v", input, err)
+			// ignore error
+			continue
+		}
+
+		log.Infof("Merging %s", input)
+		result, err = util.MergeBytes(result, content)
+		if err != nil {
+			log.Errorf("Failed to merge bytes: %v", err)
+			return nil, err
+		}
+	}
+
+	if len(result) == 0 {
+		return current, nil
+	} else {
+		return util.MergeBytes(result, current)
+	}
+}
+
+func saveCloudConfig() error {
+	var userDataBytes []byte
+	var metadata datasource.Metadata
+
+	ds, err := currentDatasource()
+	if err != nil {
+		log.Errorf("Failed to select datasource: %v", err)
+		return err
+	}
+
+	if ds != nil {
+		log.Infof("Fetching user-data from datasource %v", ds.Type())
+		userDataBytes, err = ds.FetchUserdata()
+		if err != nil {
+			log.Errorf("Failed fetching user-data from datasource: %v", err)
+			return err
+		}
+
+		log.Infof("Fetching meta-data from datasource of type %v", ds.Type())
+		metadata, err = ds.FetchMetadata()
+		if err != nil {
+			log.Errorf("Failed fetching meta-data from datasource: %v", err)
+			return err
+		}
+	}
+
+	userData := string(userDataBytes)
+	scriptBytes := []byte{}
+
+	if config.IsScript(userData) {
+		scriptBytes = userDataBytes
+		userDataBytes = []byte{}
+	} else if isCompose(userData) {
+		if userDataBytes, err = toCompose(userDataBytes); err != nil {
+			log.Errorf("Failed to convert to compose syntax: %v", err)
+			return err
+		}
+	} else if config.IsCloudConfig(userData) {
+		// nothing to do
+	} else {
+		log.Errorf("Unrecognized cloud-init\n%s", userData)
+		userDataBytes = []byte{}
+	}
+
+	if userDataBytes, err = mergeBaseConfig(userDataBytes); err != nil {
+		log.Errorf("Failed to merge base config: %v", err)
+		return err
+	}
+
+	return saveFiles(userDataBytes, scriptBytes, metadata)
+}
+
+func getSaveCloudConfig() (*config.CloudConfig, error) {
+	cloudConfig := path.Join(outputDir, outputFile)
+
+	ds := file.NewDatasource(cloudConfig)
+	if !ds.IsAvailable() {
+		log.Infof("%s does not exist", cloudConfig)
+		return nil, nil
+	}
+
+	ccBytes, err := ds.FetchUserdata()
+	if err != nil {
+		log.Errorf("Failed to read user-data from %s: %v", cloudConfig, err)
+		return nil, err
+	}
+
+	var cc config.CloudConfig
+	err = yaml.Unmarshal(ccBytes, &cc)
+	if err != nil {
+		log.Errorf("Failed to unmarshall user-data from %s: %v", cloudConfig, err)
+		return nil, err
+	}
+
+	return &cc, err
+}
+
+func executeCloudConfig() error {
+	ccu, err := getSaveCloudConfig()
+	if err != nil {
+		return err
+	}
+
+	var metadata datasource.Metadata
+
+	metaDataBytes, err := ioutil.ReadFile(path.Join(outputDir, metaDataFile))
+	if err != nil {
+		return err
+	}
+
+	if err = yaml.Unmarshal(metaDataBytes, &metadata); err != nil {
+		return err
 	}
 
 	log.Info("Merging cloud-config from meta-data and user-data")
 	cc := mergeConfigs(ccu, metadata)
 
-	if save {
-		err = saveFiles(script, userdataBytes, &cc)
-		if err != nil {
-			log.Fatal(err)
-		}
-		os.Exit(0)
-	}
-
 	if len(cc.SSHAuthorizedKeys) > 0 {
 		authorizeSSHKeys("rancher", cc.SSHAuthorizedKeys, sshKeyName)
 	}
@@ -201,6 +304,26 @@ func Main() {
 		}
 		log.Printf("Wrote file %s to filesystem", fullPath)
 	}
+
+	return nil
+}
+
+func Main() {
+	flags.Parse(rancherConfig.FilterGlobalConfig(os.Args[1:]))
+
+	if save {
+		err := saveCloudConfig()
+		if err != nil {
+			log.Fatalf("Failed to save cloud config: %v", err)
+		}
+	}
+
+	if execute {
+		err := executeCloudConfig()
+		if err != nil {
+			log.Fatalf("Failed to save cloud config: %v", err)
+		}
+	}
 }
 
 // mergeConfigs merges certain options from md (meta-data from the datasource)
@@ -229,16 +352,6 @@ func mergeConfigs(cc *config.CloudConfig, md datasource.Metadata) (out config.Cl
 func getDatasources(cfg *rancherConfig.Config) []datasource.Datasource {
 	dss := make([]datasource.Datasource, 0, 5)
 
-	if execute {
-		cloudConfig := path.Join(outputDir, outputFile)
-		if _, err := os.Stat(cloudConfig); os.IsNotExist(err) {
-			return dss
-		}
-
-		dss = append(dss, file.NewDatasource(cloudConfig))
-		return dss
-	}
-
 	for _, ds := range cfg.CloudInit.Datasources {
 		parts := strings.SplitN(ds, ":", 2)
 
@@ -355,3 +468,19 @@ func selectDatasource(sources []datasource.Datasource) datasource.Datasource {
 	close(stop)
 	return s
 }
+
+func isCompose(content string) bool {
+	return strings.HasPrefix(content, "#compose\n")
+}
+
+func toCompose(bytes []byte) ([]byte, error) {
+	result := make(map[interface{}]interface{})
+	compose := make(map[interface{}]interface{})
+	err := yaml.Unmarshal(bytes, &compose)
+	if err != nil {
+		return nil, err
+	}
+
+	result["services"] = compose
+	return yaml.Marshal(result)
+}

cmd/control/os.go

@@ -162,6 +162,7 @@ func startUpgradeContainer(image string, stage, force bool) {
 
 	container := docker.NewContainer(config.DOCKER_SYSTEM_HOST, &config.ContainerConfig{
 		Cmd: "--name=os-upgrade " +
+			"--log-driver=json-file" +
 			"--rm " +
 			"--privileged " +
 			"--net=host " +

cmd/control/service.go

@@ -37,21 +37,16 @@ func disable(c *cli.Context) {
 	}
 
 	for _, service := range c.Args() {
-		filtered := make([]string, 0, len(c.Args()))
+		if _, ok := cfg.ServicesInclude[service]; !ok {
-		for _, existing := range cfg.EnabledServices {
+			continue
-			if existing != service {
-				filtered = append(filtered, existing)
-			}
 		}
 
-		if len(filtered) != len(c.Args()) {
+		cfg.ServicesInclude[service] = false
-			cfg.EnabledServices = filtered
+		changed = true
-			changed = true
-		}
 	}
 
 	if changed {
-		if err = cfg.Set("enabled_services", cfg.EnabledServices); err != nil {
+		if err = cfg.Set("services_include", cfg.ServicesInclude); err != nil {
 			log.Fatal(err)
 		}
 	}
@@ -65,14 +60,14 @@ func enable(c *cli.Context) {
 	}
 
 	for _, service := range c.Args() {
-		if !util.Contains(cfg.EnabledServices, service) {
+		if val, ok := cfg.ServicesInclude[service]; !ok || !val {
-			cfg.EnabledServices = append(cfg.EnabledServices, service)
+			cfg.ServicesInclude[service] = true
 			changed = true
 		}
 	}
 
 	if changed {
-		if err = cfg.Set("enabled_services", cfg.EnabledServices); err != nil {
+		if err = cfg.Set("services_include", cfg.ServicesInclude); err != nil {
 			log.Fatal(err)
 		}
 	}
@@ -84,22 +79,34 @@ func list(c *cli.Context) {
 		log.Fatal(err)
 	}
 
-	enabled := map[string]bool{}
+	clone := make(map[string]bool)
-
+	for service, enabled := range cfg.ServicesInclude {
-	for _, service := range cfg.EnabledServices {
+		clone[service] = enabled
-		enabled[service] = true
 	}
 
-	for service, _ := range cfg.Services {
+	services, err := util.GetServices(cfg.Repositories.ToArray())
-		if _, ok := enabled[service]; ok {
+	if err != nil {
-			delete(enabled, service)
+		log.Fatalf("Failed to get services: %v", err)
-			fmt.Printf("enabled  %s\n", service)
+	}
+
+	for _, service := range services {
+		if enabled, ok := clone[service]; ok {
+			delete(clone, service)
+			if enabled {
+				fmt.Printf("enabled  %s\n", service)
+			} else {
+				fmt.Printf("disabled %s\n", service)
+			}
 		} else {
 			fmt.Printf("disabled %s\n", service)
 		}
 	}
 
-	for service, _ := range enabled {
+	for service, enabled := range clone {
-		fmt.Printf("enabled  %s\n", service)
+		if enabled {
+			fmt.Printf("enabled  %s\n", service)
+		} else {
+			fmt.Printf("disabled %s\n", service)
+		}
 	}
 }

config/config.go

@@ -264,3 +264,14 @@ func (d *DockerConfig) BridgeConfig() (string, string) {
 		return name, cidr
 	}
 }
+
+func (r Repositories) ToArray() []string {
+	result := make([]string, 0, len(r))
+	for _, repo := range r {
+		if repo.Url != "" {
+			result = append(result, repo.Url)
+		}
+	}
+
+	return result
+}

config/default.go

@@ -81,6 +81,7 @@ func NewConfig() *Config {
 				Privileged: true,
 				Labels: []string{
 					DETACH + "=false",
+					SCOPE + "=" + SYSTEM,
 				},
 				Volumes: []string{
 					"/dev:/host/dev",
@@ -98,6 +99,7 @@ func NewConfig() *Config {
 				Privileged: true,
 				Labels: []string{
 					DETACH + "=true",
+					SCOPE + "=" + SYSTEM,
 				},
 				Environment: []string{
 					"DAEMON=true",
@@ -113,6 +115,7 @@ func NewConfig() *Config {
 				Privileged: true,
 				Labels: []string{
 					CREATE_ONLY + "=true",
+					SCOPE + "=" + SYSTEM,
 				},
 				Volumes: []string{
 					"/dev:/host/dev",
@@ -132,6 +135,7 @@ func NewConfig() *Config {
 				Privileged: true,
 				Labels: []string{
 					CREATE_ONLY + "=true",
+					SCOPE + "=" + SYSTEM,
 				},
 				Volumes: []string{
 					"/init:/sbin/halt:ro",
@@ -156,6 +160,7 @@ func NewConfig() *Config {
 				Privileged: true,
 				Labels: []string{
 					CREATE_ONLY + "=true",
+					SCOPE + "=" + SYSTEM,
 				},
 				Volumes: []string{
 					"/home:/home",
@@ -170,6 +175,7 @@ func NewConfig() *Config {
 				Privileged: true,
 				Labels: []string{
 					CREATE_ONLY + "=true",
+					SCOPE + "=" + SYSTEM,
 				},
 				Volumes: []string{
 					"/var/lib/rancher:/var/lib/rancher",
@@ -185,6 +191,7 @@ func NewConfig() *Config {
 				Privileged: true,
 				Labels: []string{
 					CREATE_ONLY + "=true",
+					SCOPE + "=" + SYSTEM,
 				},
 				VolumesFrom: []string{
 					"docker-volumes",
@@ -201,6 +208,7 @@ func NewConfig() *Config {
 				Labels: []string{
 					RELOAD_CONFIG + "=true",
 					DETACH + "=false",
+					SCOPE + "=" + SYSTEM,
 				},
 				Environment: []string{
 					"CLOUD_INIT_NETWORK=false",
@@ -216,6 +224,7 @@ func NewConfig() *Config {
 				Net:        "host",
 				Labels: []string{
 					DETACH + "=false",
+					SCOPE + "=" + SYSTEM,
 				},
 				Links: []string{
 					"cloud-init-pre",
@@ -231,6 +240,7 @@ func NewConfig() *Config {
 				Labels: []string{
 					RELOAD_CONFIG + "=true",
 					DETACH + "=false",
+					SCOPE + "=" + SYSTEM,
 				},
 				Net: "host",
 				Links: []string{
@@ -246,6 +256,9 @@ func NewConfig() *Config {
 				Image:      "ntp",
 				Privileged: true,
 				Net:        "host",
+				Labels: []string{
+					SCOPE + "=" + SYSTEM,
+				},
 				Links: []string{
 					"cloud-init",
 					"network",
@@ -255,6 +268,9 @@ func NewConfig() *Config {
 				Image:      "syslog",
 				Privileged: true,
 				Net:        "host",
+				Labels: []string{
+					SCOPE + "=" + SYSTEM,
+				},
 				VolumesFrom: []string{
 					"system-volumes",
 				},
@@ -266,6 +282,9 @@ func NewConfig() *Config {
 				Pid:        "host",
 				Ipc:        "host",
 				Net:        "host",
+				Labels: []string{
+					SCOPE + "=" + SYSTEM,
+				},
 				Links: []string{
 					"network",
 				},
@@ -277,7 +296,8 @@ func NewConfig() *Config {
 				Image: "userdockerwait",
 				Net:   "host",
 				Labels: []string{
-					"io.rancher.os.detach=false",
+					DETACH + "=false",
+					SCOPE + "=" + SYSTEM,
 				},
 				Links: []string{
 					"userdocker",
@@ -292,6 +312,9 @@ func NewConfig() *Config {
 				Links: []string{
 					"cloud-init",
 				},
+				Labels: []string{
+					SCOPE + "=" + SYSTEM,
+				},
 				VolumesFrom: []string{
 					"all-volumes",
 				},
@@ -301,29 +324,14 @@ func NewConfig() *Config {
 				Net:     "host",
 			},
 		},
-		EnabledServices: []string{},
+		ServicesInclude: map[string]bool{
-		Services: map[string]Config{
+			"ubuntu-console": false,
-			"ubuntu-console": {
+		},
-				SystemContainers: map[string]*project.ServiceConfig{
+		Repositories: map[string]Repository{
-					"console": {
+			"core": Repository{
-						Image:      "rancher/ubuntuconsole:" + IMAGE_VERSION,
+				Url: "https://raw.githubusercontent.com/rancherio/os-services/master",
-						Privileged: true,
-						Labels: []string{
-							DETACH + "=true",
-						},
-						Links: []string{
-							"cloud-init",
-						},
-						VolumesFrom: []string{
-							"all-volumes",
-						},
-						Restart: "always",
-						Pid:     "host",
-						Ipc:     "host",
-						Net:     "host",
-					},
-				},
 			},
 		},
+		Services: map[string]*project.ServiceConfig{},
 	}
 }

config/types.go

@@ -23,6 +23,8 @@ const (
 	REMOVE        = "io.rancher.os.remove"
 	CREATE_ONLY   = "io.rancher.os.createonly"
 	RELOAD_CONFIG = "io.rancher.os.reloadconfig"
+	SCOPE         = "io.rancher.os.scope"
+	SYSTEM        = "system"
 )
 
 var (
@@ -42,17 +44,25 @@ type ContainerConfig struct {
 	Service        *project.ServiceConfig `yaml:service,omitempty`
 }
 
+type Repository struct {
+	Url string `yaml:url,omitempty`
+}
+
+type Repositories map[string]Repository
+
 type Config struct {
-	Services            map[string]Config                 `yaml:"services,omitempty"`
+	Environment         map[string]string                 `yaml:"environment,omitempty"`
+	Services            map[string]*project.ServiceConfig `yaml:"services,omitempty"`
 	BootstrapContainers map[string]*project.ServiceConfig `yaml:"bootstrap_containers,omitempty"`
 	BootstrapDocker     DockerConfig                      `yaml:"bootstrap_docker,omitempty"`
 	CloudInit           CloudInit                         `yaml:"cloud_init,omitempty"`
 	Console             ConsoleConfig                     `yaml:"console,omitempty"`
 	Debug               bool                              `yaml:"debug,omitempty"`
 	Disable             []string                          `yaml:"disable,omitempty"`
-	EnabledServices     []string                          `yaml:"enabled_services,omitempty"`
+	ServicesInclude     map[string]bool                   `yaml:"services_include,omitempty"`
 	Modules             []string                          `yaml:"modules,omitempty"`
 	Network             NetworkConfig                     `yaml:"network,omitempty"`
+	Repositories        Repositories                      `yaml:"repositories,omitempty"`
 	Ssh                 SshConfig                         `yaml:"ssh,omitempty"`
 	State               StateConfig                       `yaml:"state,omitempty"`
 	SystemContainers    map[string]*project.ServiceConfig `yaml:"system_containers,omitempty"`

docker/container.go

@@ -71,6 +71,7 @@ func NewContainerFromService(dockerHost string, name string, service *project.Se
 			Service: service,
 		},
 	}
+
 	return c.Parse()
 }
 
@@ -170,11 +171,55 @@ func (c *Container) Reset() *Container {
 	return c
 }
 
+func (c *Container) requiresSyslog() bool {
+	return (c.ContainerCfg.Service.LogDriver == "" || c.ContainerCfg.Service.LogDriver == "syslog")
+}
+
+func (c *Container) requiresUserDocker() bool {
+	if c.dockerHost == config.DOCKER_HOST {
+		return true
+	}
+
+	for _, v := range c.ContainerCfg.Service.Volumes {
+		if strings.Index(v, "/var/run/docker.sock") != -1 {
+			return true
+		}
+	}
+
+	return false
+}
+
+func (c *Container) hasLink(link string) bool {
+	return util.Contains(c.ContainerCfg.Service.Links, link)
+}
+
+func (c *Container) addLink(link string) {
+	if c.hasLink(link) {
+		return
+	}
+
+	log.Debugf("Adding %s link to %s", link, c.Name)
+	c.ContainerCfg.Service.Links = append(c.ContainerCfg.Service.Links, link)
+}
+
 func (c *Container) parseService() {
-	if (c.ContainerCfg.Service.LogDriver == "" || c.ContainerCfg.Service.LogDriver == "syslog") &&
+	if c.requiresSyslog() {
-		!util.Contains(c.ContainerCfg.Service.Links, "syslog") {
+		c.addLink("syslog")
-		log.Debugf("Adding syslog link to %s\n", c.Name)
+	}
-		c.ContainerCfg.Service.Links = append(c.ContainerCfg.Service.Links, "syslog")
+
+	if c.requiresUserDocker() {
+		c.addLink("userdockerwait")
+	} else if c.ContainerCfg.Service.Image != "" {
+		client, err := NewClient(c.dockerHost)
+		if err != nil {
+			c.Err = err
+			return
+		}
+
+		i, _ := client.InspectImage(c.ContainerCfg.Service.Image)
+		if i == nil {
+			c.addLink("network")
+		}
 	}
 
 	cfg, hostConfig, err := docker.Convert(c.ContainerCfg.Service)
@@ -190,7 +235,6 @@ func (c *Container) parseService() {
 	c.remove = c.Config.Labels[config.REMOVE] != "false"
 	c.ContainerCfg.CreateOnly = c.Config.Labels[config.CREATE_ONLY] == "true"
 	c.ContainerCfg.ReloadConfig = c.Config.Labels[config.RELOAD_CONFIG] == "true"
-
 }
 
 func (c *Container) parseCmd() {

docker/factory.go

@@ -38,6 +38,8 @@ func (c *containerBasedService) Up() error {
 
 	var event project.Event
 
+	c.project.Notify(project.CONTAINER_STARTING, c, map[string]string{})
+
 	if create {
 		container.Create()
 		event = project.CONTAINER_CREATED
@@ -71,8 +73,17 @@ func (c *containerBasedService) Name() string {
 	return c.name
 }
 
+func isSystemService(serviceConfig *project.ServiceConfig) bool {
+	return util.GetValue(serviceConfig.Labels, config.SCOPE) == config.SYSTEM
+}
+
 func (c *ContainerFactory) Create(project *project.Project, name string, serviceConfig *project.ServiceConfig) (project.Service, error) {
-	container := NewContainerFromService(config.DOCKER_SYSTEM_HOST, name, serviceConfig)
+	host := config.DOCKER_HOST
+	if isSystemService(serviceConfig) {
+		host = config.DOCKER_SYSTEM_HOST
+	}
+
+	container := NewContainerFromService(host, name, serviceConfig)
 
 	if container.Err != nil {
 		return nil, container.Err

docker/services.go

@@ -0,0 +1,144 @@
+package docker
+
+import (
+	"fmt"
+	"strings"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/rancherio/os/config"
+	"github.com/rancherio/os/util"
+	"github.com/rancherio/rancher-compose/project"
+)
+
+type configEnvironemnt struct {
+	cfg *config.Config
+}
+
+func appendEnv(array []string, key, value string) []string {
+	parts := strings.SplitN(key, "/", 2)
+	if len(parts) == 2 {
+		key = parts[1]
+	}
+
+	return append(array, fmt.Sprintf("%s=%s", key, value))
+}
+
+func lookupKeys(cfg *config.Config, keys ...string) []string {
+	for _, key := range keys {
+		if strings.HasSuffix(key, "*") {
+			result := []string{}
+			for envKey, envValue := range cfg.Environment {
+				keyPrefix := key[:len(key)-1]
+				if strings.HasPrefix(envKey, keyPrefix) {
+					result = appendEnv(result, envKey, envValue)
+				}
+			}
+
+			if len(result) > 0 {
+				return result
+			}
+		} else if value, ok := cfg.Environment[key]; ok {
+			return appendEnv([]string{}, key, value)
+		}
+	}
+
+	return []string{}
+}
+
+func (c *configEnvironemnt) Lookup(key, serviceName string, serviceConfig *project.ServiceConfig) []string {
+	fullKey := fmt.Sprintf("%s/%s", serviceName, key)
+	return lookupKeys(c.cfg, fullKey, key)
+}
+
+func RunServices(name string, cfg *config.Config, configs map[string]*project.ServiceConfig) error {
+	network := false
+	projectEvents := make(chan project.ProjectEvent)
+	p := project.NewProject(name, NewContainerFactory(cfg))
+	p.EnvironmentLookup = &configEnvironemnt{cfg: cfg}
+	p.AddListener(projectEvents)
+	enabled := make(map[string]bool)
+
+	for name, serviceConfig := range configs {
+		if err := p.AddConfig(name, serviceConfig); err != nil {
+			log.Infof("Failed loading service %s", name)
+		}
+	}
+
+	p.ReloadCallback = func() error {
+		err := cfg.Reload()
+		if err != nil {
+			return err
+		}
+
+		for service, serviceEnabled := range cfg.ServicesInclude {
+			if !serviceEnabled {
+				continue
+			}
+
+			if _, ok := enabled[service]; ok {
+				continue
+			}
+
+			//if config, ok := cfg.BundledServices[service]; ok {
+			//	for name, s := range config.SystemContainers {
+			//		if err := p.AddConfig(name, s); err != nil {
+			//			log.Errorf("Failed to load %s : %v", name, err)
+			//		}
+			//	}
+			//} else {
+			bytes, err := LoadServiceResource(service, network, cfg)
+			if err != nil {
+				if err == util.ErrNoNetwork {
+					log.Debugf("Can not load %s, networking not enabled", service)
+				} else {
+					log.Errorf("Failed to load %s : %v", service, err)
+				}
+				continue
+			}
+
+			err = p.Load(bytes)
+			if err != nil {
+				log.Errorf("Failed to load %s : %v", service, err)
+				continue
+			}
+			//}
+
+			enabled[service] = true
+		}
+
+		for service, config := range cfg.Services {
+			if _, ok := enabled[service]; ok {
+				continue
+			}
+
+			err = p.AddConfig(service, config)
+			if err != nil {
+				log.Errorf("Failed to load %s : %v", service, err)
+				continue
+			}
+
+			enabled[service] = true
+		}
+
+		return nil
+	}
+
+	go func() {
+		for event := range projectEvents {
+			if event.Event == project.CONTAINER_STARTED && event.Service.Name() == "network" {
+				network = true
+			}
+		}
+	}()
+
+	err := p.ReloadCallback()
+	if err != nil {
+		log.Errorf("Failed to reload %s : %v", name, err)
+		return err
+	}
+	return p.Up()
+}
+
+func LoadServiceResource(name string, network bool, cfg *config.Config) ([]byte, error) {
+	return util.LoadResource(name, network, cfg.Repositories.ToArray())
+}

init/bootstrap.go

@@ -7,6 +7,7 @@ import (
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/rancherio/os/config"
+	"github.com/rancherio/os/docker"
 	"github.com/rancherio/os/util"
 	"github.com/rancherio/rancher-compose/project"
 )
@@ -55,7 +56,7 @@ outer:
 
 	if format != "" {
 		log.Infof("Auto formatting : %s", format)
-		return runServices("autoformat", cfg, map[string]*project.ServiceConfig{
+		return docker.RunServices("autoformat", cfg, map[string]*project.ServiceConfig{
 			"autoformat": {
 				Net:        "none",
 				Privileged: true,
@@ -70,7 +71,7 @@ outer:
 }
 
 func runBootstrapContainers(cfg *config.Config) error {
-	return runServices("bootstrap", cfg, cfg.BootstrapContainers)
+	return docker.RunServices("bootstrap", cfg, cfg.BootstrapContainers)
 }
 
 func startDocker(cfg *config.Config) (chan interface{}, error) {

init/init.go

@@ -3,7 +3,6 @@ package init
 import (
 	"fmt"
 	"io/ioutil"
-	"net"
 	"os"
 	"os/exec"
 	"strings"
@@ -16,9 +15,10 @@ import (
 )
 
 const (
-	STATE   string = "/var"
+	STATE         string = "/var"
-	DOCKER  string = "/usr/bin/docker"
+	SYSTEM_DOCKER string = "/usr/bin/system-docker"
-	SYSINIT string = "/sbin/rancher-sysinit"
+	DOCKER        string = "/usr/bin/docker"
+	SYSINIT       string = "/sbin/rancher-sysinit"
 )
 
 var (
@@ -63,6 +63,7 @@ var (
 		"/sbin/modprobe":                     "/busybox",
 		"/usr/sbin/iptables":                 "/xtables-multi",
 		DOCKER:                               "/docker",
+		SYSTEM_DOCKER:                        "/docker",
 		SYSINIT:                              "/init",
 		"/home":                              "/var/lib/rancher/state/home",
 		"/opt":                               "/var/lib/rancher/state/opt",
@@ -237,7 +238,7 @@ func execDocker(cfg *config.Config) error {
 	}
 
 	os.Stdin.Close()
-	return syscall.Exec(DOCKER, cfg.SystemDocker.Args, os.Environ())
+	return syscall.Exec(SYSTEM_DOCKER, cfg.SystemDocker.Args, os.Environ())
 }
 
 func MainInit() {
@@ -285,10 +286,9 @@ func touchSocket(cfg *config.Config) error {
 		if err := syscall.Unlink(path); err != nil && !os.IsNotExist(err) {
 			return err
 		}
-		if l, err := net.Listen("unix", path); err != nil {
+		err := ioutil.WriteFile(path, []byte{}, 0700)
+		if err != nil {
 			return err
-		} else {
-			l.Close()
 		}
 	}
 
@@ -365,7 +365,8 @@ func RunInit() error {
 			return createMounts(postMounts...)
 		},
 		touchSocket,
-		remountRo,
+		// Disable R/O root write now to support updating modules
+		//remountRo,
 		sysInit,
 	}
 

init/sysinit.go

@@ -9,9 +9,6 @@ import (
 	dockerClient "github.com/fsouza/go-dockerclient"
 	"github.com/rancherio/os/config"
 	"github.com/rancherio/os/docker"
-	"github.com/rancherio/os/util"
-
-	"github.com/rancherio/rancher-compose/project"
 )
 
 func importImage(client *dockerClient.Client, name, fileName string) error {
@@ -109,63 +106,8 @@ func loadImages(cfg *config.Config) error {
 	return nil
 }
 
-func runServices(name string, cfg *config.Config, configs map[string]*project.ServiceConfig) error {
-	project := project.NewProject(name, docker.NewContainerFactory(cfg))
-	enabled := make(map[string]bool)
-
-	for name, serviceConfig := range configs {
-		if err := project.AddConfig(name, serviceConfig); err != nil {
-			log.Infof("Failed loading service %s", name)
-		}
-	}
-
-	project.ReloadCallback = func() error {
-		err := cfg.Reload()
-		if err != nil {
-			return err
-		}
-
-		for _, service := range cfg.EnabledServices {
-			if _, ok := enabled[service]; ok {
-				continue
-			}
-
-			if config, ok := cfg.Services[service]; ok {
-				for name, s := range config.SystemContainers {
-					if err := project.AddConfig(name, s); err != nil {
-						log.Errorf("Failed to load %s : %v", name, err)
-					}
-				}
-			} else {
-				bytes, err := util.LoadResource(service)
-				if err != nil {
-					log.Errorf("Failed to load %s : %v", service, err)
-					continue
-				}
-
-				err = project.Load(bytes)
-				if err != nil {
-					log.Errorf("Failed to load %s : %v", service, err)
-					continue
-				}
-			}
-
-			enabled[service] = true
-		}
-
-		return nil
-	}
-
-	err := project.ReloadCallback()
-	if err != nil {
-		log.Errorf("Failed to reload %s : %v", name, err)
-		return err
-	}
-	return project.Up()
-}
-
 func runContainers(cfg *config.Config) error {
-	return runServices("system-init", cfg, cfg.SystemContainers)
+	return docker.RunServices("system-init", cfg, cfg.SystemContainers)
 }
 
 func tailConsole(cfg *config.Config) error {

scripts/dockerimages/scripts/console.sh

@@ -76,7 +76,7 @@ if ! grep -q '^UseDNS no' /etc/ssh/sshd_config; then
 fi
 
 ID_TYPE="busybox"
-if grep -q 'ID_LIKE=' /etc/os-release; then
+if [ -e /etc/os-release ] && grep -q 'ID_LIKE=' /etc/os-release; then
     ID_TYPE=$(grep 'ID_LIKE=' /etc/os-release | cut -d'=' -f2)
 fi
 

scripts/extraimages/00-ubuntuconsole

@@ -3,6 +3,7 @@ RUN apt-get update && \
     apt-get upgrade --no-install-recommends -y && \
     apt-get install -y --no-install-recommends openssh-server rsync
 RUN rm -rf /etc/ssh/*key*
+COPY scripts/dockerimages/scripts/entry.sh /usr/sbin/
 COPY scripts/dockerimages/scripts/console.sh /usr/sbin/
 COPY scripts/dockerimages/scripts/update-ssh-keys /usr/bin/
 RUN echo 'RancherOS \\n \l' > /etc/issue
@@ -12,4 +13,5 @@ RUN addgroup --gid 1100 rancher && \
     useradd -u 1100 -g rancher -G docker,sudo -m -s /bin/bash rancher && \
     echo '## allow password less for rancher user' >> /etc/sudoers && \
     echo 'rancher ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
+ENTRYPOINT ["/usr/sbin/entry.sh"]
 CMD ["/usr/sbin/console.sh"]

scripts/run

@@ -82,7 +82,7 @@ qemu-system-x86_64 -serial stdio \
     -initrd ${INITRD_TEST} \
     -m 1024 \
     -net nic,vlan=0,model=virtio \
-    -net user,vlan=0,hostfwd=tcp::2222-:22,hostname=rancher \
+    -net user,vlan=0,hostfwd=tcp::2222-:22,hostname=rancher-dev \
     -drive if=virtio,file=${HD} \
     -machine accel=kvm \
     -cpu host \

scripts/version

@@ -1 +1 @@
-VERSION=v0.3.0-rc2
+VERSION=v0.3.0-rc3

util/util.go

@@ -2,6 +2,7 @@ package util
 
 import (
 	"archive/tar"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -12,12 +13,16 @@ import (
 	"strings"
 	"syscall"
 
+	log "github.com/Sirupsen/logrus"
+
 	"github.com/docker/docker/pkg/mount"
 	"gopkg.in/yaml.v2"
 )
 
 var (
-	letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+	letters      = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+	ErrNoNetwork = errors.New("Networking not available to load resource")
+	ErrNotFound  = errors.New("Failed to find resource")
 )
 
 func mountProc() error {
@@ -168,6 +173,25 @@ func Convert(from, to interface{}) error {
 	return yaml.Unmarshal(bytes, to)
 }
 
+func MergeBytes(left, right []byte) ([]byte, error) {
+	leftMap := make(map[interface{}]interface{})
+	rightMap := make(map[interface{}]interface{})
+
+	err := yaml.Unmarshal(left, &leftMap)
+	if err != nil {
+		return nil, err
+	}
+
+	err = yaml.Unmarshal(right, &rightMap)
+	if err != nil {
+		return nil, err
+	}
+
+	MergeMaps(leftMap, rightMap)
+
+	return yaml.Marshal(leftMap)
+}
+
 func MergeMaps(left, right map[interface{}]interface{}) {
 	for k, v := range right {
 		merged := false
@@ -186,15 +210,76 @@ func MergeMaps(left, right map[interface{}]interface{}) {
 	}
 }
 
-func LoadResource(location string) ([]byte, error) {
+func GetServices(urls []string) ([]string, error) {
+	result := []string{}
+
+	for _, url := range urls {
+		indexUrl := fmt.Sprintf("%s/index.yml", url)
+		content, err := LoadResource(indexUrl, true, []string{})
+		if err != nil {
+			log.Errorf("Failed to load %s: %v", indexUrl, err)
+			continue
+		}
+
+		services := make(map[string][]string)
+		err = yaml.Unmarshal(content, &services)
+		if err != nil {
+			log.Errorf("Failed to unmarshal %s: %v", indexUrl, err)
+			continue
+		}
+
+		if list, ok := services["services"]; ok {
+			result = append(result, list...)
+		}
+	}
+
+	return []string{}, nil
+}
+
+func LoadResource(location string, network bool, urls []string) ([]byte, error) {
+	var bytes []byte
+	err := ErrNotFound
+
 	if strings.HasPrefix(location, "http:/") || strings.HasPrefix(location, "https:/") {
+		if !network {
+			return nil, ErrNoNetwork
+		}
 		resp, err := http.Get(location)
 		if err != nil {
 			return nil, err
 		}
+		if resp.StatusCode != http.StatusOK {
+			return nil, fmt.Errorf("non-200 http response: %d", resp.StatusCode)
+		}
 		defer resp.Body.Close()
 		return ioutil.ReadAll(resp.Body)
-	} else {
+	} else if strings.HasPrefix(location, "/") {
 		return ioutil.ReadFile(location)
+	} else if len(location) > 0 {
+		for _, url := range urls {
+			ymlUrl := fmt.Sprintf("%s/%s/%s.yml", url, location[0:1], location)
+			log.Infof("Loading %s from %s", location, ymlUrl)
+			bytes, err = LoadResource(ymlUrl, network, []string{})
+			if err == nil {
+				return bytes, nil
+			}
+		}
 	}
+
+	return nil, err
+}
+
+func GetValue(kvPairs []string, key string) string {
+	if kvPairs == nil {
+		return ""
+	}
+
+	prefix := key + "="
+	for _, i := range kvPairs {
+		if strings.HasPrefix(i, prefix) {
+			return strings.TrimPrefix(i, prefix)
+		}
+	}
+
+	return ""
 }