Services in compose format

cmd/cloudinit/cloudinit.go

@@ -476,13 +476,11 @@ func isCompose(content string) bool {
 func toCompose(bytes []byte) ([]byte, error) {
 	result := make(map[interface{}]interface{})
 	compose := make(map[interface{}]interface{})
-	err := yaml.Unmarshal(bytes, &result)
+	err := yaml.Unmarshal(bytes, &compose)
 	if err != nil {
 		return nil, err
 	}
 
-	result["services"] = map[interface{}]interface{}{
+	result["services"] = compose
-		"cloud-config": compose,
-	}
 	return yaml.Marshal(result)
 }

cmd/control/service.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/codegangsta/cli"
 	"github.com/rancherio/os/config"
+	"github.com/rancherio/os/util"
 )
 
 func serviceSubCommands() []cli.Command {
@@ -36,16 +37,16 @@ func disable(c *cli.Context) {
 	}
 
 	for _, service := range c.Args() {
-		if _, ok := cfg.Services[service]; !ok {
+		if _, ok := cfg.ServicesInclude[service]; !ok {
 			continue
 		}
 
-		cfg.Services[service] = false
+		cfg.ServicesInclude[service] = false
 		changed = true
 	}
 
 	if changed {
-		if err = cfg.Set("services", cfg.Services); err != nil {
+		if err = cfg.Set("services_include", cfg.ServicesInclude); err != nil {
 			log.Fatal(err)
 		}
 	}
@@ -59,14 +60,14 @@ func enable(c *cli.Context) {
 	}
 
 	for _, service := range c.Args() {
-		if val, ok := cfg.Services[service]; !ok || !val {
+		if val, ok := cfg.ServicesInclude[service]; !ok || !val {
-			cfg.Services[service] = true
+			cfg.ServicesInclude[service] = true
 			changed = true
 		}
 	}
 
 	if changed {
-		if err = cfg.Set("services", cfg.Services); err != nil {
+		if err = cfg.Set("services_include", cfg.ServicesInclude); err != nil {
 			log.Fatal(err)
 		}
 	}
@@ -78,7 +79,30 @@ func list(c *cli.Context) {
 		log.Fatal(err)
 	}
 
-	for service, enabled := range cfg.Services {
+	clone := make(map[string]bool)
+	for service, enabled := range cfg.ServicesInclude {
+		clone[service] = enabled
+	}
+
+	services, err := util.GetServices(cfg.Repositories.ToArray())
+	if err != nil {
+		log.Fatalf("Failed to get services: %v", err)
+	}
+
+	for _, service := range services {
+		if enabled, ok := clone[service]; ok {
+			delete(clone, service)
+			if enabled {
+				fmt.Printf("enabled  %s\n", service)
+			} else {
+				fmt.Printf("disabled %s\n", service)
+			}
+		} else {
+			fmt.Printf("disabled %s\n", service)
+		}
+	}
+
+	for service, enabled := range clone {
 		if enabled {
 			fmt.Printf("enabled  %s\n", service)
 		} else {

config/config.go

@@ -264,3 +264,14 @@ func (d *DockerConfig) BridgeConfig() (string, string) {
 		return name, cidr
 	}
 }
+
+func (r Repositories) ToArray() []string {
+	result := make([]string, 0, len(r))
+	for _, repo := range r {
+		if repo.Url != "" {
+			result = append(result, repo.Url)
+		}
+	}
+
+	return result
+}

config/default.go

@@ -81,6 +81,7 @@ func NewConfig() *Config {
 				Privileged: true,
 				Labels: []string{
 					DETACH + "=false",
+					SCOPE + "=" + SYSTEM,
 				},
 				Volumes: []string{
 					"/dev:/host/dev",
@@ -98,6 +99,7 @@ func NewConfig() *Config {
 				Privileged: true,
 				Labels: []string{
 					DETACH + "=true",
+					SCOPE + "=" + SYSTEM,
 				},
 				Environment: []string{
 					"DAEMON=true",
@@ -113,6 +115,7 @@ func NewConfig() *Config {
 				Privileged: true,
 				Labels: []string{
 					CREATE_ONLY + "=true",
+					SCOPE + "=" + SYSTEM,
 				},
 				Volumes: []string{
 					"/dev:/host/dev",
@@ -132,6 +135,7 @@ func NewConfig() *Config {
 				Privileged: true,
 				Labels: []string{
 					CREATE_ONLY + "=true",
+					SCOPE + "=" + SYSTEM,
 				},
 				Volumes: []string{
 					"/init:/sbin/halt:ro",
@@ -156,6 +160,7 @@ func NewConfig() *Config {
 				Privileged: true,
 				Labels: []string{
 					CREATE_ONLY + "=true",
+					SCOPE + "=" + SYSTEM,
 				},
 				Volumes: []string{
 					"/home:/home",
@@ -170,6 +175,7 @@ func NewConfig() *Config {
 				Privileged: true,
 				Labels: []string{
 					CREATE_ONLY + "=true",
+					SCOPE + "=" + SYSTEM,
 				},
 				Volumes: []string{
 					"/var/lib/rancher:/var/lib/rancher",
@@ -185,6 +191,7 @@ func NewConfig() *Config {
 				Privileged: true,
 				Labels: []string{
 					CREATE_ONLY + "=true",
+					SCOPE + "=" + SYSTEM,
 				},
 				VolumesFrom: []string{
 					"docker-volumes",
@@ -201,6 +208,7 @@ func NewConfig() *Config {
 				Labels: []string{
 					RELOAD_CONFIG + "=true",
 					DETACH + "=false",
+					SCOPE + "=" + SYSTEM,
 				},
 				Environment: []string{
 					"CLOUD_INIT_NETWORK=false",
@@ -216,6 +224,7 @@ func NewConfig() *Config {
 				Net:        "host",
 				Labels: []string{
 					DETACH + "=false",
+					SCOPE + "=" + SYSTEM,
 				},
 				Links: []string{
 					"cloud-init-pre",
@@ -231,6 +240,7 @@ func NewConfig() *Config {
 				Labels: []string{
 					RELOAD_CONFIG + "=true",
 					DETACH + "=false",
+					SCOPE + "=" + SYSTEM,
 				},
 				Net: "host",
 				Links: []string{
@@ -246,6 +256,9 @@ func NewConfig() *Config {
 				Image:      "ntp",
 				Privileged: true,
 				Net:        "host",
+				Labels: []string{
+					SCOPE + "=" + SYSTEM,
+				},
 				Links: []string{
 					"cloud-init",
 					"network",
@@ -255,6 +268,9 @@ func NewConfig() *Config {
 				Image:      "syslog",
 				Privileged: true,
 				Net:        "host",
+				Labels: []string{
+					SCOPE + "=" + SYSTEM,
+				},
 				VolumesFrom: []string{
 					"system-volumes",
 				},
@@ -266,6 +282,9 @@ func NewConfig() *Config {
 				Pid:        "host",
 				Ipc:        "host",
 				Net:        "host",
+				Labels: []string{
+					SCOPE + "=" + SYSTEM,
+				},
 				Links: []string{
 					"network",
 				},
@@ -277,7 +296,8 @@ func NewConfig() *Config {
 				Image: "userdockerwait",
 				Net:   "host",
 				Labels: []string{
-					"io.rancher.os.detach=false",
+					DETACH + "=false",
+					SCOPE + "=" + SYSTEM,
 				},
 				Links: []string{
 					"userdocker",
@@ -292,6 +312,9 @@ func NewConfig() *Config {
 				Links: []string{
 					"cloud-init",
 				},
+				Labels: []string{
+					SCOPE + "=" + SYSTEM,
+				},
 				VolumesFrom: []string{
 					"all-volumes",
 				},
@@ -301,31 +324,14 @@ func NewConfig() *Config {
 				Net:     "host",
 			},
 		},
-		Services: map[string]bool{
+		ServicesInclude: map[string]bool{
 			"ubuntu-console": false,
 		},
-		BundledServices: map[string]Config{
+		Repositories: map[string]Repository{
-			"ubuntu-console": {
+			"core": Repository{
-				SystemContainers: map[string]*project.ServiceConfig{
+				Url: "https://raw.githubusercontent.com/rancherio/os-services/master/",
-					"console": {
-						Image:      "rancher/ubuntuconsole:" + IMAGE_VERSION,
-						Privileged: true,
-						Labels: []string{
-							DETACH + "=true",
-						},
-						Links: []string{
-							"cloud-init",
-						},
-						VolumesFrom: []string{
-							"all-volumes",
-						},
-						Restart: "always",
-						Pid:     "host",
-						Ipc:     "host",
-						Net:     "host",
-					},
-				},
 			},
 		},
+		Services: map[string]*project.ServiceConfig{},
 	}
 }

config/types.go

@@ -23,6 +23,8 @@ const (
 	REMOVE        = "io.rancher.os.remove"
 	CREATE_ONLY   = "io.rancher.os.createonly"
 	RELOAD_CONFIG = "io.rancher.os.reloadconfig"
+	SCOPE         = "io.rancher.os.scope"
+	SYSTEM        = "system"
 )
 
 var (
@@ -42,18 +44,25 @@ type ContainerConfig struct {
 	Service        *project.ServiceConfig `yaml:service,omitempty`
 }
 
+type Repository struct {
+	Url string `yaml:url,omitempty`
+}
+
+type Repositories map[string]Repository
+
 type Config struct {
 	Environment         map[string]string                 `yaml:"environment,omitempty"`
-	BundledServices     map[string]Config                 `yaml:"bundled_services,omitempty"`
+	Services            map[string]*project.ServiceConfig `yaml:"services,omitempty"`
 	BootstrapContainers map[string]*project.ServiceConfig `yaml:"bootstrap_containers,omitempty"`
 	BootstrapDocker     DockerConfig                      `yaml:"bootstrap_docker,omitempty"`
 	CloudInit           CloudInit                         `yaml:"cloud_init,omitempty"`
 	Console             ConsoleConfig                     `yaml:"console,omitempty"`
 	Debug               bool                              `yaml:"debug,omitempty"`
 	Disable             []string                          `yaml:"disable,omitempty"`
-	Services            map[string]bool                   `yaml:"services,omitempty"`
+	ServicesInclude     map[string]bool                   `yaml:"services_include,omitempty"`
 	Modules             []string                          `yaml:"modules,omitempty"`
 	Network             NetworkConfig                     `yaml:"network,omitempty"`
+	Repositories        Repositories                      `yaml:"repositories,omitempty"`
 	Ssh                 SshConfig                         `yaml:"ssh,omitempty"`
 	State               StateConfig                       `yaml:"state,omitempty"`
 	SystemContainers    map[string]*project.ServiceConfig `yaml:"system_containers,omitempty"`

docker/container.go

@@ -71,6 +71,7 @@ func NewContainerFromService(dockerHost string, name string, service *project.Se
 			Service: service,
 		},
 	}
+
 	return c.Parse()
 }
 
@@ -174,34 +175,53 @@ func (c *Container) requiresSyslog() bool {
 	return (c.ContainerCfg.Service.LogDriver == "" || c.ContainerCfg.Service.LogDriver == "syslog")
 }
 
+func (c *Container) requiresUserDocker() bool {
+	if c.dockerHost == config.DOCKER_HOST {
+		return true
+	}
+
+	for _, v := range c.ContainerCfg.Service.Volumes {
+		if strings.Index(v, "/var/run/docker.sock") != -1 {
+			return true
+		}
+	}
+
+	return false
+}
+
 func (c *Container) hasLink(link string) bool {
 	return util.Contains(c.ContainerCfg.Service.Links, link)
 }
 
 func (c *Container) addLink(link string) {
+	if c.hasLink(link) {
+		return
+	}
+
+	log.Debugf("Adding %s link to %s", link, c.Name)
 	c.ContainerCfg.Service.Links = append(c.ContainerCfg.Service.Links, link)
 }
 
 func (c *Container) parseService() {
+	if c.requiresSyslog() {
+		c.addLink("syslog")
+	}
+
+	if c.requiresUserDocker() {
+		c.addLink("userdockerwait")
+	} else if c.ContainerCfg.Service.Image != "" {
 		client, err := NewClient(c.dockerHost)
 		if err != nil {
 			c.Err = err
 			return
 		}
 
-	if c.ContainerCfg.Service.Image != "" {
 		i, _ := client.InspectImage(c.ContainerCfg.Service.Image)
-		if i == nil && !c.hasLink("network") {
+		if i == nil {
-			log.Debugf("Adding network link to %s", c.Name)
 			c.addLink("network")
 		}
 	}
 
-	if c.requiresSyslog() && !c.hasLink("syslog") {
-		log.Debugf("Adding syslog link to %s\n", c.Name)
-		c.addLink("syslog")
-	}
-
 	cfg, hostConfig, err := docker.Convert(c.ContainerCfg.Service)
 	if err != nil {
 		c.Err = err
@@ -215,7 +235,6 @@ func (c *Container) parseService() {
 	c.remove = c.Config.Labels[config.REMOVE] != "false"
 	c.ContainerCfg.CreateOnly = c.Config.Labels[config.CREATE_ONLY] == "true"
 	c.ContainerCfg.ReloadConfig = c.Config.Labels[config.RELOAD_CONFIG] == "true"
-
 }
 
 func (c *Container) parseCmd() {

docker/factory.go

@@ -38,6 +38,8 @@ func (c *containerBasedService) Up() error {
 
 	var event project.Event
 
+	c.project.Notify(project.CONTAINER_STARTING, c, map[string]string{})
+
 	if create {
 		container.Create()
 		event = project.CONTAINER_CREATED
@@ -71,8 +73,17 @@ func (c *containerBasedService) Name() string {
 	return c.name
 }
 
+func isSystemService(serviceConfig *project.ServiceConfig) bool {
+	return util.GetValue(serviceConfig.Labels, config.SCOPE) == config.SYSTEM
+}
+
 func (c *ContainerFactory) Create(project *project.Project, name string, serviceConfig *project.ServiceConfig) (project.Service, error) {
-	container := NewContainerFromService(config.DOCKER_SYSTEM_HOST, name, serviceConfig)
+	host := config.DOCKER_HOST
+	if isSystemService(serviceConfig) {
+		host = config.DOCKER_SYSTEM_HOST
+	}
+
+	container := NewContainerFromService(host, name, serviceConfig)
 
 	if container.Err != nil {
 		return nil, container.Err

docker/services.go

@@ -0,0 +1,123 @@
+package docker
+
+import (
+	"fmt"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/rancherio/os/config"
+	"github.com/rancherio/os/util"
+	"github.com/rancherio/rancher-compose/project"
+)
+
+type configEnvironemnt struct {
+	cfg *config.Config
+}
+
+func (c *configEnvironemnt) Lookup(key, serviceName string, serviceConfig *project.ServiceConfig) []string {
+	result := ""
+	fullKey := fmt.Sprintf("%s/%s", serviceName, key)
+	if value, ok := c.cfg.Environment[fullKey]; ok {
+		result = value
+	} else if value, ok := c.cfg.Environment[key]; ok {
+		result = value
+	}
+
+	if result == "" {
+		return []string{}
+	} else {
+		return []string{fmt.Sprintf("%s=%s", key, result)}
+	}
+}
+
+func RunServices(name string, cfg *config.Config, configs map[string]*project.ServiceConfig) error {
+	network := false
+	projectEvents := make(chan project.ProjectEvent)
+	p := project.NewProject(name, NewContainerFactory(cfg))
+	p.EnvironmentLookup = &configEnvironemnt{cfg: cfg}
+	p.AddListener(projectEvents)
+	enabled := make(map[string]bool)
+
+	for name, serviceConfig := range configs {
+		if err := p.AddConfig(name, serviceConfig); err != nil {
+			log.Infof("Failed loading service %s", name)
+		}
+	}
+
+	p.ReloadCallback = func() error {
+		err := cfg.Reload()
+		if err != nil {
+			return err
+		}
+
+		for service, serviceEnabled := range cfg.ServicesInclude {
+			if !serviceEnabled {
+				continue
+			}
+
+			if _, ok := enabled[service]; ok {
+				continue
+			}
+
+			//if config, ok := cfg.BundledServices[service]; ok {
+			//	for name, s := range config.SystemContainers {
+			//		if err := p.AddConfig(name, s); err != nil {
+			//			log.Errorf("Failed to load %s : %v", name, err)
+			//		}
+			//	}
+			//} else {
+			bytes, err := LoadServiceResource(service, network, cfg)
+			if err != nil {
+				if err == util.ErrNoNetwork {
+					log.Debugf("Can not load %s, networking not enabled", service)
+				} else {
+					log.Errorf("Failed to load %s : %v", service, err)
+				}
+				continue
+			}
+
+			err = p.Load(bytes)
+			if err != nil {
+				log.Errorf("Failed to load %s : %v", service, err)
+				continue
+			}
+			//}
+
+			enabled[service] = true
+		}
+
+		for service, config := range cfg.Services {
+			if _, ok := enabled[service]; ok {
+				continue
+			}
+
+			err = p.AddConfig(service, config)
+			if err != nil {
+				log.Errorf("Failed to load %s : %v", service, err)
+				continue
+			}
+
+			enabled[service] = true
+		}
+
+		return nil
+	}
+
+	go func() {
+		for event := range projectEvents {
+			if event.Event == project.CONTAINER_STARTED && event.Service.Name() == "network" {
+				network = true
+			}
+		}
+	}()
+
+	err := p.ReloadCallback()
+	if err != nil {
+		log.Errorf("Failed to reload %s : %v", name, err)
+		return err
+	}
+	return p.Up()
+}
+
+func LoadServiceResource(name string, network bool, cfg *config.Config) ([]byte, error) {
+	return util.LoadResource(name, network, cfg.Repositories.ToArray())
+}

init/bootstrap.go

@@ -7,6 +7,7 @@ import (
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/rancherio/os/config"
+	"github.com/rancherio/os/docker"
 	"github.com/rancherio/os/util"
 	"github.com/rancherio/rancher-compose/project"
 )
@@ -55,7 +56,7 @@ outer:
 
 	if format != "" {
 		log.Infof("Auto formatting : %s", format)
-		return runServices("autoformat", cfg, map[string]*project.ServiceConfig{
+		return docker.RunServices("autoformat", cfg, map[string]*project.ServiceConfig{
 			"autoformat": {
 				Net:        "none",
 				Privileged: true,
@@ -70,7 +71,7 @@ outer:
 }
 
 func runBootstrapContainers(cfg *config.Config) error {
-	return runServices("bootstrap", cfg, cfg.BootstrapContainers)
+	return docker.RunServices("bootstrap", cfg, cfg.BootstrapContainers)
 }
 
 func startDocker(cfg *config.Config) (chan interface{}, error) {

init/init.go

@@ -3,7 +3,6 @@ package init
 import (
 	"fmt"
 	"io/ioutil"
-	"net"
 	"os"
 	"os/exec"
 	"strings"
@@ -17,6 +16,7 @@ import (
 
 const (
 	STATE         string = "/var"
+	SYSTEM_DOCKER string = "/usr/bin/system-docker"
 	DOCKER        string = "/usr/bin/docker"
 	SYSINIT       string = "/sbin/rancher-sysinit"
 )
@@ -63,6 +63,7 @@ var (
 		"/sbin/modprobe":                     "/busybox",
 		"/usr/sbin/iptables":                 "/xtables-multi",
 		DOCKER:                               "/docker",
+		SYSTEM_DOCKER:                        "/docker",
 		SYSINIT:                              "/init",
 		"/home":                              "/var/lib/rancher/state/home",
 		"/opt":                               "/var/lib/rancher/state/opt",
@@ -237,7 +238,7 @@ func execDocker(cfg *config.Config) error {
 	}
 
 	os.Stdin.Close()
-	return syscall.Exec(DOCKER, cfg.SystemDocker.Args, os.Environ())
+	return syscall.Exec(SYSTEM_DOCKER, cfg.SystemDocker.Args, os.Environ())
 }
 
 func MainInit() {
@@ -285,10 +286,9 @@ func touchSocket(cfg *config.Config) error {
 		if err := syscall.Unlink(path); err != nil && !os.IsNotExist(err) {
 			return err
 		}
-		if l, err := net.Listen("unix", path); err != nil {
+		err := ioutil.WriteFile(path, []byte{}, 0700)
+		if err != nil {
 			return err
-		} else {
-			l.Close()
 		}
 	}
 
@@ -365,7 +365,8 @@ func RunInit() error {
 			return createMounts(postMounts...)
 		},
 		touchSocket,
-		remountRo,
+		// Disable R/O root write now to support updating modules
+		//remountRo,
 		sysInit,
 	}
 

init/sysinit.go

@@ -9,9 +9,6 @@ import (
 	dockerClient "github.com/fsouza/go-dockerclient"
 	"github.com/rancherio/os/config"
 	"github.com/rancherio/os/docker"
-	"github.com/rancherio/os/util"
-
-	"github.com/rancherio/rancher-compose/project"
 )
 
 func importImage(client *dockerClient.Client, name, fileName string) error {
@@ -109,82 +106,8 @@ func loadImages(cfg *config.Config) error {
 	return nil
 }
 
-func runServices(name string, cfg *config.Config, configs map[string]*project.ServiceConfig) error {
-	network := false
-	projectEvents := make(chan project.ProjectEvent)
-	p := project.NewProject(name, docker.NewContainerFactory(cfg))
-	p.AddListener(projectEvents)
-	enabled := make(map[string]bool)
-
-	for name, serviceConfig := range configs {
-		if err := p.AddConfig(name, serviceConfig); err != nil {
-			log.Infof("Failed loading service %s", name)
-		}
-	}
-
-	p.ReloadCallback = func() error {
-		err := cfg.Reload()
-		if err != nil {
-			return err
-		}
-
-		for service, serviceEnabled := range cfg.Services {
-			if !serviceEnabled {
-				continue
-			}
-
-			if _, ok := enabled[service]; ok {
-				continue
-			}
-
-			if config, ok := cfg.BundledServices[service]; ok {
-				for name, s := range config.SystemContainers {
-					if err := p.AddConfig(name, s); err != nil {
-						log.Errorf("Failed to load %s : %v", name, err)
-					}
-				}
-			} else {
-				bytes, err := util.LoadResource(service, network)
-				if err != nil {
-					if err == util.ErrNoNetwork {
-						log.Debugf("Can not load %s, networking not enabled", service)
-					} else {
-						log.Errorf("Failed to load %s : %v", service, err)
-					}
-					continue
-				}
-
-				err = p.Load(bytes)
-				if err != nil {
-					log.Errorf("Failed to load %s : %v", service, err)
-					continue
-				}
-			}
-
-			enabled[service] = true
-		}
-
-		return nil
-	}
-
-	go func() {
-		for event := range projectEvents {
-			if event.Event == project.CONTAINER_STARTED && event.Service.Name() == "network" {
-				network = true
-			}
-		}
-	}()
-
-	err := p.ReloadCallback()
-	if err != nil {
-		log.Errorf("Failed to reload %s : %v", name, err)
-		return err
-	}
-	return p.Up()
-}
-
 func runContainers(cfg *config.Config) error {
-	return runServices("system-init", cfg, cfg.SystemContainers)
+	return docker.RunServices("system-init", cfg, cfg.SystemContainers)
 }
 
 func tailConsole(cfg *config.Config) error {

util/util.go

@@ -13,6 +13,8 @@ import (
 	"strings"
 	"syscall"
 
+	log "github.com/Sirupsen/logrus"
+
 	"github.com/docker/docker/pkg/mount"
 	"gopkg.in/yaml.v2"
 )
@@ -20,6 +22,7 @@ import (
 var (
 	letters      = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
 	ErrNoNetwork = errors.New("Networking not available to load resource")
+	ErrNotFound  = errors.New("Failed to find resource")
 )
 
 func mountProc() error {
@@ -207,7 +210,36 @@ func MergeMaps(left, right map[interface{}]interface{}) {
 	}
 }
 
-func LoadResource(location string, network bool) ([]byte, error) {
+func GetServices(urls []string) ([]string, error) {
+	result := []string{}
+
+	for _, url := range urls {
+		indexUrl := fmt.Sprintf("%s/index.yml", url)
+		content, err := LoadResource(indexUrl, true, []string{})
+		if err != nil {
+			log.Errorf("Failed to load %s: %v", indexUrl, err)
+			continue
+		}
+
+		services := make(map[string][]string)
+		err = yaml.Unmarshal(content, &services)
+		if err != nil {
+			log.Errorf("Failed to unmarshal %s: %v", indexUrl, err)
+			continue
+		}
+
+		if list, ok := services["services"]; ok {
+			result = append(result, list...)
+		}
+	}
+
+	return []string{}, nil
+}
+
+func LoadResource(location string, network bool, urls []string) ([]byte, error) {
+	var bytes []byte
+	err := ErrNotFound
+
 	if strings.HasPrefix(location, "http:/") || strings.HasPrefix(location, "https:/") {
 		if !network {
 			return nil, ErrNoNetwork
@@ -216,9 +248,38 @@ func LoadResource(location string, network bool) ([]byte, error) {
 		if err != nil {
 			return nil, err
 		}
+		if resp.StatusCode != http.StatusOK {
+			return nil, fmt.Errorf("non-200 http response: %d", resp.StatusCode)
+		}
 		defer resp.Body.Close()
 		return ioutil.ReadAll(resp.Body)
-	} else {
+	} else if strings.HasPrefix(location, "/") {
 		return ioutil.ReadFile(location)
+	} else if len(location) > 0 {
+		for _, url := range urls {
+			ymlUrl := fmt.Sprintf("%s/%s/%s.yml", url, location[0:1], location)
+			log.Infof("Loading %s from %s", location, ymlUrl)
+			bytes, err = LoadResource(ymlUrl, network, []string{})
+			if err == nil {
+				return bytes, nil
 			}
+		}
+	}
+
+	return nil, err
+}
+
+func GetValue(kvPairs []string, key string) string {
+	if kvPairs == nil {
+		return ""
+	}
+
+	prefix := key + "="
+	for _, i := range kvPairs {
+		if strings.HasPrefix(i, prefix) {
+			return strings.TrimPrefix(i, prefix)
+		}
+	}
+
+	return ""
 }