Add multi docker command

cmd/control/console_init.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/codegangsta/cli"
 	"github.com/rancher/os/cmd/cloudinitexecute"
+	"github.com/rancher/os/compose"
 	"github.com/rancher/os/config"
 	"github.com/rancher/os/config/cmdline"
 	"github.com/rancher/os/log"
@@ -90,6 +91,27 @@ func consoleInitFunc() error {
 		log.Error(err)
 	}
 
+	p, err := compose.GetProject(cfg, false, true)
+	if err != nil {
+		log.Error(err)
+	}
+
+	// check the multi engine service & generate the multi engine script
+	for _, key := range p.ServiceConfigs.Keys() {
+		serviceConfig, ok := p.ServiceConfigs.Get(key)
+		if !ok {
+			log.Errorf("Failed to get service config from the project")
+			continue
+		}
+		if _, ok := serviceConfig.Labels[config.UserDockerLabel]; ok {
+			err = util.GenerateEngineScript(serviceConfig.Labels[config.UserDockerLabel])
+			if err != nil {
+				log.Errorf("Failed to generate engine script: %v", err)
+				continue
+			}
+		}
+	}
+
 	for _, link := range []symlink{
 		{"/var/lib/rancher/engine/docker", "/usr/bin/docker"},
 		{"/var/lib/rancher/engine/docker-init", "/usr/bin/docker-init"},

cmd/control/engine.go

@@ -2,15 +2,27 @@ package control
 
 import (
 	"fmt"
+	"io/ioutil"
+	"net"
+	"os"
+	"os/user"
+	"path"
 	"sort"
+	"strconv"
 	"strings"
 
 	"golang.org/x/net/context"
 
+	yaml "github.com/cloudfoundry-incubator/candiedyaml"
 	"github.com/codegangsta/cli"
 	"github.com/docker/docker/reference"
+	"github.com/docker/engine-api/types"
+	composeConfig "github.com/docker/libcompose/config"
 	"github.com/docker/libcompose/project/options"
+	composeYaml "github.com/docker/libcompose/yaml"
+	"github.com/pkg/errors"
 	"github.com/rancher/os/cmd/control/service"
+	"github.com/rancher/os/cmd/control/service/app"
 	"github.com/rancher/os/compose"
 	"github.com/rancher/os/config"
 	"github.com/rancher/os/docker"
@@ -19,6 +31,11 @@ import (
 	"github.com/rancher/os/util/network"
 )
 
+var (
+	SupportedEngineVersions = []string{"docker:17.12.1-dind", "docker:18.03-dind", "docker:18.03.1-dind"}
+	SSHKeyPathDefault       = "%s/.ssh/authorized_keys"
+)
+
 func engineSubcommands() []cli.Command {
 	return []cli.Command{
 		{
@@ -36,6 +53,64 @@ func engineSubcommands() []cli.Command {
 				},
 			},
 		},
+		{
+			Name:      "create",
+			Usage:     "create Docker engine without a reboot",
+			ArgsUsage: "<name>",
+			Before:    preFlightValidate,
+			Action:    engineCreate,
+			Flags: []cli.Flag{
+				cli.StringFlag{
+					Name:  "version, v",
+					Value: SupportedEngineVersions[0],
+					Usage: "set the version for the engine",
+				},
+				cli.StringFlag{
+					Name:  "network",
+					Usage: "set the network for the engine",
+				},
+				cli.StringFlag{
+					Name:  "fixed-ip",
+					Usage: "set the fix ip for the engine",
+				},
+				cli.IntFlag{
+					Name:  "ssh-port",
+					Value: randomSSHPort(),
+					Usage: "set the ssh port for the engine",
+				},
+				cli.StringFlag{
+					Name:  "authorized-keys",
+					Usage: "set the ssh authorized_keys path for the engine",
+				},
+			},
+		},
+		{
+			Name:      "rm",
+			Usage:     "remove Docker engine without a reboot",
+			ArgsUsage: "<name>",
+			Before: func(c *cli.Context) error {
+				if len(c.Args()) != 1 {
+					return errors.New("Must specify exactly one Docker engine to remove")
+				}
+				return nil
+			},
+			Action: engineRemove,
+			Flags: []cli.Flag{
+				cli.IntFlag{
+					Name:  "timeout,t",
+					Usage: "Specify a shutdown timeout in seconds.",
+					Value: 10,
+				},
+				cli.BoolFlag{
+					Name:  "force,f",
+					Usage: "Allow deletion of all services",
+				},
+				cli.BoolFlag{
+					Name:  "v",
+					Usage: "Remove volumes associated with containers",
+				},
+			},
+		},
 		{
 			Name:   "enable",
 			Usage:  "set Docker engine to be switched on next reboot",
@@ -82,6 +157,96 @@ func engineSwitch(c *cli.Context) error {
 	return nil
 }
 
+func engineCreate(c *cli.Context) error {
+	name := c.Args()[0]
+	version := c.String("version")
+	sshPort := c.Int("ssh-port")
+	authorizedKeys := c.String("authorized-keys")
+	network := c.String("network")
+	fixedIP := c.String("fixed-ip")
+
+	if authorizedKeys == "" {
+		authorizedKeys = authorizedKeysPath()
+	}
+
+	// generate & create engine compose
+	err := generateEngineCompose(name, version, sshPort, authorizedKeys, network, fixedIP)
+	if err != nil {
+		return err
+	}
+
+	// stage engine service
+	cfg := config.LoadConfig()
+	var enabledServices []string
+	if val, ok := cfg.Rancher.ServicesInclude[name]; !ok || !val {
+		cfg.Rancher.ServicesInclude[name] = true
+		enabledServices = append(enabledServices, name)
+	}
+
+	if len(enabledServices) > 0 {
+		if err := compose.StageServices(cfg, enabledServices...); err != nil {
+			log.Fatal(err)
+		}
+
+		if err := config.Set("rancher.services_include", cfg.Rancher.ServicesInclude); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	// generate engine script
+	err = util.GenerateEngineScript(name)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return nil
+}
+
+func engineRemove(c *cli.Context) error {
+	name := c.Args()[0]
+	cfg := config.LoadConfig()
+	p, err := compose.GetProject(cfg, true, false)
+	if err != nil {
+		log.Fatalf("Get project failed: %v", err)
+	}
+
+	// 1. service stop
+	err = app.ProjectStop(p, c)
+	if err != nil {
+		log.Fatalf("Stop project service failed: %v", err)
+	}
+
+	// 2. service delete
+	err = app.ProjectDelete(p, c)
+	if err != nil {
+		log.Fatalf("Delete project service failed: %v", err)
+	}
+
+	// 3. service delete
+	changed := false
+
+	if _, ok := cfg.Rancher.ServicesInclude[name]; !ok {
+		log.Fatalf("Failed to found enabled service %s", name)
+	}
+
+	delete(cfg.Rancher.ServicesInclude, name)
+	changed = true
+
+	if changed {
+		if err = config.Set("rancher.services_include", cfg.Rancher.ServicesInclude); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	// 4. remove service from file
+	err = RemoveEngineFromCompose(name)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return nil
+}
+
 func engineEnable(c *cli.Context) error {
 	if len(c.Args()) != 1 {
 		log.Fatal("Must specify exactly one Docker engine to enable")
@@ -166,3 +331,207 @@ func CurrentEngine() (engine string) {
 
 	return
 }
+
+func preFlightValidate(c *cli.Context) error {
+	if len(c.Args()) != 1 {
+		return errors.New("Must specify one engine name")
+	}
+	name := c.Args()[0]
+	if name == "" {
+		return errors.New("Must specify one engine name")
+	}
+
+	version := c.String("version")
+	if version == "" {
+		return errors.New("Must specify one engine version")
+	}
+
+	port := c.Int("ssh-port")
+	if port == 0 {
+		return errors.New("Must specify one engine ssh port")
+	}
+
+	network := c.String("network")
+	if network == "" {
+		return errors.New("Must specify network")
+	}
+
+	userDefineNetwork, err := CheckUserDefineNetwork(network)
+	if err != nil {
+		return err
+	}
+
+	fixedIP := c.String("fixed-ip")
+	if fixedIP == "" {
+		return errors.New("Must specify fix ip")
+	}
+
+	err = CheckUserDefineIPv4Address(fixedIP, *userDefineNetwork)
+	if err != nil {
+		return err
+	}
+
+	isVersionMatch := false
+	for _, v := range SupportedEngineVersions {
+		if v == version {
+			isVersionMatch = true
+			break
+		}
+	}
+
+	if !isVersionMatch {
+		return errors.Errorf("Engine version not supported only %v are supported", SupportedEngineVersions)
+	}
+
+	addr, err := net.ResolveTCPAddr("tcp", "localhost:"+strconv.Itoa(port))
+	if err != nil {
+		return errors.Errorf("Failed to resolve tcp addr: %v", err)
+	}
+	l, err := net.ListenTCP("tcp", addr)
+	if err != nil {
+		return errors.Errorf("Failed to listen tcp: %v", err)
+	}
+	defer l.Close()
+
+	return nil
+}
+
+func authorizedKeysPath() string {
+	home := "/home/rancher"
+	user, err := user.Current()
+	if err == nil {
+		home = user.HomeDir
+	}
+
+	return fmt.Sprintf(SSHKeyPathDefault, home)
+}
+
+func randomSSHPort() int {
+	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
+	if err != nil {
+		log.Errorf("Failed to resolve tcp addr: %v", err)
+		return 0
+	}
+
+	l, err := net.ListenTCP("tcp", addr)
+	if err != nil {
+		return 0
+	}
+	defer l.Close()
+	return l.Addr().(*net.TCPAddr).Port
+}
+
+func generateEngineCompose(name, version string, sshPort int, authorizedKeys, network, fixedIP string) error {
+	if err := os.MkdirAll(path.Dir(config.MultiDockerConfFile), 0700); err != nil && !os.IsExist(err) {
+		log.Errorf("Failed to create directory for file %s: %v", config.MultiDockerConfFile, err)
+		return err
+	}
+
+	composeConfigs := map[string]composeConfig.ServiceConfigV1{}
+
+	if _, err := os.Stat(config.MultiDockerConfFile); err == nil {
+		// read from engine compose
+		bytes, err := ioutil.ReadFile(config.MultiDockerConfFile)
+		if err != nil {
+			return err
+		}
+		err = yaml.Unmarshal(bytes, &composeConfigs)
+		if err != nil {
+			return err
+		}
+	}
+
+	if err := os.MkdirAll(config.MultiDockerDataDir+"/"+name, 0700); err != nil && !os.IsExist(err) {
+		log.Errorf("Failed to create directory for file %s: %v", config.MultiDockerDataDir+"/"+name, err)
+		return err
+	}
+
+	composeConfigs[name] = composeConfig.ServiceConfigV1{
+		Image:      "${REGISTRY_DOMAIN}/" + version,
+		Restart:    "always",
+		Privileged: true,
+		Net:        network,
+		Ports:      []string{strconv.Itoa(sshPort) + ":22"},
+		Volumes: []string{
+			"/lib/modules:/lib/modules",
+			config.MultiDockerDataDir + "/" + name + ":" + config.MultiDockerDataDir + "/" + name,
+			authorizedKeys + ":/root/.ssh/authorized_keys",
+		},
+		VolumesFrom: []string{},
+		Command: composeYaml.Command{
+			"dockerd-entrypoint.sh",
+			"--storage-driver=overlay2",
+			"--data-root=" + config.MultiDockerDataDir + "/" + name,
+			"--host=unix://" + config.MultiDockerDataDir + "/" + name + "/docker-" + name + ".sock",
+		},
+		Labels: composeYaml.SliceorMap{
+			"io.rancher.os.scope":     "system",
+			"io.rancher.os.after":     "console",
+			config.UserDockerLabel:    name,
+			config.UserDockerNetLabel: network,
+			config.UserDockerFIPLabel: fixedIP,
+		},
+	}
+
+	bytes, err := yaml.Marshal(composeConfigs)
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(config.MultiDockerConfFile, bytes, 0640)
+}
+
+func RemoveEngineFromCompose(name string) error {
+	composeConfigs := map[string]composeConfig.ServiceConfigV1{}
+
+	if _, err := os.Stat(config.MultiDockerConfFile); err == nil {
+		// read from engine compose
+		bytes, err := ioutil.ReadFile(config.MultiDockerConfFile)
+		if err != nil {
+			return err
+		}
+		err = yaml.Unmarshal(bytes, &composeConfigs)
+		if err != nil {
+			return err
+		}
+	}
+
+	delete(composeConfigs, name)
+
+	bytes, err := yaml.Marshal(composeConfigs)
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(config.MultiDockerConfFile, bytes, 0640)
+}
+
+func CheckUserDefineNetwork(name string) (*types.NetworkResource, error) {
+	systemClient, err := docker.NewSystemClient()
+	if err != nil {
+		return nil, err
+	}
+
+	networks, err := systemClient.NetworkList(context.Background(), types.NetworkListOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	for _, network := range networks {
+		if network.Name == name {
+			return &network, nil
+		}
+	}
+
+	return nil, errors.Errorf("Failed to found the user define network: %s", name)
+}
+
+func CheckUserDefineIPv4Address(ipv4 string, network types.NetworkResource) error {
+	for _, config := range network.IPAM.Config {
+		_, ipnet, _ := net.ParseCIDR(config.Subnet)
+		if ipnet.Contains(net.ParseIP(ipv4)) {
+			return nil
+		}
+	}
+	return errors.Errorf("IP %s is not in the specified cidr", ipv4)
+}

cmd/control/service/app/app.go

@@ -85,6 +85,7 @@ func ProjectUp(p project.APIProject, c *cli.Context) error {
 	if err != nil {
 		return cli.NewExitError(err.Error(), 1)
 	}
+
 	if c.Bool("foreground") {
 		signalChan := make(chan os.Signal, 1)
 		cleanupDone := make(chan bool)

compose/project.go

@@ -2,6 +2,8 @@ package compose
 
 import (
 	"fmt"
+	"io/ioutil"
+	"os"
 
 	"golang.org/x/net/context"
 
@@ -224,8 +226,31 @@ func StageServices(cfg *config.CloudConfig, services ...string) error {
 		return err
 	}
 
+	// read engine services
+	composeConfigs := map[string]composeConfig.ServiceConfigV1{}
+	if _, err := os.Stat(config.MultiDockerConfFile); err == nil {
+		// read from engine compose
+		multiEngineBytes, err := ioutil.ReadFile(config.MultiDockerConfFile)
+		if err != nil {
+			return fmt.Errorf("Failed to read %s : %v", config.MultiDockerConfFile, err)
+		}
+		err = yaml.Unmarshal(multiEngineBytes, &composeConfigs)
+		if err != nil {
+			return fmt.Errorf("Failed to unmarshal %s : %v", config.MultiDockerConfFile, err)
+		}
+	}
+
 	for _, service := range services {
-		bytes, err := network.LoadServiceResource(service, true, cfg)
+		var bytes []byte
+		foundServiceConfig := map[string]composeConfig.ServiceConfigV1{}
+
+		if _, ok := composeConfigs[service]; ok {
+			foundServiceConfig[service] = composeConfigs[service]
+			bytes, err = yaml.Marshal(foundServiceConfig)
+		} else {
+			bytes, err = network.LoadServiceResource(service, true, cfg)
+		}
+
 		if err != nil {
 			return fmt.Errorf("Failed to load %s : %v", service, err)
 		}

compose/reload.go

@@ -13,10 +13,16 @@ import (
 )
 
 func LoadService(p *project.Project, cfg *config.CloudConfig, useNetwork bool, service string) error {
-	bytes, err := network.LoadServiceResource(service, useNetwork, cfg)
-	if err != nil {
-		log.Error(err)
-		return err
+	// First check the multi engine service file.
+	// If the name has been found in multi enging service file and matches, will not execute network.LoadServiceResource
+	// Otherwise will execute network.LoadServiceResource
+	bytes, err := network.LoadMultiEngineResource(service)
+	if err != nil || bytes == nil {
+		bytes, err = network.LoadServiceResource(service, useNetwork, cfg)
+		if err != nil {
+			log.Error(err)
+			return err
+		}
 	}
 
 	m := map[interface{}]interface{}{}

config/types.go

@@ -27,15 +27,18 @@ const (
 	SystemDockerLog  = "/var/log/system-docker.log"
 	SystemDockerBin  = "/usr/bin/system-dockerd"
 
-	HashLabel         = "io.rancher.os.hash"
-	IDLabel           = "io.rancher.os.id"
-	DetachLabel       = "io.rancher.os.detach"
-	CreateOnlyLabel   = "io.rancher.os.createonly"
-	ReloadConfigLabel = "io.rancher.os.reloadconfig"
-	ConsoleLabel      = "io.rancher.os.console"
-	ScopeLabel        = "io.rancher.os.scope"
-	RebuildLabel      = "io.docker.compose.rebuild"
-	System            = "system"
+	HashLabel          = "io.rancher.os.hash"
+	IDLabel            = "io.rancher.os.id"
+	DetachLabel        = "io.rancher.os.detach"
+	CreateOnlyLabel    = "io.rancher.os.createonly"
+	ReloadConfigLabel  = "io.rancher.os.reloadconfig"
+	ConsoleLabel       = "io.rancher.os.console"
+	ScopeLabel         = "io.rancher.os.scope"
+	RebuildLabel       = "io.docker.compose.rebuild"
+	UserDockerLabel    = "io.rancher.user_docker.name"
+	UserDockerNetLabel = "io.rancher.user_docker.net"
+	UserDockerFIPLabel = "io.rancher.user_docker.fix_ip"
+	System             = "system"
 
 	OsConfigFile           = "/usr/share/ros/os-config.yml"
 	VarRancherDir          = "/var/lib/rancher"
@@ -47,6 +50,8 @@ const (
 	MetaDataFile           = "/var/lib/rancher/conf/metadata"
 	CloudConfigFile        = "/var/lib/rancher/conf/cloud-config.yml"
 	EtcResolvConfFile      = "/etc/resolv.conf"
+	MultiDockerConfFile    = "/var/lib/rancher/conf.d/m-user-docker.yml"
+	MultiDockerDataDir     = "/var/lib/m-user-docker"
 )
 
 var (

os-config.tpl.yml

@@ -139,6 +139,7 @@ rancher:
       read_only: true
       volumes:
       - /var/lib/user-docker:/var/lib/docker
+      - /var/lib/m-user-docker:/var/lib/m-user-docker
     user-volumes:
       image: {{.OS_REPO}}/os-base:{{.VERSION}}{{.SUFFIX}}
       command: echo

util/network/network.go

@@ -10,6 +10,7 @@ import (
 
 	yaml "github.com/cloudfoundry-incubator/candiedyaml"
 
+	composeConfig "github.com/docker/libcompose/config"
 	"github.com/rancher/os/config"
 	"github.com/rancher/os/log"
 )
@@ -165,3 +166,30 @@ func LoadServiceResource(name string, useNetwork bool, cfg *config.CloudConfig)
 
 	return nil, err
 }
+
+func LoadMultiEngineResource(name string) ([]byte, error) {
+	composeConfigs := map[string]composeConfig.ServiceConfigV1{}
+	if _, err := os.Stat(config.MultiDockerConfFile); err == nil {
+		multiEngineBytes, err := ioutil.ReadFile(config.MultiDockerConfFile)
+		if err != nil {
+			return nil, err
+		}
+		err = yaml.Unmarshal(multiEngineBytes, &composeConfigs)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if _, ok := composeConfigs[name]; !ok {
+		return nil, errors.New("Failed to found " + name + " from " + config.MultiDockerConfFile + " will load from network")
+	}
+
+	foundServiceConfig := map[string]composeConfig.ServiceConfigV1{}
+	foundServiceConfig[name] = composeConfigs[name]
+	bytes, err := yaml.Marshal(foundServiceConfig)
+	if err == nil {
+		return bytes, err
+	}
+
+	return nil, err
+}

util/util.go

@@ -295,3 +295,21 @@ func RunCommandSequence(commandSequence []osYaml.StringandSlice) error {
 	}
 	return nil
 }
+
+func GenerateEngineScript(name string) error {
+	if _, err := os.Stat("/usr/bin/docker-" + name); err == nil {
+		err = os.Remove("/usr/bin/docker-" + name)
+		if err != nil {
+			return err
+		}
+	}
+
+	bytes := []byte("/usr/bin/docker -H unix:///var/lib/m-user-docker/" + name + "/docker-" + name + ".sock $@")
+
+	err := ioutil.WriteFile("/usr/bin/docker-"+name, bytes, 755)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}