Make it easier to switch between Docker engines

Dockerfile.dapper

@@ -139,9 +139,6 @@ RUN ln -sf go-6 /usr/bin/go && \
 RUN curl -fL ${!BUILD_DOCKER_URL} > /usr/bin/docker && \
     chmod +x /usr/bin/docker
 
-# Install Target Docker
-RUN curl -fL ${!DOCKER_URL} > ${DOWNLOADS}/docker.tgz
-
 # Install Trash
 RUN go get github.com/rancher/trash
 

cmd/console/console.go

@@ -22,6 +22,10 @@ const (
 	startScript = "/opt/rancher/bin/start.sh"
 )
 
+type symlink struct {
+	oldname, newname string
+}
+
 func Main() {
 	cfg := config.LoadConfig()
 
@@ -88,6 +92,20 @@ func Main() {
 		log.Error(err)
 	}
 
+	for _, link := range []symlink{
+		{"/var/lib/rancher/engine/docker", "/usr/bin/docker"},
+		{"/var/lib/rancher/engine/docker-containerd", "/usr/bin/docker-containerd"},
+		{"/var/lib/rancher/engine/docker-containerd-ctr", "/usr/bin/docker-containerd-ctr"},
+		{"/var/lib/rancher/engine/docker-containerd-shim", "/usr/bin/docker-containerd-shim"},
+		{"/var/lib/rancher/engine/dockerd", "/usr/bin/dockerd"},
+		{"/var/lib/rancher/engine/docker-runc", "/usr/bin/docker-runc"},
+	} {
+		syscall.Unlink(link.newname)
+		if err = os.Symlink(link.oldname, link.newname); err != nil {
+			log.Error(err)
+		}
+	}
+
 	cmd = exec.Command("bash", "-c", `echo 'RancherOS \n \l' > /etc/issue`)
 	if err = cmd.Run(); err != nil {
 		log.Error(err)

cmd/control/cli.go

@@ -33,7 +33,7 @@ func Main() {
 		},
 		{
 			Name:        "console",
-			Usage:       "console container commands",
+			Usage:       "manage which console container is used",
 			HideHelp:    true,
 			Subcommands: consoleSubcommands(),
 		},
@@ -45,6 +45,12 @@ func Main() {
 			SkipFlagParsing: true,
 			Action:          devAction,
 		},
+		{
+			Name:        "engine",
+			Usage:       "manage which Docker engine is used",
+			HideHelp:    true,
+			Subcommands: engineSubcommands(),
+		},
 		{
 			Name:            "entrypoint",
 			HideHelp:        true,

cmd/control/docker.go

@@ -0,0 +1,115 @@
+package control
+
+import (
+	"fmt"
+	"sort"
+
+	"golang.org/x/net/context"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/codegangsta/cli"
+	composeConfig "github.com/docker/libcompose/config"
+	"github.com/docker/libcompose/project/options"
+	"github.com/rancher/os/compose"
+	"github.com/rancher/os/config"
+	"github.com/rancher/os/util/network"
+)
+
+func engineSubcommands() []cli.Command {
+	return []cli.Command{
+		{
+			Name:   "switch",
+			Usage:  "switch Docker engine without a reboot",
+			Action: engineSwitch,
+			Flags: []cli.Flag{
+				cli.BoolFlag{
+					Name:  "force, f",
+					Usage: "do not prompt for input",
+				},
+				cli.BoolFlag{
+					Name:  "no-pull",
+					Usage: "don't pull console image",
+				},
+			},
+		},
+		{
+			Name:   "enable",
+			Usage:  "set Docker engine to be switched on next reboot",
+			Action: engineEnable,
+		},
+		{
+			Name:   "list",
+			Usage:  "list available Docker engines",
+			Action: engineList,
+		},
+	}
+}
+
+func engineSwitch(c *cli.Context) error {
+	if len(c.Args()) != 1 {
+		log.Fatal("Must specify exactly one Docker engine to switch to")
+	}
+	newEngine := c.Args()[0]
+
+	cfg := config.LoadConfig()
+
+	project, err := compose.GetProject(cfg, true, false)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if err = project.Stop(context.Background(), 10, "docker"); err != nil {
+		log.Fatal(err)
+	}
+
+	project.ServiceConfigs.Add("docker", &composeConfig.ServiceConfig{})
+
+	if err = compose.LoadService(project, cfg, true, newEngine); err != nil {
+		log.Fatal(err)
+	}
+
+	if err = project.Up(context.Background(), options.Up{}, "docker"); err != nil {
+		log.Fatal(err)
+	}
+
+	if err := config.Set("rancher.docker.engine", newEngine); err != nil {
+		log.Errorf("Failed to update rancher.docker.engine: %v", err)
+	}
+
+	return nil
+}
+
+func engineEnable(c *cli.Context) error {
+	if len(c.Args()) != 1 {
+		log.Fatal("Must specify exactly one Docker engine to enable")
+	}
+	newEngine := c.Args()[0]
+
+	cfg := config.LoadConfig()
+
+	if err := compose.StageServices(cfg, newEngine); err != nil {
+		return err
+	}
+
+	if err := config.Set("rancher.docker.engine", newEngine); err != nil {
+		log.Errorf("Failed to update 'rancher.docker.engine': %v", err)
+	}
+
+	return nil
+}
+
+func engineList(c *cli.Context) error {
+	cfg := config.LoadConfig()
+
+	engines, err := network.GetEngines(cfg.Rancher.Repositories.ToArray())
+	if err != nil {
+		return err
+	}
+	sort.Strings(engines)
+
+	for _, engine := range engines {
+		fmt.Println(engine)
+	}
+
+	return nil
+}

cmd/userdocker/main.go

@@ -5,6 +5,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/signal"
+	"path"
 	"strconv"
 	"syscall"
 	"time"
@@ -29,9 +30,15 @@ const (
 	DOCKER_PID_FILE         = "/var/run/docker.pid"
 	DOCKER_COMMAND          = "docker-init"
 	userDocker              = "user-docker"
+	sourceDirectory         = "/engine"
+	destDirectory           = "/var/lib/rancher/engine"
 )
 
 func Main() {
+	if err := copyBinaries(sourceDirectory, destDirectory); err != nil {
+		log.Fatal(err)
+	}
+
 	cfg := config.LoadConfig()
 
 	execID, resp, err := startDocker(cfg)
@@ -64,6 +71,43 @@ func Main() {
 	os.Exit(state.ExitCode)
 }
 
+func copyBinaries(source, dest string) error {
+	if err := os.MkdirAll(dest, 0755); err != nil {
+		return err
+	}
+
+	files, err := ioutil.ReadDir(dest)
+	if err != nil {
+		return err
+	}
+
+	for _, file := range files {
+		if err = os.RemoveAll(path.Join(dest, file.Name())); err != nil {
+			return err
+		}
+	}
+
+	files, err = ioutil.ReadDir(source)
+	if err != nil {
+		return err
+	}
+
+	for _, file := range files {
+		sourceFile := path.Join(source, file.Name())
+		destFile := path.Join(dest, file.Name())
+
+		data, err := ioutil.ReadFile(sourceFile)
+		if err != nil {
+			return err
+		}
+		if err := ioutil.WriteFile(destFile, data, 0751); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func writeCerts(cfg *config.CloudConfig) error {
 	outDir := control.ServerTlsPath
 	if err := os.MkdirAll(outDir, 0700); err != nil {

compose/reload.go

@@ -5,6 +5,7 @@ import (
 
 	log "github.com/Sirupsen/logrus"
 	yaml "github.com/cloudfoundry-incubator/candiedyaml"
+	composeConfig "github.com/docker/libcompose/config"
 	"github.com/docker/libcompose/project"
 	"github.com/rancher/os/config"
 	"github.com/rancher/os/docker"
@@ -61,12 +62,17 @@ func projectReload(p *project.Project, useNetwork *bool, loadConsole bool, envir
 			enabled[service] = service
 		}
 
-		if !loadConsole || cfg.Rancher.Console == "" || cfg.Rancher.Console == "default" {
-			return nil
+		if loadConsole && cfg.Rancher.Console != "" && cfg.Rancher.Console != "default" {
+			if err := LoadService(p, cfg, *useNetwork, cfg.Rancher.Console); err != nil && err != network.ErrNoNetwork {
+				log.Error(err)
+			}
 		}
 
-		if err := LoadService(p, cfg, *useNetwork, cfg.Rancher.Console); err != nil && err != network.ErrNoNetwork {
-			log.Error(err)
+		if cfg.Rancher.Docker.Engine != "" {
+			p.ServiceConfigs.Add("docker", &composeConfig.ServiceConfig{})
+			if err := LoadService(p, cfg, *useNetwork, cfg.Rancher.Docker.Engine); err != nil && err != network.ErrNoNetwork {
+				log.Error(err)
+			}
 		}
 
 		return nil

config/types.go

@@ -134,6 +134,7 @@ type UpgradeConfig struct {
 }
 
 type DockerConfig struct {
+	Engine         string   `yaml:"engine,omitempty"`
 	TLS            bool     `yaml:"tls,omitempty"`
 	TLSArgs        []string `yaml:"tls_args,flow,omitempty"`
 	Args           []string `yaml:"args,flow,omitempty"`

images/02-console/Dockerfile

@@ -7,10 +7,6 @@ RUN sed -i 's/rancher:!/rancher:*/g' /etc/shadow && \
     echo '## allow password less for rancher user' >> /etc/sudoers && \
     echo 'rancher ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers && \
     echo '## allow password less for docker user' >> /etc/sudoers && \
-    echo 'docker ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers && \
-    ln -sf /usr/bin/docker.dist /usr/bin/docker && \
-    ln -sf /usr/bin/docker-containerd.dist /usr/bin/docker-containerd && \
-    ln -sf /usr/bin/docker-containerd-shim.dist /usr/bin/docker-containerd-shim && \
-    ln -sf /usr/bin/docker-runc.dist /usr/bin/docker-runc
+    echo 'docker ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
 COPY prompt.sh /etc/profile.d/
 CMD ["/usr/sbin/console.sh"]

images/02-preload/Dockerfile

@@ -1,7 +1,4 @@
 FROM rancher/os-base
-RUN ln -sf /usr/bin/docker.dist /usr/bin/docker && \
-    ln -sf /usr/bin/docker-containerd.dist /usr/bin/docker-containerd && \
-    ln -sf /usr/bin/docker-containerd-shim.dist /usr/bin/docker-containerd-shim && \
-    ln -sf /usr/bin/docker-runc.dist /usr/bin/docker-runc
+RUN ln -sf /var/lib/rancher/docker/docker /usr/bin/docker
 COPY preload.sh  /
 CMD ["/preload.sh"]

os-config.tpl.yml

@@ -161,10 +161,6 @@ rancher:
       privileged: true
       read_only: true
       volumes:
-      - /usr/bin/docker-containerd:/usr/bin/docker-containerd.dist:ro
-      - /usr/bin/docker-containerd-shim:/usr/bin/docker-containerd-shim.dist:ro
-      - /usr/bin/docker-runc:/usr/bin/docker-runc.dist:ro
-      - /usr/bin/docker:/usr/bin/docker.dist:ro
       - /usr/bin/ros:/usr/bin/dockerlaunch:ro
       - /usr/bin/ros:/usr/bin/user-docker:ro
       - /usr/bin/ros:/usr/bin/system-docker:ro
@@ -359,7 +355,7 @@ rancher:
       - /home:/home
       - /opt:/opt
     docker:
-      image: {{.OS_REPO}}/os-base:{{.VERSION}}{{.SUFFIX}}
+      image: {{.OS_REPO}}/os-docker:1.11.2{{.SUFFIX}}
       command: /usr/bin/user-docker
       environment:
       - HTTP_PROXY
@@ -376,8 +372,6 @@ rancher:
       restart: always
       volumes_from:
       - all-volumes
-      volumes:
-      - /sys/fs/cgroup:/host/sys/fs/cgroup
   system_docker:
     exec: true
     args: [daemon, --log-opt, max-size=25m, --log-opt, max-file=2, -s, overlay,

scripts/layout

@@ -25,8 +25,6 @@ ln -s ros                      ${INITRD_DIR}/usr/bin/system-docker
 ln -s ../../../../usr/bin/ros  ${INITRD_DIR}/usr/var/lib/cni/bin/bridge
 ln -s ../../../../usr/bin/ros  ${INITRD_DIR}/usr/var/lib/cni/bin/host-local
 
-tar xvzf ${DOWNLOADS}/docker.tgz -C ${INITRD_DIR}/usr/bin --strip-components=1
-
 if [ -e ${DOWNLOADS}/kernel.tar.gz ]; then
     mkdir -p ${BUILD}/kernel
     tar xf ${DOWNLOADS}/kernel.tar.gz -C ${BUILD}/kernel

tests/assets/test_05/cloud-config.yml

@@ -1,5 +1,6 @@
 #cloud-config
 rancher:
-  console: debian
+  docker:
+    engine: docker-1.10.3
 ssh_authorized_keys:
   - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC85w9stZyiLQp/DkVO6fqwiShYcj1ClKdtCqgHtf+PLpJkFReSFu8y21y+ev09gsSMRRrjF7yt0pUHV6zncQhVeqsZtgc5WbELY2DOYUGmRn/CCvPbXovoBrQjSorqlBmpuPwsStYLr92Xn+VVsMNSUIegHY22DphGbDKG85vrKB8HxUxGIDxFBds/uE8FhSy+xsoyT/jUZDK6pgq2HnGl6D81ViIlKecpOpWlW3B+fea99ADNyZNVvDzbHE5pcI3VRw8u59WmpWOUgT6qacNVACl8GqpBvQk8sw7O/X9DSZHCKafeD9G5k+GYbAUz92fKWrx/lOXfUXPS3+c8dRIF

tests/assets/test_25/cloud-config.yml

@@ -0,0 +1,7 @@
+#cloud-config
+rancher:
+  console: debian
+  docker:
+    engine: docker-1.10.3
+ssh_authorized_keys:
+  - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC85w9stZyiLQp/DkVO6fqwiShYcj1ClKdtCqgHtf+PLpJkFReSFu8y21y+ev09gsSMRRrjF7yt0pUHV6zncQhVeqsZtgc5WbELY2DOYUGmRn/CCvPbXovoBrQjSorqlBmpuPwsStYLr92Xn+VVsMNSUIegHY22DphGbDKG85vrKB8HxUxGIDxFBds/uE8FhSy+xsoyT/jUZDK6pgq2HnGl6D81ViIlKecpOpWlW3B+fea99ADNyZNVvDzbHE5pcI3VRw8u59WmpWOUgT6qacNVACl8GqpBvQk8sw7O/X9DSZHCKafeD9G5k+GYbAUz92fKWrx/lOXfUXPS3+c8dRIF

tests/consoles_test.go

@@ -0,0 +1,55 @@
+package integration
+
+import . "gopkg.in/check.v1"
+
+func (s *QemuSuite) TestCloudConfigConsole(c *C) {
+	err := s.RunQemu("--cloud-config", "./tests/assets/test_03/cloud-config.yml")
+	c.Assert(err, IsNil)
+
+	s.CheckCall(c, "apt-get --version")
+	s.CheckCall(c, `
+sudo ros console list | grep default | grep disabled
+sudo ros console list | grep debian | grep current`)
+}
+
+func (s *QemuSuite) TestConsoleCommand(c *C) {
+	err := s.RunQemu()
+	c.Assert(err, IsNil)
+
+	s.CheckCall(c, `
+sudo ros console list | grep default | grep current
+sudo ros console list | grep debian | grep disabled`)
+
+	s.MakeCall("sudo ros console switch -f debian")
+	c.Assert(s.WaitForSSH(), IsNil)
+
+	s.CheckCall(c, "apt-get --version")
+	s.CheckCall(c, `
+sudo ros console list | grep default | grep disabled
+sudo ros console list | grep debian | grep current`)
+
+	s.Reboot(c)
+
+	s.CheckCall(c, "apt-get --version")
+	s.CheckCall(c, `
+sudo ros console list | grep default | grep disabled
+sudo ros console list | grep debian | grep current`)
+
+	s.MakeCall("sudo ros console switch -f default")
+	c.Assert(s.WaitForSSH(), IsNil)
+
+	s.CheckCall(c, `
+sudo ros console list | grep default | grep current
+sudo ros console list | grep debian | grep disabled`)
+
+	s.CheckCall(c, "sudo ros console enable debian")
+
+	s.CheckCall(c, "sudo ros console list | grep default | grep current")
+	s.CheckCall(c, "sudo ros console list | grep debian | grep enabled")
+
+	s.Reboot(c)
+
+	s.CheckCall(c, `
+sudo ros console list | grep default | grep disabled
+sudo ros console list | grep debian | grep current`)
+}

tests/custom_docker_in_persistent_console_test.go

@@ -1,14 +0,0 @@
-package integration
-
-import . "gopkg.in/check.v1"
-
-func (s *QemuSuite) TestCustomDockerInPersistentConsole(c *C) {
-	err := s.RunQemu("--cloud-config", "./tests/assets/test_05/cloud-config.yml")
-	c.Assert(err, IsNil)
-
-	s.CheckCall(c, "curl", "-Lfo", "./docker", DockerUrl)
-	s.CheckCall(c, "chmod", "+x", "/home/rancher/docker")
-	s.CheckCall(c, "sudo", "ln", "-sf", "/home/rancher/docker", "/usr/bin/docker")
-	s.CheckCall(c, "sudo", "system-docker", "restart", "docker")
-	s.CheckCall(c, "sudo", "system-docker", "version")
-}

tests/custom_docker_test.go

@@ -0,0 +1,60 @@
+package integration
+
+import . "gopkg.in/check.v1"
+
+func (s *QemuSuite) TestCustomDocker(c *C) {
+	err := s.RunQemu("--cloud-config", "./tests/assets/test_05/cloud-config.yml")
+	c.Assert(err, IsNil)
+
+	s.CheckCall(c, `
+set -ex
+
+docker version | grep 1.10.3
+docker run -d --restart=always nginx
+docker ps | grep nginx`)
+
+	s.CheckCall(c, `
+set -ex
+
+sudo ros engine switch docker-1.11.2
+/usr/sbin/wait-for-docker
+docker version | grep 1.11.2
+docker ps | grep nginx`)
+
+	s.Reboot(c)
+
+	s.CheckCall(c, `
+set -ex
+
+docker version | grep 1.11.2
+docker ps | grep nginx`)
+}
+
+func (s *QemuSuite) TestCustomDockerInPersistentConsole(c *C) {
+	err := s.RunQemu("--cloud-config", "./tests/assets/test_25/cloud-config.yml")
+	c.Assert(err, IsNil)
+
+	s.CheckCall(c, `
+set -ex
+
+apt-get --version
+docker version | grep 1.10.3
+docker run -d --restart=always nginx
+docker ps | grep nginx`)
+
+	s.CheckCall(c, `
+set -ex
+
+sudo ros engine switch docker-1.11.2
+/usr/sbin/wait-for-docker
+docker version | grep 1.11.2
+docker ps | grep nginx`)
+
+	s.Reboot(c)
+
+	s.CheckCall(c, `
+set -ex
+
+docker version | grep 1.11.2
+docker ps | grep nginx`)
+}

tests/docker_in_persistent_console_test.go

@@ -1,20 +0,0 @@
-package integration
-
-import (
-	"fmt"
-
-	. "gopkg.in/check.v1"
-)
-
-func (s *QemuSuite) TestRebootWithContainerRunning(c *C) {
-	err := s.RunQemu("--cloud-config", "./tests/assets/test_03/cloud-config.yml")
-	c.Assert(err, IsNil)
-
-	s.CheckCall(c, fmt.Sprintf(`
-set -e -x
-docker run -d --restart=always %s`, NginxImage))
-
-	s.Reboot(c)
-
-	s.CheckCall(c, "docker ps -f status=running | grep nginx")
-}

tests/http_proxy_test.go

@@ -9,9 +9,9 @@ func (s *QemuSuite) TestHttpProxy(c *C) {
 	s.CheckCall(c, `
 set -x -e
 
-sudo system-docker exec docker env | grep HTTP_PROXY=invalid
-sudo system-docker exec docker env | grep HTTPS_PROXY=invalid
-sudo system-docker exec docker env | grep NO_PROXY=invalid
+sudo system-docker inspect docker env | grep HTTP_PROXY=invalid
+sudo system-docker inspect docker env | grep HTTPS_PROXY=invalid
+sudo system-docker inspect docker env | grep NO_PROXY=invalid
 
 if docker pull busybox; then
     exit 1

tests/misc_test.go

@@ -22,7 +22,7 @@ docker --tlsverify version`)
 
 	s.CheckCall(c, `
 set -e -x
-for i in $(pidof docker); do
+for i in $(pidof system-docker); do
     if [ $i = 1 ]; then
         found=true
     fi

trash.conf

@@ -37,7 +37,7 @@ github.com/packethost/packngo 92012705236896736875186c9e49557897c6af90 https://g
 github.com/pkg/errors d62207b3dc916c342cd6a7180fa861d898cf42ee
 github.com/pmezard/go-difflib d8ed2627bdf02c080bf22230dbb337003b7aba2d
 github.com/rancher/cniglue b56bd68e5df113ad3fcc59c58034c22afaede877
-github.com/rancher/docker-from-scratch 152ddfa8d618d83238d987e7b8ae7287fc69f327
+github.com/rancher/docker-from-scratch 084910d99b12408467a559f338e5a27bf4a1aed3
 github.com/rancher/netconf ddd7e35a6aacd7e80991920774083dd4408ec018
 github.com/rcrowley/go-metrics eeba7bd0dd01ace6e690fa833b3f22aaec29af43
 github.com/ryanuber/go-glob 0067a9abd927e50aed5190662702f81231413ae0

util/network/network.go

@@ -28,6 +28,10 @@ func GetConsoles(urls []string) ([]string, error) {
 	return getServices(urls, "consoles")
 }
 
+func GetEngines(urls []string) ([]string, error) {
+	return getServices(urls, "engines")
+}
+
 func getServices(urls []string, key string) ([]string, error) {
 	result := []string{}
 

vendor/github.com/rancher/docker-from-scratch/Dockerfile.dapper

@@ -30,7 +30,7 @@ RUN wget https://github.com/rancher/docker-from-scratch/releases/download/bin-v0
     wget https://github.com/rancher/docker-from-scratch/releases/download/bin-v0.4.0/base-files_arm64.tar.gz
 
 ENV DOCKER_VERSION=1.11.2 DOCKER_PATCH_VERSION=v1.11.2-ros1
-ENV VERSION=v${DOCKER_VERSION}
+ENV VERSION=v${DOCKER_VERSION}-2
 
 RUN wget -O docker-${DOCKER_VERSION}_amd64.tgz -L https://get.docker.com/builds/Linux/x86_64/docker-${DOCKER_VERSION}.tgz && \
     wget -L https://github.com/rancher/docker/releases/download/${DOCKER_PATCH_VERSION}/docker-${DOCKER_VERSION}_arm.tgz && \

vendor/github.com/rancher/docker-from-scratch/scratch.go

@@ -108,9 +108,9 @@ func mountCgroups(hierarchyConfig map[string]string) error {
 	for scanner.Scan() {
 		text := scanner.Text()
 		log.Debugf("/proc/cgroups: %s", text)
-		fields := strings.SplitN(text, "\t", 3)
+		fields := strings.Split(text, "\t")
 		cgroup := fields[0]
-		if cgroup == "" || cgroup[0] == '#' || len(fields) < 3 || cgroup[2] == '0' {
+		if cgroup == "" || cgroup[0] == '#' || (len(fields) > 3 && fields[3] == "0") {
 			continue
 		}
 
@@ -661,11 +661,15 @@ func runOrExec(config *Config, docker string, args ...string) (*exec.Cmd, error)
 		return nil, err
 	}
 
-	cmd := "docker"
+	cmd := path.Base(docker)
 	if config != nil && config.CommandName != "" {
 		cmd = config.CommandName
 	}
 
+	if cmd == "dockerd" && len(args) > 1 && args[0] == "daemon" {
+		args = args[1:]
+	}
+
 	return execDocker(config, docker, cmd, args)
 }