Run user docker with a configurable storage context

cmd/control/tlsconf.go

@@ -131,13 +131,19 @@ func tlsConfCreate(c *cli.Context) {
 }
 
 func generate(c *cli.Context) error {
+	generateServer := c.Bool("server")
+	outDir := c.String("dir")
+	hostnames := c.StringSlice("hostname")
+
+	return Generate(generateServer, outDir, hostnames)
+}
+
+func Generate(generateServer bool, outDir string, hostnames []string) error {
 	cfg, err := config.LoadConfig()
 	if err != nil {
 		return err
 	}
 
-	generateServer := c.Bool("server")
-	outDir := c.String("dir")
 	if outDir == "" {
 		return fmt.Errorf("out directory (-d, --dir) not specified")
 	}
@@ -161,6 +167,5 @@ func generate(c *cli.Context) error {
 		return err
 	}
 
-	hostnames := c.StringSlice("hostname")
 	return writeCerts(generateServer, hostnames, cfg, certPath, keyPath, caCertPath, caKeyPath)
 }

cmd/userdocker/main.go

@@ -0,0 +1,267 @@
+package userdocker
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+	"os/signal"
+	"strings"
+	"syscall"
+	"time"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/docker/libcompose/docker"
+	"github.com/docker/libcompose/project"
+	"github.com/rancherio/os/cmd/control"
+	"github.com/rancherio/os/compose"
+	"github.com/rancherio/os/config"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	_ "github.com/opencontainers/runc/libcontainer/nsenter"
+	"github.com/opencontainers/runc/libcontainer/system"
+)
+
+const (
+	DEFAULT_STORAGE_CONTEXT = "console"
+	userDocker              = "user-docker"
+)
+
+func Main() {
+	cfg, err := config.LoadConfig()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if len(os.Args) == 1 {
+		if err := enter(cfg); err != nil {
+			log.Fatal(err)
+		}
+	} else {
+		if err := main(cfg); err != nil {
+			log.Fatal(err)
+		}
+	}
+}
+
+func enter(cfg *config.CloudConfig) error {
+	context := cfg.Rancher.UserDocker.StorageContext
+	if context == "" {
+		context = DEFAULT_STORAGE_CONTEXT
+	}
+
+	p, err := compose.GetProject(cfg)
+	if err != nil {
+		return err
+	}
+
+	pid, err := waitForPid(context, p)
+	if err != nil {
+		return err
+	}
+
+	log.Infof("%s PID %d", context, pid)
+
+	return runNsenter(pid)
+}
+
+type result struct {
+	Pid int `json:"Pid"`
+}
+
+func expand(first, second string, args []string) ([]string, error) {
+	var err error
+	prog := ""
+
+	prog, err = exec.LookPath(first)
+	if err != nil {
+		prog, err = exec.LookPath(second)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return append([]string{prog}, args...), nil
+}
+
+func runNsenter(pid int) error {
+	args, err := expand(userDocker, "", []string{"main"})
+	if err != nil {
+		return err
+	}
+
+	r, w, err := os.Pipe()
+	if err != nil {
+		return err
+	}
+
+	cmd := &exec.Cmd{
+		Path:       args[0],
+		Args:       args,
+		Stdin:      os.Stdin,
+		Stdout:     os.Stdout,
+		Stderr:     os.Stderr,
+		ExtraFiles: []*os.File{w},
+		Env: append(os.Environ(),
+			"_LIBCONTAINER_INITPIPE=3",
+			fmt.Sprintf("_LIBCONTAINER_INITPID=%d", pid),
+		),
+	}
+
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	w.Close()
+
+	var result result
+	if err := json.NewDecoder(r).Decode(&result); err != nil {
+		return err
+	}
+
+	if err := cmd.Wait(); err != nil {
+		return err
+	}
+
+	log.Infof("Docker PID %d", result.Pid)
+
+	p, err := os.FindProcess(result.Pid)
+	if err != nil {
+		return err
+	}
+
+	handleTerm(p)
+
+	if err := switchCgroup(result.Pid, pid); err != nil {
+		return err
+	}
+
+	_, err = p.Wait()
+	return err
+}
+
+func handleTerm(p *os.Process) {
+	term := make(chan os.Signal)
+	signal.Notify(term, syscall.SIGTERM)
+	go func() {
+		<-term
+		p.Signal(syscall.SIGTERM)
+	}()
+}
+
+func waitForPid(service string, project *project.Project) (int, error) {
+	for {
+		if pid, err := getPid(service, project); err != nil || pid == 0 {
+			log.Infof("Waiting for %s : %d : %v", service, pid, err)
+			time.Sleep(250)
+		} else {
+			return pid, err
+		}
+	}
+}
+
+func getPid(service string, project *project.Project) (int, error) {
+	s, err := project.CreateService(service)
+	if err != nil {
+		return 0, err
+	}
+
+	containers, err := s.Containers()
+	if err != nil {
+		return 0, err
+	}
+
+	if len(containers) == 0 {
+		return 0, nil
+	}
+
+	client, err := docker.CreateClient(docker.ClientOpts{
+		Host: config.DOCKER_SYSTEM_HOST,
+	})
+	if err != nil {
+		return 0, err
+	}
+
+	id, err := containers[0].Id()
+	if err != nil {
+		return 0, err
+	}
+
+	info, err := client.InspectContainer(id)
+	if err != nil || info == nil {
+		return 0, err
+	}
+
+	if info.State.Running {
+		return info.State.Pid, nil
+	}
+
+	return 0, nil
+}
+
+func main(cfg *config.CloudConfig) error {
+	os.Unsetenv("_LIBCONTAINER_INITPIPE")
+	os.Unsetenv("_LIBCONTAINER_INITPID")
+
+	if err := system.ParentDeathSignal(syscall.SIGKILL).Set(); err != nil {
+		return err
+	}
+
+	if err := os.Remove("/var/run/docker.pid"); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+
+	dockerCfg := cfg.Rancher.UserDocker
+
+	args := dockerCfg.FullArgs()
+
+	log.Debugf("User Docker args: %v", args)
+
+	if dockerCfg.TLS {
+		log.Debug("Generating TLS certs if needed")
+		if err := control.Generate(true, "/etc/docker/tls", []string{"localhost"}); err != nil {
+			return err
+		}
+	}
+
+	args, err := expand("docker-init", "docker", args)
+	if err != nil {
+		return err
+	}
+
+	log.Infof("Running %v", args)
+	err = syscall.Exec(args[0], args, dockerCfg.AppendEnv())
+	return err
+}
+
+func switchCgroup(src, target int) error {
+	cgroupFile := fmt.Sprintf("/proc/%d/cgroup", target)
+	f, err := os.Open(cgroupFile)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	targetCgroups := map[string]string{}
+
+	s := bufio.NewScanner(f)
+	for s.Scan() {
+		text := s.Text()
+		parts := strings.Split(text, ":")
+		subparts := strings.Split(parts[1], "=")
+		subsystem := subparts[0]
+		if len(subparts) > 1 {
+			subsystem = subparts[1]
+		}
+
+		targetPath := fmt.Sprintf("/host/sys/fs/cgroup/%s%s", subsystem, parts[2])
+		log.Infof("Moving Docker to cgroup %s", targetPath)
+		targetCgroups[subsystem] = targetPath
+	}
+
+	if err := s.Err(); err != nil {
+		return err
+	}
+
+	return cgroups.EnterPid(targetCgroups, src)
+}

compose/project.go

@@ -49,6 +49,10 @@ func RunServices(cfg *config.CloudConfig) error {
 	return p.Up()
 }
 
+func GetProject(cfg *config.CloudConfig) (*project.Project, error) {
+	return newCoreServiceProject(cfg)
+}
+
 func newProject(name string, cfg *config.CloudConfig) (*project.Project, error) {
 	clientFactory, err := rosDocker.NewClientFactory(docker.ClientOpts{})
 	if err != nil {

config/docker_config.go

@@ -0,0 +1,17 @@
+package config
+
+import "os"
+
+func (d *DockerConfig) FullArgs() []string {
+	args := append(d.Args, d.ExtraArgs...)
+
+	if d.TLS {
+		args = append(args, d.TLSArgs...)
+	}
+
+	return args
+}
+
+func (d *DockerConfig) AppendEnv() []string {
+	return append(os.Environ(), d.Environment...)
+}

config/types.go

@@ -91,14 +91,16 @@ type UpgradeConfig struct {
 }
 
 type DockerConfig struct {
-	TLS        bool     `yaml:"tls,omitempty"`
-	TLSArgs    []string `yaml:"tls_args,flow,omitempty"`
-	Args       []string `yaml:"args,flow,omitempty"`
-	ExtraArgs  []string `yaml:"extra_args,flow,omitempty"`
-	ServerCert string   `yaml:"server_cert,omitempty"`
-	ServerKey  string   `yaml:"server_key,omitempty"`
-	CACert     string   `yaml:"ca_cert,omitempty"`
-	CAKey      string   `yaml:"ca_key,omitempty"`
+	TLS            bool     `yaml:"tls,omitempty"`
+	TLSArgs        []string `yaml:"tls_args,flow,omitempty"`
+	Args           []string `yaml:"args,flow,omitempty"`
+	ExtraArgs      []string `yaml:"extra_args,flow,omitempty"`
+	ServerCert     string   `yaml:"server_cert,omitempty"`
+	ServerKey      string   `yaml:"server_key,omitempty"`
+	CACert         string   `yaml:"ca_cert,omitempty"`
+	CAKey          string   `yaml:"ca_key,omitempty"`
+	Environment    []string `yaml:"environment,omitempty"`
+	StorageContext string   `yaml:"storage_context,omitempty"`
 }
 
 type SshConfig struct {

init/init.go

@@ -139,6 +139,7 @@ func getLaunchConfig(cfg *config.CloudConfig, dockerCfg *config.DockerConfig) (*
 
 	launchConfig.DnsConfig.Nameservers = cfg.Rancher.Network.Dns.Nameservers
 	launchConfig.DnsConfig.Search = cfg.Rancher.Network.Dns.Search
+	launchConfig.Environment = dockerCfg.Environment
 
 	if !cfg.Rancher.Debug {
 		launchConfig.LogFile = config.SYSTEM_DOCKER_LOG

main.go

@@ -14,6 +14,7 @@ import (
 	"github.com/rancherio/os/cmd/respawn"
 	"github.com/rancherio/os/cmd/sysinit"
 	"github.com/rancherio/os/cmd/systemdocker"
+	"github.com/rancherio/os/cmd/userdocker"
 	"github.com/rancherio/os/cmd/wait"
 	"github.com/rancherio/os/config"
 	osInit "github.com/rancherio/os/init"
@@ -41,6 +42,7 @@ func main() {
 	registerCmd("/init", osInit.MainInit)
 	registerCmd(config.SYSINIT_BIN, sysinit.Main)
 	registerCmd("/usr/bin/dockerlaunch", dockerlaunchMain.Main)
+	registerCmd("/usr/bin/user-docker", userdocker.Main)
 	registerCmd("/usr/bin/system-docker", systemdocker.Main)
 	registerCmd("/sbin/poweroff", power.PowerOff)
 	registerCmd("/sbin/reboot", power.Reboot)

os-config.yml

@@ -120,19 +120,20 @@ rancher:
       privileged: true
       read_only: true
       volumes:
+      - /usr/bin/docker:/usr/bin/docker.dist:ro
       - /usr/bin/ros:/sbin/halt:ro
+      - /usr/bin/ros:/sbin/netconf:ro
       - /usr/bin/ros:/sbin/poweroff:ro
       - /usr/bin/ros:/sbin/reboot:ro
       - /usr/bin/ros:/sbin/shutdown:ro
-      - /usr/bin/ros:/sbin/netconf:ro
       - /usr/bin/ros:/usr/bin/cloud-init:ro
-      - /usr/bin/ros:/usr/bin/rancherctl:ro
-      - /usr/bin/ros:/usr/bin/ros:ro
-      - /usr/bin/ros:/usr/bin/respawn:ro
-      - /usr/bin/ros:/usr/bin/system-docker:ro
-      - /usr/bin/ros:/usr/sbin/wait-for-docker:ro
       - /usr/bin/ros:/usr/bin/dockerlaunch:ro
-      - /usr/bin/docker:/usr/bin/docker.dist:ro
+      - /usr/bin/ros:/usr/bin/rancherctl:ro
+      - /usr/bin/ros:/usr/bin/respawn:ro
+      - /usr/bin/ros:/usr/bin/ros:ro
+      - /usr/bin/ros:/usr/bin/system-docker:ro
+      - /usr/bin/ros:/usr/bin/user-docker:ro
+      - /usr/bin/ros:/usr/sbin/wait-for-docker:ro
     console:
       image: rancher/os-console:v0.4.0-dev
       labels:
@@ -225,15 +226,16 @@ rancher:
       read_only: true
       volumes:
       - /dev:/host/dev
-      - /usr/share/ros/os-config.yml:/usr/share/ros/os-config.yml
-      - /var/lib/rancher:/var/lib/rancher
-      - /var/lib/rancher/conf:/var/lib/rancher/conf
+      - /etc/docker:/etc/docker
       - /etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt.rancher
-      - /lib/modules:/lib/modules
       - /lib/firmware:/lib/firmware
+      - /lib/modules:/lib/modules
       - /run:/run
-      - /var/run:/var/run
+      - /usr/share/ros/os-config.yml:/usr/share/ros/os-config.yml
+      - /var/lib/rancher/conf:/var/lib/rancher/conf
+      - /var/lib/rancher:/var/lib/rancher
       - /var/log:/var/log
+      - /var/run:/var/run
     udev-cold:
       image: rancher/os-udev:v0.4.0-dev
       labels:
@@ -269,6 +271,20 @@ rancher:
       volumes:
       - /home:/home
       - /opt:/opt
+    docker:
+      image: rancher/os-docker:v0.4.0-dev
+      labels:
+        io.rancher.os.scope: system
+      net: host
+      pid: host
+      ipc: host
+      uts: host
+      privileged: true
+      restart: always
+      volumes_from:
+      - all-volumes
+      volumes:
+      - /sys/fs/cgroup:/host/sys/fs/cgroup
   system_docker:
     args: [daemon, --log-opt, max-size=25m, --log-opt, max-file=2, -s, overlay, -b, docker-sys,
       --fixed-cidr, 172.18.42.1/16, --restart=false, -g, /var/lib/system-docker, -G, root,