diff --git a/cmd/control/tlsconf.go b/cmd/control/tlsconf.go index b23e8b38..7842c431 100644 --- a/cmd/control/tlsconf.go +++ b/cmd/control/tlsconf.go @@ -131,13 +131,19 @@ func tlsConfCreate(c *cli.Context) { } func generate(c *cli.Context) error { + generateServer := c.Bool("server") + outDir := c.String("dir") + hostnames := c.StringSlice("hostname") + + return Generate(generateServer, outDir, hostnames) +} + +func Generate(generateServer bool, outDir string, hostnames []string) error { cfg, err := config.LoadConfig() if err != nil { return err } - generateServer := c.Bool("server") - outDir := c.String("dir") if outDir == "" { return fmt.Errorf("out directory (-d, --dir) not specified") } @@ -161,6 +167,5 @@ func generate(c *cli.Context) error { return err } - hostnames := c.StringSlice("hostname") return writeCerts(generateServer, hostnames, cfg, certPath, keyPath, caCertPath, caKeyPath) } diff --git a/cmd/userdocker/main.go b/cmd/userdocker/main.go new file mode 100644 index 00000000..bcc75e55 --- /dev/null +++ b/cmd/userdocker/main.go @@ -0,0 +1,267 @@ +package userdocker + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "os/exec" + "os/signal" + "strings" + "syscall" + "time" + + log "github.com/Sirupsen/logrus" + "github.com/docker/libcompose/docker" + "github.com/docker/libcompose/project" + "github.com/rancherio/os/cmd/control" + "github.com/rancherio/os/compose" + "github.com/rancherio/os/config" + + "github.com/opencontainers/runc/libcontainer/cgroups" + _ "github.com/opencontainers/runc/libcontainer/nsenter" + "github.com/opencontainers/runc/libcontainer/system" +) + +const ( + DEFAULT_STORAGE_CONTEXT = "console" + userDocker = "user-docker" +) + +func Main() { + cfg, err := config.LoadConfig() + if err != nil { + log.Fatal(err) + } + + if len(os.Args) == 1 { + if err := enter(cfg); err != nil { + log.Fatal(err) + } + } else { + if err := main(cfg); err != nil { + log.Fatal(err) + } + } +} + +func enter(cfg *config.CloudConfig) error { + context := cfg.Rancher.UserDocker.StorageContext + if context == "" { + context = DEFAULT_STORAGE_CONTEXT + } + + p, err := compose.GetProject(cfg) + if err != nil { + return err + } + + pid, err := waitForPid(context, p) + if err != nil { + return err + } + + log.Infof("%s PID %d", context, pid) + + return runNsenter(pid) +} + +type result struct { + Pid int `json:"Pid"` +} + +func expand(first, second string, args []string) ([]string, error) { + var err error + prog := "" + + prog, err = exec.LookPath(first) + if err != nil { + prog, err = exec.LookPath(second) + } + if err != nil { + return nil, err + } + + return append([]string{prog}, args...), nil +} + +func runNsenter(pid int) error { + args, err := expand(userDocker, "", []string{"main"}) + if err != nil { + return err + } + + r, w, err := os.Pipe() + if err != nil { + return err + } + + cmd := &exec.Cmd{ + Path: args[0], + Args: args, + Stdin: os.Stdin, + Stdout: os.Stdout, + Stderr: os.Stderr, + ExtraFiles: []*os.File{w}, + Env: append(os.Environ(), + "_LIBCONTAINER_INITPIPE=3", + fmt.Sprintf("_LIBCONTAINER_INITPID=%d", pid), + ), + } + + if err := cmd.Start(); err != nil { + return err + } + w.Close() + + var result result + if err := json.NewDecoder(r).Decode(&result); err != nil { + return err + } + + if err := cmd.Wait(); err != nil { + return err + } + + log.Infof("Docker PID %d", result.Pid) + + p, err := os.FindProcess(result.Pid) + if err != nil { + return err + } + + handleTerm(p) + + if err := switchCgroup(result.Pid, pid); err != nil { + return err + } + + _, err = p.Wait() + return err +} + +func handleTerm(p *os.Process) { + term := make(chan os.Signal) + signal.Notify(term, syscall.SIGTERM) + go func() { + <-term + p.Signal(syscall.SIGTERM) + }() +} + +func waitForPid(service string, project *project.Project) (int, error) { + for { + if pid, err := getPid(service, project); err != nil || pid == 0 { + log.Infof("Waiting for %s : %d : %v", service, pid, err) + time.Sleep(250) + } else { + return pid, err + } + } +} + +func getPid(service string, project *project.Project) (int, error) { + s, err := project.CreateService(service) + if err != nil { + return 0, err + } + + containers, err := s.Containers() + if err != nil { + return 0, err + } + + if len(containers) == 0 { + return 0, nil + } + + client, err := docker.CreateClient(docker.ClientOpts{ + Host: config.DOCKER_SYSTEM_HOST, + }) + if err != nil { + return 0, err + } + + id, err := containers[0].Id() + if err != nil { + return 0, err + } + + info, err := client.InspectContainer(id) + if err != nil || info == nil { + return 0, err + } + + if info.State.Running { + return info.State.Pid, nil + } + + return 0, nil +} + +func main(cfg *config.CloudConfig) error { + os.Unsetenv("_LIBCONTAINER_INITPIPE") + os.Unsetenv("_LIBCONTAINER_INITPID") + + if err := system.ParentDeathSignal(syscall.SIGKILL).Set(); err != nil { + return err + } + + if err := os.Remove("/var/run/docker.pid"); err != nil && !os.IsNotExist(err) { + return err + } + + dockerCfg := cfg.Rancher.UserDocker + + args := dockerCfg.FullArgs() + + log.Debugf("User Docker args: %v", args) + + if dockerCfg.TLS { + log.Debug("Generating TLS certs if needed") + if err := control.Generate(true, "/etc/docker/tls", []string{"localhost"}); err != nil { + return err + } + } + + args, err := expand("docker-init", "docker", args) + if err != nil { + return err + } + + log.Infof("Running %v", args) + err = syscall.Exec(args[0], args, dockerCfg.AppendEnv()) + return err +} + +func switchCgroup(src, target int) error { + cgroupFile := fmt.Sprintf("/proc/%d/cgroup", target) + f, err := os.Open(cgroupFile) + if err != nil { + return err + } + defer f.Close() + + targetCgroups := map[string]string{} + + s := bufio.NewScanner(f) + for s.Scan() { + text := s.Text() + parts := strings.Split(text, ":") + subparts := strings.Split(parts[1], "=") + subsystem := subparts[0] + if len(subparts) > 1 { + subsystem = subparts[1] + } + + targetPath := fmt.Sprintf("/host/sys/fs/cgroup/%s%s", subsystem, parts[2]) + log.Infof("Moving Docker to cgroup %s", targetPath) + targetCgroups[subsystem] = targetPath + } + + if err := s.Err(); err != nil { + return err + } + + return cgroups.EnterPid(targetCgroups, src) +} diff --git a/compose/project.go b/compose/project.go index 168bd599..c05faf3b 100644 --- a/compose/project.go +++ b/compose/project.go @@ -49,6 +49,10 @@ func RunServices(cfg *config.CloudConfig) error { return p.Up() } +func GetProject(cfg *config.CloudConfig) (*project.Project, error) { + return newCoreServiceProject(cfg) +} + func newProject(name string, cfg *config.CloudConfig) (*project.Project, error) { clientFactory, err := rosDocker.NewClientFactory(docker.ClientOpts{}) if err != nil { diff --git a/config/docker_config.go b/config/docker_config.go new file mode 100644 index 00000000..c2ccd739 --- /dev/null +++ b/config/docker_config.go @@ -0,0 +1,17 @@ +package config + +import "os" + +func (d *DockerConfig) FullArgs() []string { + args := append(d.Args, d.ExtraArgs...) + + if d.TLS { + args = append(args, d.TLSArgs...) + } + + return args +} + +func (d *DockerConfig) AppendEnv() []string { + return append(os.Environ(), d.Environment...) +} diff --git a/config/types.go b/config/types.go index 81a45a4f..4435ba35 100644 --- a/config/types.go +++ b/config/types.go @@ -91,14 +91,16 @@ type UpgradeConfig struct { } type DockerConfig struct { - TLS bool `yaml:"tls,omitempty"` - TLSArgs []string `yaml:"tls_args,flow,omitempty"` - Args []string `yaml:"args,flow,omitempty"` - ExtraArgs []string `yaml:"extra_args,flow,omitempty"` - ServerCert string `yaml:"server_cert,omitempty"` - ServerKey string `yaml:"server_key,omitempty"` - CACert string `yaml:"ca_cert,omitempty"` - CAKey string `yaml:"ca_key,omitempty"` + TLS bool `yaml:"tls,omitempty"` + TLSArgs []string `yaml:"tls_args,flow,omitempty"` + Args []string `yaml:"args,flow,omitempty"` + ExtraArgs []string `yaml:"extra_args,flow,omitempty"` + ServerCert string `yaml:"server_cert,omitempty"` + ServerKey string `yaml:"server_key,omitempty"` + CACert string `yaml:"ca_cert,omitempty"` + CAKey string `yaml:"ca_key,omitempty"` + Environment []string `yaml:"environment,omitempty"` + StorageContext string `yaml:"storage_context,omitempty"` } type SshConfig struct { diff --git a/init/init.go b/init/init.go index 6d4b2855..528670c6 100644 --- a/init/init.go +++ b/init/init.go @@ -139,6 +139,7 @@ func getLaunchConfig(cfg *config.CloudConfig, dockerCfg *config.DockerConfig) (* launchConfig.DnsConfig.Nameservers = cfg.Rancher.Network.Dns.Nameservers launchConfig.DnsConfig.Search = cfg.Rancher.Network.Dns.Search + launchConfig.Environment = dockerCfg.Environment if !cfg.Rancher.Debug { launchConfig.LogFile = config.SYSTEM_DOCKER_LOG diff --git a/main.go b/main.go index 15a8bd81..0d1bcfd7 100644 --- a/main.go +++ b/main.go @@ -14,6 +14,7 @@ import ( "github.com/rancherio/os/cmd/respawn" "github.com/rancherio/os/cmd/sysinit" "github.com/rancherio/os/cmd/systemdocker" + "github.com/rancherio/os/cmd/userdocker" "github.com/rancherio/os/cmd/wait" "github.com/rancherio/os/config" osInit "github.com/rancherio/os/init" @@ -41,6 +42,7 @@ func main() { registerCmd("/init", osInit.MainInit) registerCmd(config.SYSINIT_BIN, sysinit.Main) registerCmd("/usr/bin/dockerlaunch", dockerlaunchMain.Main) + registerCmd("/usr/bin/user-docker", userdocker.Main) registerCmd("/usr/bin/system-docker", systemdocker.Main) registerCmd("/sbin/poweroff", power.PowerOff) registerCmd("/sbin/reboot", power.Reboot) diff --git a/os-config.yml b/os-config.yml index 854a9a9a..05361b95 100644 --- a/os-config.yml +++ b/os-config.yml @@ -120,19 +120,20 @@ rancher: privileged: true read_only: true volumes: + - /usr/bin/docker:/usr/bin/docker.dist:ro - /usr/bin/ros:/sbin/halt:ro + - /usr/bin/ros:/sbin/netconf:ro - /usr/bin/ros:/sbin/poweroff:ro - /usr/bin/ros:/sbin/reboot:ro - /usr/bin/ros:/sbin/shutdown:ro - - /usr/bin/ros:/sbin/netconf:ro - /usr/bin/ros:/usr/bin/cloud-init:ro - - /usr/bin/ros:/usr/bin/rancherctl:ro - - /usr/bin/ros:/usr/bin/ros:ro - - /usr/bin/ros:/usr/bin/respawn:ro - - /usr/bin/ros:/usr/bin/system-docker:ro - - /usr/bin/ros:/usr/sbin/wait-for-docker:ro - /usr/bin/ros:/usr/bin/dockerlaunch:ro - - /usr/bin/docker:/usr/bin/docker.dist:ro + - /usr/bin/ros:/usr/bin/rancherctl:ro + - /usr/bin/ros:/usr/bin/respawn:ro + - /usr/bin/ros:/usr/bin/ros:ro + - /usr/bin/ros:/usr/bin/system-docker:ro + - /usr/bin/ros:/usr/bin/user-docker:ro + - /usr/bin/ros:/usr/sbin/wait-for-docker:ro console: image: rancher/os-console:v0.4.0-dev labels: @@ -225,15 +226,16 @@ rancher: read_only: true volumes: - /dev:/host/dev - - /usr/share/ros/os-config.yml:/usr/share/ros/os-config.yml - - /var/lib/rancher:/var/lib/rancher - - /var/lib/rancher/conf:/var/lib/rancher/conf + - /etc/docker:/etc/docker - /etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt.rancher - - /lib/modules:/lib/modules - /lib/firmware:/lib/firmware + - /lib/modules:/lib/modules - /run:/run - - /var/run:/var/run + - /usr/share/ros/os-config.yml:/usr/share/ros/os-config.yml + - /var/lib/rancher/conf:/var/lib/rancher/conf + - /var/lib/rancher:/var/lib/rancher - /var/log:/var/log + - /var/run:/var/run udev-cold: image: rancher/os-udev:v0.4.0-dev labels: @@ -269,6 +271,20 @@ rancher: volumes: - /home:/home - /opt:/opt + docker: + image: rancher/os-docker:v0.4.0-dev + labels: + io.rancher.os.scope: system + net: host + pid: host + ipc: host + uts: host + privileged: true + restart: always + volumes_from: + - all-volumes + volumes: + - /sys/fs/cgroup:/host/sys/fs/cgroup system_docker: args: [daemon, --log-opt, max-size=25m, --log-opt, max-file=2, -s, overlay, -b, docker-sys, --fixed-cidr, 172.18.42.1/16, --restart=false, -g, /var/lib/system-docker, -G, root,