From 19f9a1b281b3435c52c32f7d2a1a8453fd91fca8 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Wed, 29 Jul 2015 00:51:49 -0700 Subject: [PATCH] Rebase on docker-from-scratch --- config/types.go | 7 +- init/bootstrap.go | 25 ++- init/init.go | 408 ++++++++++++---------------------------------- init/root.go | 96 +++++++++++ main.go | 3 +- os-config.yml | 31 ++-- scripts/run | 9 +- 7 files changed, 237 insertions(+), 342 deletions(-) create mode 100644 init/root.go diff --git a/config/types.go b/config/types.go index 29445ecb..eb08cf7b 100644 --- a/config/types.go +++ b/config/types.go @@ -9,15 +9,16 @@ import ( const ( CONSOLE_CONTAINER = "console" DOCKER_BIN = "/usr/bin/docker" + ROS_BIN = "/usr/bin/ros" + SYSINIT_BIN = "/usr/bin/ros-sysinit" DOCKER_SYSTEM_HOME = "/var/lib/system-docker" DOCKER_SYSTEM_HOST = "unix:///var/run/system-docker.sock" DOCKER_HOST = "unix:///var/run/docker.sock" - IMAGES_PATH = "/" + IMAGES_PATH = "/usr/share/ros" IMAGES_PATTERN = "images*.tar" - SYS_INIT = "/sbin/init-sys" - USER_INIT = "/sbin/init-user" MODULES_ARCHIVE = "/modules.tar" DEBUG = false + SYSTEM_DOCKER_LOG = "/var/log/system-docker.log" LABEL = "label" HASH = "io.rancher.os.hash" diff --git a/init/bootstrap.go b/init/bootstrap.go index 1d64fa71..1ed20b73 100644 --- a/init/bootstrap.go +++ b/init/bootstrap.go @@ -2,16 +2,17 @@ package init import ( "os" - "os/exec" "syscall" "fmt" + "strings" + log "github.com/Sirupsen/logrus" + "github.com/rancher/docker-from-scratch" "github.com/rancherio/os/config" "github.com/rancherio/os/docker" "github.com/rancherio/os/util" "github.com/rancherio/rancher-compose/librcompose/project" - "strings" ) func autoformat(cfg *config.CloudConfig) error { @@ -31,20 +32,14 @@ func runBootstrapContainers(cfg *config.CloudConfig) error { return docker.RunServices("bootstrap", cfg, cfg.Rancher.BootstrapContainers) } -func startDocker(cfg *config.CloudConfig) (chan interface{}, error) { - for _, d := range []string{config.DOCKER_SYSTEM_HOST, "/var/run"} { - err := os.MkdirAll(d, 0700) - if err != nil { - return nil, err - } - } +func startDocker(cfg *config.Config) (chan interface{}, error) { - cmd := exec.Command(cfg.Rancher.BootstrapDocker.Args[0], cfg.Rancher.BootstrapDocker.Args[1:]...) - if cfg.Rancher.Debug { - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - } - err := cmd.Start() + launchConfig, args := getLaunchConfig(cfg, &cfg.Rancher.BootstrapDocker) + launchConfig.Fork = true + launchConfig.LogFile = "" + launchConfig.NoLog = true + + cmd, err := dockerlaunch.LaunchDocker(launchConfig, config.DOCKER_BIN, args...) if err != nil { return nil, err } diff --git a/init/init.go b/init/init.go index 26dc4eec..deb28a70 100644 --- a/init/init.go +++ b/init/init.go @@ -3,203 +3,54 @@ package init import ( + "bufio" "fmt" - "io/ioutil" "os" "os/exec" "strings" - "syscall" log "github.com/Sirupsen/logrus" - "github.com/rancherio/os/cmd/network" + "github.com/rancher/docker-from-scratch" "github.com/rancherio/os/config" "github.com/rancherio/os/util" ) const ( - STATE string = "/var" - SYSTEM_DOCKER string = "/usr/bin/system-docker" - DOCKER string = "/usr/bin/docker" - SYSINIT string = "/sbin/rancher-sysinit" + STATE string = "/state" ) var ( - dirs []string = []string{ - "/etc/ssl/certs", - "/sbin", - "/usr/bin", - "/usr/sbin", - } - postDirs []string = []string{ - "/var/log", - "/var/lib/rancher/state/home", - "/var/lib/rancher/state/opt", - } - mounts [][]string = [][]string{ - {"devtmpfs", "/dev", "devtmpfs", ""}, - {"none", "/dev/pts", "devpts", ""}, - {"none", "/etc/docker", "tmpfs", ""}, - {"none", "/proc", "proc", ""}, - {"none", "/run", "tmpfs", ""}, - {"none", "/sys", "sysfs", ""}, - {"none", "/sys/fs/cgroup", "tmpfs", ""}, - } - postMounts [][]string = [][]string{ - {"none", "/var/run", "tmpfs", ""}, - } - cgroups []string = []string{ - "blkio", - "cpu", - "cpuacct", - "cpuset", - "devices", - "freezer", - "memory", - "net_cls", - "perf_event", - } - // Notice this map is the reverse order of a "ln -s x y" command - // so map[y] = x - symlinks map[string]string = map[string]string{ - "/etc/ssl/certs/ca-certificates.crt": "/ca.crt", - "/sbin/modprobe": "/busybox", - "/usr/sbin/iptables": "/xtables-multi", - DOCKER: "/docker", - SYSTEM_DOCKER: "/docker", - SYSINIT: "/init", - "/home": "/var/lib/rancher/state/home", - "/opt": "/var/lib/rancher/state/opt", + mountConfig = dockerlaunch.Config{ + CgroupHierarchy: map[string]string{ + "cpu": "cpu", + "cpuacct": "cpu", + "net_cls": "net_cls", + "net_prio": "net_cls", + }, } ) -func createSymlinks(cfg *config.CloudConfig, symlinks map[string]string) error { - log.Debug("Creating symlinking") - for dest, src := range symlinks { - if _, err := os.Stat(dest); os.IsNotExist(err) { - log.Debugf("Symlinking %s => %s", src, dest) - if err = os.Symlink(src, dest); err != nil { - return err - } - } - } +func loadModules(cfg *config.Config) error { + mounted := map[string]bool{} - return nil -} - -func createDirs(dirs ...string) error { - for _, dir := range dirs { - if _, err := os.Stat(dir); os.IsNotExist(err) { - log.Debugf("Creating %s", dir) - err = os.MkdirAll(dir, 0755) - if err != nil { - return err - } - } - } - - return nil -} - -func createMounts(mounts ...[]string) error { - for _, mount := range mounts { - log.Debugf("Mounting %s %s %s %s", mount[0], mount[1], mount[2], mount[3]) - err := util.Mount(mount[0], mount[1], mount[2], mount[3]) - if err != nil { - return err - } - } - - return nil -} - -func remountRo(cfg *config.CloudConfig) error { - log.Info("Remouting root read only") - return util.Remount("/", "ro") -} - -func mountCgroups(cfg *config.CloudConfig) error { - for _, cgroup := range cgroups { - err := createDirs("/sys/fs/cgroup/" + cgroup) - if err != nil { - return err - } - - err = createMounts([][]string{ - {"none", "sys/fs/cgroup/" + cgroup, "cgroup", cgroup}, - }...) - if err != nil { - return err - } - } - - log.Debug("Done mouting cgroupfs") - - return nil -} - -func extractModules(cfg *config.CloudConfig) error { - if _, err := os.Stat(config.MODULES_ARCHIVE); os.IsNotExist(err) { - log.Debug("Modules do not exist") - return nil - } - - log.Debug("Extracting modules") - return util.ExtractTar(config.MODULES_ARCHIVE, "/") -} - -func setResolvConf(cfg *config.CloudConfig) error { - log.Debug("Creating /etc/resolv.conf") - //f, err := os.OpenFile("/etc/resolv.conf", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) - f, err := os.Create("/etc/resolv.conf") + f, err := os.Open("/proc/modules") if err != nil { return err } - defer f.Close() - for _, dns := range cfg.Rancher.Network.Dns.Nameservers { - content := fmt.Sprintf("nameserver %s\n", dns) - if _, err = f.Write([]byte(content)); err != nil { - return err + reader := bufio.NewScanner(f) + for reader.Scan() { + mounted[strings.SplitN(reader.Text(), " ", 2)[0]] = true + } + + for _, module := range cfg.Modules { + if mounted[module] { + continue } - } - search := strings.Join(cfg.Rancher.Network.Dns.Search, " ") - if search != "" { - content := fmt.Sprintf("search %s\n", search) - if _, err = f.Write([]byte(content)); err != nil { - return err - } - } - - if cfg.Rancher.Network.Dns.Domain != "" { - content := fmt.Sprintf("domain %s\n", cfg.Rancher.Network.Dns.Domain) - if _, err = f.Write([]byte(content)); err != nil { - return err - } - } - - return nil -} - -func loadModules(cfg *config.CloudConfig) error { - filesystems, err := ioutil.ReadFile("/proc/filesystems") - if err != nil { - return err - } - - if !strings.Contains(string(filesystems), "nodev\toverlay\n") { - log.Debug("Loading overlay module") - err = exec.Command("/sbin/modprobe", "overlay").Run() - if err != nil { - return err - } - } - - for _, module := range cfg.Rancher.Modules { log.Debugf("Loading module %s", module) - err = exec.Command("/sbin/modprobe", module).Run() - if err != nil { + if err := exec.Command("modprobe", module).Run(); err != nil { log.Errorf("Could not load module %s, err %v", module, err) } } @@ -207,19 +58,18 @@ func loadModules(cfg *config.CloudConfig) error { return nil } -func sysInit(cfg *config.CloudConfig) error { - args := append([]string{SYSINIT}, os.Args[1:]...) +func sysInit(cfg *config.Config) error { + args := append([]string{config.SYSINIT_BIN}, os.Args[1:]...) - var cmd *exec.Cmd - if util.IsRunningInTty() { - cmd = exec.Command(args[0], args[1:]...) - cmd.Stdin = os.Stdin - cmd.Stderr = os.Stderr - cmd.Stdout = os.Stdout - } else { - cmd = exec.Command(args[0], args[1:]...) + cmd := &exec.Cmd{ + Path: config.ROS_BIN, + Args: args, } + cmd.Stdin = os.Stdin + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + if err := cmd.Start(); err != nil { return err } @@ -227,128 +77,88 @@ func sysInit(cfg *config.CloudConfig) error { return os.Stdin.Close() } -func execDocker(cfg *config.CloudConfig) error { - log.Info("Launching System Docker") - if !cfg.Rancher.Debug { - output, err := os.Create("/var/log/system-docker.log") - if err != nil { - return err - } - - syscall.Dup2(int(output.Fd()), int(os.Stdout.Fd())) - syscall.Dup2(int(output.Fd()), int(os.Stderr.Fd())) - } - - os.Stdin.Close() - return syscall.Exec(SYSTEM_DOCKER, cfg.Rancher.SystemDocker.Args, os.Environ()) -} - func MainInit() { if err := RunInit(); err != nil { log.Fatal(err) } } -func mountStateTmpfs(cfg *config.CloudConfig) error { - log.Debugf("State will not be persisted") - return util.Mount("none", STATE, "tmpfs", "") -} - -func mountState(cfg *config.CloudConfig) error { +func mountState(cfg *config.Config) error { var err error - if cfg.Rancher.State.Dev != "" { - dev := util.ResolveDevice(cfg.Rancher.State.Dev) - if dev == "" { - msg := fmt.Sprintf("Could not resolve device %q", cfg.Rancher.State.Dev) - log.Infof(msg) - return fmt.Errorf(msg) - } - log.Infof("Mounting state device %s to %s", dev, STATE) - - fsType := cfg.Rancher.State.FsType - if fsType == "auto" { - fsType, err = util.GetFsType(dev) - } - - if err == nil { - log.Debugf("FsType has been set to %s", fsType) - err = util.Mount(dev, STATE, fsType, "") - } - } else { - return mountStateTmpfs(cfg) - } - - return err -} - -func tryMountAndBootstrap(cfg *config.CloudConfig) error { - if err := mountState(cfg); err != nil { - if err := bootstrap(cfg); err != nil { - if cfg.Rancher.State.Required { - return err - } - return mountStateTmpfs(cfg) - } - if err := mountState(cfg); err != nil { - if cfg.Rancher.State.Required { - return err - } - return mountStateTmpfs(cfg) - } - } - return nil -} - -func createGroups(cfg *config.CloudConfig) error { - return ioutil.WriteFile("/etc/group", []byte("root:x:0:\n"), 0644) -} - -func touchSocket(cfg *config.CloudConfig) error { - for _, path := range []string{"/var/run/docker.sock", "/var/run/system-docker.sock"} { - if err := syscall.Unlink(path); err != nil && !os.IsNotExist(err) { - return err - } - err := ioutil.WriteFile(path, []byte{}, 0700) - if err != nil { - return err - } - } - - return nil -} - -func setupSystemBridge(cfg *config.CloudConfig) error { - bridge, cidr := cfg.Rancher.SystemDocker.BridgeConfig() - if bridge == "" { + if cfg.State.Dev == "" { return nil } - return network.ApplyNetworkConfigs(&config.NetworkConfig{ - Interfaces: map[string]config.InterfaceConfig{ - bridge: { - Bridge: true, - Address: cidr, - }, - }, - }) + dev := util.ResolveDevice(cfg.State.Dev) + if dev == "" { + return fmt.Errorf("Could not resolve device %q", cfg.State.Dev) + } + fsType := cfg.State.FsType + if fsType == "auto" { + fsType, err = util.GetFsType(dev) + } + + if err != nil { + return err + } + + log.Debugf("FsType has been set to %s", fsType) + log.Infof("Mounting state device %s to %s", dev, STATE) + return util.Mount(dev, STATE, fsType, "") +} + +func tryMountState(cfg *config.Config) error { + if mountState(cfg) == nil { + return nil + } + + // If we failed to mount lets run bootstrap and try again + if err := bootstrap(cfg); err != nil { + return err + } + + return mountState(cfg) +} + +func tryMountAndBootstrap(cfg *config.Config) error { + if err := tryMountState(cfg); !cfg.State.Required && err != nil { + return nil + } else if err != nil { + return err + } + + log.Debugf("Switching to new root at %s", STATE) + return switchRoot(STATE) +} + +func getLaunchConfig(cfg *config.Config, dockerCfg *config.DockerConfig) (*dockerlaunch.Config, []string) { + var launchConfig dockerlaunch.Config + + args := dockerlaunch.ParseConfig(&launchConfig, append(dockerCfg.Args, dockerCfg.ExtraArgs...)...) + + launchConfig.DnsConfig.Nameservers = cfg.Network.Dns.Nameservers + launchConfig.DnsConfig.Search = cfg.Network.Dns.Search + + if !cfg.Debug { + launchConfig.LogFile = config.SYSTEM_DOCKER_LOG + } + + return &launchConfig, args } func RunInit() error { - var cfg config.CloudConfig + var cfg config.Config os.Setenv("PATH", "/sbin:/usr/sbin:/usr/bin") + // Magic setting to tell Docker to do switch_root and not pivot_root os.Setenv("DOCKER_RAMDISK", "true") initFuncs := []config.InitFunc{ - func(cfg *config.CloudConfig) error { - return createDirs(dirs...) + func(cfg *config.Config) error { + return dockerlaunch.PrepareFs(&mountConfig) }, - func(cfg *config.CloudConfig) error { - log.Info("Setting up mounts") - return createMounts(mounts...) - }, - func(cfg *config.CloudConfig) error { + func(cfg *config.Config) error { newCfg, err := config.LoadConfig() if err == nil { newCfg, err = config.LoadConfig() @@ -357,37 +167,21 @@ func RunInit() error { *cfg = *newCfg } - if cfg.Rancher.Debug { + if cfg.Debug { cfgString, _ := config.Dump(false, true) - log.Debugf("os-config dump: \n%s", cfgString) + if cfgString != "" { + log.Debugf("Config: %s", cfgString) + } } return err }, - mountCgroups, - func(cfg *config.CloudConfig) error { - return createSymlinks(cfg, symlinks) - }, - createGroups, - extractModules, loadModules, - setResolvConf, - setupSystemBridge, tryMountAndBootstrap, - func(cfg *config.CloudConfig) error { + func(cfg *config.Config) error { return cfg.Reload() }, loadModules, - setResolvConf, - func(cfg *config.CloudConfig) error { - return createDirs(postDirs...) - }, - func(cfg *config.CloudConfig) error { - return createMounts(postMounts...) - }, - touchSocket, - // Disable R/O root write now to support updating modules - //remountRo, sysInit, } @@ -395,5 +189,9 @@ func RunInit() error { return err } - return execDocker(&cfg) + launchConfig, args := getLaunchConfig(&cfg, &cfg.SystemDocker) + + log.Info("Launching System Docker") + _, err := dockerlaunch.LaunchDocker(launchConfig, config.DOCKER_BIN, args...) + return err } diff --git a/init/root.go b/init/root.go new file mode 100644 index 00000000..241bd2a4 --- /dev/null +++ b/init/root.go @@ -0,0 +1,96 @@ +package init + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "syscall" + + log "github.com/Sirupsen/logrus" + "github.com/docker/docker/pkg/archive" + "github.com/rancher/docker-from-scratch" + "github.com/rancherio/os/config" +) + +func prepareRoot(rootfs string) error { + usr := path.Join(rootfs, "usr") + if err := os.Remove(usr); err != nil && !os.IsNotExist(err) { + log.Errorf("Failed to delete %s, possibly invalid RancherOS state partition: %v", usr, err) + return err + } + + return nil +} + +func copyMoveRoot(rootfs string) error { + usrVer := fmt.Sprintf("usr-%s", config.VERSION) + usr := path.Join(rootfs, usrVer) + + if err := archive.CopyWithTar("/usr", usr); err != nil { + return err + } + + if err := dockerlaunch.CreateSymlink(usrVer, path.Join(rootfs, "usr")); err != nil { + return err + } + + files, err := ioutil.ReadDir("/") + if err != nil { + return err + } + + for _, file := range files { + filename := path.Join("/", file.Name()) + + if filename == rootfs { + continue + } + + log.Debugf("Deleting %s", filename) + //if err := os.Remove(filename); err != nil { + if err := os.RemoveAll(filename); err != nil { + return err + } + //} + } + + return nil +} + +func switchRoot(rootfs string) error { + for _, i := range []string{"/dev", "/sys", "/proc", "/run"} { + log.Debugf("Moving mount %s to %s", i, path.Join(rootfs, i)) + if err := os.MkdirAll(path.Join(rootfs, i), 0755); err != nil { + return err + } + if err := syscall.Mount(i, path.Join(rootfs, i), "", syscall.MS_MOVE, ""); err != nil { + return err + } + } + + if err := copyMoveRoot(rootfs); err != nil { + return err + } + + if err := syscall.Chdir(rootfs); err != nil { + return err + } + + if err := syscall.Mount(rootfs, "/", "", syscall.MS_MOVE, ""); err != nil { + return err + } + + if err := syscall.Chroot("."); err != nil { + return err + } + + if err := syscall.Chdir("/"); err != nil { + return err + } + + log.Debugf("Successfully moved to new root at %s", rootfs) + os.Setenv("DOCKER_RAMDISK", "false") + + return nil +} diff --git a/main.go b/main.go index 6f65c249..3b8546af 100644 --- a/main.go +++ b/main.go @@ -14,6 +14,7 @@ import ( "github.com/rancherio/os/cmd/sysinit" "github.com/rancherio/os/cmd/systemdocker" "github.com/rancherio/os/cmd/wait" + "github.com/rancherio/os/config" osInit "github.com/rancherio/os/init" ) @@ -37,7 +38,7 @@ func registerCmd(cmd string, mainFunc func()) { func main() { registerCmd("/init", osInit.MainInit) - registerCmd(osInit.SYSINIT, sysinit.Main) + registerCmd(config.SYSINIT_BIN, sysinit.Main) registerCmd("/usr/bin/system-docker", systemdocker.Main) registerCmd("/sbin/poweroff", power.PowerOff) registerCmd("/sbin/reboot", power.Reboot) diff --git a/os-config.yml b/os-config.yml index 852a5efb..206aecfa 100644 --- a/os-config.yml +++ b/os-config.yml @@ -38,8 +38,8 @@ rancher: - /lib/modules:/lib/modules - /lib/firmware:/lib/firmware bootstrap_docker: - args: [docker, -d, -s, overlay, -b, none, --restart=false, -g, /var/lib/system-docker, - -G, root, -H, 'unix:///var/run/system-docker.sock'] + args: [-d, -s, overlay, -b, none, --restart=false, -g, /var/lib/system-docker, + -G, root, -H, 'unix:///var/run/system-docker.sock', --userland-proxy=false] cloud_init: datasources: - configdrive:/media/config-2 @@ -90,6 +90,7 @@ rancher: io.rancher.os.reloadconfig: true io.rancher.os.scope: system links: + - preload-user-images - cloud-init-pre - network net: host @@ -124,17 +125,17 @@ rancher: privileged: true read_only: true volumes: - - /init:/sbin/halt:ro - - /init:/sbin/poweroff:ro - - /init:/sbin/reboot:ro - - /init:/sbin/shutdown:ro - - /init:/sbin/netconf:ro - - /init:/usr/bin/cloud-init:ro - - /init:/usr/bin/rancherctl:ro - - /init:/usr/bin/ros:ro - - /init:/usr/bin/respawn:ro - - /init:/usr/bin/system-docker:ro - - /init:/usr/sbin/wait-for-docker:ro + - /usr/bin/ros:/sbin/halt:ro + - /usr/bin/ros:/sbin/poweroff:ro + - /usr/bin/ros:/sbin/reboot:ro + - /usr/bin/ros:/sbin/shutdown:ro + - /usr/bin/ros:/sbin/netconf:ro + - /usr/bin/ros:/usr/bin/cloud-init:ro + - /usr/bin/ros:/usr/bin/rancherctl:ro + - /usr/bin/ros:/usr/bin/ros:ro + - /usr/bin/ros:/usr/bin/respawn:ro + - /usr/bin/ros:/usr/bin/system-docker:ro + - /usr/bin/ros:/usr/sbin/wait-for-docker:ro - /lib/modules:/lib/modules - /usr/bin/docker:/usr/bin/docker:ro console: @@ -144,7 +145,6 @@ rancher: io.rancher.os.scope: system links: - cloud-init - - dockerwait # because console runs `loud-init -execute`, which may need docker net: host uts: host pid: host @@ -158,7 +158,6 @@ rancher: labels: io.rancher.os.scope: system links: - - cloud-init - network net: host uts: host @@ -298,7 +297,7 @@ rancher: - /home:/home - /opt:/opt system_docker: - args: [docker, -d, --log-driver, syslog, -s, overlay, -b, docker-sys, --fixed-cidr, + args: [docker, -d, -s, overlay, -b, docker-sys, --fixed-cidr, 172.18.42.1/16, --restart=false, -g, /var/lib/system-docker, -G, root, -H, 'unix:///var/run/system-docker.sock', --userland-proxy=false] upgrade: diff --git a/scripts/run b/scripts/run index 869aa8f8..b233d586 100755 --- a/scripts/run +++ b/scripts/run @@ -64,10 +64,15 @@ if [ ! -d ${INITRD_TMP} ]; then popd fi +if [ -e ${INITRD_CURRENT} ]; then + rm -f ${INITRD_CURRENT} +fi + ln -sf ${INITRD_TMP} ${INITRD_CURRENT} -cp bin/rancheros ${INITRD_TMP}/init -cp -f os-config.yml ${INITRD_TMP}/ +mkdir -p ${INITRD_TMP}/usr/{bin,share/ros} +cp bin/rancheros ${INITRD_TMP}/usr/bin/ros +cp -f os-config.yml ${INITRD_TMP}/usr/share/ros cd ${INITRD_TMP} find . | cpio -H newc -o > ${INITRD_TEST}