Merge pull request #2014 from SvenDowideit/v1.0.x-open-vm-tools-exit

[v1.0.x] Improve the way shutdown works

cmd/control/cli.go

@@ -1,6 +1,7 @@
 package control
 
 import (
+	"fmt"
 	"os"
 
 	"github.com/codegangsta/cli"
@@ -14,7 +15,7 @@ func Main() {
 	app := cli.NewApp()
 
 	app.Name = os.Args[0]
-	app.Usage = "Control and configure RancherOS"
+	app.Usage = fmt.Sprintf("Control and configure RancherOS\nbuilt: %s", config.BuildDate)
 	app.Version = config.Version
 	app.Author = "Rancher Labs, Inc."
 	app.EnableBashCompletion = true

cmd/control/selinux.go

@@ -11,7 +11,6 @@ import (
 func selinuxCommand() cli.Command {
 	app := cli.Command{}
 	app.Name = "selinux"
-	app.Usage = "Launch SELinux tools container."
 	app.Action = func(c *cli.Context) error {
 		argv := []string{"system-docker", "run", "-it", "--privileged", "--rm",
 			"--net", "host", "--pid", "host", "--ipc", "host",

cmd/control/service/service.go

@@ -36,7 +36,6 @@ func Commands() cli.Command {
 	app := cli.Command{}
 	app.Name = "service"
 	app.ShortName = "s"
-	app.Usage = "Command line interface for services and compose."
 	app.Before = beforeApp
 	app.Flags = append(dockerApp.DockerClientFlags(), cli.BoolFlag{
 		Name: "verbose,debug",

cmd/power/power.go

@@ -2,17 +2,20 @@ package power
 
 import (
 	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 	"syscall"
+	"time"
 
 	"golang.org/x/net/context"
 
 	"github.com/docker/engine-api/types"
 	"github.com/docker/engine-api/types/container"
 	"github.com/docker/engine-api/types/filters"
+	"github.com/rancher/os/config"
 	"github.com/rancher/os/log"
 
 	"github.com/rancher/os/docker"
@@ -76,21 +79,35 @@ func runDocker(name string) error {
 		return err
 	}
 
-	go func() {
-		client.ContainerAttach(context.Background(), types.ContainerAttachOptions{
-			ContainerID: powerContainer.ID,
-			Stream:      true,
-			Stderr:      true,
-			Stdout:      true,
-		})
-	}()
-
 	err = client.ContainerStart(context.Background(), powerContainer.ID)
 	if err != nil {
 		return err
 	}
 
-	_, err = client.ContainerWait(context.Background(), powerContainer.ID)
+	reader, err := client.ContainerLogs(context.Background(), types.ContainerLogsOptions{
+		ContainerID: powerContainer.ID,
+		ShowStderr:  true,
+		ShowStdout:  true,
+		Follow:      true,
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for {
+		p := make([]byte, 4096)
+		n, err := reader.Read(p)
+		if err != nil {
+			log.Error(err)
+			if n == 0 {
+				reader.Close()
+				break
+			}
+		}
+		if n > 0 {
+			fmt.Print(string(p))
+		}
+	}
 
 	if err != nil {
 		log.Fatal(err)
@@ -126,13 +143,39 @@ func Halt() {
 }
 
 func reboot(code uint) {
+	cfg := config.LoadConfig()
+	timeoutValue := cfg.Rancher.ShutdownTimeout
+	if timeoutValue == 0 {
+		timeoutValue = 60
+	}
+	if timeoutValue < 5 {
+		timeoutValue = 5
+	}
+	log.Infof("Setting %s timeout to %d (rancher.shutdown_timeout set to %d)", os.Args[0], timeoutValue, cfg.Rancher.ShutdownTimeout)
+
+	go func() {
+		timeout := time.After(time.Duration(timeoutValue) * time.Second)
+		tick := time.Tick(100 * time.Millisecond)
+		// Keep trying until we're timed out or got a result or got an error
+		for {
+			select {
+			// Got a timeout! fail with a timeout error
+			case <-timeout:
+				log.Errorf("Container shutdown taking too long, forcing %s.", os.Args[0])
+				syscall.Sync()
+				syscall.Reboot(int(code))
+			case <-tick:
+				fmt.Printf(".")
+			}
+		}
+	}()
+
 	err := shutDownContainers()
 	if err != nil {
 		log.Error(err)
 	}
 
 	syscall.Sync()
-
 	err = syscall.Reboot(int(code))
 	if err != nil {
 		log.Fatal(err)
@@ -187,27 +230,63 @@ func shutDownContainers() error {
 	}
 
 	var stopErrorStrings []string
+	consoleContainerIdx := -1
 
-	for _, container := range containers {
+	for idx, container := range containers {
 		if container.ID == currentContainerID {
 			continue
 		}
+		if container.Names[0] == "/console" {
+			consoleContainerIdx = idx
+			continue
+		}
 
-		log.Infof("Stopping %s : %v", container.ID[:12], container.Names)
+		log.Infof("Stopping %s : %s", container.Names[0], container.ID[:12])
 		stopErr := client.ContainerStop(context.Background(), container.ID, timeout)
 		if stopErr != nil {
+			log.Errorf("------- Error Stopping %s : %s", container.Names[0], stopErr.Error())
 			stopErrorStrings = append(stopErrorStrings, " ["+container.ID+"] "+stopErr.Error())
 		}
 	}
 
+	// lets see what containers are still running and only wait on those
+	containers, err = client.ContainerList(context.Background(), opts)
+	if err != nil {
+		return err
+	}
+
 	var waitErrorStrings []string
 
-	for _, container := range containers {
+	for idx, container := range containers {
 		if container.ID == currentContainerID {
 			continue
 		}
+		if container.Names[0] == "/console" {
+			consoleContainerIdx = idx
+			continue
+		}
+		log.Infof("Waiting %s : %s", container.Names[0], container.ID[:12])
 		_, waitErr := client.ContainerWait(context.Background(), container.ID)
 		if waitErr != nil {
+			log.Errorf("------- Error Waiting %s : %s", container.Names[0], waitErr.Error())
+			waitErrorStrings = append(waitErrorStrings, " ["+container.ID+"] "+waitErr.Error())
+		}
+	}
+
+	// and now stop the console
+	if consoleContainerIdx != -1 {
+		container := containers[consoleContainerIdx]
+		log.Infof("Console Stopping %v : %s", container.Names, container.ID[:12])
+		stopErr := client.ContainerStop(context.Background(), container.ID, timeout)
+		if stopErr != nil {
+			log.Errorf("------- Error Stopping %v : %s", container.Names, stopErr.Error())
+			stopErrorStrings = append(stopErrorStrings, " ["+container.ID+"] "+stopErr.Error())
+		}
+
+		log.Infof("Console Waiting %v : %s", container.Names, container.ID[:12])
+		_, waitErr := client.ContainerWait(context.Background(), container.ID)
+		if waitErr != nil {
+			log.Errorf("------- Error Waiting %v : %s", container.Names, waitErr.Error())
 			waitErrorStrings = append(waitErrorStrings, " ["+container.ID+"] "+waitErr.Error())
 		}
 	}

cmd/power/shutdown.go

@@ -1,6 +1,7 @@
 package power
 
 import (
+	"fmt"
 	"os"
 
 	"github.com/codegangsta/cli"
@@ -13,10 +14,9 @@ func Main() {
 	app := cli.NewApp()
 
 	app.Name = os.Args[0]
-	app.Usage = "Control and configure RancherOS"
+	app.Usage = fmt.Sprintf("%s RancherOS\nbuilt: %s", app.Name, config.BuildDate)
 	app.Version = config.Version
 	app.Author = "Rancher Labs, Inc."
-	app.Email = "sid@rancher.com"
 	app.EnableBashCompletion = true
 	app.Action = shutdown
 	app.Flags = []cli.Flag{
@@ -30,6 +30,7 @@ func Main() {
 		},
 	}
 	app.HideHelp = true
+
 	app.Run(os.Args)
 }
 

cmd/respawn/respawn.go

@@ -1,6 +1,7 @@
 package respawn
 
 import (
+	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
@@ -13,6 +14,7 @@ import (
 	"time"
 
 	"github.com/codegangsta/cli"
+	"github.com/rancher/os/config"
 	"github.com/rancher/os/log"
 )
 
@@ -28,6 +30,11 @@ func Main() {
 	runtime.LockOSThread()
 	app := cli.NewApp()
 
+	app.Name = os.Args[0]
+	app.Usage = fmt.Sprintf("%s RancherOS\nbuilt: %s", app.Name, config.BuildDate)
+	app.Version = config.Version
+	app.Author = "Rancher Labs, Inc."
+
 	app.Flags = []cli.Flag{
 		cli.StringFlag{
 			Name:  "file, f",
@@ -36,6 +43,9 @@ func Main() {
 	}
 	app.Action = run
 
+	log.Infof("%s, %s", app.Usage, app.Version)
+	fmt.Printf("%s, %s", app.Usage, app.Version)
+
 	app.Run(os.Args)
 }
 
@@ -69,17 +79,21 @@ func run(c *cli.Context) error {
 		panic(err)
 	}
 
-	var wg sync.WaitGroup
+	lines := strings.Split(string(input), "\n")
+	doneChannel := make(chan string, len(lines))
 
-	for _, line := range strings.Split(string(input), "\n") {
+	for _, line := range lines {
 		if strings.TrimSpace(line) == "" || strings.Index(strings.TrimSpace(line), "#") == 0 {
 			continue
 		}
-		wg.Add(1)
-		go execute(line, &wg)
+		go execute(line, doneChannel)
 	}
 
-	wg.Wait()
+	for i := 0; i < len(lines); i++ {
+		line := <-doneChannel
+		log.Infof("FINISHED: %s", line)
+		fmt.Printf("FINISHED: %s", line)
+	}
 	return nil
 }
 
@@ -101,19 +115,20 @@ func termPids() {
 	defer processLock.Unlock()
 
 	for _, process := range processes {
+		log.Infof("sending SIGTERM to %d", process.Pid)
 		process.Signal(syscall.SIGTERM)
 	}
 }
 
-func execute(line string, wg *sync.WaitGroup) {
-	defer wg.Done()
+func execute(line string, doneChannel chan string) {
+	defer func() { doneChannel <- line }()
 
 	start := time.Now()
 	count := 0
 
-	for {
-		args := strings.Split(line, " ")
+	args := strings.Split(line, " ")
 
+	for {
 		cmd := exec.Command(args[0], args[1:]...)
 		cmd.Stdout = os.Stdout
 		cmd.Stderr = os.Stderr

config/schema.go

@@ -52,7 +52,8 @@ var schema = `{
         "defaults": {"$ref": "#/definitions/defaults_config"},
         "resize_device": {"type": "string"},
         "sysctl": {"type": "object"},
-        "restart_services": {"type": "array"}
+        "restart_services": {"type": "array"},
+        "shutdown_timeout": {"type": "integer"}
       }
     },
 

config/types.go

@@ -51,6 +51,7 @@ const (
 var (
 	OemConfigFile = OEM + "/oem-config.yml"
 	Version       string
+	BuildDate     string
 	Arch          string
 	Suffix        string
 	OsRepo        string
@@ -129,6 +130,7 @@ type RancherConfig struct {
 	ResizeDevice        string                                    `yaml:"resize_device,omitempty"`
 	Sysctl              map[string]string                         `yaml:"sysctl,omitempty"`
 	RestartServices     []string                                  `yaml:"restart_services,omitempty"`
+	ShutdownTimeout     int                                       `yaml:"shutdown_timeout,omitempty"`
 }
 
 type UpgradeConfig struct {

os-config.tpl.yml

@@ -1,4 +1,5 @@
 rancher:
+  shutdown_timeout: 60
   environment:
     VERSION: {{.VERSION}}
     SUFFIX: {{.SUFFIX}}

scripts/build-target

@@ -14,6 +14,7 @@ fi
 
 OUTPUT=${OUTPUT:-bin/ros}
 echo Building $OUTPUT
+BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
 
-CONST="-X github.com/docker/docker/dockerversion.GitCommit=${COMMIT} -X github.com/docker/docker/dockerversion.Version=${DOCKER_PATCH_VERSION} -X github.com/docker/docker/dockerversion.BuildTime=$(date -u +'%Y-%m-%dT%H:%M:%SZ') -X github.com/docker/docker/dockerversion.IAmStatic=true -X github.com/rancher/os/config.Version=${VERSION} -X github.com/rancher/os/config.OsRepo=${OS_REPO}"
+CONST="-X github.com/docker/docker/dockerversion.GitCommit=${COMMIT} -X github.com/docker/docker/dockerversion.Version=${DOCKER_PATCH_VERSION} -X github.com/docker/docker/dockerversion.BuildTime='${BUILDDATE}' -X github.com/docker/docker/dockerversion.IAmStatic=true -X github.com/rancher/os/config.Version=${VERSION} -X github.com/rancher/os/config.OsRepo=${OS_REPO} -X github.com/rancher/os/config.BuildDate='${BUILDDATE}'"
 go build -tags "selinux cgo daemon netgo dnspatch" -installsuffix netgo -ldflags "$CONST -linkmode external -extldflags -static -s -w" -o ${OUTPUT}

scripts/create-installed

@@ -36,7 +36,7 @@ EOF
 
 rm -f build/{success,hd.img}
 qemu-img create -f qcow2 build/hd.img 8G
-qemu-system-${QEMUARCH} -serial stdio \
+qemu-system-${QEMUARCH} -serial mon:stdio \
     -enable-kvm \
     -drive if=virtio,file=build/hd.img \
     -boot d -cdrom ./dist/artifacts/rancheros.iso \
@@ -51,4 +51,4 @@ qemu-system-${QEMUARCH} -serial stdio \
 
 mkdir -p state
 cp build/hd.img state/hd.img
-echo "------------------------ RancherOS installed to hd.img."
+echo "------------------------ RancherOS installed to hd.img."

scripts/installer/kexec/Dockerfile.dapper

@@ -10,7 +10,7 @@ RUN echo "Acquire::http { Proxy \"$APTPROXY\"; };" >> /etc/apt/apt.conf.d/01prox
     && apt-get install -yq build-essential autoconf libtool gawk alien fakeroot \
         zlib1g-dev uuid-dev libattr1-dev libblkid-dev libselinux-dev libudev-dev libdevmapper-dev \
         module-init-tools \
-        parted lsscsi ksh curl git
+        parted lsscsi ksh curl git wget
 
 WORKDIR /source
 
@@ -20,8 +20,9 @@ WORKDIR /source
 #    && tar zxvf /source/build-linux-4.9.15-rancher-x86.tar.gz
 
 # https://www.kernel.org/pub/linux/utils/kernel/kexec/
-ENV VERSION 2.0.14
-ADD https://www.kernel.org/pub/linux/utils/kernel/kexec/kexec-tools-$VERSION.tar.gz .
+ENV VERSION 2.0.15
+RUN wget https://www.kernel.org/pub/linux/utils/kernel/kexec/kexec-tools-$VERSION.tar.gz \
+    && tar zxvf kexec-tools-$VERSION.tar.gz
 
 RUN zcat kexec-tools-$VERSION.tar.gz | tar xvf - \
   && cd kexec-tools-$VERSION \

scripts/schema.json

@@ -50,7 +50,8 @@
         "defaults": {"$ref": "#/definitions/defaults_config"},
         "resize_device": {"type": "string"},
         "sysctl": {"type": "object"},
-        "restart_services": {"type": "array"}
+        "restart_services": {"type": "array"},
+	"shutdown_timeout": {"type": "integer"}
       }
     },
 

tests/cmdline_test.go

@@ -19,7 +19,7 @@ func (s *QemuSuite) TestElideCmdLine(c *C) {
 	s.RunQemuWith(c, runArgs...)
 
 	s.CheckOutput(c, "nope\n", Equals, "hostname")
-	cmdline := s.CheckOutput(c, "", Not(Equals), "cat /proc/cmdline",)
+	cmdline := s.CheckOutput(c, "", Not(Equals), "cat /proc/cmdline")
 	if strings.Contains(cmdline, extra) {
 		c.Errorf("/proc/cmdline (%s) contains info that should be elided (%s)", cmdline, extra)
 	}