From bc92fd6a02a37de4a7e348a898e2acc91c2046e6 Mon Sep 17 00:00:00 2001
From: James DeFelice <james@mesosphere.io>
Date: Fri, 8 Jan 2016 16:14:55 +0000
Subject: [PATCH] add a timeout for job runs in case something gets stuck

---
 cluster/test-e2e.sh |  2 +-
 test/e2e/kubectl.go |  4 ++++
 test/e2e/util.go    | 25 ++++++++++++++++++++++---
 3 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/cluster/test-e2e.sh b/cluster/test-e2e.sh
index f6caae2581a..073a6eb7a30 100755
--- a/cluster/test-e2e.sh
+++ b/cluster/test-e2e.sh
@@ -30,4 +30,4 @@ TEST_ARGS="$@"
 
 echo "Running e2e tests:" 1>&2
 echo "./hack/ginkgo-e2e.sh ${TEST_ARGS}" 1>&2
-exec "${KUBE_ROOT}/hack/ginkgo-e2e.sh" ${TEST_ARGS}
+exec "${KUBE_ROOT}/hack/ginkgo-e2e.sh" "$@"
diff --git a/test/e2e/kubectl.go b/test/e2e/kubectl.go
index 55a85cc2196..e12e4176a08 100644
--- a/test/e2e/kubectl.go
+++ b/test/e2e/kubectl.go
@@ -70,6 +70,7 @@ const (
 	simplePodName            = "nginx"
 	nginxDefaultOutput       = "Welcome to nginx!"
 	simplePodPort            = 80
+	runJobTimeout            = 5 * time.Minute
 )
 
 var proxyRegexp = regexp.MustCompile("Starting to serve on 127.0.0.1:([0-9]+)")
@@ -911,8 +912,11 @@ var _ = Describe("Kubectl client", func() {
 
 		It("should create a job from an image, then delete the job [Conformance]", func() {
 			By("executing a command with run --rm and attach with stdin")
+			t := time.NewTimer(runJobTimeout)
+			defer t.Stop()
 			runOutput := newKubectlCommand(nsFlag, "run", jobName, "--image=busybox", "--rm=true", "--restart=Never", "--attach=true", "--stdin", "--", "sh", "-c", "cat && echo 'stdin closed'").
 				withStdinData("abcd1234").
+				withTimeout(t.C).
 				execOrDie()
 			Expect(runOutput).To(ContainSubstring("abcd1234"))
 			Expect(runOutput).To(ContainSubstring("stdin closed"))
diff --git a/test/e2e/util.go b/test/e2e/util.go
index 004d73d4714..c713e950f54 100644
--- a/test/e2e/util.go
+++ b/test/e2e/util.go
@@ -1189,7 +1189,8 @@ func kubectlCmd(args ...string) *exec.Cmd {
 // kubectlBuilder is used to build, custimize and execute a kubectl Command.
 // Add more functions to customize the builder as needed.
 type kubectlBuilder struct {
-	cmd *exec.Cmd
+	cmd     *exec.Cmd
+	timeout <-chan time.Time
 }
 
 func newKubectlCommand(args ...string) *kubectlBuilder {
@@ -1198,6 +1199,11 @@ func newKubectlCommand(args ...string) *kubectlBuilder {
 	return b
 }
 
+func (b *kubectlBuilder) withTimeout(t <-chan time.Time) *kubectlBuilder {
+	b.timeout = t
+	return b
+}
+
 func (b kubectlBuilder) withStdinData(data string) *kubectlBuilder {
 	b.cmd.Stdin = strings.NewReader(data)
 	return &b
@@ -1220,8 +1226,21 @@ func (b kubectlBuilder) exec() (string, error) {
 	cmd.Stdout, cmd.Stderr = &stdout, &stderr
 
 	Logf("Running '%s %s'", cmd.Path, strings.Join(cmd.Args[1:], " ")) // skip arg[0] as it is printed separately
-	if err := cmd.Run(); err != nil {
-		return "", fmt.Errorf("Error running %v:\nCommand stdout:\n%v\nstderr:\n%v\nerror:\n%v\n", cmd, cmd.Stdout, cmd.Stderr, err)
+	if err := cmd.Start(); err != nil {
+		return "", fmt.Errorf("Error starting %v:\nCommand stdout:\n%v\nstderr:\n%v\nerror:\n%v\n", cmd, cmd.Stdout, cmd.Stderr, err)
+	}
+	errCh := make(chan error, 1)
+	go func() {
+		errCh <- cmd.Wait()
+	}()
+	select {
+	case err := <-errCh:
+		if err != nil {
+			return "", fmt.Errorf("Error running %v:\nCommand stdout:\n%v\nstderr:\n%v\nerror:\n%v\n", cmd, cmd.Stdout, cmd.Stderr, err)
+		}
+	case <-b.timeout:
+		b.cmd.Process.Kill()
+		return "", fmt.Errorf("Timed out waiting for command %v:\nCommand stdout:\n%v\nstderr:\n%v\n", cmd, cmd.Stdout, cmd.Stderr)
 	}
 	Logf("stdout: %q", stdout.String())
 	Logf("stderr: %q", stderr.String())