Add optional timeout logic to e2e.go

hack/e2e.go

@@ -34,6 +34,8 @@ import (
 )
 
 var (
+	interrupt = time.NewTimer(time.Duration(0)) // interrupt testing at this time.
+	terminate = time.NewTimer(time.Duration(0)) // terminate testing at this time.
 	// TODO(fejta): change all these _ flags to -
 	build            = flag.Bool("build", false, "If true, build a new release. Otherwise, use whatever is there.")
 	checkVersionSkew = flag.Bool("check_version_skew", true, ""+
@@ -48,6 +50,7 @@ var (
 	skewTests            = flag.Bool("skew", false, "If true, run tests in another version at ../kubernetes/hack/e2e.go")
 	testArgs             = flag.String("test_args", "", "Space-separated list of arguments to pass to Ginkgo test runner.")
 	test                 = flag.Bool("test", false, "Run Ginkgo tests.")
+	timeout              = flag.Duration("timeout", time.Duration(0), "Terminate testing after the timeout duration (s/m/h)")
 	up                   = flag.Bool("up", false, "If true, start the the e2e cluster. If cluster is already up, recreate it.")
 	upgradeArgs          = flag.String("upgrade_args", "", "If set, run upgrade tests before other tests")
 	verbose              = flag.Bool("v", false, "If true, print all command output.")
@@ -152,6 +155,18 @@ func main() {
 	log.SetFlags(log.LstdFlags | log.Lshortfile)
 	flag.Parse()
 
+	if !terminate.Stop() {
+		<-terminate.C // Drain the value if necessary.
+	}
+	if !interrupt.Stop() {
+		<-interrupt.C // Drain value
+	}
+
+	if *timeout > 0 {
+		log.Printf("Limiting testing to %s", *timeout)
+		interrupt.Reset(*timeout)
+	}
+
 	if err := validWorkingDirectory(); err != nil {
 		log.Fatalf("Called from invalid working directory: %v", err)
 	}
@@ -770,8 +785,28 @@ func finishRunning(stepName string, cmd *exec.Cmd) error {
 		log.Printf("Step '%s' finished in %s", stepName, time.Since(start))
 	}(time.Now())
 
-	if err := cmd.Run(); err != nil {
+	if err := cmd.Start(); err != nil {
-		return fmt.Errorf("error running %v: %v", stepName, err)
+		return fmt.Errorf("error starting %v: %v", stepName, err)
+	}
+
+	finished := make(chan error)
+
+	go func() {
+		finished <- cmd.Wait()
+	}()
+
+	for {
+		select {
+		case <-terminate.C:
+			terminate.Reset(time.Duration(0)) // Kill subsequent processes immediately.
+			cmd.Process.Kill()
+			return fmt.Errorf("Terminate testing after 15m after %s timeout during %s", *timeout, stepName)
+		case <-interrupt.C:
+			log.Printf("Interrupt testing after %s timeout. Will terminate in another 15m", *timeout)
+			terminate.Reset(15 * time.Minute)
+			cmd.Process.Signal(os.Interrupt)
+		case err := <-finished:
+			return err
+		}
 	}
-	return nil
 }