diff --git a/hack/e2e.go b/hack/e2e.go index a50f1abdf05..91e9ac252bb 100644 --- a/hack/e2e.go +++ b/hack/e2e.go @@ -36,6 +36,8 @@ import ( ) var ( + interrupt = time.NewTimer(time.Duration(0)) // interrupt testing at this time. + terminate = time.NewTimer(time.Duration(0)) // terminate testing at this time. // TODO(fejta): change all these _ flags to - build = flag.Bool("build", false, "If true, build a new release. Otherwise, use whatever is there.") checkVersionSkew = flag.Bool("check_version_skew", true, ""+ @@ -50,6 +52,7 @@ var ( skewTests = flag.Bool("skew", false, "If true, run tests in another version at ../kubernetes/hack/e2e.go") testArgs = flag.String("test_args", "", "Space-separated list of arguments to pass to Ginkgo test runner.") test = flag.Bool("test", false, "Run Ginkgo tests.") + timeout = flag.Duration("timeout", time.Duration(0), "Terminate testing after the timeout duration (s/m/h)") up = flag.Bool("up", false, "If true, start the the e2e cluster. If cluster is already up, recreate it.") upgradeArgs = flag.String("upgrade_args", "", "If set, run upgrade tests before other tests") verbose = flag.Bool("v", false, "If true, print all command output.") @@ -184,6 +187,18 @@ func main() { log.SetFlags(log.LstdFlags | log.Lshortfile) flag.Parse() + if !terminate.Stop() { + <-terminate.C // Drain the value if necessary. + } + if !interrupt.Stop() { + <-interrupt.C // Drain value + } + + if *timeout > 0 { + log.Printf("Limiting testing to %s", *timeout) + interrupt.Reset(*timeout) + } + if err := validWorkingDirectory(); err != nil { log.Fatalf("Called from invalid working directory: %v", err) } @@ -921,8 +936,28 @@ func finishRunning(stepName string, cmd *exec.Cmd) error { log.Printf("Step '%s' finished in %s", stepName, time.Since(start)) }(time.Now()) - if err := cmd.Run(); err != nil { - return fmt.Errorf("error running %v: %v", stepName, err) + if err := cmd.Start(); err != nil { + return fmt.Errorf("error starting %v: %v", stepName, err) + } + + finished := make(chan error) + + go func() { + finished <- cmd.Wait() + }() + + for { + select { + case <-terminate.C: + terminate.Reset(time.Duration(0)) // Kill subsequent processes immediately. + cmd.Process.Kill() + return fmt.Errorf("Terminate testing after 15m after %s timeout during %s", *timeout, stepName) + case <-interrupt.C: + log.Printf("Interrupt testing after %s timeout. Will terminate in another 15m", *timeout) + terminate.Reset(15 * time.Minute) + cmd.Process.Signal(os.Interrupt) + case err := <-finished: + return err + } } - return nil }