From 83dd910175cc7201fc6abc89274dc0e93e4f1fce Mon Sep 17 00:00:00 2001 From: Jeff Grafton Date: Mon, 16 May 2016 18:33:51 -0700 Subject: [PATCH] Add a timeout to the node e2e Ginkgo test runner Also add a few debugging statements to indicate progress. --- hack/verify-flags/known-flags.txt | 1 + test/e2e_node/e2e_remote.go | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 6befaeb17d1..b60bdcf4774 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -416,6 +416,7 @@ system-reserved target-port tcp-services terminated-pod-gc-threshold +test-timeout tls-cert-file tls-private-key-file to-version diff --git a/test/e2e_node/e2e_remote.go b/test/e2e_node/e2e_remote.go index f1b6f9decab..d6c3faa53a4 100644 --- a/test/e2e_node/e2e_remote.go +++ b/test/e2e_node/e2e_remote.go @@ -32,6 +32,7 @@ import ( var sshOptions = flag.String("ssh-options", "", "Commandline options passed to ssh.") var sshEnv = flag.String("ssh-env", "", "Use predefined ssh options for environment. Options: gce") +var testTimeoutSeconds = flag.Int("test-timeout", 45*60, "How long (in seconds) to wait for ginkgo tests to complete.") var sshOptionsMap map[string]string @@ -120,6 +121,7 @@ func CreateTestArchive() string { // RunRemote copies the archive file to a /tmp file on host, unpacks it, and runs the e2e_node.test func RunRemote(archive string, host string, cleanup bool) (string, error) { // Create the temp staging directory + glog.Infof("Staging test binaries on %s", host) tmp := fmt.Sprintf("/tmp/gcloud-e2e-%d", rand.Int31()) _, err := RunSshCommand("ssh", host, "--", "mkdir", tmp) if err != nil { @@ -149,14 +151,16 @@ func RunRemote(archive string, host string, cleanup bool) (string, error) { // No need to log an error if pkill fails since pkill will fail if the commands are not running. // If we are unable to stop existing running k8s processes, we should see messages in the kubelet/apiserver/etcd // logs about failing to bind the required ports. + glog.Infof("Killing any existing node processes on %s", host) RunSshCommand("ssh", host, "--", "sh", "-c", cmd) // Extract the archive and run the tests cmd = getSshCommand(" && ", fmt.Sprintf("cd %s", tmp), fmt.Sprintf("tar -xzvf ./%s", archiveName), - fmt.Sprintf("./e2e_node.test --logtostderr --v 2 --build-services=false --stop-services=%t --node-name=%s", cleanup, host), + fmt.Sprintf("timeout -k 30s %ds ./e2e_node.test --logtostderr --v 2 --build-services=false --stop-services=%t --node-name=%s", *testTimeoutSeconds, cleanup, host), ) + glog.Infof("Starting tests on %s", host) output, err := RunSshCommand("ssh", host, "--", "sh", "-c", cmd) if err != nil { return "", err