From 36917ccf47473bf708b5e49e24e57c582845b676 Mon Sep 17 00:00:00 2001 From: Joe Finney Date: Thu, 3 Mar 2016 13:23:40 -0800 Subject: [PATCH] Print out a message if e2e times out. --- hack/jenkins/job-configs/global.yaml | 9 +++++++++ hack/jenkins/job-configs/kubernetes-build.yaml | 4 +++- .../job-configs/kubernetes-e2e-gce-enormous-startup.yaml | 3 ++- hack/jenkins/job-configs/kubernetes-e2e.yaml | 5 +++-- hack/jenkins/job-configs/kubernetes-kubemark.yaml | 3 ++- hack/jenkins/job-configs/kubernetes-soak.yaml | 6 ++++-- hack/jenkins/job-configs/kubernetes-test-go.yaml | 4 +++- hack/jenkins/job-configs/kubernetes-upgrades.yaml | 6 ++++-- 8 files changed, 30 insertions(+), 10 deletions(-) diff --git a/hack/jenkins/job-configs/global.yaml b/hack/jenkins/job-configs/global.yaml index a778e3e17b0..fcdde7835e3 100644 --- a/hack/jenkins/job-configs/global.yaml +++ b/hack/jenkins/job-configs/global.yaml @@ -131,6 +131,15 @@ kill-timeout: 15 # Just to be safe, use the Jenkins timeout after a long time. jenkins-timeout: 600 + # report-rc assumes that $rc is set to the exit status of the runner. + report-rc: | + if [[ ${{rc}} -eq 124 || ${{rc}} -eq 137 ]]; then + echo "Build timed out" >&2 + elif [[ ${{rc}} -ne 0 ]]; then + echo "Build failed" >&2 + fi + echo "Exiting with code: ${{rc}}" + exit ${{rc}} branch: 'master' job-env: '' runner: curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/e2e-runner.sh" | bash - diff --git a/hack/jenkins/job-configs/kubernetes-build.yaml b/hack/jenkins/job-configs/kubernetes-build.yaml index 67e88789afe..15fade56406 100644 --- a/hack/jenkins/job-configs/kubernetes-build.yaml +++ b/hack/jenkins/job-configs/kubernetes-build.yaml @@ -6,7 +6,9 @@ numToKeep: 200 builders: - shell: 'bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-started.sh")' - - shell: 'timeout -k {kill-timeout}m 30m ./hack/jenkins/build.sh' + - shell: | + timeout -k {kill-timeout}m 30m ./hack/jenkins/build.sh && rc=$? || rc=$? + {report-rc} properties: - mail-watcher publishers: diff --git a/hack/jenkins/job-configs/kubernetes-e2e-gce-enormous-startup.yaml b/hack/jenkins/job-configs/kubernetes-e2e-gce-enormous-startup.yaml index a0a8b82232c..39931bec721 100644 --- a/hack/jenkins/job-configs/kubernetes-e2e-gce-enormous-startup.yaml +++ b/hack/jenkins/job-configs/kubernetes-e2e-gce-enormous-startup.yaml @@ -30,7 +30,8 @@ export TEST_CLUSTER_RESYNC_PERIOD="--min-resync-period=12h" {post-env} export KUBE_GCE_NETWORK="e2e-enormous-cluster" - timeout -k {kill-timeout}m 480m {runner} + timeout -k {kill-timeout}m 480m {runner} && rc=$? || rc=$? + {report-rc} properties: - mail-watcher publishers: diff --git a/hack/jenkins/job-configs/kubernetes-e2e.yaml b/hack/jenkins/job-configs/kubernetes-e2e.yaml index 52ad61bb6c3..17804f81c45 100644 --- a/hack/jenkins/job-configs/kubernetes-e2e.yaml +++ b/hack/jenkins/job-configs/kubernetes-e2e.yaml @@ -21,13 +21,14 @@ {provider-env} {job-env} {post-env} - if ! timeout -k {kill-timeout}m {timeout}m {runner}; then + timeout -k {kill-timeout}m {timeout}m {runner} && rc=$? || rc=$? + if [[ ${{rc}} -ne 0 ]]; then if [[ -x kubernetes/cluster/log-dump.sh && -d _artifacts ]]; then echo "Dumping logs for any remaining nodes" ./kubernetes/cluster/log-dump.sh _artifacts fi - exit 1 fi + {report-rc} properties: - mail-watcher wrappers: diff --git a/hack/jenkins/job-configs/kubernetes-kubemark.yaml b/hack/jenkins/job-configs/kubernetes-kubemark.yaml index 2560d4dcf24..ad7fd596fc1 100644 --- a/hack/jenkins/job-configs/kubernetes-kubemark.yaml +++ b/hack/jenkins/job-configs/kubernetes-kubemark.yaml @@ -8,7 +8,8 @@ {provider-env} {job-env} {post-env} - timeout -k {kill-timeout}m {timeout}m {runner} + timeout -k {kill-timeout}m {timeout}m {runner} && rc=$? || rc=$? + {report-rc} properties: - mail-watcher publishers: diff --git a/hack/jenkins/job-configs/kubernetes-soak.yaml b/hack/jenkins/job-configs/kubernetes-soak.yaml index 46669a98011..4de7187db76 100644 --- a/hack/jenkins/job-configs/kubernetes-soak.yaml +++ b/hack/jenkins/job-configs/kubernetes-soak.yaml @@ -9,7 +9,8 @@ {soak-deploy} {job-env} {post-env} - timeout -k {kill-timeout}m 90m {runner} + timeout -k {kill-timeout}m 90m {runner} && rc=$? || rc=$? + {report-rc} properties: - build-blocker: use-build-blocker: true @@ -41,7 +42,8 @@ {soak-continuous} {job-env} {post-env} - timeout -k {kill-timeout}m 360m {runner} + timeout -k {kill-timeout}m 360m {runner} && rc=$? || rc=$? + {report-rc} properties: - build-blocker: use-build-blocker: true diff --git a/hack/jenkins/job-configs/kubernetes-test-go.yaml b/hack/jenkins/job-configs/kubernetes-test-go.yaml index 0b425adf4d1..fc76a208992 100644 --- a/hack/jenkins/job-configs/kubernetes-test-go.yaml +++ b/hack/jenkins/job-configs/kubernetes-test-go.yaml @@ -8,7 +8,9 @@ node: unittest builders: - shell: 'bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-started.sh")' - - shell: 'timeout -k {kill-timeout}m {timeout}m ./hack/jenkins/gotest-dockerized.sh' + - shell: | + timeout -k {kill-timeout}m {timeout}m ./hack/jenkins/gotest-dockerized.sh && rc=$? || rc=$? + {report-rc} publishers: - claim-build - gcs-uploader diff --git a/hack/jenkins/job-configs/kubernetes-upgrades.yaml b/hack/jenkins/job-configs/kubernetes-upgrades.yaml index 61a99a724e1..044323833a7 100644 --- a/hack/jenkins/job-configs/kubernetes-upgrades.yaml +++ b/hack/jenkins/job-configs/kubernetes-upgrades.yaml @@ -70,7 +70,8 @@ # per-step variables, such as whether to run tests {job-env} {post-env} - timeout -k {kill-timeout}m 60m {runner} + timeout -k {kill-timeout}m 60m {runner} && rc=$? || rc=$? + {report-rc} properties: - mail-watcher publishers: @@ -105,7 +106,8 @@ # per-step variables, such as whether to run tests {job-env} {post-env} - timeout -k {kill-timeout}m 300m {runner} + timeout -k {kill-timeout}m 300m {runner} && rc=$? || rc=$? + {report-rc} properties: - mail-watcher publishers: