diff --git a/hack/jenkins/job-configs/global.yaml b/hack/jenkins/job-configs/global.yaml index 9fc4f226fb9..fcdde7835e3 100644 --- a/hack/jenkins/job-configs/global.yaml +++ b/hack/jenkins/job-configs/global.yaml @@ -127,6 +127,19 @@ name: global emails: '$DEFAULT_RECIPIENTS' cron-string: 'H/30 * * * *' + # How long to wait after sending TERM to send KILL (minutes) + kill-timeout: 15 + # Just to be safe, use the Jenkins timeout after a long time. + jenkins-timeout: 600 + # report-rc assumes that $rc is set to the exit status of the runner. + report-rc: | + if [[ ${{rc}} -eq 124 || ${{rc}} -eq 137 ]]; then + echo "Build timed out" >&2 + elif [[ ${{rc}} -ne 0 ]]; then + echo "Build failed" >&2 + fi + echo "Exiting with code: ${{rc}}" + exit ${{rc}} branch: 'master' job-env: '' runner: curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/e2e-runner.sh" | bash - diff --git a/hack/jenkins/job-configs/kubernetes-build.yaml b/hack/jenkins/job-configs/kubernetes-build.yaml index 31b94301af4..15fade56406 100644 --- a/hack/jenkins/job-configs/kubernetes-build.yaml +++ b/hack/jenkins/job-configs/kubernetes-build.yaml @@ -6,7 +6,9 @@ numToKeep: 200 builders: - shell: 'bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-started.sh")' - - shell: './hack/jenkins/build.sh' + - shell: | + timeout -k {kill-timeout}m 30m ./hack/jenkins/build.sh && rc=$? || rc=$? + {report-rc} properties: - mail-watcher publishers: @@ -35,8 +37,8 @@ cron: 'H/2 * * * *' wrappers: - timeout: - timeout: 30 - abort: true + timeout: '{jenkins-timeout}' + fail: true - timestamps - project: diff --git a/hack/jenkins/job-configs/kubernetes-e2e-gce-enormous-startup.yaml b/hack/jenkins/job-configs/kubernetes-e2e-gce-enormous-startup.yaml index 6f4297efafd..39931bec721 100644 --- a/hack/jenkins/job-configs/kubernetes-e2e-gce-enormous-startup.yaml +++ b/hack/jenkins/job-configs/kubernetes-e2e-gce-enormous-startup.yaml @@ -30,7 +30,8 @@ export TEST_CLUSTER_RESYNC_PERIOD="--min-resync-period=12h" {post-env} export KUBE_GCE_NETWORK="e2e-enormous-cluster" - {runner} + timeout -k {kill-timeout}m 480m {runner} && rc=$? || rc=$? + {report-rc} properties: - mail-watcher publishers: @@ -45,7 +46,7 @@ - ansicolor: colormap: xterm - timeout: - timeout: 480 + timeout: '{jenkins-timeout}' fail: true - timestamps - workspace-cleanup diff --git a/hack/jenkins/job-configs/kubernetes-e2e.yaml b/hack/jenkins/job-configs/kubernetes-e2e.yaml index ae4d9525be3..ba4e9b2091a 100644 --- a/hack/jenkins/job-configs/kubernetes-e2e.yaml +++ b/hack/jenkins/job-configs/kubernetes-e2e.yaml @@ -21,14 +21,21 @@ {provider-env} {job-env} {post-env} - {runner} + timeout -k {kill-timeout}m {timeout}m {runner} && rc=$? || rc=$? + if [[ ${{rc}} -ne 0 ]]; then + if [[ -x kubernetes/cluster/log-dump.sh && -d _artifacts ]]; then + echo "Dumping logs for any remaining nodes" + ./kubernetes/cluster/log-dump.sh _artifacts + fi + fi + {report-rc} properties: - mail-watcher wrappers: - ansicolor: colormap: xterm - timeout: - timeout: '{timeout}' + timeout: '{jenkins-timeout}' fail: true - timestamps - workspace-cleanup diff --git a/hack/jenkins/job-configs/kubernetes-kubemark.yaml b/hack/jenkins/job-configs/kubernetes-kubemark.yaml index 27e946e92e3..ad7fd596fc1 100644 --- a/hack/jenkins/job-configs/kubernetes-kubemark.yaml +++ b/hack/jenkins/job-configs/kubernetes-kubemark.yaml @@ -8,7 +8,8 @@ {provider-env} {job-env} {post-env} - {runner} + timeout -k {kill-timeout}m {timeout}m {runner} && rc=$? || rc=$? + {report-rc} properties: - mail-watcher publishers: @@ -27,7 +28,7 @@ - ansicolor: colormap: xterm - timeout: - timeout: '{timeout}' + timeout: '{jenkins-timeout}' fail: true - timestamps - workspace-cleanup diff --git a/hack/jenkins/job-configs/kubernetes-soak.yaml b/hack/jenkins/job-configs/kubernetes-soak.yaml index 57c6b492f14..4de7187db76 100644 --- a/hack/jenkins/job-configs/kubernetes-soak.yaml +++ b/hack/jenkins/job-configs/kubernetes-soak.yaml @@ -9,7 +9,8 @@ {soak-deploy} {job-env} {post-env} - {runner} + timeout -k {kill-timeout}m 90m {runner} && rc=$? || rc=$? + {report-rc} properties: - build-blocker: use-build-blocker: true @@ -24,7 +25,7 @@ - ansicolor: colormap: xterm - timeout: - timeout: 90 + timeout: '{jenkins-timeout}' fail: true - timestamps - workspace-cleanup @@ -41,7 +42,8 @@ {soak-continuous} {job-env} {post-env} - {runner} + timeout -k {kill-timeout}m 360m {runner} && rc=$? || rc=$? + {report-rc} properties: - build-blocker: use-build-blocker: true @@ -60,7 +62,7 @@ - ansicolor: colormap: xterm - timeout: - timeout: 360 + timeout: '{jenkins-timeout}' fail: true - timestamps diff --git a/hack/jenkins/job-configs/kubernetes-test-go.yaml b/hack/jenkins/job-configs/kubernetes-test-go.yaml index b9c2adc6e5c..fc76a208992 100644 --- a/hack/jenkins/job-configs/kubernetes-test-go.yaml +++ b/hack/jenkins/job-configs/kubernetes-test-go.yaml @@ -8,7 +8,9 @@ node: unittest builders: - shell: 'bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-started.sh")' - - shell: './hack/jenkins/gotest-dockerized.sh' + - shell: | + timeout -k {kill-timeout}m {timeout}m ./hack/jenkins/gotest-dockerized.sh && rc=$? || rc=$? + {report-rc} publishers: - claim-build - gcs-uploader @@ -47,7 +49,7 @@ - ansicolor: colormap: xterm - timeout: - timeout: '{timeout}' + timeout: '{jenkins-timeout}' fail: true - timestamps - raw: diff --git a/hack/jenkins/job-configs/kubernetes-upgrades.yaml b/hack/jenkins/job-configs/kubernetes-upgrades.yaml index dec18d4f49c..044323833a7 100644 --- a/hack/jenkins/job-configs/kubernetes-upgrades.yaml +++ b/hack/jenkins/job-configs/kubernetes-upgrades.yaml @@ -70,7 +70,8 @@ # per-step variables, such as whether to run tests {job-env} {post-env} - {runner} + timeout -k {kill-timeout}m 60m {runner} && rc=$? || rc=$? + {report-rc} properties: - mail-watcher publishers: @@ -84,7 +85,7 @@ - ansicolor: colormap: xterm - timeout: - timeout: 60 + timeout: '{jenkins-timeout}' fail: true - timestamps - workspace-cleanup @@ -105,7 +106,8 @@ # per-step variables, such as whether to run tests {job-env} {post-env} - {runner} + timeout -k {kill-timeout}m 300m {runner} && rc=$? || rc=$? + {report-rc} properties: - mail-watcher publishers: @@ -119,8 +121,7 @@ - ansicolor: colormap: xterm - timeout: - timeout: 300 - abort: true + timeout: '{jenkins-timeout}' fail: true - timestamps # Don't clean the workspace; we want to keep configs intact across steps in the multijob