Merge pull request #22374 from spxtr/dump

Dump master/node logs at the end of E2E Jenkins jobs.

hack/jenkins/job-configs/global.yaml

@@ -127,6 +127,19 @@
     name: global
     emails: '$DEFAULT_RECIPIENTS'
     cron-string: 'H/30 * * * *'
+    # How long to wait after sending TERM to send KILL (minutes)
+    kill-timeout: 15
+    # Just to be safe, use the Jenkins timeout after a long time.
+    jenkins-timeout: 600
+    # report-rc assumes that $rc is set to the exit status of the runner.
+    report-rc: |
+        if [[ ${{rc}} -eq 124 || ${{rc}} -eq 137 ]]; then
+            echo "Build timed out" >&2
+        elif [[ ${{rc}} -ne 0 ]]; then
+            echo "Build failed" >&2
+        fi
+        echo "Exiting with code: ${{rc}}"
+        exit ${{rc}}
     branch: 'master'
     job-env: ''
     runner: curl -fsS --retry 3  "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/e2e-runner.sh" | bash -

hack/jenkins/job-configs/kubernetes-build.yaml

@@ -6,7 +6,9 @@
         numToKeep: 200
     builders:
         - shell: 'bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-started.sh")'
-        - shell: './hack/jenkins/build.sh'
+        - shell: |
+            timeout -k {kill-timeout}m 30m ./hack/jenkins/build.sh && rc=$? || rc=$?
+            {report-rc}
     properties:
         - mail-watcher
     publishers:
@@ -35,8 +37,8 @@
             cron: 'H/2 * * * *'
     wrappers:
         - timeout:
-            timeout: 30
+            timeout: '{jenkins-timeout}'
-            abort: true
+            fail: true
         - timestamps
 
 - project:

hack/jenkins/job-configs/kubernetes-e2e-gce-enormous-startup.yaml

@@ -30,7 +30,8 @@
             export TEST_CLUSTER_RESYNC_PERIOD="--min-resync-period=12h"
             {post-env}
             export KUBE_GCE_NETWORK="e2e-enormous-cluster"
-            {runner}
+            timeout -k {kill-timeout}m 480m {runner} && rc=$? || rc=$?
+            {report-rc}
     properties:
         - mail-watcher
     publishers:
@@ -45,7 +46,7 @@
         - ansicolor:
             colormap: xterm
         - timeout:
-            timeout: 480
+            timeout: '{jenkins-timeout}'
             fail: true
         - timestamps
         - workspace-cleanup

hack/jenkins/job-configs/kubernetes-e2e.yaml

@@ -21,14 +21,21 @@
             {provider-env}
             {job-env}
             {post-env}
-            {runner}
+            timeout -k {kill-timeout}m {timeout}m {runner} && rc=$? || rc=$?
+            if [[ ${{rc}} -ne 0 ]]; then
+                if [[ -x kubernetes/cluster/log-dump.sh && -d _artifacts ]]; then
+                    echo "Dumping logs for any remaining nodes"
+                    ./kubernetes/cluster/log-dump.sh _artifacts
+                fi
+            fi
+            {report-rc}
     properties:
         - mail-watcher
     wrappers:
         - ansicolor:
             colormap: xterm
         - timeout:
-            timeout: '{timeout}'
+            timeout: '{jenkins-timeout}'
             fail: true
         - timestamps
         - workspace-cleanup

hack/jenkins/job-configs/kubernetes-kubemark.yaml

@@ -8,7 +8,8 @@
             {provider-env}
             {job-env}
             {post-env}
-            {runner}
+            timeout -k {kill-timeout}m {timeout}m {runner} && rc=$? || rc=$?
+            {report-rc}
     properties:
         - mail-watcher
     publishers:
@@ -27,7 +28,7 @@
         - ansicolor:
             colormap: xterm
         - timeout:
-            timeout: '{timeout}'
+            timeout: '{jenkins-timeout}'
             fail: true
         - timestamps
         - workspace-cleanup

hack/jenkins/job-configs/kubernetes-soak.yaml

@@ -9,7 +9,8 @@
             {soak-deploy}
             {job-env}
             {post-env}
-            {runner}
+            timeout -k {kill-timeout}m 90m {runner} && rc=$? || rc=$?
+            {report-rc}
     properties:
         - build-blocker:
             use-build-blocker: true
@@ -24,7 +25,7 @@
         - ansicolor:
             colormap: xterm
         - timeout:
-            timeout: 90
+            timeout: '{jenkins-timeout}'
             fail: true
         - timestamps
         - workspace-cleanup
@@ -41,7 +42,8 @@
             {soak-continuous}
             {job-env}
             {post-env}
-            {runner}
+            timeout -k {kill-timeout}m 360m {runner} && rc=$? || rc=$?
+            {report-rc}
     properties:
         - build-blocker:
             use-build-blocker: true
@@ -60,7 +62,7 @@
         - ansicolor:
             colormap: xterm
         - timeout:
-            timeout: 360
+            timeout: '{jenkins-timeout}'
             fail: true
         - timestamps
 

hack/jenkins/job-configs/kubernetes-test-go.yaml

@@ -8,7 +8,9 @@
     node: unittest
     builders:
         - shell: 'bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-started.sh")'
-        - shell: './hack/jenkins/gotest-dockerized.sh'
+        - shell: |
+            timeout -k {kill-timeout}m {timeout}m ./hack/jenkins/gotest-dockerized.sh && rc=$? || rc=$?
+            {report-rc}
     publishers:
         - claim-build
         - gcs-uploader
@@ -47,7 +49,7 @@
         - ansicolor:
             colormap: xterm
         - timeout:
-            timeout: '{timeout}'
+            timeout: '{jenkins-timeout}'
             fail: true
         - timestamps
         - raw:

hack/jenkins/job-configs/kubernetes-upgrades.yaml

@@ -70,7 +70,8 @@
             # per-step variables, such as whether to run tests
             {job-env}
             {post-env}
-            {runner}
+            timeout -k {kill-timeout}m 60m {runner} && rc=$? || rc=$?
+            {report-rc}
     properties:
         - mail-watcher
     publishers:
@@ -84,7 +85,7 @@
         - ansicolor:
             colormap: xterm
         - timeout:
-            timeout: 60
+            timeout: '{jenkins-timeout}'
             fail: true
         - timestamps
         - workspace-cleanup
@@ -105,7 +106,8 @@
             # per-step variables, such as whether to run tests
             {job-env}
             {post-env}
-            {runner}
+            timeout -k {kill-timeout}m 300m {runner} && rc=$? || rc=$?
+            {report-rc}
     properties:
         - mail-watcher
     publishers:
@@ -119,8 +121,7 @@
         - ansicolor:
             colormap: xterm
         - timeout:
-            timeout: 300
+            timeout: '{jenkins-timeout}'
-            abort: true
             fail: true
         - timestamps
         # Don't clean the workspace; we want to keep configs intact across steps in the multijob