Merge pull request #23415 from ixdy/upload-to-gcs-script

Refactor upload-started.sh and upload-finished.sh into upload-to-gcs.sh

hack/jenkins/e2e-runner.sh

@@ -207,8 +207,9 @@ fi
 cd kubernetes
 
 # Upload build start time and k8s version to GCS, but not on PR Jenkins.
+# On PR Jenkins this is done before the build.
 if [[ ! "${JOB_NAME}" =~ -pull- ]]; then
-    bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-started.sh")
+    JENKINS_BUILD_STARTED=true bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh")
 fi
 
 # Have cmd/e2e run by goe2e.sh generate JUnit report in ${WORKSPACE}/junit*.xml

hack/jenkins/job-configs/global.yaml

@@ -14,40 +14,33 @@
         - postbuildscript:
             builders:
                 - shell: |
-                    curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-finished.sh" > upload-finished.sh
+                    mkdir -p _tmp
-                    chmod +x upload-finished.sh
+                    curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh" > ./_tmp/upload-to-gcs.sh
+                    chmod +x ./_tmp/upload-to-gcs.sh
                 - conditional-step:
                     condition-kind: current-status
                     condition-worst: SUCCESS
                     condition-best: SUCCESS
                     steps:
-                        - shell: './upload-finished.sh SUCCESS'
+                        - shell: 'JENKINS_BUILD_FINISHED=SUCCESS ./_tmp/upload-to-gcs.sh'
                 - conditional-step:
                     condition-kind: current-status
                     condition-worst: UNSTABLE
                     condition-best: UNSTABLE
                     steps:
-                        - shell: './upload-finished.sh UNSTABLE'
+                        - shell: 'JENKINS_BUILD_FINISHED=UNSTABLE ./_tmp/upload-to-gcs.sh'
                 - conditional-step:
                     condition-kind: current-status
                     condition-worst: FAILURE
                     condition-best: FAILURE
                     steps:
-                        - shell: './upload-finished.sh FAILURE'
+                        - shell: 'JENKINS_BUILD_FINISHED=FAILURE ./_tmp/upload-to-gcs.sh'
                 - conditional-step:
                     condition-kind: current-status
                     condition-worst: ABORTED
                     condition-best: ABORTED
                     steps:
-                        - shell: './upload-finished.sh ABORTED'
+                        - shell: 'JENKINS_BUILD_FINISHED=ABORTED ./_tmp/upload-to-gcs.sh'
-                # Use our script for build artifacts, since it's more flexible.
-                # Run last since it updates latest-build.txt.
-                - shell: |
-                    if [[ -x ./hack/jenkins/upload-to-gcs.sh ]]; then
-                      ./hack/jenkins/upload-to-gcs.sh
-                    else
-                      curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh" | bash -
-                    fi
             script-only-if-succeeded: False
             script-only-if-failed: False
         # Use the plugin for the build log, since it isn't available on Jenkins slaves.

hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-build.yaml

@@ -5,7 +5,7 @@
     logrotate:
         numToKeep: 200
     builders:
-        - shell: 'bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-started.sh")'
+        - shell: 'JENKINS_BUILD_STARTED=true bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh")'
         - shell: |
             timeout -k {kill-timeout}m 30m ./hack/jenkins/build.sh && rc=$? || rc=$?
             {report-rc}

hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-test-go.yaml

@@ -7,7 +7,7 @@
         numToKeep: 200
     node: unittest
     builders:
-        - shell: 'bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-started.sh")'
+        - shell: 'JENKINS_BUILD_STARTED=true bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh")'
         - shell: |
             export KUBE_FORCE_VERIFY_CHECKS='y'
             export KUBE_VERIFY_GIT_BRANCH='{branch}'

hack/jenkins/upload-finished.sh

@@ -29,24 +29,15 @@ if [[ $# -ne 1 ]]; then
     exit 1
 fi
 
-# TODO: DRY. Refactor into upload-to-gcs.sh ?
+export JENKINS_BUILD_FINISHED="$1"
-: ${JENKINS_GCS_LOGS_PATH:="gs://kubernetes-jenkins/logs"}
-: ${JENKINS_UPLOAD_TO_GCS:="y"}
 
-if [[ ! ${JENKINS_UPLOAD_TO_GCS:-} =~ ^[yY]$ ]]; then
+echo
-  exit 0
+echo "Passing through to upload-to-gcs.sh with JENKINS_BUILD_FINISHED=${JENKINS_BUILD_FINISHED}"
+echo "Please update configs to call upload-to-gcs.sh directly."
+echo
+
+if [[ -x ./hack/jenkins/upload-to-gcs.sh ]]; then
+  ./hack/jenkins/upload-to-gcs.sh
+else
+  curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh" | bash -
 fi
-
-readonly result="$1"
-readonly timestamp=$(date +%s)
-readonly location="${JENKINS_GCS_LOGS_PATH}/${JOB_NAME}/${BUILD_NUMBER}/finished.json"
-
-echo -n 'Run finished at '; date -d "@${timestamp}"
-
-echo "Uploading build result to: ${location}"
-gsutil -q cp -a "public-read" <(
-    echo "{"
-    echo "    \"result\": \"${result}\","
-    echo "    \"timestamp\": ${timestamp}"
-    echo "}"
-) "${location}"

hack/jenkins/upload-started.sh

@@ -24,42 +24,15 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-# TODO: DRY. Refactor into upload-to-gcs.sh ?
+export JENKINS_BUILD_STARTED="true"
-: ${JENKINS_GCS_LOGS_PATH:="gs://kubernetes-jenkins/logs"}
-: ${JENKINS_UPLOAD_TO_GCS:="y"}
 
-if [[ ! ${JENKINS_UPLOAD_TO_GCS:-} =~ ^[yY]$ ]]; then
+echo
-  exit 0
+echo "Passing through to upload-to-gcs.sh with JENKINS_BUILD_STARTED=${JENKINS_BUILD_STARTED}"
-fi
+echo "Please update configs to call upload-to-gcs.sh directly."
+echo
 
-version=""
+if [[ -x ./hack/jenkins/upload-to-gcs.sh ]]; then
-readonly timestamp=$(date +%s)
+  ./hack/jenkins/upload-to-gcs.sh
-readonly location="${JENKINS_GCS_LOGS_PATH}/${JOB_NAME}/${BUILD_NUMBER}/started.json"
-
-echo -n 'Run starting at '; date -d "@${timestamp}"
-
-# Try to discover the kubernetes version.
-if [[ -e "version" ]]; then
-    version=$(cat "version")
-elif [[ -e "hack/lib/version.sh" ]]; then
-    version=$(
-        export KUBE_ROOT="."
-        source "hack/lib/version.sh"
-        kube::version::get_version_vars
-        echo "${KUBE_GIT_VERSION-}"
-    )
-fi
-
-if [[ -n "${version}" ]]; then
-    echo "Found Kubernetes version: ${version}"
 else
-    echo "Could not find Kubernetes version"
+  curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh" | bash -
 fi
-
-echo "Uploading version to: ${location}"
-gsutil -q cp -a "public-read" <(
-    echo "{"
-    echo "    \"version\": \"${version}\","
-    echo "    \"timestamp\": ${timestamp}"
-    echo "}"
-) "${location}"

hack/jenkins/upload-to-gcs.sh

@@ -14,22 +14,36 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Run this script in the Jenkins "Execute shell" build action to upload test
+# This script uploads metadata and test results to Google Cloud Storage, in the
-# artifacts to GCS. Since it uses gsutil directly, it's a bit faster at
+# location indicated by JENKINS_GCS_LOGS_PATH. By default, we use the Google
-# uploading large numbers of files than the GCS Jenkins plugin.
+# kubernetes-jenkins bucket.
-# We also intentionally ignore gsutil errors since we don't want failed uploads
+#
-# to fail the entire test run (#13548).
+# The script looks for one of two environment variables to be set:
-
+#   JENKINS_BUILD_STARTED: set to a nonempty string to upload version
+#     information to 'started.json'. The value of the variable is not
+#     currently used.
+#   JENKINS_BUILD_FINISHED: set to the Jenkins build result to upload the build
+#     result to 'finished.json', any test artifacts, and update the
+#     'latest-build.txt' file pointer. Since this script uses gsutil directly,
+#     it's a bit faster at uploading large numbers of files than the GCS Jenkins
+#     plugin. It also makes use of gsutil's gzip functionality.
+#
 # Note: for magicfile support to work correctly, the "file" utility must be
 # installed.
 
-# TODO: eventually fold this all into upload-finished.sh, once every job is
-# using it.
-
 set -o errexit
 set -o nounset
 set -o pipefail
 
+if [[ -n "${JENKINS_BUILD_STARTED:-}" && -n "${JENKINS_BUILD_FINISHED:-}" ]]; then
+  echo "Error: JENKINS_BUILD_STARTED and JENKINS_BUILD_FINISHED should not both be set!"
+  exit 1
+fi
+
+if [[ ! ${JENKINS_UPLOAD_TO_GCS:-y} =~ ^[yY]$ ]]; then
+  exit 0
+fi
+
 if [[ ${JOB_NAME} =~ -pull- ]]; then
   : ${JENKINS_GCS_LOGS_PATH:="gs://kubernetes-jenkins/pr-logs/pull/${ghprbPullId:-unknown}"}
 else
@@ -40,19 +54,79 @@ readonly artifacts_path="${WORKSPACE}/_artifacts"
 readonly gcs_job_path="${JENKINS_GCS_LOGS_PATH}/${JOB_NAME}"
 readonly gcs_build_path="${gcs_job_path}/${BUILD_NUMBER}"
 readonly gcs_acl="public-read"
-
-for upload_attempt in $(seq 3); do
-  echo "Uploading to ${gcs_build_path} (attempt ${upload_attempt})"
-  if [[ -d "${artifacts_path}" && -n $(ls -A "${artifacts_path}") ]]; then
-    gsutil -m -q -o "GSUtil:use_magicfile=True" cp -a "${gcs_acl}" -r -c \
-      -z log,txt,xml "${artifacts_path}" "${gcs_build_path}/artifacts" || continue
-  fi
-  # Mark this build as the latest completed.
-  echo "${BUILD_NUMBER}" | \
-    gsutil -q -h "Content-Type:text/plain" -h "Cache-Control:private, max-age=0, no-transform" \
-      cp -a "${gcs_acl}" - "${gcs_job_path}/latest-build.txt" || continue
-  break  # all uploads succeeded if we hit this point
-done
-
 readonly results_url=${gcs_build_path//"gs:/"/"https://storage.cloud.google.com"}
-echo -e "\n\n\n*** View logs and artifacts at ${results_url} ***\n\n"
+readonly timestamp=$(date +%s)
+
+function upload_version() {
+  echo -n 'Run starting at '; date -d "@${timestamp}"
+
+  # Try to discover the kubernetes version.
+  local version=""
+  if [[ -e "version" ]]; then
+    version=$(cat "version")
+  elif [[ -e "hack/lib/version.sh" ]]; then
+    version=$(
+      export KUBE_ROOT="."
+      source "hack/lib/version.sh"
+      kube::version::get_version_vars
+      echo "${KUBE_GIT_VERSION-}"
+    )
+  fi
+
+  if [[ -n "${version}" ]]; then
+    echo "Found Kubernetes version: ${version}"
+  else
+    echo "Could not find Kubernetes version"
+  fi
+
+  local -r json_file="${gcs_build_path}/started.json"
+  for upload_attempt in $(seq 3); do
+    echo "Uploading version to: ${json_file} (attempt ${upload_attempt})"
+    gsutil -q -h "Content-Type:application/json" cp -a "${gcs_acl}" <(
+      echo "{"
+      echo "    \"version\": \"${version}\","
+      echo "    \"timestamp\": ${timestamp}"
+      echo "}"
+    ) "${json_file}" || continue
+    break
+  done
+}
+
+function upload_artifacts_and_build_result() {
+  local -r build_result=$1
+  echo -n 'Run finished at '; date -d "@${timestamp}"
+
+  for upload_attempt in $(seq 3); do
+    echo "Uploading to ${gcs_build_path} (attempt ${upload_attempt})"
+    echo "Uploading build result: ${build_result}"
+    gsutil -q -h "Content-Type:application/json" cp -a "${gcs_acl}" <(
+      echo "{"
+      echo "    \"result\": \"${build_result}\","
+      echo "    \"timestamp\": ${timestamp}"
+      echo "}"
+    ) "${gcs_build_path}/finished.json" || continue
+    if [[ -d "${artifacts_path}" && -n $(ls -A "${artifacts_path}") ]]; then
+      echo "Uploading artifacts"
+      gsutil -m -q -o "GSUtil:use_magicfile=True" cp -a "${gcs_acl}" -r -c \
+        -z log,txt,xml "${artifacts_path}" "${gcs_build_path}/artifacts" || continue
+    fi
+    # Mark this build as the latest completed.
+    echo "Marking build ${BUILD_NUMBER} as the latest completed build"
+    echo "${BUILD_NUMBER}" | \
+      gsutil -q -h "Content-Type:text/plain" -h "Cache-Control:private, max-age=0, no-transform" \
+        cp -a "${gcs_acl}" - "${gcs_job_path}/latest-build.txt" || continue
+    break  # all uploads succeeded if we hit this point
+  done
+
+  echo -e "\n\n\n*** View logs and artifacts at ${results_url} ***\n\n"
+}
+
+if [[ -n "${JENKINS_BUILD_STARTED:-}" ]]; then
+  upload_version
+elif [[ -n "${JENKINS_BUILD_FINISHED:-}" ]]; then
+  upload_artifacts_and_build_result ${JENKINS_BUILD_FINISHED}
+else
+  echo "Called without JENKINS_BUILD_STARTED or JENKINS_BUILD_FINISHED set."
+  echo "Assuming a legacy invocation."
+  upload_artifacts_and_build_result "[UNSET]"
+fi