From f83a6f1665a01410b4bb3bed36f194c18246536a Mon Sep 17 00:00:00 2001 From: Jeff Grafton Date: Wed, 23 Mar 2016 16:48:00 -0700 Subject: [PATCH 1/2] Refactor upload-started.sh and upload-finished.sh into upload-to-gcs.sh. --- hack/jenkins/upload-finished.sh | 29 +++----- hack/jenkins/upload-started.sh | 43 +++-------- hack/jenkins/upload-to-gcs.sh | 122 +++++++++++++++++++++++++------- 3 files changed, 116 insertions(+), 78 deletions(-) diff --git a/hack/jenkins/upload-finished.sh b/hack/jenkins/upload-finished.sh index 710d9b4e4ff..e76f7908d89 100755 --- a/hack/jenkins/upload-finished.sh +++ b/hack/jenkins/upload-finished.sh @@ -29,24 +29,15 @@ if [[ $# -ne 1 ]]; then exit 1 fi -# TODO: DRY. Refactor into upload-to-gcs.sh ? -: ${JENKINS_GCS_LOGS_PATH:="gs://kubernetes-jenkins/logs"} -: ${JENKINS_UPLOAD_TO_GCS:="y"} +export JENKINS_BUILD_FINISHED="$1" -if [[ ! ${JENKINS_UPLOAD_TO_GCS:-} =~ ^[yY]$ ]]; then - exit 0 +echo +echo "Passing through to upload-to-gcs.sh with JENKINS_BUILD_FINISHED=${JENKINS_BUILD_FINISHED}" +echo "Please update configs to call upload-to-gcs.sh directly." +echo + +if [[ -x ./hack/jenkins/upload-to-gcs.sh ]]; then + ./hack/jenkins/upload-to-gcs.sh +else + curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh" | bash - fi - -readonly result="$1" -readonly timestamp=$(date +%s) -readonly location="${JENKINS_GCS_LOGS_PATH}/${JOB_NAME}/${BUILD_NUMBER}/finished.json" - -echo -n 'Run finished at '; date -d "@${timestamp}" - -echo "Uploading build result to: ${location}" -gsutil -q cp -a "public-read" <( - echo "{" - echo " \"result\": \"${result}\"," - echo " \"timestamp\": ${timestamp}" - echo "}" -) "${location}" diff --git a/hack/jenkins/upload-started.sh b/hack/jenkins/upload-started.sh index a6f1f1701d9..fe2adf1c613 100755 --- a/hack/jenkins/upload-started.sh +++ b/hack/jenkins/upload-started.sh @@ -24,42 +24,15 @@ set -o errexit set -o nounset set -o pipefail -# TODO: DRY. Refactor into upload-to-gcs.sh ? -: ${JENKINS_GCS_LOGS_PATH:="gs://kubernetes-jenkins/logs"} -: ${JENKINS_UPLOAD_TO_GCS:="y"} +export JENKINS_BUILD_STARTED="true" -if [[ ! ${JENKINS_UPLOAD_TO_GCS:-} =~ ^[yY]$ ]]; then - exit 0 -fi +echo +echo "Passing through to upload-to-gcs.sh with JENKINS_BUILD_STARTED=${JENKINS_BUILD_STARTED}" +echo "Please update configs to call upload-to-gcs.sh directly." +echo -version="" -readonly timestamp=$(date +%s) -readonly location="${JENKINS_GCS_LOGS_PATH}/${JOB_NAME}/${BUILD_NUMBER}/started.json" - -echo -n 'Run starting at '; date -d "@${timestamp}" - -# Try to discover the kubernetes version. -if [[ -e "version" ]]; then - version=$(cat "version") -elif [[ -e "hack/lib/version.sh" ]]; then - version=$( - export KUBE_ROOT="." - source "hack/lib/version.sh" - kube::version::get_version_vars - echo "${KUBE_GIT_VERSION-}" - ) -fi - -if [[ -n "${version}" ]]; then - echo "Found Kubernetes version: ${version}" +if [[ -x ./hack/jenkins/upload-to-gcs.sh ]]; then + ./hack/jenkins/upload-to-gcs.sh else - echo "Could not find Kubernetes version" + curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh" | bash - fi - -echo "Uploading version to: ${location}" -gsutil -q cp -a "public-read" <( - echo "{" - echo " \"version\": \"${version}\"," - echo " \"timestamp\": ${timestamp}" - echo "}" -) "${location}" diff --git a/hack/jenkins/upload-to-gcs.sh b/hack/jenkins/upload-to-gcs.sh index ca5c14de785..15791b65a6e 100755 --- a/hack/jenkins/upload-to-gcs.sh +++ b/hack/jenkins/upload-to-gcs.sh @@ -14,22 +14,36 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Run this script in the Jenkins "Execute shell" build action to upload test -# artifacts to GCS. Since it uses gsutil directly, it's a bit faster at -# uploading large numbers of files than the GCS Jenkins plugin. -# We also intentionally ignore gsutil errors since we don't want failed uploads -# to fail the entire test run (#13548). - +# This script uploads metadata and test results to Google Cloud Storage, in the +# location indicated by JENKINS_GCS_LOGS_PATH. By default, we use the Google +# kubernetes-jenkins bucket. +# +# The script looks for one of two environment variables to be set: +# JENKINS_BUILD_STARTED: set to a nonempty string to upload version +# information to 'started.json'. The value of the variable is not +# currently used. +# JENKINS_BUILD_FINISHED: set to the Jenkins build result to upload the build +# result to 'finished.json', any test artifacts, and update the +# 'latest-build.txt' file pointer. Since this script uses gsutil directly, +# it's a bit faster at uploading large numbers of files than the GCS Jenkins +# plugin. It also makes use of gsutil's gzip functionality. +# # Note: for magicfile support to work correctly, the "file" utility must be # installed. -# TODO: eventually fold this all into upload-finished.sh, once every job is -# using it. - set -o errexit set -o nounset set -o pipefail +if [[ -n "${JENKINS_BUILD_STARTED:-}" && -n "${JENKINS_BUILD_FINISHED:-}" ]]; then + echo "Error: JENKINS_BUILD_STARTED and JENKINS_BUILD_FINISHED should not both be set!" + exit 1 +fi + +if [[ ! ${JENKINS_UPLOAD_TO_GCS:-y} =~ ^[yY]$ ]]; then + exit 0 +fi + if [[ ${JOB_NAME} =~ -pull- ]]; then : ${JENKINS_GCS_LOGS_PATH:="gs://kubernetes-jenkins/pr-logs/pull/${ghprbPullId:-unknown}"} else @@ -40,19 +54,79 @@ readonly artifacts_path="${WORKSPACE}/_artifacts" readonly gcs_job_path="${JENKINS_GCS_LOGS_PATH}/${JOB_NAME}" readonly gcs_build_path="${gcs_job_path}/${BUILD_NUMBER}" readonly gcs_acl="public-read" - -for upload_attempt in $(seq 3); do - echo "Uploading to ${gcs_build_path} (attempt ${upload_attempt})" - if [[ -d "${artifacts_path}" && -n $(ls -A "${artifacts_path}") ]]; then - gsutil -m -q -o "GSUtil:use_magicfile=True" cp -a "${gcs_acl}" -r -c \ - -z log,txt,xml "${artifacts_path}" "${gcs_build_path}/artifacts" || continue - fi - # Mark this build as the latest completed. - echo "${BUILD_NUMBER}" | \ - gsutil -q -h "Content-Type:text/plain" -h "Cache-Control:private, max-age=0, no-transform" \ - cp -a "${gcs_acl}" - "${gcs_job_path}/latest-build.txt" || continue - break # all uploads succeeded if we hit this point -done - readonly results_url=${gcs_build_path//"gs:/"/"https://storage.cloud.google.com"} -echo -e "\n\n\n*** View logs and artifacts at ${results_url} ***\n\n" +readonly timestamp=$(date +%s) + +function upload_version() { + echo -n 'Run starting at '; date -d "@${timestamp}" + + # Try to discover the kubernetes version. + local version="" + if [[ -e "version" ]]; then + version=$(cat "version") + elif [[ -e "hack/lib/version.sh" ]]; then + version=$( + export KUBE_ROOT="." + source "hack/lib/version.sh" + kube::version::get_version_vars + echo "${KUBE_GIT_VERSION-}" + ) + fi + + if [[ -n "${version}" ]]; then + echo "Found Kubernetes version: ${version}" + else + echo "Could not find Kubernetes version" + fi + + local -r json_file="${gcs_build_path}/started.json" + for upload_attempt in $(seq 3); do + echo "Uploading version to: ${json_file} (attempt ${upload_attempt})" + gsutil -q -h "Content-Type:application/json" cp -a "${gcs_acl}" <( + echo "{" + echo " \"version\": \"${version}\"," + echo " \"timestamp\": ${timestamp}" + echo "}" + ) "${json_file}" || continue + break + done +} + +function upload_artifacts_and_build_result() { + local -r build_result=$1 + echo -n 'Run finished at '; date -d "@${timestamp}" + + for upload_attempt in $(seq 3); do + echo "Uploading to ${gcs_build_path} (attempt ${upload_attempt})" + echo "Uploading build result: ${build_result}" + gsutil -q -h "Content-Type:application/json" cp -a "${gcs_acl}" <( + echo "{" + echo " \"result\": \"${build_result}\"," + echo " \"timestamp\": ${timestamp}" + echo "}" + ) "${gcs_build_path}/finished.json" || continue + if [[ -d "${artifacts_path}" && -n $(ls -A "${artifacts_path}") ]]; then + echo "Uploading artifacts" + gsutil -m -q -o "GSUtil:use_magicfile=True" cp -a "${gcs_acl}" -r -c \ + -z log,txt,xml "${artifacts_path}" "${gcs_build_path}/artifacts" || continue + fi + # Mark this build as the latest completed. + echo "Marking build ${BUILD_NUMBER} as the latest completed build" + echo "${BUILD_NUMBER}" | \ + gsutil -q -h "Content-Type:text/plain" -h "Cache-Control:private, max-age=0, no-transform" \ + cp -a "${gcs_acl}" - "${gcs_job_path}/latest-build.txt" || continue + break # all uploads succeeded if we hit this point + done + + echo -e "\n\n\n*** View logs and artifacts at ${results_url} ***\n\n" +} + +if [[ -n "${JENKINS_BUILD_STARTED:-}" ]]; then + upload_version +elif [[ -n "${JENKINS_BUILD_FINISHED:-}" ]]; then + upload_artifacts_and_build_result ${JENKINS_BUILD_FINISHED} +else + echo "Called without JENKINS_BUILD_STARTED or JENKINS_BUILD_FINISHED set." + echo "Assuming a legacy invocation." + upload_artifacts_and_build_result "[UNSET]" +fi From 62b3587b629dbfe01b29e1bda2bb4bc72705585e Mon Sep 17 00:00:00 2001 From: Jeff Grafton Date: Wed, 23 Mar 2016 16:55:17 -0700 Subject: [PATCH 2/2] Use upload-to-gcs.sh everywhere --- hack/jenkins/e2e-runner.sh | 3 ++- hack/jenkins/job-configs/global.yaml | 21 +++++++------------ .../kubernetes-jenkins/kubernetes-build.yaml | 2 +- .../kubernetes-test-go.yaml | 2 +- 4 files changed, 11 insertions(+), 17 deletions(-) diff --git a/hack/jenkins/e2e-runner.sh b/hack/jenkins/e2e-runner.sh index d83a1256c73..f0126db4fcc 100755 --- a/hack/jenkins/e2e-runner.sh +++ b/hack/jenkins/e2e-runner.sh @@ -207,8 +207,9 @@ fi cd kubernetes # Upload build start time and k8s version to GCS, but not on PR Jenkins. +# On PR Jenkins this is done before the build. if [[ ! "${JOB_NAME}" =~ -pull- ]]; then - bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-started.sh") + JENKINS_BUILD_STARTED=true bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh") fi # Have cmd/e2e run by goe2e.sh generate JUnit report in ${WORKSPACE}/junit*.xml diff --git a/hack/jenkins/job-configs/global.yaml b/hack/jenkins/job-configs/global.yaml index 838d285dfdf..e44b6b3d247 100644 --- a/hack/jenkins/job-configs/global.yaml +++ b/hack/jenkins/job-configs/global.yaml @@ -14,40 +14,33 @@ - postbuildscript: builders: - shell: | - curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-finished.sh" > upload-finished.sh - chmod +x upload-finished.sh + mkdir -p _tmp + curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh" > ./_tmp/upload-to-gcs.sh + chmod +x ./_tmp/upload-to-gcs.sh - conditional-step: condition-kind: current-status condition-worst: SUCCESS condition-best: SUCCESS steps: - - shell: './upload-finished.sh SUCCESS' + - shell: 'JENKINS_BUILD_FINISHED=SUCCESS ./_tmp/upload-to-gcs.sh' - conditional-step: condition-kind: current-status condition-worst: UNSTABLE condition-best: UNSTABLE steps: - - shell: './upload-finished.sh UNSTABLE' + - shell: 'JENKINS_BUILD_FINISHED=UNSTABLE ./_tmp/upload-to-gcs.sh' - conditional-step: condition-kind: current-status condition-worst: FAILURE condition-best: FAILURE steps: - - shell: './upload-finished.sh FAILURE' + - shell: 'JENKINS_BUILD_FINISHED=FAILURE ./_tmp/upload-to-gcs.sh' - conditional-step: condition-kind: current-status condition-worst: ABORTED condition-best: ABORTED steps: - - shell: './upload-finished.sh ABORTED' - # Use our script for build artifacts, since it's more flexible. - # Run last since it updates latest-build.txt. - - shell: | - if [[ -x ./hack/jenkins/upload-to-gcs.sh ]]; then - ./hack/jenkins/upload-to-gcs.sh - else - curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh" | bash - - fi + - shell: 'JENKINS_BUILD_FINISHED=ABORTED ./_tmp/upload-to-gcs.sh' script-only-if-succeeded: False script-only-if-failed: False # Use the plugin for the build log, since it isn't available on Jenkins slaves. diff --git a/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-build.yaml b/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-build.yaml index 7da1ce916c4..b57904bc0a1 100644 --- a/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-build.yaml +++ b/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-build.yaml @@ -5,7 +5,7 @@ logrotate: numToKeep: 200 builders: - - shell: 'bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-started.sh")' + - shell: 'JENKINS_BUILD_STARTED=true bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh")' - shell: | timeout -k {kill-timeout}m 30m ./hack/jenkins/build.sh && rc=$? || rc=$? {report-rc} diff --git a/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-test-go.yaml b/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-test-go.yaml index 6df68e08a8c..687bb2d6d4b 100644 --- a/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-test-go.yaml +++ b/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-test-go.yaml @@ -7,7 +7,7 @@ numToKeep: 200 node: unittest builders: - - shell: 'bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-started.sh")' + - shell: 'JENKINS_BUILD_STARTED=true bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh")' - shell: | export KUBE_FORCE_VERIFY_CHECKS='y' export KUBE_VERIFY_GIT_BRANCH='{branch}'