github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-24 19:12:02 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #21180 from ixdy/cleanup-upload-to-gcs-script

Browse Source 
Auto commit by PR queue bot
This commit is contained in:
k8s-merge-robot 2016-02-21 03:14:31 -08:00
commit 7462f8cb47
1 changed files with 25 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
hack/jenkins/upload-to-gcs.sh
View File

@ -14,26 +14,21 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# Source this script in the Jenkins "Execute shell" build action to have all # Run this script in the Jenkins "Execute shell" build action to upload test
# test artifacts and the console log uploaded at the end of the test run. # artifacts to GCS. Since it uses gsutil directly, it's a bit faster at
# uploading large numbers of files than the GCS Jenkins plugin.
# We also intentionally ignore gsutil errors since we don't want failed uploads
# to fail the entire test run (#13548).
# For example, you might use the following snippet as the first few lines: # Note: for magicfile support to work correctly, the "file" utility must be
#
# if [[ -f ./hack/jenkins/upload-to-gcs.sh ]]; then
# source ./hack/jenkins/upload-to-gcs.sh
# else
# curl -fsS -o upload-to-gcs.sh --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh" && source upload-to-gcs.sh; rm -f upload-to-gcs.sh
# fi
# Note that this script requires the Jenkins shell binary to be set to bash, not
# the system default (which may be dash on Debian-based systems).
# Also, for magicfile support to work correctly, the "file" utility must be
# installed. # installed.
# TODO: eventually fold this all into upload-finished.sh, once every job is
# using it.
set -o errexit set -o errexit
set -o nounset set -o nounset
set -o pipefail set -o pipefail
set -o xtrace
if [[ ${JOB_NAME} =~ -pull- ]]; then if [[ ${JOB_NAME} =~ -pull- ]]; then
: ${JENKINS_GCS_LOGS_PATH:="gs://kubernetes-jenkins/pr-logs/pull/${ghprbPullId:-unknown}"} : ${JENKINS_GCS_LOGS_PATH:="gs://kubernetes-jenkins/pr-logs/pull/${ghprbPullId:-unknown}"}
@ -41,47 +36,23 @@ else
: ${JENKINS_GCS_LOGS_PATH:="gs://kubernetes-jenkins/logs"} : ${JENKINS_GCS_LOGS_PATH:="gs://kubernetes-jenkins/logs"}
fi fi
: ${JENKINS_UPLOAD_TO_GCS:="y"} readonly artifacts_path="${WORKSPACE}/_artifacts"
readonly gcs_job_path="${JENKINS_GCS_LOGS_PATH}/${JOB_NAME}"
readonly gcs_build_path="${gcs_job_path}/${BUILD_NUMBER}"
readonly gcs_acl="public-read"
# Upload the build log and all test artifacts (under _artifacts) to GCS when for upload_attempt in $(seq 3); do
# JENKINS_UPLOAD_TO_GCS is set to y. echo "Uploading to ${gcs_build_path} (attempt ${upload_attempt})"
# We intentionally ignore gsutil errors since we don't want failed uploads to if [[ -d "${artifacts_path}" && -n $(ls -A "${artifacts_path}") ]]; then
# fail the entire test run (#13548). gsutil -m -q -o "GSUtil:use_magicfile=True" cp -a "${gcs_acl}" -r -c \
function upload_logs_to_gcs() { -z log,txt,xml "${artifacts_path}" "${gcs_build_path}/artifacts" || continue
if [[ ! ${JENKINS_UPLOAD_TO_GCS:-} =~ ^[yY]$ ]]; then
return
fi fi
local -r artifacts_path="${WORKSPACE}/_artifacts" # Mark this build as the latest completed.
local -r gcs_job_path="${JENKINS_GCS_LOGS_PATH}/${JOB_NAME}" echo "${BUILD_NUMBER}" | \
local -r gcs_build_path="${gcs_job_path}/${BUILD_NUMBER}" gsutil -q -h "Content-Type:text/plain" -h "Cache-Control:private, max-age=0, no-transform" \
local -r gcs_acl="public-read"
local -r console_log="${JENKINS_HOME}/jobs/${JOB_NAME}/builds/${BUILD_NUMBER}/log"
local -r filtered_console_log="${WORKSPACE}/console-log.txt"
for upload_attempt in $(seq 3); do
echo "Uploading to ${gcs_build_path} (attempt ${upload_attempt})"
if [[ -d "${artifacts_path}" && -n $(ls -A "${artifacts_path}") ]]; then
gsutil -m -q -o "GSUtil:use_magicfile=True" cp -a "${gcs_acl}" -r -c \
-z log,txt,xml "${artifacts_path}" "${gcs_build_path}/artifacts" || continue
fi
# The console log only exists on the Jenkins master, so don't fail if it
# doesn't exist.
if [[ -f "${console_log}" ]]; then
# Remove ANSI escape sequences from the console log before uploading.
sed -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" \
"${console_log}" > "${filtered_console_log}"
gsutil -q cp -a "${gcs_acl}" -z txt "${filtered_console_log}" "${gcs_build_path}/" || continue
fi
# Mark this build as the latest completed.
{
echo "BUILD_NUMBER=${BUILD_NUMBER}"
echo "GIT_COMMIT=${GIT_COMMIT:-}"
} | gsutil -q -h "Content-Type:text/plain" -h "Cache-Control:private, max-age=0, no-transform" \
cp -a "${gcs_acl}" - "${gcs_job_path}/latest-build.txt" || continue cp -a "${gcs_acl}" - "${gcs_job_path}/latest-build.txt" || continue
break # all uploads succeeded if we hit this point break # all uploads succeeded if we hit this point
done done
local -r results_url=${gcs_build_path//"gs:/"/"https://storage.cloud.google.com"}
echo -e "\n\n\n*** View logs and artifacts at ${results_url} ***\n\n"
}
# Automatically upload logs to GCS on exit or timeout. readonly results_url=${gcs_build_path//"gs:/"/"https://storage.cloud.google.com"}
trap upload_logs_to_gcs EXIT SIGTERM SIGINT echo -e "\n\n\n*** View logs and artifacts at ${results_url} ***\n\n"