mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-08-05 10:19:50 +00:00
Merge pull request #23415 from ixdy/upload-to-gcs-script
Refactor upload-started.sh and upload-finished.sh into upload-to-gcs.sh
This commit is contained in:
Jeff Grafton
commit
d7f7c7796c
@ -207,8 +207,9 @@ fi
|
||||
cd kubernetes
|
||||
|
||||
# Upload build start time and k8s version to GCS, but not on PR Jenkins.
|
||||
# On PR Jenkins this is done before the build.
|
||||
if [[ ! "${JOB_NAME}" =~ -pull- ]]; then
|
||||
bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-started.sh")
|
||||
JENKINS_BUILD_STARTED=true bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh")
|
||||
fi
|
||||
|
||||
# Have cmd/e2e run by goe2e.sh generate JUnit report in ${WORKSPACE}/junit*.xml
|
||||
|
||||
@ -14,40 +14,33 @@
|
||||
- postbuildscript:
|
||||
builders:
|
||||
- shell: |
|
||||
curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-finished.sh" > upload-finished.sh
|
||||
chmod +x upload-finished.sh
|
||||
mkdir -p _tmp
|
||||
curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh" > ./_tmp/upload-to-gcs.sh
|
||||
chmod +x ./_tmp/upload-to-gcs.sh
|
||||
- conditional-step:
|
||||
condition-kind: current-status
|
||||
condition-worst: SUCCESS
|
||||
condition-best: SUCCESS
|
||||
steps:
|
||||
- shell: './upload-finished.sh SUCCESS'
|
||||
- shell: 'JENKINS_BUILD_FINISHED=SUCCESS ./_tmp/upload-to-gcs.sh'
|
||||
- conditional-step:
|
||||
condition-kind: current-status
|
||||
condition-worst: UNSTABLE
|
||||
condition-best: UNSTABLE
|
||||
steps:
|
||||
- shell: './upload-finished.sh UNSTABLE'
|
||||
- shell: 'JENKINS_BUILD_FINISHED=UNSTABLE ./_tmp/upload-to-gcs.sh'
|
||||
- conditional-step:
|
||||
condition-kind: current-status
|
||||
condition-worst: FAILURE
|
||||
condition-best: FAILURE
|
||||
steps:
|
||||
- shell: './upload-finished.sh FAILURE'
|
||||
- shell: 'JENKINS_BUILD_FINISHED=FAILURE ./_tmp/upload-to-gcs.sh'
|
||||
- conditional-step:
|
||||
condition-kind: current-status
|
||||
condition-worst: ABORTED
|
||||
condition-best: ABORTED
|
||||
steps:
|
||||
- shell: './upload-finished.sh ABORTED'
|
||||
# Use our script for build artifacts, since it's more flexible.
|
||||
# Run last since it updates latest-build.txt.
|
||||
- shell: |
|
||||
if [[ -x ./hack/jenkins/upload-to-gcs.sh ]]; then
|
||||
./hack/jenkins/upload-to-gcs.sh
|
||||
else
|
||||
curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh" | bash -
|
||||
fi
|
||||
- shell: 'JENKINS_BUILD_FINISHED=ABORTED ./_tmp/upload-to-gcs.sh'
|
||||
script-only-if-succeeded: False
|
||||
script-only-if-failed: False
|
||||
# Use the plugin for the build log, since it isn't available on Jenkins slaves.
|
||||
|
||||
@ -5,7 +5,7 @@
|
||||
logrotate:
|
||||
numToKeep: 200
|
||||
builders:
|
||||
- shell: 'bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-started.sh")'
|
||||
- shell: 'JENKINS_BUILD_STARTED=true bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh")'
|
||||
- shell: |
|
||||
timeout -k {kill-timeout}m 30m ./hack/jenkins/build.sh && rc=$? || rc=$?
|
||||
{report-rc}
|
||||
|
||||
@ -7,7 +7,7 @@
|
||||
numToKeep: 200
|
||||
node: unittest
|
||||
builders:
|
||||
- shell: 'bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-started.sh")'
|
||||
- shell: 'JENKINS_BUILD_STARTED=true bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh")'
|
||||
- shell: |
|
||||
export KUBE_FORCE_VERIFY_CHECKS='y'
|
||||
export KUBE_VERIFY_GIT_BRANCH='{branch}'
|
||||
|
||||
@ -29,24 +29,15 @@ if [[ $# -ne 1 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# TODO: DRY. Refactor into upload-to-gcs.sh ?
|
||||
: ${JENKINS_GCS_LOGS_PATH:="gs://kubernetes-jenkins/logs"}
|
||||
: ${JENKINS_UPLOAD_TO_GCS:="y"}
|
||||
export JENKINS_BUILD_FINISHED="$1"
|
||||
|
||||
if [[ ! ${JENKINS_UPLOAD_TO_GCS:-} =~ ^[yY]$ ]]; then
|
||||
exit 0
|
||||
echo
|
||||
echo "Passing through to upload-to-gcs.sh with JENKINS_BUILD_FINISHED=${JENKINS_BUILD_FINISHED}"
|
||||
echo "Please update configs to call upload-to-gcs.sh directly."
|
||||
echo
|
||||
|
||||
if [[ -x ./hack/jenkins/upload-to-gcs.sh ]]; then
|
||||
./hack/jenkins/upload-to-gcs.sh
|
||||
else
|
||||
curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh" | bash -
|
||||
fi
|
||||
|
||||
readonly result="$1"
|
||||
readonly timestamp=$(date +%s)
|
||||
readonly location="${JENKINS_GCS_LOGS_PATH}/${JOB_NAME}/${BUILD_NUMBER}/finished.json"
|
||||
|
||||
echo -n 'Run finished at '; date -d "@${timestamp}"
|
||||
|
||||
echo "Uploading build result to: ${location}"
|
||||
gsutil -q cp -a "public-read" <(
|
||||
echo "{"
|
||||
echo " \"result\": \"${result}\","
|
||||
echo " \"timestamp\": ${timestamp}"
|
||||
echo "}"
|
||||
) "${location}"
|
||||
|
||||
@ -24,42 +24,15 @@ set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
# TODO: DRY. Refactor into upload-to-gcs.sh ?
|
||||
: ${JENKINS_GCS_LOGS_PATH:="gs://kubernetes-jenkins/logs"}
|
||||
: ${JENKINS_UPLOAD_TO_GCS:="y"}
|
||||
export JENKINS_BUILD_STARTED="true"
|
||||
|
||||
if [[ ! ${JENKINS_UPLOAD_TO_GCS:-} =~ ^[yY]$ ]]; then
|
||||
exit 0
|
||||
fi
|
||||
echo
|
||||
echo "Passing through to upload-to-gcs.sh with JENKINS_BUILD_STARTED=${JENKINS_BUILD_STARTED}"
|
||||
echo "Please update configs to call upload-to-gcs.sh directly."
|
||||
echo
|
||||
|
||||
version=""
|
||||
readonly timestamp=$(date +%s)
|
||||
readonly location="${JENKINS_GCS_LOGS_PATH}/${JOB_NAME}/${BUILD_NUMBER}/started.json"
|
||||
|
||||
echo -n 'Run starting at '; date -d "@${timestamp}"
|
||||
|
||||
# Try to discover the kubernetes version.
|
||||
if [[ -e "version" ]]; then
|
||||
version=$(cat "version")
|
||||
elif [[ -e "hack/lib/version.sh" ]]; then
|
||||
version=$(
|
||||
export KUBE_ROOT="."
|
||||
source "hack/lib/version.sh"
|
||||
kube::version::get_version_vars
|
||||
echo "${KUBE_GIT_VERSION-}"
|
||||
)
|
||||
fi
|
||||
|
||||
if [[ -n "${version}" ]]; then
|
||||
echo "Found Kubernetes version: ${version}"
|
||||
if [[ -x ./hack/jenkins/upload-to-gcs.sh ]]; then
|
||||
./hack/jenkins/upload-to-gcs.sh
|
||||
else
|
||||
echo "Could not find Kubernetes version"
|
||||
curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh" | bash -
|
||||
fi
|
||||
|
||||
echo "Uploading version to: ${location}"
|
||||
gsutil -q cp -a "public-read" <(
|
||||
echo "{"
|
||||
echo " \"version\": \"${version}\","
|
||||
echo " \"timestamp\": ${timestamp}"
|
||||
echo "}"
|
||||
) "${location}"
|
||||
|
||||
@ -14,22 +14,36 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Run this script in the Jenkins "Execute shell" build action to upload test
|
||||
# artifacts to GCS. Since it uses gsutil directly, it's a bit faster at
|
||||
# uploading large numbers of files than the GCS Jenkins plugin.
|
||||
# We also intentionally ignore gsutil errors since we don't want failed uploads
|
||||
# to fail the entire test run (#13548).
|
||||
|
||||
# This script uploads metadata and test results to Google Cloud Storage, in the
|
||||
# location indicated by JENKINS_GCS_LOGS_PATH. By default, we use the Google
|
||||
# kubernetes-jenkins bucket.
|
||||
#
|
||||
# The script looks for one of two environment variables to be set:
|
||||
# JENKINS_BUILD_STARTED: set to a nonempty string to upload version
|
||||
# information to 'started.json'. The value of the variable is not
|
||||
# currently used.
|
||||
# JENKINS_BUILD_FINISHED: set to the Jenkins build result to upload the build
|
||||
# result to 'finished.json', any test artifacts, and update the
|
||||
# 'latest-build.txt' file pointer. Since this script uses gsutil directly,
|
||||
# it's a bit faster at uploading large numbers of files than the GCS Jenkins
|
||||
# plugin. It also makes use of gsutil's gzip functionality.
|
||||
#
|
||||
# Note: for magicfile support to work correctly, the "file" utility must be
|
||||
# installed.
|
||||
|
||||
# TODO: eventually fold this all into upload-finished.sh, once every job is
|
||||
# using it.
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
if [[ -n "${JENKINS_BUILD_STARTED:-}" && -n "${JENKINS_BUILD_FINISHED:-}" ]]; then
|
||||
echo "Error: JENKINS_BUILD_STARTED and JENKINS_BUILD_FINISHED should not both be set!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! ${JENKINS_UPLOAD_TO_GCS:-y} =~ ^[yY]$ ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ ${JOB_NAME} =~ -pull- ]]; then
|
||||
: ${JENKINS_GCS_LOGS_PATH:="gs://kubernetes-jenkins/pr-logs/pull/${ghprbPullId:-unknown}"}
|
||||
else
|
||||
@ -40,19 +54,79 @@ readonly artifacts_path="${WORKSPACE}/_artifacts"
|
||||
readonly gcs_job_path="${JENKINS_GCS_LOGS_PATH}/${JOB_NAME}"
|
||||
readonly gcs_build_path="${gcs_job_path}/${BUILD_NUMBER}"
|
||||
readonly gcs_acl="public-read"
|
||||
|
||||
for upload_attempt in $(seq 3); do
|
||||
echo "Uploading to ${gcs_build_path} (attempt ${upload_attempt})"
|
||||
if [[ -d "${artifacts_path}" && -n $(ls -A "${artifacts_path}") ]]; then
|
||||
gsutil -m -q -o "GSUtil:use_magicfile=True" cp -a "${gcs_acl}" -r -c \
|
||||
-z log,txt,xml "${artifacts_path}" "${gcs_build_path}/artifacts" || continue
|
||||
fi
|
||||
# Mark this build as the latest completed.
|
||||
echo "${BUILD_NUMBER}" | \
|
||||
gsutil -q -h "Content-Type:text/plain" -h "Cache-Control:private, max-age=0, no-transform" \
|
||||
cp -a "${gcs_acl}" - "${gcs_job_path}/latest-build.txt" || continue
|
||||
break # all uploads succeeded if we hit this point
|
||||
done
|
||||
|
||||
readonly results_url=${gcs_build_path//"gs:/"/"https://storage.cloud.google.com"}
|
||||
echo -e "\n\n\n*** View logs and artifacts at ${results_url} ***\n\n"
|
||||
readonly timestamp=$(date +%s)
|
||||
|
||||
function upload_version() {
|
||||
echo -n 'Run starting at '; date -d "@${timestamp}"
|
||||
|
||||
# Try to discover the kubernetes version.
|
||||
local version=""
|
||||
if [[ -e "version" ]]; then
|
||||
version=$(cat "version")
|
||||
elif [[ -e "hack/lib/version.sh" ]]; then
|
||||
version=$(
|
||||
export KUBE_ROOT="."
|
||||
source "hack/lib/version.sh"
|
||||
kube::version::get_version_vars
|
||||
echo "${KUBE_GIT_VERSION-}"
|
||||
)
|
||||
fi
|
||||
|
||||
if [[ -n "${version}" ]]; then
|
||||
echo "Found Kubernetes version: ${version}"
|
||||
else
|
||||
echo "Could not find Kubernetes version"
|
||||
fi
|
||||
|
||||
local -r json_file="${gcs_build_path}/started.json"
|
||||
for upload_attempt in $(seq 3); do
|
||||
echo "Uploading version to: ${json_file} (attempt ${upload_attempt})"
|
||||
gsutil -q -h "Content-Type:application/json" cp -a "${gcs_acl}" <(
|
||||
echo "{"
|
||||
echo " \"version\": \"${version}\","
|
||||
echo " \"timestamp\": ${timestamp}"
|
||||
echo "}"
|
||||
) "${json_file}" || continue
|
||||
break
|
||||
done
|
||||
}
|
||||
|
||||
function upload_artifacts_and_build_result() {
|
||||
local -r build_result=$1
|
||||
echo -n 'Run finished at '; date -d "@${timestamp}"
|
||||
|
||||
for upload_attempt in $(seq 3); do
|
||||
echo "Uploading to ${gcs_build_path} (attempt ${upload_attempt})"
|
||||
echo "Uploading build result: ${build_result}"
|
||||
gsutil -q -h "Content-Type:application/json" cp -a "${gcs_acl}" <(
|
||||
echo "{"
|
||||
echo " \"result\": \"${build_result}\","
|
||||
echo " \"timestamp\": ${timestamp}"
|
||||
echo "}"
|
||||
) "${gcs_build_path}/finished.json" || continue
|
||||
if [[ -d "${artifacts_path}" && -n $(ls -A "${artifacts_path}") ]]; then
|
||||
echo "Uploading artifacts"
|
||||
gsutil -m -q -o "GSUtil:use_magicfile=True" cp -a "${gcs_acl}" -r -c \
|
||||
-z log,txt,xml "${artifacts_path}" "${gcs_build_path}/artifacts" || continue
|
||||
fi
|
||||
# Mark this build as the latest completed.
|
||||
echo "Marking build ${BUILD_NUMBER} as the latest completed build"
|
||||
echo "${BUILD_NUMBER}" | \
|
||||
gsutil -q -h "Content-Type:text/plain" -h "Cache-Control:private, max-age=0, no-transform" \
|
||||
cp -a "${gcs_acl}" - "${gcs_job_path}/latest-build.txt" || continue
|
||||
break # all uploads succeeded if we hit this point
|
||||
done
|
||||
|
||||
echo -e "\n\n\n*** View logs and artifacts at ${results_url} ***\n\n"
|
||||
}
|
||||
|
||||
if [[ -n "${JENKINS_BUILD_STARTED:-}" ]]; then
|
||||
upload_version
|
||||
elif [[ -n "${JENKINS_BUILD_FINISHED:-}" ]]; then
|
||||
upload_artifacts_and_build_result ${JENKINS_BUILD_FINISHED}
|
||||
else
|
||||
echo "Called without JENKINS_BUILD_STARTED or JENKINS_BUILD_FINISHED set."
|
||||
echo "Assuming a legacy invocation."
|
||||
upload_artifacts_and_build_result "[UNSET]"
|
||||
fi
|
||||
|
||||
Loading…
Reference in New Issue
Block a user