mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-08-09 12:07:47 +00:00
Merge pull request #22073 from ihmccreery/upgrade-job-infra
Prep for upgrade job infra fixups
This commit is contained in:
Isaac Hollander McCreery
commit
361cd2cad3
@ -21,96 +21,129 @@ set -o nounset
|
|||||||
set -o pipefail
|
set -o pipefail
|
||||||
set -o xtrace
|
set -o xtrace
|
||||||
|
|
||||||
|
function check_dirty_workspace() {
|
||||||
|
if [[ "${JENKINS_TOLERATE_DIRTY_WORKSPACE:-}" =~ ^[yY]$ ]]; then
|
||||||
|
echo "Tolerating dirty workspace (because JENKINS_TOLERATE_DIRTY_WORKSPACE)."
|
||||||
|
else
|
||||||
|
echo "Checking dirty workspace."
|
||||||
|
# .config and its children are created by the gcloud call that we use to
|
||||||
|
# get the GCE service account.
|
||||||
|
#
|
||||||
|
# console-log.txt is created by Jenkins, but is usually not flushed out
|
||||||
|
# this early in this script.
|
||||||
|
if [[ $(find . -not -path "./.config*" -not -name "console-log.txt" | wc -l) != 1 ]]; then
|
||||||
|
echo "${PWD} not empty, bailing!"
|
||||||
|
find .
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function fetch_output_tars() {
|
||||||
|
clean_binaries
|
||||||
|
echo "Using binaries from _output."
|
||||||
|
cp _output/release-tars/kubernetes*.tar.gz .
|
||||||
|
unpack_binaries
|
||||||
|
}
|
||||||
|
|
||||||
|
function fetch_server_version_tars() {
|
||||||
|
clean_binaries
|
||||||
|
local -r msg=$(gcloud ${CMD_GROUP} container get-server-config --project=${PROJECT} --zone=${ZONE} | grep defaultClusterVersion)
|
||||||
|
# msg will look like "defaultClusterVersion: 1.0.1". Strip
|
||||||
|
# everything up to, including ": "
|
||||||
|
local -r build_version="v${msg##*: }"
|
||||||
|
fetch_tars_from_gcs "release" "${build_version}"
|
||||||
|
unpack_binaries
|
||||||
|
}
|
||||||
|
|
||||||
|
# Use a published version like "ci/latest" (default), "release/latest",
|
||||||
|
# "release/latest-1", or "release/stable"
|
||||||
|
function fetch_published_version_tars() {
|
||||||
|
clean_binaries
|
||||||
|
local -r published_version="${1}"
|
||||||
|
IFS='/' read -a varr <<< "${published_version}"
|
||||||
|
bucket="${varr[0]}"
|
||||||
|
build_version=$(gsutil cat gs://kubernetes-release/${published_version}.txt)
|
||||||
|
echo "Using published version $bucket/$build_version (from ${published_version})"
|
||||||
|
fetch_tars_from_gcs "${bucket}" "${build_version}"
|
||||||
|
unpack_binaries
|
||||||
|
# Set CLUSTER_API_VERSION for GKE CI
|
||||||
|
export CLUSTER_API_VERSION=$(echo ${build_version} | cut -c 2-)
|
||||||
|
}
|
||||||
|
|
||||||
|
# TODO(ihmccreery) I'm not sure if this is necesssary, with the workspace check
|
||||||
|
# below.
|
||||||
|
function clean_binaries() {
|
||||||
|
echo "Cleaning up binaries."
|
||||||
|
rm -rf kubernetes*
|
||||||
|
}
|
||||||
|
|
||||||
|
function fetch_tars_from_gcs() {
|
||||||
|
local -r bucket="${1}"
|
||||||
|
local -r build_version="${1}"
|
||||||
|
echo "Pulling binaries from GCS; using server version ${bucket}/${build_version}."
|
||||||
|
gsutil -mq cp \
|
||||||
|
"gs://kubernetes-release/${bucket}/${build_version}/kubernetes.tar.gz" \
|
||||||
|
"gs://kubernetes-release/${bucket}/${build_version}/kubernetes-test.tar.gz" \
|
||||||
|
.
|
||||||
|
}
|
||||||
|
|
||||||
|
function unpack_binaries() {
|
||||||
|
md5sum kubernetes*.tar.gz
|
||||||
|
tar -xzf kubernetes.tar.gz
|
||||||
|
tar -xzf kubernetes-test.tar.gz
|
||||||
|
}
|
||||||
|
|
||||||
echo "--------------------------------------------------------------------------------"
|
echo "--------------------------------------------------------------------------------"
|
||||||
echo "Test Environment:"
|
echo "Test Environment:"
|
||||||
printenv | sort
|
printenv | sort
|
||||||
echo "--------------------------------------------------------------------------------"
|
echo "--------------------------------------------------------------------------------"
|
||||||
|
|
||||||
# We get the Kubernetes tarballs on either cluster creation or when we want to
|
# We get the Kubernetes tarballs unless we are going to use old ones
|
||||||
# replace existing ones in a multi-step job (e.g. a cluster upgrade).
|
if [[ "${JENKINS_USE_EXISTING_BINARIES:-}" =~ ^[yY]$ ]]; then
|
||||||
if [[ "${E2E_UP,,}" == "true" || "${JENKINS_FORCE_GET_TARS:-}" =~ ^[yY]$ ]]; then
|
echo "Using existing binaries; not cleaning, fetching, or unpacking new ones."
|
||||||
if [[ ${KUBE_RUN_FROM_OUTPUT:-} =~ ^[yY]$ ]]; then
|
elif [[ "${KUBE_RUN_FROM_OUTPUT:-}" =~ ^[yY]$ ]]; then
|
||||||
echo "Found KUBE_RUN_FROM_OUTPUT=y; will use binaries from _output"
|
# TODO(spxtr) This should probably be JENKINS_USE_BINARIES_FROM_OUTPUT or
|
||||||
cp _output/release-tars/kubernetes*.tar.gz .
|
# something, rather than being prepended with KUBE, since it's sort of a
|
||||||
else
|
# meta-thing.
|
||||||
echo "Pulling binaries from GCS"
|
fetch_output_tars
|
||||||
# In a multi-step job, clean up just the kubernetes build files.
|
elif [[ "${JENKINS_USE_SERVER_VERSION:-}" =~ ^[yY]$ ]]; then
|
||||||
# Otherwise, we want a completely empty directory.
|
# This is for test, staging, and prod jobs on GKE, where we want to
|
||||||
if [[ "${JENKINS_FORCE_GET_TARS:-}" =~ ^[yY]$ ]]; then
|
# test what's running in GKE by default rather than some CI build.
|
||||||
rm -rf kubernetes*
|
check_dirty_workspace
|
||||||
# .config and its children are created by the gcloud call that we use to
|
fetch_server_version_tars
|
||||||
# get the GCE service account.
|
else
|
||||||
# console-log.txt is created by Jenkins, but is usually not flushed out
|
# use JENKINS_PUBLISHED_VERSION, default to 'ci/latest', since that's
|
||||||
# this early in the script.
|
# usually what we're testing.
|
||||||
elif [[ $(find . -not -path "./.config*" -not -name "console-log.txt" \
|
check_dirty_workspace
|
||||||
| wc -l) != 1 ]]; then
|
fetch_published_version_tars "${JENKINS_PUBLISHED_VERSION:-'ci/latest'}"
|
||||||
echo "${PWD} not empty, bailing!"
|
fi
|
||||||
find .
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# This is for test, staging, and prod jobs on GKE, where we want to
|
# Copy GCE keys so we don't keep cycling them.
|
||||||
# test what's running in GKE by default rather than some CI build.
|
# To set this up, you must know the <project>, <zone>, and <instance>
|
||||||
if [[ ${JENKINS_USE_SERVER_VERSION:-} =~ ^[yY]$ ]]; then
|
# on which your jenkins jobs are running. Then do:
|
||||||
# for GKE we can use server default version.
|
#
|
||||||
bucket="release"
|
# # SSH from your computer into the instance.
|
||||||
msg=$(gcloud ${CMD_GROUP} container get-server-config --project=${PROJECT} --zone=${ZONE} | grep defaultClusterVersion)
|
# $ gcloud compute ssh --project="<prj>" ssh --zone="<zone>" <instance>
|
||||||
# msg will look like "defaultClusterVersion: 1.0.1". Strip
|
#
|
||||||
# everything up to, including ": "
|
# # Generate a key by ssh'ing from the instance into itself, then exit.
|
||||||
build_version="v${msg##*: }"
|
# $ gcloud compute ssh --project="<prj>" ssh --zone="<zone>" <instance>
|
||||||
echo "Using server version $bucket/$build_version"
|
# $ ^D
|
||||||
else # use JENKINS_PUBLISHED_VERSION, for CI
|
#
|
||||||
# Use a published version like "ci/latest" (default),
|
# # Copy the keys to the desired location (e.g. /var/lib/jenkins/gce_keys/).
|
||||||
# "release/latest", "release/latest-1", or "release/stable"
|
# $ sudo mkdir -p /var/lib/jenkins/gce_keys/
|
||||||
JENKINS_PUBLISHED_VERSION=${JENKINS_PUBLISHED_VERSION:-'ci/latest'}
|
# $ sudo cp ~/.ssh/google_compute_engine /var/lib/jenkins/gce_keys/
|
||||||
IFS='/' read -a varr <<< "${JENKINS_PUBLISHED_VERSION}"
|
# $ sudo cp ~/.ssh/google_compute_engine.pub /var/lib/jenkins/gce_keys/
|
||||||
bucket="${varr[0]}"
|
#
|
||||||
build_version=$(gsutil cat gs://kubernetes-release/${JENKINS_PUBLISHED_VERSION}.txt)
|
# # Move the permissions for the keys to Jenkins.
|
||||||
echo "Using published version $bucket/$build_version (from ${JENKINS_PUBLISHED_VERSION})"
|
# $ sudo chown -R jenkins /var/lib/jenkins/gce_keys/
|
||||||
# Set CLUSTER_API_VERSION for GKE CI
|
# $ sudo chgrp -R jenkins /var/lib/jenkins/gce_keys/
|
||||||
export CLUSTER_API_VERSION=$(echo ${build_version} | cut -c 2-)
|
if [[ "${KUBERNETES_PROVIDER}" == "aws" ]]; then
|
||||||
fi
|
echo "Skipping SSH key copying for AWS"
|
||||||
# At this point, we want to have the following vars set:
|
else
|
||||||
# - bucket
|
mkdir -p ${WORKSPACE}/.ssh/
|
||||||
# - build_version
|
cp /var/lib/jenkins/gce_keys/google_compute_engine ${WORKSPACE}/.ssh/
|
||||||
gsutil -mq cp \
|
cp /var/lib/jenkins/gce_keys/google_compute_engine.pub ${WORKSPACE}/.ssh/
|
||||||
"gs://kubernetes-release/${bucket}/${build_version}/kubernetes.tar.gz" \
|
|
||||||
"gs://kubernetes-release/${bucket}/${build_version}/kubernetes-test.tar.gz" \
|
|
||||||
.
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ ! "${CIRCLECI:-}" == "true" ]]; then
|
|
||||||
# Copy GCE keys so we don't keep cycling them.
|
|
||||||
# To set this up, you must know the <project>, <zone>, and <instance>
|
|
||||||
# on which your jenkins jobs are running. Then do:
|
|
||||||
#
|
|
||||||
# # SSH from your computer into the instance.
|
|
||||||
# $ gcloud compute ssh --project="<prj>" ssh --zone="<zone>" <instance>
|
|
||||||
#
|
|
||||||
# # Generate a key by ssh'ing from the instance into itself, then exit.
|
|
||||||
# $ gcloud compute ssh --project="<prj>" ssh --zone="<zone>" <instance>
|
|
||||||
# $ ^D
|
|
||||||
#
|
|
||||||
# # Copy the keys to the desired location (e.g. /var/lib/jenkins/gce_keys/).
|
|
||||||
# $ sudo mkdir -p /var/lib/jenkins/gce_keys/
|
|
||||||
# $ sudo cp ~/.ssh/google_compute_engine /var/lib/jenkins/gce_keys/
|
|
||||||
# $ sudo cp ~/.ssh/google_compute_engine.pub /var/lib/jenkins/gce_keys/
|
|
||||||
#
|
|
||||||
# # Move the permissions for the keys to Jenkins.
|
|
||||||
# $ sudo chown -R jenkins /var/lib/jenkins/gce_keys/
|
|
||||||
# $ sudo chgrp -R jenkins /var/lib/jenkins/gce_keys/
|
|
||||||
if [[ "${KUBERNETES_PROVIDER}" == "aws" ]]; then
|
|
||||||
echo "Skipping SSH key copying for AWS"
|
|
||||||
else
|
|
||||||
mkdir -p ${WORKSPACE}/.ssh/
|
|
||||||
cp /var/lib/jenkins/gce_keys/google_compute_engine ${WORKSPACE}/.ssh/
|
|
||||||
cp /var/lib/jenkins/gce_keys/google_compute_engine.pub ${WORKSPACE}/.ssh/
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
md5sum kubernetes*.tar.gz
|
|
||||||
tar -xzf kubernetes.tar.gz
|
|
||||||
tar -xzf kubernetes-test.tar.gz
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
cd kubernetes
|
cd kubernetes
|
||||||
@ -177,9 +210,8 @@ if [[ "${E2E_TEST,,}" == "true" ]]; then
|
|||||||
${GINKGO_TEST_ARGS:+--test_args="${GINKGO_TEST_ARGS}"} \
|
${GINKGO_TEST_ARGS:+--test_args="${GINKGO_TEST_ARGS}"} \
|
||||||
&& exitcode=0 || exitcode=$?
|
&& exitcode=0 || exitcode=$?
|
||||||
if [[ "${E2E_PUBLISH_GREEN_VERSION:-}" == "true" && ${exitcode} == 0 && -n ${build_version:-} ]]; then
|
if [[ "${E2E_PUBLISH_GREEN_VERSION:-}" == "true" && ${exitcode} == 0 && -n ${build_version:-} ]]; then
|
||||||
echo "publish build_version to ci/latest-green.txt: ${build_version}"
|
echo "Publish build_version to ci/latest-green.txt: ${build_version}"
|
||||||
echo "${build_version}" > ${WORKSPACE}/build_version.txt
|
gsutil cp ./version gs://kubernetes-release/ci/latest-green.txt
|
||||||
gsutil cp ${WORKSPACE}/build_version.txt gs://kubernetes-release/ci/latest-green.txt
|
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|||||||
@ -71,6 +71,7 @@
|
|||||||
export E2E_TEST="false"
|
export E2E_TEST="false"
|
||||||
export E2E_DOWN="false"
|
export E2E_DOWN="false"
|
||||||
soak-continuous: |
|
soak-continuous: |
|
||||||
|
export JENKINS_USE_EXISTING_BINARIES="y"
|
||||||
export FAIL_ON_GCP_RESOURCE_LEAK="false"
|
export FAIL_ON_GCP_RESOURCE_LEAK="false"
|
||||||
export E2E_UP="false"
|
export E2E_UP="false"
|
||||||
export E2E_DOWN="false"
|
export E2E_DOWN="false"
|
||||||
|
|||||||
@ -192,7 +192,7 @@
|
|||||||
name: 'upgrade-gke'
|
name: 'upgrade-gke'
|
||||||
provider-env: |
|
provider-env: |
|
||||||
{gke-provider-env}
|
{gke-provider-env}
|
||||||
export JENKINS_FORCE_GET_TARS="y"
|
export JENKINS_TOLERATE_DIRTY_WORKSPACE="y"
|
||||||
export FAIL_ON_GCP_RESOURCE_LEAK="false"
|
export FAIL_ON_GCP_RESOURCE_LEAK="false"
|
||||||
jobs:
|
jobs:
|
||||||
- '{provider}-{version-old}-{version-new}-upgrades':
|
- '{provider}-{version-old}-{version-new}-upgrades':
|
||||||
@ -233,7 +233,7 @@
|
|||||||
provider-env: |
|
provider-env: |
|
||||||
{gce-provider-env}
|
{gce-provider-env}
|
||||||
export NUM_NODES=5
|
export NUM_NODES=5
|
||||||
export JENKINS_FORCE_GET_TARS="y"
|
export JENKINS_TOLERATE_DIRTY_WORKSPACE="y"
|
||||||
export FAIL_ON_GCP_RESOURCE_LEAK="false"
|
export FAIL_ON_GCP_RESOURCE_LEAK="false"
|
||||||
jobs:
|
jobs:
|
||||||
- '{provider}-{version-old}-{version-new}-upgrades':
|
- '{provider}-{version-old}-{version-new}-upgrades':
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user