mirror of
https://github.com/k3s-io/kubernetes.git
synced 2026-01-05 07:27:21 +00:00
Move kubemark, skew, upgrade, leak logic into hack/e2e.go
This commit is contained in:
Erick Fejta
@@ -161,35 +161,6 @@ function install_google_cloud_sdk_tarball() {
|
||||
export PATH=${install_dir}/google-cloud-sdk/bin:${PATH}
|
||||
}
|
||||
|
||||
# Only call after attempting to bring the cluster up. Don't call after
|
||||
# bringing the cluster down.
|
||||
function dump_cluster_logs_and_exit() {
|
||||
local -r exit_status=$?
|
||||
dump_cluster_logs
|
||||
if [[ "${USE_KUBEMARK:-}" == "true" ]]; then
|
||||
# If we tried to bring the Kubemark cluster up, make a courtesy
|
||||
# attempt to bring it down so we're not leaving resources around.
|
||||
./test/kubemark/stop-kubemark.sh || true
|
||||
fi
|
||||
if [[ "${E2E_DOWN,,}" == "true" ]]; then
|
||||
# If we tried to bring the cluster up, make a courtesy attempt
|
||||
# to bring the cluster down so we're not leaving resources
|
||||
# around. Unlike later, don't sleep beforehand, though. (We're
|
||||
# just trying to tear down as many resources as we can as fast
|
||||
# as possible and don't even know if we brought the master up.)
|
||||
go run ./hack/e2e.go ${E2E_OPT:-} -v --down || true
|
||||
fi
|
||||
exit ${exit_status}
|
||||
}
|
||||
|
||||
# Only call after attempting to bring the cluster up. Don't call after
|
||||
# bringing the cluster down.
|
||||
function dump_cluster_logs() {
|
||||
if [[ -x "cluster/log-dump.sh" ]]; then
|
||||
./cluster/log-dump.sh "${ARTIFACTS}"
|
||||
fi
|
||||
}
|
||||
|
||||
# Figures out the builtin k8s version of a GCI image.
|
||||
function get_gci_k8s_version() {
|
||||
local -r image_description=$(gcloud compute images describe ${KUBE_GCE_MASTER_IMAGE} --project=${KUBE_GCE_MASTER_PROJECT})
|
||||
@@ -265,24 +236,6 @@ elif [[ -n "${KUBEKINS_SERVICE_ACCOUNT_FILE:-}" ]]; then
|
||||
echo "ERROR: cannot access service account file at: ${KUBEKINS_SERVICE_ACCOUNT_FILE}"
|
||||
fi
|
||||
|
||||
|
||||
function e2e_test() {
|
||||
local -r ginkgo_test_args="${1}"
|
||||
# Check to make sure the cluster is up before running tests, and fail if it's not.
|
||||
go run ./hack/e2e.go ${E2E_OPT:-} -v --isup
|
||||
# Jenkins will look at the junit*.xml files for test failures, so don't exit with a nonzero
|
||||
# error code if it was only tests that failed.
|
||||
go run ./hack/e2e.go ${E2E_OPT:-} -v --test \
|
||||
${ginkgo_test_args:+--test_args="${ginkgo_test_args}"} \
|
||||
&& exitcode=0 || exitcode=$?
|
||||
if [[ "${E2E_PUBLISH_GREEN_VERSION:-}" == "true" && ${exitcode} == 0 ]]; then
|
||||
# Use plaintext version file packaged with kubernetes.tar.gz
|
||||
echo "Publish version to ci/latest-green.txt: $(cat version)"
|
||||
gsutil cp ./version "gs://${KUBE_GCS_DEV_RELEASE_BUCKET}/ci/latest-green.txt"
|
||||
fi
|
||||
return ${exitcode}
|
||||
}
|
||||
|
||||
echo "--------------------------------------------------------------------------------"
|
||||
echo "Test Environment:"
|
||||
printenv | sort
|
||||
@@ -360,47 +313,6 @@ case "${KUBERNETES_PROVIDER}" in
|
||||
;;
|
||||
esac
|
||||
|
||||
cd kubernetes
|
||||
|
||||
# Upload build start time and k8s version to GCS, but not on PR Jenkins.
|
||||
# On PR Jenkins this is done before the build.
|
||||
if [[ ! "${JOB_NAME}" =~ -pull- ]]; then
|
||||
JENKINS_BUILD_STARTED=true bash <(curl -fsS --retry 3 --keepalive-time 2 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh")
|
||||
fi
|
||||
|
||||
# When run inside Docker, we need to make sure all files are world-readable
|
||||
# (since they will be owned by root on the host).
|
||||
trap "chmod -R o+r '${ARTIFACTS}'" EXIT SIGINT SIGTERM
|
||||
export E2E_REPORT_DIR=${ARTIFACTS}
|
||||
declare -r gcp_list_resources_script="./cluster/gce/list-resources.sh"
|
||||
declare -r gcp_resources_before="${ARTIFACTS}/gcp-resources-before.txt"
|
||||
declare -r gcp_resources_cluster_up="${ARTIFACTS}/gcp-resources-cluster-up.txt"
|
||||
declare -r gcp_resources_after="${ARTIFACTS}/gcp-resources-after.txt"
|
||||
if [[ ( ${KUBERNETES_PROVIDER} == "gce" || ${KUBERNETES_PROVIDER} == "gke" ) && -x "${gcp_list_resources_script}" ]]; then
|
||||
gcp_list_resources="true"
|
||||
# Always pull the script from HEAD, overwriting the local one if it exists.
|
||||
# We do this to pick up fixes if we are running tests from a branch or tag.
|
||||
curl -fsS --retry 3 --keepalive-time 2 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/cluster/gce/list-resources.sh" > "${gcp_list_resources_script}"
|
||||
else
|
||||
gcp_list_resources="false"
|
||||
fi
|
||||
|
||||
### Set up ###
|
||||
if [[ "${E2E_UP,,}" == "true" ]]; then
|
||||
go run ./hack/e2e.go ${E2E_OPT:-} -v --down
|
||||
fi
|
||||
if [[ "${gcp_list_resources}" == "true" ]]; then
|
||||
${gcp_list_resources_script} > "${gcp_resources_before}"
|
||||
fi
|
||||
if [[ "${E2E_UP,,}" == "true" ]]; then
|
||||
# We want to try to gather logs even if kube-up fails, so collect the
|
||||
# result here and fail after dumping logs if it's nonzero.
|
||||
go run ./hack/e2e.go ${E2E_OPT:-} -v --up || dump_cluster_logs_and_exit
|
||||
go run ./hack/e2e.go -v --ctl="version --match-server-version=false"
|
||||
if [[ "${gcp_list_resources}" == "true" ]]; then
|
||||
${gcp_list_resources_script} > "${gcp_resources_cluster_up}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Allow download & unpack of alternate version of tests, for cross-version & upgrade testing.
|
||||
#
|
||||
@@ -414,87 +326,75 @@ fi
|
||||
# JENKINS_USE_SKEW_TESTS=true will run tests from the skewed version rather
|
||||
# than the original version.
|
||||
if [[ -n "${JENKINS_PUBLISHED_SKEW_VERSION:-}" ]]; then
|
||||
cd ..
|
||||
mv kubernetes kubernetes_old
|
||||
fetch_published_version_tars "${JENKINS_PUBLISHED_SKEW_VERSION}"
|
||||
cd kubernetes
|
||||
# Upgrade the cluster before running other tests
|
||||
if [[ "${E2E_UPGRADE_TEST:-}" == "true" ]]; then
|
||||
# Add a report prefix for the e2e tests so that the tests don't get overwritten when we run
|
||||
# the rest of the e2es.
|
||||
E2E_REPORT_PREFIX='upgrade' e2e_test "${GINKGO_UPGRADE_TEST_ARGS:-}" || EXIT_CODE=1
|
||||
fi
|
||||
if [[ "${JENKINS_USE_SKEW_TESTS:-}" != "true" ]]; then
|
||||
# Back out into the old tests now that we've downloaded & maybe upgraded.
|
||||
cd ../kubernetes_old
|
||||
# Append kubectl-path of skewed kubectl to test args, since we always
|
||||
# want that to use the skewed kubectl version:
|
||||
#
|
||||
# - for upgrade jobs, we want kubectl to be at the same version as master.
|
||||
# - for client skew tests, we want to use the skewed kubectl (that's what we're testing).
|
||||
GINKGO_TEST_ARGS="${GINKGO_TEST_ARGS:-} --kubectl-path=$(pwd)/../kubernetes/cluster/kubectl.sh"
|
||||
fi
|
||||
mv kubernetes kubernetes_orig
|
||||
fetch_published_version_tars "${JENKINS_PUBLISHED_SKEW_VERSION}"
|
||||
mv kubernetes kubernetes_skew
|
||||
mv kubernetes_orig kubernetes
|
||||
if [[ "${JENKINS_USE_SKEW_TESTS:-}" != "true" ]]; then
|
||||
# Append kubectl-path of skewed kubectl to test args, since we always
|
||||
# # want that to use the skewed kubectl version:
|
||||
# #
|
||||
# # - for upgrade jobs, we want kubectl to be at the same version as
|
||||
# master.
|
||||
# # - for client skew tests, we want to use the skewed kubectl
|
||||
# (that's what we're testing).
|
||||
GINKGO_TEST_ARGS="${GINKGO_TEST_ARGS:-} --kubectl-path=$(pwd)/../kubernetes_skew/cluster/kubectl.sh"
|
||||
fi
|
||||
fi
|
||||
|
||||
cd kubernetes
|
||||
|
||||
# Upload build start time and k8s version to GCS, but not on PR Jenkins.
|
||||
# On PR Jenkins this is done before the build.
|
||||
if [[ ! "${JOB_NAME}" =~ -pull- ]]; then
|
||||
JENKINS_BUILD_STARTED=true bash <(curl -fsS --retry 3 --keepalive-time 2 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh")
|
||||
fi
|
||||
|
||||
# When run inside Docker, we need to make sure all files are world-readable
|
||||
# (since they will be owned by root on the host).
|
||||
trap "chmod -R o+r '${ARTIFACTS}'" EXIT SIGINT SIGTERM
|
||||
export E2E_REPORT_DIR=${ARTIFACTS}
|
||||
|
||||
e2e_go_args=( \
|
||||
-v \
|
||||
--dump="${ARTIFACTS}" \
|
||||
)
|
||||
|
||||
|
||||
case "${KUBERNETES_PROVIDER}" in
|
||||
gce|gke)
|
||||
e2e_go_args+=(--check_leaked_resources)
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ "${E2E_UP,,}" == "true" ]]; then
|
||||
e2e_go_args+=(--up --ctl="version --match-server-version=false")
|
||||
fi
|
||||
|
||||
if [[ "${E2E_DOWN,,}" == "true" ]]; then
|
||||
e2e_go_args+=(--down)
|
||||
fi
|
||||
|
||||
if [[ "${E2E_TEST,,}" == "true" ]]; then
|
||||
e2e_test "${GINKGO_TEST_ARGS:-}" || EXIT_CODE=1
|
||||
e2e_go_args+=(--test --test_args="${GINKGO_TEST_ARGS}")
|
||||
fi
|
||||
|
||||
### Start Kubemark ###
|
||||
if [[ "${USE_KUBEMARK:-}" == "true" ]]; then
|
||||
export RUN_FROM_DISTRO=true
|
||||
NUM_NODES_BKP=${NUM_NODES}
|
||||
MASTER_SIZE_BKP=${MASTER_SIZE}
|
||||
./test/kubemark/stop-kubemark.sh
|
||||
NUM_NODES=${KUBEMARK_NUM_NODES:-$NUM_NODES}
|
||||
MASTER_SIZE=${KUBEMARK_MASTER_SIZE:-$MASTER_SIZE}
|
||||
./test/kubemark/start-kubemark.sh || dump_cluster_logs_and_exit
|
||||
# Similarly, if tests fail, we trigger empty set of tests that would trigger storing logs from the base cluster.
|
||||
# We intentionally overwrite the exit-code from `run-e2e-tests.sh` because we want jenkins to look at the
|
||||
# junit.xml results for test failures and not process the exit code. This is needed by jenkins to more gracefully
|
||||
# handle blocking the merge queue as a result of test failure flakes. Infrastructure failures should continue to
|
||||
# exit non-0.
|
||||
# TODO: The above comment is no longer accurate. Need to fix this before
|
||||
# turning xunit off for the postsubmit tests. See: #28200
|
||||
./test/kubemark/run-e2e-tests.sh --ginkgo.focus="${KUBEMARK_TESTS:-starting\s30\spods}" "${KUBEMARK_TEST_ARGS:-}" || dump_cluster_logs_and_exit
|
||||
./test/kubemark/stop-kubemark.sh
|
||||
NUM_NODES=${NUM_NODES_BKP}
|
||||
MASTER_SIZE=${MASTER_SIZE_BKP}
|
||||
unset RUN_FROM_DISTRO
|
||||
unset NUM_NODES_BKP
|
||||
unset MASTER_SIZE_BKP
|
||||
# Optionally run tests from the version in kubernetes_skew
|
||||
if [[ "${JENKINS_USE_SKEW_TESTS:-}" == "true" ]]; then
|
||||
e2e_go_args+=(--skew)
|
||||
fi
|
||||
|
||||
### Clean up ###
|
||||
if [[ "${E2E_DOWN,,}" == "true" ]]; then
|
||||
# Sleep before deleting the cluster to give the controller manager time to
|
||||
# delete any cloudprovider resources still around from the last test.
|
||||
# This is calibrated to allow enough time for 3 attempts to delete the
|
||||
# resources. Each attempt is allocated 5 seconds for requests to the
|
||||
# cloudprovider plus the processingRetryInterval from servicecontroller.go
|
||||
# for the wait between attempts.
|
||||
sleep 30
|
||||
go run ./hack/e2e.go ${E2E_OPT:-} -v --down
|
||||
fi
|
||||
if [[ "${gcp_list_resources}" == "true" ]]; then
|
||||
${gcp_list_resources_script} > "${gcp_resources_after}"
|
||||
# Optionally run upgrade tests before other tests.
|
||||
if [[ "${E2E_UPGRADE_TEST:-}" == "true" ]]; then
|
||||
e2e_go_args+=(--upgrade_args="${GINKGO_UPGRADE_TEST_ARGS}")
|
||||
fi
|
||||
|
||||
# Compare resources if either the cluster was
|
||||
# * started and destroyed (normal e2e)
|
||||
# * neither started nor destroyed (soak test)
|
||||
if [[ "${E2E_UP:-}" == "${E2E_DOWN:-}" && -f "${gcp_resources_before}" && -f "${gcp_resources_after}" ]]; then
|
||||
difference=$(diff -sw -U0 -F'^\[.*\]$' "${gcp_resources_before}" "${gcp_resources_after}") || true
|
||||
noleak=true
|
||||
if [[ -n $(echo "${difference}" | tail -n +3 | grep -E "^\+") ]] && [[ "${FAIL_ON_GCP_RESOURCE_LEAK:-}" == "true" ]]; then
|
||||
noleak=false
|
||||
fi
|
||||
if ! ${noleak} ; then
|
||||
echo "${difference}"
|
||||
echo "!!! FAIL: Google Cloud Platform resources leaked while running tests!"
|
||||
EXIT_CODE=1
|
||||
fi
|
||||
record_command "${STAGE_CLEANUP}" "gcp_resource_leak_check" ${noleak}
|
||||
fi
|
||||
go run ./hack/e2e.go \
|
||||
${E2E_OPT:-} \
|
||||
"${e2e_go_args[@]}"
|
||||
|
||||
exit ${EXIT_CODE}
|
||||
if [[ "${E2E_PUBLISH_GREEN_VERSION:-}" == "true" ]]; then
|
||||
# Use plaintext version file packaged with kubernetes.tar.gz
|
||||
echo "Publish version to ci/latest-green.txt: $(cat version)"
|
||||
gsutil cp ./version "gs://${KUBE_GCS_DEV_RELEASE_BUCKET}/ci/latest-green.txt"
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user