From 53c3a5626d9795413003e31ce6b9dfdb463fc133 Mon Sep 17 00:00:00 2001 From: Isaac Hollander McCreery Date: Mon, 14 Mar 2016 17:07:45 -0700 Subject: [PATCH] Add configuration for splitting upgrade jobs instead of using Jenkins steps --- hack/ginkgo-e2e.sh | 1 + hack/jenkins/e2e-runner.sh | 49 ++++++++++++----- .../kubernetes-jenkins/kubernetes-e2e.yaml | 52 ++++++++++++++++++- hack/verify-flags/known-flags.txt | 1 + test/e2e/e2e.go | 3 +- test/e2e/util.go | 1 + 6 files changed, 91 insertions(+), 16 deletions(-) diff --git a/hack/ginkgo-e2e.sh b/hack/ginkgo-e2e.sh index 8da5acb8fec..ae60dfba2ac 100755 --- a/hack/ginkgo-e2e.sh +++ b/hack/ginkgo-e2e.sh @@ -107,4 +107,5 @@ export PATH=$(dirname "${e2e_test}"):"${PATH}" ${E2E_CLEAN_START:+"--clean-start=true"} \ ${E2E_MIN_STARTUP_PODS:+"--minStartupPods=${E2E_MIN_STARTUP_PODS}"} \ ${E2E_REPORT_DIR:+"--report-dir=${E2E_REPORT_DIR}"} \ + ${E2E_REPORT_PREFIX:+"--report-prefix=${E2E_REPORT_PREFIX}"} \ "${@:-}" diff --git a/hack/jenkins/e2e-runner.sh b/hack/jenkins/e2e-runner.sh index f0126db4fcc..04f52780be0 100755 --- a/hack/jenkins/e2e-runner.sh +++ b/hack/jenkins/e2e-runner.sh @@ -152,6 +152,22 @@ if [[ "${JENKINS_USE_TRUSTY_IMAGES:-}" =~ ^[yY]$ ]]; then export KUBE_OS_DISTRIBUTION="trusty" fi +function e2e_test() { + local -r ginkgo_test_args="${1}" + # Check to make sure the cluster is up before running tests, and fail if it's not. + go run ./hack/e2e.go ${E2E_OPT:-} -v --isup + # Jenkins will look at the junit*.xml files for test failures, so don't exit with a nonzero + # error code if it was only tests that failed. + go run ./hack/e2e.go ${E2E_OPT:-} -v --test \ + ${ginkgo_test_args:+--test_args="${ginkgo_test_args}"} \ + && exitcode=0 || exitcode=$? + if [[ "${E2E_PUBLISH_GREEN_VERSION:-}" == "true" && ${exitcode} == 0 ]]; then + # Use plaintext version file packaged with kubernetes.tar.gz + echo "Publish version to ci/latest-green.txt: $(cat version)" + gsutil cp ./version gs://kubernetes-release/ci/latest-green.txt + fi +} + echo "--------------------------------------------------------------------------------" echo "Test Environment:" printenv | sort @@ -231,7 +247,7 @@ fi ### Pre Set Up ### # Install gcloud from a custom path if provided. Used to test GKE with gcloud # at HEAD, release candidate. -if [[ ! -z "${CLOUDSDK_BUCKET:-}" ]]; then +if [[ -n "${CLOUDSDK_BUCKET:-}" ]]; then gsutil -mq cp -r "${CLOUDSDK_BUCKET}" ~ rm -rf ~/repo ~/cloudsdk mv ~/$(basename "${CLOUDSDK_BUCKET}") ~/repo @@ -267,21 +283,28 @@ if [[ "${E2E_UP,,}" == "true" ]]; then fi fi -### Run tests ### -# Jenkins will look at the junit*.xml files for test failures, so don't exit -# with a nonzero error code if it was only tests that failed. -if [[ "${E2E_TEST,,}" == "true" ]]; then - # Check to make sure the cluster is up before running tests, and fail if it's not. - go run ./hack/e2e.go ${E2E_OPT:-} -v --isup - go run ./hack/e2e.go ${E2E_OPT:-} -v --test \ - ${GINKGO_TEST_ARGS:+--test_args="${GINKGO_TEST_ARGS}"} \ - && exitcode=0 || exitcode=$? - if [[ "${E2E_PUBLISH_GREEN_VERSION:-}" == "true" && ${exitcode} == 0 && -n ${build_version:-} ]]; then - echo "Publish build_version to ci/latest-green.txt: ${build_version}" - gsutil cp ./version gs://kubernetes-release/ci/latest-green.txt +# Allow download & unpack of alternate version of tests, for cross-version & upgrade testing. +if [[ -n "${JENKINS_PUBLISHED_TEST_VERSION:-}" ]]; then + cd .. + mv kubernetes kubernetes_old + fetch_published_version_tars "${JENKINS_PUBLISHED_TEST_VERSION}" + cd kubernetes + # Upgrade the cluster before running other tests + if [[ "${E2E_UPGRADE_TEST,,}" == "true" ]]; then + # Add a report prefix for the e2e tests so that the tests don't get overwritten when we run + # the rest of the e2es. + E2E_REPORT_PREFIX='upgrade' e2e_test "${GINKGO_UPGRADE_TEST_ARGS:-}" + # If JENKINS_USE_OLD_TESTS is set, back out into the old tests now that we've upgraded. + if [[ "${JENKINS_USE_OLD_TESTS:-}" == "true" ]]; then + cd ../kubernetes_old + fi fi fi +if [[ "${E2E_TEST,,}" == "true" ]]; then + e2e_test "${GINKGO_TEST_ARGS:-}" +fi + ### Start Kubemark ### if [[ "${USE_KUBEMARK:-}" == "true" ]]; then export RUN_FROM_DISTRO=true diff --git a/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-e2e.yaml b/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-e2e.yaml index 8be460fdfff..fa9aa951e4c 100644 --- a/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-e2e.yaml +++ b/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-e2e.yaml @@ -572,7 +572,6 @@ jobs: - 'kubernetes-e2e-{suffix}' -# ============================================================================== # NOTE: From here on all jobs use Trusty as the image for master and/or nodes. # Please add templates/groups/projects/jobs that use ContainerVm above/below # this section (search "End of Trusty jobs" for the ending separator). @@ -842,4 +841,53 @@ export KUBE_OS_DISTRIBUTION="trusty" jobs: - 'kubernetes-e2e-{suffix}' -#============================== End of Trusty jobs ============================= + +# End of Trusty jobs + +- project: + name: kubernetes-e2e-gke-upgrades-experimental + trigger-job: 'kubernetes-build' + test-owner: 'ihmccreery' + emails: 'ihmccreery@google.com' + provider-env: '{gke-provider-env}' + suffix: + - 'gke-kubectl-skew-1.1-1.2': + description: 'Deploys a cluster at v1.1 and runs the v1.2 Kubectl tests.' + timeout: 30 + job-env: | + export PROJECT="kube-jks-gke-upg-experimental" + export E2E_NAME="gke-ctl-skew-1-1-1-2" + export JENKINS_PUBLISHED_TEST_VERSION="ci/latest-1.2" + export GINKGO_TEST_ARGS="--ginkgo.focus=Kubectl" + export GINKGO_PARALLEL="y" + - 'gke-upgrade-master-1.1-1.2': + description: 'Deploys a cluster at v1.1, upgrades its master to v1.2, and runs v1.1 tests against it.' + timeout: 30 + job-env: | + export PROJECT="kube-jks-gke-upg-experimental" + export E2E_NAME="gke-upg-mas-1-1-1-2" + export JENKINS_PUBLISHED_TEST_VERSION="ci/latest-1.2" + export E2E_UPGRADE_TEST="true" + export GINKGO_UPGRADE_TEST_ARGS="--ginkgo.focus=\[Feature:MasterUpgrade\] --upgrade-target=ci/latest-1.2" + export JENKINS_USE_OLD_TESTS="true" + - 'gke-upgrade-cluster-1.1-1.2': + description: 'Deploys a cluster at v1.1, upgrades the cluster to v1.2, and runs v1.1 tests against it.' + timeout: 30 + job-env: | + export PROJECT="kube-jks-gke-upg-experimental" + export E2E_NAME="gke-upg-clu-1-1-1-2" + export JENKINS_PUBLISHED_TEST_VERSION="ci/latest-1.2" + export E2E_UPGRADE_TEST="true" + export GINKGO_UPGRADE_TEST_ARGS="--ginkgo.focus=\[Feature:ClusterUpgrade\] --upgrade-target=ci/latest-1.2" + export JENKINS_USE_OLD_TESTS="true" + - 'gke-upgrade-cluster-new-1.1-1.2': + description: 'Deploys a cluster at v1.1, upgrades the cluster to v1.2, and runs v1.2 tests against it.' + timeout: 30 + job-env: | + export PROJECT="kube-jks-gke-upg-experimental" + export E2E_NAME="gke-upg-clu-new-1-1-1-2" + export JENKINS_PUBLISHED_TEST_VERSION="ci/latest-1.2" + export E2E_UPGRADE_TEST="true" + export GINKGO_UPGRADE_TEST_ARGS="--ginkgo.focus=\[Feature:ClusterUpgrade\] --upgrade-target=ci/latest-1.2" + jobs: + - 'kubernetes-e2e-{suffix}' diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 496ffa3f11e..990b79cf0b8 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -334,6 +334,7 @@ replicaset-lookup-cache-size replication-controller-lookup-cache-size repo-root report-dir +report-prefix required-contexts resolv-conf resource-container diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index fd296e044bb..1ab91961b1b 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -71,6 +71,7 @@ func RegisterFlags() { flag.StringVar(&testContext.KubectlPath, "kubectl-path", "kubectl", "The kubectl binary to use. For development, you might use 'cluster/kubectl.sh' here.") flag.StringVar(&testContext.OutputDir, "e2e-output-dir", "/tmp", "Output directory for interesting/useful test data, like performance data, benchmarks, and other metrics.") flag.StringVar(&testContext.ReportDir, "report-dir", "", "Path to the directory where the JUnit XML reports should be saved. Default is empty, which doesn't generate these reports.") + flag.StringVar(&testContext.ReportPrefix, "report-prefix", "", "Optional prefix for JUnit XML reports. Default is empty, which doesn't prepend anything to the default name.") flag.StringVar(&testContext.prefix, "prefix", "e2e", "A prefix to be added to cloud resources created during testing.") flag.StringVar(&testContext.OSDistro, "os-distro", "debian", "The OS distribution of cluster VM instances (debian, trusty, or coreos).") @@ -269,7 +270,7 @@ func RunE2ETests(t *testing.T) { if err := os.MkdirAll(testContext.ReportDir, 0755); err != nil { glog.Errorf("Failed creating report directory: %v", err) } else { - r = append(r, reporters.NewJUnitReporter(path.Join(testContext.ReportDir, fmt.Sprintf("junit_%02d.xml", config.GinkgoConfig.ParallelNode)))) + r = append(r, reporters.NewJUnitReporter(path.Join(testContext.ReportDir, fmt.Sprintf("junit_%v%02d.xml", testContext.ReportPrefix, config.GinkgoConfig.ParallelNode)))) } } glog.Infof("Starting e2e run %q on Ginkgo node %d", runId, config.GinkgoConfig.ParallelNode) diff --git a/test/e2e/util.go b/test/e2e/util.go index 7338c1fb3a9..96ae79c0465 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -186,6 +186,7 @@ type TestContextType struct { KubectlPath string OutputDir string ReportDir string + ReportPrefix string prefix string MinStartupPods int UpgradeTarget string