diff --git a/hack/jenkins/e2e.sh b/hack/jenkins/e2e.sh index e7d33c17e09..8256ce0c339 100755 --- a/hack/jenkins/e2e.sh +++ b/hack/jenkins/e2e.sh @@ -47,24 +47,6 @@ function join_regex_no_empty() { fi } -# GCP Project to fetch Trusty images. -TRUSTY_IMAGE_PROJECT="${TRUSTY_IMAGE_PROJECT:-}" - -# Get the latest Trusty image for a Jenkins job. -function get_latest_trusty_image() { - local job_name="${1:-${JOB_NAME}}" - local image_index="" - if [[ "${job_name}" =~ trusty-beta ]]; then - image_index="trusty-beta" - elif [[ "${job_name}" =~ trusty ]]; then - image_index="trusty" - fi - gsutil cat "gs://${TRUSTY_IMAGE_PROJECT}/image-indices/latest-test-image-${image_index}" - # Clean up gsutil artifacts otherwise the later test stage will complain. - rm -rf .config &> /dev/null - rm -rf .gsutil &> /dev/null -} - # Properly configure globals for an upgrade step in a GKE or GCE upgrade suite # # These suites: @@ -399,28 +381,10 @@ GCE_SOAK_CONTINUOUS_SKIP_TESTS=( GCE_RELEASE_SKIP_TESTS=( ) -TRUSTY_DEFAULT_SKIP_TESTS=( - # TODO(wonderfly): Remove this once - # https://github.com/kubernetes/kubernetes/issues/12689 is fixed. - "Services.*should\swork\safter\srestarting\skube-proxy" - # This test was broken and fixed after v1.1.1 release cut. As Trusty release - # tests pin to the latest release version of k8s, we'd have to wait till - # next release to pick up the fix. - # https://github.com/kubernetes/kubernetes/pull/14603 - # TODO(wonderfly): Remove this once v1.1.2 is out. - "Kubelet\sregular\sresource\susage\stracking\sover\s30m0s\swith\s0\spods\sper\snode" -) - -TRUSTY_SKIP_TESTS=( - "${TRUSTY_DEFAULT_SKIP_TESTS[@]}" - "Monitoring\sshould\sverify\smonitoring\spods\sand\sall\scluster\snodes\sare\savailable\son\sinfluxdb\susing\sheapster" -) - -TRUSTY_BETA_SKIP_TESTS=( - "${TRUSTY_DEFAULT_SKIP_TESTS[@]}" -) - # Define environment variables based on the Jenkins project name. +# NOTE: Not all jobs are defined here. The hack/jenkins/e2e.sh in master and +# release branches defines relevant jobs for that particular version of +# Kubernetes. case ${JOB_NAME} in # Runs all non-flaky, non-slow tests on GCE, sequentially. kubernetes-e2e-gce) @@ -606,28 +570,6 @@ case ${JOB_NAME} in TEST_CLUSTER_RESYNC_PERIOD="--min-resync-period=12h" ;; - # Runs the performance/scalability tests on GCE using the latest 1.1 ci - # release. A larger cluster is used. - kubernetes-e2e-gce-scalability-1.1) - : ${E2E_CLUSTER_NAME:="jenkins-gce-e2e-scalability-1.1"} - : ${E2E_NETWORK:="e2e-scalability-1-1"} - : ${GINKGO_TEST_ARGS:="--ginkgo.focus=\[Performance\]"} - : ${JENKINS_PUBLISHED_VERSION:="ci/latest-1.1"} - : ${KUBE_GCE_INSTANCE_PREFIX:="e2e-scalability-1-1"} - : ${KUBE_GCS_STAGING_PATH_SUFFIX:="scalability-1.1"} - : ${PROJECT:="k8s-e2e-gce-scalability-1-1"} - # Override GCE defaults. - E2E_ZONE="us-east1-b" - MASTER_SIZE="n1-standard-4" - NODE_SIZE="n1-standard-2" - NODE_DISK_SIZE="50GB" - NUM_NODES="100" - # Reduce logs verbosity - TEST_CLUSTER_LOG_LEVEL="--v=2" - # Increase resync period to simulate production - TEST_CLUSTER_RESYNC_PERIOD="--min-resync-period=12h" - ;; - # Sets up the GCE soak cluster weekly using the latest CI release. kubernetes-soak-weekly-deploy-gce) : ${E2E_CLUSTER_NAME:="gce-soak-weekly"} @@ -656,249 +598,6 @@ case ${JOB_NAME} in : ${PROJECT:="kubernetes-jenkins"} ;; - # Sets up the GCE soak cluster weekly using the latest 1.1 ci release. - kubernetes-soak-weekly-deploy-gce-1.1) - : ${E2E_CLUSTER_NAME:="gce-soak-weekly-1.1"} - : ${E2E_DOWN:="false"} - : ${E2E_NETWORK:="gce-soak-weekly-1-1"} - : ${E2E_TEST:="false"} - : ${E2E_UP:="true"} - : ${JENKINS_PUBLISHED_VERSION:="ci/latest-1.1"} - : ${KUBE_GCE_INSTANCE_PREFIX:="gce-soak-weekly-1-1"} - : ${KUBE_GCS_STAGING_PATH_SUFFIX:="soak-1.1"} - : ${PROJECT:="kubernetes-jenkins"} - : ${FAIL_ON_GCP_RESOURCE_LEAK:="true"} - ;; - - # Runs tests on GCE soak cluster for latest 1.1 ci release. - kubernetes-soak-continuous-e2e-gce-1.1) - : ${E2E_CLUSTER_NAME:="gce-soak-weekly-1.1"} - : ${E2E_DOWN:="false"} - : ${E2E_NETWORK:="gce-soak-weekly-1-1"} - : ${E2E_UP:="false"} - # Clear out any orphaned namespaces in case previous run was interrupted. - : ${E2E_CLEAN_START:="true"} - : ${GINKGO_TEST_ARGS:="--ginkgo.skip=$(join_regex_allow_empty \ - ${GCE_DEFAULT_SKIP_TESTS[@]:+${GCE_DEFAULT_SKIP_TESTS[@]}} \ - ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}} \ - ${GCE_SOAK_CONTINUOUS_SKIP_TESTS[@]:+${GCE_SOAK_CONTINUOUS_SKIP_TESTS[@]}} \ - )"} - : ${KUBE_GCE_INSTANCE_PREFIX:="gce-soak-weekly-1-1"} - : ${PROJECT:="kubernetes-jenkins"} - ;; - - # Runs non-flaky tests on GCE on the release-1.0 branch, - # sequentially. As a reminder, if you need to change the skip list - # or flaky test list on the release branch, you'll need to propose a - # pull request directly to the release branch itself. - kubernetes-e2e-gce-release-1.0) - : ${E2E_CLUSTER_NAME:="jenkins-gce-e2e-release-1.0"} - : ${E2E_DOWN:="false"} - : ${E2E_NETWORK:="e2e-gce-release-1-0"} - : ${GINKGO_TEST_ARGS:="--ginkgo.skip=$(join_regex_allow_empty \ - ${GCE_DEFAULT_SKIP_TESTS[@]:+${GCE_DEFAULT_SKIP_TESTS[@]}} \ - ${GCE_RELEASE_SKIP_TESTS[@]:+${GCE_RELEASE_SKIP_TESTS[@]}} \ - ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}} \ - )"} - : ${KUBE_GCE_INSTANCE_PREFIX="e2e-gce-1-0"} - : ${KUBE_GCS_STAGING_PATH_SUFFIX:="release-1.0"} - : ${PROJECT:="k8s-jkns-e2e-gce-release"} - ;; - - # Runs non-flaky tests on GCE with Trusty as base image for minions, - # sequentially. - kubernetes-e2e-gce-trusty-release) - : ${E2E_CLUSTER_NAME:="jenkins-gce-e2e-trusty-release"} - : ${E2E_DOWN:="false"} - : ${E2E_NETWORK:="e2e-gce-trusty-release"} - : ${GINKGO_TEST_ARGS:="--ginkgo.skip=$(join_regex_allow_empty \ - ${GCE_DEFAULT_SKIP_TESTS[@]:+${GCE_DEFAULT_SKIP_TESTS[@]}} \ - ${GCE_RELEASE_SKIP_TESTS[@]:+${GCE_RELEASE_SKIP_TESTS[@]}} \ - ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}} \ - ${GCE_SLOW_TESTS[@]:+${GCE_SLOW_TESTS[@]}} \ - ${TRUSTY_SKIP_TESTS[@]:+${TRUSTY_SKIP_TESTS[@]}} \ - )"} - : ${KUBE_GCE_INSTANCE_PREFIX="e2e-gce"} - : ${PROJECT:="kubekins-e2e-gce-trusty-rls"} - : ${KUBE_GCE_NODE_PROJECT:="${TRUSTY_IMAGE_PROJECT}"} - : ${KUBE_GCE_NODE_IMAGE:="$(get_latest_trusty_image ${JOB_NAME})"} - : ${KUBE_OS_DISTRIBUTION:="trusty"} - : ${ENABLE_CLUSTER_REGISTRY:=false} - : ${JENKINS_EXPLICIT_VERSION:="release/v1.1.1"} - ;; - - # Runs slow tests on GCE with Trusy as base image for minions, sequentially. - kubernetes-e2e-gce-trusty-slow) - : ${E2E_CLUSTER_NAME:="jenkins-gce-e2e-trusty-slow"} - : ${E2E_NETWORK:="e2e-gce-trusty-slow"} - : ${GINKGO_TEST_ARGS:="--ginkgo.focus=$(join_regex_no_empty \ - ${GCE_SLOW_TESTS[@]:+${GCE_SLOW_TESTS[@]}} \ - ) --ginkgo.skip=$(join_regex_allow_empty \ - ${TRUSTY_SKIP_TESTS[@]:+${TRUSTY_SKIP_TESTS[@]}} \ - )"} - : ${KUBE_GCE_INSTANCE_PREFIX="e2e-trusty-slow"} - : ${PROJECT:="k8s-e2e-gce-trusty-slow"} - : ${KUBE_GCE_NODE_PROJECT:="${TRUSTY_IMAGE_PROJECT}"} - : ${KUBE_GCE_NODE_IMAGE:="$(get_latest_trusty_image ${JOB_NAME})"} - : ${KUBE_OS_DISTRIBUTION:="trusty"} - : ${ENABLE_CLUSTER_REGISTRY:=false} - : ${JENKINS_EXPLICIT_VERSION:="release/v1.1.1"} - : ${FAIL_ON_GCP_RESOURCE_LEAK:="true"} - ;; - - # Runs non-flaky tests on GCE with Trusty-beta as base image for minions, - # sequentially. - kubernetes-e2e-gce-trusty-beta-release) - : ${E2E_CLUSTER_NAME:="jenkins-gce-e2e-trusty-beta-release"} - : ${E2E_DOWN:="false"} - : ${E2E_NETWORK:="e2e-gce-trusty-beta-release"} - : ${GINKGO_TEST_ARGS:="--ginkgo.skip=$(join_regex_allow_empty \ - ${GCE_DEFAULT_SKIP_TESTS[@]:+${GCE_DEFAULT_SKIP_TESTS[@]}} \ - ${GCE_RELEASE_SKIP_TESTS[@]:+${GCE_RELEASE_SKIP_TESTS[@]}} \ - ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}} \ - ${GCE_SLOW_TESTS[@]:+${GCE_SLOW_TESTS[@]}} \ - ${TRUSTY_BETA_SKIP_TESTS[@]:+${TRUSTY_BETA_SKIP_TESTS[@]}} \ - )"} - : ${KUBE_GCE_INSTANCE_PREFIX="e2e-gce"} - : ${PROJECT:="k8s-e2e-gce-trusty-beta"} - : ${KUBE_GCE_NODE_PROJECT:="${TRUSTY_IMAGE_PROJECT}"} - : ${KUBE_GCE_NODE_IMAGE:="$(get_latest_trusty_image ${JOB_NAME})"} - : ${KUBE_OS_DISTRIBUTION:="trusty"} - : ${ENABLE_CLUSTER_REGISTRY:=false} - : ${JENKINS_EXPLICIT_VERSION:="release/v1.1.1"} - ;; - - # Runs slow tests on GCE with Trusy-beta as base image for minions, - # sequentially. - kubernetes-e2e-gce-trusty-beta-slow) - : ${E2E_CLUSTER_NAME:="jenkins-gce-e2e-trusty-beta-slow"} - : ${E2E_NETWORK:="e2e-gce-trusty-beta-slow"} - : ${GINKGO_TEST_ARGS:="--ginkgo.focus=$(join_regex_no_empty \ - ${GCE_SLOW_TESTS[@]:+${GCE_SLOW_TESTS[@]}} \ - ) --ginkgo.skip=$(join_regex_allow_empty \ - ${TRUSTY_BETA_SKIP_TESTS[@]:+${TRUSTY_BETA_SKIP_TESTS[@]}} \ - )"} - : ${KUBE_GCE_INSTANCE_PREFIX="e2e-trusty-beta-slow"} - : ${PROJECT:="k8s-e2e-gce-trusty-beta-slow"} - : ${KUBE_GCE_NODE_PROJECT:="${TRUSTY_IMAGE_PROJECT}"} - : ${KUBE_GCE_NODE_IMAGE:="$(get_latest_trusty_image ${JOB_NAME})"} - : ${KUBE_OS_DISTRIBUTION:="trusty"} - : ${ENABLE_CLUSTER_REGISTRY:=false} - : ${JENKINS_EXPLICIT_VERSION:="release/v1.1.1"} - : ${FAIL_ON_GCP_RESOURCE_LEAK:="true"} - ;; - - # Runs non-flaky tests on GCE on the release candidate branch, - # sequentially. As a reminder, if you need to change the skip list - # or flaky test list on the release branch, you'll need to propose a - # pull request directly to the release branch itself. - kubernetes-e2e-gce-release-1.1) - : ${E2E_CLUSTER_NAME:="jenkins-gce-e2e-release-1.1"} - : ${E2E_DOWN:="false"} - : ${E2E_NETWORK:="gce-e2e-1-1"} - : ${GINKGO_TEST_ARGS:="--ginkgo.skip=$(join_regex_allow_empty \ - ${GCE_DEFAULT_SKIP_TESTS[@]:+${GCE_DEFAULT_SKIP_TESTS[@]}} \ - ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}} \ - ${GCE_SLOW_TESTS[@]:+${GCE_SLOW_TESTS[@]}} \ - )"} - : ${JENKINS_PUBLISHED_VERSION:="ci/latest-1.1"} - : ${KUBE_GCE_INSTANCE_PREFIX="e2e-gce-1-1"} - : ${KUBE_GCS_STAGING_PATH_SUFFIX:="release-1.1"} - : ${PROJECT:="k8s-jkns-e2e-gce-release"} - : ${ENABLE_DEPLOYMENTS:=true} - ;; - - # Runs "disruptive" tests on GCE on the release candidate branch, - # sequentially, against the latest 1.1 ci release. - kubernetes-e2e-gce-disruptive-1.1) - : ${E2E_CLUSTER_NAME:="jenkins-gce-e2e-disruptive-1.1"} - : ${E2E_DOWN:="false"} - : ${E2E_NETWORK:="gce-e2e-disruptive-1-1"} - : ${GINKGO_TEST_ARGS:="--ginkgo.skip=$(join_regex_allow_empty \ - ${REBOOT_SKIP_TESTS[@]:+${REBOOT_SKIP_TESTS[@]}}\ - ) --ginkgo.focus=$(join_regex_no_empty \ - ${DISRUPTIVE_TESTS[@]:+${DISRUPTIVE_TESTS[@]}} \ - "\[Autoscaling\]\sReplicationController" \ - "GCE\sL7\sLoadBalancer\sController" - )"} - : ${JENKINS_PUBLISHED_VERSION:="ci/latest-1.1"} - : ${KUBE_GCE_INSTANCE_PREFIX="e2e-gce-disruptive-1-1"} - : ${KUBE_GCS_STAGING_PATH_SUFFIX:="disruptive-1.1"} - # TODO: move into its own project - : ${PROJECT:="kubernetes-jenkins"} - : ${ENABLE_DEPLOYMENTS:=true} - : ${ENABLE_DAEMONSETS:=true} - ;; - - kubernetes-e2e-gke-subnet) - : ${DOGFOOD_GCLOUD:="true"} - : ${E2E_CLUSTER_NAME:="jkns-gke-subnet"} - # auto-subnet manually created - if deleted, it will need to be recreated - # gcloud alpha compute networks create auto-subnet --mode auto - : ${E2E_NETWORK:="auto-subnet"} - : ${E2E_SET_CLUSTER_API_VERSION:=y} - : ${JENKINS_USE_SERVER_VERSION:=y} - : ${PROJECT:="k8s-jkns-e2e-gke-subnet"} - : ${FAIL_ON_GCP_RESOURCE_LEAK:="true"} - : ${GINKGO_TEST_ARGS:="--ginkgo.skip=$(join_regex_allow_empty \ - ${GKE_DEFAULT_SKIP_TESTS[@]:+${GKE_DEFAULT_SKIP_TESTS[@]}} \ - ${GCE_DEFAULT_SKIP_TESTS[@]:+${GCE_DEFAULT_SKIP_TESTS[@]}} \ - ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}} \ - ${GKE_FLAKY_TESTS[@]:+${GKE_FLAKY_TESTS[@]}} \ - )"} - ;; - - kubernetes-e2e-gke-prod) - : ${DOGFOOD_GCLOUD:="true"} - : ${E2E_CLUSTER_NAME:="jkns-gke-e2e-prod"} - : ${E2E_NETWORK:="e2e-gke-prod"} - : ${E2E_SET_CLUSTER_API_VERSION:=y} - : ${JENKINS_USE_SERVER_VERSION:=y} - : ${PROJECT:="k8s-jkns-e2e-gke-prod"} - : ${FAIL_ON_GCP_RESOURCE_LEAK:="true"} - : ${GINKGO_TEST_ARGS:="--ginkgo.skip=$(join_regex_allow_empty \ - ${GKE_DEFAULT_SKIP_TESTS[@]:+${GKE_DEFAULT_SKIP_TESTS[@]}} \ - ${GCE_DEFAULT_SKIP_TESTS[@]:+${GCE_DEFAULT_SKIP_TESTS[@]}} \ - ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}} \ - ${GKE_FLAKY_TESTS[@]:+${GKE_FLAKY_TESTS[@]}} \ - )"} - ;; - - kubernetes-e2e-gke-staging) - : ${DOGFOOD_GCLOUD:="true"} - : ${GKE_API_ENDPOINT:="https://staging-container.sandbox.googleapis.com/"} - : ${E2E_CLUSTER_NAME:="jkns-gke-e2e-staging"} - : ${E2E_NETWORK:="e2e-gke-staging"} - : ${E2E_SET_CLUSTER_API_VERSION:=y} - : ${JENKINS_USE_SERVER_VERSION:=y} - : ${PROJECT:="k8s-jkns-e2e-gke-staging"} - : ${FAIL_ON_GCP_RESOURCE_LEAK:="true"} - : ${GINKGO_TEST_ARGS:="--ginkgo.skip=$(join_regex_allow_empty \ - ${GKE_DEFAULT_SKIP_TESTS[@]:+${GKE_DEFAULT_SKIP_TESTS[@]}} \ - ${GCE_DEFAULT_SKIP_TESTS[@]:+${GCE_DEFAULT_SKIP_TESTS[@]}} \ - ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}} \ - ${GKE_FLAKY_TESTS[@]:+${GKE_FLAKY_TESTS[@]}} \ - )"} - ;; - - kubernetes-e2e-gke-test) - : ${DOGFOOD_GCLOUD:="true"} - : ${CLOUDSDK_BUCKET:="gs://cloud-sdk-build/testing/rc"} - : ${GKE_API_ENDPOINT:="https://test-container.sandbox.googleapis.com/"} - : ${E2E_CLUSTER_NAME:="jkns-gke-e2e-test"} - : ${E2E_NETWORK:="e2e-gke-test"} - : ${E2E_SET_CLUSTER_API_VERSION:=y} - : ${JENKINS_USE_SERVER_VERSION:=y} - : ${PROJECT:="k8s-jkns-e2e-gke-test"} - : ${FAIL_ON_GCP_RESOURCE_LEAK:="true"} - : ${GINKGO_TEST_ARGS:="--ginkgo.skip=$(join_regex_allow_empty \ - ${GKE_DEFAULT_SKIP_TESTS[@]:+${GKE_DEFAULT_SKIP_TESTS[@]}} \ - ${GCE_DEFAULT_SKIP_TESTS[@]:+${GCE_DEFAULT_SKIP_TESTS[@]}} \ - ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}} \ - ${GKE_FLAKY_TESTS[@]:+${GKE_FLAKY_TESTS[@]}} \ - )"} - ;; - kubernetes-e2e-gke-ci) : ${DOGFOOD_GCLOUD:="true"} : ${CLOUDSDK_BUCKET:="gs://cloud-sdk-build/testing/staging"} @@ -950,40 +649,6 @@ case ${JOB_NAME} in )"} ;; - kubernetes-e2e-gke-1.1) - : ${DOGFOOD_GCLOUD:="true"} - : ${GKE_API_ENDPOINT:="https://test-container.sandbox.googleapis.com/"} - : ${E2E_CLUSTER_NAME:="gke-release-1-1"} - : ${E2E_NETWORK:="gke-release-1-1"} - : ${E2E_SET_CLUSTER_API_VERSION:=y} - : ${JENKINS_PUBLISHED_VERSION:="ci/latest-1.1"} - : ${PROJECT:="k8s-jkns-e2e-gke-release-1-1"} - : ${FAIL_ON_GCP_RESOURCE_LEAK:="true"} - # Note Daemon sets and Deployments are experimental features in 1.1, - # and are turned off in GKE. - : ${GINKGO_TEST_ARGS:="--ginkgo.skip=$(join_regex_allow_empty \ - ${GKE_DEFAULT_SKIP_TESTS[@]:+${GKE_DEFAULT_SKIP_TESTS[@]}} \ - ${GCE_DEFAULT_SKIP_TESTS[@]:+${GCE_DEFAULT_SKIP_TESTS[@]}} \ - ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}} \ - ${GKE_FLAKY_TESTS[@]:+${GKE_FLAKY_TESTS[@]}} \ - )"} - ;; - - kubernetes-e2e-gke-1.1-features) - : ${DOGFOOD_GCLOUD:="true"} - : ${GKE_API_ENDPOINT:="https://test-container.sandbox.googleapis.com/"} - : ${E2E_CLUSTER_NAME:="gke-1-1-features"} - : ${E2E_NETWORK:="gke-1-1-features"} - : ${E2E_SET_CLUSTER_API_VERSION:=y} - : ${JENKINS_PUBLISHED_VERSION:="ci/latest-1.1"} - : ${PROJECT:="k8s-jkns-e2e-gke-1-1-features"} - : ${FAIL_ON_GCP_RESOURCE_LEAK:="true"} - : ${GINKGO_TEST_ARGS:="--ginkgo.focus=GCE\sL7\sLoadBalancer\sController|Job|Horizontal\spod\sautoscaling"} - # At least n1-standard-2 nodes are required for the cluster to - # have enough cpu/ram to run the Horizontal pod autoscaling tests. - NODE_SIZE="n1-standard-2" - ;; - # Sets up the GKE soak cluster weekly using the latest CI release. kubernetes-soak-weekly-deploy-gke) : ${GKE_API_ENDPOINT:="https://test-container.sandbox.googleapis.com/"}