diff --git a/cluster/gke/config-common.sh b/cluster/gke/config-common.sh index 790c26b2ded..0b4dbea69dd 100644 --- a/cluster/gke/config-common.sh +++ b/cluster/gke/config-common.sh @@ -25,7 +25,7 @@ NETWORK="${NETWORK:-default}" NETWORK_RANGE="${NETWORK_RANGE:-10.240.0.0/16}" FIREWALL_SSH="${FIREWALL_SSH:-${NETWORK}-allow-ssh}" GCLOUD="${GCLOUD:-gcloud}" -CMD_GROUP="${CMD_GROUP:-beta}" +CMD_GROUP="${CMD_GROUP:-}" GCLOUD_CONFIG_DIR="${GCLOUD_CONFIG_DIR:-${HOME}/.config/gcloud/kubernetes}" MINION_SCOPES="${MINION_SCOPES:-"compute-rw,storage-ro"}" MACHINE_TYPE="${MACHINE_TYPE:-n1-standard-1}" diff --git a/cluster/gke/util.sh b/cluster/gke/util.sh index b2812adae0f..37c582a3358 100755 --- a/cluster/gke/util.sh +++ b/cluster/gke/util.sh @@ -99,7 +99,7 @@ function verify-prereqs() { sudo_prefix="sudo" fi ${sudo_prefix} gcloud ${gcloud_prompt:-} components update preview || true - ${sudo_prefix} gcloud ${gcloud_prompt:-} components update "${CMD_GROUP}"|| true + ${sudo_prefix} gcloud ${gcloud_prompt:-} components update ${CMD_GROUP:-} || true ${sudo_prefix} gcloud ${gcloud_prompt:-} components update kubectl|| true ${sudo_prefix} gcloud ${gcloud_prompt:-} components update || true } @@ -150,7 +150,7 @@ function kube-up() { ) # Bring up the cluster. - "${GCLOUD}" "${CMD_GROUP}" container clusters create "${CLUSTER_NAME}" "${create_args[@]}" + "${GCLOUD}" ${CMD_GROUP:-} container clusters create "${CLUSTER_NAME}" "${create_args[@]}" } # Execute prior to running tests to initialize required structure. This is @@ -200,7 +200,7 @@ function test-setup() { function detect-master() { echo "... in gke:detect-master()" >&2 detect-project >&2 - KUBE_MASTER_IP=$("${GCLOUD}" "${CMD_GROUP}" container clusters describe \ + KUBE_MASTER_IP=$("${GCLOUD}" ${CMD_GROUP:-} container clusters describe \ --project="${PROJECT}" --zone="${ZONE}" "${CLUSTER_NAME}" \ | grep endpoint | cut -f 2 -d ' ') } @@ -242,7 +242,7 @@ function detect-minion-names { # NODE_INSTANCE_GROUP function detect-node-instance-group { echo "... in gke:detect-node-instance-group()" >&2 - NODE_INSTANCE_GROUP=$("${GCLOUD}" "${CMD_GROUP}" container clusters describe \ + NODE_INSTANCE_GROUP=$("${GCLOUD}" ${CMD_GROUP:-} container clusters describe \ --project="${PROJECT}" --zone="${ZONE}" "${CLUSTER_NAME}" \ | grep instanceGroupManagers | cut -d '/' -f 11) } @@ -318,6 +318,6 @@ function test-teardown() { function kube-down() { echo "... in gke:kube-down()" >&2 detect-project >&2 - "${GCLOUD}" "${CMD_GROUP}" container clusters delete --project="${PROJECT}" \ + "${GCLOUD}" ${CMD_GROUP:-} container clusters delete --project="${PROJECT}" \ --zone="${ZONE}" "${CLUSTER_NAME}" --quiet } diff --git a/hack/jenkins/e2e.sh b/hack/jenkins/e2e.sh index bec74401388..0c9626f14e2 100755 --- a/hack/jenkins/e2e.sh +++ b/hack/jenkins/e2e.sh @@ -70,6 +70,10 @@ if [[ ${JOB_NAME} =~ ^kubernetes-.*-gce ]]; then : ${E2E_MIN_STARTUP_PODS:="1"} : ${E2E_ZONE:="us-central1-f"} : ${NUM_MINIONS_PARALLEL:="6"} # Number of nodes required to run all of the tests in parallel + +elif [[ ${JOB_NAME} =~ ^kubernetes-.*-gke ]]; then + KUBERNETES_PROVIDER="gke" + : ${E2E_ZONE:="us-central1-f"} fi if [[ "${KUBERNETES_PROVIDER}" == "aws" ]]; then @@ -84,8 +88,8 @@ if [[ "${KUBERNETES_PROVIDER}" == "aws" ]]; then fi fi -# Specialized tests which should be skipped by default for projects. -GCE_DEFAULT_SKIP_TESTS=( +# Specialized to skip when running reboot tests. +REBOOT_SKIP_TESTS=( "Autoscaling\sSuite" "Skipped" "Reboot" @@ -93,6 +97,20 @@ GCE_DEFAULT_SKIP_TESTS=( "Example" ) +# Specialized tests which should be skipped by default for projects. +GCE_DEFAULT_SKIP_TESTS=( + "${REBOOT_SKIP_TESTS[@]}" + "Reboot") + +# Tests which cannot be run on GKE, e.g. because they require +# master ssh access. +GKE_REQUIRED_SKIP_TESTS=( + "Nodes" + "Etcd\sFailure" + "MasterCerts" + "Shell" + ) + # The following tests are known to be flaky, and are thus run only in their own # -flaky- build variants. GCE_FLAKY_TESTS=( @@ -345,6 +363,81 @@ case ${JOB_NAME} in : ${KUBE_GCE_INSTANCE_PREFIX="e2e-gce"} : ${PROJECT:="k8s-jkns-e2e-gce-release"} ;; + + kubernetes-e2e-gke-prod) + : ${DOGFOOD_GCLOUD:="true"} + : ${E2E_CLUSTER_NAME:="jkns-gke-e2e-prod"} + : ${E2E_NETWORK:="e2e-gke-prod"} + : ${E2E_SET_CLUSTER_API_VERSION:=y} + : ${JENKINS_USE_SERVER_VERSION:=y} + : ${PROJECT:="k8s-jkns-e2e-gke-prod"} + : ${GINKGO_TEST_ARGS:="--ginkgo.skip=$(join_regex_allow_empty \ + ${GKE_REQUIRED_SKIP_TESTS[@]:+${GKE_REQUIRED_SKIP_TESTS[@]}} \ + ${GCE_DEFAULT_SKIP_TESTS[@]:+${GCE_DEFAULT_SKIP_TESTS[@]}} \ + ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}} \ + )"} + ;; + + kubernetes-e2e-gke-staging) + : ${DOGFOOD_GCLOUD:="true"} + : ${GKE_API_ENDPOINT:="https://staging-container.sandbox.googleapis.com/"} + : ${E2E_CLUSTER_NAME:="jkns-gke-e2e-staging"} + : ${E2E_NETWORK:="e2e-gke-staging"} + : ${E2E_SET_CLUSTER_API_VERSION:=y} + : ${JENKINS_USE_SERVER_VERSION:=y} + : ${PROJECT:="k8s-jkns-e2e-gke-staging"} + : ${GINKGO_TEST_ARGS:="--ginkgo.skip=$(join_regex_allow_empty \ + ${GKE_REQUIRED_SKIP_TESTS[@]:+${GKE_REQUIRED_SKIP_TESTS[@]}} \ + ${GCE_DEFAULT_SKIP_TESTS[@]:+${GCE_DEFAULT_SKIP_TESTS[@]}} \ + ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}} \ + )"} + ;; + + kubernetes-e2e-gke-test) + : ${DOGFOOD_GCLOUD:="true"} + : ${CLOUDSDK_BUCKET:="gs://cloud-sdk-build/testing/rc"} + : ${GKE_API_ENDPOINT:="https://test-container.sandbox.googleapis.com/"} + : ${E2E_CLUSTER_NAME:="jkns-gke-e2e-test"} + : ${E2E_NETWORK:="e2e-gke-test"} + : ${JENKINS_USE_RELEASE_TARS:=y} + : ${PROJECT:="k8s-jkns-e2e-gke-ci"} + : ${GINKGO_TEST_ARGS:="--ginkgo.skip=$(join_regex_allow_empty \ + ${GKE_REQUIRED_SKIP_TESTS[@]:+${GKE_REQUIRED_SKIP_TESTS[@]}} \ + ${GCE_DEFAULT_SKIP_TESTS[@]:+${GCE_DEFAULT_SKIP_TESTS[@]}} \ + ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}} \ + )"} + ;; + + kubernetes-e2e-gke-ci) + : ${DOGFOOD_GCLOUD:="true"} + : ${CLOUDSDK_BUCKET:="gs://cloud-sdk-build/testing/staging"} + : ${GKE_API_ENDPOINT:="https://test-container.sandbox.googleapis.com/"} + : ${E2E_CLUSTER_NAME:="jkns-gke-e2e-ci"} + : ${E2E_NETWORK:="e2e-gke-ci"} + : ${E2E_SET_CLUSTER_API_VERSION:=y} + : ${PROJECT:="k8s-jkns-e2e-gke-ci"} + : ${GINKGO_TEST_ARGS:="--ginkgo.skip=$(join_regex_allow_empty \ + ${GKE_REQUIRED_SKIP_TESTS[@]:+${GKE_REQUIRED_SKIP_TESTS[@]}} \ + ${GCE_DEFAULT_SKIP_TESTS[@]:+${GCE_DEFAULT_SKIP_TESTS[@]}} \ + ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}} \ + )"} + ;; + + kubernetes-e2e-gke-ci-reboot) + : ${DOGFOOD_GCLOUD:="true"} + : ${CLOUDSDK_BUCKET:="gs://cloud-sdk-build/testing/staging"} + : ${GKE_API_ENDPOINT:="https://test-container.sandbox.googleapis.com/"} + : ${E2E_CLUSTER_NAME:="jkns-gke-e2e-ci-reboot"} + : ${E2E_NETWORK:="e2e-gke-ci"} + : ${E2E_SET_CLUSTER_API_VERSION:=y} + : ${PROJECT:="k8s-jkns-e2e-gke-ci"} + : ${GINKGO_TEST_ARGS:="--ginkgo.skip=$(join_regex_allow_empty \ + ${GKE_REQUIRED_SKIP_TESTS[@]:+${GKE_REQUIRED_SKIP_TESTS[@]}} \ + ${REBOOT_SKIP_TESTS[@]:+${REBOOT_SKIP_TESTS[@]}} \ + ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}} \ + ${GCE_PARALLEL_SKIP_TESTS[@]:+${GCE_PARALLEL_SKIP_TESTS[@]}} \ + )"} + ;; esac # AWS variables @@ -362,6 +455,13 @@ export KUBE_GCS_STAGING_PATH_SUFFIX=${KUBE_GCS_STAGING_PATH_SUFFIX:-} export CLUSTER_NAME=${E2E_CLUSTER_NAME} export ZONE=${E2E_ZONE} export KUBE_GKE_NETWORK=${E2E_NETWORK} +export E2E_SET_CLUSTER_API_VERSION=${E2E_SET_CLUSTER_API_VERSION:-} +export DOGFOOD_GCLOUD=${DOGFOOD_GCLOUD:-} +export CMD_GROUP=${CMD_GROUP:-} + +if [[ ! -z "${GKE_API_ENDPOINT:-}" ]]; then + export CLOUDSDK_API_ENDPOINT_OVERRIDES_CONTAINER=${GKE_API_ENDPOINT} +fi # Shared cluster variables export E2E_MIN_STARTUP_PODS=${E2E_MIN_STARTUP_PODS:-} @@ -371,6 +471,7 @@ export MINION_SIZE=${MINION_SIZE:-} export NUM_MINIONS=${NUM_MINIONS:-} export PROJECT=${PROJECT:-} +export KUBERNETES_PROVIDER=${KUBERNETES_PROVIDER} export PATH=${PATH}:/usr/local/go/bin export KUBE_SKIP_CONFIRMATIONS=y @@ -407,10 +508,13 @@ if [[ "${E2E_UP,,}" == "true" || "${JENKINS_FORCE_GET_TARS:-}" =~ ^[yY]$ ]]; the # gcloud bug can cause racing component updates to stomp on each # other. export KUBE_SKIP_UPDATE=y - sudo flock -x -n /var/run/lock/gcloud-components.lock -c "gcloud components update -q" || true - sudo flock -x -n /var/run/lock/gcloud-components.lock -c "gcloud components update preview -q" || true - sudo flock -x -n /var/run/lock/gcloud-components.lock -c "gcloud components update alpha -q" || true - sudo flock -x -n /var/run/lock/gcloud-components.lock -c "gcloud components update beta -q" || true + { + sudo flock -x -n 9 + gcloud components update -q || true + gcloud components update preview -q || true + gcloud components update alpha -q || true + gcloud components update beta -q || true + } 9>/var/run/lock/gcloud-components.lock if [[ ! -z ${JENKINS_EXPLICIT_VERSION:-} ]]; then # Use an explicit pinned version like "ci/v0.10.0-101-g6c814c4" or @@ -496,6 +600,21 @@ ARTIFACTS=${WORKSPACE}/_artifacts mkdir -p ${ARTIFACTS} export E2E_REPORT_DIR=${ARTIFACTS} +### Pre Set Up ### +# Install gcloud from a custom path if provided. Used to test GKE with gcloud +# at HEAD, release candidate. +if [[ ! -z "${CLOUDSDK_BUCKET:-}" ]]; then + sudo gsutil -m cp -r "${CLOUDSDK_BUCKET}" ~ + mv ~/$(basename "${CLOUDSDK_BUCKET}") ~/repo + mkdir ~/cloudsdk + tar zvxf ~/repo/google-cloud-sdk.tar.gz -C ~/cloudsdk + export CLOUDSDK_CORE_DISABLE_PROMPTS=1 + export CLOUDSDK_COMPONENT_MANAGER_SNAPSHOT_URL=file://${HOME}/repo/components-2.json + ~/cloudsdk/google-cloud-sdk/install.sh --disable-installation-options --bash-completion=false --path-update=false --usage-reporting=false + export PATH=${HOME}/cloudsdk/google-cloud-sdk/bin:${PATH} + export CLOUDSDK_CONFIG=/var/lib/jenkins/.config/gcloud +fi + ### Set up ### if [[ "${E2E_UP,,}" == "true" ]]; then go run ./hack/e2e.go ${E2E_OPT} -v --down