diff --git a/build/push-ci-build.sh b/build/push-ci-build.sh index fbea9261e2d..6d74878ffad 100755 --- a/build/push-ci-build.sh +++ b/build/push-ci-build.sh @@ -49,7 +49,5 @@ if [[ ! ${attempt} -lt ${MAX_ATTEMPTS} ]];then fi if [[ "${FEDERATION:-}" == "true" ]];then - source "${KUBE_ROOT}/federation/cluster/common.sh" - # Docker compatiblity - FEDERATION_IMAGE_TAG="$(kube::release::semantic_image_tag_version)" push-federated-images + "${KUBE_ROOT}/build/push-federation-images.sh" fi diff --git a/build/push-federation-images.sh b/build/push-federation-images.sh new file mode 100755 index 00000000000..6c44cec6d8a --- /dev/null +++ b/build/push-federation-images.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Pushes federation container images to existing repositories + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. + +source "${KUBE_ROOT}/build/util.sh" + +source "${KUBE_ROOT}/federation/cluster/common.sh" + +FEDERATION_IMAGE_TAG="$(kube::release::semantic_image_tag_version)" push-federation-images diff --git a/cluster/kube-util.sh b/cluster/kube-util.sh index 416e0cb4af1..8bc820b645a 100644 --- a/cluster/kube-util.sh +++ b/cluster/kube-util.sh @@ -104,7 +104,7 @@ fi # Should NOT be called within the global scope, unless setting the desired global zone vars # This function is currently NOT USED in the global scope -function set-federated-zone-vars { +function set-federation-zone-vars { zone="$1" export OVERRIDE_CONTEXT="federation-e2e-${KUBERNETES_PROVIDER}-$zone" echo "Setting zone vars to: $OVERRIDE_CONTEXT" diff --git a/docs/devel/e2e-tests.md b/docs/devel/e2e-tests.md index d09ab9e7dc2..b67d3a5efb0 100644 --- a/docs/devel/e2e-tests.md +++ b/docs/devel/e2e-tests.md @@ -45,6 +45,14 @@ Updated: 5/3/2016 - [Cleaning up](#cleaning-up) - [Advanced testing](#advanced-testing) - [Bringing up a cluster for testing](#bringing-up-a-cluster-for-testing) + - [Federation e2e tests](#federation-e2e-tests) + - [Configuring federation e2e tests](#configuring-federation-e2e-tests) + - [Image Push Repository](#image-push-repository) + - [Build](#build) + - [Deploy federation control plane](#deploy-federation-control-plane) + - [Run the Tests](#run-the-tests) + - [Teardown](#teardown) + - [Shortcuts for test developers](#shortcuts-for-test-developers) - [Debugging clusters](#debugging-clusters) - [Local clusters](#local-clusters) - [Testing against local clusters](#testing-against-local-clusters) @@ -232,6 +240,119 @@ stale permissions can cause problems. - `sudo iptables -F`, clear ip tables rules left by the kube-proxy. +### Federation e2e tests + +By default, `e2e.go` provisions a single Kubernetes cluster, and any `Feature:Federation` ginkgo tests will be skipped. + +Federation e2e testing involve bringing up multiple "underlying" Kubernetes clusters, +and deploying the federation control plane as a Kubernetes application on the underlying clusters. + +The federation e2e tests are still managed via `e2e.go`, but require some extra configuration items. + +#### Configuring federation e2e tests + +The following environment variables will enable federation e2e building, provisioning and testing. + +```sh +$ export FEDERATION=true +$ export E2E_ZONES="us-central1-a us-central1-b us-central1-f" +``` + +A Kubernetes cluster will be provisioned in each zone listed in `E2E_ZONES`. A zone can only appear once in the `E2E_ZONES` list. + +#### Image Push Repository + +Next, specify the docker repository where your ci images will be pushed. + +* **If `KUBERNETES_PROVIDER=gce` or `KUBERNETES_PROVIDER=gke`**: + + You can simply set your push repo base based on your project name, and the necessary repositories will be auto-created when you + first push your container images. + + ```sh + $ export FEDERATION_PUSH_REPO_BASE="gcr.io/${GCE_PROJECT_NAME}" + ``` + + Skip ahead to the **Build** section. + +* **For all other providers**: + + You'll be responsible for creating and managing access to the repositories manually. + + ```sh + $ export FEDERATION_PUSH_REPO_BASE="quay.io/colin_hom" + ``` + + Given this example, the `federation-apiserver` container image will be pushed to the repository + `quay.io/colin_hom/federation-apiserver`. + + The docker client on the machine running `e2e.go` must have push access for the following pre-existing repositories: + + * `${FEDERATION_PUSH_REPO_BASE}/federation-apiserver` + * `${FEDERATION_PUSH_REPO_BASE}/federation-controller-manager` + + These repositories must allow public read access, as the e2e node docker daemons will not have any credentials. If you're using + gce/gke as your provider, the repositories will have read-access by default. + +#### Build + +* Compile the binaries and build container images: + + ```sh + $ KUBE_RELEASE_RUN_TESTS=n KUBE_FASTBUILD=true go run hack/e2e.go -v -build + ``` + +* Push the federation container images + + ```sh + $ build/push-federation-images.sh + ``` + +#### Deploy federation control plane + +The following command will create the underlying Kubernetes clusters in each of `E2E_ZONES`, and then provision the +federation control plane in the cluster occupying the last zone in the `E2E_ZONES` list. + +```sh +$ go run hack/e2e.go -v -up +``` + +#### Run the Tests + +This will run only the `Feature:Federation` e2e tests. You can omit the `ginkgo.focus` argument to run the entire e2e suite. + +```sh +$ go run hack/e2e.go -v -test --test_args="--ginkgo.focus=\[Feature:Federation\]" +``` + +#### Teardown + +```sh +$ go run hack/e2e.go -v -down +``` + +#### Shortcuts for test developers + +* To speed up `e2e.go -up`, provision a single-node kubernetes cluster in a single e2e zone: + + `NUM_NODES=1 E2E_ZONES="us-central1-f"` + + Keep in mind that some tests may require multiple underlying clusters and/or minimum compute resource availability. + +* You can quickly recompile the e2e testing framework via `go install ./test/e2e`. This will not do anything besides + allow you to verify that the go code compiles. + +* If you want to run your e2e testing framework without re-provisioning the e2e setup, you can do so via + `make WHAT=test/e2e/e2e.test` and then re-running the ginkgo tests. + +* If you're hacking around with the federation control plane deployment itself, + you can quickly re-deploy the federation control plane Kubernetes manifests without tearing any resources down. + To re-deploy the federation control plane after running `-up` for the first time: + + ```sh + $ federation/cluster/federation-up.sh + ``` + ### Debugging clusters If a cluster fails to initialize, or you'd like to better understand cluster diff --git a/federation/cluster/common.sh b/federation/cluster/common.sh index ec1f442d4d2..bfb7392c8f0 100644 --- a/federation/cluster/common.sh +++ b/federation/cluster/common.sh @@ -41,7 +41,7 @@ host_kubectl="${KUBE_ROOT}/cluster/kubectl.sh --namespace=${FEDERATION_NAMESPACE # Optional # FEDERATION_IMAGE_TAG: reference and pull all federated images with this tag. Used for ci testing -function create-federated-api-objects { +function create-federation-api-objects { ( : "${FEDERATION_PUSH_REPO_BASE?Must set FEDERATION_PUSH_REPO_BASE env var}" export FEDERATION_APISERVER_DEPLOYMENT_NAME="federation-apiserver" @@ -69,7 +69,7 @@ function create-federated-api-objects { $template "${manifests_root}/federation-ns.yaml" | $host_kubectl apply -f - - cleanup-federated-api-objects + cleanup-federation-api-objects export FEDERATION_API_HOST="" export KUBE_MASTER_IP="" @@ -181,7 +181,7 @@ function create-federated-api-objects { # Optional # FEDERATION_IMAGE_TAG: push all federated images with this tag. Used for ci testing -function push-federated-images { +function push-federation-images { : "${FEDERATION_PUSH_REPO_BASE?Must set FEDERATION_PUSH_REPO_BASE env var}" local FEDERATION_BINARIES=${FEDERATION_BINARIES:-"federation-apiserver federation-controller-manager"} @@ -230,7 +230,7 @@ function push-federated-images { done } -function cleanup-federated-api-objects { +function cleanup-federation-api-objects { # Delete all resources with the federated-cluster label. $host_kubectl delete pods,svc,rc,deployment,secret -lapp=federated-cluster # Delete all resources in FEDERATION_NAMESPACE. diff --git a/federation/cluster/federated-down.sh b/federation/cluster/federation-down.sh similarity index 96% rename from federation/cluster/federated-down.sh rename to federation/cluster/federation-down.sh index 92f19307956..dd849add5ed 100755 --- a/federation/cluster/federated-down.sh +++ b/federation/cluster/federation-down.sh @@ -22,6 +22,6 @@ KUBE_ROOT=$(readlink -m $(dirname "${BASH_SOURCE}")/../../) . ${KUBE_ROOT}/federation/cluster/common.sh -cleanup-federated-api-objects +cleanup-federation-api-objects $host_kubectl delete ns/${FEDERATION_NAMESPACE} diff --git a/federation/cluster/federated-push.sh b/federation/cluster/federation-push.sh similarity index 97% rename from federation/cluster/federated-push.sh rename to federation/cluster/federation-push.sh index 80d66641972..8f3030e55db 100755 --- a/federation/cluster/federated-push.sh +++ b/federation/cluster/federation-push.sh @@ -28,5 +28,5 @@ KUBE_ROOT=$(readlink -m $(dirname "${BASH_SOURCE}")/../../) . ${KUBE_ROOT}/federation/cluster/common.sh -push-federated-images +push-federation-images diff --git a/federation/cluster/federated-up.sh b/federation/cluster/federation-up.sh similarity index 96% rename from federation/cluster/federated-up.sh rename to federation/cluster/federation-up.sh index 750b907aafb..90c80982302 100755 --- a/federation/cluster/federated-up.sh +++ b/federation/cluster/federation-up.sh @@ -22,4 +22,4 @@ KUBE_ROOT=$(readlink -m $(dirname "${BASH_SOURCE}")/../../) . ${KUBE_ROOT}/federation/cluster/common.sh -create-federated-api-objects +create-federation-api-objects diff --git a/hack/e2e-internal/e2e-down.sh b/hack/e2e-internal/e2e-down.sh index 93dddd007b2..61b563c81b9 100755 --- a/hack/e2e-internal/e2e-down.sh +++ b/hack/e2e-internal/e2e-down.sh @@ -34,8 +34,8 @@ if [[ "${FEDERATION:-}" == "true" ]];then for zone in ${E2E_ZONES};do # bring up e2e cluster ( - set-federated-zone-vars "$zone" - cleanup-federated-api-objects || echo "Couldn't cleanup federated api objects" + set-federation-zone-vars "$zone" + cleanup-federation-api-objects || echo "Couldn't cleanup federation api objects" test-teardown ) done diff --git a/hack/e2e-internal/e2e-status.sh b/hack/e2e-internal/e2e-status.sh index f6e37dc0e85..df171bb5463 100755 --- a/hack/e2e-internal/e2e-status.sh +++ b/hack/e2e-internal/e2e-status.sh @@ -37,7 +37,7 @@ if [[ "${FEDERATION:-}" == "true" ]];then # select each one and call federated-up for zone in ${E2E_ZONES};do ( - set-federated-zone-vars "$zone" + set-federation-zone-vars "$zone" printf "\n\tChecking version for $OVERRIDE_CONTEXT\n" ${KUBECTL} --context="$OVERRIDE_CONTEXT" version ) diff --git a/hack/e2e-internal/e2e-up.sh b/hack/e2e-internal/e2e-up.sh index 4d5dadc4e1c..8d2a501dcf3 100755 --- a/hack/e2e-internal/e2e-up.sh +++ b/hack/e2e-internal/e2e-up.sh @@ -36,14 +36,14 @@ if [[ "${FEDERATION:-}" == "true" ]];then # select each one and call federated-up for zone in ${E2E_ZONES};do ( - set-federated-zone-vars "$zone" + set-federation-zone-vars "$zone" test-setup ) done if [[ -f "${KUBE_ROOT}/federation/manifests/federated-image.tag" ]];then export FEDERATION_IMAGE_TAG="$(cat "${KUBE_ROOT}/federation/manifests/federated-image.tag")" fi - "${KUBE_ROOT}/federation/cluster/federated-up.sh" + "${KUBE_ROOT}/federation/cluster/federation-up.sh" else test-setup fi diff --git a/hack/federated-ginkgo-e2e.sh b/hack/federated-ginkgo-e2e.sh index b147e719915..c42e9f2fa97 100755 --- a/hack/federated-ginkgo-e2e.sh +++ b/hack/federated-ginkgo-e2e.sh @@ -29,6 +29,6 @@ for zone in ${E2E_ZONES};do lastZone="$zone" done ( - set-federated-zone-vars "$zone" + set-federation-zone-vars "$zone" "${KUBE_ROOT}/hack/ginkgo-e2e.sh" $@ )