From 5a7644c5021fc03d9bf34d44ea64069dc801b134 Mon Sep 17 00:00:00 2001 From: "Madhusudan.C.S" Date: Mon, 31 Oct 2016 20:56:04 -0700 Subject: [PATCH 1/2] [Federation][init-11] Switch federation e2e tests to use the new federation control plane bootstrap via the `kubefed init` command. --- build/push-federation-images.sh | 7 +-- build/release.sh | 8 ---- cluster/kube-util.sh | 12 ++++- federation/cluster/common.sh | 2 +- federation/cluster/federation-up.sh | 69 +++++++++++++++++++++++++---- federation/develop/develop.sh | 7 ++- hack/e2e-internal/e2e-up.sh | 11 ++--- hack/ginkgo-e2e.sh | 1 + test/e2e/framework/framework.go | 2 +- test/e2e/framework/test_context.go | 2 +- test/e2e/framework/util.go | 2 +- 11 files changed, 88 insertions(+), 35 deletions(-) diff --git a/build/push-federation-images.sh b/build/push-federation-images.sh index 3ebf7d4dce2..0db1cd9184c 100755 --- a/build/push-federation-images.sh +++ b/build/push-federation-images.sh @@ -22,8 +22,5 @@ set -o pipefail KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. -source "${KUBE_ROOT}/build/util.sh" - -source "${KUBE_ROOT}/federation/cluster/common.sh" - -FEDERATION_IMAGE_TAG="$(kube::release::semantic_image_tag_version)" push-federation-images +make -C "${KUBE_ROOT}/federation/" build_image +make -C "${KUBE_ROOT}/federation/" push diff --git a/build/release.sh b/build/release.sh index e0931213080..4a587b7c460 100755 --- a/build/release.sh +++ b/build/release.sh @@ -42,13 +42,5 @@ fi kube::build::copy_output -if [[ "${FEDERATION:-}" == "true" ]];then - ( - source "${KUBE_ROOT}/build/util.sh" - # Write federated docker image tag to workspace - kube::release::semantic_image_tag_version > "${KUBE_ROOT}/federation/manifests/federated-image.tag" - ) -fi - kube::release::package_tarballs kube::release::package_hyperkube diff --git a/cluster/kube-util.sh b/cluster/kube-util.sh index 2d73569a554..9f12aec3080 100644 --- a/cluster/kube-util.sh +++ b/cluster/kube-util.sh @@ -35,11 +35,21 @@ fi # Federation utils +# Sets the kubeconfig context value for the current cluster. +# +# Vars set: +# CLUSTER_CONTEXT +function kubeconfig-federation-context() { + CLUSTER_CONTEXT="federation-e2e-${KUBERNETES_PROVIDER}-$zone" +} + + # Should NOT be called within the global scope, unless setting the desired global zone vars # This function is currently NOT USED in the global scope function set-federation-zone-vars { zone="$1" - export OVERRIDE_CONTEXT="federation-e2e-${KUBERNETES_PROVIDER}-$zone" + kubeconfig-federation-context "${zone}" + export OVERRIDE_CONTEXT="${CLUSTER_CONTEXT}" echo "Setting zone vars to: $OVERRIDE_CONTEXT" if [[ "$KUBERNETES_PROVIDER" == "gce" ]];then diff --git a/federation/cluster/common.sh b/federation/cluster/common.sh index d8319de4a02..e80f76d37a0 100644 --- a/federation/cluster/common.sh +++ b/federation/cluster/common.sh @@ -46,7 +46,7 @@ if [[ -z "${FEDERATION_PUSH_REPO_BASE}" ]]; then fi FEDERATION_IMAGE_REPO_BASE=${FEDERATION_IMAGE_REPO_BASE:-'gcr.io/google_containers'} -FEDERATION_NAMESPACE=${FEDERATION_NAMESPACE:-federation} +FEDERATION_NAMESPACE=${FEDERATION_NAMESPACE:-federation-system} KUBE_PLATFORM=${KUBE_PLATFORM:-linux} KUBE_ARCH=${KUBE_ARCH:-amd64} diff --git a/federation/cluster/federation-up.sh b/federation/cluster/federation-up.sh index ded90026965..f756e633ead 100755 --- a/federation/cluster/federation-up.sh +++ b/federation/cluster/federation-up.sh @@ -18,15 +18,66 @@ set -o errexit set -o nounset set -o pipefail -KUBE_ROOT=$(readlink -m $(dirname "${BASH_SOURCE}")/../../) +# This script is only used for e2e tests! Don't use it in production! +# This is also a temporary bridge to slowly switch over everything to +# federation/develop.sh. Carefully moving things step-by-step, ensuring +# things don't break. +# TODO(madhusudancs): Remove this script and its dependencies. -. ${KUBE_ROOT}/federation/cluster/common.sh -tagfile="${KUBE_ROOT}/federation/manifests/federated-image.tag" -if [[ ! -f "$tagfile" ]]; then - echo "FATAL: tagfile ${tagfile} does not exist. Make sure that you have run build/push-federation-images.sh" - exit 1 -fi -export FEDERATION_IMAGE_TAG="$(cat "${KUBE_ROOT}/federation/manifests/federated-image.tag")" +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. +# For `kube::log::status` function since it already sources +# "${KUBE_ROOT}/cluster/lib/logging.sh" and DEFAULT_KUBECONFIG +source "${KUBE_ROOT}/cluster/common.sh" +# For $KUBE_PLATFORM, $KUBE_ARCH, $KUBE_BUILD_STAGE, +# $FEDERATION_PUSH_REPO_BASE and $FEDERATION_NAMESPACE. +source "${KUBE_ROOT}/federation/cluster/common.sh" +# For `get_version` function and $KUBE_REGISTRY. +# TODO(madhusudancs): Remove this when the code here is moved +# to federation/develop.sh +source "${KUBE_ROOT}/federation/develop/develop.sh" -create-federation-api-objects +FEDERATION_NAME="${FEDERATION_NAME:-e2e-federation}" +DNS_ZONE_NAME="${FEDERATION_DNS_ZONE_NAME:-}" +HOST_CLUSTER_CONTEXT="${FEDERATION_HOST_CLUSTER_CONTEXT:-${1}}" + +readonly CLIENT_BIN_DIR="${KUBE_ROOT}/_output/${KUBE_BUILD_STAGE}/client/${KUBE_PLATFORM}-${KUBE_ARCH}/kubernetes/client/bin" +kubefed="${CLIENT_BIN_DIR}/kubefed" +kubectl="${CLIENT_BIN_DIR}/kubectl" + +# Initializes the control plane. +# TODO(madhusudancs): Move this to federation/develop.sh. +function init() { + kube::log::status "Deploying federation control plane for ${FEDERATION_NAME} in cluster ${HOST_CLUSTER_CONTEXT}" + + local -r kube_version="$(get_version)" + + ${kubefed} init \ + "${FEDERATION_NAME}" \ + --host-cluster-context="${HOST_CLUSTER_CONTEXT}" \ + --dns-zone-name="${DNS_ZONE_NAME}" \ + --image="${KUBE_REGISTRY}/hyperkube-amd64:${kube_version}" +} + +# create_cluster_secrets creates the secrets containing the kubeconfigs +# of the participating clusters in the host cluster. The kubeconfigs itself +# are created while deploying clusters, i.e. when kube-up is run. +function create_cluster_secrets() { + local -r kubeconfig_dir="$(dirname ${DEFAULT_KUBECONFIG})" + local -r base_dir="${kubeconfig_dir}/federation/kubernetes-apiserver" + + # Create secrets with all the kubernetes-apiserver's kubeconfigs. + for dir in $(ls "${base_dir}"); do + # We create a secret with the same name as the directory name (which is + # same as cluster name in kubeconfig). + # Massage the name so that it is valid (should not contain "_" and max 253 + # chars) + name=$(echo "${dir}" | sed -e "s/_/-/g") # Replace "_" by "-" + name=${name:0:252} + kube::log::status "Creating secret with name: ${name} in namespace ${FEDERATION_NAMESPACE}" + ${kubectl} create secret generic ${name} --from-file="${base_dir}/${dir}/kubeconfig" --namespace="${FEDERATION_NAMESPACE}" + done +} + +init +create_cluster_secrets diff --git a/federation/develop/develop.sh b/federation/develop/develop.sh index 59f76211196..31f3b1c51b6 100755 --- a/federation/develop/develop.sh +++ b/federation/develop/develop.sh @@ -117,7 +117,7 @@ function build_image() { make -C "${KUBE_ROOT}/cluster/images/hyperkube" build } -function push() { +function get_version() { local kube_version="" if [[ -n "${KUBE_VERSION:-}" ]]; then kube_version="${KUBE_VERSION}" @@ -125,6 +125,11 @@ function push() { # Read the version back from the versions file if no version is given. kube_version="$(jq -r '.KUBE_VERSION' ${VERSIONS_FILE})" fi + echo "${kube_version}" +} + +function push() { + local -r kube_version="$(get_version)" kube::log::status "Pushing hyperkube image to the registry" gcloud docker -- push "${KUBE_REGISTRY}/hyperkube-amd64:${kube_version}" diff --git a/hack/e2e-internal/e2e-up.sh b/hack/e2e-internal/e2e-up.sh index fb80a23b4df..a6a9dd01190 100755 --- a/hack/e2e-internal/e2e-up.sh +++ b/hack/e2e-internal/e2e-up.sh @@ -43,14 +43,11 @@ if [[ "${FEDERATION:-}" == "true" ]]; then ) cur_ip_octet2="$((cur_ip_octet2 + 1))" done - tagfile="${KUBE_ROOT}/federation/manifests/federated-image.tag" - if [[ ! -f "$tagfile" ]]; then - echo "FATAL: tagfile ${tagfile} does not exist. Make sure that you have run build/push-federation-images.sh" - exit 1 - fi - export FEDERATION_IMAGE_TAG="$(cat "${KUBE_ROOT}/federation/manifests/federated-image.tag")" - "${KUBE_ROOT}/federation/cluster/federation-up.sh" + # Sets ${CLUSTER_CONTEXT} + kubeconfig-federation-context "${zone}" + + "${KUBE_ROOT}/federation/cluster/federation-up.sh" "${CLUSTER_CONTEXT}" else test-setup fi diff --git a/hack/ginkgo-e2e.sh b/hack/ginkgo-e2e.sh index 42a22df823a..fddadcb2d65 100755 --- a/hack/ginkgo-e2e.sh +++ b/hack/ginkgo-e2e.sh @@ -128,6 +128,7 @@ export PATH=$(dirname "${e2e_test}"):"${PATH}" --node-instance-group="${NODE_INSTANCE_GROUP:-}" \ --prefix="${KUBE_GCE_INSTANCE_PREFIX:-e2e}" \ --network="${KUBE_GCE_NETWORK:-${KUBE_GKE_NETWORK:-e2e}}" \ + --federated-kube-context="${FEDERATION_KUBE_CONTEXT:-e2e-federation}" \ ${KUBE_CONTAINER_RUNTIME:+"--container-runtime=${KUBE_CONTAINER_RUNTIME}"} \ ${MASTER_OS_DISTRIBUTION:+"--master-os-distro=${MASTER_OS_DISTRIBUTION}"} \ ${NODE_OS_DISTRIBUTION:+"--node-os-distro=${NODE_OS_DISTRIBUTION}"} \ diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 86282972df2..bc57f5b8647 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -750,7 +750,7 @@ func (f *Framework) GetUnderlyingFederatedContexts() []E2EContext { e2eContexts := []E2EContext{} for _, context := range kubeconfig.Contexts { - if strings.HasPrefix(context.Name, "federation") && context.Name != "federation-cluster" { + if strings.HasPrefix(context.Name, "federation") && context.Name != federatedKubeContext { user := kubeconfig.findUser(context.Context.User) if user == nil { diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index af793d35071..4ba8ce20929 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -163,7 +163,7 @@ func RegisterClusterFlags() { flag.StringVar(&TestContext.KubeConfig, clientcmd.RecommendedConfigPathFlag, os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to kubeconfig containing embedded authinfo.") flag.StringVar(&TestContext.KubeContext, clientcmd.FlagContext, "", "kubeconfig context to use/override. If unset, will use value from 'current-context'") flag.StringVar(&TestContext.KubeAPIContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "ContentType used to communicate with apiserver") - flag.StringVar(&federatedKubeContext, "federated-kube-context", "federation-cluster", "kubeconfig context for federation-cluster.") + flag.StringVar(&federatedKubeContext, "federated-kube-context", "e2e-federation", "kubeconfig context for federation.") flag.StringVar(&TestContext.KubeVolumeDir, "volume-dir", "/var/lib/kubelet", "Path to the directory containing the kubelet volumes.") flag.StringVar(&TestContext.CertDir, "cert-dir", "", "Path to the directory containing the certs. Default is empty, which doesn't use certs.") diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 6a10ee91b5b..ad1de77e327 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -363,7 +363,7 @@ func SkipUnlessServerVersionGTE(v *utilversion.Version, c discovery.ServerVersio func SkipUnlessFederated(c clientset.Interface) { federationNS := os.Getenv("FEDERATION_NAMESPACE") if federationNS == "" { - federationNS = "federation" + federationNS = "federation-system" } _, err := c.Core().Namespaces().Get(federationNS, metav1.GetOptions{}) From fd01ed8fdb343ff9883300a8e0ad6b52532ae5a9 Mon Sep 17 00:00:00 2001 From: shashidharatd Date: Mon, 21 Nov 2016 13:04:42 +0530 Subject: [PATCH 2/2] [Federation][init-11.2] use USE_KUBEFED env var to choose bw old and new federation deployment --- federation/cluster/common.sh | 10 +++++----- federation/cluster/federation-up.sh | 12 +++++++++--- .../manifests/federation-apiserver-deployment.yaml | 2 +- .../federation-controller-manager-deployment.yaml | 2 +- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/federation/cluster/common.sh b/federation/cluster/common.sh index e80f76d37a0..e4e10302b37 100644 --- a/federation/cluster/common.sh +++ b/federation/cluster/common.sh @@ -65,11 +65,11 @@ function create-federation-api-objects { : "${FEDERATION_IMAGE_TAG?Must set FEDERATION_IMAGE_TAG env var}" export FEDERATION_APISERVER_DEPLOYMENT_NAME="federation-apiserver" - export FEDERATION_APISERVER_IMAGE_REPO="${FEDERATION_PUSH_REPO_BASE}/hyperkube" + export FEDERATION_APISERVER_IMAGE_REPO="${FEDERATION_PUSH_REPO_BASE}/hyperkube-amd64" export FEDERATION_APISERVER_IMAGE_TAG="${FEDERATION_IMAGE_TAG}" export FEDERATION_CONTROLLER_MANAGER_DEPLOYMENT_NAME="federation-controller-manager" - export FEDERATION_CONTROLLER_MANAGER_IMAGE_REPO="${FEDERATION_PUSH_REPO_BASE}/hyperkube" + export FEDERATION_CONTROLLER_MANAGER_IMAGE_REPO="${FEDERATION_PUSH_REPO_BASE}/hyperkube-amd64" export FEDERATION_CONTROLLER_MANAGER_IMAGE_TAG="${FEDERATION_IMAGE_TAG}" if [[ -z "${FEDERATION_DNS_PROVIDER:-}" ]]; then @@ -165,7 +165,7 @@ function create-federation-api-objects { # controller manager can use to talk to the federation-apiserver. # Note that the file name should be "kubeconfig" so that the secret key gets the same name. KUBECONFIG_DIR=$(dirname ${KUBECONFIG:-$DEFAULT_KUBECONFIG}) - CONTEXT=federation-cluster \ + CONTEXT=${FEDERATION_KUBE_CONTEXT} \ KUBE_BEARER_TOKEN="$FEDERATION_API_TOKEN" \ KUBE_USER="${KUBE_USER}" \ KUBE_PASSWORD="${KUBE_PASSWORD}" \ @@ -207,7 +207,7 @@ function create-federation-api-objects { done # Update the users kubeconfig to include federation-apiserver credentials. - CONTEXT=federation-cluster \ + CONTEXT=${FEDERATION_KUBE_CONTEXT} \ KUBE_BEARER_TOKEN="$FEDERATION_API_TOKEN" \ KUBE_USER="${KUBE_USER}" \ KUBE_PASSWORD="${KUBE_PASSWORD}" \ @@ -296,7 +296,7 @@ function push-federation-images { source "${KUBE_ROOT}/build/common.sh" source "${KUBE_ROOT}/hack/lib/util.sh" - local FEDERATION_BINARIES=${FEDERATION_BINARIES:-"hyperkube"} + local FEDERATION_BINARIES=${FEDERATION_BINARIES:-"hyperkube-amd64"} local bin_dir="${KUBE_ROOT}/_output/${KUBE_BUILD_STAGE}/server/${KUBE_PLATFORM}-${KUBE_ARCH}/kubernetes/server/bin" diff --git a/federation/cluster/federation-up.sh b/federation/cluster/federation-up.sh index f756e633ead..d9ed22bc194 100755 --- a/federation/cluster/federation-up.sh +++ b/federation/cluster/federation-up.sh @@ -38,9 +38,9 @@ source "${KUBE_ROOT}/federation/cluster/common.sh" source "${KUBE_ROOT}/federation/develop/develop.sh" FEDERATION_NAME="${FEDERATION_NAME:-e2e-federation}" +FEDERATION_KUBE_CONTEXT="${FEDERATION_KUBE_CONTEXT:-e2e-federation}" DNS_ZONE_NAME="${FEDERATION_DNS_ZONE_NAME:-}" HOST_CLUSTER_CONTEXT="${FEDERATION_HOST_CLUSTER_CONTEXT:-${1}}" - readonly CLIENT_BIN_DIR="${KUBE_ROOT}/_output/${KUBE_BUILD_STAGE}/client/${KUBE_PLATFORM}-${KUBE_ARCH}/kubernetes/client/bin" kubefed="${CLIENT_BIN_DIR}/kubefed" kubectl="${CLIENT_BIN_DIR}/kubectl" @@ -79,5 +79,11 @@ function create_cluster_secrets() { done } -init -create_cluster_secrets +USE_KUBEFED="${USE_KUBEFED:-}" +if [[ "${USE_KUBEFED}" == "true" ]]; then + init + create_cluster_secrets +else + export FEDERATION_IMAGE_TAG="$(get_version)" + create-federation-api-objects +fi diff --git a/federation/manifests/federation-apiserver-deployment.yaml b/federation/manifests/federation-apiserver-deployment.yaml index 5113f5f00b3..aa4fb98dfb5 100644 --- a/federation/manifests/federation-apiserver-deployment.yaml +++ b/federation/manifests/federation-apiserver-deployment.yaml @@ -17,7 +17,7 @@ spec: - name: apiserver image: {{.FEDERATION_APISERVER_IMAGE_REPO}}:{{.FEDERATION_APISERVER_IMAGE_TAG}} command: - - /usr/local/bin/hyperkube + - /hyperkube - federation-apiserver - --bind-address=0.0.0.0 - --etcd-servers=http://localhost:2379 diff --git a/federation/manifests/federation-controller-manager-deployment.yaml b/federation/manifests/federation-controller-manager-deployment.yaml index c339a306995..9b3263b0caa 100644 --- a/federation/manifests/federation-controller-manager-deployment.yaml +++ b/federation/manifests/federation-controller-manager-deployment.yaml @@ -31,7 +31,7 @@ spec: mountPath: "/etc/federation/controller-manager" image: {{.FEDERATION_CONTROLLER_MANAGER_IMAGE_REPO}}:{{.FEDERATION_CONTROLLER_MANAGER_IMAGE_TAG}} command: - - /usr/local/bin/hyperkube + - /hyperkube - federation-controller-manager - --master=https://{{.FEDERATION_APISERVER_DEPLOYMENT_NAME}}:443 - --kubeconfig=/etc/federation/controller-manager/kubeconfig