Merge pull request #37215 from shashidharatd/fed-ci

Automatic merge from submit-queue

[Federation][init-11.2] use USE_KUBEFED env var to choose bw old and new federation deployment

This is continuation of #35961
USE_KUBEFED variable is used for deploying federation control plane. if not defined, federation will be brought up using old method i.e scripts.

Have verified that federation comes up using the old method, using following steps
```
$ export FEDERATION=true
$ export E2E_ZONES="asia-east1-c"
$ export FEDERATION_PUSH_REPO_BASE=gcr.io/<my-project>
$ KUBE_RELEASE_RUN_TESTS=n KUBE_FASTBUILD=true go run hack/e2e.go -v -build
$ build-tools/push-federation-images.sh
$ go run hack/e2e.go -v --up
```
Should merge #35961 before this PR

@madhusudancs

build/push-federation-images.sh

@@ -22,8 +22,5 @@ set -o pipefail
 
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 
-source "${KUBE_ROOT}/build/util.sh"
-
-source "${KUBE_ROOT}/federation/cluster/common.sh"
-
-FEDERATION_IMAGE_TAG="$(kube::release::semantic_image_tag_version)" push-federation-images
+make -C "${KUBE_ROOT}/federation/" build_image
+make -C "${KUBE_ROOT}/federation/" push

build/release.sh

@@ -42,13 +42,5 @@ fi
 
 kube::build::copy_output
 
-if [[ "${FEDERATION:-}" == "true" ]];then
-    (
-	source "${KUBE_ROOT}/build/util.sh"
-	# Write federated docker image tag to workspace
-	kube::release::semantic_image_tag_version > "${KUBE_ROOT}/federation/manifests/federated-image.tag"
-    )
-fi
-
 kube::release::package_tarballs
 kube::release::package_hyperkube

cluster/kube-util.sh

@@ -35,11 +35,21 @@ fi
 
 # Federation utils
 
+# Sets the kubeconfig context value for the current cluster.
+#
+# Vars set:
+#   CLUSTER_CONTEXT
+function kubeconfig-federation-context() {
+  CLUSTER_CONTEXT="federation-e2e-${KUBERNETES_PROVIDER}-$zone"
+}
+
+
 # Should NOT be called within the global scope, unless setting the desired global zone vars
 # This function is currently NOT USED in the global scope
 function set-federation-zone-vars {
   zone="$1"
-  export OVERRIDE_CONTEXT="federation-e2e-${KUBERNETES_PROVIDER}-$zone"
+  kubeconfig-federation-context "${zone}"
+  export OVERRIDE_CONTEXT="${CLUSTER_CONTEXT}"
   echo "Setting zone vars to: $OVERRIDE_CONTEXT"
   if [[ "$KUBERNETES_PROVIDER" == "gce"  ]];then
 

federation/cluster/common.sh

@@ -46,7 +46,7 @@ if [[ -z "${FEDERATION_PUSH_REPO_BASE}" ]]; then
 fi
 
 FEDERATION_IMAGE_REPO_BASE=${FEDERATION_IMAGE_REPO_BASE:-'gcr.io/google_containers'}
-FEDERATION_NAMESPACE=${FEDERATION_NAMESPACE:-federation}
+FEDERATION_NAMESPACE=${FEDERATION_NAMESPACE:-federation-system}
 
 KUBE_PLATFORM=${KUBE_PLATFORM:-linux}
 KUBE_ARCH=${KUBE_ARCH:-amd64}
@@ -65,11 +65,11 @@ function create-federation-api-objects {
     : "${FEDERATION_IMAGE_TAG?Must set FEDERATION_IMAGE_TAG env var}"
 
     export FEDERATION_APISERVER_DEPLOYMENT_NAME="federation-apiserver"
-    export FEDERATION_APISERVER_IMAGE_REPO="${FEDERATION_PUSH_REPO_BASE}/hyperkube"
+    export FEDERATION_APISERVER_IMAGE_REPO="${FEDERATION_PUSH_REPO_BASE}/hyperkube-amd64"
     export FEDERATION_APISERVER_IMAGE_TAG="${FEDERATION_IMAGE_TAG}"
 
     export FEDERATION_CONTROLLER_MANAGER_DEPLOYMENT_NAME="federation-controller-manager"
-    export FEDERATION_CONTROLLER_MANAGER_IMAGE_REPO="${FEDERATION_PUSH_REPO_BASE}/hyperkube"
+    export FEDERATION_CONTROLLER_MANAGER_IMAGE_REPO="${FEDERATION_PUSH_REPO_BASE}/hyperkube-amd64"
     export FEDERATION_CONTROLLER_MANAGER_IMAGE_TAG="${FEDERATION_IMAGE_TAG}"
 
     if [[ -z "${FEDERATION_DNS_PROVIDER:-}" ]]; then
@@ -165,7 +165,7 @@ function create-federation-api-objects {
     # controller manager can use to talk to the federation-apiserver.
     # Note that the file name should be "kubeconfig" so that the secret key gets the same name.
     KUBECONFIG_DIR=$(dirname ${KUBECONFIG:-$DEFAULT_KUBECONFIG})
-    CONTEXT=federation-cluster \
+    CONTEXT=${FEDERATION_KUBE_CONTEXT} \
 	   KUBE_BEARER_TOKEN="$FEDERATION_API_TOKEN" \
            KUBE_USER="${KUBE_USER}" \
            KUBE_PASSWORD="${KUBE_PASSWORD}" \
@@ -207,7 +207,7 @@ function create-federation-api-objects {
     done
 
     # Update the users kubeconfig to include federation-apiserver credentials.
-    CONTEXT=federation-cluster \
+    CONTEXT=${FEDERATION_KUBE_CONTEXT} \
 	   KUBE_BEARER_TOKEN="$FEDERATION_API_TOKEN" \
            KUBE_USER="${KUBE_USER}" \
            KUBE_PASSWORD="${KUBE_PASSWORD}" \
@@ -296,7 +296,7 @@ function push-federation-images {
     source "${KUBE_ROOT}/build/common.sh"
     source "${KUBE_ROOT}/hack/lib/util.sh"
 
-    local FEDERATION_BINARIES=${FEDERATION_BINARIES:-"hyperkube"}
+    local FEDERATION_BINARIES=${FEDERATION_BINARIES:-"hyperkube-amd64"}
 
     local bin_dir="${KUBE_ROOT}/_output/${KUBE_BUILD_STAGE}/server/${KUBE_PLATFORM}-${KUBE_ARCH}/kubernetes/server/bin"
 

federation/cluster/federation-up.sh

@@ -18,15 +18,72 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-KUBE_ROOT=$(readlink -m $(dirname "${BASH_SOURCE}")/../../)
+# This script is only used for e2e tests! Don't use it in production!
+# This is also a temporary bridge to slowly switch over everything to
+# federation/develop.sh. Carefully moving things step-by-step, ensuring
+# things don't break.
+# TODO(madhusudancs): Remove this script and its dependencies.
 
-. ${KUBE_ROOT}/federation/cluster/common.sh
 
-tagfile="${KUBE_ROOT}/federation/manifests/federated-image.tag"
-if [[ ! -f "$tagfile" ]]; then
-    echo "FATAL: tagfile ${tagfile} does not exist. Make sure that you have run build/push-federation-images.sh"
-    exit 1
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
+# For `kube::log::status` function since it already sources
+# "${KUBE_ROOT}/cluster/lib/logging.sh" and DEFAULT_KUBECONFIG
+source "${KUBE_ROOT}/cluster/common.sh"
+# For $KUBE_PLATFORM, $KUBE_ARCH, $KUBE_BUILD_STAGE,
+# $FEDERATION_PUSH_REPO_BASE and $FEDERATION_NAMESPACE.
+source "${KUBE_ROOT}/federation/cluster/common.sh"
+# For `get_version` function and $KUBE_REGISTRY.
+# TODO(madhusudancs): Remove this when the code here is moved
+# to federation/develop.sh
+source "${KUBE_ROOT}/federation/develop/develop.sh"
+
+FEDERATION_NAME="${FEDERATION_NAME:-e2e-federation}"
+FEDERATION_KUBE_CONTEXT="${FEDERATION_KUBE_CONTEXT:-e2e-federation}"
+DNS_ZONE_NAME="${FEDERATION_DNS_ZONE_NAME:-}"
+HOST_CLUSTER_CONTEXT="${FEDERATION_HOST_CLUSTER_CONTEXT:-${1}}"
+readonly CLIENT_BIN_DIR="${KUBE_ROOT}/_output/${KUBE_BUILD_STAGE}/client/${KUBE_PLATFORM}-${KUBE_ARCH}/kubernetes/client/bin"
+kubefed="${CLIENT_BIN_DIR}/kubefed"
+kubectl="${CLIENT_BIN_DIR}/kubectl"
+
+# Initializes the control plane.
+# TODO(madhusudancs): Move this to federation/develop.sh.
+function init() {
+  kube::log::status "Deploying federation control plane for ${FEDERATION_NAME} in cluster ${HOST_CLUSTER_CONTEXT}"
+
+  local -r kube_version="$(get_version)"
+
+  ${kubefed} init \
+      "${FEDERATION_NAME}" \
+      --host-cluster-context="${HOST_CLUSTER_CONTEXT}" \
+      --dns-zone-name="${DNS_ZONE_NAME}" \
+      --image="${KUBE_REGISTRY}/hyperkube-amd64:${kube_version}"
+}
+
+# create_cluster_secrets creates the secrets containing the kubeconfigs
+# of the participating clusters in the host cluster. The kubeconfigs itself
+# are created while deploying clusters, i.e. when kube-up is run.
+function create_cluster_secrets() {
+  local -r kubeconfig_dir="$(dirname ${DEFAULT_KUBECONFIG})"
+  local -r base_dir="${kubeconfig_dir}/federation/kubernetes-apiserver"
+
+  # Create secrets with all the kubernetes-apiserver's kubeconfigs.
+  for dir in $(ls "${base_dir}"); do
+    # We create a secret with the same name as the directory name (which is
+    # same as cluster name in kubeconfig).
+    # Massage the name so that it is valid (should not contain "_" and max 253
+    # chars)
+    name=$(echo "${dir}" | sed -e "s/_/-/g")  # Replace "_" by "-"
+    name=${name:0:252}
+    kube::log::status "Creating secret with name: ${name} in namespace ${FEDERATION_NAMESPACE}"
+    ${kubectl} create secret generic ${name} --from-file="${base_dir}/${dir}/kubeconfig" --namespace="${FEDERATION_NAMESPACE}"
+  done
+}
+
+USE_KUBEFED="${USE_KUBEFED:-}"
+if [[ "${USE_KUBEFED}" == "true" ]]; then
+  init
+  create_cluster_secrets
+else
+  export FEDERATION_IMAGE_TAG="$(get_version)"
+  create-federation-api-objects
 fi
-export FEDERATION_IMAGE_TAG="$(cat "${KUBE_ROOT}/federation/manifests/federated-image.tag")"
-
-create-federation-api-objects

federation/develop/develop.sh

@@ -117,7 +117,7 @@ function build_image() {
     make -C "${KUBE_ROOT}/cluster/images/hyperkube" build
 }
 
-function push() {
+function get_version() {
   local kube_version=""
   if [[ -n "${KUBE_VERSION:-}" ]]; then
     kube_version="${KUBE_VERSION}"
@@ -125,6 +125,11 @@ function push() {
     # Read the version back from the versions file if no version is given.
     kube_version="$(jq -r '.KUBE_VERSION' ${VERSIONS_FILE})"
   fi
+  echo "${kube_version}"
+}
+
+function push() {
+  local -r kube_version="$(get_version)"
 
   kube::log::status "Pushing hyperkube image to the registry"
   gcloud docker -- push "${KUBE_REGISTRY}/hyperkube-amd64:${kube_version}"

federation/manifests/federation-apiserver-deployment.yaml

@@ -17,7 +17,7 @@ spec:
       - name: apiserver
         image: {{.FEDERATION_APISERVER_IMAGE_REPO}}:{{.FEDERATION_APISERVER_IMAGE_TAG}}
         command:
-        - /usr/local/bin/hyperkube
+        - /hyperkube
         - federation-apiserver
         - --bind-address=0.0.0.0
         - --etcd-servers=http://localhost:2379

federation/manifests/federation-controller-manager-deployment.yaml

@@ -31,7 +31,7 @@ spec:
           mountPath: "/etc/federation/controller-manager"
         image: {{.FEDERATION_CONTROLLER_MANAGER_IMAGE_REPO}}:{{.FEDERATION_CONTROLLER_MANAGER_IMAGE_TAG}}
         command:
-        - /usr/local/bin/hyperkube
+        - /hyperkube
         - federation-controller-manager
         - --master=https://{{.FEDERATION_APISERVER_DEPLOYMENT_NAME}}:443
         - --kubeconfig=/etc/federation/controller-manager/kubeconfig

hack/e2e-internal/e2e-up.sh

@@ -43,14 +43,11 @@ if [[ "${FEDERATION:-}" == "true" ]]; then
         )
         cur_ip_octet2="$((cur_ip_octet2 + 1))"
     done
-    tagfile="${KUBE_ROOT}/federation/manifests/federated-image.tag"
-    if [[ ! -f "$tagfile" ]]; then
-        echo "FATAL: tagfile ${tagfile} does not exist. Make sure that you have run build/push-federation-images.sh"
-        exit 1
-    fi
-    export FEDERATION_IMAGE_TAG="$(cat "${KUBE_ROOT}/federation/manifests/federated-image.tag")"
 
-    "${KUBE_ROOT}/federation/cluster/federation-up.sh"
+    # Sets ${CLUSTER_CONTEXT}
+    kubeconfig-federation-context "${zone}"
+
+    "${KUBE_ROOT}/federation/cluster/federation-up.sh" "${CLUSTER_CONTEXT}"
 else
     test-setup
 fi

hack/ginkgo-e2e.sh

@@ -128,6 +128,7 @@ export PATH=$(dirname "${e2e_test}"):"${PATH}"
   --node-instance-group="${NODE_INSTANCE_GROUP:-}" \
   --prefix="${KUBE_GCE_INSTANCE_PREFIX:-e2e}" \
   --network="${KUBE_GCE_NETWORK:-${KUBE_GKE_NETWORK:-e2e}}" \
+  --federated-kube-context="${FEDERATION_KUBE_CONTEXT:-e2e-federation}" \
   ${KUBE_CONTAINER_RUNTIME:+"--container-runtime=${KUBE_CONTAINER_RUNTIME}"} \
   ${MASTER_OS_DISTRIBUTION:+"--master-os-distro=${MASTER_OS_DISTRIBUTION}"} \
   ${NODE_OS_DISTRIBUTION:+"--node-os-distro=${NODE_OS_DISTRIBUTION}"} \

test/e2e/framework/framework.go

@@ -750,7 +750,7 @@ func (f *Framework) GetUnderlyingFederatedContexts() []E2EContext {
 
 	e2eContexts := []E2EContext{}
 	for _, context := range kubeconfig.Contexts {
-		if strings.HasPrefix(context.Name, "federation") && context.Name != "federation-cluster" {
+		if strings.HasPrefix(context.Name, "federation") && context.Name != federatedKubeContext {
 
 			user := kubeconfig.findUser(context.Context.User)
 			if user == nil {

test/e2e/framework/test_context.go

@@ -163,7 +163,7 @@ func RegisterClusterFlags() {
 	flag.StringVar(&TestContext.KubeConfig, clientcmd.RecommendedConfigPathFlag, os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to kubeconfig containing embedded authinfo.")
 	flag.StringVar(&TestContext.KubeContext, clientcmd.FlagContext, "", "kubeconfig context to use/override. If unset, will use value from 'current-context'")
 	flag.StringVar(&TestContext.KubeAPIContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "ContentType used to communicate with apiserver")
-	flag.StringVar(&federatedKubeContext, "federated-kube-context", "federation-cluster", "kubeconfig context for federation-cluster.")
+	flag.StringVar(&federatedKubeContext, "federated-kube-context", "e2e-federation", "kubeconfig context for federation.")
 
 	flag.StringVar(&TestContext.KubeVolumeDir, "volume-dir", "/var/lib/kubelet", "Path to the directory containing the kubelet volumes.")
 	flag.StringVar(&TestContext.CertDir, "cert-dir", "", "Path to the directory containing the certs. Default is empty, which doesn't use certs.")

test/e2e/framework/util.go

@@ -363,7 +363,7 @@ func SkipUnlessServerVersionGTE(v *utilversion.Version, c discovery.ServerVersio
 func SkipUnlessFederated(c clientset.Interface) {
 	federationNS := os.Getenv("FEDERATION_NAMESPACE")
 	if federationNS == "" {
-		federationNS = "federation"
+		federationNS = "federation-system"
 	}
 
 	_, err := c.Core().Namespaces().Get(federationNS, metav1.GetOptions{})