From 657a7ef6a4b144398d9018c9e1ecbc1f9f50915d Mon Sep 17 00:00:00 2001 From: nikhiljindal Date: Mon, 13 Jun 2016 18:37:07 -0700 Subject: [PATCH 1/3] Create kubeconfig for cluster secrets on other cloud providers as well --- cluster/aws/util.sh | 11 +++++++++++ cluster/gke/config-common.sh | 1 + cluster/gke/util.sh | 16 ++++++++++++++++ cluster/vagrant/util.sh | 10 ++++++++++ 4 files changed, 38 insertions(+) diff --git a/cluster/aws/util.sh b/cluster/aws/util.sh index 3a3d7c759be..76e6a8cd1fb 100755 --- a/cluster/aws/util.sh +++ b/cluster/aws/util.sh @@ -1304,7 +1304,18 @@ function build-config() { export CONTEXT="aws_${INSTANCE_PREFIX}" ( umask 077 + + # Update the user's kubeconfig to include credentials for this apiserver. create-kubeconfig + + if [[ "${FEDERATION:-}" == "true" ]]; then + # Create a kubeconfig with credentials for this apiserver. We will later use + # this kubeconfig to create a secret which the federation control plane can + # use to talk to this apiserver. + KUBECONFIG_DIR=$(dirname ${KUBECONFIG:-$DEFAULT_KUBECONFIG}) + KUBECONFIG="${KUBECONFIG_DIR}/federation/kubernetes-apiserver/${CONTEXT}/kubeconfig" \ + create-kubeconfig + fi ) } diff --git a/cluster/gke/config-common.sh b/cluster/gke/config-common.sh index 9850542611a..f84004ae9a0 100644 --- a/cluster/gke/config-common.sh +++ b/cluster/gke/config-common.sh @@ -26,6 +26,7 @@ NETWORK="${NETWORK:-default}" NETWORK_RANGE="${NETWORK_RANGE:-10.240.0.0/16}" FIREWALL_SSH="${FIREWALL_SSH:-${NETWORK}-allow-ssh}" GCLOUD="${GCLOUD:-gcloud}" +KUBECTL="${KUBECTL:-kubectl}" CMD_GROUP="${CMD_GROUP:-}" GCLOUD_CONFIG_DIR="${GCLOUD_CONFIG_DIR:-${HOME}/.config/gcloud/kubernetes}" MACHINE_TYPE="${MACHINE_TYPE:-n1-standard-2}" diff --git a/cluster/gke/util.sh b/cluster/gke/util.sh index ae7064d1c15..607b8efdeeb 100755 --- a/cluster/gke/util.sh +++ b/cluster/gke/util.sh @@ -193,6 +193,22 @@ function kube-up() { # Bring up the cluster. "${GCLOUD}" ${CMD_GROUP:-} container clusters create "${CLUSTER_NAME}" "${create_args[@]}" + if [[ "${FEDERATION:-}" == "true" ]]; then + # Create a kubeconfig with credentials for this apiserver. We will later use + # this kubeconfig to create a secret which the federation control plane can + # use to talk to this apiserver. + KUBECONFIG=${KUBECONFIG:-${HOME}/.kube/config} + KUBECONFIG_DIR=$(dirname $KUBECONFIG) + CONTEXT=$($KUBECTL config current-context) + DEST_KUBECONFIG="${KUBECONFIG_DIR}/federation/kubernetes-apiserver/${CONTEXT}/kubeconfig" + mkdir -p $(dirname $DEST_KUBECONFIG) >&2 + # TODO: Original kubeconfig can contain credential information about + # other clusters as well. Extract the information about only this cluster + # and then create a file with that. + # For now, we use the whole kubeconfig file. + cp $KUBECONFIG $DEST_KUBECONFIG >&2 + fi + if [[ ! -z "${HEAPSTER_MACHINE_TYPE:-}" ]]; then "${GCLOUD}" ${CMD_GROUP:-} container node-pools create "heapster-pool" --cluster "${CLUSTER_NAME}" --num-nodes=1 --machine-type="${HEAPSTER_MACHINE_TYPE}" "${shared_args[@]}" fi diff --git a/cluster/vagrant/util.sh b/cluster/vagrant/util.sh index a04dd8c9d1a..b97afba7a05 100755 --- a/cluster/vagrant/util.sh +++ b/cluster/vagrant/util.sh @@ -295,7 +295,17 @@ function kube-up { vagrant ssh master -- sudo cat /srv/kubernetes/kubecfg.key >"${KUBE_KEY}" 2>/dev/null vagrant ssh master -- sudo cat /srv/kubernetes/ca.crt >"${CA_CERT}" 2>/dev/null + # Update the user's kubeconfig to include credentials for this apiserver. create-kubeconfig + + if [[ "${FEDERATION:-}" == "true" ]]; then + # Create a kubeconfig with credentials for this apiserver. We will later use + # this kubeconfig to create a secret which the federation control plane can + # use to talk to this apiserver. + KUBECONFIG_DIR=$(dirname ${KUBECONFIG:-$DEFAULT_KUBECONFIG}) + KUBECONFIG="${KUBECONFIG_DIR}/federation/kubernetes-apiserver/${CONTEXT}/kubeconfig" \ + create-kubeconfig + fi ) verify-cluster From 9026195614245538630c62a950c99de7baad370d Mon Sep 17 00:00:00 2001 From: nikhiljindal Date: Wed, 15 Jun 2016 15:19:49 -0700 Subject: [PATCH 2/3] Updating the code to fix the context name before using it as secret and cluster names --- cluster/gke/util.sh | 4 +++ federation/cluster/common.sh | 10 ++++++- test/e2e/framework/federation_util.go | 42 +++++++++++++++++++++++++++ test/e2e/framework/framework.go | 10 ++++++- 4 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 test/e2e/framework/federation_util.go diff --git a/cluster/gke/util.sh b/cluster/gke/util.sh index 607b8efdeeb..70ff52ffd43 100755 --- a/cluster/gke/util.sh +++ b/cluster/gke/util.sh @@ -206,6 +206,10 @@ function kube-up() { # other clusters as well. Extract the information about only this cluster # and then create a file with that. # For now, we use the whole kubeconfig file. + # Note: This is not as dangerous as it sounds because this code path is + # only expected to run during tests. Users are not expected to set + # FEDERATION=true while bringing up their kubernetes clusters. + # But there is nothing stopping them from doing so. cp $KUBECONFIG $DEST_KUBECONFIG >&2 fi diff --git a/federation/cluster/common.sh b/federation/cluster/common.sh index a326543cab1..7955dd65817 100644 --- a/federation/cluster/common.sh +++ b/federation/cluster/common.sh @@ -159,10 +159,18 @@ function create-federation-api-objects { $host_kubectl create secret generic federation-apiserver-secret --from-file="${KUBECONFIG_DIR}/federation/federation-apiserver/kubeconfig" --namespace="${FEDERATION_NAMESPACE}" # Create secrets with all the kubernetes-apiserver's kubeconfigs. + # Note: This is used only by the test setup (where kubernetes clusters are + # brought up with FEDERATION=true). Users are expected to create this secret + # themselves. for dir in ${KUBECONFIG_DIR}/federation/kubernetes-apiserver/*; do # We create a secret with the same name as the directory name (which is - # same as cluster name in kubeconfig) + # same as cluster name in kubeconfig). + # Massage the name so that it is valid (should not contain "_" and max 253 + # chars) name=$(basename $dir) + name=$(echo "$name" | sed -e "s/_/-/g") # Replace "_" by "-" + name=${name:0:252} + echo "Creating secret with name: $name" $host_kubectl create secret generic ${name} --from-file="${dir}/kubeconfig" --namespace="${FEDERATION_NAMESPACE}" done diff --git a/test/e2e/framework/federation_util.go b/test/e2e/framework/federation_util.go new file mode 100644 index 00000000000..a018d2b6021 --- /dev/null +++ b/test/e2e/framework/federation_util.go @@ -0,0 +1,42 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "fmt" + "regexp" + + "k8s.io/kubernetes/pkg/api/validation" + validation_util "k8s.io/kubernetes/pkg/util/validation" +) + +// GetValidDNSSubdomainName massages the given name to be a valid dns subdomain name. +// Most resources (such as secrets, clusters) require the names to be valid dns subdomain. +// This is a generic function (not specific to federation). Should be moved to a more generic location if others want to use it. +func GetValidDNSSubdomainName(name string) (string, error) { + // "_" are not allowed. Replace them by "-". + name = regexp.MustCompile("_").ReplaceAllLiteralString(name, "-") + maxLength := validation_util.DNS1123SubdomainMaxLength + if len(name) > maxLength { + name = name[0 : maxLength-1] + } + // Verify that name now passes the validation. + if errors := validation.NameIsDNSSubdomain(name, false); len(errors) != 0 { + return "", fmt.Errorf("errors in converting name to a valid DNS subdomain %s", errors) + } + return name, nil +} diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index c982b221915..cdc7e27a8cf 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -585,6 +585,9 @@ func (kc *KubeConfig) findCluster(name string) *KubeCluster { } type E2EContext struct { + // Raw context name, + RawName string `yaml:"rawName"` + // A valid dns subdomain which can be used as the name of kubernetes resources. Name string `yaml:"name"` Cluster *KubeCluster `yaml:"cluster"` User *KubeUser `yaml:"user"` @@ -615,8 +618,13 @@ func (f *Framework) GetUnderlyingFederatedContexts() []E2EContext { Failf("Could not find cluster for context %+v", context) } + dnsSubdomainName, err := GetValidDNSSubdomainName(context.Name) + if err != nil { + Failf("Could not convert context name %s to a valid dns subdomain name, error: %s", context.Name, err) + } e2eContexts = append(e2eContexts, E2EContext{ - Name: context.Name, + RawName: context.Name, + Name: dnsSubdomainName, Cluster: cluster, User: user, }) From 864b267a8d3f4a57c12c978d2f97e44414438594 Mon Sep 17 00:00:00 2001 From: nikhiljindal Date: Mon, 20 Jun 2016 16:10:47 -0700 Subject: [PATCH 3/3] Simplifying the code to create kubeconfig --- cluster/aws/util.sh | 9 +-------- cluster/common.sh | 13 +++++++++++++ cluster/gce/util.sh | 9 +-------- cluster/gke/config-common.sh | 1 - cluster/gke/util.sh | 21 ++------------------- cluster/vagrant/util.sh | 9 +-------- 6 files changed, 18 insertions(+), 44 deletions(-) diff --git a/cluster/aws/util.sh b/cluster/aws/util.sh index 76e6a8cd1fb..8d0f03090ef 100755 --- a/cluster/aws/util.sh +++ b/cluster/aws/util.sh @@ -1308,14 +1308,7 @@ function build-config() { # Update the user's kubeconfig to include credentials for this apiserver. create-kubeconfig - if [[ "${FEDERATION:-}" == "true" ]]; then - # Create a kubeconfig with credentials for this apiserver. We will later use - # this kubeconfig to create a secret which the federation control plane can - # use to talk to this apiserver. - KUBECONFIG_DIR=$(dirname ${KUBECONFIG:-$DEFAULT_KUBECONFIG}) - KUBECONFIG="${KUBECONFIG_DIR}/federation/kubernetes-apiserver/${CONTEXT}/kubeconfig" \ - create-kubeconfig - fi + create-kubeconfig-for-federation ) } diff --git a/cluster/common.sh b/cluster/common.sh index dd7dd01c89a..78f0835777d 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -152,6 +152,19 @@ function clear-kubeconfig() { echo "Cleared config for ${CONTEXT} from ${KUBECONFIG}" } +# Creates a kubeconfig file with the credentials for only the current-context +# cluster. This is used by federation to create secrets in test setup. +function create-kubeconfig-for-federation() { + if [[ "${FEDERATION:-}" == "true" ]]; then + echo "creating kubeconfig for federation secret" + local kubectl="${KUBE_ROOT}/cluster/kubectl.sh" + local cc=$("${kubectl}" config view -o jsonpath='{.current-context}') + KUBECONFIG_DIR=$(dirname ${KUBECONFIG:-$DEFAULT_KUBECONFIG}) + KUBECONFIG_PATH="${KUBECONFIG_DIR}/federation/kubernetes-apiserver/${cc}" + mkdir -p "${KUBECONFIG_PATH}" + "${kubectl}" config view --minify --flatten > "${KUBECONFIG_PATH}/kubeconfig" + fi +} function tear_down_alive_resources() { local kubectl="${KUBE_ROOT}/cluster/kubectl.sh" diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index ec0d7daf6ef..64a0af42e32 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -920,14 +920,7 @@ function check-cluster() { # Update the user's kubeconfig to include credentials for this apiserver. create-kubeconfig - if [[ "${FEDERATION:-}" == "true" ]]; then - # Create a kubeconfig with credentials for this apiserver. We will later use - # this kubeconfig to create a secret which the federation control plane can - # use to talk to this apiserver. - KUBECONFIG_DIR=$(dirname ${KUBECONFIG:-$DEFAULT_KUBECONFIG}) - KUBECONFIG="${KUBECONFIG_DIR}/federation/kubernetes-apiserver/${CONTEXT}/kubeconfig" \ - create-kubeconfig - fi + create-kubeconfig-for-federation ) # ensures KUBECONFIG is set diff --git a/cluster/gke/config-common.sh b/cluster/gke/config-common.sh index f84004ae9a0..9850542611a 100644 --- a/cluster/gke/config-common.sh +++ b/cluster/gke/config-common.sh @@ -26,7 +26,6 @@ NETWORK="${NETWORK:-default}" NETWORK_RANGE="${NETWORK_RANGE:-10.240.0.0/16}" FIREWALL_SSH="${FIREWALL_SSH:-${NETWORK}-allow-ssh}" GCLOUD="${GCLOUD:-gcloud}" -KUBECTL="${KUBECTL:-kubectl}" CMD_GROUP="${CMD_GROUP:-}" GCLOUD_CONFIG_DIR="${GCLOUD_CONFIG_DIR:-${HOME}/.config/gcloud/kubernetes}" MACHINE_TYPE="${MACHINE_TYPE:-n1-standard-2}" diff --git a/cluster/gke/util.sh b/cluster/gke/util.sh index 70ff52ffd43..0fb3cef8cd9 100755 --- a/cluster/gke/util.sh +++ b/cluster/gke/util.sh @@ -23,6 +23,7 @@ KUBE_PROMPT_FOR_UPDATE=y KUBE_SKIP_UPDATE=${KUBE_SKIP_UPDATE-"n"} KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. source "${KUBE_ROOT}/cluster/gke/${KUBE_CONFIG_FILE:-config-default.sh}" +source "${KUBE_ROOT}/cluster/common.sh" source "${KUBE_ROOT}/cluster/lib/util.sh" # Perform preparations required to run e2e tests @@ -193,25 +194,7 @@ function kube-up() { # Bring up the cluster. "${GCLOUD}" ${CMD_GROUP:-} container clusters create "${CLUSTER_NAME}" "${create_args[@]}" - if [[ "${FEDERATION:-}" == "true" ]]; then - # Create a kubeconfig with credentials for this apiserver. We will later use - # this kubeconfig to create a secret which the federation control plane can - # use to talk to this apiserver. - KUBECONFIG=${KUBECONFIG:-${HOME}/.kube/config} - KUBECONFIG_DIR=$(dirname $KUBECONFIG) - CONTEXT=$($KUBECTL config current-context) - DEST_KUBECONFIG="${KUBECONFIG_DIR}/federation/kubernetes-apiserver/${CONTEXT}/kubeconfig" - mkdir -p $(dirname $DEST_KUBECONFIG) >&2 - # TODO: Original kubeconfig can contain credential information about - # other clusters as well. Extract the information about only this cluster - # and then create a file with that. - # For now, we use the whole kubeconfig file. - # Note: This is not as dangerous as it sounds because this code path is - # only expected to run during tests. Users are not expected to set - # FEDERATION=true while bringing up their kubernetes clusters. - # But there is nothing stopping them from doing so. - cp $KUBECONFIG $DEST_KUBECONFIG >&2 - fi + create-kubeconfig-for-federation if [[ ! -z "${HEAPSTER_MACHINE_TYPE:-}" ]]; then "${GCLOUD}" ${CMD_GROUP:-} container node-pools create "heapster-pool" --cluster "${CLUSTER_NAME}" --num-nodes=1 --machine-type="${HEAPSTER_MACHINE_TYPE}" "${shared_args[@]}" diff --git a/cluster/vagrant/util.sh b/cluster/vagrant/util.sh index b97afba7a05..48a7d21b419 100755 --- a/cluster/vagrant/util.sh +++ b/cluster/vagrant/util.sh @@ -298,14 +298,7 @@ function kube-up { # Update the user's kubeconfig to include credentials for this apiserver. create-kubeconfig - if [[ "${FEDERATION:-}" == "true" ]]; then - # Create a kubeconfig with credentials for this apiserver. We will later use - # this kubeconfig to create a secret which the federation control plane can - # use to talk to this apiserver. - KUBECONFIG_DIR=$(dirname ${KUBECONFIG:-$DEFAULT_KUBECONFIG}) - KUBECONFIG="${KUBECONFIG_DIR}/federation/kubernetes-apiserver/${CONTEXT}/kubeconfig" \ - create-kubeconfig - fi + create-kubeconfig-for-federation ) verify-cluster