Merge pull request #5264 from jlowdermilk/kube-up

Standalone kubeconfig for gce kube-up (mulligan)

cluster/common.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Common utilites for kube-up/kube-down
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
+
+# Generate kubeconfig data for the created cluster.
+# Assumed vars:
+#   KUBE_USER
+#   KUBE_PASSWORD
+#   KUBE_MASTER_IP
+#   KUBECONFIG
+#
+#   KUBE_CERT
+#   KUBE_KEY
+#   CA_CERT
+#   CONTEXT
+function create-kubeconfig() {
+  local kubectl="${KUBE_ROOT}/cluster/kubectl.sh"
+
+  # KUBECONFIG determines the file we write to, but it may not exist yet
+  if [[ ! -e "${KUBECONFIG}" ]]; then
+    mkdir -p $(dirname "${KUBECONFIG}")
+    touch "${KUBECONFIG}"
+  fi
+  "${kubectl}" config set-cluster "${CONTEXT}" --server="https://${KUBE_MASTER_IP}" \
+                                               --certificate-authority="${CA_CERT}" \
+                                               --embed-certs=true
+  "${kubectl}" config set-credentials "${CONTEXT}" --username="${KUBE_USER}" \
+                                                --password="${KUBE_PASSWORD}" \
+                                                --client-certificate="${KUBE_CERT}" \
+                                                --client-key="${KUBE_KEY}" \
+                                                --embed-certs=true
+  "${kubectl}" config set-context "${CONTEXT}" --cluster="${CONTEXT}" --user="${CONTEXT}"
+  "${kubectl}" config use-context "${CONTEXT}"  --cluster="${CONTEXT}"
+
+   echo "Wrote config for ${CONTEXT} to ${KUBECONFIG}"
+}
+
+# Clear kubeconfig data for a context
+# Assumed vars:
+#   KUBECONFIG
+#   CONTEXT
+function clear-kubeconfig() {
+  local kubectl="${KUBE_ROOT}/cluster/kubectl.sh"
+  "${kubectl}" config unset "clusters.${CONTEXT}"
+  "${kubectl}" config unset "users.${CONTEXT}"
+  "${kubectl}" config unset "contexts.${CONTEXT}"
+
+  local current
+  current=$("${kubectl}" config view -o template --template='{{ index . "current-context" }}')
+  if [[ "${current}" == "${CONTEXT}" ]]; then
+    "${kubectl}" config unset current-context
+  fi
+
+  echo "Cleared config for ${CONTEXT} from ${KUBECONFIG}"
+}

cluster/gce/util.sh

@@ -20,6 +20,7 @@
 # config-default.sh.
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
 source "${KUBE_ROOT}/cluster/gce/${KUBE_CONFIG_FILE-"config-default.sh"}"
+source "${KUBE_ROOT}/cluster/common.sh"
 
 NODE_INSTANCE_PREFIX="${INSTANCE_PREFIX}-minion"
 
@@ -234,17 +235,16 @@ function detect-master () {
 #   KUBE_USER
 #   KUBE_PASSWORD
 function get-password {
-  # go template to extract the auth-path of the current-context user
+  # templates to extract the username,password for the current-context user
   # Note: we save dot ('.') to $dot because the 'with' action overrides dot
-  local template='{{$dot := .}}{{with $ctx := index $dot "current-context"}}{{$user := index $dot "contexts" $ctx "user"}}{{index $dot "users" $user "auth-path"}}{{end}}'
+  local username='{{$dot := .}}{{with $ctx := index $dot "current-context"}}{{$user := index $dot "contexts" $ctx "user"}}{{index $dot "users" $user "username"}}{{end}}'
-  local file=$("${KUBE_ROOT}/cluster/kubectl.sh" config view -o template --template="${template}")
+  local password='{{$dot := .}}{{with $ctx := index $dot "current-context"}}{{$user := index $dot "contexts" $ctx "user"}}{{index $dot "users" $user "password"}}{{end}}'
-  if [[ ! -z "$file" && -r "$file" ]]; then
+  KUBE_USER=$("${KUBE_ROOT}/cluster/kubectl.sh" config view -o template --template="${username}")
-    KUBE_USER=$(cat "$file" | python -c 'import json,sys;print json.load(sys.stdin)["User"]')
+  KUBE_PASSWORD=$("${KUBE_ROOT}/cluster/kubectl.sh" config view -o template --template="${password}")
-    KUBE_PASSWORD=$(cat "$file" | python -c 'import json,sys;print json.load(sys.stdin)["Password"]')
+  if [[ -z "${KUBE_USER}" || -z "${KUBE_PASSWORD}" ]]; then
-    return
+    KUBE_USER=admin
+    KUBE_PASSWORD=$(python -c 'import string,random; print "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16))')
   fi
-  KUBE_USER=admin
-  KUBE_PASSWORD=$(python -c 'import string,random; print "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16))')
 }
 
 # Set MASTER_HTPASSWD
@@ -643,44 +643,22 @@ function kube-up {
 
   echo "Kubernetes cluster created."
 
-  local kube_cert="kubecfg.crt"
+  # TODO use token instead of basic auth
-  local kube_key="kubecfg.key"
+  export KUBECONFIG="${HOME}/.kube/.kubeconfig"
-  local ca_cert="kubernetes.ca.crt"
+  export KUBE_CERT="/tmp/kubecfg.crt"
-  # TODO use token instead of kube_auth
+  export KUBE_KEY="/tmp/kubecfg.key"
-  local kube_auth="kubernetes_auth"
+  export CA_CERT="/tmp/kubernetes.ca.crt"
-
+  export CONTEXT="${PROJECT}_${INSTANCE_PREFIX}"
-  local kubectl="${KUBE_ROOT}/cluster/kubectl.sh"
-  local context="${PROJECT}_${INSTANCE_PREFIX}"
-  local user="${context}-admin"
-  local config_dir="${HOME}/.kube/${context}"
 
   # TODO: generate ADMIN (and KUBELET) tokens and put those in the master's
   # config file.  Distribute the same way the htpasswd is done.
   (
-   mkdir -p "${config_dir}"
    umask 077
-   gcloud compute ssh --project "${PROJECT}" --zone "$ZONE" "${MASTER_NAME}" --command "sudo cat /srv/kubernetes/kubecfg.crt" >"${config_dir}/${kube_cert}" 2>/dev/null
+   gcloud compute ssh --project "${PROJECT}" --zone "$ZONE" "${MASTER_NAME}" --command "sudo cat /srv/kubernetes/kubecfg.crt" >"${KUBE_CERT}" 2>/dev/null
-   gcloud compute ssh --project "${PROJECT}" --zone "$ZONE" "${MASTER_NAME}" --command "sudo cat /srv/kubernetes/kubecfg.key" >"${config_dir}/${kube_key}" 2>/dev/null
+   gcloud compute ssh --project "${PROJECT}" --zone "$ZONE" "${MASTER_NAME}" --command "sudo cat /srv/kubernetes/kubecfg.key" >"${KUBE_KEY}" 2>/dev/null
-   gcloud compute ssh --project "${PROJECT}" --zone "$ZONE" "${MASTER_NAME}" --command "sudo cat /srv/kubernetes/ca.crt" >"${config_dir}/${ca_cert}" 2>/dev/null
+   gcloud compute ssh --project "${PROJECT}" --zone "$ZONE" "${MASTER_NAME}" --command "sudo cat /srv/kubernetes/ca.crt" >"${CA_CERT}" 2>/dev/null
 
-   "${kubectl}" config set-cluster "${context}" --server="https://${KUBE_MASTER_IP}" --certificate-authority="${config_dir}/${ca_cert}" --global
+   create-kubeconfig
-   "${kubectl}" config set-credentials "${user}" --auth-path="${config_dir}/${kube_auth}" --global
-   "${kubectl}" config set-context "${context}" --cluster="${context}" --user="${user}" --global
-   "${kubectl}" config use-context "${context}" --global
-
-   cat << EOF > "${config_dir}/${kube_auth}"
-{
-  "User": "$KUBE_USER",
-  "Password": "$KUBE_PASSWORD",
-  "CAFile": "${config_dir}/${ca_cert}",
-  "CertFile": "${config_dir}/${kube_cert}",
-  "KeyFile": "${config_dir}/${kube_key}"
-}
-EOF
-
-   chmod 0600 "${config_dir}/${kube_auth}" "${config_dir}/$kube_cert" \
-     "${config_dir}/${kube_key}" "${config_dir}/${ca_cert}"
-   echo "Wrote ${config_dir}/${kube_auth}"
   )
 
   echo "Sanity checking cluster..."
@@ -728,7 +706,7 @@ EOF
   echo
   echo -e "${color_yellow}  https://${KUBE_MASTER_IP}"
   echo
-  echo -e "${color_green}The user name and password to use is located in ${config_dir}/${kube_auth}.${color_norm}"
+  echo -e "${color_green}The user name and password to use is located in ${KUBECONFIG}.${color_norm}"
   echo
 
 }
@@ -823,6 +801,9 @@ function kube-down {
     --quiet \
     "${MASTER_NAME}-ip" || true
 
+  export KUBECONFIG="${HOME}/.kube/.kubeconfig"
+  export CONTEXT="${PROJECT}_${INSTANCE_PREFIX}"
+  clear-kubeconfig
 }
 
 # Update a kubernetes cluster with latest source

cmd/e2e/e2e.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	goruntime "runtime"
 
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
 	"github.com/GoogleCloudPlatform/kubernetes/test/e2e"
 	"github.com/golang/glog"
@@ -27,7 +28,8 @@ import (
 )
 
 var (
-	authConfig = flag.String("auth_config", os.Getenv("HOME")+"/.kubernetes_auth", "Path to the auth info file.")
+	kubeConfig = flag.String(clientcmd.RecommendedConfigPathFlag, "", "Path to kubeconfig containing embeded authinfo. Will use cluster/user info from 'current-context'")
+	authConfig = flag.String("auth_config", "", "Path to the auth info file.")
 	certDir    = flag.String("cert_dir", "", "Path to the directory containing the certs. Default is empty, which doesn't use certs.")
 	gceProject = flag.String("gce_project", "", "The GCE project being used, if applicable")
 	gceZone    = flag.String("gce_zone", "", "GCE zone being used, if applicable")
@@ -61,5 +63,5 @@ func main() {
 		Zone:       *gceZone,
 		MasterName: *masterName,
 	}
-	e2e.RunE2ETests(*authConfig, *certDir, *host, *repoRoot, *provider, gceConfig, *orderseed, *times, *reportDir, testList)
+	e2e.RunE2ETests(*kubeConfig, *authConfig, *certDir, *host, *repoRoot, *provider, gceConfig, *orderseed, *times, *reportDir, testList)
 }

hack/ginkgo-e2e.sh

@@ -93,7 +93,7 @@ elif [[ "${KUBERNETES_PROVIDER}" == "gke" ]]; then
   )
 elif [[ "${KUBERNETES_PROVIDER}" == "gce" ]]; then
   auth_config=(
-    "--auth_config=${HOME}/.kube/${PROJECT}_${INSTANCE_PREFIX}/kubernetes_auth"
+    "--kubeconfig=${HOME}/.kube/.kubeconfig"
   )
 elif [[ "${KUBERNETES_PROVIDER}" == "aws" ]]; then
   auth_config=(

pkg/client/clientcmd/auth_loaders.go

@@ -80,7 +80,7 @@ func promptForString(field string, r io.Reader) string {
 	return result
 }
 
-// NewDefaultAuthLoader is an AuthLoader that parses an AuthInfo object from a file path. It prompts user and creates file if it doesn't exist.
+// NewPromptingAuthLoader is an AuthLoader that parses an AuthInfo object from a file path. It prompts user and creates file if it doesn't exist.
 func NewPromptingAuthLoader(reader io.Reader) *PromptingAuthLoader {
 	return &PromptingAuthLoader{reader}
 }

test/e2e/driver.go

@@ -53,8 +53,8 @@ func (t *testResult) Fail() { *t = false }
 
 // Run each Go end-to-end-test. This function assumes the
 // creation of a test cluster.
-func RunE2ETests(authConfig, certDir, host, repoRoot, provider string, gceConfig *GCEConfig, orderseed int64, times int, reportDir string, testList []string) {
+func RunE2ETests(kubeConfig, authConfig, certDir, host, repoRoot, provider string, gceConfig *GCEConfig, orderseed int64, times int, reportDir string, testList []string) {
-	testContext = testContextType{authConfig, certDir, host, repoRoot, provider, *gceConfig}
+	testContext = testContextType{kubeConfig, authConfig, certDir, host, repoRoot, provider, *gceConfig}
 	util.ReallyCrash = true
 	util.InitLogs()
 	defer util.FlushLogs()

test/e2e/kubectl.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd"
 
 	. "github.com/onsi/ginkgo"
 )
@@ -182,12 +183,17 @@ func getData(c *client.Client, podID string) (*updateDemoData, error) {
 }
 
 func kubectlCmd(args ...string) *exec.Cmd {
-	defaultArgs := []string{"--auth-path=" + testContext.authConfig}
+	defaultArgs := []string{}
-	if testContext.certDir != "" {
+	if testContext.kubeConfig != "" {
-		defaultArgs = append(defaultArgs,
+		defaultArgs = append(defaultArgs, "--"+clientcmd.RecommendedConfigPathFlag+"="+testContext.kubeConfig)
-			fmt.Sprintf("--certificate-authority=%s", filepath.Join(testContext.certDir, "ca.crt")),
+	} else {
-			fmt.Sprintf("--client-certificate=%s", filepath.Join(testContext.certDir, "kubecfg.crt")),
+		defaultArgs = append(defaultArgs, "--"+clientcmd.FlagAuthPath+"="+testContext.authConfig)
-			fmt.Sprintf("--client-key=%s", filepath.Join(testContext.certDir, "kubecfg.key")))
+		if testContext.certDir != "" {
+			defaultArgs = append(defaultArgs,
+				fmt.Sprintf("--certificate-authority=%s", filepath.Join(testContext.certDir, "ca.crt")),
+				fmt.Sprintf("--client-certificate=%s", filepath.Join(testContext.certDir, "kubecfg.crt")),
+				fmt.Sprintf("--client-key=%s", filepath.Join(testContext.certDir, "kubecfg.key")))
+		}
 	}
 	kubectlArgs := append(defaultArgs, args...)
 	// TODO: Remove this once gcloud writes a proper entry in the kubeconfig file.

test/e2e/util.go

@@ -25,8 +25,8 @@ import (
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/clientauth"
-
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -38,6 +38,7 @@ const (
 )
 
 type testContextType struct {
+	kubeConfig string
 	authConfig string
 	certDir    string
 	host       string
@@ -117,32 +118,44 @@ func waitForPodSuccess(c *client.Client, podName string, contName string) error
 }
 
 func loadConfig() (*client.Config, error) {
-	config := &client.Config{
+	switch {
-		Host: testContext.host,
+	case testContext.kubeConfig != "":
+		fmt.Printf(">>> testContext.kubeConfig: %s\n", testContext.kubeConfig)
+		c, err := clientcmd.LoadFromFile(testContext.kubeConfig)
+		if err != nil {
+			return nil, fmt.Errorf("error loading kubeConfig: %v", err.Error())
+		}
+		return clientcmd.NewDefaultClientConfig(*c, &clientcmd.ConfigOverrides{}).ClientConfig()
+	case testContext.authConfig != "":
+		config := &client.Config{
+			Host: testContext.host,
+		}
+		info, err := clientauth.LoadFromFile(testContext.authConfig)
+		if err != nil {
+			return nil, fmt.Errorf("error loading authConfig: %v", err.Error())
+		}
+		// If the certificate directory is provided, set the cert paths to be there.
+		if testContext.certDir != "" {
+			Logf("Expecting certs in %v.", testContext.certDir)
+			info.CAFile = filepath.Join(testContext.certDir, "ca.crt")
+			info.CertFile = filepath.Join(testContext.certDir, "kubecfg.crt")
+			info.KeyFile = filepath.Join(testContext.certDir, "kubecfg.key")
+		}
+		mergedConfig, err := info.MergeWithConfig(*config)
+		return &mergedConfig, err
+	default:
+		return nil, fmt.Errorf("either kubeConfig or authConfig must be specified to load client config")
 	}
-	info, err := clientauth.LoadFromFile(testContext.authConfig)
-	if err != nil {
-		return nil, fmt.Errorf("Error loading auth: %v", err.Error())
-	}
-	// If the certificate directory is provided, set the cert paths to be there.
-	if testContext.certDir != "" {
-		Logf("Expecting certs in %v.", testContext.certDir)
-		info.CAFile = filepath.Join(testContext.certDir, "ca.crt")
-		info.CertFile = filepath.Join(testContext.certDir, "kubecfg.crt")
-		info.KeyFile = filepath.Join(testContext.certDir, "kubecfg.key")
-	}
-	mergedConfig, err := info.MergeWithConfig(*config)
-	return &mergedConfig, err
 }
 
 func loadClient() (*client.Client, error) {
 	config, err := loadConfig()
 	if err != nil {
-		return nil, fmt.Errorf("Error creating client: %v", err.Error())
+		return nil, fmt.Errorf("error creating client: %v", err.Error())
 	}
 	c, err := client.New(config)
 	if err != nil {
-		return nil, fmt.Errorf("Error creating client: %v", err.Error())
+		return nil, fmt.Errorf("error creating client: %v", err.Error())
 	}
 	return c, nil
 }