kubeadm: EnvParam struct and GlobalEnvParam

Previously, GetEnvParams (now called SetEnvParams) had no way of being altered unless
it was through enviroment variables. These changes allow for a global
EnvParam to be set and also altered while still initally getting their value from
set enviroment variables. This change is especially helpful for testing
(see kubeadm/app/util/kubeconfig_test.go).

cmd/kubeadm/app/apis/kubeadm/env.go

@@ -23,9 +23,11 @@ import (
 	"strings"
 )
 
+var GlobalEnvParams = SetEnvParams()
+
 // TODO(phase2) use componentconfig
 // we need some params for testing etc, let's keep these hidden for now
-func GetEnvParams() map[string]string {
+func SetEnvParams() *EnvParams {
 
 	envParams := map[string]string{
 		// TODO(phase1+): Mode prefix and host_pki_path to another place as constants, and use them everywhere
@@ -45,5 +47,13 @@ func GetEnvParams() map[string]string {
 		}
 	}
 
-	return envParams
+        return &EnvParams{
+                KubernetesDir:     envParams["kubernetes_dir"],
+                HostPKIPath:       envParams["host_pki_path"],
+                HostEtcdPath:      envParams["host_etcd_path"],
+                HyperkubeImage:    envParams["hyperkube_image"],
+                DiscoveryImage:    envParams["discovery_image"],
+                EtcdImage:         envParams["etcd_image"],
+                ComponentLoglevel: envParams["component_loglevel"],
+        }
 }

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -18,6 +18,16 @@ package kubeadm
 
 import "k8s.io/kubernetes/pkg/api/unversioned"
 
+type EnvParams struct {
+        KubernetesDir     string
+        HostPKIPath       string
+        HostEtcdPath      string
+        HyperkubeImage    string
+        DiscoveryImage    string
+        EtcdImage         string
+        ComponentLoglevel string
+}
+
 type MasterConfiguration struct {
 	unversioned.TypeMeta
 

cmd/kubeadm/app/master/addons.go

@@ -31,13 +31,12 @@ import (
 )
 
 func createKubeProxyPodSpec(cfg *kubeadmapi.MasterConfiguration) api.PodSpec {
-	envParams := kubeadmapi.GetEnvParams()
 	privilegedTrue := true
 	return api.PodSpec{
 		SecurityContext: &api.PodSecurityContext{HostNetwork: true},
 		Containers: []api.Container{{
 			Name:            kubeProxy,
-			Image:           images.GetCoreImage(images.KubeProxyImage, cfg, envParams["hyperkube_image"]),
+                        Image:           images.GetCoreImage(images.KubeProxyImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
 			Command:         append(getProxyCommand(cfg), "--kubeconfig=/run/kubeconfig"),
 			SecurityContext: &api.SecurityContext{Privileged: &privilegedTrue},
 			VolumeMounts: []api.VolumeMount{
@@ -67,7 +66,7 @@ func createKubeProxyPodSpec(cfg *kubeadmapi.MasterConfiguration) api.PodSpec {
 			{
 				Name: "kubeconfig",
 				VolumeSource: api.VolumeSource{
-					HostPath: &api.HostPathVolumeSource{Path: path.Join(envParams["kubernetes_dir"], "kubelet.conf")},
+                                        HostPath: &api.HostPathVolumeSource{Path: path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "kubelet.conf")},
 				},
 			},
 			{

cmd/kubeadm/app/master/discovery.go

@@ -61,7 +61,6 @@ func encodeKubeDiscoverySecretData(cfg *kubeadmapi.MasterConfiguration, caCert *
 }
 
 func newKubeDiscoveryPodSpec(cfg *kubeadmapi.MasterConfiguration) api.PodSpec {
-	envParams := kubeadmapi.GetEnvParams()
 	return api.PodSpec{
 		// We have to use host network namespace, as `HostPort`/`HostIP` are Docker's
 		// buisness and CNI support isn't quite there yet (except for kubenet)
@@ -70,7 +69,7 @@ func newKubeDiscoveryPodSpec(cfg *kubeadmapi.MasterConfiguration) api.PodSpec {
 		SecurityContext: &api.PodSecurityContext{HostNetwork: true},
 		Containers: []api.Container{{
 			Name:    kubeDiscoveryName,
-			Image:   envParams["discovery_image"],
+                        Image:   kubeadmapi.GlobalEnvParams.DiscoveryImage,
 			Command: []string{"/usr/local/bin/kube-discovery"},
 			VolumeMounts: []api.VolumeMount{{
 				Name:      kubeDiscoverySecretName,

cmd/kubeadm/app/master/manifests.go

@@ -54,12 +54,11 @@ const (
 // WriteStaticPodManifests builds manifest objects based on user provided configuration and then dumps it to disk
 // where kubelet will pick and schedule them.
 func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error {
-	envParams := kubeadmapi.GetEnvParams()
 	// Prepare static pod specs
 	staticPodSpecs := map[string]api.Pod{
 		kubeAPIServer: componentPod(api.Container{
 			Name:          kubeAPIServer,
-			Image:         images.GetCoreImage(images.KubeAPIServerImage, cfg, envParams["hyperkube_image"]),
+                        Image:         images.GetCoreImage(images.KubeAPIServerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
 			Command:       getAPIServerCommand(cfg),
 			VolumeMounts:  []api.VolumeMount{certsVolumeMount(), k8sVolumeMount()},
 			LivenessProbe: componentProbe(8080, "/healthz"),
@@ -67,7 +66,7 @@ func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error {
 		}, certsVolume(cfg), k8sVolume(cfg)),
 		kubeControllerManager: componentPod(api.Container{
 			Name:          kubeControllerManager,
-			Image:         images.GetCoreImage(images.KubeControllerManagerImage, cfg, envParams["hyperkube_image"]),
+                        Image:         images.GetCoreImage(images.KubeControllerManagerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
 			Command:       getControllerManagerCommand(cfg),
 			VolumeMounts:  []api.VolumeMount{certsVolumeMount(), k8sVolumeMount()},
 			LivenessProbe: componentProbe(10252, "/healthz"),
@@ -75,7 +74,7 @@ func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error {
 		}, certsVolume(cfg), k8sVolume(cfg)),
 		kubeScheduler: componentPod(api.Container{
 			Name:          kubeScheduler,
-			Image:         images.GetCoreImage(images.KubeSchedulerImage, cfg, envParams["hyperkube_image"]),
+                        Image:         images.GetCoreImage(images.KubeSchedulerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
 			Command:       getSchedulerCommand(cfg),
 			LivenessProbe: componentProbe(10251, "/healthz"),
 			Resources:     componentResources("100m"),
@@ -93,7 +92,7 @@ func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error {
 				"--data-dir=/var/etcd/data",
 			},
 			VolumeMounts:  []api.VolumeMount{certsVolumeMount(), etcdVolumeMount(), k8sVolumeMount()},
-			Image:         images.GetCoreImage(images.KubeEtcdImage, cfg, envParams["etcd_image"]),
+                        Image:         images.GetCoreImage(images.KubeEtcdImage, cfg, kubeadmapi.GlobalEnvParams.EtcdImage),
 			LivenessProbe: componentProbe(2379, "/health"),
 			Resources:     componentResources("200m"),
 			SecurityContext: &api.SecurityContext{
@@ -108,7 +107,7 @@ func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error {
 		}, certsVolume(cfg), etcdVolume(cfg), k8sVolume(cfg))
 	}
 
-	manifestsPath := path.Join(envParams["kubernetes_dir"], "manifests")
+        manifestsPath := path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "manifests")
 	if err := os.MkdirAll(manifestsPath, 0700); err != nil {
 		return fmt.Errorf("<master/manifests> failed to create directory %q [%v]", manifestsPath, err)
 	}
@@ -127,11 +126,10 @@ func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error {
 
 // etcdVolume exposes a path on the host in order to guarantee data survival during reboot.
 func etcdVolume(cfg *kubeadmapi.MasterConfiguration) api.Volume {
-	envParams := kubeadmapi.GetEnvParams()
 	return api.Volume{
 		Name: "etcd",
 		VolumeSource: api.VolumeSource{
-			HostPath: &api.HostPathVolumeSource{Path: envParams["host_etcd_path"]},
+                        HostPath: &api.HostPathVolumeSource{Path: kubeadmapi.GlobalEnvParams.HostEtcdPath},
 		},
 	}
 }
@@ -162,11 +160,10 @@ func certsVolumeMount() api.VolumeMount {
 }
 
 func k8sVolume(cfg *kubeadmapi.MasterConfiguration) api.Volume {
-	envParams := kubeadmapi.GetEnvParams()
 	return api.Volume{
 		Name: "pki",
 		VolumeSource: api.VolumeSource{
-			HostPath: &api.HostPathVolumeSource{Path: envParams["kubernetes_dir"]},
+                        HostPath: &api.HostPathVolumeSource{Path: kubeadmapi.GlobalEnvParams.KubernetesDir},
 		},
 	}
 }
@@ -222,13 +219,12 @@ func componentPod(container api.Container, volumes ...api.Volume) api.Pod {
 }
 
 func getComponentBaseCommand(component string) (command []string) {
-	envParams := kubeadmapi.GetEnvParams()
+        if kubeadmapi.GlobalEnvParams.HyperkubeImage != "" {
-	if envParams["hyperkube_image"] != "" {
 		command = []string{"/hyperkube", component}
 	} else {
 		command = []string{"kube-" + component}
 	}
-	command = append(command, envParams["component_loglevel"])
+        command = append(command, kubeadmapi.GlobalEnvParams.ComponentLoglevel)
 	return
 }
 

cmd/kubeadm/app/master/pki.go

@@ -158,7 +158,7 @@ func CreatePKIAssets(cfg *kubeadmapi.MasterConfiguration) (*rsa.PrivateKey, *x50
 	}
 	altNames.DNSNames = append(altNames.DNSNames, cfg.API.ExternalDNSNames...)
 
-	pkiPath := path.Join(kubeadmapi.GetEnvParams()["host_pki_path"])
+        pkiPath := path.Join(kubeadmapi.GlobalEnvParams.HostPKIPath)
 
 	caKey, caCert, err := newCertificateAuthority()
 	if err != nil {

cmd/kubeadm/app/master/tokens.go

@@ -48,12 +48,12 @@ func generateTokenIfNeeded(s *kubeadmapi.Secrets) error {
 }
 
 func CreateTokenAuthFile(s *kubeadmapi.Secrets) error {
-	tokenAuthFilePath := path.Join(kubeadmapi.GetEnvParams()["host_pki_path"], "tokens.csv")
+        tokenAuthFilePath := path.Join(kubeadmapi.GlobalEnvParams.HostPKIPath, "tokens.csv")
 	if err := generateTokenIfNeeded(s); err != nil {
 		return fmt.Errorf("<master/tokens> failed to generate token(s) [%v]", err)
 	}
-	if err := os.MkdirAll(kubeadmapi.GetEnvParams()["host_pki_path"], 0700); err != nil {
+        if err := os.MkdirAll(kubeadmapi.GlobalEnvParams.HostPKIPath, 0700); err != nil {
-		return fmt.Errorf("<master/tokens> failed to create directory %q [%v]", kubeadmapi.GetEnvParams()["host_pki_path"], err)
+                return fmt.Errorf("<master/tokens> failed to create directory %q [%v]", kubeadmapi.GlobalEnvParams.HostPKIPath, err)
 	}
 	serialized := []byte(fmt.Sprintf("%s,kubeadm-node-csr,%s,system:kubelet-bootstrap\n", s.BearerToken, uuid.NewUUID()))
 	// DumpReaderToFile create a file with mode 0600

cmd/kubeadm/app/node/BUILD

@@ -33,3 +33,17 @@ go_library(
         "//vendor:github.com/square/go-jose",
     ],
 )
+
+go_test(
+    name = "go_default_test",
+    srcs = ["bootstrap_test.go"],
+    library = "go_default_library",
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/api/unversioned:go_default_library",
+        "//pkg/client/clientset_generated/internalclientset:go_default_library",
+        "//pkg/client/restclient:go_default_library",
+        "//pkg/client/typed/discovery:go_default_library",
+        "//pkg/version:go_default_library",
+    ],
+)

cmd/kubeadm/app/util/BUILD

@@ -30,8 +30,16 @@ go_library(
 
 go_test(
     name = "go_default_test",
-    srcs = ["tokens_test.go"],
+    srcs = [
+        "error_test.go",
+        "kubeconfig_test.go",
+        "tokens_test.go",
+    ],
     library = "go_default_library",
     tags = ["automanaged"],
-    deps = ["//cmd/kubeadm/app/apis/kubeadm:go_default_library"],
+    deps = [
+        "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
+        "//cmd/kubeadm/app/preflight:go_default_library",
+        "//pkg/client/unversioned/clientcmd/api:go_default_library",
+    ],
 )

cmd/kubeadm/app/util/error_test.go

@@ -34,11 +34,12 @@ func TestCheckErr(t *testing.T) {
                 expected int
         }{
                 {nil, 0},
-                {fmt.Errorf(""), 1},
+                {fmt.Errorf(""), DefaultErrorExitCode},
-                {&preflight.PreFlightError{}, 2},
+                {&preflight.PreFlightError{}, PreFlight},
         }
 
         for _, rt := range tokenTest {
+                codeReturned = 0
                 checkErr("", rt.e, errHandle)
                 if codeReturned != rt.expected {
                         t.Errorf(

cmd/kubeadm/app/util/kubeconfig.go

@@ -76,12 +76,11 @@ func MakeClientConfigWithToken(config *clientcmdapi.Config, clusterName string,
 }
 
 func WriteKubeconfigIfNotExists(name string, kubeconfig *clientcmdapi.Config) error {
-	envParams := kubeadmapi.GetEnvParams()
+        if err := os.MkdirAll(kubeadmapi.GlobalEnvParams.KubernetesDir, 0700); err != nil {
-	if err := os.MkdirAll(envParams["kubernetes_dir"], 0700); err != nil {
+                return fmt.Errorf("<util/kubeconfig> failed to create directory %q [%v]", kubeadmapi.GlobalEnvParams.KubernetesDir, err)
-		return fmt.Errorf("<util/kubeconfig> failed to create directory %q [%v]", envParams["kubernetes_dir"], err)
 	}
 
-	filename := path.Join(envParams["kubernetes_dir"], fmt.Sprintf("%s.conf", name))
+        filename := path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, fmt.Sprintf("%s.conf", name))
 	// Create and open the file, only if it does not already exist.
 	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0600)
 	if err != nil {

cmd/kubeadm/app/util/kubeconfig_test.go

@@ -17,18 +17,42 @@ limitations under the License.
 package util
 
 import (
+        "bytes"
         "fmt"
-        "log"
+        "io/ioutil"
-        "math/rand"
         "os"
-        "runtime"
+        "path/filepath"
-        "strings"
         "testing"
-        "time"
 
+        kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
         clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api"
 )
 
+const (
+        configOut1 = `apiVersion: v1
+clusters:
+- cluster:
+    server: ""
+  name: ""
+contexts: []
+current-context: ""
+kind: Config
+preferences: {}
+users: []
+`
+        configOut2 = `apiVersion: v1
+clusters:
+- cluster:
+    server: ""
+  name: kubernetes
+contexts: []
+current-context: ""
+kind: Config
+preferences: {}
+users: []
+`
+)
+
 type configClient struct {
         c  string
         s  string
@@ -50,10 +74,6 @@ type configClientWithToken struct {
         token       string
 }
 
-func init() {
-        rand.Seed(time.Now().UTC().UnixNano())
-}
-
 func TestCreateBasicClientConfig(t *testing.T) {
         var createBasicTest = []struct {
                 cc       configClient
@@ -131,44 +151,49 @@ func TestMakeClientConfigWithToken(t *testing.T) {
         }
 }
 
-func setEnvs() {
-        rand := rand.Int()
-        var envParams = map[string]string{
-                "kubernetes_dir":     fmt.Sprintf("/tmp/%d/etc/kubernetes", rand),
-                "host_pki_path":      fmt.Sprintf("/tmp/%d/etc/kubernetes/pki", rand),
-                "host_etcd_path":     fmt.Sprintf("/tmp/%d/var/lib/etcd", rand),
-                "hyperkube_image":    "",
-                "discovery_image":    fmt.Sprintf("gcr.io/google_containers/kube-discovery-%s:%s", runtime.GOARCH, "1.0"),
-                "etcd_image":         "",
-                "component_loglevel": "--v=4",
-        }
-        for k, v := range envParams {
-                err := os.Setenv(fmt.Sprintf("KUBE_%s", strings.ToUpper(k)), v)
-                if err != nil {
-                        log.Fatal(err)
-                }
-        }
-}
-
 func TestWriteKubeconfigIfNotExists(t *testing.T) {
-        setEnvs()
+        tmpdir, err := ioutil.TempDir("", "")
+        if err != nil {
+                t.Fatalf("Couldn't create tmpdir")
+        }
+        defer os.Remove(tmpdir)
+
+        // set up tmp GlobalEnvParams values for testing
+        oldEnv := kubeadmapi.GlobalEnvParams
+        kubeadmapi.GlobalEnvParams = kubeadmapi.SetEnvParams()
+        kubeadmapi.GlobalEnvParams.KubernetesDir = fmt.Sprintf("%s/etc/kubernetes", tmpdir)
+        kubeadmapi.GlobalEnvParams.HostPKIPath = fmt.Sprintf("%s/etc/kubernetes/pki", tmpdir)
+        kubeadmapi.GlobalEnvParams.HostEtcdPath = fmt.Sprintf("%s/var/lib/etcd", tmpdir)
+        kubeadmapi.GlobalEnvParams.DiscoveryImage = fmt.Sprintf("%s/var/lib/etcd", tmpdir)
+        defer func() { kubeadmapi.GlobalEnvParams = oldEnv }()
+
         var writeConfig = []struct {
                 name     string
                 cc       configClient
                 expected error
+                file     []byte
         }{
-                {fmt.Sprintf("%d", rand.Int()), configClient{}, nil},
+                {"test1", configClient{}, nil, []byte(configOut1)},
-                {fmt.Sprintf("%d", rand.Int()), configClient{c: "kubernetes"}, nil},
+                {"test2", configClient{c: "kubernetes"}, nil, []byte(configOut2)},
         }
         for _, rt := range writeConfig {
                 c := CreateBasicClientConfig(rt.cc.c, rt.cc.s, rt.cc.ca)
                 err := WriteKubeconfigIfNotExists(rt.name, c)
                 if err != rt.expected {
                         t.Errorf(
-                                "failed WriteKubeconfigIfNotExists:\n\texpected: %s\n\t  actual: %s",
+                                "failed WriteKubeconfigIfNotExists with an error:\n\texpected: %s\n\t  actual: %s",
                                 err,
                                 rt.expected,
                         )
                 }
+                configPath := filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, fmt.Sprintf("%s.conf", rt.name))
+                newFile, err := ioutil.ReadFile(configPath)
+                if !bytes.Equal(newFile, rt.file) {
+                        t.Errorf(
+                                "failed WriteKubeconfigIfNotExists config write:\n\texpected: %s\n\t  actual: %s",
+                                newFile,
+                                rt.file,
+                        )
+                }
         }
 }

cmd/kubeadm/app/util/tokens_test.go

@@ -99,14 +99,14 @@ func TestGenerateToken(t *testing.T) {
                 givenToken := strings.Split(strings.ToLower(rt.s.GivenToken), ".")
                 if len(givenToken) != rt.l {
                         t.Errorf(
-                                "failed GenerateToken:\n\texpected: %d\n\t  actual: %d",
+                                "failed GenerateToken num parts:\n\texpected: %d\n\t  actual: %d",
                                 rt.l,
                                 len(givenToken),
                         )
                 }
                 if len(givenToken[0]) != rt.n {
                         t.Errorf(
-                                "failed GenerateToken:\n\texpected: %d\n\t  actual: %d",
+                                "failed GenerateToken first part length:\n\texpected: %d\n\t  actual: %d",
                                 rt.l,
                                 len(givenToken),
                         )
@@ -129,9 +129,10 @@ func TestUseGivenTokenIfValid(t *testing.T) {
                 actual, _ := UseGivenTokenIfValid(&rt.s)
                 if actual != rt.expected {
                         t.Errorf(
-                                "failed UseGivenTokenIfValid:\n\texpected: %t\n\t  actual: %t",
+                                "failed UseGivenTokenIfValid:\n\texpected: %t\n\t  actual: %t\n\t token:%s",
                                 rt.expected,
                                 actual,
+                                rt.s.GivenToken,
                         )
                 }
         }