kubeadm: Generate kubeconfig files for controller-manager and scheduler and use them; secures the control plane communication

cmd/kubeadm/app/constants/constants.go

@@ -49,12 +49,23 @@ const (
 	FrontProxyClientCertName           = "front-proxy-client.crt"
 	FrontProxyClientKeyName            = "front-proxy-client.key"
 
-	AdminKubeConfigFileName   = "admin.conf"
+	AdminKubeConfigFileName             = "admin.conf"
-	KubeletKubeConfigFileName = "kubelet.conf"
+	KubeletKubeConfigFileName           = "kubelet.conf"
+	ControllerManagerKubeConfigFileName = "controller-manager.conf"
+	SchedulerKubeConfigFileName         = "scheduler.conf"
+
+	DefaultCertDir = "/etc/kubernetes/pki"
 
 	// Important: a "v"-prefix shouldn't exist here; semver doesn't allow that
 	MinimumControlPlaneVersion = "1.6.0-alpha.2"
 
+	// Some well-known users and groups in the core Kubernetes authorization system
+
+	ControllerManagerUser = "system:kube-controller-manager"
+	SchedulerUser         = "system:kube-scheduler"
+	MastersGroup          = "system:masters"
+	NodesGroup            = "system:nodes"
+
 	// Constants for what we name our ServiceAccounts with limited access to the cluster in case of RBAC
 	KubeDNSServiceAccountName   = "kube-dns"
 	KubeProxyServiceAccountName = "kube-proxy"

cmd/kubeadm/app/master/manifests.go

@@ -91,10 +91,11 @@ func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error {
 			Name:          kubeScheduler,
 			Image:         images.GetCoreImage(images.KubeSchedulerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
 			Command:       getSchedulerCommand(cfg, false),
+			VolumeMounts:  []api.VolumeMount{k8sVolumeMount()},
 			LivenessProbe: componentProbe(10251, "/healthz"),
 			Resources:     componentResources("100m"),
 			Env:           getProxyEnvVars(),
-		}),
+		}, k8sVolume(cfg)),
 	}
 
 	// Add etcd static pod spec only if external etcd is not configured
@@ -378,7 +379,7 @@ func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted
 	command = append(getComponentBaseCommand(controllerManager),
 		"--address=127.0.0.1",
 		"--leader-elect",
-		"--master=127.0.0.1:8080",
+		"--kubeconfig="+path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName),
 		"--root-ca-file="+getCertFilePath(kubeadmconstants.CACertName),
 		"--service-account-private-key-file="+getCertFilePath(kubeadmconstants.ServiceAccountPrivateKeyName),
 		"--cluster-signing-cert-file="+getCertFilePath(kubeadmconstants.CACertName),
@@ -416,7 +417,7 @@ func getSchedulerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool) [
 	command = append(getComponentBaseCommand(scheduler),
 		"--address=127.0.0.1",
 		"--leader-elect",
-		"--master=127.0.0.1:8080",
+		"--kubeconfig="+path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.SchedulerKubeConfigFileName),
 	)
 
 	return command

cmd/kubeadm/app/master/manifests_test.go

@@ -483,7 +483,7 @@ func TestGetControllerManagerCommand(t *testing.T) {
 				"kube-controller-manager",
 				"--address=127.0.0.1",
 				"--leader-elect",
-				"--master=127.0.0.1:8080",
+				"--kubeconfig=" + kubeadmapi.GlobalEnvParams.KubernetesDir + "/controller-manager.conf",
 				"--root-ca-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt",
 				"--service-account-private-key-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/sa.key",
 				"--cluster-signing-cert-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt",
@@ -498,7 +498,7 @@ func TestGetControllerManagerCommand(t *testing.T) {
 				"kube-controller-manager",
 				"--address=127.0.0.1",
 				"--leader-elect",
-				"--master=127.0.0.1:8080",
+				"--kubeconfig=" + kubeadmapi.GlobalEnvParams.KubernetesDir + "/controller-manager.conf",
 				"--root-ca-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt",
 				"--service-account-private-key-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/sa.key",
 				"--cluster-signing-cert-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt",
@@ -514,7 +514,7 @@ func TestGetControllerManagerCommand(t *testing.T) {
 				"kube-controller-manager",
 				"--address=127.0.0.1",
 				"--leader-elect",
-				"--master=127.0.0.1:8080",
+				"--kubeconfig=" + kubeadmapi.GlobalEnvParams.KubernetesDir + "/controller-manager.conf",
 				"--root-ca-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt",
 				"--service-account-private-key-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/sa.key",
 				"--cluster-signing-cert-file=" + kubeadmapi.GlobalEnvParams.HostPKIPath + "/ca.crt",
@@ -552,7 +552,7 @@ func TestGetSchedulerCommand(t *testing.T) {
 				"kube-scheduler",
 				"--address=127.0.0.1",
 				"--leader-elect",
-				"--master=127.0.0.1:8080",
+				"--kubeconfig=" + kubeadmapi.GlobalEnvParams.KubernetesDir + "/scheduler.conf",
 			},
 		},
 	}

cmd/kubeadm/app/phases/certs/certs.go

@@ -151,7 +151,7 @@ func CreatePKIAssets(cfg *kubeadmapi.MasterConfiguration, pkiDir string) error {
 		// TODO: Add a test case to verify that this cert has the x509.ExtKeyUsageClientAuth flag
 		config := certutil.Config{
 			CommonName:   "kube-apiserver-kubelet-client",
-			Organization: []string{"system:masters"},
+			Organization: []string{kubeadmconstants.MastersGroup},
 			Usages:       []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
 		}
 		apiClientCert, apiClientKey, err := pkiutil.NewCertAndKey(caCert, caKey, config)

cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go

@@ -18,7 +18,6 @@ package kubeconfig
 
 import (
 	"bytes"
-	"crypto/rsa"
 	"crypto/x509"
 	"fmt"
 	"os"
@@ -32,6 +31,15 @@ import (
 	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
 )
 
+type KubeConfigProperties struct {
+	CertDir         string
+	ClientName      string
+	Organization    []string
+	APIServer       string
+	Token           string
+	MakeClientCerts bool
+}
+
 // TODO: Make an integration test for this function that runs after the certificates phase
 // and makes sure that those two phases work well together...
 
@@ -42,61 +50,112 @@ import (
 // /etc/kubernetes/{admin,kubelet}.conf exist but the CA cert doesn't match what's in the pki dir => error
 // /etc/kubernetes/{admin,kubelet}.conf exist but not certs => certs will be generated and conflict with the kubeconfig files => error
 
-// CreateAdminAndKubeletKubeConfig is called from the main init and does the work for the default phase behaviour
+// CreateInitKubeConfigFiles is called from the main init and does the work for the default phase behaviour
-func CreateAdminAndKubeletKubeConfig(masterEndpoint, pkiDir, outDir string) error {
+func CreateInitKubeConfigFiles(masterEndpoint, pkiDir, outDir string) error {
 
-	// Try to load ca.crt and ca.key from the PKI directory
+	hostname, err := os.Hostname()
-	caCert, caKey, err := pkiutil.TryLoadCertAndKeyFromDisk(pkiDir, kubeadmconstants.CACertAndKeyBaseName)
 	if err != nil {
-		return fmt.Errorf("couldn't create a kubeconfig; the CA files couldn't be loaded: %v", err)
+		return err
 	}
 
-	// User admin should have full access to the cluster
+	// Create a lightweight specification for what the files should look like
-	// TODO: Add test case that make sure this cert has the x509.ExtKeyUsageClientAuth flag
+	filesToCreateFromSpec := map[string]KubeConfigProperties{
-	adminCertConfig := certutil.Config{
+		kubeadmconstants.AdminKubeConfigFileName: {
-		CommonName:   "kubernetes-admin",
+			ClientName:      "kubernetes-admin",
-		Organization: []string{"system:masters"},
+			APIServer:       masterEndpoint,
-		Usages:       []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+			CertDir:         pkiDir,
-	}
+			Organization:    []string{kubeadmconstants.MastersGroup},
-	adminKubeConfigFilePath := filepath.Join(outDir, kubeadmconstants.AdminKubeConfigFileName)
+			MakeClientCerts: true,
-	if err := createKubeConfigFileForClient(masterEndpoint, adminKubeConfigFilePath, adminCertConfig, caCert, caKey); err != nil {
+		},
-		return fmt.Errorf("couldn't create config for the admin: %v", err)
+		kubeadmconstants.KubeletKubeConfigFileName: {
+			ClientName:      fmt.Sprintf("system:node:%s", hostname),
+			APIServer:       masterEndpoint,
+			CertDir:         pkiDir,
+			Organization:    []string{kubeadmconstants.NodesGroup},
+			MakeClientCerts: true,
+		},
+		kubeadmconstants.ControllerManagerKubeConfigFileName: {
+			ClientName:      kubeadmconstants.ControllerManagerUser,
+			APIServer:       masterEndpoint,
+			CertDir:         pkiDir,
+			MakeClientCerts: true,
+		},
+		kubeadmconstants.SchedulerKubeConfigFileName: {
+			ClientName:      kubeadmconstants.SchedulerUser,
+			APIServer:       masterEndpoint,
+			CertDir:         pkiDir,
+			MakeClientCerts: true,
+		},
 	}
 
-	// TODO: The kubelet should have limited access to the cluster. Right now, this gives kubelet basically root access
+	// Loop through all specs for kubeconfig files and create them if necessary
-	// and we do need that in the bootstrap phase, but we should swap it out after the control plane is up
+	for filename, config := range filesToCreateFromSpec {
-	// TODO: Add test case that make sure this cert has the x509.ExtKeyUsageClientAuth flag
+		kubeconfig, err := buildKubeConfig(config)
-	kubeletCertConfig := certutil.Config{
+		if err != nil {
-		CommonName:   "kubelet",
+			return err
-		Organization: []string{"system:nodes"},
+		}
-		Usages:       []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+
-	}
+		kubeConfigFilePath := filepath.Join(outDir, filename)
-	kubeletKubeConfigFilePath := filepath.Join(outDir, kubeadmconstants.KubeletKubeConfigFileName)
+		err = writeKubeconfigToDiskIfNotExists(kubeConfigFilePath, kubeconfig)
-	if err := createKubeConfigFileForClient(masterEndpoint, kubeletKubeConfigFilePath, kubeletCertConfig, caCert, caKey); err != nil {
+		if err != nil {
-		return fmt.Errorf("couldn't create a kubeconfig file for the kubelet: %v", err)
+			return err
+		}
 	}
 
-	// TODO make credentials for the controller-manager and scheduler
 	return nil
 }
 
-func createKubeConfigFileForClient(masterEndpoint, kubeConfigFilePath string, config certutil.Config, caCert *x509.Certificate, caKey *rsa.PrivateKey) error {
+func GetKubeConfigBytesFromSpec(config KubeConfigProperties) ([]byte, error) {
-	cert, key, err := pkiutil.NewCertAndKey(caCert, caKey, config)
+	kubeconfig, err := buildKubeConfig(config)
 	if err != nil {
-		return fmt.Errorf("failure while creating %s client certificate [%v]", config.CommonName, err)
+		return []byte{}, err
 	}
 
-	kubeconfig := kubeconfigutil.CreateWithCerts(
+	kubeConfigBytes, err := clientcmd.Write(*kubeconfig)
-		masterEndpoint,
+	if err != nil {
-		"kubernetes",
+		return []byte{}, err
-		config.CommonName,
+	}
-		certutil.EncodeCertPEM(caCert),
+	return kubeConfigBytes, nil
-		certutil.EncodePrivateKeyPEM(key),
+}
-		certutil.EncodeCertPEM(cert),
-	)
 
-	// Write it now to a file if there already isn't a valid one
+// buildKubeConfig creates a kubeconfig object from some commonly specified properties in the struct above
-	return writeKubeconfigToDiskIfNotExists(kubeConfigFilePath, kubeconfig)
+func buildKubeConfig(config KubeConfigProperties) (*clientcmdapi.Config, error) {
+
+	// Try to load ca.crt and ca.key from the PKI directory
+	caCert, caKey, err := pkiutil.TryLoadCertAndKeyFromDisk(config.CertDir, kubeadmconstants.CACertAndKeyBaseName)
+	if err != nil {
+		return nil, fmt.Errorf("couldn't create a kubeconfig; the CA files couldn't be loaded: %v", err)
+	}
+
+	// If this file should have client certs, generate one from the spec
+	if config.MakeClientCerts {
+		certConfig := certutil.Config{
+			CommonName:   config.ClientName,
+			Organization: config.Organization,
+			Usages:       []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+		}
+		cert, key, err := pkiutil.NewCertAndKey(caCert, caKey, certConfig)
+		if err != nil {
+			return nil, fmt.Errorf("failure while creating %s client certificate [%v]", certConfig.CommonName, err)
+		}
+		return kubeconfigutil.CreateWithCerts(
+			config.APIServer,
+			"kubernetes",
+			config.ClientName,
+			certutil.EncodeCertPEM(caCert),
+			certutil.EncodePrivateKeyPEM(key),
+			certutil.EncodeCertPEM(cert),
+		), nil
+	}
+
+	// otherwise, create a kubeconfig with a token
+	return kubeconfigutil.CreateWithToken(
+		config.APIServer,
+		"kubernetes",
+		config.ClientName,
+		certutil.EncodeCertPEM(caCert),
+		config.Token,
+	), nil
 }
 
 // writeKubeconfigToDiskIfNotExists saves the KubeConfig struct to disk if there isn't any file at the given path