kubelet: Add --bootstrap-kubeconfig to get TLS client cert.

Add --bootstrap-kubeconfig flag to kubelet. If the flag is non-empty
and --kubeconfig doesn't exist, then the kubelet will use the bootstrap
kubeconfig to create rest client and generate certificate signing request
to request a client cert from API server.

Once succeeds, the result cert will be written down to
--cert-dir/kubelet-client.crt, and the kubeconfig will be populated with
certfile, keyfile path pointing to the result certificate file, key file.
(The key file is generated before creating the CSR).

cmd/kubelet/app/options/options.go

@@ -41,7 +41,9 @@ const (
 type KubeletServer struct {
 	componentconfig.KubeletConfiguration
 
-	KubeConfig util.StringFlag
+	KubeConfig          util.StringFlag
+	BootstrapKubeconfig string
+
 	// If true, an invalid KubeConfig will result in the Kubelet exiting with an error.
 	RequireKubeConfig bool
 	AuthPath          util.StringFlag // Deprecated -- use KubeConfig instead
@@ -97,6 +99,10 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.TLSPrivateKeyFile, "tls-private-key-file", s.TLSPrivateKeyFile, "File containing x509 private key matching --tls-cert-file.")
 	fs.StringVar(&s.CertDirectory, "cert-dir", s.CertDirectory, "The directory where the TLS certs are located (by default /var/run/kubernetes). "+
 		"If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.")
+	fs.StringVar(&s.BootstrapKubeconfig, "experimental-bootstrap-kubeconfig", s.BootstrapKubeconfig, "<Warning: Experimental feature> Path to a kubeconfig file that will be used to get client certificate for kubelet. "+
+		"If the file specified by --kubeconfig does not exist, the bootstrap kubeconfig is used to request a client certificate from the API server. "+
+		"On success, a kubeconfig file referencing the generated key and obtained certificate is written to the path specified by --kubeconfig. "+
+		"The certificate and key file will be stored in /var/run/kubernetes/.")
 	fs.StringVar(&s.HostnameOverride, "hostname-override", s.HostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname.")
 	fs.StringVar(&s.PodInfraContainerImage, "pod-infra-container-image", s.PodInfraContainerImage, "The image whose network/ipc namespaces containers in each pod will use.")
 	fs.StringVar(&s.DockerEndpoint, "docker-endpoint", s.DockerEndpoint, "Use this for the docker endpoint to communicate with")

cmd/kubelet/app/server.go

@@ -19,6 +19,8 @@ package app
 
 import (
 	"crypto/tls"
+	"crypto/x509/pkix"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -39,11 +41,14 @@ import (
 	"k8s.io/kubernetes/cmd/kubelet/app/options"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/resource"
+	"k8s.io/kubernetes/pkg/api/unversioned"
+	"k8s.io/kubernetes/pkg/apis/certificates"
 	"k8s.io/kubernetes/pkg/apis/componentconfig"
 	kubeExternal "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
 	"k8s.io/kubernetes/pkg/capabilities"
 	"k8s.io/kubernetes/pkg/client/chaosclient"
 	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
+	unversionedcertificates "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/certificates/unversioned"
 	unversionedcore "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/unversioned"
 	"k8s.io/kubernetes/pkg/client/record"
 	"k8s.io/kubernetes/pkg/client/restclient"
@@ -64,6 +69,7 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/network"
 	"k8s.io/kubernetes/pkg/kubelet/server"
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
+	utilcertificates "k8s.io/kubernetes/pkg/util/certificates"
 	utilconfig "k8s.io/kubernetes/pkg/util/config"
 	"k8s.io/kubernetes/pkg/util/configz"
 	"k8s.io/kubernetes/pkg/util/crypto"
@@ -77,6 +83,12 @@ import (
 	"k8s.io/kubernetes/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/version"
 	"k8s.io/kubernetes/pkg/volume"
+	"k8s.io/kubernetes/pkg/watch"
+)
+
+const (
+	defaultKubeletClientCertificateFile = "/var/run/kubernetes/kubelet-client.crt"
+	defaultKubeletClientKeyFile         = "/var/run/kubernetes/kubelet-client.key"
 )
 
 // bootstrapping interface for kubelet, targets the initialization protocol
@@ -335,6 +347,12 @@ func run(s *options.KubeletServer, kcfg *KubeletConfig) (err error) {
 
 	if kcfg == nil {
 		var kubeClient, eventClient *clientset.Clientset
+		if s.BootstrapKubeconfig != "" {
+			if err := bootstrapClientCert(s); err != nil {
+				return err
+			}
+		}
+
 		clientConfig, err := CreateAPIServerClientConfig(s)
 		if err == nil {
 			kubeClient, err = clientset.NewForConfig(clientConfig)
@@ -439,6 +457,242 @@ func run(s *options.KubeletServer, kcfg *KubeletConfig) (err error) {
 	return nil
 }
 
+// bootstrapClientCert will request a client cert for kubelet.
+// If the file specified by --kubeconfig does not exist, the bootstrap kubeconfig is used
+// to request a client certificate from the API server.
+// On success, a kubeconfig file referencing the generated key and obtained certificate is
+// written to the path specified by --kubeconfig.
+// The certificate and key file will be stored in /var/run/kubernetes/.
+func bootstrapClientCert(s *options.KubeletServer) error {
+	// Check the if --kubeconfig already has sufficient TLS client info.
+	kcfg, err := (&clientcmd.ClientConfigLoadingRules{ExplicitPath: s.KubeConfig.Value()}).Load()
+	if err == nil {
+		authInfo, err := getCurrentContextAuthInfo(kcfg)
+		if err == nil {
+			if containsSufficientTLSInfo(authInfo) {
+				return nil
+			}
+		}
+	}
+
+	// At this point, we need to use the bootstrap kubeconfig to generate TLS client cert, key, and a kubeconfig
+	// to stored in --kubeconfig.
+	glog.V(2).Info("Using bootstrap kubeconfig to generate TLS client cert, key and kubeconfig file")
+
+	kcfg, err = (&clientcmd.ClientConfigLoadingRules{ExplicitPath: s.BootstrapKubeconfig}).Load()
+	if err != nil {
+		return fmt.Errorf("unable to load boostrap kubeconfig: %v", err)
+	}
+	authInfo, err := getCurrentContextAuthInfo(kcfg)
+	if err != nil {
+		return fmt.Errorf("unable to load auth info in bootstrap kubeconfig: %v", err)
+	}
+
+	authInfo.ClientCertificate, authInfo.ClientKey, err = getClientCertAndKey(s, authInfo.ClientCertificate, authInfo.ClientKey)
+	if err != nil {
+		return fmt.Errorf("unable to get cert from API server: %v", err)
+	}
+
+	// Marshal and write the kubeconfig to disk.
+	data, err := json.Marshal(kcfg)
+	if err != nil {
+		return fmt.Errorf("unable to marshal the kubeconfig: %v", err)
+	}
+
+	if err := ioutil.WriteFile(s.KubeConfig.Value(), data, 0644); err != nil {
+		return fmt.Errorf("unable to write the kubeconfig file at %q: %v", s.KubeConfig.Value(), err)
+	}
+
+	return nil
+}
+
+// getCurrentContextAuthInfo returns the AuthInfo object that's referenced
+// by the current context.
+// If current context or auth info name is empty, then it will return an error.
+// If AuthInfo is empty, a new auth info object will be created.
+func getCurrentContextAuthInfo(config *clientcmdapi.Config) (*clientcmdapi.AuthInfo, error) {
+	ctx, ok := config.Contexts[config.CurrentContext]
+	if !ok {
+		return nil, fmt.Errorf("unable to find current context %q", config.CurrentContext)
+	}
+
+	if ctx.AuthInfo == "" {
+		return nil, fmt.Errorf("unable to find the name of the authInfo in current context %q", config.CurrentContext)
+	}
+
+	if _, ok := config.AuthInfos[ctx.AuthInfo]; !ok {
+		config.AuthInfos[ctx.AuthInfo] = clientcmdapi.NewAuthInfo()
+	}
+	return config.AuthInfos[ctx.AuthInfo], nil
+}
+
+// TODO(yifan): More detailed check on the cert / key content.
+// CheckTLSInfo returns true if the authInfo contains client certificate data for client key data.
+// Or if the client certificate or key file exists.
+func containsSufficientTLSInfo(authInfo *clientcmdapi.AuthInfo) bool {
+	// We use '||' so that we won't override existing data in case of wrong setup.
+	if len(authInfo.ClientCertificateData) > 0 || len(authInfo.ClientKeyData) > 0 {
+		return true
+	}
+
+	if crypto.FoundCertOrKey(authInfo.ClientCertificate, authInfo.ClientKey) {
+		return true
+	}
+
+	return false
+}
+
+// getClientCertAndKey will:
+// (1) Create a restful client for doing the certificate signing request.
+// (2) Read existing key data from existingKeyPath if possible.
+// (3) Pass 'requestClientCertificate()' the CSR client, existing key data, and node name to
+//     request for client certificate from the API server.
+// (4) Once (3) succeeds, dump the certificate and key data to the given paths.
+// On failure, the the certificate and key file will be cleaned up.
+// If the existingCertPath or existingKeyPath is empty, then the function will use the default path, respectively:
+// /var/run/kubernetes/kubelet-client.crt, /var/run/kubernetes/kubelet-client.key.
+func getClientCertAndKey(s *options.KubeletServer, existingCertPath, existingKeyPath string) (certPath, keyPath string, err error) {
+	// (1).
+	clientConfig, err := kubeconfigClientConfig(s.BootstrapKubeconfig, s.APIServerList)
+	if err != nil {
+		return "", "", fmt.Errorf("unable to create client config: %v", err)
+	}
+	client, err := unversionedcertificates.NewForConfig(clientConfig)
+	if err != nil {
+		return "", "", fmt.Errorf("unable to create certificates signing request client: %v", err)
+	}
+	csrClient := client.CertificateSigningRequests()
+
+	// (2).
+	certPath, keyPath = existingCertPath, existingKeyPath
+	if certPath == "" {
+		certPath = defaultKubeletClientCertificateFile
+	}
+	if keyPath == "" {
+		keyPath = defaultKubeletClientKeyFile
+
+	}
+	existingKeyData, err := ioutil.ReadFile(keyPath)
+	if err != nil && !os.IsNotExist(err) {
+		return "", "", fmt.Errorf("unable to read key file %q: %v", keyPath, err)
+	}
+
+	// (3).
+	nodeName, err := getNodeName(s)
+	if err != nil {
+		return "", "", fmt.Errorf("unable to get node name: %v", err)
+	}
+	certData, keyData, err := requestClientCertificate(csrClient, existingKeyData, nodeName)
+	if err != nil {
+		return "", "", fmt.Errorf("unable to request certificate from API server: %v", err)
+	}
+
+	// (4).
+	if err = crypto.WriteCertToPath(certPath, certData); err != nil {
+		return "", "", fmt.Errorf("unable to write certificate file %q: %v", certPath, err)
+	}
+	if err = crypto.WriteKeyToPath(keyPath, keyData); err != nil {
+		if err := os.Remove(certPath); err != nil {
+			glog.Warningf("Cannot clean up the certificate file %q: %v", certPath, err)
+		}
+		return "", "", fmt.Errorf("unable to write key file %q: %v", keyPath, err)
+	}
+
+	return certPath, keyPath, nil
+}
+
+// getNodeName returns the node name according to the cloud provider
+// if cloud provider is specified. Otherwise, returns the host name of the node.
+func getNodeName(s *options.KubeletServer) (string, error) {
+	var err error
+	var cloud cloudprovider.Interface
+
+	if s.CloudProvider != kubeExternal.AutoDetectCloudProvider {
+		cloud, err = cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	hostName := nodeutil.GetHostname(s.HostnameOverride)
+	if cloud != nil {
+		instances, ok := cloud.Instances()
+		if !ok {
+			return "", fmt.Errorf("failed to get instances from cloud provider")
+		}
+		return instances.CurrentNodeName(hostName)
+	}
+	return hostName, nil
+}
+
+// requestClientCertificate will create a certificate signing request and send it to API server,
+// then it will watch the object's status, once approved by API server, it will return the API
+// server's issued certificate (pem-encoded). If there is any errors, or the watch timeouts,
+// it will return an error.
+// If the existingKeyData is empty, a new private key will be generated to create the certificate
+// signing request.
+func requestClientCertificate(client unversionedcertificates.CertificateSigningRequestInterface, existingKeyData []byte, nodeName string) (certData []byte, keyData []byte, err error) {
+	subject := &pkix.Name{
+		Organization: []string{"system:nodes"},
+		CommonName:   fmt.Sprintf("system:node:%s", nodeName),
+	}
+
+	csr, keyData, err := utilcertificates.NewCertificateRequest(existingKeyData, subject, nil, nil)
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to generate certificate request: %v", err)
+	}
+
+	req, err := client.Create(&certificates.CertificateSigningRequest{
+		TypeMeta:   unversioned.TypeMeta{Kind: "CertificateSigningRequest"},
+		ObjectMeta: api.ObjectMeta{GenerateName: "csr-"},
+
+		// Username, UID, Groups will be injected by API server.
+		Spec: certificates.CertificateSigningRequestSpec{Request: csr},
+	})
+	if err != nil {
+		return nil, nil, fmt.Errorf("cannot create certificate signing request: %v", err)
+
+	}
+
+	// Make a default timeout = 3600s
+	var defaultTimeoutSeconds int64 = 3600
+	resultCh, err := client.Watch(api.ListOptions{
+		Watch:          true,
+		TimeoutSeconds: &defaultTimeoutSeconds,
+		// Label and field selector are not used now.
+	})
+	if err != nil {
+		return nil, nil, fmt.Errorf("cannot watch on the certificate signing request: %v", err)
+	}
+
+	var status certificates.CertificateSigningRequestStatus
+	ch := resultCh.ResultChan()
+
+	for {
+		event, ok := <-ch
+		if !ok {
+			break
+		}
+
+		if event.Type == watch.Modified {
+			if event.Object.(*certificates.CertificateSigningRequest).UID != req.UID {
+				continue
+			}
+			status = event.Object.(*certificates.CertificateSigningRequest).Status
+			for _, c := range status.Conditions {
+				if c.Type == certificates.CertificateDenied {
+					return nil, nil, fmt.Errorf("certificate signing request is not approved: %v, %v", c.Reason, c.Message)
+				}
+				if c.Type == certificates.CertificateApproved && status.Certificate != nil {
+					return status.Certificate, keyData, nil
+				}
+			}
+		}
+	}
+
+	return nil, nil, fmt.Errorf("watch channel closed")
+}
+
 // InitializeTLS checks for a configured TLSCertFile and TLSPrivateKeyFile: if unspecified a new self-signed
 // certificate and key file are generated. Returns a configured server.TLSOptions object.
 func InitializeTLS(s *options.KubeletServer) (*server.TLSOptions, error) {
@@ -528,13 +782,13 @@ func createClientConfig(s *options.KubeletServer) (*restclient.Config, error) {
 		return nil, fmt.Errorf("cannot specify both --kubeconfig and --auth-path")
 	}
 	if s.KubeConfig.Provided() {
-		return kubeconfigClientConfig(s)
+		return kubeconfigClientConfig(s.KubeConfig.Value(), s.APIServerList)
 	}
 	if s.AuthPath.Provided() {
 		return authPathClientConfig(s, false)
 	}
 	// Try the kubeconfig default first, falling back to the auth path default.
-	clientConfig, err := kubeconfigClientConfig(s)
+	clientConfig, err := kubeconfigClientConfig(s.KubeConfig.Value(), s.APIServerList)
 	if err != nil {
 		glog.Warningf("Could not load kubeconfig file %s: %v. Trying auth path instead.", s.KubeConfig, err)
 		return authPathClientConfig(s, true)

pkg/util/certificates/csr.go

@@ -26,9 +26,7 @@ import (
 	"encoding/pem"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"net"
-	"os"
 
 	"k8s.io/kubernetes/pkg/apis/certificates"
 )
@@ -49,49 +47,39 @@ func ParseCertificateRequestObject(obj *certificates.CertificateSigningRequest)
 }
 
 // NewCertificateRequest generates a PEM-encoded CSR using the supplied private
-// key file, subject, and SANs. If the private key file does not exist, it generates a
-// new ECDSA P256 key to use and writes it to the keyFile path.
-func NewCertificateRequest(keyFile string, subject *pkix.Name, dnsSANs []string, ipSANs []net.IP) ([]byte, error) {
+// key data, subject, and SANs. If the private key data is empty, it generates a
+// new ECDSA P256 key to use and returns it together with the CSR data.
+func NewCertificateRequest(keyData []byte, subject *pkix.Name, dnsSANs []string, ipSANs []net.IP) (csr []byte, key []byte, err error) {
 	var privateKey interface{}
+	var privateKeyPemBlock *pem.Block
 
-	if _, err := os.Stat(keyFile); os.IsNotExist(err) {
+	if len(keyData) == 0 {
 		privateKey, err = ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 
 		ecdsaKey := privateKey.(*ecdsa.PrivateKey)
 		derBytes, err := x509.MarshalECPrivateKey(ecdsaKey)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 
-		pemBlock := &pem.Block{
+		privateKeyPemBlock = &pem.Block{
 			Type:  "EC PRIVATE KEY",
 			Bytes: derBytes,
 		}
-
-		err = ioutil.WriteFile(keyFile, pem.EncodeToMemory(pemBlock), os.FileMode(0600))
-		if err != nil {
-			return nil, err
-		}
+	} else {
+		privateKeyPemBlock, _ = pem.Decode(keyData)
 	}
 
-	keyBytes, err := ioutil.ReadFile(keyFile)
-	if err != nil {
-		return nil, err
-	}
-
-	var block *pem.Block
 	var sigType x509.SignatureAlgorithm
 
-	block, _ = pem.Decode(keyBytes)
-
-	switch block.Type {
+	switch privateKeyPemBlock.Type {
 	case "EC PRIVATE KEY":
-		privateKey, err = x509.ParseECPrivateKey(block.Bytes)
+		privateKey, err = x509.ParseECPrivateKey(privateKeyPemBlock.Bytes)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		ecdsaKey := privateKey.(*ecdsa.PrivateKey)
 		switch ecdsaKey.Curve.Params().BitSize {
@@ -103,9 +91,9 @@ func NewCertificateRequest(keyFile string, subject *pkix.Name, dnsSANs []string,
 			sigType = x509.ECDSAWithSHA256
 		}
 	case "RSA PRIVATE KEY":
-		privateKey, err = x509.ParsePKCS1PrivateKey(block.Bytes)
+		privateKey, err = x509.ParsePKCS1PrivateKey(privateKeyPemBlock.Bytes)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		rsaKey := privateKey.(*rsa.PrivateKey)
 		keySize := rsaKey.N.BitLen()
@@ -118,7 +106,7 @@ func NewCertificateRequest(keyFile string, subject *pkix.Name, dnsSANs []string,
 			sigType = x509.SHA256WithRSA
 		}
 	default:
-		return nil, fmt.Errorf("unsupported key type: %s", block.Type)
+		return nil, nil, fmt.Errorf("unsupported key type: %s", privateKeyPemBlock.Type)
 	}
 
 	template := &x509.CertificateRequest{
@@ -128,15 +116,15 @@ func NewCertificateRequest(keyFile string, subject *pkix.Name, dnsSANs []string,
 		IPAddresses:        ipSANs,
 	}
 
-	csr, err := x509.CreateCertificateRequest(cryptorand.Reader, template, privateKey)
+	csr, err = x509.CreateCertificateRequest(cryptorand.Reader, template, privateKey)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
-	pemBlock := &pem.Block{
+	csrPemBlock := &pem.Block{
 		Type:  "CERTIFICATE REQUEST",
 		Bytes: csr,
 	}
 
-	return pem.EncodeToMemory(pemBlock), nil
+	return pem.EncodeToMemory(csrPemBlock), pem.EncodeToMemory(privateKeyPemBlock), nil
 }

pkg/util/certificates/csr_test.go

@@ -2,6 +2,7 @@ package certificates
 
 import (
 	"crypto/x509/pkix"
+	"io/ioutil"
 	"net"
 	"testing"
 )
@@ -14,7 +15,11 @@ func TestNewCertificateRequest(t *testing.T) {
 	dnsSANs := []string{"localhost"}
 	ipSANs := []net.IP{net.ParseIP("127.0.0.1")}
 
-	_, err := NewCertificateRequest(keyFile, subject, dnsSANs, ipSANs)
+	keyData, err := ioutil.ReadFile(keyFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, _, err = NewCertificateRequest(keyData, subject, dnsSANs, ipSANs)
 	if err != nil {
 		t.Error(err)
 	}

pkg/util/crypto/crypto.go

@@ -109,24 +109,46 @@ func GenerateSelfSignedCert(host, certPath, keyPath string, alternateIPs []net.I
 	}
 
 	// Write cert
-	if err := os.MkdirAll(filepath.Dir(certPath), os.FileMode(0755)); err != nil {
-		return err
-	}
-	if err := ioutil.WriteFile(certPath, certBuffer.Bytes(), os.FileMode(0644)); err != nil {
+	if err := WriteCertToPath(certPath, certBuffer.Bytes()); err != nil {
 		return err
 	}
 
 	// Write key
-	if err := os.MkdirAll(filepath.Dir(keyPath), os.FileMode(0755)); err != nil {
-		return err
-	}
-	if err := ioutil.WriteFile(keyPath, keyBuffer.Bytes(), os.FileMode(0600)); err != nil {
+	if err := WriteKeyToPath(keyPath, keyBuffer.Bytes()); err != nil {
 		return err
 	}
 
 	return nil
 }
 
+// WriteCertToPath writes the pem-encoded certificate data to certPath.
+// The certificate file will be created with file mode 0644.
+// If the certificate file already exists, it will be overwritten.
+// The parent directory of the certPath will be created as needed with file mode 0755.
+func WriteCertToPath(certPath string, data []byte) error {
+	if err := os.MkdirAll(filepath.Dir(certPath), os.FileMode(0755)); err != nil {
+		return err
+	}
+	if err := ioutil.WriteFile(certPath, data, os.FileMode(0644)); err != nil {
+		return err
+	}
+	return nil
+}
+
+// writeCertToPath writes the pem-encoded key data to keyPath.
+// The key file will be created with file mode 0600.
+// If the key file already exists, it will be overwritten.
+// The parent directory of the keyPath will be created as needed with file mode 0755.
+func WriteKeyToPath(keyPath string, data []byte) error {
+	if err := os.MkdirAll(filepath.Dir(keyPath), os.FileMode(0755)); err != nil {
+		return err
+	}
+	if err := ioutil.WriteFile(keyPath, data, os.FileMode(0600)); err != nil {
+		return err
+	}
+	return nil
+}
+
 // CertPoolFromFile returns an x509.CertPool containing the certificates in the given PEM-encoded file.
 // Returns an error if the file could not be read, a certificate could not be parsed, or if the file does not contain any certificates
 func CertPoolFromFile(filename string) (*x509.CertPool, error) {