Merge pull request #45059 from jcbsmpsn/rotate-server-certificate

Automatic merge from submit-queue (batch tested with PRs 46635, 45619, 46637, 45059, 46415)

Certificate rotation for kubelet server certs.

Replaces the current kubelet server side self signed certs with certs signed by
the Certificate Request Signing API on the API server. Also renews expiring
kubelet server certs as expiration approaches.

Two Points:
1. With `--feature-gates=RotateKubeletServerCertificate=true` set, the kubelet will
    request a certificate during the boot cycle and pause waiting for the request to
    be satisfied.
2. In order to have the kubelet's certificate signing request auto approved,
    `--insecure-experimental-approve-all-kubelet-csrs-for-group=` must be set on
    the cluster controller manager. There is an improved mechanism for auto
    approval [proposed](https://github.com/kubernetes/kubernetes/issues/45030).

**Release note**:
```release-note
With `--feature-gates=RotateKubeletServerCertificate=true` set, the kubelet will
request a server certificate from the API server during the boot cycle and pause
waiting for the request to be satisfied. It will continually refresh the certificate as
the certificates expiration approaches.
```

cmd/kubelet/app/server.go

@@ -288,7 +288,6 @@ func initKubeletConfigSync(s *options.KubeletServer) (*componentconfig.KubeletCo
 func Run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) error {
 	if err := run(s, kubeDeps); err != nil {
 		return fmt.Errorf("failed to run Kubelet: %v", err)
-
 	}
 	return nil
 }
@@ -623,7 +622,7 @@ func getNodeName(cloud cloudprovider.Interface, hostname string) (types.NodeName
 // InitializeTLS checks for a configured TLSCertFile and TLSPrivateKeyFile: if unspecified a new self-signed
 // certificate and key file are generated. Returns a configured server.TLSOptions object.
 func InitializeTLS(kf *options.KubeletFlags, kc *componentconfig.KubeletConfiguration) (*server.TLSOptions, error) {
-	if kc.TLSCertFile == "" && kc.TLSPrivateKeyFile == "" {
+	if !utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) && kc.TLSCertFile == "" && kc.TLSPrivateKeyFile == "" {
 		kc.TLSCertFile = path.Join(kc.CertDirectory, "kubelet.crt")
 		kc.TLSPrivateKeyFile = path.Join(kc.CertDirectory, "kubelet.key")
 

pkg/features/kube_features.go

@@ -89,6 +89,14 @@ const (
 	// to take advantage of NoExecute Taints and Tolerations.
 	TaintBasedEvictions utilfeature.Feature = "TaintBasedEvictions"
 
+	// owner: @jcbsmpsn
+	// alpha: v1.7
+	//
+	// Gets a server certificate for the kubelet from the Certificate Signing
+	// Request API instead of generating one self signed and auto rotates the
+	// certificate as expiration approaches.
+	RotateKubeletServerCertificate utilfeature.Feature = "RotateKubeletServerCertificate"
+
 	// owner: @msau
 	// alpha: v1.7
 	//
@@ -113,6 +121,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
 	AffinityInAnnotations:                       {Default: false, PreRelease: utilfeature.Alpha},
 	Accelerators:                                {Default: false, PreRelease: utilfeature.Alpha},
 	TaintBasedEvictions:                         {Default: false, PreRelease: utilfeature.Alpha},
+	RotateKubeletServerCertificate:              {Default: false, PreRelease: utilfeature.Alpha},
 	PersistentLocalVolumes:                      {Default: false, PreRelease: utilfeature.Alpha},
 
 	// inherited features from generic apiserver, relisted here to get a conflict if it is changed

pkg/kubelet/BUILD

@@ -42,16 +42,19 @@ go_library(
         "//pkg/api/v1/pod:go_default_library",
         "//pkg/api/v1/resource:go_default_library",
         "//pkg/api/v1/validation:go_default_library",
+        "//pkg/apis/certificates/v1beta1:go_default_library",
         "//pkg/apis/componentconfig:go_default_library",
         "//pkg/apis/componentconfig/v1alpha1:go_default_library",
         "//pkg/capabilities:go_default_library",
         "//pkg/client/clientset_generated/clientset:go_default_library",
+        "//pkg/client/clientset_generated/clientset/typed/certificates/v1beta1:go_default_library",
         "//pkg/client/listers/core/v1:go_default_library",
         "//pkg/cloudprovider:go_default_library",
         "//pkg/features:go_default_library",
         "//pkg/fieldpath:go_default_library",
         "//pkg/kubelet/apis/cri:go_default_library",
         "//pkg/kubelet/cadvisor:go_default_library",
+        "//pkg/kubelet/certificate:go_default_library",
         "//pkg/kubelet/cm:go_default_library",
         "//pkg/kubelet/config:go_default_library",
         "//pkg/kubelet/container:go_default_library",

pkg/kubelet/certificate/certificate_store.go

@@ -195,6 +195,9 @@ func (s *fileStore) Update(certData, keyData []byte) (*tls.Certificate, error) {
 	ts := time.Now().Format("2006-01-02-15-04-05")
 	pemFilename := s.filename(ts)
 
+	if err := os.MkdirAll(s.certDirectory, 0755); err != nil {
+		return nil, fmt.Errorf("could not create directory %q to store certificates: %v", s.certDirectory, err)
+	}
 	certPath := filepath.Join(s.certDirectory, pemFilename)
 
 	f, err := os.OpenFile(certPath, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0600)

pkg/kubelet/kubelet.go

@@ -17,6 +17,9 @@ limitations under the License.
 package kubelet
 
 import (
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
 	"fmt"
 	"net"
 	"net/http"
@@ -52,14 +55,17 @@ import (
 	"k8s.io/kubernetes/cmd/kubelet/app/options"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/v1"
+	certificates "k8s.io/kubernetes/pkg/apis/certificates/v1beta1"
 	"k8s.io/kubernetes/pkg/apis/componentconfig"
 	componentconfigv1alpha1 "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
 	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
+	clientcertificates "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/certificates/v1beta1"
 	corelisters "k8s.io/kubernetes/pkg/client/listers/core/v1"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	"k8s.io/kubernetes/pkg/features"
 	internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri"
 	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
+	"k8s.io/kubernetes/pkg/kubelet/certificate"
 	"k8s.io/kubernetes/pkg/kubelet/cm"
 	"k8s.io/kubernetes/pkg/kubelet/config"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
@@ -305,6 +311,8 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
 	hostname := nodeutil.GetHostname(hostnameOverride)
 	// Query the cloud provider for our node name, default to hostname
 	nodeName := types.NodeName(hostname)
+	cloudIPs := []net.IP{}
+	cloudNames := []string{}
 	if kubeDeps.Cloud != nil {
 		var err error
 		instances, ok := kubeDeps.Cloud.Instances()
@@ -318,6 +326,25 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
 		}
 
 		glog.V(2).Infof("cloud provider determined current node name to be %s", nodeName)
+
+		if utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) {
+			nodeAddresses, err := instances.NodeAddresses(nodeName)
+			if err != nil {
+				return nil, fmt.Errorf("failed to get the addresses of the current instance from the cloud provider: %v", err)
+			}
+			for _, nodeAddress := range nodeAddresses {
+				switch nodeAddress.Type {
+				case v1.NodeExternalIP, v1.NodeInternalIP:
+					ip := net.ParseIP(nodeAddress.Address)
+					if ip != nil && !ip.IsLoopback() {
+						cloudIPs = append(cloudIPs, ip)
+					}
+				case v1.NodeExternalDNS, v1.NodeInternalDNS, v1.NodeHostName:
+					cloudNames = append(cloudNames, nodeAddress.Address)
+				}
+			}
+		}
+
 	}
 
 	if kubeDeps.PodConfig == nil {
@@ -655,6 +682,33 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
 
 	klet.statusManager = status.NewManager(klet.kubeClient, klet.podManager, klet)
 
+	if utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) && kubeDeps.TLSOptions != nil {
+		var ips []net.IP
+		cfgAddress := net.ParseIP(kubeCfg.Address)
+		if cfgAddress == nil || cfgAddress.IsUnspecified() {
+			if localIPs, err := allLocalIPsWithoutLoopback(); err != nil {
+				return nil, err
+			} else {
+				ips = localIPs
+			}
+		} else {
+			ips = []net.IP{cfgAddress}
+		}
+		ips = append(ips, cloudIPs...)
+		names := append([]string{klet.GetHostname(), hostnameOverride}, cloudNames...)
+		klet.serverCertificateManager, err = initializeServerCertificateManager(klet.kubeClient, kubeCfg, klet.nodeName, ips, names)
+		if err != nil {
+			return nil, fmt.Errorf("failed to initialize certificate manager: %v", err)
+		}
+		kubeDeps.TLSOptions.Config.GetCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+			cert := klet.serverCertificateManager.Current()
+			if cert == nil {
+				return nil, fmt.Errorf("no certificate available")
+			}
+			return cert, nil
+		}
+	}
+
 	klet.probeManager = prober.NewManager(
 		klet.statusManager,
 		klet.livenessManager,
@@ -866,6 +920,9 @@ type Kubelet struct {
 	// Cached MachineInfo returned by cadvisor.
 	machineInfo *cadvisorapi.MachineInfo
 
+	// Handles certificate rotations.
+	serverCertificateManager certificate.Manager
+
 	// Syncs pods statuses with apiserver; also used as a cache of statuses.
 	statusManager status.Manager
 
@@ -1038,6 +1095,62 @@ type Kubelet struct {
 	dockerLegacyService dockershim.DockerLegacyService
 }
 
+func initializeServerCertificateManager(kubeClient clientset.Interface, kubeCfg *componentconfig.KubeletConfiguration, nodeName types.NodeName, ips []net.IP, hostnames []string) (certificate.Manager, error) {
+	var certSigningRequestClient clientcertificates.CertificateSigningRequestInterface
+	if kubeClient != nil && kubeClient.Certificates() != nil {
+		certSigningRequestClient = kubeClient.Certificates().CertificateSigningRequests()
+	}
+	certificateStore, err := certificate.NewFileStore(
+		"kubelet-server",
+		kubeCfg.CertDirectory,
+		kubeCfg.CertDirectory,
+		kubeCfg.TLSCertFile,
+		kubeCfg.TLSPrivateKeyFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize certificate store: %v", err)
+	}
+	return certificate.NewManager(&certificate.Config{
+		CertificateSigningRequestClient: certSigningRequestClient,
+		Template: &x509.CertificateRequest{
+			Subject: pkix.Name{
+				CommonName:   string(nodeName),
+				Organization: []string{"system:nodes"},
+			},
+			DNSNames:    hostnames,
+			IPAddresses: ips,
+		},
+		Usages: []certificates.KeyUsage{
+			certificates.UsageKeyEncipherment,
+			certificates.UsageServerAuth,
+			certificates.UsageSigning,
+		},
+		CertificateStore: certificateStore,
+	})
+}
+
+func allLocalIPsWithoutLoopback() ([]net.IP, error) {
+	interfaces, err := net.Interfaces()
+	if err != nil {
+		return nil, fmt.Errorf("could not list network interfaces: %v", err)
+	}
+	var ips []net.IP
+	for _, i := range interfaces {
+		addresses, err := i.Addrs()
+		if err != nil {
+			return nil, fmt.Errorf("could not list the addresses for network interface %v: %v\n", i, err)
+		}
+		for _, address := range addresses {
+			switch v := address.(type) {
+			case *net.IPNet:
+				if !v.IP.IsLoopback() {
+					ips = append(ips, v.IP)
+				}
+			}
+		}
+	}
+	return ips, nil
+}
+
 // setupDataDirs creates:
 // 1.  the root directory
 // 2.  the pods directory
@@ -1101,25 +1214,30 @@ func (kl *Kubelet) StartGarbageCollection() {
 // initializeModules will initialize internal modules that do not require the container runtime to be up.
 // Note that the modules here must not depend on modules that are not initialized here.
 func (kl *Kubelet) initializeModules() error {
-	// Step 1: Prometheus metrics.
+	// Prometheus metrics.
 	metrics.Register(kl.runtimeCache)
 
-	// Step 2: Setup filesystem directories.
+	// Setup filesystem directories.
 	if err := kl.setupDataDirs(); err != nil {
 		return err
 	}
 
-	// Step 3: If the container logs directory does not exist, create it.
+	// If the container logs directory does not exist, create it.
 	if _, err := os.Stat(ContainerLogsDir); err != nil {
 		if err := kl.os.MkdirAll(ContainerLogsDir, 0755); err != nil {
 			glog.Errorf("Failed to create directory %q: %v", ContainerLogsDir, err)
 		}
 	}
 
-	// Step 4: Start the image manager.
+	// Start the image manager.
 	kl.imageManager.Start()
 
-	// Step 5: Start container manager.
+	// Start the certificate manager.
+	if utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) {
+		kl.serverCertificateManager.Start()
+	}
+
+	// Start container manager.
 	node, err := kl.getNodeAnyWay()
 	if err != nil {
 		return fmt.Errorf("Kubelet failed to get node info: %v", err)
@@ -1129,17 +1247,17 @@ func (kl *Kubelet) initializeModules() error {
 		return fmt.Errorf("Failed to start ContainerManager %v", err)
 	}
 
-	// Step 6: Start out of memory watcher.
+	// Start out of memory watcher.
 	if err := kl.oomWatcher.Start(kl.nodeRef); err != nil {
 		return fmt.Errorf("Failed to start OOM watcher %v", err)
 	}
 
-	// Step 7: Initialize GPUs
+	// Initialize GPUs
 	if err := kl.gpuManager.Start(); err != nil {
 		glog.Errorf("Failed to start gpuManager %v", err)
 	}
 
-	// Step 8: Start resource analyzer
+	// Start resource analyzer
 	kl.resourceAnalyzer.Start()
 
 	return nil

pkg/kubelet/server/server.go

@@ -134,6 +134,9 @@ func ListenAndServeKubeletServer(
 	}
 	if tlsOptions != nil {
 		s.TLSConfig = tlsOptions.Config
+		// Passing empty strings as the cert and key files means no
+		// cert/keys are specified and GetCertificate in the TLSConfig
+		// should be called instead.
 		glog.Fatal(s.ListenAndServeTLS(tlsOptions.CertFile, tlsOptions.KeyFile))
 	} else {
 		glog.Fatal(s.ListenAndServe())