Make bootstrap client cert loading part of rotation

Ensure that bootstrap+clientcert-rotation in the Kubelet can:

1. happen in the background so that static pods aren't blocked by bootstrap
2. collapse down to a single call path for requesting a CSR
3. reorganize the code to allow future flexibility in retrieving bootstrap creds

Fetching the first certificate and later certificates when the kubelet
is using client rotation and bootstrapping should share the same code
path. We also want to start the Kubelet static pod loop before
bootstrapping completes. Finally, we want to take an incremental step
towards improving how the bootstrap credentials are loaded from disk
(potentially allowing for a CLI call to get credentials, or a remote
plugin that better integrates with cloud providers or KSMs).

Reorganize how the kubelet client config is determined. If rotation is
off, simplify the code path. If rotation is on, load the config
from disk, and then pass that into the cert manager. The cert manager
creates a client each time it tries to request a new cert.

Preserve existing behavior where:

1. bootstrap kubeconfig is used if the current kubeconfig is invalid/expired
2. we create the kubeconfig file based on the bootstrap kubeconfig, pointing to
   the location that new client certs will be placed
3. the newest client cert is used once it has been loaded

cmd/kubelet/app/BUILD

@@ -113,12 +113,12 @@ go_library(
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/typed/authentication/v1beta1:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/typed/authorization/v1beta1:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
         "//staging/src/k8s.io/client-go/rest:go_default_library",
         "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library",
         "//staging/src/k8s.io/client-go/tools/record:go_default_library",
         "//staging/src/k8s.io/client-go/util/cert:go_default_library",
-        "//staging/src/k8s.io/client-go/util/certificate:go_default_library",
         "//staging/src/k8s.io/cloud-provider:go_default_library",
         "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library",
         "//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library",

cmd/kubelet/app/server.go

@@ -50,12 +50,12 @@ import (
 	"k8s.io/apiserver/pkg/util/flag"
 	"k8s.io/client-go/dynamic"
 	clientset "k8s.io/client-go/kubernetes"
+	certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/record"
 	certutil "k8s.io/client-go/util/cert"
-	"k8s.io/client-go/util/certificate"
 	cloudprovider "k8s.io/cloud-provider"
 	csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned"
 	kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
@@ -537,66 +537,39 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan
 		return err
 	}
 
-	if s.BootstrapKubeconfig != "" {
-		if err := bootstrap.LoadClientCert(s.KubeConfig, s.BootstrapKubeconfig, s.CertDirectory, nodeName); err != nil {
-			return err
-		}
-	}
-
 	// if in standalone mode, indicate as much by setting all clients to nil
-	if standaloneMode {
+	switch {
+	case standaloneMode:
 		kubeDeps.KubeClient = nil
 		kubeDeps.DynamicKubeClient = nil
 		kubeDeps.EventClient = nil
 		kubeDeps.HeartbeatClient = nil
 		klog.Warningf("standalone mode, no API client")
-	} else if kubeDeps.KubeClient == nil || kubeDeps.EventClient == nil || kubeDeps.HeartbeatClient == nil || kubeDeps.DynamicKubeClient == nil {
-		// initialize clients if not standalone mode and any of the clients are not provided
-		var kubeClient clientset.Interface
-		var eventClient v1core.EventsGetter
-		var heartbeatClient clientset.Interface
-		var dynamicKubeClient dynamic.Interface
 
-		clientConfig, err := createAPIServerClientConfig(s)
-		if err != nil {
-			return fmt.Errorf("invalid kubeconfig: %v", err)
-		}
-
-		var clientCertificateManager certificate.Manager
-		if s.RotateCertificates && utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletClientCertificate) {
-			clientCertificateManager, err = kubeletcertificate.NewKubeletClientCertificateManager(s.CertDirectory, nodeName, clientConfig.CertData, clientConfig.KeyData, clientConfig.CertFile, clientConfig.KeyFile)
+	case kubeDeps.KubeClient == nil, kubeDeps.EventClient == nil, kubeDeps.HeartbeatClient == nil, kubeDeps.DynamicKubeClient == nil:
+		clientConfig, closeAllConns, err := buildKubeletClientConfig(s, nodeName)
 		if err != nil {
 			return err
 		}
-		}
-		// we set exitAfter to five minutes because we use this client configuration to request new certs - if we are unable
-		// to request new certs, we will be unable to continue normal operation. Exiting the process allows a wrapper
-		// or the bootstrapping credentials to potentially lay down new initial config.
-		closeAllConns, err := kubeletcertificate.UpdateTransport(wait.NeverStop, clientConfig, clientCertificateManager, 5*time.Minute)
+		kubeDeps.OnHeartbeatFailure = closeAllConns
+
+		kubeDeps.KubeClient, err = clientset.NewForConfig(clientConfig)
 		if err != nil {
-			return err
+			return fmt.Errorf("failed to initialize kubelet client: %v", err)
 		}
 
-		kubeClient, err = clientset.NewForConfig(clientConfig)
+		kubeDeps.DynamicKubeClient, err = dynamic.NewForConfig(clientConfig)
 		if err != nil {
-			klog.Warningf("New kubeClient from clientConfig error: %v", err)
-		} else if kubeClient.CertificatesV1beta1() != nil && clientCertificateManager != nil {
-			klog.V(2).Info("Starting client certificate rotation.")
-			clientCertificateManager.SetCertificateSigningRequestClient(kubeClient.CertificatesV1beta1().CertificateSigningRequests())
-			clientCertificateManager.Start()
-		}
-		dynamicKubeClient, err = dynamic.NewForConfig(clientConfig)
-		if err != nil {
-			klog.Warningf("Failed to initialize dynamic KubeClient: %v", err)
+			return fmt.Errorf("failed to initialize kubelet dynamic client: %v", err)
 		}
 
 		// make a separate client for events
 		eventClientConfig := *clientConfig
 		eventClientConfig.QPS = float32(s.EventRecordQPS)
 		eventClientConfig.Burst = int(s.EventBurst)
-		eventClient, err = v1core.NewForConfig(&eventClientConfig)
+		kubeDeps.EventClient, err = v1core.NewForConfig(&eventClientConfig)
 		if err != nil {
-			klog.Warningf("Failed to create API Server client for Events: %v", err)
+			return fmt.Errorf("failed to initialize kubelet event client: %v", err)
 		}
 
 		// make a separate client for heartbeat with throttling disabled and a timeout attached
@@ -610,28 +583,18 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan
 			}
 		}
 		heartbeatClientConfig.QPS = float32(-1)
-		heartbeatClient, err = clientset.NewForConfig(&heartbeatClientConfig)
+		kubeDeps.HeartbeatClient, err = clientset.NewForConfig(&heartbeatClientConfig)
 		if err != nil {
-			klog.Warningf("Failed to create API Server client for heartbeat: %v", err)
+			return fmt.Errorf("failed to initialize kubelet heartbeat client: %v", err)
 		}
 
-		// csiClient works with CRDs that support json only
-		clientConfig.ContentType = "application/json"
-		csiClient, err := csiclientset.NewForConfig(clientConfig)
+		// CRDs are JSON only, and client renegotiation for streaming is not correct as per #67803
+		csiClientConfig := restclient.CopyConfig(clientConfig)
+		csiClientConfig.ContentType = "application/json"
+		kubeDeps.CSIClient, err = csiclientset.NewForConfig(csiClientConfig)
 		if err != nil {
-			klog.Warningf("Failed to create CSI API client: %v", err)
+			return fmt.Errorf("failed to initialize kubelet storage client: %v", err)
 		}
-
-		kubeDeps.KubeClient = kubeClient
-		kubeDeps.DynamicKubeClient = dynamicKubeClient
-		if heartbeatClient != nil {
-			kubeDeps.HeartbeatClient = heartbeatClient
-			kubeDeps.OnHeartbeatFailure = closeAllConns
-		}
-		if eventClient != nil {
-			kubeDeps.EventClient = eventClient
-		}
-		kubeDeps.CSIClient = csiClient
 	}
 
 	// If the kubelet config controller is available, and dynamic config is enabled, start the config and status sync loops
@@ -771,6 +734,74 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan
 	return nil
 }
 
+// buildKubeletClientConfig constructs the appropriate client config for the kubelet depending on whether
+// bootstrapping is enabled or client certificate rotation is enabled.
+func buildKubeletClientConfig(s *options.KubeletServer, nodeName types.NodeName) (*restclient.Config, func(), error) {
+	if s.RotateCertificates && utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletClientCertificate) {
+		certConfig, clientConfig, err := bootstrap.LoadClientConfig(s.KubeConfig, s.BootstrapKubeconfig, s.CertDirectory)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		newClientFn := func(current *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) {
+			// If we have a valid certificate, use that to fetch CSRs. Otherwise use the bootstrap
+			// credentials.
+			// XXX: When an external bootstrap source is available, it should be possible to always use that source
+			//      to retrieve new credentials.
+			config := certConfig
+			if current != nil {
+				config = clientConfig
+			}
+			client, err := clientset.NewForConfig(config)
+			if err != nil {
+				return nil, err
+			}
+			return client.CertificatesV1beta1().CertificateSigningRequests(), nil
+		}
+
+		clientCertificateManager, err := kubeletcertificate.NewKubeletClientCertificateManager(
+			s.CertDirectory,
+			nodeName,
+			clientConfig.CertFile,
+			clientConfig.KeyFile,
+			newClientFn,
+		)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		// the rotating transport will use the cert from the cert manager instead of these files
+		transportConfig := *clientConfig
+		transportConfig.CertFile = ""
+		transportConfig.KeyFile = ""
+
+		// we set exitAfter to five minutes because we use this client configuration to request new certs - if we are unable
+		// to request new certs, we will be unable to continue normal operation. Exiting the process allows a wrapper
+		// or the bootstrapping credentials to potentially lay down new initial config.
+		closeAllConns, err := kubeletcertificate.UpdateTransport(wait.NeverStop, &transportConfig, clientCertificateManager, 5*time.Minute)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		klog.V(2).Info("Starting client certificate rotation.")
+		clientCertificateManager.Start()
+
+		return &transportConfig, closeAllConns, nil
+	}
+
+	if len(s.BootstrapKubeconfig) > 0 {
+		if err := bootstrap.LoadClientCert(s.KubeConfig, s.BootstrapKubeconfig, s.CertDirectory, nodeName); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	clientConfig, err := createAPIServerClientConfig(s)
+	if err != nil {
+		return nil, nil, fmt.Errorf("invalid kubeconfig: %v", err)
+	}
+	return clientConfig, nil, nil
+}
+
 // getNodeName returns the node name according to the cloud provider
 // if cloud provider is specified. Otherwise, returns the hostname of the node.
 func getNodeName(cloud cloudprovider.Interface, hostname string) (types.NodeName, error) {
@@ -869,11 +900,10 @@ func kubeconfigClientConfig(s *options.KubeletServer) (*restclient.Config, error
 // createClientConfig creates a client configuration from the command line arguments.
 // If --kubeconfig is explicitly set, it will be used.
 func createClientConfig(s *options.KubeletServer) (*restclient.Config, error) {
-	if s.BootstrapKubeconfig != "" || len(s.KubeConfig) > 0 {
+	if len(s.BootstrapKubeconfig) > 0 || len(s.KubeConfig) > 0 {
 		return kubeconfigClientConfig(s)
-	} else {
-		return nil, fmt.Errorf("createClientConfig called in standalone mode")
 	}
+	return nil, fmt.Errorf("createClientConfig called in standalone mode")
 }
 
 // createAPIServerClientConfig generates a client.Config from command line flags

pkg/kubelet/certificate/bootstrap/bootstrap.go

@@ -47,11 +47,60 @@ import (
 
 const tmpPrivateKeyFile = "kubelet-client.key.tmp"
 
+// LoadClientConfig tries to load the appropriate client config for retrieving certs and for use by users.
+// If bootstrapPath is empty, only kubeconfigPath is checked. If bootstrap path is set and the contents
+// of kubeconfigPath are valid, both certConfig and userConfig will point to that file. Otherwise the
+// kubeconfigPath on disk is populated based on bootstrapPath but pointing to the location of the client cert
+// in certDir. This preserves the historical behavior of bootstrapping where on subsequent restarts the
+// most recent client cert is used to request new client certs instead of the initial token.
+func LoadClientConfig(kubeconfigPath, bootstrapPath, certDir string) (certConfig, userConfig *restclient.Config, err error) {
+	if len(bootstrapPath) == 0 {
+		clientConfig, err := loadRESTClientConfig(kubeconfigPath)
+		if err != nil {
+			return nil, nil, fmt.Errorf("unable to load kubeconfig: %v", err)
+		}
+		return clientConfig, clientConfig, nil
+	}
+
+	store, err := certificate.NewFileStore("kubelet-client", certDir, certDir, "", "")
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to build bootstrap cert store")
+	}
+
+	ok, err := verifyBootstrapClientConfig(kubeconfigPath)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// use the current client config
+	if ok {
+		clientConfig, err := loadRESTClientConfig(kubeconfigPath)
+		if err != nil {
+			return nil, nil, fmt.Errorf("unable to load kubeconfig: %v", err)
+		}
+		return clientConfig, clientConfig, nil
+	}
+
+	bootstrapClientConfig, err := loadRESTClientConfig(bootstrapPath)
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to load bootstrap kubeconfig: %v", err)
+	}
+
+	clientConfig := restclient.AnonymousClientConfig(bootstrapClientConfig)
+	pemPath := store.CurrentPath()
+	clientConfig.KeyFile = pemPath
+	clientConfig.CertFile = pemPath
+	if err := writeKubeconfigFromBootstrapping(clientConfig, kubeconfigPath, pemPath); err != nil {
+		return nil, nil, err
+	}
+	return bootstrapClientConfig, clientConfig, nil
+}
+
 // LoadClientCert requests a client cert for kubelet if the kubeconfigPath file does not exist.
 // The kubeconfig at bootstrapPath is used to request a client certificate from the API server.
 // On success, a kubeconfig file referencing the generated key and obtained certificate is written to kubeconfigPath.
 // The certificate and key file are stored in certDir.
-func LoadClientCert(kubeconfigPath string, bootstrapPath string, certDir string, nodeName types.NodeName) error {
+func LoadClientCert(kubeconfigPath, bootstrapPath, certDir string, nodeName types.NodeName) error {
 	// Short-circuit if the kubeconfig file exists and is valid.
 	ok, err := verifyBootstrapClientConfig(kubeconfigPath)
 	if err != nil {
@@ -117,8 +166,10 @@ func LoadClientCert(kubeconfigPath string, bootstrapPath string, certDir string,
 		klog.V(2).Infof("failed cleaning up private key file %q: %v", privKeyPath, err)
 	}
 
-	pemPath := store.CurrentPath()
+	return writeKubeconfigFromBootstrapping(bootstrapClientConfig, kubeconfigPath, store.CurrentPath())
+}
 
+func writeKubeconfigFromBootstrapping(bootstrapClientConfig *restclient.Config, kubeconfigPath, pemPath string) error {
 	// Get the CA data from the bootstrap client config.
 	caFile, caData := bootstrapClientConfig.CAFile, []byte{}
 	if len(caFile) == 0 {

pkg/kubelet/certificate/kubelet.go

@@ -17,6 +17,7 @@ limitations under the License.
 package certificate
 
 import (
+	"crypto/tls"
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"fmt"
@@ -29,7 +30,7 @@ import (
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	clientset "k8s.io/client-go/kubernetes"
-	clientcertificates "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
+	certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
 	"k8s.io/client-go/util/certificate"
 	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
 	"k8s.io/kubernetes/pkg/kubelet/metrics"
@@ -38,7 +39,7 @@ import (
 // NewKubeletServerCertificateManager creates a certificate manager for the kubelet when retrieving a server certificate
 // or returns an error.
 func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg *kubeletconfig.KubeletConfiguration, nodeName types.NodeName, getAddresses func() []v1.NodeAddress, certDirectory string) (certificate.Manager, error) {
-	var certSigningRequestClient clientcertificates.CertificateSigningRequestInterface
+	var certSigningRequestClient certificatesclient.CertificateSigningRequestInterface
 	if kubeClient != nil && kubeClient.CertificatesV1beta1() != nil {
 		certSigningRequestClient = kubeClient.CertificatesV1beta1().CertificateSigningRequests()
 	}
@@ -78,7 +79,9 @@ func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg
 	}
 
 	m, err := certificate.NewManager(&certificate.Config{
-		CertificateSigningRequestClient: certSigningRequestClient,
+		ClientFn: func(current *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) {
+			return certSigningRequestClient, nil
+		},
 		GetTemplate: getTemplate,
 		Usages: []certificates.KeyUsage{
 			// https://tools.ietf.org/html/rfc5280#section-4.2.1.3
@@ -142,10 +145,9 @@ func addressesToHostnamesAndIPs(addresses []v1.NodeAddress) (dnsNames []string,
 }
 
 // NewKubeletClientCertificateManager sets up a certificate manager without a
-// client that can be used to sign new certificates (or rotate). It answers with
-// whatever certificate it is initialized with. If a CSR client is set later, it
-// may begin rotating/renewing the client cert
-func NewKubeletClientCertificateManager(certDirectory string, nodeName types.NodeName, certData []byte, keyData []byte, certFile string, keyFile string) (certificate.Manager, error) {
+// client that can be used to sign new certificates (or rotate). If a CSR
+// client is set later, it may begin rotating/renewing the client cert.
+func NewKubeletClientCertificateManager(certDirectory string, nodeName types.NodeName, certFile string, keyFile string, clientFn certificate.CSRClientFunc) (certificate.Manager, error) {
 	certificateStore, err := certificate.NewFileStore(
 		"kubelet-client",
 		certDirectory,
@@ -166,6 +168,7 @@ func NewKubeletClientCertificateManager(certDirectory string, nodeName types.Nod
 	prometheus.MustRegister(certificateExpiration)
 
 	m, err := certificate.NewManager(&certificate.Config{
+		ClientFn: clientFn,
 		Template: &x509.CertificateRequest{
 			Subject: pkix.Name{
 				CommonName:   fmt.Sprintf("system:node:%s", nodeName),
@@ -188,8 +191,6 @@ func NewKubeletClientCertificateManager(certDirectory string, nodeName types.Nod
 			certificates.UsageClientAuth,
 		},
 		CertificateStore:      certificateStore,
-		BootstrapCertificatePEM: certData,
-		BootstrapKeyPEM:         keyData,
 		CertificateExpiration: certificateExpiration,
 	})
 	if err != nil {

staging/src/k8s.io/client-go/util/certificate/certificate_manager.go

@@ -48,9 +48,6 @@ var certificateWaitBackoff = wait.Backoff{Duration: 30 * time.Second, Steps: 4,
 // manager. In the background it communicates with the API server to get new
 // certificates for certificates about to expire.
 type Manager interface {
-	// CertificateSigningRequestClient sets the client interface that is used for
-	// signing new certificates generated as part of rotation.
-	SetCertificateSigningRequestClient(certificatesclient.CertificateSigningRequestInterface) error
 	// Start the API server status sync loop.
 	Start()
 	// Current returns the currently selected certificate from the
@@ -67,11 +64,11 @@ type Manager interface {
 
 // Config is the set of configuration parameters available for a new Manager.
 type Config struct {
-	// CertificateSigningRequestClient will be used for signing new certificate
-	// requests generated when a key rotation occurs. It must be set either at
-	// initialization or by using CertificateSigningRequestClient before
-	// Manager.Start() is called.
-	CertificateSigningRequestClient certificatesclient.CertificateSigningRequestInterface
+	// ClientFn will be used to create a client for
+	// signing new certificate requests generated when a key rotation occurs.
+	// It must be set at initialization. The function will never be invoked
+	// in parallel. It is passed the current client certificate if one exists.
+	ClientFn CSRClientFunc
 	// Template is the CertificateRequest that will be used as a template for
 	// generating certificate signing requests for all new keys generated as
 	// part of rotation. It follows the same rules as the template parameter of
@@ -141,21 +138,32 @@ type Gauge interface {
 // NoCertKeyError indicates there is no cert/key currently available.
 type NoCertKeyError string
 
+// CSRClientFunc returns a new client for requesting CSRs. It passes the
+// current certificate if one is available and valid.
+type CSRClientFunc func(current *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error)
+
 func (e *NoCertKeyError) Error() string { return string(*e) }
 
 type manager struct {
-	certSigningRequestClient certificatesclient.CertificateSigningRequestInterface
 	getTemplate     func() *x509.CertificateRequest
 	lastRequestLock sync.Mutex
 	lastRequest     *x509.CertificateRequest
 	dynamicTemplate bool
 	usages          []certificates.KeyUsage
+	forceRotation   bool
+
 	certStore Store
+
+	certificateExpiration Gauge
+
+	// the following variables must only be accessed under certAccessLock
 	certAccessLock sync.RWMutex
 	cert           *tls.Certificate
-	forceRotation            bool
-	certificateExpiration    Gauge
 	serverHealth   bool
+
+	// the clientFn must only be accessed under the clientAccessLock
+	clientAccessLock sync.Mutex
+	clientFn         CSRClientFunc
 }
 
 // NewManager returns a new certificate manager. A certificate manager is
@@ -176,7 +184,7 @@ func NewManager(config *Config) (Manager, error) {
 	}
 
 	m := manager{
-		certSigningRequestClient: config.CertificateSigningRequestClient,
+		clientFn:              config.ClientFn,
 		getTemplate:           getTemplate,
 		dynamicTemplate:       config.GetTemplate != nil,
 		usages:                config.Usages,
@@ -192,10 +200,14 @@ func NewManager(config *Config) (Manager, error) {
 // Current returns the currently selected certificate from the certificate
 // manager. This can be nil if the manager was initialized without a
 // certificate and has not yet received one from the
-// CertificateSigningRequestClient.
+// CertificateSigningRequestClient, or if the current cert has expired.
 func (m *manager) Current() *tls.Certificate {
 	m.certAccessLock.RLock()
 	defer m.certAccessLock.RUnlock()
+	if m.cert != nil && m.cert.Leaf != nil && time.Now().After(m.cert.Leaf.NotAfter) {
+		klog.V(2).Infof("Current certificate is expired.")
+		return nil
+	}
 	return m.cert
 }
 
@@ -207,26 +219,12 @@ func (m *manager) ServerHealthy() bool {
 	return m.serverHealth
 }
 
-// SetCertificateSigningRequestClient sets the client interface that is used
-// for signing new certificates generated as part of rotation. It must be
-// called before Start() and can not be used to change the
-// CertificateSigningRequestClient that has already been set. This method is to
-// support the one specific scenario where the CertificateSigningRequestClient
-// uses the CertificateManager.
-func (m *manager) SetCertificateSigningRequestClient(certSigningRequestClient certificatesclient.CertificateSigningRequestInterface) error {
-	if m.certSigningRequestClient == nil {
-		m.certSigningRequestClient = certSigningRequestClient
-		return nil
-	}
-	return fmt.Errorf("property CertificateSigningRequestClient is already set")
-}
-
 // Start will start the background work of rotating the certificates.
 func (m *manager) Start() {
 	// Certificate rotation depends on access to the API server certificate
 	// signing API, so don't start the certificate manager if we don't have a
 	// client.
-	if m.certSigningRequestClient == nil {
+	if m.clientFn == nil {
 		klog.V(2).Infof("Certificate rotation is not enabled, no connection to the apiserver.")
 		return
 	}
@@ -327,6 +325,13 @@ func getCurrentCertificateOrBootstrap(
 	return &bootstrapCert, true, nil
 }
 
+func (m *manager) getClient() (certificatesclient.CertificateSigningRequestInterface, error) {
+	current := m.Current()
+	m.clientAccessLock.Lock()
+	defer m.clientAccessLock.Unlock()
+	return m.clientFn(current)
+}
+
 // rotateCerts attempts to request a client cert from the server, wait a reasonable
 // period of time for it to be signed, and then update the cert on disk. If it cannot
 // retrieve a cert, it will return false. It will only return error in exceptional cases.
@@ -341,9 +346,16 @@ func (m *manager) rotateCerts() (bool, error) {
 		return false, nil
 	}
 
+	// request the client each time
+	client, err := m.getClient()
+	if err != nil {
+		utilruntime.HandleError(fmt.Errorf("Unable to load a client to request certificates: %v", err))
+		return false, nil
+	}
+
 	// Call the Certificate Signing Request API to get a certificate for the
 	// new private key.
-	req, err := csr.RequestCertificate(m.certSigningRequestClient, csrPEM, "", m.usages, privateKey)
+	req, err := csr.RequestCertificate(client, csrPEM, "", m.usages, privateKey)
 	if err != nil {
 		utilruntime.HandleError(fmt.Errorf("Failed while requesting a signed certificate from the master: %v", err))
 		return false, m.updateServerError(err)
@@ -359,7 +371,7 @@ func (m *manager) rotateCerts() (bool, error) {
 	var crtPEM []byte
 	watchDuration := time.Minute
 	if err := wait.ExponentialBackoff(certificateWaitBackoff, func() (bool, error) {
-		data, err := csr.WaitForCertificate(m.certSigningRequestClient, req, watchDuration)
+		data, err := csr.WaitForCertificate(client, req, watchDuration)
 		switch {
 		case err == nil:
 			crtPEM = data

staging/src/k8s.io/client-go/util/certificate/certificate_manager_test.go

@@ -60,6 +60,23 @@ iQIgZX08DA8VfvcA5/Xj1Zjdey9FVY6POLXen6RPiabE97UCICp6eUW7ht+2jjar
 e35EltCRCjoejRHTuN9TC0uCoVipAiAXaJIx/Q47vGwiw6Y8KXsNU6y54gTbOSxX
 54LzHNk/+Q==
 -----END RSA PRIVATE KEY-----`)
+var expiredStoreCertData = newCertificateData(`-----BEGIN CERTIFICATE-----
+MIIBFzCBwgIJALhygXnxXmN1MA0GCSqGSIb3DQEBCwUAMBMxETAPBgNVBAMMCGhv
+c3QtMTIzMB4XDTE4MTEwNDIzNTc1NFoXDTE4MTEwNTIzNTc1NFowEzERMA8GA1UE
+AwwIaG9zdC0xMjMwXDANBgkqhkiG9w0BAQEFAANLADBIAkEAtBMa7NWpv3BVlKTC
+PGO/LEsguKqWHBtKzweMY2CVtAL1rQm913huhxF9w+ai76KQ3MHK5IVnLJjYYA5M
+zP2H5QIDAQABMA0GCSqGSIb3DQEBCwUAA0EAN2DPFUtCzqnidL+5nh+46Sk6dkMI
+T5DD11UuuIjZusKvThsHKVCIsyJ2bDo7cTbI+/nklLRP+FcC2wESFUgXbA==
+-----END CERTIFICATE-----`, `-----BEGIN RSA PRIVATE KEY-----
+MIIBUwIBADANBgkqhkiG9w0BAQEFAASCAT0wggE5AgEAAkEAtBMa7NWpv3BVlKTC
+PGO/LEsguKqWHBtKzweMY2CVtAL1rQm913huhxF9w+ai76KQ3MHK5IVnLJjYYA5M
+zP2H5QIDAQABAkAS9BfXab3OKpK3bIgNNyp+DQJKrZnTJ4Q+OjsqkpXvNltPJosf
+G8GsiKu/vAt4HGqI3eU77NvRI+mL4MnHRmXBAiEA3qM4FAtKSRBbcJzPxxLEUSwg
+XSCcosCktbkXvpYrS30CIQDPDxgqlwDEJQ0uKuHkZI38/SPWWqfUmkecwlbpXABK
+iQIgZX08DA8VfvcA5/Xj1Zjdey9FVY6POLXen6RPiabE97UCICp6eUW7ht+2jjar
+e35EltCRCjoejRHTuN9TC0uCoVipAiAXaJIx/Q47vGwiw6Y8KXsNU6y54gTbOSxX
+54LzHNk/+Q==
+-----END RSA PRIVATE KEY-----`)
 var bootstrapCertData = newCertificateData(
 	`-----BEGIN CERTIFICATE-----
 MIICRzCCAfGgAwIBAgIJANXr+UzRFq4TMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNV
@@ -388,8 +405,8 @@ func TestRotateCertCreateCSRError(t *testing.T) {
 		},
 		getTemplate: func() *x509.CertificateRequest { return &x509.CertificateRequest{} },
 		usages:      []certificates.KeyUsage{},
-		certSigningRequestClient: fakeClient{
-			failureType: createError,
+		clientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) {
+			return fakeClient{failureType: createError}, nil
 		},
 	}
 
@@ -411,8 +428,8 @@ func TestRotateCertWaitingForResultError(t *testing.T) {
 		},
 		getTemplate: func() *x509.CertificateRequest { return &x509.CertificateRequest{} },
 		usages:      []certificates.KeyUsage{},
-		certSigningRequestClient: fakeClient{
-			failureType: watchError,
+		clientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) {
+			return fakeClient{failureType: watchError}, nil
 		},
 	}
 
@@ -598,6 +615,14 @@ func TestInitializeCertificateSigningRequestClient(t *testing.T) {
 			expectedCertBeforeStart: storeCertData,
 			expectedCertAfterStart:  storeCertData,
 		},
+		{
+			description:             "Current certificate expired, no bootstrap certificate",
+			storeCert:               expiredStoreCertData,
+			bootstrapCert:           nilCertificate,
+			apiCert:                 apiServerCertData,
+			expectedCertBeforeStart: nil,
+			expectedCertAfterStart:  apiServerCertData,
+		},
 	}
 
 	for _, tc := range testCases {
@@ -621,19 +646,25 @@ func TestInitializeCertificateSigningRequestClient(t *testing.T) {
 				CertificateStore:        certificateStore,
 				BootstrapCertificatePEM: tc.bootstrapCert.certificatePEM,
 				BootstrapKeyPEM:         tc.bootstrapCert.keyPEM,
+				ClientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) {
+					return &fakeClient{
+						certificatePEM: tc.apiCert.certificatePEM,
+					}, nil
+				},
 			})
 			if err != nil {
 				t.Errorf("Got %v, wanted no error.", err)
 			}
 
 			certificate := certificateManager.Current()
+			if tc.expectedCertBeforeStart == nil {
+				if certificate != nil {
+					t.Errorf("Expected certificate to be nil, was %s", certificate.Leaf.NotAfter)
+				}
+			} else {
 				if !certificatesEqual(certificate, tc.expectedCertBeforeStart.certificate) {
 					t.Errorf("Got %v, wanted %v", certificateString(certificate), certificateString(tc.expectedCertBeforeStart.certificate))
 				}
-			if err := certificateManager.SetCertificateSigningRequestClient(&fakeClient{
-				certificatePEM: tc.apiCert.certificatePEM,
-			}); err != nil {
-				t.Errorf("Got error %v, expected none.", err)
 			}
 
 			if m, ok := certificateManager.(*manager); !ok {
@@ -649,6 +680,12 @@ func TestInitializeCertificateSigningRequestClient(t *testing.T) {
 			}
 
 			certificate = certificateManager.Current()
+			if tc.expectedCertAfterStart == nil {
+				if certificate != nil {
+					t.Errorf("Expected certificate to be nil, was %s", certificate.Leaf.NotAfter)
+				}
+				return
+			}
 			if !certificatesEqual(certificate, tc.expectedCertAfterStart.certificate) {
 				t.Errorf("Got %v, wanted %v", certificateString(certificate), certificateString(tc.expectedCertAfterStart.certificate))
 			}
@@ -721,8 +758,10 @@ func TestInitializeOtherRESTClients(t *testing.T) {
 				CertificateStore:        certificateStore,
 				BootstrapCertificatePEM: tc.bootstrapCert.certificatePEM,
 				BootstrapKeyPEM:         tc.bootstrapCert.keyPEM,
-				CertificateSigningRequestClient: &fakeClient{
+				ClientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) {
+					return &fakeClient{
 						certificatePEM: tc.apiCert.certificatePEM,
+					}, nil
 				},
 			})
 			if err != nil {
@@ -873,10 +912,12 @@ func TestServerHealth(t *testing.T) {
 				CertificateStore:        certificateStore,
 				BootstrapCertificatePEM: tc.bootstrapCert.certificatePEM,
 				BootstrapKeyPEM:         tc.bootstrapCert.keyPEM,
-				CertificateSigningRequestClient: &fakeClient{
+				ClientFn: func(_ *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) {
+					return &fakeClient{
 						certificatePEM: tc.apiCert.certificatePEM,
 						failureType:    tc.failureType,
 						err:            tc.clientErr,
+					}, nil
 				},
 			})
 			if err != nil {