From 7555dec82ed0afcff7fef5f7a6df36cebabecc68 Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Sun, 1 Oct 2017 16:52:07 -0400 Subject: [PATCH] Kubelet should exit if the current client cert has expired The client cert manager uses the most recent cert to request new certificates. If that certificate is expired, it will be unable to complete new CSR requests. This commit alters the manager to force process exit if no further client cert rotation is possible, which is expected to trigger a restart of the kubelet and either a re-bootstrap from the bootstrap kubeconfig or a re-read of the current disk state (assuming that some other agent is managing the bootstrap configuration). This prevents the Kubelet from wedging in a state where it cannot make API calls. --- cmd/kubelet/app/server.go | 4 +++- pkg/kubelet/certificate/transport.go | 9 ++++++--- pkg/kubelet/certificate/transport_test.go | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 099b5f50da3..66badd8f2d6 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -342,7 +342,9 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies) (err error) { if err != nil { return err } - if err := kubeletcertificate.UpdateTransport(wait.NeverStop, clientConfig, clientCertificateManager); err != nil { + // we set exitIfExpired to true because we use this client configuration to request new certs - if we are unable + // to request new certs, we will be unable to continue normal operation + if err := kubeletcertificate.UpdateTransport(wait.NeverStop, clientConfig, clientCertificateManager, true); err != nil { return err } } diff --git a/pkg/kubelet/certificate/transport.go b/pkg/kubelet/certificate/transport.go index 49c089b44a9..d206f23cdeb 100644 --- a/pkg/kubelet/certificate/transport.go +++ b/pkg/kubelet/certificate/transport.go @@ -45,13 +45,13 @@ import ( // // stopCh should be used to indicate when the transport is unused and doesn't need // to continue checking the manager. -func UpdateTransport(stopCh <-chan struct{}, clientConfig *restclient.Config, clientCertificateManager certificate.Manager) error { - return updateTransport(stopCh, 10*time.Second, clientConfig, clientCertificateManager) +func UpdateTransport(stopCh <-chan struct{}, clientConfig *restclient.Config, clientCertificateManager certificate.Manager, exitIfExpired bool) error { + return updateTransport(stopCh, 10*time.Second, clientConfig, clientCertificateManager, exitIfExpired) } // updateTransport is an internal method that exposes how often this method checks that the // client cert has changed. Intended for testing. -func updateTransport(stopCh <-chan struct{}, period time.Duration, clientConfig *restclient.Config, clientCertificateManager certificate.Manager) error { +func updateTransport(stopCh <-chan struct{}, period time.Duration, clientConfig *restclient.Config, clientCertificateManager certificate.Manager, exitIfExpired bool) error { if clientConfig.Transport != nil { return fmt.Errorf("there is already a transport configured") } @@ -80,6 +80,9 @@ func updateTransport(stopCh <-chan struct{}, period time.Duration, clientConfig lastCert := clientCertificateManager.Current() go wait.Until(func() { curr := clientCertificateManager.Current() + if exitIfExpired && curr != nil && time.Now().After(curr.Leaf.NotAfter) { + glog.Fatalf("The currently active client certificate has expired, exiting.") + } if curr == nil || lastCert == curr { // Cert hasn't been rotated. return diff --git a/pkg/kubelet/certificate/transport_test.go b/pkg/kubelet/certificate/transport_test.go index 306c3c18a03..5dce3eafb4c 100644 --- a/pkg/kubelet/certificate/transport_test.go +++ b/pkg/kubelet/certificate/transport_test.go @@ -184,7 +184,7 @@ func TestRotateShutsDownConnections(t *testing.T) { } // Check for a new cert every 10 milliseconds - if err := updateTransport(stop, 10*time.Millisecond, c, m); err != nil { + if err := updateTransport(stop, 10*time.Millisecond, c, m, false); err != nil { t.Fatal(err) }