diff --git a/cmd/kubelet/app/BUILD b/cmd/kubelet/app/BUILD index 0202f3d5b0f..64d34049287 100644 --- a/cmd/kubelet/app/BUILD +++ b/cmd/kubelet/app/BUILD @@ -37,6 +37,7 @@ go_library( "//cmd/kubelet/app/options:go_default_library", "//pkg/api:go_default_library", "//pkg/api/v1:go_default_library", + "//pkg/apis/certificates/v1beta1:go_default_library", "//pkg/apis/componentconfig:go_default_library", "//pkg/apis/componentconfig/v1alpha1:go_default_library", "//pkg/capabilities:go_default_library", @@ -52,6 +53,7 @@ go_library( "//pkg/features:go_default_library", "//pkg/kubelet:go_default_library", "//pkg/kubelet/cadvisor:go_default_library", + "//pkg/kubelet/certificate:go_default_library", "//pkg/kubelet/cm:go_default_library", "//pkg/kubelet/config:go_default_library", "//pkg/kubelet/container:go_default_library", @@ -110,6 +112,7 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 6d23dc92568..5642a4d9dda 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -19,6 +19,8 @@ package app import ( "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" "errors" "fmt" "io/ioutil" @@ -41,6 +43,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + utilnet "k8s.io/apimachinery/pkg/util/net" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" @@ -58,6 +61,7 @@ import ( "k8s.io/kubernetes/cmd/kubelet/app/options" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/v1" + certificates "k8s.io/kubernetes/pkg/apis/certificates/v1beta1" "k8s.io/kubernetes/pkg/apis/componentconfig" componentconfigv1alpha1 "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1" "k8s.io/kubernetes/pkg/capabilities" @@ -68,6 +72,7 @@ import ( "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet" "k8s.io/kubernetes/pkg/kubelet/cadvisor" + "k8s.io/kubernetes/pkg/kubelet/certificate" "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/pkg/kubelet/config" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" @@ -446,10 +451,30 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) (err error) { } clientConfig, err := CreateAPIServerClientConfig(s) + + var clientCertificateManager certificate.Manager if err == nil { + if utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletClientCertificate) { + nodeName, err := getNodeName(cloud, nodeutil.GetHostname(s.HostnameOverride)) + if err != nil { + return err + } + clientCertificateManager, err = initializeClientCertificateManager(s.CertDirectory, nodeName, clientConfig.CertData, clientConfig.KeyData) + if err != nil { + return err + } + if err := updateTransport(clientConfig, clientCertificateManager); err != nil { + return err + } + } + kubeClient, err = clientset.NewForConfig(clientConfig) if err != nil { glog.Warningf("New kubeClient from clientConfig error: %v", err) + } else if kubeClient.Certificates() != nil && clientCertificateManager != nil { + glog.V(2).Info("Starting client certificate rotation.") + clientCertificateManager.SetCertificateSigningRequestClient(kubeClient.Certificates().CertificateSigningRequests()) + clientCertificateManager.Start() } externalKubeClient, err = clientgoclientset.NewForConfig(clientConfig) if err != nil { @@ -597,6 +622,90 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) (err error) { return nil } +func updateTransport(clientConfig *restclient.Config, clientCertificateManager certificate.Manager) error { + if clientConfig.Transport != nil { + return fmt.Errorf("there is already a transport configured") + } + tlsConfig, err := restclient.TLSConfigFor(clientConfig) + if err != nil { + return fmt.Errorf("unable to configure TLS for the rest client: %v", err) + } + if tlsConfig == nil { + tlsConfig = &tls.Config{} + } + tlsConfig.Certificates = nil + tlsConfig.GetClientCertificate = func(requestInfo *tls.CertificateRequestInfo) (*tls.Certificate, error) { + cert := clientCertificateManager.Current() + if cert == nil { + return &tls.Certificate{Certificate: nil}, nil + } + return cert, nil + } + clientConfig.Transport = utilnet.SetTransportDefaults(&http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSHandshakeTimeout: 10 * time.Second, + TLSClientConfig: tlsConfig, + MaxIdleConnsPerHost: 25, + Dial: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).Dial, + }) + clientConfig.CertData = nil + clientConfig.KeyData = nil + clientConfig.CertFile = "" + clientConfig.KeyFile = "" + clientConfig.CAData = nil + clientConfig.CAFile = "" + return nil +} + +// initializeClientCertificateManager sets up a certificate manager without a +// client that can be used to sign new certificates (or rotate). It answers with +// whatever certificate it is initialized with. If a CSR client is set later, it +// may begin rotating/renewing the client cert +func initializeClientCertificateManager(certDirectory string, nodeName types.NodeName, certData []byte, keyData []byte) (certificate.Manager, error) { + certificateStore, err := certificate.NewFileStore( + "kubelet-client", + certDirectory, + certDirectory, + "", + "") + if err != nil { + return nil, fmt.Errorf("failed to initialize certificate store: %v", err) + } + clientCertificateManager, err := certificate.NewManager(&certificate.Config{ + Template: &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"system:nodes"}, + CommonName: fmt.Sprintf("system:node:%s", nodeName), + }, + }, + Usages: []certificates.KeyUsage{ + // https://tools.ietf.org/html/rfc5280#section-4.2.1.3 + // + // DigitalSignature allows the certificate to be used to verify + // digital signatures including signatures used during TLS + // negotiation. + certificates.UsageDigitalSignature, + // KeyEncipherment allows the cert/key pair to be used to encrypt + // keys, including the symetric keys negotiated during TLS setup + // and used for data transfer.. + certificates.UsageKeyEncipherment, + // ClientAuth allows the cert to be used by a TLS client to + // authenticate itself to the TLS server. + certificates.UsageClientAuth, + }, + CertificateStore: certificateStore, + BootstrapCertificatePEM: certData, + BootstrapKeyPEM: keyData, + }) + if err != nil { + return nil, fmt.Errorf("failed to initialize certificate manager: %v", err) + } + return clientCertificateManager, nil +} + // getNodeName returns the node name according to the cloud provider // if cloud provider is specified. Otherwise, returns the hostname of the node. func getNodeName(cloud cloudprovider.Interface, hostname string) (types.NodeName, error) { diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 3e6828f69e0..7920d4bc5cf 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -97,6 +97,13 @@ const ( // certificate as expiration approaches. RotateKubeletServerCertificate utilfeature.Feature = "RotateKubeletServerCertificate" + // owner: @jcbsmpsn + // alpha: v1.7 + // + // Automatically renews the client certificate used for communicating with + // the API server as the certificate approaches expiration. + RotateKubeletClientCertificate utilfeature.Feature = "RotateKubeletClientCertificate" + // owner: @msau // alpha: v1.7 // @@ -128,6 +135,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS Accelerators: {Default: false, PreRelease: utilfeature.Alpha}, TaintBasedEvictions: {Default: false, PreRelease: utilfeature.Alpha}, RotateKubeletServerCertificate: {Default: false, PreRelease: utilfeature.Alpha}, + RotateKubeletClientCertificate: {Default: false, PreRelease: utilfeature.Alpha}, PersistentLocalVolumes: {Default: false, PreRelease: utilfeature.Alpha}, LocalStorageCapacityIsolation: {Default: false, PreRelease: utilfeature.Alpha}, diff --git a/pkg/kubelet/certificate/certificate_manager.go b/pkg/kubelet/certificate/certificate_manager.go index d02ef4f3468..c381010902b 100644 --- a/pkg/kubelet/certificate/certificate_manager.go +++ b/pkg/kubelet/certificate/certificate_manager.go @@ -52,14 +52,17 @@ type Manager interface { // Start the API server status sync loop. Start() // Current returns the currently selected certificate from the - // certificate manager. + // certificate manager, as well as the associated certificate and key data + // in PEM format. Current() *tls.Certificate } // Config is the set of configuration parameters available for a new Manager. type Config struct { // CertificateSigningRequestClient will be used for signing new certificate - // requests generated when a key rotation occurs. + // requests generated when a key rotation occurs. It must be set either at + // initialization or by using CertificateSigningRequestClient before + // Manager.Start() is called. CertificateSigningRequestClient certificatesclient.CertificateSigningRequestInterface // Template is the CertificateRequest that will be used as a template for // generating certificate signing requests for all new keys generated as @@ -99,7 +102,8 @@ type Config struct { // Depending on the concrete implementation, the backing store for this // behavior may vary. type Store interface { - // Current returns the currently selected certificate. If the Store doesn't + // Current returns the currently selected certificate, as well as the + // associated certificate and key data in PEM format. If the Store doesn't // have a cert/key pair currently, it should return a NoCertKeyError so // that the Manager can recover by using bootstrap certificates to request // a new cert/key pair. diff --git a/pkg/kubelet/certificate/certificate_manager_test.go b/pkg/kubelet/certificate/certificate_manager_test.go index a9fcf7b2dcc..ca493d810a8 100644 --- a/pkg/kubelet/certificate/certificate_manager_test.go +++ b/pkg/kubelet/certificate/certificate_manager_test.go @@ -173,7 +173,7 @@ func TestShouldRotate(t *testing.T) { } m.setRotationDeadline() if m.shouldRotate() != test.shouldRotate { - t.Errorf("For time %v, a certificate issued for (%v, %v) should rotate should be %t.", + t.Errorf("Time %v, a certificate issued for (%v, %v) should rotate should be %t.", now, m.cert.Leaf.NotBefore, m.cert.Leaf.NotAfter, @@ -296,7 +296,6 @@ func TestNewManagerBootstrap(t *testing.T) { if cert == nil { t.Errorf("Certificate was nil, expected something.") } - if m, ok := cm.(*manager); !ok { t.Errorf("Expected a '*manager' from 'NewManager'") } else if !m.shouldRotate() { @@ -335,7 +334,6 @@ func TestNewManagerNoBootstrap(t *testing.T) { if currentCert == nil { t.Errorf("Certificate was nil, expected something.") } - if m, ok := cm.(*manager); !ok { t.Errorf("Expected a '*manager' from 'NewManager'") } else { @@ -386,8 +384,8 @@ func TestGetCurrentCertificateOrBootstrap(t *testing.T) { store, tc.bootstrapCertData, tc.bootstrapKeyData) - if certResult == nil || tc.expectedCert == nil { - if certResult != tc.expectedCert { + if certResult == nil || certResult.Certificate == nil || tc.expectedCert == nil { + if certResult != nil && tc.expectedCert != nil { t.Errorf("Got certificate %v, wanted %v", certResult, tc.expectedCert) } } else {