From 4c22e6bc6a2be95a05077ef95e67a4ead9f3f3d2 Mon Sep 17 00:00:00 2001 From: Jacob Simpson Date: Fri, 17 Feb 2017 11:32:41 -0800 Subject: [PATCH] Certificate rotation for kubelet server certs. Replaces the current kubelet server side self signed certs with certs signed by the Certificate Request Signing API on the API server. Also renews expiring kubelet server certs as expiration approaches. --- cmd/kubelet/app/server.go | 3 +- pkg/features/kube_features.go | 9 ++ pkg/kubelet/BUILD | 3 + pkg/kubelet/certificate/certificate_store.go | 3 + pkg/kubelet/kubelet.go | 134 +++++++++++++++++-- pkg/kubelet/server/server.go | 3 + 6 files changed, 145 insertions(+), 10 deletions(-) diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 5588b85c118..7972066973d 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -288,7 +288,6 @@ func initKubeletConfigSync(s *options.KubeletServer) (*componentconfig.KubeletCo func Run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) error { if err := run(s, kubeDeps); err != nil { return fmt.Errorf("failed to run Kubelet: %v", err) - } return nil } @@ -623,7 +622,7 @@ func getNodeName(cloud cloudprovider.Interface, hostname string) (types.NodeName // InitializeTLS checks for a configured TLSCertFile and TLSPrivateKeyFile: if unspecified a new self-signed // certificate and key file are generated. Returns a configured server.TLSOptions object. func InitializeTLS(kf *options.KubeletFlags, kc *componentconfig.KubeletConfiguration) (*server.TLSOptions, error) { - if kc.TLSCertFile == "" && kc.TLSPrivateKeyFile == "" { + if !utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) && kc.TLSCertFile == "" && kc.TLSPrivateKeyFile == "" { kc.TLSCertFile = path.Join(kc.CertDirectory, "kubelet.crt") kc.TLSPrivateKeyFile = path.Join(kc.CertDirectory, "kubelet.key") diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 6c7c42d38ac..67e74e04d15 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -89,6 +89,14 @@ const ( // to take advantage of NoExecute Taints and Tolerations. TaintBasedEvictions utilfeature.Feature = "TaintBasedEvictions" + // owner: @jcbsmpsn + // alpha: v1.7 + // + // Gets a server certificate for the kubelet from the Certificate Signing + // Request API instead of generating one self signed and auto rotates the + // certificate as expiration approaches. + RotateKubeletServerCertificate utilfeature.Feature = "RotateKubeletServerCertificate" + // owner: @msau // alpha: v1.7 // @@ -113,6 +121,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS AffinityInAnnotations: {Default: false, PreRelease: utilfeature.Alpha}, Accelerators: {Default: false, PreRelease: utilfeature.Alpha}, TaintBasedEvictions: {Default: false, PreRelease: utilfeature.Alpha}, + RotateKubeletServerCertificate: {Default: false, PreRelease: utilfeature.Alpha}, PersistentLocalVolumes: {Default: false, PreRelease: utilfeature.Alpha}, // inherited features from generic apiserver, relisted here to get a conflict if it is changed diff --git a/pkg/kubelet/BUILD b/pkg/kubelet/BUILD index 4853e7fcb1d..1ae7c962341 100644 --- a/pkg/kubelet/BUILD +++ b/pkg/kubelet/BUILD @@ -42,16 +42,19 @@ go_library( "//pkg/api/v1/pod:go_default_library", "//pkg/api/v1/resource:go_default_library", "//pkg/api/v1/validation:go_default_library", + "//pkg/apis/certificates/v1beta1:go_default_library", "//pkg/apis/componentconfig:go_default_library", "//pkg/apis/componentconfig/v1alpha1:go_default_library", "//pkg/capabilities:go_default_library", "//pkg/client/clientset_generated/clientset:go_default_library", + "//pkg/client/clientset_generated/clientset/typed/certificates/v1beta1:go_default_library", "//pkg/client/listers/core/v1:go_default_library", "//pkg/cloudprovider:go_default_library", "//pkg/features:go_default_library", "//pkg/fieldpath:go_default_library", "//pkg/kubelet/apis/cri:go_default_library", "//pkg/kubelet/cadvisor:go_default_library", + "//pkg/kubelet/certificate:go_default_library", "//pkg/kubelet/cm:go_default_library", "//pkg/kubelet/config:go_default_library", "//pkg/kubelet/container:go_default_library", diff --git a/pkg/kubelet/certificate/certificate_store.go b/pkg/kubelet/certificate/certificate_store.go index 890961d83f2..53f4d98a4a4 100644 --- a/pkg/kubelet/certificate/certificate_store.go +++ b/pkg/kubelet/certificate/certificate_store.go @@ -195,6 +195,9 @@ func (s *fileStore) Update(certData, keyData []byte) (*tls.Certificate, error) { ts := time.Now().Format("2006-01-02-15-04-05") pemFilename := s.filename(ts) + if err := os.MkdirAll(s.certDirectory, 0755); err != nil { + return nil, fmt.Errorf("could not create directory %q to store certificates: %v", s.certDirectory, err) + } certPath := filepath.Join(s.certDirectory, pemFilename) f, err := os.OpenFile(certPath, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0600) diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index b06c860a3df..c75c48eba4a 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -17,6 +17,9 @@ limitations under the License. package kubelet import ( + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" "fmt" "net" "net/http" @@ -51,14 +54,17 @@ import ( "k8s.io/client-go/util/integer" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/v1" + certificates "k8s.io/kubernetes/pkg/apis/certificates/v1beta1" "k8s.io/kubernetes/pkg/apis/componentconfig" componentconfigv1alpha1 "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" + clientcertificates "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/certificates/v1beta1" corelisters "k8s.io/kubernetes/pkg/client/listers/core/v1" "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/features" internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" "k8s.io/kubernetes/pkg/kubelet/cadvisor" + "k8s.io/kubernetes/pkg/kubelet/certificate" "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/pkg/kubelet/config" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" @@ -304,6 +310,8 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub hostname := nodeutil.GetHostname(hostnameOverride) // Query the cloud provider for our node name, default to hostname nodeName := types.NodeName(hostname) + cloudIPs := []net.IP{} + cloudNames := []string{} if kubeDeps.Cloud != nil { var err error instances, ok := kubeDeps.Cloud.Instances() @@ -317,6 +325,25 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub } glog.V(2).Infof("cloud provider determined current node name to be %s", nodeName) + + if utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) { + nodeAddresses, err := instances.NodeAddresses(nodeName) + if err != nil { + return nil, fmt.Errorf("failed to get the addresses of the current instance from the cloud provider: %v", err) + } + for _, nodeAddress := range nodeAddresses { + switch nodeAddress.Type { + case v1.NodeExternalIP, v1.NodeInternalIP: + ip := net.ParseIP(nodeAddress.Address) + if ip != nil && !ip.IsLoopback() { + cloudIPs = append(cloudIPs, ip) + } + case v1.NodeExternalDNS, v1.NodeInternalDNS, v1.NodeHostName: + cloudNames = append(cloudNames, nodeAddress.Address) + } + } + } + } if kubeDeps.PodConfig == nil { @@ -654,6 +681,33 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub klet.statusManager = status.NewManager(klet.kubeClient, klet.podManager, klet) + if utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) && kubeDeps.TLSOptions != nil { + var ips []net.IP + cfgAddress := net.ParseIP(kubeCfg.Address) + if cfgAddress == nil || cfgAddress.IsUnspecified() { + if localIPs, err := allLocalIPsWithoutLoopback(); err != nil { + return nil, err + } else { + ips = localIPs + } + } else { + ips = []net.IP{cfgAddress} + } + ips = append(ips, cloudIPs...) + names := append([]string{klet.GetHostname(), hostnameOverride}, cloudNames...) + klet.serverCertificateManager, err = initializeServerCertificateManager(klet.kubeClient, kubeCfg, klet.nodeName, ips, names) + if err != nil { + return nil, fmt.Errorf("failed to initialize certificate manager: %v", err) + } + kubeDeps.TLSOptions.Config.GetCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + cert := klet.serverCertificateManager.Current() + if cert == nil { + return nil, fmt.Errorf("no certificate available") + } + return cert, nil + } + } + klet.probeManager = prober.NewManager( klet.statusManager, klet.livenessManager, @@ -865,6 +919,9 @@ type Kubelet struct { // Cached MachineInfo returned by cadvisor. machineInfo *cadvisorapi.MachineInfo + // Handles certificate rotations. + serverCertificateManager certificate.Manager + // Syncs pods statuses with apiserver; also used as a cache of statuses. statusManager status.Manager @@ -1037,6 +1094,62 @@ type Kubelet struct { dockerLegacyService dockershim.DockerLegacyService } +func initializeServerCertificateManager(kubeClient clientset.Interface, kubeCfg *componentconfig.KubeletConfiguration, nodeName types.NodeName, ips []net.IP, hostnames []string) (certificate.Manager, error) { + var certSigningRequestClient clientcertificates.CertificateSigningRequestInterface + if kubeClient != nil && kubeClient.Certificates() != nil { + certSigningRequestClient = kubeClient.Certificates().CertificateSigningRequests() + } + certificateStore, err := certificate.NewFileStore( + "kubelet-server", + kubeCfg.CertDirectory, + kubeCfg.CertDirectory, + kubeCfg.TLSCertFile, + kubeCfg.TLSPrivateKeyFile) + if err != nil { + return nil, fmt.Errorf("failed to initialize certificate store: %v", err) + } + return certificate.NewManager(&certificate.Config{ + CertificateSigningRequestClient: certSigningRequestClient, + Template: &x509.CertificateRequest{ + Subject: pkix.Name{ + CommonName: string(nodeName), + Organization: []string{"system:nodes"}, + }, + DNSNames: hostnames, + IPAddresses: ips, + }, + Usages: []certificates.KeyUsage{ + certificates.UsageKeyEncipherment, + certificates.UsageServerAuth, + certificates.UsageSigning, + }, + CertificateStore: certificateStore, + }) +} + +func allLocalIPsWithoutLoopback() ([]net.IP, error) { + interfaces, err := net.Interfaces() + if err != nil { + return nil, fmt.Errorf("could not list network interfaces: %v", err) + } + var ips []net.IP + for _, i := range interfaces { + addresses, err := i.Addrs() + if err != nil { + return nil, fmt.Errorf("could not list the addresses for network interface %v: %v\n", i, err) + } + for _, address := range addresses { + switch v := address.(type) { + case *net.IPNet: + if !v.IP.IsLoopback() { + ips = append(ips, v.IP) + } + } + } + } + return ips, nil +} + // setupDataDirs creates: // 1. the root directory // 2. the pods directory @@ -1100,25 +1213,30 @@ func (kl *Kubelet) StartGarbageCollection() { // initializeModules will initialize internal modules that do not require the container runtime to be up. // Note that the modules here must not depend on modules that are not initialized here. func (kl *Kubelet) initializeModules() error { - // Step 1: Prometheus metrics. + // Prometheus metrics. metrics.Register(kl.runtimeCache) - // Step 2: Setup filesystem directories. + // Setup filesystem directories. if err := kl.setupDataDirs(); err != nil { return err } - // Step 3: If the container logs directory does not exist, create it. + // If the container logs directory does not exist, create it. if _, err := os.Stat(ContainerLogsDir); err != nil { if err := kl.os.MkdirAll(ContainerLogsDir, 0755); err != nil { glog.Errorf("Failed to create directory %q: %v", ContainerLogsDir, err) } } - // Step 4: Start the image manager. + // Start the image manager. kl.imageManager.Start() - // Step 5: Start container manager. + // Start the certificate manager. + if utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) { + kl.serverCertificateManager.Start() + } + + // Start container manager. node, err := kl.getNodeAnyWay() if err != nil { return fmt.Errorf("Kubelet failed to get node info: %v", err) @@ -1128,17 +1246,17 @@ func (kl *Kubelet) initializeModules() error { return fmt.Errorf("Failed to start ContainerManager %v", err) } - // Step 6: Start out of memory watcher. + // Start out of memory watcher. if err := kl.oomWatcher.Start(kl.nodeRef); err != nil { return fmt.Errorf("Failed to start OOM watcher %v", err) } - // Step 7: Initialize GPUs + // Initialize GPUs if err := kl.gpuManager.Start(); err != nil { glog.Errorf("Failed to start gpuManager %v", err) } - // Step 8: Start resource analyzer + // Start resource analyzer kl.resourceAnalyzer.Start() return nil diff --git a/pkg/kubelet/server/server.go b/pkg/kubelet/server/server.go index bbb04dc2fb9..a633d3fefda 100644 --- a/pkg/kubelet/server/server.go +++ b/pkg/kubelet/server/server.go @@ -134,6 +134,9 @@ func ListenAndServeKubeletServer( } if tlsOptions != nil { s.TLSConfig = tlsOptions.Config + // Passing empty strings as the cert and key files means no + // cert/keys are specified and GetCertificate in the TLSConfig + // should be called instead. glog.Fatal(s.ListenAndServeTLS(tlsOptions.CertFile, tlsOptions.KeyFile)) } else { glog.Fatal(s.ListenAndServe())