diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 122ab9eea00..6d23dc92568 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -288,7 +288,6 @@ func initKubeletConfigSync(s *options.KubeletServer) (*componentconfig.KubeletCo func Run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) error { if err := run(s, kubeDeps); err != nil { return fmt.Errorf("failed to run Kubelet: %v", err) - } return nil } @@ -623,7 +622,7 @@ func getNodeName(cloud cloudprovider.Interface, hostname string) (types.NodeName // InitializeTLS checks for a configured TLSCertFile and TLSPrivateKeyFile: if unspecified a new self-signed // certificate and key file are generated. Returns a configured server.TLSOptions object. func InitializeTLS(kf *options.KubeletFlags, kc *componentconfig.KubeletConfiguration) (*server.TLSOptions, error) { - if kc.TLSCertFile == "" && kc.TLSPrivateKeyFile == "" { + if !utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) && kc.TLSCertFile == "" && kc.TLSPrivateKeyFile == "" { kc.TLSCertFile = path.Join(kc.CertDirectory, "kubelet.crt") kc.TLSPrivateKeyFile = path.Join(kc.CertDirectory, "kubelet.key") diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 6c7c42d38ac..67e74e04d15 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -89,6 +89,14 @@ const ( // to take advantage of NoExecute Taints and Tolerations. TaintBasedEvictions utilfeature.Feature = "TaintBasedEvictions" + // owner: @jcbsmpsn + // alpha: v1.7 + // + // Gets a server certificate for the kubelet from the Certificate Signing + // Request API instead of generating one self signed and auto rotates the + // certificate as expiration approaches. + RotateKubeletServerCertificate utilfeature.Feature = "RotateKubeletServerCertificate" + // owner: @msau // alpha: v1.7 // @@ -113,6 +121,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS AffinityInAnnotations: {Default: false, PreRelease: utilfeature.Alpha}, Accelerators: {Default: false, PreRelease: utilfeature.Alpha}, TaintBasedEvictions: {Default: false, PreRelease: utilfeature.Alpha}, + RotateKubeletServerCertificate: {Default: false, PreRelease: utilfeature.Alpha}, PersistentLocalVolumes: {Default: false, PreRelease: utilfeature.Alpha}, // inherited features from generic apiserver, relisted here to get a conflict if it is changed diff --git a/pkg/kubelet/BUILD b/pkg/kubelet/BUILD index 4853e7fcb1d..1ae7c962341 100644 --- a/pkg/kubelet/BUILD +++ b/pkg/kubelet/BUILD @@ -42,16 +42,19 @@ go_library( "//pkg/api/v1/pod:go_default_library", "//pkg/api/v1/resource:go_default_library", "//pkg/api/v1/validation:go_default_library", + "//pkg/apis/certificates/v1beta1:go_default_library", "//pkg/apis/componentconfig:go_default_library", "//pkg/apis/componentconfig/v1alpha1:go_default_library", "//pkg/capabilities:go_default_library", "//pkg/client/clientset_generated/clientset:go_default_library", + "//pkg/client/clientset_generated/clientset/typed/certificates/v1beta1:go_default_library", "//pkg/client/listers/core/v1:go_default_library", "//pkg/cloudprovider:go_default_library", "//pkg/features:go_default_library", "//pkg/fieldpath:go_default_library", "//pkg/kubelet/apis/cri:go_default_library", "//pkg/kubelet/cadvisor:go_default_library", + "//pkg/kubelet/certificate:go_default_library", "//pkg/kubelet/cm:go_default_library", "//pkg/kubelet/config:go_default_library", "//pkg/kubelet/container:go_default_library", diff --git a/pkg/kubelet/certificate/certificate_store.go b/pkg/kubelet/certificate/certificate_store.go index 890961d83f2..53f4d98a4a4 100644 --- a/pkg/kubelet/certificate/certificate_store.go +++ b/pkg/kubelet/certificate/certificate_store.go @@ -195,6 +195,9 @@ func (s *fileStore) Update(certData, keyData []byte) (*tls.Certificate, error) { ts := time.Now().Format("2006-01-02-15-04-05") pemFilename := s.filename(ts) + if err := os.MkdirAll(s.certDirectory, 0755); err != nil { + return nil, fmt.Errorf("could not create directory %q to store certificates: %v", s.certDirectory, err) + } certPath := filepath.Join(s.certDirectory, pemFilename) f, err := os.OpenFile(certPath, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0600) diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index d9997c29afc..c87641c2c46 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -17,6 +17,9 @@ limitations under the License. package kubelet import ( + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" "fmt" "net" "net/http" @@ -52,14 +55,17 @@ import ( "k8s.io/kubernetes/cmd/kubelet/app/options" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/v1" + certificates "k8s.io/kubernetes/pkg/apis/certificates/v1beta1" "k8s.io/kubernetes/pkg/apis/componentconfig" componentconfigv1alpha1 "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" + clientcertificates "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/certificates/v1beta1" corelisters "k8s.io/kubernetes/pkg/client/listers/core/v1" "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/features" internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" "k8s.io/kubernetes/pkg/kubelet/cadvisor" + "k8s.io/kubernetes/pkg/kubelet/certificate" "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/pkg/kubelet/config" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" @@ -305,6 +311,8 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub hostname := nodeutil.GetHostname(hostnameOverride) // Query the cloud provider for our node name, default to hostname nodeName := types.NodeName(hostname) + cloudIPs := []net.IP{} + cloudNames := []string{} if kubeDeps.Cloud != nil { var err error instances, ok := kubeDeps.Cloud.Instances() @@ -318,6 +326,25 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub } glog.V(2).Infof("cloud provider determined current node name to be %s", nodeName) + + if utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) { + nodeAddresses, err := instances.NodeAddresses(nodeName) + if err != nil { + return nil, fmt.Errorf("failed to get the addresses of the current instance from the cloud provider: %v", err) + } + for _, nodeAddress := range nodeAddresses { + switch nodeAddress.Type { + case v1.NodeExternalIP, v1.NodeInternalIP: + ip := net.ParseIP(nodeAddress.Address) + if ip != nil && !ip.IsLoopback() { + cloudIPs = append(cloudIPs, ip) + } + case v1.NodeExternalDNS, v1.NodeInternalDNS, v1.NodeHostName: + cloudNames = append(cloudNames, nodeAddress.Address) + } + } + } + } if kubeDeps.PodConfig == nil { @@ -655,6 +682,33 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub klet.statusManager = status.NewManager(klet.kubeClient, klet.podManager, klet) + if utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) && kubeDeps.TLSOptions != nil { + var ips []net.IP + cfgAddress := net.ParseIP(kubeCfg.Address) + if cfgAddress == nil || cfgAddress.IsUnspecified() { + if localIPs, err := allLocalIPsWithoutLoopback(); err != nil { + return nil, err + } else { + ips = localIPs + } + } else { + ips = []net.IP{cfgAddress} + } + ips = append(ips, cloudIPs...) + names := append([]string{klet.GetHostname(), hostnameOverride}, cloudNames...) + klet.serverCertificateManager, err = initializeServerCertificateManager(klet.kubeClient, kubeCfg, klet.nodeName, ips, names) + if err != nil { + return nil, fmt.Errorf("failed to initialize certificate manager: %v", err) + } + kubeDeps.TLSOptions.Config.GetCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + cert := klet.serverCertificateManager.Current() + if cert == nil { + return nil, fmt.Errorf("no certificate available") + } + return cert, nil + } + } + klet.probeManager = prober.NewManager( klet.statusManager, klet.livenessManager, @@ -866,6 +920,9 @@ type Kubelet struct { // Cached MachineInfo returned by cadvisor. machineInfo *cadvisorapi.MachineInfo + // Handles certificate rotations. + serverCertificateManager certificate.Manager + // Syncs pods statuses with apiserver; also used as a cache of statuses. statusManager status.Manager @@ -1038,6 +1095,62 @@ type Kubelet struct { dockerLegacyService dockershim.DockerLegacyService } +func initializeServerCertificateManager(kubeClient clientset.Interface, kubeCfg *componentconfig.KubeletConfiguration, nodeName types.NodeName, ips []net.IP, hostnames []string) (certificate.Manager, error) { + var certSigningRequestClient clientcertificates.CertificateSigningRequestInterface + if kubeClient != nil && kubeClient.Certificates() != nil { + certSigningRequestClient = kubeClient.Certificates().CertificateSigningRequests() + } + certificateStore, err := certificate.NewFileStore( + "kubelet-server", + kubeCfg.CertDirectory, + kubeCfg.CertDirectory, + kubeCfg.TLSCertFile, + kubeCfg.TLSPrivateKeyFile) + if err != nil { + return nil, fmt.Errorf("failed to initialize certificate store: %v", err) + } + return certificate.NewManager(&certificate.Config{ + CertificateSigningRequestClient: certSigningRequestClient, + Template: &x509.CertificateRequest{ + Subject: pkix.Name{ + CommonName: string(nodeName), + Organization: []string{"system:nodes"}, + }, + DNSNames: hostnames, + IPAddresses: ips, + }, + Usages: []certificates.KeyUsage{ + certificates.UsageKeyEncipherment, + certificates.UsageServerAuth, + certificates.UsageSigning, + }, + CertificateStore: certificateStore, + }) +} + +func allLocalIPsWithoutLoopback() ([]net.IP, error) { + interfaces, err := net.Interfaces() + if err != nil { + return nil, fmt.Errorf("could not list network interfaces: %v", err) + } + var ips []net.IP + for _, i := range interfaces { + addresses, err := i.Addrs() + if err != nil { + return nil, fmt.Errorf("could not list the addresses for network interface %v: %v\n", i, err) + } + for _, address := range addresses { + switch v := address.(type) { + case *net.IPNet: + if !v.IP.IsLoopback() { + ips = append(ips, v.IP) + } + } + } + } + return ips, nil +} + // setupDataDirs creates: // 1. the root directory // 2. the pods directory @@ -1101,25 +1214,30 @@ func (kl *Kubelet) StartGarbageCollection() { // initializeModules will initialize internal modules that do not require the container runtime to be up. // Note that the modules here must not depend on modules that are not initialized here. func (kl *Kubelet) initializeModules() error { - // Step 1: Prometheus metrics. + // Prometheus metrics. metrics.Register(kl.runtimeCache) - // Step 2: Setup filesystem directories. + // Setup filesystem directories. if err := kl.setupDataDirs(); err != nil { return err } - // Step 3: If the container logs directory does not exist, create it. + // If the container logs directory does not exist, create it. if _, err := os.Stat(ContainerLogsDir); err != nil { if err := kl.os.MkdirAll(ContainerLogsDir, 0755); err != nil { glog.Errorf("Failed to create directory %q: %v", ContainerLogsDir, err) } } - // Step 4: Start the image manager. + // Start the image manager. kl.imageManager.Start() - // Step 5: Start container manager. + // Start the certificate manager. + if utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) { + kl.serverCertificateManager.Start() + } + + // Start container manager. node, err := kl.getNodeAnyWay() if err != nil { return fmt.Errorf("Kubelet failed to get node info: %v", err) @@ -1129,17 +1247,17 @@ func (kl *Kubelet) initializeModules() error { return fmt.Errorf("Failed to start ContainerManager %v", err) } - // Step 6: Start out of memory watcher. + // Start out of memory watcher. if err := kl.oomWatcher.Start(kl.nodeRef); err != nil { return fmt.Errorf("Failed to start OOM watcher %v", err) } - // Step 7: Initialize GPUs + // Initialize GPUs if err := kl.gpuManager.Start(); err != nil { glog.Errorf("Failed to start gpuManager %v", err) } - // Step 8: Start resource analyzer + // Start resource analyzer kl.resourceAnalyzer.Start() return nil diff --git a/pkg/kubelet/server/server.go b/pkg/kubelet/server/server.go index bbb04dc2fb9..a633d3fefda 100644 --- a/pkg/kubelet/server/server.go +++ b/pkg/kubelet/server/server.go @@ -134,6 +134,9 @@ func ListenAndServeKubeletServer( } if tlsOptions != nil { s.TLSConfig = tlsOptions.Config + // Passing empty strings as the cert and key files means no + // cert/keys are specified and GetCertificate in the TLSConfig + // should be called instead. glog.Fatal(s.ListenAndServeTLS(tlsOptions.CertFile, tlsOptions.KeyFile)) } else { glog.Fatal(s.ListenAndServe())