From 1b303fe5da01b80c7447dc5abde863fd65a72b56 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Fri, 26 Apr 2024 03:45:36 +0000 Subject: [PATCH] enable kubelet server to dynamically load tls certificate files --- pkg/kubelet/certificate/kubelet.go | 67 ++++++++++++++++++++++++++++++ pkg/kubelet/kubelet.go | 31 +++++++++----- 2 files changed, 88 insertions(+), 10 deletions(-) diff --git a/pkg/kubelet/certificate/kubelet.go b/pkg/kubelet/certificate/kubelet.go index 8293894acea..1e238030ab4 100644 --- a/pkg/kubelet/certificate/kubelet.go +++ b/pkg/kubelet/certificate/kubelet.go @@ -17,6 +17,7 @@ limitations under the License. package certificate import ( + "context" "crypto/tls" "crypto/x509" "crypto/x509/pkix" @@ -24,15 +25,18 @@ import ( "math" "net" "sort" + "sync/atomic" "time" certificates "k8s.io/api/certificates/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apiserver/pkg/server/dynamiccertificates" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/util/certificate" compbasemetrics "k8s.io/component-base/metrics" "k8s.io/component-base/metrics/legacyregistry" + "k8s.io/klog/v2" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" "k8s.io/kubernetes/pkg/kubelet/metrics" netutils "k8s.io/utils/net" @@ -234,3 +238,66 @@ func NewKubeletClientCertificateManager( return m, nil } + +// NewKubeletServerCertificateDynamicFileManager creates a certificate manager based on reading and watching certificate and key files. +// The returned struct implements certificate.Manager interface, enabling using it like other CertificateManager in this package. +// But the struct doesn't communicate with API server to perform certificate request at all. +func NewKubeletServerCertificateDynamicFileManager(certFile, keyFile string) (certificate.Manager, error) { + c, err := dynamiccertificates.NewDynamicServingContentFromFiles("kubelet-server-cert-files", certFile, keyFile) + if err != nil { + return nil, fmt.Errorf("unable to set up dynamic certificate manager for kubelet server cert files: %w", err) + } + m := &kubeletServerCertificateDynamicFileManager{ + dynamicCertificateContent: c, + certFile: certFile, + keyFile: keyFile, + } + m.Enqueue() + c.AddListener(m) + return m, nil +} + +// kubeletServerCertificateDynamicFileManager uses a dynamic CertKeyContentProvider based on cert and key files. +type kubeletServerCertificateDynamicFileManager struct { + cancelFn context.CancelFunc + certFile string + keyFile string + dynamicCertificateContent *dynamiccertificates.DynamicCertKeyPairContent + currentTLSCertificate atomic.Pointer[tls.Certificate] +} + +// Enqueue implements the functions to be notified when the serving cert content changes. +func (m *kubeletServerCertificateDynamicFileManager) Enqueue() { + certContent, keyContent := m.dynamicCertificateContent.CurrentCertKeyContent() + cert, err := tls.X509KeyPair(certContent, keyContent) + if err != nil { + klog.ErrorS(err, "invalid certificate and key pair from file", "certFile", m.certFile, "keyFile", m.keyFile) + return + } + m.currentTLSCertificate.Store(&cert) + klog.V(4).InfoS("loaded certificate and key pair in kubelet server certificate manager", "certFile", m.certFile, "keyFile", m.keyFile) +} + +// Current returns the last valid certificate key pair loaded from files. +func (m *kubeletServerCertificateDynamicFileManager) Current() *tls.Certificate { + return m.currentTLSCertificate.Load() +} + +// Start starts watching the certificate and key files +func (m *kubeletServerCertificateDynamicFileManager) Start() { + var ctx context.Context + ctx, m.cancelFn = context.WithCancel(context.Background()) + go m.dynamicCertificateContent.Run(ctx, 1) +} + +// Stop stops watching the certificate and key files +func (m *kubeletServerCertificateDynamicFileManager) Stop() { + if m.cancelFn != nil { + m.cancelFn() + } +} + +// ServerHealthy always returns true since the file manager doesn't communicate with any server +func (m *kubeletServerCertificateDynamicFileManager) ServerHealthy() bool { + return true +} diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index af74a095628..1d32a0d3297 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -774,17 +774,28 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, } klet.imageManager = imageManager - if kubeCfg.ServerTLSBootstrap && kubeDeps.TLSOptions != nil && utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) { - klet.serverCertificateManager, err = kubeletcertificate.NewKubeletServerCertificateManager(klet.kubeClient, kubeCfg, klet.nodeName, klet.getLastObservedNodeAddresses, certDirectory) - if err != nil { - return nil, fmt.Errorf("failed to initialize certificate manager: %v", err) - } - kubeDeps.TLSOptions.Config.GetCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) { - cert := klet.serverCertificateManager.Current() - if cert == nil { - return nil, fmt.Errorf("no serving certificate available for the kubelet") + if kubeDeps.TLSOptions != nil { + if kubeCfg.ServerTLSBootstrap && utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) { + klet.serverCertificateManager, err = kubeletcertificate.NewKubeletServerCertificateManager(klet.kubeClient, kubeCfg, klet.nodeName, klet.getLastObservedNodeAddresses, certDirectory) + if err != nil { + return nil, fmt.Errorf("failed to initialize certificate manager: %w", err) + } + + } else if kubeDeps.TLSOptions.CertFile != "" && kubeDeps.TLSOptions.KeyFile != "" && utilfeature.DefaultFeatureGate.Enabled(features.ReloadKubeletServerCertificateFile) { + klet.serverCertificateManager, err = kubeletcertificate.NewKubeletServerCertificateDynamicFileManager(kubeDeps.TLSOptions.CertFile, kubeDeps.TLSOptions.KeyFile) + if err != nil { + return nil, fmt.Errorf("failed to initialize file based certificate manager: %w", err) + } + } + + if klet.serverCertificateManager != nil { + kubeDeps.TLSOptions.Config.GetCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + cert := klet.serverCertificateManager.Current() + if cert == nil { + return nil, fmt.Errorf("no serving certificate available for the kubelet") + } + return cert, nil } - return cert, nil } }