From 3f5fbfbfac281f40c11de2f57d58cc332affc37b Mon Sep 17 00:00:00 2001 From: Jack Kleeman Date: Tue, 22 Oct 2019 19:10:40 +0100 Subject: [PATCH] Dynamic serving certificate Reload certificate cert and key file from disk every minute and notify the dynamic certificate controller when they change, allowing serving tls config to be updated. --- .../pkg/server/dynamiccertificates/BUILD | 3 +- ...e_content.go => dynamic_cafile_content.go} | 0 .../dynamic_serving_content.go | 179 ++++++++++++++++++ .../apiserver/pkg/server/options/serving.go | 2 +- .../apiserver/pkg/server/secure_serving.go | 11 ++ 5 files changed, 193 insertions(+), 2 deletions(-) rename staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/{dynamicfile_content.go => dynamic_cafile_content.go} (100%) create mode 100644 staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_serving_content.go diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD index 55d9aa65204..1ef2bd57659 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD @@ -5,7 +5,8 @@ go_library( srcs = [ "cert_key.go", "client_ca.go", - "dynamicfile_content.go", + "dynamic_cafile_content.go", + "dynamic_serving_content.go", "named_certificates.go", "static_content.go", "tlsconfig.go", diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamicfile_content.go b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_cafile_content.go similarity index 100% rename from staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamicfile_content.go rename to staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_cafile_content.go diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_serving_content.go b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_serving_content.go new file mode 100644 index 00000000000..39be47fb07d --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_serving_content.go @@ -0,0 +1,179 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dynamiccertificates + +import ( + "crypto/tls" + "fmt" + "io/ioutil" + "sync/atomic" + "time" + + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog" +) + +// DynamicFileServingContent provides a CertKeyContentProvider that can dynamically react to new file content +type DynamicFileServingContent struct { + name string + + // certFile is the name of the certificate file to read. + certFile string + // keyFile is the name of the key file to read. + keyFile string + + // servingCert is a certKeyContent that contains the last read, non-zero length content of the key and cert + servingCert atomic.Value + + listeners []Listener + + // queue only ever has one item, but it has nice error handling backoff/retry semantics + queue workqueue.RateLimitingInterface +} + +var _ Notifier = &DynamicFileServingContent{} +var _ CertKeyContentProvider = &DynamicFileServingContent{} +var _ ControllerRunner = &DynamicFileServingContent{} + +// NewDynamicServingContentFromFiles returns a dynamic CertKeyContentProvider based on a cert and key filename +func NewDynamicServingContentFromFiles(purpose, certFile, keyFile string) (*DynamicFileServingContent, error) { + if len(certFile) == 0 || len(keyFile) == 0 { + return nil, fmt.Errorf("missing filename for serving cert") + } + name := fmt.Sprintf("%s::%s::%s", purpose, certFile, keyFile) + + ret := &DynamicFileServingContent{ + name: name, + certFile: certFile, + keyFile: keyFile, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), fmt.Sprintf("DynamicCABundle-%s", purpose)), + } + if err := ret.loadServingCert(); err != nil { + return nil, err + } + + return ret, nil +} + +// AddListener adds a listener to be notified when the serving cert content changes. +func (c *DynamicFileServingContent) AddListener(listener Listener) { + c.listeners = append(c.listeners, listener) +} + +// loadServingCert determines the next set of content for the file. +func (c *DynamicFileServingContent) loadServingCert() error { + cert, err := ioutil.ReadFile(c.certFile) + if err != nil { + return err + } + key, err := ioutil.ReadFile(c.keyFile) + if err != nil { + return err + } + if len(cert) == 0 || len(key) == 0 { + return fmt.Errorf("missing content for serving cert %q", c.Name()) + } + + // Ensure that the key matches the cert and both are valid + _, err = tls.X509KeyPair(cert, key) + if err != nil { + return err + } + + newCertKey := &certKeyContent{ + cert: cert, + key: key, + } + + // check to see if we have a change. If the values are the same, do nothing. + existing, ok := c.servingCert.Load().(*certKeyContent) + if ok && existing != nil && existing.Equal(newCertKey) { + return nil + } + + c.servingCert.Store(newCertKey) + + for _, listener := range c.listeners { + listener.Enqueue() + } + + return nil +} + +// RunOnce runs a single sync loop +func (c *DynamicFileServingContent) RunOnce() error { + return c.loadServingCert() +} + +// Run starts the controller and blocks until stopCh is closed. +func (c *DynamicFileServingContent) Run(workers int, stopCh <-chan struct{}) { + defer utilruntime.HandleCrash() + defer c.queue.ShutDown() + + klog.Infof("Starting %s", c.name) + defer klog.Infof("Shutting down %s", c.name) + + // doesn't matter what workers say, only start one. + go wait.Until(c.runWorker, time.Second, stopCh) + + // start timer that rechecks every minute, just in case. this also serves to prime the controller quickly. + _ = wait.PollImmediateUntil(FileRefreshDuration, func() (bool, error) { + c.queue.Add(workItemKey) + return false, nil + }, stopCh) + + // TODO this can be wired to an fsnotifier as well. + + <-stopCh +} + +func (c *DynamicFileServingContent) runWorker() { + for c.processNextWorkItem() { + } +} + +func (c *DynamicFileServingContent) processNextWorkItem() bool { + dsKey, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(dsKey) + + err := c.loadServingCert() + if err == nil { + c.queue.Forget(dsKey) + return true + } + + utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err)) + c.queue.AddRateLimited(dsKey) + + return true +} + +// Name is just an identifier +func (c *DynamicFileServingContent) Name() string { + return c.name +} + +// CurrentCertKeyContent provides serving cert byte content +func (c *DynamicFileServingContent) CurrentCertKeyContent() ([]byte, []byte) { + certKeyContent := c.servingCert.Load().(*certKeyContent) + return certKeyContent.cert, certKeyContent.key +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving.go index b20a8b38644..822207cb2cc 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/serving.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving.go @@ -226,7 +226,7 @@ func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error // load main cert if len(serverCertFile) != 0 || len(serverKeyFile) != 0 { var err error - c.Cert, err = dynamiccertificates.NewStaticCertKeyContentFromFiles(serverCertFile, serverKeyFile) + c.Cert, err = dynamiccertificates.NewDynamicServingContentFromFiles("serving-cert", serverCertFile, serverKeyFile) if err != nil { return err } diff --git a/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go b/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go index 66c3bbd2968..586327daa0e 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go +++ b/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go @@ -76,6 +76,9 @@ func (s *SecureServingInfo) tlsConfig(stopCh <-chan struct{}) (*tls.Config, erro if notifier, ok := s.ClientCA.(dynamiccertificates.Notifier); ok { notifier.AddListener(dynamicCertificateController) } + if notifier, ok := s.Cert.(dynamiccertificates.Notifier); ok { + notifier.AddListener(dynamicCertificateController) + } // start controllers if possible if controller, ok := s.ClientCA.(dynamiccertificates.ControllerRunner); ok { // runonce to be sure that we have a value. @@ -85,6 +88,14 @@ func (s *SecureServingInfo) tlsConfig(stopCh <-chan struct{}) (*tls.Config, erro go controller.Run(1, stopCh) } + if controller, ok := s.Cert.(dynamiccertificates.ControllerRunner); ok { + // runonce to be sure that we have a value. + if err := controller.RunOnce(); err != nil { + return nil, err + } + + go controller.Run(1, stopCh) + } // runonce to be sure that we have a value. if err := dynamicCertificateController.RunOnce(); err != nil {