diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD index 55d9aa65204..1ef2bd57659 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD @@ -5,7 +5,8 @@ go_library( srcs = [ "cert_key.go", "client_ca.go", - "dynamicfile_content.go", + "dynamic_cafile_content.go", + "dynamic_serving_content.go", "named_certificates.go", "static_content.go", "tlsconfig.go", diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamicfile_content.go b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_cafile_content.go similarity index 100% rename from staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamicfile_content.go rename to staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_cafile_content.go diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_serving_content.go b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_serving_content.go new file mode 100644 index 00000000000..39be47fb07d --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_serving_content.go @@ -0,0 +1,179 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dynamiccertificates + +import ( + "crypto/tls" + "fmt" + "io/ioutil" + "sync/atomic" + "time" + + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog" +) + +// DynamicFileServingContent provides a CertKeyContentProvider that can dynamically react to new file content +type DynamicFileServingContent struct { + name string + + // certFile is the name of the certificate file to read. + certFile string + // keyFile is the name of the key file to read. + keyFile string + + // servingCert is a certKeyContent that contains the last read, non-zero length content of the key and cert + servingCert atomic.Value + + listeners []Listener + + // queue only ever has one item, but it has nice error handling backoff/retry semantics + queue workqueue.RateLimitingInterface +} + +var _ Notifier = &DynamicFileServingContent{} +var _ CertKeyContentProvider = &DynamicFileServingContent{} +var _ ControllerRunner = &DynamicFileServingContent{} + +// NewDynamicServingContentFromFiles returns a dynamic CertKeyContentProvider based on a cert and key filename +func NewDynamicServingContentFromFiles(purpose, certFile, keyFile string) (*DynamicFileServingContent, error) { + if len(certFile) == 0 || len(keyFile) == 0 { + return nil, fmt.Errorf("missing filename for serving cert") + } + name := fmt.Sprintf("%s::%s::%s", purpose, certFile, keyFile) + + ret := &DynamicFileServingContent{ + name: name, + certFile: certFile, + keyFile: keyFile, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), fmt.Sprintf("DynamicCABundle-%s", purpose)), + } + if err := ret.loadServingCert(); err != nil { + return nil, err + } + + return ret, nil +} + +// AddListener adds a listener to be notified when the serving cert content changes. +func (c *DynamicFileServingContent) AddListener(listener Listener) { + c.listeners = append(c.listeners, listener) +} + +// loadServingCert determines the next set of content for the file. +func (c *DynamicFileServingContent) loadServingCert() error { + cert, err := ioutil.ReadFile(c.certFile) + if err != nil { + return err + } + key, err := ioutil.ReadFile(c.keyFile) + if err != nil { + return err + } + if len(cert) == 0 || len(key) == 0 { + return fmt.Errorf("missing content for serving cert %q", c.Name()) + } + + // Ensure that the key matches the cert and both are valid + _, err = tls.X509KeyPair(cert, key) + if err != nil { + return err + } + + newCertKey := &certKeyContent{ + cert: cert, + key: key, + } + + // check to see if we have a change. If the values are the same, do nothing. + existing, ok := c.servingCert.Load().(*certKeyContent) + if ok && existing != nil && existing.Equal(newCertKey) { + return nil + } + + c.servingCert.Store(newCertKey) + + for _, listener := range c.listeners { + listener.Enqueue() + } + + return nil +} + +// RunOnce runs a single sync loop +func (c *DynamicFileServingContent) RunOnce() error { + return c.loadServingCert() +} + +// Run starts the controller and blocks until stopCh is closed. +func (c *DynamicFileServingContent) Run(workers int, stopCh <-chan struct{}) { + defer utilruntime.HandleCrash() + defer c.queue.ShutDown() + + klog.Infof("Starting %s", c.name) + defer klog.Infof("Shutting down %s", c.name) + + // doesn't matter what workers say, only start one. + go wait.Until(c.runWorker, time.Second, stopCh) + + // start timer that rechecks every minute, just in case. this also serves to prime the controller quickly. + _ = wait.PollImmediateUntil(FileRefreshDuration, func() (bool, error) { + c.queue.Add(workItemKey) + return false, nil + }, stopCh) + + // TODO this can be wired to an fsnotifier as well. + + <-stopCh +} + +func (c *DynamicFileServingContent) runWorker() { + for c.processNextWorkItem() { + } +} + +func (c *DynamicFileServingContent) processNextWorkItem() bool { + dsKey, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(dsKey) + + err := c.loadServingCert() + if err == nil { + c.queue.Forget(dsKey) + return true + } + + utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err)) + c.queue.AddRateLimited(dsKey) + + return true +} + +// Name is just an identifier +func (c *DynamicFileServingContent) Name() string { + return c.name +} + +// CurrentCertKeyContent provides serving cert byte content +func (c *DynamicFileServingContent) CurrentCertKeyContent() ([]byte, []byte) { + certKeyContent := c.servingCert.Load().(*certKeyContent) + return certKeyContent.cert, certKeyContent.key +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving.go index b20a8b38644..822207cb2cc 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/serving.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving.go @@ -226,7 +226,7 @@ func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error // load main cert if len(serverCertFile) != 0 || len(serverKeyFile) != 0 { var err error - c.Cert, err = dynamiccertificates.NewStaticCertKeyContentFromFiles(serverCertFile, serverKeyFile) + c.Cert, err = dynamiccertificates.NewDynamicServingContentFromFiles("serving-cert", serverCertFile, serverKeyFile) if err != nil { return err } diff --git a/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go b/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go index 66c3bbd2968..586327daa0e 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go +++ b/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go @@ -76,6 +76,9 @@ func (s *SecureServingInfo) tlsConfig(stopCh <-chan struct{}) (*tls.Config, erro if notifier, ok := s.ClientCA.(dynamiccertificates.Notifier); ok { notifier.AddListener(dynamicCertificateController) } + if notifier, ok := s.Cert.(dynamiccertificates.Notifier); ok { + notifier.AddListener(dynamicCertificateController) + } // start controllers if possible if controller, ok := s.ClientCA.(dynamiccertificates.ControllerRunner); ok { // runonce to be sure that we have a value. @@ -85,6 +88,14 @@ func (s *SecureServingInfo) tlsConfig(stopCh <-chan struct{}) (*tls.Config, erro go controller.Run(1, stopCh) } + if controller, ok := s.Cert.(dynamiccertificates.ControllerRunner); ok { + // runonce to be sure that we have a value. + if err := controller.RunOnce(); err != nil { + return nil, err + } + + go controller.Run(1, stopCh) + } // runonce to be sure that we have a value. if err := dynamicCertificateController.RunOnce(); err != nil { diff --git a/test/integration/apiserver/certreload/certreload_test.go b/test/integration/apiserver/certreload/certreload_test.go index 6c89abc6c8b..f17370ad051 100644 --- a/test/integration/apiserver/certreload/certreload_test.go +++ b/test/integration/apiserver/certreload/certreload_test.go @@ -17,15 +17,18 @@ limitations under the License. package podlogs import ( + "bytes" "crypto/tls" + "crypto/x509" + "encoding/base64" "io/ioutil" "net/url" + "path" "strings" "testing" "time" "k8s.io/apiserver/pkg/server/dynamiccertificates" - "k8s.io/kubernetes/cmd/kube-apiserver/app/options" "k8s.io/kubernetes/test/integration/framework" ) @@ -129,3 +132,124 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps= } } } + +func TestServingCert(t *testing.T) { + stopCh := make(chan struct{}) + defer close(stopCh) + + var serverKey = []byte(`-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA13f50PPWuR/InxLIoJjHdNSG+jVUd25CY7ZL2J023X2BAY+1 +M6jkLR6C2nSFZnn58ubiB74/d1g/Fg1Twd419iR615A013f+qOoyFx3LFHxU1S6e +v22fgJ6ntK/+4QD5MwNgOwD8k1jN2WxHqNWn16IF4Tidbv8M9A35YHAdtYDYaOJC +kzjVztzRw1y6bKRakpMXxHylQyWmAKDJ2GSbRTbGtjr7Ji54WBfG43k94tO5X8K4 +VGbz/uxrKe1IFMHNOlrjR438dbOXusksx9EIqDA9a42J3qjr5NKSqzCIbgBFl6qu +45V3A7cdRI/sJ2G1aqlWIXh2fAQiaFQAEBrPfwIDAQABAoIBAAZbxgWCjJ2d8H+x +QDZtC8XI18redAWqPU9P++ECkrHqmDoBkalanJEwS1BDDATAKL4gTh9IX/sXoZT3 +A7e+5PzEitN9r/GD2wIFF0FTYcDTAnXgEFM52vEivXQ5lV3yd2gn+1kCaHG4typp +ZZv34iIc5+uDjjHOWQWCvA86f8XxX5EfYH+GkjfixTtN2xhWWlfi9vzYeESS4Jbt +tqfH0iEaZ1Bm/qvb8vFgKiuSTOoSpaf+ojAdtPtXDjf1bBtQQG+RSQkP59O/taLM +FCVuRrU8EtdB0+9anwmAP+O2UqjL5izA578lQtdIh13jHtGEgOcnfGNUphK11y9r +Mg5V28ECgYEA9fwI6Xy1Rb9b9irp4bU5Ec99QXa4x2bxld5cDdNOZWJQu9OnaIbg +kw/1SyUkZZCGMmibM/BiWGKWoDf8E+rn/ujGOtd70sR9U0A94XMPqEv7iHxhpZmD +rZuSz4/snYbOWCZQYXFoD/nqOwE7Atnz7yh+Jti0qxBQ9bmkb9o0QW8CgYEA4D3d +okzodg5QQ1y9L0J6jIC6YysoDedveYZMd4Un9bKlZEJev4OwiT4xXmSGBYq/7dzo +OJOvN6qgPfibr27mSB8NkAk6jL/VdJf3thWxNYmjF4E3paLJ24X31aSipN1Ta6K3 +KKQUQRvixVoI1q+8WHAubBDEqvFnNYRHD+AjKvECgYBkekjhpvEcxme4DBtw+OeQ +4OJXJTmhKemwwB12AERboWc88d3GEqIVMEWQJmHRotFOMfCDrMNfOxYv5+5t7FxL +gaXHT1Hi7CQNJ4afWrKgmjjqrXPtguGIvq2fXzjVt8T9uNjIlNxe+kS1SXFjXsgH +ftDY6VgTMB0B4ozKq6UAvQKBgQDER8K5buJHe+3rmMCMHn+Qfpkndr4ftYXQ9Kn4 +MFiy6sV0hdfTgRzEdOjXu9vH/BRVy3iFFVhYvIR42iTEIal2VaAUhM94Je5cmSyd +eE1eFHTqfRPNazmPaqttmSc4cfa0D4CNFVoZR6RupIl6Cect7jvkIaVUD+wMXxWo +osOFsQKBgDLwVhZWoQ13RV/jfQxS3veBUnHJwQJ7gKlL1XZ16mpfEOOVnJF7Es8j +TIIXXYhgSy/XshUbsgXQ+YGliye/rXSCTXHBXvWShOqxEMgeMYMRkcm8ZLp/DH7C +kC2pemkLPUJqgSh1PASGcJbDJIvFGUfP69tUCYpHpk3nHzexuAg3 +-----END RSA PRIVATE KEY-----`) + + var serverCert = []byte(`-----BEGIN CERTIFICATE----- +MIIDQDCCAiigAwIBAgIJANWw74P5KJk2MA0GCSqGSIb3DQEBCwUAMDQxMjAwBgNV +BAMMKWdlbmVyaWNfd2ViaG9va19hZG1pc3Npb25fcGx1Z2luX3Rlc3RzX2NhMCAX +DTE3MTExNjAwMDUzOVoYDzIyOTEwOTAxMDAwNTM5WjAjMSEwHwYDVQQDExh3ZWJo +b29rLXRlc3QuZGVmYXVsdC5zdmMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQDXd/nQ89a5H8ifEsigmMd01Ib6NVR3bkJjtkvYnTbdfYEBj7UzqOQtHoLa +dIVmefny5uIHvj93WD8WDVPB3jX2JHrXkDTXd/6o6jIXHcsUfFTVLp6/bZ+Anqe0 +r/7hAPkzA2A7APyTWM3ZbEeo1afXogXhOJ1u/wz0DflgcB21gNho4kKTONXO3NHD +XLpspFqSkxfEfKVDJaYAoMnYZJtFNsa2OvsmLnhYF8bjeT3i07lfwrhUZvP+7Gsp +7UgUwc06WuNHjfx1s5e6ySzH0QioMD1rjYneqOvk0pKrMIhuAEWXqq7jlXcDtx1E +j+wnYbVqqVYheHZ8BCJoVAAQGs9/AgMBAAGjZDBiMAkGA1UdEwQCMAAwCwYDVR0P +BAQDAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATApBgNVHREEIjAg +hwR/AAABghh3ZWJob29rLXRlc3QuZGVmYXVsdC5zdmMwDQYJKoZIhvcNAQELBQAD +ggEBAD/GKSPNyQuAOw/jsYZesb+RMedbkzs18sSwlxAJQMUrrXwlVdHrA8q5WhE6 +ABLqU1b8lQ8AWun07R8k5tqTmNvCARrAPRUqls/ryER+3Y9YEcxEaTc3jKNZFLbc +T6YtcnkdhxsiO136wtiuatpYL91RgCmuSpR8+7jEHhuFU01iaASu7ypFrUzrKHTF +bKwiLRQi1cMzVcLErq5CDEKiKhUkoDucyARFszrGt9vNIl/YCcBOkcNvM3c05Hn3 +M++C29JwS3Hwbubg6WO3wjFjoEhpCwU6qRYUz3MRp4tHO4kxKXx+oQnUiFnR7vW0 +YkNtGc1RUDHwecCTFpJtPb7Yu/E= +-----END CERTIFICATE-----`) + + var servingCertPath string + + _, kubeconfig := framework.StartTestServer(t, stopCh, framework.TestServerSetup{ + ModifyServerRunOptions: func(opts *options.ServerRunOptions) { + opts.GenericServerRunOptions.MaxRequestBodyBytes = 1024 * 1024 + servingCertPath = opts.SecureServing.ServerCert.CertDirectory + dynamiccertificates.FileRefreshDuration = 1 * time.Second + }, + }) + apiserverURL, err := url.Parse(kubeconfig.Host) + if err != nil { + t.Fatal(err) + } + + // when we run this the second time, we know which one we are expecting + acceptableCerts := [][]byte{} + tlsConfig := &tls.Config{ + InsecureSkipVerify: true, + VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { + acceptableCerts = make([][]byte, 0, len(rawCerts)) + for _, r := range rawCerts { + acceptableCerts = append(acceptableCerts, r) + } + return nil + }, + } + + conn, err := tls.Dial("tcp", apiserverURL.Host, tlsConfig) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + if err := ioutil.WriteFile(path.Join(servingCertPath, "apiserver.key"), serverKey, 0644); err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile(path.Join(servingCertPath, "apiserver.crt"), serverCert, 0644); err != nil { + t.Fatal(err) + } + + time.Sleep(4 * time.Second) + + conn2, err := tls.Dial("tcp", apiserverURL.Host, tlsConfig) + if err != nil { + t.Fatal(err) + } + defer conn2.Close() + + cert, err := tls.X509KeyPair(serverCert, serverKey) + if err != nil { + t.Fatal(err) + } + + expectedCerts := cert.Certificate + if len(expectedCerts) != len(acceptableCerts) { + var certs []string + for _, a := range acceptableCerts { + certs = append(certs, base64.StdEncoding.EncodeToString(a)) + } + t.Fatalf("Unexpected number of certs: %v", strings.Join(certs, ":")) + } + for i := range expectedCerts { + if !bytes.Equal(acceptableCerts[i], expectedCerts[i]) { + t.Errorf("expected %q, got %q", base64.StdEncoding.EncodeToString(expectedCerts[i]), base64.StdEncoding.EncodeToString(acceptableCerts[i])) + } + } +}