Merge pull request #83580 from jackkleeman/apiserver-cert-file-reload5

Plumbing for dynamic apiserver serving certificates

staging/src/k8s.io/apiserver/pkg/server/config.go

@@ -233,9 +233,10 @@ type SecureServingInfo struct {
 
 	// Cert is the main server cert which is used if SNI does not match. Cert must be non-nil and is
 	// allowed to be in SNICerts.
-	Cert *tls.Certificate
+	Cert dynamiccertificates.CertKeyContentProvider
 
 	// SNICerts are the TLS certificates by name used for SNI.
+	// todo: use dynamic certificates
 	SNICerts map[string]*tls.Certificate
 
 	// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD

@@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 go_library(
     name = "go_default_library",
     srcs = [
+        "cert_key.go",
         "client_ca.go",
         "static_content.go",
         "tlsconfig.go",
@@ -40,6 +41,7 @@ filegroup(
 go_test(
     name = "go_default_test",
     srcs = [
+        "cert_key_test.go",
         "client_ca_test.go",
         "tlsconfig_test.go",
     ],

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/cert_key.go

@@ -0,0 +1,43 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"bytes"
+)
+
+// CertKeyContentProvider provides a certificate and matching private key
+type CertKeyContentProvider interface {
+	// Name is just an identifier
+	Name() string
+	// CurrentCertKeyContent provides cert and key byte content
+	CurrentCertKeyContent() ([]byte, []byte)
+}
+
+// caBundleContent holds the content for the cert and key
+type certKeyContent struct {
+	cert []byte
+	key  []byte
+}
+
+func (c *certKeyContent) Equal(rhs *certKeyContent) bool {
+	if c == nil || rhs == nil {
+		return c == rhs
+	}
+
+	return bytes.Equal(c.key, rhs.key) && bytes.Equal(c.cert, rhs.cert)
+}

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/cert_key_test.go

@@ -0,0 +1,76 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import "testing"
+
+func TestCertKeyContentEquals(t *testing.T) {
+	tests := []struct {
+		name     string
+		lhs      *certKeyContent
+		rhs      *certKeyContent
+		expected bool
+	}{
+		{
+			name:     "both nil",
+			expected: true,
+		},
+		{
+			name:     "lhs nil",
+			rhs:      &certKeyContent{},
+			expected: false,
+		},
+		{
+			name:     "rhs nil",
+			lhs:      &certKeyContent{},
+			expected: false,
+		},
+		{
+			name:     "same",
+			lhs:      &certKeyContent{cert: []byte("foo"), key: []byte("baz")},
+			rhs:      &certKeyContent{cert: []byte("foo"), key: []byte("baz")},
+			expected: true,
+		},
+		{
+			name:     "different cert",
+			lhs:      &certKeyContent{cert: []byte("foo"), key: []byte("baz")},
+			rhs:      &certKeyContent{cert: []byte("bar"), key: []byte("baz")},
+			expected: false,
+		},
+		{
+			name:     "different key",
+			lhs:      &certKeyContent{cert: []byte("foo"), key: []byte("baz")},
+			rhs:      &certKeyContent{cert: []byte("foo"), key: []byte("qux")},
+			expected: false,
+		},
+		{
+			name:     "different cert and key",
+			lhs:      &certKeyContent{cert: []byte("foo"), key: []byte("baz")},
+			rhs:      &certKeyContent{cert: []byte("bar"), key: []byte("qux")},
+			expected: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			actual := test.lhs.Equal(test.rhs)
+			if actual != test.expected {
+				t.Error(actual)
+			}
+		})
+	}
+}

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/client_ca.go

@@ -30,10 +30,10 @@ type CAContentProvider interface {
 }
 
 // dynamicCertificateContent holds the content that overrides the baseTLSConfig
-// TODO add the serving certs to this struct
 type dynamicCertificateContent struct {
 	// clientCA holds the content for the clientCA bundle
 	clientCA    caBundleContent
+	servingCert certKeyContent
 }
 
 // caBundleContent holds the content for the clientCA bundle.  Wrapping the bytes makes the Equals work nicely with the
@@ -51,6 +51,10 @@ func (c *dynamicCertificateContent) Equal(rhs *dynamicCertificateContent) bool {
 		return false
 	}
 
+	if !c.servingCert.Equal(&rhs.servingCert) {
+		return false
+	}
+
 	return true
 }
 

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/client_ca_test.go

@@ -59,6 +59,42 @@ func TestDynamicCertificateContentEquals(t *testing.T) {
 			},
 			expected: false,
 		},
+		{
+			name: "same with serving",
+			lhs: &dynamicCertificateContent{
+				clientCA:    caBundleContent{caBundle: []byte("foo")},
+				servingCert: certKeyContent{cert: []byte("foo"), key: []byte("foo")},
+			},
+			rhs: &dynamicCertificateContent{
+				clientCA:    caBundleContent{caBundle: []byte("foo")},
+				servingCert: certKeyContent{cert: []byte("foo"), key: []byte("foo")},
+			},
+			expected: true,
+		},
+		{
+			name: "different serving cert",
+			lhs: &dynamicCertificateContent{
+				clientCA:    caBundleContent{caBundle: []byte("foo")},
+				servingCert: certKeyContent{cert: []byte("foo"), key: []byte("foo")},
+			},
+			rhs: &dynamicCertificateContent{
+				clientCA:    caBundleContent{caBundle: []byte("foo")},
+				servingCert: certKeyContent{cert: []byte("bar"), key: []byte("foo")},
+			},
+			expected: false,
+		},
+		{
+			name: "different serving key",
+			lhs: &dynamicCertificateContent{
+				clientCA:    caBundleContent{caBundle: []byte("foo")},
+				servingCert: certKeyContent{cert: []byte("foo"), key: []byte("foo")},
+			},
+			rhs: &dynamicCertificateContent{
+				clientCA:    caBundleContent{caBundle: []byte("foo")},
+				servingCert: certKeyContent{cert: []byte("foo"), key: []byte("bar")},
+			},
+			expected: false,
+		},
 	}
 
 	for _, test := range tests {

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/static_content.go

@@ -17,6 +17,7 @@ limitations under the License.
 package dynamiccertificates
 
 import (
+	"crypto/tls"
 	"fmt"
 	"io/ioutil"
 )
@@ -56,3 +57,55 @@ func (c *staticCAContent) Name() string {
 func (c *staticCAContent) CurrentCABundleContent() (cabundle []byte) {
 	return c.caBundle
 }
+
+type staticCertKeyContent struct {
+	name string
+	cert []byte
+	key  []byte
+}
+
+// NewStaticCertKeyContentFromFiles returns a CertKeyContentProvider based on a filename
+func NewStaticCertKeyContentFromFiles(certFile, keyFile string) (CertKeyContentProvider, error) {
+	if len(certFile) == 0 {
+		return nil, fmt.Errorf("missing filename for certificate")
+	}
+	if len(keyFile) == 0 {
+		return nil, fmt.Errorf("missing filename for key")
+	}
+
+	certPEMBlock, err := ioutil.ReadFile(certFile)
+	if err != nil {
+		return nil, err
+	}
+	keyPEMBlock, err := ioutil.ReadFile(keyFile)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewStaticCertKeyContent(fmt.Sprintf("cert: %s, key: %s", certFile, keyFile), certPEMBlock, keyPEMBlock)
+}
+
+// NewStaticCertKeyContent returns a CertKeyContentProvider that always returns the same value
+func NewStaticCertKeyContent(name string, cert, key []byte) (CertKeyContentProvider, error) {
+	// Ensure that the key matches the cert and both are valid
+	_, err := tls.X509KeyPair(cert, key)
+	if err != nil {
+		return nil, err
+	}
+
+	return &staticCertKeyContent{
+		name: name,
+		cert: cert,
+		key:  key,
+	}, nil
+}
+
+// Name is just an identifier
+func (c *staticCertKeyContent) Name() string {
+	return c.name
+}
+
+// CurrentCertKeyContent provides cert and key content
+func (c *staticCertKeyContent) CurrentCertKeyContent() ([]byte, []byte) {
+	return c.cert, c.key
+}

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig.go

@@ -44,6 +44,8 @@ type DynamicServingCertificateController struct {
 
 	// clientCA provides the very latest content of the ca bundle
 	clientCA CAContentProvider
+	// servingCert provides the very latest content of the default serving certificate
+	servingCert CertKeyContentProvider
 
 	// currentlyServedContent holds the original bytes that we are serving. This is used to decide if we need to set a
 	// new atomic value. The types used for efficient TLSConfig preclude using the processed value.
@@ -60,11 +62,13 @@ type DynamicServingCertificateController struct {
 func NewDynamicServingCertificateController(
 	baseTLSConfig tls.Config,
 	clientCA CAContentProvider,
+	servingCert CertKeyContentProvider,
 	eventRecorder events.EventRecorder,
 ) *DynamicServingCertificateController {
 	c := &DynamicServingCertificateController{
 		baseTLSConfig: baseTLSConfig,
 		clientCA:      clientCA,
+		servingCert:   servingCert,
 
 		queue:         workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "DynamicServingCertificateController"),
 		eventRecorder: eventRecorder,
@@ -91,13 +95,23 @@ func (c *DynamicServingCertificateController) GetConfigForClient(clientHello *tl
 func (c *DynamicServingCertificateController) newTLSContent() (*dynamicCertificateContent, error) {
 	newContent := &dynamicCertificateContent{}
 
+	if c.clientCA != nil {
 		currClientCABundle := c.clientCA.CurrentCABundleContent()
 		// don't remove all content.  The value was configured at one time, so continue using that.
-	// Errors reading content can be reported by lower level controllers.
 		if len(currClientCABundle) == 0 {
 			return nil, fmt.Errorf("not loading an empty client ca bundle from %q", c.clientCA.Name())
 		}
 		newContent.clientCA = caBundleContent{caBundle: currClientCABundle}
+	}
+
+	if c.servingCert != nil {
+		currServingCert, currServingKey := c.servingCert.CurrentCertKeyContent()
+		if len(currServingCert) == 0 || len(currServingKey) == 0 {
+			return nil, fmt.Errorf("not loading an empty serving certificate from %q", c.servingCert.Name())
+		}
+
+		newContent.servingCert = certKeyContent{cert: currServingCert, key: currServingKey}
+	}
 
 	return newContent, nil
 }
@@ -115,9 +129,12 @@ func (c *DynamicServingCertificateController) syncCerts() error {
 		return nil
 	}
 
+	// make a shallow copy and override the dynamic pieces which have changed.
+	newTLSConfigCopy := c.baseTLSConfig.Clone()
+
 	// parse new content to add to TLSConfig
-	newClientCAPool := x509.NewCertPool()
 	if len(newContent.clientCA.caBundle) > 0 {
+		newClientCAPool := x509.NewCertPool()
 		newClientCAs, err := cert.ParseCertsPEM(newContent.clientCA.caBundle)
 		if err != nil {
 			return fmt.Errorf("unable to load client CA file: %v", err)
@@ -130,11 +147,36 @@ func (c *DynamicServingCertificateController) syncCerts() error {
 
 			newClientCAPool.AddCert(cert)
 		}
+
+		newTLSConfigCopy.ClientCAs = newClientCAPool
 	}
 
-	// make a copy and override the dynamic pieces which have changed.
-	newTLSConfigCopy := c.baseTLSConfig.Clone()
-	newTLSConfigCopy.ClientCAs = newClientCAPool
+	if len(newContent.servingCert.cert) > 0 && len(newContent.servingCert.key) > 0 {
+		cert, err := tls.X509KeyPair(newContent.servingCert.cert, newContent.servingCert.key)
+		if err != nil {
+			return fmt.Errorf("invalid serving cert keypair: %v", err)
+		}
+
+		x509Cert, err := x509.ParseCertificate(cert.Certificate[0])
+		if err != nil {
+			return fmt.Errorf("invalid serving cert: %v", err)
+		}
+
+		klog.V(2).Infof("loaded serving cert [%q]: %s", c.servingCert.Name(), GetHumanCertDetail(x509Cert))
+		if c.eventRecorder != nil {
+			c.eventRecorder.Eventf(nil, nil, v1.EventTypeWarning, "TLSConfigChanged", "ServingCertificateReload", "loaded serving cert [%q]: %s", c.clientCA.Name(), GetHumanCertDetail(x509Cert))
+		}
+
+		newTLSConfigCopy.Certificates = []tls.Certificate{cert}
+
+		// append all named certs. Otherwise, the go tls stack will think no SNI processing
+		// is necessary because there is only one cert anyway.
+		// Moreover, if ServerCert.CertFile/ServerCert.KeyFile are not set, the first SNI
+		// cert will become the default cert. That's what we expect anyway.
+		for _, c := range newTLSConfigCopy.NameToCertificate {
+			newTLSConfigCopy.Certificates = append(newTLSConfigCopy.Certificates, *c)
+		}
+	}
 
 	// store new values of content for serving.
 	c.currentServingTLSConfig.Store(newTLSConfigCopy)

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig_test.go

@@ -23,10 +23,65 @@ import (
 	"github.com/davecgh/go-spew/spew"
 )
 
+var serverKey = []byte(`-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA13f50PPWuR/InxLIoJjHdNSG+jVUd25CY7ZL2J023X2BAY+1
+M6jkLR6C2nSFZnn58ubiB74/d1g/Fg1Twd419iR615A013f+qOoyFx3LFHxU1S6e
+v22fgJ6ntK/+4QD5MwNgOwD8k1jN2WxHqNWn16IF4Tidbv8M9A35YHAdtYDYaOJC
+kzjVztzRw1y6bKRakpMXxHylQyWmAKDJ2GSbRTbGtjr7Ji54WBfG43k94tO5X8K4
+VGbz/uxrKe1IFMHNOlrjR438dbOXusksx9EIqDA9a42J3qjr5NKSqzCIbgBFl6qu
+45V3A7cdRI/sJ2G1aqlWIXh2fAQiaFQAEBrPfwIDAQABAoIBAAZbxgWCjJ2d8H+x
+QDZtC8XI18redAWqPU9P++ECkrHqmDoBkalanJEwS1BDDATAKL4gTh9IX/sXoZT3
+A7e+5PzEitN9r/GD2wIFF0FTYcDTAnXgEFM52vEivXQ5lV3yd2gn+1kCaHG4typp
+ZZv34iIc5+uDjjHOWQWCvA86f8XxX5EfYH+GkjfixTtN2xhWWlfi9vzYeESS4Jbt
+tqfH0iEaZ1Bm/qvb8vFgKiuSTOoSpaf+ojAdtPtXDjf1bBtQQG+RSQkP59O/taLM
+FCVuRrU8EtdB0+9anwmAP+O2UqjL5izA578lQtdIh13jHtGEgOcnfGNUphK11y9r
+Mg5V28ECgYEA9fwI6Xy1Rb9b9irp4bU5Ec99QXa4x2bxld5cDdNOZWJQu9OnaIbg
+kw/1SyUkZZCGMmibM/BiWGKWoDf8E+rn/ujGOtd70sR9U0A94XMPqEv7iHxhpZmD
+rZuSz4/snYbOWCZQYXFoD/nqOwE7Atnz7yh+Jti0qxBQ9bmkb9o0QW8CgYEA4D3d
+okzodg5QQ1y9L0J6jIC6YysoDedveYZMd4Un9bKlZEJev4OwiT4xXmSGBYq/7dzo
+OJOvN6qgPfibr27mSB8NkAk6jL/VdJf3thWxNYmjF4E3paLJ24X31aSipN1Ta6K3
+KKQUQRvixVoI1q+8WHAubBDEqvFnNYRHD+AjKvECgYBkekjhpvEcxme4DBtw+OeQ
+4OJXJTmhKemwwB12AERboWc88d3GEqIVMEWQJmHRotFOMfCDrMNfOxYv5+5t7FxL
+gaXHT1Hi7CQNJ4afWrKgmjjqrXPtguGIvq2fXzjVt8T9uNjIlNxe+kS1SXFjXsgH
+ftDY6VgTMB0B4ozKq6UAvQKBgQDER8K5buJHe+3rmMCMHn+Qfpkndr4ftYXQ9Kn4
+MFiy6sV0hdfTgRzEdOjXu9vH/BRVy3iFFVhYvIR42iTEIal2VaAUhM94Je5cmSyd
+eE1eFHTqfRPNazmPaqttmSc4cfa0D4CNFVoZR6RupIl6Cect7jvkIaVUD+wMXxWo
+osOFsQKBgDLwVhZWoQ13RV/jfQxS3veBUnHJwQJ7gKlL1XZ16mpfEOOVnJF7Es8j
+TIIXXYhgSy/XshUbsgXQ+YGliye/rXSCTXHBXvWShOqxEMgeMYMRkcm8ZLp/DH7C
+kC2pemkLPUJqgSh1PASGcJbDJIvFGUfP69tUCYpHpk3nHzexuAg3
+-----END RSA PRIVATE KEY-----`)
+
+var serverCert = []byte(`-----BEGIN CERTIFICATE-----
+MIIDQDCCAiigAwIBAgIJANWw74P5KJk2MA0GCSqGSIb3DQEBCwUAMDQxMjAwBgNV
+BAMMKWdlbmVyaWNfd2ViaG9va19hZG1pc3Npb25fcGx1Z2luX3Rlc3RzX2NhMCAX
+DTE3MTExNjAwMDUzOVoYDzIyOTEwOTAxMDAwNTM5WjAjMSEwHwYDVQQDExh3ZWJo
+b29rLXRlc3QuZGVmYXVsdC5zdmMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQDXd/nQ89a5H8ifEsigmMd01Ib6NVR3bkJjtkvYnTbdfYEBj7UzqOQtHoLa
+dIVmefny5uIHvj93WD8WDVPB3jX2JHrXkDTXd/6o6jIXHcsUfFTVLp6/bZ+Anqe0
+r/7hAPkzA2A7APyTWM3ZbEeo1afXogXhOJ1u/wz0DflgcB21gNho4kKTONXO3NHD
+XLpspFqSkxfEfKVDJaYAoMnYZJtFNsa2OvsmLnhYF8bjeT3i07lfwrhUZvP+7Gsp
+7UgUwc06WuNHjfx1s5e6ySzH0QioMD1rjYneqOvk0pKrMIhuAEWXqq7jlXcDtx1E
+j+wnYbVqqVYheHZ8BCJoVAAQGs9/AgMBAAGjZDBiMAkGA1UdEwQCMAAwCwYDVR0P
+BAQDAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATApBgNVHREEIjAg
+hwR/AAABghh3ZWJob29rLXRlc3QuZGVmYXVsdC5zdmMwDQYJKoZIhvcNAQELBQAD
+ggEBAD/GKSPNyQuAOw/jsYZesb+RMedbkzs18sSwlxAJQMUrrXwlVdHrA8q5WhE6
+ABLqU1b8lQ8AWun07R8k5tqTmNvCARrAPRUqls/ryER+3Y9YEcxEaTc3jKNZFLbc
+T6YtcnkdhxsiO136wtiuatpYL91RgCmuSpR8+7jEHhuFU01iaASu7ypFrUzrKHTF
+bKwiLRQi1cMzVcLErq5CDEKiKhUkoDucyARFszrGt9vNIl/YCcBOkcNvM3c05Hn3
+M++C29JwS3Hwbubg6WO3wjFjoEhpCwU6qRYUz3MRp4tHO4kxKXx+oQnUiFnR7vW0
+YkNtGc1RUDHwecCTFpJtPb7Yu/E=
+-----END CERTIFICATE-----`)
+
 func TestNewTLSContent(t *testing.T) {
+	testCertProvider, err := NewStaticCertKeyContent("test-cert", serverCert, serverKey)
+	if err != nil {
+		t.Error(err)
+	}
+
 	tests := []struct {
 		name        string
 		clientCA    CAContentProvider
+		servingCert CertKeyContentProvider
 
 		expected    *dynamicCertificateContent
 		expectedErr string
@@ -34,8 +89,10 @@ func TestNewTLSContent(t *testing.T) {
 		{
 			name:        "filled",
 			clientCA:    NewStaticCAContent("test-ca", []byte("content-1")),
+			servingCert: testCertProvider,
 			expected: &dynamicCertificateContent{
 				clientCA:    caBundleContent{caBundle: []byte("content-1")},
+				servingCert: certKeyContent{cert: serverCert, key: serverKey},
 			},
 		},
 		{
@@ -44,12 +101,17 @@ func TestNewTLSContent(t *testing.T) {
 			expected:    nil,
 			expectedErr: `not loading an empty client ca bundle from "test-ca"`,
 		},
+		{
+			name:     "nil",
+			expected: &dynamicCertificateContent{clientCA: caBundleContent{}, servingCert: certKeyContent{}},
+		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			c := &DynamicServingCertificateController{
 				clientCA:    test.clientCA,
+				servingCert: test.servingCert,
 			}
 			actual, err := c.newTLSContent()
 			if !reflect.DeepEqual(actual, test.expected) {

staging/src/k8s.io/apiserver/pkg/server/options/BUILD

@@ -59,6 +59,7 @@ go_library(
         "//staging/src/k8s.io/apiserver/pkg/registry/generic:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server/egressselector:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig:go_default_library",

staging/src/k8s.io/apiserver/pkg/server/options/serving.go

@@ -29,6 +29,7 @@ import (
 
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 	certutil "k8s.io/client-go/util/cert"
 	"k8s.io/client-go/util/keyutil"
 	cliflag "k8s.io/component-base/cli/flag"
@@ -88,7 +89,7 @@ type GeneratableKeyCert struct {
 	PairName string
 
 	// GeneratedCert holds an in-memory generated certificate if CertFile/KeyFile aren't explicitly set, and CertDirectory/PairName are not set.
-	GeneratedCert *tls.Certificate
+	GeneratedCert dynamiccertificates.CertKeyContentProvider
 
 	// FixtureDirectory is a directory that contains test fixture used to avoid regeneration of certs during tests.
 	// The format is:
@@ -225,11 +226,11 @@ func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error
 	serverCertFile, serverKeyFile := s.ServerCert.CertKey.CertFile, s.ServerCert.CertKey.KeyFile
 	// load main cert
 	if len(serverCertFile) != 0 || len(serverKeyFile) != 0 {
-		tlsCert, err := tls.LoadX509KeyPair(serverCertFile, serverKeyFile)
+		var err error
+		c.Cert, err = dynamiccertificates.NewStaticCertKeyContentFromFiles(serverCertFile, serverKeyFile)
 		if err != nil {
-			return fmt.Errorf("unable to load server certificate: %v", err)
+			return err
 		}
-		c.Cert = &tlsCert
 	} else if s.ServerCert.GeneratedCert != nil {
 		c.Cert = s.ServerCert.GeneratedCert
 	}
@@ -311,11 +312,10 @@ func (s *SecureServingOptions) MaybeDefaultWithSelfSignedCerts(publicAddress str
 			}
 			klog.Infof("Generated self-signed cert (%s, %s)", keyCert.CertFile, keyCert.KeyFile)
 		} else {
-			tlsCert, err := tls.X509KeyPair(cert, key)
+			s.ServerCert.GeneratedCert, err = dynamiccertificates.NewStaticCertKeyContent("Generated self signed cert", cert, key)
 			if err != nil {
-				return fmt.Errorf("unable to generate self signed cert: %v", err)
+				return err
 			}
-			s.ServerCert.GeneratedCert = &tlsCert
 			klog.Infof("Generated self-signed cert in-memory")
 		}
 	}

staging/src/k8s.io/apiserver/pkg/server/secure_serving.go

@@ -61,9 +61,9 @@ func (s *SecureServingInfo) tlsConfig(stopCh <-chan struct{}) (*tls.Config, erro
 	if len(s.CipherSuites) > 0 {
 		tlsConfig.CipherSuites = s.CipherSuites
 	}
-	if s.Cert != nil {
-		tlsConfig.Certificates = []tls.Certificate{*s.Cert}
-	}
+
+	// if s.Cert is not nil, this logic is contained within the dynamic serving controller
+	if s.Cert == nil {
 		// append all named certs. Otherwise, the go tls stack will think no SNI processing
 		// is necessary because there is only one cert anyway.
 		// Moreover, if ServerCert.CertFile/ServerCert.KeyFile are not set, the first SNI
@@ -71,15 +71,19 @@ func (s *SecureServingInfo) tlsConfig(stopCh <-chan struct{}) (*tls.Config, erro
 		for _, c := range s.SNICerts {
 			tlsConfig.Certificates = append(tlsConfig.Certificates, *c)
 		}
+	}
 
 	if s.ClientCA != nil {
 		// Populate PeerCertificates in requests, but don't reject connections without certificates
 		// This allows certificates to be validated by authenticators, while still allowing other auth types
 		tlsConfig.ClientAuth = tls.RequestClientCert
+	}
 
+	if s.ClientCA != nil || s.Cert != nil {
 		dynamicCertificateController := dynamiccertificates.NewDynamicServingCertificateController(
 			*tlsConfig,
 			s.ClientCA,
+			s.Cert,
 			nil, // TODO see how to plumb an event recorder down in here. For now this results in simply klog messages.
 		)
 		// runonce to be sure that we have a value.