Merge pull request #83555 from deads2k/dynamic-cert-kube-apiserver-wiring

Dynamic cert kube apiserver wiring

staging/src/k8s.io/apiserver/go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e
 	github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea
+	github.com/davecgh/go-spew v1.1.1
 	github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0
 	github.com/emicklei/go-restful v2.9.5+incompatible
 	github.com/evanphx/json-patch v4.2.0+incompatible

staging/src/k8s.io/apiserver/pkg/server/BUILD

@@ -98,6 +98,7 @@ go_library(
         "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/registry/generic:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/registry/rest:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server/egressselector:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server/filters:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library",
@@ -107,7 +108,6 @@ go_library(
         "//staging/src/k8s.io/apiserver/pkg/util/openapi:go_default_library",
         "//staging/src/k8s.io/client-go/informers:go_default_library",
         "//staging/src/k8s.io/client-go/rest:go_default_library",
-        "//staging/src/k8s.io/client-go/util/cert:go_default_library",
         "//staging/src/k8s.io/component-base/logs:go_default_library",
         "//vendor/github.com/coreos/go-systemd/daemon:go_default_library",
         "//vendor/github.com/emicklei/go-restful:go_default_library",
@@ -135,6 +135,7 @@ filegroup(
     name = "all-srcs",
     srcs = [
         ":package-srcs",
+        "//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:all-srcs",
         "//staging/src/k8s.io/apiserver/pkg/server/egressselector:all-srcs",
         "//staging/src/k8s.io/apiserver/pkg/server/filters:all-srcs",
         "//staging/src/k8s.io/apiserver/pkg/server/healthz:all-srcs",

staging/src/k8s.io/apiserver/pkg/server/config.go

@@ -18,7 +18,6 @@ package server
 
 import (
 	"crypto/tls"
-	"crypto/x509"
 	"fmt"
 	"net"
 	"net/http"
@@ -56,6 +55,7 @@ import (
 	apiopenapi "k8s.io/apiserver/pkg/endpoints/openapi"
 	apirequest "k8s.io/apiserver/pkg/endpoints/request"
 	genericregistry "k8s.io/apiserver/pkg/registry/generic"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 	"k8s.io/apiserver/pkg/server/egressselector"
 	genericfilters "k8s.io/apiserver/pkg/server/filters"
 	"k8s.io/apiserver/pkg/server/healthz"
@@ -63,7 +63,6 @@ import (
 	serverstore "k8s.io/apiserver/pkg/server/storage"
 	"k8s.io/client-go/informers"
 	restclient "k8s.io/client-go/rest"
-	certutil "k8s.io/client-go/util/cert"
 	"k8s.io/component-base/logs"
 	"k8s.io/klog"
 	openapicommon "k8s.io/kube-openapi/pkg/common"
@@ -240,7 +239,7 @@ type SecureServingInfo struct {
 	SNICerts map[string]*tls.Certificate
 
 	// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates
-	ClientCA *x509.CertPool
+	ClientCA dynamiccertificates.CAContentProvider
 
 	// MinTLSVersion optionally overrides the minimum TLS version supported.
 	// Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants).
@@ -350,15 +349,14 @@ func DefaultOpenAPIConfig(getDefinitions openapicommon.GetOpenAPIDefinitions, de
 func (c *AuthenticationInfo) ApplyClientCert(clientCAFile string, servingInfo *SecureServingInfo) error {
 	if servingInfo != nil {
 		if len(clientCAFile) > 0 {
-			clientCAs, err := certutil.CertsFromFile(clientCAFile)
+			clientCAProvider, err := dynamiccertificates.NewStaticCAContentFromFile(clientCAFile)
 			if err != nil {
 				return fmt.Errorf("unable to load client CA file: %v", err)
 			}
 			if servingInfo.ClientCA == nil {
-				servingInfo.ClientCA = x509.NewCertPool()
+				servingInfo.ClientCA = clientCAProvider
-			}
+			} else {
-			for _, cert := range clientCAs {
+				servingInfo.ClientCA = dynamiccertificates.NewUnionCAContentProvider(servingInfo.ClientCA, clientCAProvider)
-				servingInfo.ClientCA.AddCert(cert)
 			}
 		}
 	}

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD

@@ -0,0 +1,48 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "client_ca.go",
+        "static_content.go",
+        "tlsconfig.go",
+        "union_content.go",
+        "util.go",
+    ],
+    importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/dynamiccertificates",
+    importpath = "k8s.io/apiserver/pkg/server/dynamiccertificates",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/events:go_default_library",
+        "//staging/src/k8s.io/client-go/util/cert:go_default_library",
+        "//staging/src/k8s.io/client-go/util/workqueue:go_default_library",
+        "//vendor/k8s.io/klog:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "client_ca_test.go",
+        "tlsconfig_test.go",
+    ],
+    embed = [":go_default_library"],
+    deps = ["//vendor/github.com/davecgh/go-spew/spew:go_default_library"],
+)

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/client_ca.go

@@ -0,0 +1,63 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"bytes"
+)
+
+// CAContentProvider provides ca bundle byte content
+type CAContentProvider interface {
+	// Name is just an identifier
+	Name() string
+	// CurrentCABundleContent provides ca bundle byte content.  Errors can be contained to the controllers initializing
+	// the value.  By the time you get here, you should always be returning a value that won't fail.
+	CurrentCABundleContent() []byte
+}
+
+// dynamicCertificateContent holds the content that overrides the baseTLSConfig
+// TODO add the serving certs to this struct
+type dynamicCertificateContent struct {
+	// clientCA holds the content for the clientCA bundle
+	clientCA caBundleContent
+}
+
+// caBundleContent holds the content for the clientCA bundle.  Wrapping the bytes makes the Equals work nicely with the
+// method receiver.
+type caBundleContent struct {
+	caBundle []byte
+}
+
+func (c *dynamicCertificateContent) Equal(rhs *dynamicCertificateContent) bool {
+	if c == nil || rhs == nil {
+		return c == rhs
+	}
+
+	if !c.clientCA.Equal(&rhs.clientCA) {
+		return false
+	}
+
+	return true
+}
+
+func (c *caBundleContent) Equal(rhs *caBundleContent) bool {
+	if c == nil || rhs == nil {
+		return c == rhs
+	}
+
+	return bytes.Equal(c.caBundle, rhs.caBundle)
+}

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/client_ca_test.go

@@ -0,0 +1,117 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import "testing"
+
+func TestDynamicCertificateContentEquals(t *testing.T) {
+	tests := []struct {
+		name     string
+		lhs      *dynamicCertificateContent
+		rhs      *dynamicCertificateContent
+		expected bool
+	}{
+		{
+			name:     "both nil",
+			expected: true,
+		},
+		{
+			name:     "lhs nil",
+			rhs:      &dynamicCertificateContent{},
+			expected: false,
+		},
+		{
+			name:     "rhs nil",
+			lhs:      &dynamicCertificateContent{},
+			expected: false,
+		},
+		{
+			name: "same",
+			lhs: &dynamicCertificateContent{
+				clientCA: caBundleContent{caBundle: []byte("foo")},
+			},
+			rhs: &dynamicCertificateContent{
+				clientCA: caBundleContent{caBundle: []byte("foo")},
+			},
+			expected: true,
+		},
+		{
+			name: "different",
+			lhs: &dynamicCertificateContent{
+				clientCA: caBundleContent{caBundle: []byte("foo")},
+			},
+			rhs: &dynamicCertificateContent{
+				clientCA: caBundleContent{caBundle: []byte("bar")},
+			},
+			expected: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			actual := test.lhs.Equal(test.rhs)
+			if actual != test.expected {
+				t.Error(actual)
+			}
+		})
+	}
+}
+
+func TestCABundleContentEquals(t *testing.T) {
+	tests := []struct {
+		name     string
+		lhs      *caBundleContent
+		rhs      *caBundleContent
+		expected bool
+	}{
+		{
+			name:     "both nil",
+			expected: true,
+		},
+		{
+			name:     "lhs nil",
+			rhs:      &caBundleContent{},
+			expected: false,
+		},
+		{
+			name:     "rhs nil",
+			lhs:      &caBundleContent{},
+			expected: false,
+		},
+		{
+			name:     "same",
+			lhs:      &caBundleContent{caBundle: []byte("foo")},
+			rhs:      &caBundleContent{caBundle: []byte("foo")},
+			expected: true,
+		},
+		{
+			name:     "different",
+			lhs:      &caBundleContent{caBundle: []byte("foo")},
+			rhs:      &caBundleContent{caBundle: []byte("bar")},
+			expected: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			actual := test.lhs.Equal(test.rhs)
+			if actual != test.expected {
+				t.Error(actual)
+			}
+		})
+	}
+}

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/static_content.go

@@ -0,0 +1,58 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"fmt"
+	"io/ioutil"
+)
+
+type staticCAContent struct {
+	name     string
+	caBundle []byte
+}
+
+// NewStaticCAContentFromFile returns a CAContentProvider based on a filename
+func NewStaticCAContentFromFile(filename string) (CAContentProvider, error) {
+	if len(filename) == 0 {
+		return nil, fmt.Errorf("missing filename for ca bundle")
+	}
+
+	caBundle, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	return NewStaticCAContent(filename, caBundle), nil
+}
+
+// NewStaticCAContent returns a CAContentProvider that always returns the same value
+func NewStaticCAContent(name string, caBundle []byte) CAContentProvider {
+	return &staticCAContent{
+		name:     name,
+		caBundle: caBundle,
+	}
+}
+
+// Name is just an identifier
+func (c *staticCAContent) Name() string {
+	return c.name
+}
+
+// CurrentCABundleContent provides ca bundle byte content
+func (c *staticCAContent) CurrentCABundleContent() (cabundle []byte) {
+	return c.caBundle
+}

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig.go

@@ -0,0 +1,200 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"sync/atomic"
+	"time"
+
+	v1 "k8s.io/api/core/v1"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/tools/events"
+	"k8s.io/client-go/util/cert"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog"
+)
+
+const workItemKey = "key"
+
+// DynamicServingCertificateController dynamically loads certificates and provides a golang tls compatible dynamic GetCertificate func.
+type DynamicServingCertificateController struct {
+	// baseTLSConfig is the static portion of the tlsConfig for serving to clients.  It is copied and the copy is mutated
+	// based on the dynamic cert state.
+	baseTLSConfig tls.Config
+
+	// clientCA provides the very latest content of the ca bundle
+	clientCA CAContentProvider
+
+	// currentlyServedContent holds the original bytes that we are serving. This is used to decide if we need to set a
+	// new atomic value. The types used for efficient TLSConfig preclude using the processed value.
+	currentlyServedContent *dynamicCertificateContent
+	// currentServingTLSConfig holds a *tls.Config that will be used to serve requests
+	currentServingTLSConfig atomic.Value
+
+	// queue only ever has one item, but it has nice error handling backoff/retry semantics
+	queue         workqueue.RateLimitingInterface
+	eventRecorder events.EventRecorder
+}
+
+// NewDynamicServingCertificateController returns a controller that can be used to keep a TLSConfig up to date.
+func NewDynamicServingCertificateController(
+	baseTLSConfig tls.Config,
+	clientCA CAContentProvider,
+	eventRecorder events.EventRecorder,
+) *DynamicServingCertificateController {
+	c := &DynamicServingCertificateController{
+		baseTLSConfig: baseTLSConfig,
+		clientCA:      clientCA,
+
+		queue:         workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "DynamicServingCertificateController"),
+		eventRecorder: eventRecorder,
+	}
+
+	return c
+}
+
+// GetConfigForClient is an implementation of tls.Config.GetConfigForClient
+func (c *DynamicServingCertificateController) GetConfigForClient(clientHello *tls.ClientHelloInfo) (*tls.Config, error) {
+	uncastObj := c.currentServingTLSConfig.Load()
+	if uncastObj == nil {
+		return nil, errors.New("dynamiccertificates: configuration not ready")
+	}
+	tlsConfig, ok := uncastObj.(*tls.Config)
+	if !ok {
+		return nil, errors.New("dynamiccertificates: unexpected config type")
+	}
+
+	return tlsConfig.Clone(), nil
+}
+
+// newTLSContent determines the next set of content for overriding the baseTLSConfig.
+func (c *DynamicServingCertificateController) newTLSContent() (*dynamicCertificateContent, error) {
+	newContent := &dynamicCertificateContent{}
+
+	currClientCABundle := c.clientCA.CurrentCABundleContent()
+	// don't remove all content.  The value was configured at one time, so continue using that.
+	// Errors reading content can be reported by lower level controllers.
+	if len(currClientCABundle) == 0 {
+		return nil, fmt.Errorf("not loading an empty client ca bundle from %q", c.clientCA.Name())
+	}
+	newContent.clientCA = caBundleContent{caBundle: currClientCABundle}
+
+	return newContent, nil
+}
+
+// syncCerts gets newTLSContent, if it has changed from the existing, the content is parsed and stored for usage in
+// GetConfigForClient.
+func (c *DynamicServingCertificateController) syncCerts() error {
+	newContent, err := c.newTLSContent()
+	if err != nil {
+		return err
+	}
+	// if the content is the same as what we currently have, we can simply skip it.  This works because we are single
+	// threaded.  If you ever make this multi-threaded, add a lock.
+	if newContent.Equal(c.currentlyServedContent) {
+		return nil
+	}
+
+	// parse new content to add to TLSConfig
+	newClientCAPool := x509.NewCertPool()
+	if len(newContent.clientCA.caBundle) > 0 {
+		newClientCAs, err := cert.ParseCertsPEM(newContent.clientCA.caBundle)
+		if err != nil {
+			return fmt.Errorf("unable to load client CA file: %v", err)
+		}
+		for i, cert := range newClientCAs {
+			klog.V(2).Infof("loaded client CA [%d/%q]: %s", i, c.clientCA.Name(), GetHumanCertDetail(cert))
+			if c.eventRecorder != nil {
+				c.eventRecorder.Eventf(nil, nil, v1.EventTypeWarning, "TLSConfigChanged", "CACertificateReload", "loaded client CA [%d/%q]: %s", i, c.clientCA.Name(), GetHumanCertDetail(cert))
+			}
+
+			newClientCAPool.AddCert(cert)
+		}
+	}
+
+	// make a copy and override the dynamic pieces which have changed.
+	newTLSConfigCopy := c.baseTLSConfig.Clone()
+	newTLSConfigCopy.ClientCAs = newClientCAPool
+
+	// store new values of content for serving.
+	c.currentServingTLSConfig.Store(newTLSConfigCopy)
+	c.currentlyServedContent = newContent // this is single threaded, so we have no locking issue
+
+	return nil
+}
+
+// RunOnce runs a single sync step to ensure that we have a valid starting configuration.
+func (c *DynamicServingCertificateController) RunOnce() error {
+	return c.syncCerts()
+}
+
+// Run starts the kube-apiserver and blocks until stopCh is closed.
+func (c *DynamicServingCertificateController) Run(workers int, stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.Infof("Starting DynamicServingCertificateController")
+	defer klog.Infof("Shutting down DynamicServingCertificateController")
+
+	// synchronously load once.  We will trigger again, so ignoring any error is fine
+	_ = c.RunOnce()
+
+	// doesn't matter what workers say, only start one.
+	go wait.Until(c.runWorker, time.Second, stopCh)
+
+	// start timer that rechecks every minute, just in case.  this also serves to prime the controller quickly.
+	go wait.Until(func() {
+		c.Enqueue()
+	}, 1*time.Minute, stopCh)
+
+	<-stopCh
+}
+
+func (c *DynamicServingCertificateController) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+func (c *DynamicServingCertificateController) processNextWorkItem() bool {
+	dsKey, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	defer c.queue.Done(dsKey)
+
+	err := c.syncCerts()
+	if err == nil {
+		c.queue.Forget(dsKey)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err))
+	c.queue.AddRateLimited(dsKey)
+
+	return true
+}
+
+// Enqueue a method to allow separate control loops to cause the certificate controller to trigger and read content.
+func (c *DynamicServingCertificateController) Enqueue() {
+	c.queue.Add(workItemKey)
+}

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig_test.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/davecgh/go-spew/spew"
+)
+
+func TestNewTLSContent(t *testing.T) {
+	tests := []struct {
+		name     string
+		clientCA CAContentProvider
+
+		expected    *dynamicCertificateContent
+		expectedErr string
+	}{
+		{
+			name:     "filled",
+			clientCA: NewStaticCAContent("test-ca", []byte("content-1")),
+			expected: &dynamicCertificateContent{
+				clientCA: caBundleContent{caBundle: []byte("content-1")},
+			},
+		},
+		{
+			name:        "missingCA",
+			clientCA:    NewStaticCAContent("test-ca", []byte("")),
+			expected:    nil,
+			expectedErr: `not loading an empty client ca bundle from "test-ca"`,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			c := &DynamicServingCertificateController{
+				clientCA: test.clientCA,
+			}
+			actual, err := c.newTLSContent()
+			if !reflect.DeepEqual(actual, test.expected) {
+				t.Error(spew.Sdump(actual))
+			}
+			switch {
+			case err == nil && len(test.expectedErr) == 0:
+			case err == nil && len(test.expectedErr) != 0:
+				t.Errorf("missing %q", test.expectedErr)
+			case err != nil && len(test.expectedErr) == 0:
+				t.Error(err)
+			case err != nil && err.Error() != test.expectedErr:
+				t.Error(err)
+			}
+		})
+	}
+}

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/union_content.go

@@ -0,0 +1,48 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"bytes"
+	"strings"
+)
+
+type unionCAContent []CAContentProvider
+
+// NewUnionCAContentProvider returns a CAContentProvider that is a union of other CAContentProviders
+func NewUnionCAContentProvider(caContentProviders ...CAContentProvider) CAContentProvider {
+	return unionCAContent(caContentProviders)
+}
+
+// Name is just an identifier
+func (c unionCAContent) Name() string {
+	names := []string{}
+	for _, curr := range c {
+		names = append(names, curr.Name())
+	}
+	return strings.Join(names, ",")
+}
+
+// CurrentCABundleContent provides ca bundle byte content
+func (c unionCAContent) CurrentCABundleContent() []byte {
+	caBundles := [][]byte{}
+	for _, curr := range c {
+		caBundles = append(caBundles, curr.CurrentCABundleContent())
+	}
+
+	return bytes.Join(caBundles, []byte("\n"))
+}

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/util.go

@@ -0,0 +1,68 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"crypto/x509"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// GetHumanCertDetail is a convenient method for printing compact details of certificate that helps when debugging
+// kube-apiserver usage of certs.
+func GetHumanCertDetail(certificate *x509.Certificate) string {
+	humanName := certificate.Subject.CommonName
+	signerHumanName := certificate.Issuer.CommonName
+	if certificate.Subject.CommonName == certificate.Issuer.CommonName {
+		signerHumanName = "<self>"
+	}
+
+	usages := []string{}
+	for _, curr := range certificate.ExtKeyUsage {
+		if curr == x509.ExtKeyUsageClientAuth {
+			usages = append(usages, "client")
+			continue
+		}
+		if curr == x509.ExtKeyUsageServerAuth {
+			usages = append(usages, "serving")
+			continue
+		}
+
+		usages = append(usages, fmt.Sprintf("%d", curr))
+	}
+
+	validServingNames := []string{}
+	for _, ip := range certificate.IPAddresses {
+		validServingNames = append(validServingNames, ip.String())
+	}
+	for _, dnsName := range certificate.DNSNames {
+		validServingNames = append(validServingNames, dnsName)
+	}
+	servingString := ""
+	if len(validServingNames) > 0 {
+		servingString = fmt.Sprintf(" validServingFor=[%s]", strings.Join(validServingNames, ","))
+	}
+
+	groupString := ""
+	if len(certificate.Subject.Organization) > 0 {
+		groupString = fmt.Sprintf(" groups=[%s]", strings.Join(certificate.Subject.Organization, ","))
+	}
+
+	return fmt.Sprintf("%q [%s]%s%s issuer=%q (%v to %v (now=%v))", humanName, strings.Join(usages, ","), groupString, servingString, signerHumanName, certificate.NotBefore.UTC(), certificate.NotAfter.UTC(),
+		time.Now().UTC())
+}

staging/src/k8s.io/apiserver/pkg/server/secure_serving.go

@@ -31,6 +31,7 @@ import (
 
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/validation"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 )
 
 const (
@@ -71,13 +72,23 @@ func (s *SecureServingInfo) tlsConfig(stopCh <-chan struct{}) (*tls.Config, erro
 		tlsConfig.Certificates = append(tlsConfig.Certificates, *c)
 	}
 
-	// TODO this will become dynamic.
 	if s.ClientCA != nil {
 		// Populate PeerCertificates in requests, but don't reject connections without certificates
 		// This allows certificates to be validated by authenticators, while still allowing other auth types
 		tlsConfig.ClientAuth = tls.RequestClientCert
-		// Specify allowed CAs for client certificates
+
-		tlsConfig.ClientCAs = s.ClientCA
+		dynamicCertificateController := dynamiccertificates.NewDynamicServingCertificateController(
+			*tlsConfig,
+			s.ClientCA,
+			nil, // TODO see how to plumb an event recorder down in here. For now this results in simply klog messages.
+		)
+		// runonce to be sure that we have a value.
+		if err := dynamicCertificateController.RunOnce(); err != nil {
+			return nil, err
+		}
+		go dynamicCertificateController.Run(1, stopCh)
+
+		tlsConfig.GetConfigForClient = dynamicCertificateController.GetConfigForClient
 	}
 
 	return tlsConfig, nil

vendor/modules.txt

@@ -1337,6 +1337,7 @@ k8s.io/apiserver/pkg/registry/generic/testing
 k8s.io/apiserver/pkg/registry/rest
 k8s.io/apiserver/pkg/registry/rest/resttest
 k8s.io/apiserver/pkg/server
+k8s.io/apiserver/pkg/server/dynamiccertificates
 k8s.io/apiserver/pkg/server/egressselector
 k8s.io/apiserver/pkg/server/filters
 k8s.io/apiserver/pkg/server/healthz