wire up a means to dynamically reload ca bundles for kube-apiserver

cmd/kubelet/app/BUILD

@@ -118,10 +118,10 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory:go_default_library",
-        "//staging/src/k8s.io/apiserver/pkg/authentication/request/x509:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",

cmd/kubelet/app/auth.go

@@ -24,9 +24,9 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
-	"k8s.io/apiserver/pkg/authentication/request/x509"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
 	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 	clientset "k8s.io/client-go/kubernetes"
 	authenticationclient "k8s.io/client-go/kubernetes/typed/authentication/v1beta1"
 	authorizationclient "k8s.io/client-go/kubernetes/typed/authorization/v1beta1"
@@ -64,15 +64,19 @@ func BuildAuth(nodeName types.NodeName, client clientset.Interface, config kubel
 
 // BuildAuthn creates an authenticator compatible with the kubelet's needs
 func BuildAuthn(client authenticationclient.TokenReviewInterface, authn kubeletconfig.KubeletAuthentication) (authenticator.Request, error) {
-	clientCertVerifier, err := x509.NewStaticVerifierFromFile(authn.X509.ClientCAFile)
+	var clientCertificateCAContentProvider authenticatorfactory.CAContentProvider
+	var err error
+	if len(authn.X509.ClientCAFile) > 0 {
+		clientCertificateCAContentProvider, err = dynamiccertificates.NewDynamicCAContentFromFile("client-ca-bundle", authn.X509.ClientCAFile)
 		if err != nil {
 			return nil, err
 		}
+	}
 
 	authenticatorConfig := authenticatorfactory.DelegatingAuthenticatorConfig{
 		Anonymous:                          authn.Anonymous.Enabled,
 		CacheTTL:                           authn.Webhook.CacheTTL.Duration,
-		ClientVerifyOptionFn: clientCertVerifier,
+		ClientCertificateCAContentProvider: clientCertificateCAContentProvider,
 	}
 
 	if authn.Webhook.Enabled {

pkg/kubeapiserver/authenticator/BUILD

@@ -24,6 +24,7 @@ go_library(
         "//staging/src/k8s.io/apiserver/pkg/authentication/token/cache:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/authentication/token/tokenfile:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/authentication/token/union:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//staging/src/k8s.io/apiserver/plugin/pkg/authenticator/password/passwordfile:go_default_library",
         "//staging/src/k8s.io/apiserver/plugin/pkg/authenticator/request/basicauth:go_default_library",

pkg/kubeapiserver/authenticator/config.go

@@ -33,6 +33,7 @@ import (
 	tokencache "k8s.io/apiserver/pkg/authentication/token/cache"
 	"k8s.io/apiserver/pkg/authentication/token/tokenfile"
 	tokenunion "k8s.io/apiserver/pkg/authentication/token/union"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/apiserver/plugin/pkg/authenticator/password/passwordfile"
 	"k8s.io/apiserver/plugin/pkg/authenticator/request/basicauth"
@@ -77,10 +78,10 @@ type Config struct {
 	// TODO, this is the only non-serializable part of the entire config.  Factor it out into a clientconfig
 	ServiceAccountTokenGetter   serviceaccount.ServiceAccountTokenGetter
 	BootstrapTokenAuthenticator authenticator.Token
-	// ClientVerifyOptionFn are the options for verifying incoming connections using mTLS and directly assigning to users.
+	// ClientCAContentProvider are the options for verifying incoming connections using mTLS and directly assigning to users.
 	// Generally this is the CA bundle file used to authenticate client certificates
 	// If this value is nil, then mutual TLS is disabled.
-	ClientVerifyOptionFn x509.VerifyOptionFunc
+	ClientCAContentProvider dynamiccertificates.CAContentProvider
 }
 
 // New returns an authenticator.Request or an error that supports the standard
@@ -94,7 +95,7 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, er
 	// Add the front proxy authenticator if requested
 	if config.RequestHeaderConfig != nil {
 		requestHeaderAuthenticator := headerrequest.NewDynamicVerifyOptionsSecure(
-			config.RequestHeaderConfig.VerifyOptionFn,
+			config.RequestHeaderConfig.CAContentProvider.VerifyOptions,
 			config.RequestHeaderConfig.AllowedClientNames,
 			config.RequestHeaderConfig.UsernameHeaders,
 			config.RequestHeaderConfig.GroupHeaders,
@@ -120,8 +121,8 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, er
 	}
 
 	// X509 methods
-	if config.ClientVerifyOptionFn != nil {
+	if config.ClientCAContentProvider != nil {
-		certAuth := x509.NewDynamic(config.ClientVerifyOptionFn, x509.CommonNameUserConversion)
+		certAuth := x509.NewDynamic(config.ClientCAContentProvider.VerifyOptions, x509.CommonNameUserConversion)
 		authenticators = append(authenticators, certAuth)
 	}
 

pkg/kubeapiserver/options/authentication.go

@@ -324,7 +324,7 @@ func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() (kubeauthenticat
 
 	if s.ClientCert != nil {
 		var err error
-		ret.ClientVerifyOptionFn, err = s.ClientCert.GetClientVerifyOptionFn()
+		ret.ClientCAContentProvider, err = s.ClientCert.GetClientCAContentProvider()
 		if err != nil {
 			return kubeauthenticator.Config{}, err
 		}
@@ -390,17 +390,26 @@ func (o *BuiltInAuthenticationOptions) ApplyTo(c *genericapiserver.Config) error
 		return nil
 	}
 
-	var err error
 	if o.ClientCert != nil {
-		if err = c.Authentication.ApplyClientCert(o.ClientCert.ClientCA, c.SecureServing); err != nil {
+		clientCertificateCAContentProvider, err := o.ClientCert.GetClientCAContentProvider()
+		if err != nil {
+			return fmt.Errorf("unable to load client CA file: %v", err)
+		}
+		if err = c.Authentication.ApplyClientCert(clientCertificateCAContentProvider, c.SecureServing); err != nil {
 			return fmt.Errorf("unable to load client CA file: %v", err)
 		}
 	}
 	if o.RequestHeader != nil {
-		if err = c.Authentication.ApplyClientCert(o.RequestHeader.ClientCAFile, c.SecureServing); err != nil {
+		requestHeaderConfig, err := o.RequestHeader.ToAuthenticationRequestHeaderConfig()
+		if err != nil {
+			return fmt.Errorf("unable to create request header authentication config: %v", err)
+		}
+		if requestHeaderConfig != nil {
+			if err = c.Authentication.ApplyClientCert(requestHeaderConfig.CAContentProvider, c.SecureServing); err != nil {
 				return fmt.Errorf("unable to load client CA file: %v", err)
 			}
 		}
+	}
 
 	c.Authentication.SupportsBasicAuth = o.PasswordFile != nil && len(o.PasswordFile.BasicAuthFile) > 0
 

pkg/kubeapiserver/options/authentication_test.go

@@ -146,7 +146,7 @@ func TestToAuthenticationConfig(t *testing.T) {
 		Anonymous:                   false,
 		BasicAuthFile:               "/testBasicAuthFile",
 		BootstrapToken:              false,
-		ClientVerifyOptionFn:        nil, // this is nil because you can't compare functions
+		ClientCAContentProvider:     nil, // this is nil because you can't compare functions
 		TokenAuthFile:               "/testTokenFile",
 		OIDCIssuerURL:               "testIssuerURL",
 		OIDCClientID:                "testClientID",
@@ -165,7 +165,7 @@ func TestToAuthenticationConfig(t *testing.T) {
 			UsernameHeaders:     headerrequest.StaticStringSlice{"x-remote-user"},
 			GroupHeaders:        headerrequest.StaticStringSlice{"x-remote-group"},
 			ExtraHeaderPrefixes: headerrequest.StaticStringSlice{"x-remote-extra-"},
-			VerifyOptionFn:      nil, // this is nil because you can't compare functions
+			CAContentProvider:   nil, // this is nil because you can't compare functions
 			AllowedClientNames:  headerrequest.StaticStringSlice{"kube-aggregator"},
 		},
 	}
@@ -176,14 +176,14 @@ func TestToAuthenticationConfig(t *testing.T) {
 	}
 
 	// nil these out because you cannot compare pointers.  Ensure they are non-nil first
-	if resultConfig.ClientVerifyOptionFn == nil {
+	if resultConfig.ClientCAContentProvider == nil {
 		t.Error("missing client verify")
 	}
-	if resultConfig.RequestHeaderConfig.VerifyOptionFn == nil {
+	if resultConfig.RequestHeaderConfig.CAContentProvider == nil {
 		t.Error("missing requestheader verify")
 	}
-	resultConfig.ClientVerifyOptionFn = nil
+	resultConfig.ClientCAContentProvider = nil
-	resultConfig.RequestHeaderConfig.VerifyOptionFn = nil
+	resultConfig.RequestHeaderConfig.CAContentProvider = nil
 
 	if !reflect.DeepEqual(resultConfig, expectConfig) {
 		t.Error(cmp.Diff(resultConfig, expectConfig))

staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/delegating.go

@@ -30,7 +30,6 @@ import (
 	unionauth "k8s.io/apiserver/pkg/authentication/request/union"
 	"k8s.io/apiserver/pkg/authentication/request/websocket"
 	"k8s.io/apiserver/pkg/authentication/request/x509"
-	x509request "k8s.io/apiserver/pkg/authentication/request/x509"
 	"k8s.io/apiserver/pkg/authentication/token/cache"
 	webhooktoken "k8s.io/apiserver/plugin/pkg/authenticator/token/webhook"
 	authenticationclient "k8s.io/client-go/kubernetes/typed/authentication/v1beta1"
@@ -47,9 +46,10 @@ type DelegatingAuthenticatorConfig struct {
 	// CacheTTL is the length of time that a token authentication answer will be cached.
 	CacheTTL time.Duration
 
-	// ClientVerifyOptionFn are the options for verifying incoming connections using mTLS and directly assigning to users.
+	// CAContentProvider are the options for verifying incoming connections using mTLS and directly assigning to users.
 	// Generally this is the CA bundle file used to authenticate client certificates
-	ClientVerifyOptionFn x509request.VerifyOptionFunc
+	// If this is nil, then mTLS will not be used.
+	ClientCertificateCAContentProvider CAContentProvider
 
 	APIAudiences authenticator.Audiences
 
@@ -64,7 +64,7 @@ func (c DelegatingAuthenticatorConfig) New() (authenticator.Request, *spec.Secur
 	// Add the front proxy authenticator if requested
 	if c.RequestHeaderConfig != nil {
 		requestHeaderAuthenticator := headerrequest.NewDynamicVerifyOptionsSecure(
-			c.RequestHeaderConfig.VerifyOptionFn,
+			c.RequestHeaderConfig.CAContentProvider.VerifyOptions,
 			c.RequestHeaderConfig.AllowedClientNames,
 			c.RequestHeaderConfig.UsernameHeaders,
 			c.RequestHeaderConfig.GroupHeaders,
@@ -74,8 +74,8 @@ func (c DelegatingAuthenticatorConfig) New() (authenticator.Request, *spec.Secur
 	}
 
 	// x509 client cert auth
-	if c.ClientVerifyOptionFn != nil {
+	if c.ClientCertificateCAContentProvider != nil {
-		authenticators = append(authenticators, x509.NewDynamic(c.ClientVerifyOptionFn, x509.CommonNameUserConversion))
+		authenticators = append(authenticators, x509.NewDynamic(c.ClientCertificateCAContentProvider.VerifyOptions, x509.CommonNameUserConversion))
 	}
 
 	if c.TokenAccessReviewClient != nil {

staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/requestheader.go

@@ -17,8 +17,9 @@ limitations under the License.
 package authenticatorfactory
 
 import (
+	"crypto/x509"
+
 	"k8s.io/apiserver/pkg/authentication/request/headerrequest"
-	x509request "k8s.io/apiserver/pkg/authentication/request/x509"
 )
 
 type RequestHeaderConfig struct {
@@ -29,9 +30,19 @@ type RequestHeaderConfig struct {
 	// ExtraHeaderPrefixes are the head prefixes to check (case-insentively) for filling in
 	// the user.Info.Extra.  All values of all matching headers will be added.
 	ExtraHeaderPrefixes headerrequest.StringSliceProvider
-	// VerifyOptionFn are the options for verifying incoming connections using mTLS.  Generally this points to CA bundle file which is used verify the identity of the front proxy.
+	// CAContentProvider the options for verifying incoming connections using mTLS.  Generally this points to CA bundle file which is used verify the identity of the front proxy.
 	//	It may produce different options at will.
-	VerifyOptionFn x509request.VerifyOptionFunc
+	CAContentProvider CAContentProvider
 	// AllowedClientNames is a list of common names that may be presented by the authenticating front proxy.  Empty means: accept any.
 	AllowedClientNames headerrequest.StringSliceProvider
 }
+
+// CAContentProvider provides ca bundle byte content
+type CAContentProvider interface {
+	// Name is just an identifier
+	Name() string
+	// CurrentCABundleContent provides ca bundle byte content
+	CurrentCABundleContent() []byte
+	// VerifyOptions provides VerifyOptions for authenticators
+	VerifyOptions() x509.VerifyOptions
+}

staging/src/k8s.io/apiserver/pkg/server/config.go

@@ -345,21 +345,19 @@ func DefaultOpenAPIConfig(getDefinitions openapicommon.GetOpenAPIDefinitions, de
 	}
 }
 
-func (c *AuthenticationInfo) ApplyClientCert(clientCAFile string, servingInfo *SecureServingInfo) error {
+func (c *AuthenticationInfo) ApplyClientCert(clientCA dynamiccertificates.CAContentProvider, servingInfo *SecureServingInfo) error {
-	if servingInfo != nil {
+	if servingInfo == nil {
-		if len(clientCAFile) > 0 {
+		return nil
-			clientCAProvider, err := dynamiccertificates.NewStaticCAContentFromFile(clientCAFile)
+	}
-			if err != nil {
+	if clientCA == nil {
-				return fmt.Errorf("unable to load client CA file: %v", err)
+		return nil
 	}
 	if servingInfo.ClientCA == nil {
-				servingInfo.ClientCA = clientCAProvider
+		servingInfo.ClientCA = clientCA
-			} else {
+		return nil
-				servingInfo.ClientCA = dynamiccertificates.NewUnionCAContentProvider(servingInfo.ClientCA, clientCAProvider)
-			}
-		}
 	}
 
+	servingInfo.ClientCA = dynamiccertificates.NewUnionCAContentProvider(servingInfo.ClientCA, clientCA)
 	return nil
 }
 

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD

@@ -17,6 +17,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/client_ca.go

@@ -18,6 +18,7 @@ package dynamiccertificates
 
 import (
 	"bytes"
+	"crypto/x509"
 )
 
 // CAContentProvider provides ca bundle byte content
@@ -27,6 +28,8 @@ type CAContentProvider interface {
 	// CurrentCABundleContent provides ca bundle byte content.  Errors can be contained to the controllers initializing
 	// the value.  By the time you get here, you should always be returning a value that won't fail.
 	CurrentCABundleContent() []byte
+	// VerifyOptions provides VerifyOptions for authenticators
+	VerifyOptions() x509.VerifyOptions
 }
 
 // dynamicCertificateContent holds the content that overrides the baseTLSConfig

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamicfile_content.go

@@ -17,10 +17,10 @@ limitations under the License.
 package dynamiccertificates
 
 import (
+	"bytes"
 	"crypto/x509"
 	"fmt"
 	"io/ioutil"
-	"reflect"
 	"sync/atomic"
 	"time"
 
@@ -32,10 +32,30 @@ import (
 	"k8s.io/klog"
 )
 
-type CAListener interface {
+// FileRefreshDuration is exposed so that integration tests can crank up the reload speed.
+var FileRefreshDuration = 1 * time.Minute
+
+// Listener is an interface to use to notify interested parties of a change.
+type Listener interface {
+	// Enqueue should be called when an input may have changed
 	Enqueue()
 }
 
+// Notifier is a way to add listeners
+type Notifier interface {
+	// AddListener is adds a listener to be notified of potential input changes
+	AddListener(listener Listener)
+}
+
+// ControllerRunner is a generic interface for starting a controller
+type ControllerRunner interface {
+	// RunOnce runs the sync loop a single time.  This useful for synchronous priming
+	RunOnce() error
+
+	// Run should be called a go .Run
+	Run(workers int, stopCh <-chan struct{})
+}
+
 // DynamicFileCAContent provies a CAContentProvider that can dynamically react to new file content
 // It also fulfills the authenticator interface to provide verifyoptions
 type DynamicFileCAContent struct {
@@ -47,18 +67,22 @@ type DynamicFileCAContent struct {
 	// caBundle is a caBundleAndVerifier that contains the last read, non-zero length content of the file
 	caBundle atomic.Value
 
-	listeners []CAListener
+	listeners []Listener
 
 	// queue only ever has one item, but it has nice error handling backoff/retry semantics
 	queue workqueue.RateLimitingInterface
 }
 
+var _ Notifier = &DynamicFileCAContent{}
+var _ CAContentProvider = &DynamicFileCAContent{}
+var _ ControllerRunner = &DynamicFileCAContent{}
+
 type caBundleAndVerifier struct {
 	caBundle      []byte
 	verifyOptions x509.VerifyOptions
 }
 
-// NewStaticCAContentFromFile returns a CAContentProvider based on a filename
+// NewDynamicCAContentFromFile returns a CAContentProvider based on a filename that automatically reloads content
 func NewDynamicCAContentFromFile(purpose, filename string) (*DynamicFileCAContent, error) {
 	if len(filename) == 0 {
 		return nil, fmt.Errorf("missing filename for ca bundle")
@@ -78,7 +102,7 @@ func NewDynamicCAContentFromFile(purpose, filename string) (*DynamicFileCAConten
 }
 
 // AddListener adds a listener to be notified when the CA content changes.
-func (c *DynamicFileCAContent) AddListener(listener CAListener) {
+func (c *DynamicFileCAContent) AddListener(listener Listener) {
 	c.listeners = append(c.listeners, listener)
 }
 
@@ -93,8 +117,7 @@ func (c *DynamicFileCAContent) loadCABundle() error {
 	}
 
 	// check to see if we have a change. If the values are the same, do nothing.
-	existing, ok := c.caBundle.Load().(*caBundleAndVerifier)
+	if !c.hasCAChanged(caBundle) {
-	if ok && existing != nil && reflect.DeepEqual(existing.caBundle, caBundle) {
 		return nil
 	}
 
@@ -111,6 +134,30 @@ func (c *DynamicFileCAContent) loadCABundle() error {
 	return nil
 }
 
+// hasCAChanged returns true if the caBundle is different than the current.
+func (c *DynamicFileCAContent) hasCAChanged(caBundle []byte) bool {
+	uncastExisting := c.caBundle.Load()
+	if uncastExisting == nil {
+		return true
+	}
+
+	// check to see if we have a change. If the values are the same, do nothing.
+	existing, ok := uncastExisting.(*caBundleAndVerifier)
+	if !ok {
+		return true
+	}
+	if !bytes.Equal(existing.caBundle, caBundle) {
+		return true
+	}
+
+	return false
+}
+
+// RunOnce runs a single sync loop
+func (c *DynamicFileCAContent) RunOnce() error {
+	return c.loadCABundle()
+}
+
 // Run starts the kube-apiserver and blocks until stopCh is closed.
 func (c *DynamicFileCAContent) Run(workers int, stopCh <-chan struct{}) {
 	defer utilruntime.HandleCrash()
@@ -123,7 +170,7 @@ func (c *DynamicFileCAContent) Run(workers int, stopCh <-chan struct{}) {
 	go wait.Until(c.runWorker, time.Second, stopCh)
 
 	// start timer that rechecks every minute, just in case.  this also serves to prime the controller quickly.
-	_ = wait.PollImmediateUntil(1*time.Minute, func() (bool, error) {
+	_ = wait.PollImmediateUntil(FileRefreshDuration, func() (bool, error) {
 		c.queue.Add(workItemKey)
 		return false, nil
 	}, stopCh)
@@ -164,11 +211,12 @@ func (c *DynamicFileCAContent) Name() string {
 
 // CurrentCABundleContent provides ca bundle byte content
 func (c *DynamicFileCAContent) CurrentCABundleContent() (cabundle []byte) {
-	return c.caBundle.Load().(caBundleAndVerifier).caBundle
+	return c.caBundle.Load().(*caBundleAndVerifier).caBundle
 }
 
+// VerifyOptions provides verifyoptions compatible with authenticators
 func (c *DynamicFileCAContent) VerifyOptions() x509.VerifyOptions {
-	return c.caBundle.Load().(caBundleAndVerifier).verifyOptions
+	return c.caBundle.Load().(*caBundleAndVerifier).verifyOptions
 }
 
 // newVerifyOptions creates a new verification func from a file.  It reads the content and then fails.

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/static_content.go

@@ -18,15 +18,18 @@ package dynamiccertificates
 
 import (
 	"crypto/tls"
+	"crypto/x509"
 	"fmt"
 	"io/ioutil"
 )
 
 type staticCAContent struct {
 	name     string
-	caBundle []byte
+	caBundle *caBundleAndVerifier
 }
 
+var _ CAContentProvider = &staticCAContent{}
+
 // NewStaticCAContentFromFile returns a CAContentProvider based on a filename
 func NewStaticCAContentFromFile(filename string) (CAContentProvider, error) {
 	if len(filename) == 0 {
@@ -37,15 +40,20 @@ func NewStaticCAContentFromFile(filename string) (CAContentProvider, error) {
 	if err != nil {
 		return nil, err
 	}
-	return NewStaticCAContent(filename, caBundle), nil
+	return NewStaticCAContent(filename, caBundle)
 }
 
 // NewStaticCAContent returns a CAContentProvider that always returns the same value
-func NewStaticCAContent(name string, caBundle []byte) CAContentProvider {
+func NewStaticCAContent(name string, caBundle []byte) (CAContentProvider, error) {
+	caBundleAndVerifier, err := newCABundleAndVerifier(name, caBundle)
+	if err != nil {
+		return nil, err
+	}
+
 	return &staticCAContent{
 		name:     name,
-		caBundle: caBundle,
+		caBundle: caBundleAndVerifier,
-	}
+	}, nil
 }
 
 // Name is just an identifier
@@ -55,7 +63,11 @@ func (c *staticCAContent) Name() string {
 
 // CurrentCABundleContent provides ca bundle byte content
 func (c *staticCAContent) CurrentCABundleContent() (cabundle []byte) {
-	return c.caBundle
+	return c.caBundle.caBundle
+}
+
+func (c *staticCAContent) VerifyOptions() x509.VerifyOptions {
+	return c.caBundle.verifyOptions
 }
 
 type staticCertKeyContent struct {

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig.go

@@ -60,6 +60,8 @@ type DynamicServingCertificateController struct {
 	eventRecorder events.EventRecorder
 }
 
+var _ Listener = &DynamicServingCertificateController{}
+
 // NewDynamicServingCertificateController returns a controller that can be used to keep a TLSConfig up to date.
 func NewDynamicServingCertificateController(
 	baseTLSConfig tls.Config,

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig_test.go

@@ -89,7 +89,7 @@ func TestNewStaticCertKeyContent(t *testing.T) {
 	}{
 		{
 			name:        "filled",
-			clientCA:    NewStaticCAContent("test-ca", []byte("content-1")),
+			clientCA:    &staticCAContent{name: "test-ca", caBundle: &caBundleAndVerifier{caBundle: []byte("content-1")}},
 			servingCert: testCertProvider,
 			sniCerts:    []SNICertKeyContentProvider{testCertProvider},
 			expected: &dynamicCertificateContent{
@@ -101,7 +101,7 @@ func TestNewStaticCertKeyContent(t *testing.T) {
 		},
 		{
 			name:        "missingCA",
-			clientCA:    NewStaticCAContent("test-ca", []byte("")),
+			clientCA:    &staticCAContent{name: "test-ca", caBundle: &caBundleAndVerifier{caBundle: []byte("")}},
 			expected:    nil,
 			expectedErr: `not loading an empty client ca bundle from "test-ca"`,
 		},

staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/union_content.go

@@ -18,11 +18,18 @@ package dynamiccertificates
 
 import (
 	"bytes"
+	"crypto/x509"
 	"strings"
+
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 )
 
 type unionCAContent []CAContentProvider
 
+var _ Notifier = &unionCAContent{}
+var _ CAContentProvider = &unionCAContent{}
+var _ ControllerRunner = &unionCAContent{}
+
 // NewUnionCAContentProvider returns a CAContentProvider that is a union of other CAContentProviders
 func NewUnionCAContentProvider(caContentProviders ...CAContentProvider) CAContentProvider {
 	return unionCAContent(caContentProviders)
@@ -46,3 +53,48 @@ func (c unionCAContent) CurrentCABundleContent() []byte {
 
 	return bytes.Join(caBundles, []byte("\n"))
 }
+
+// CurrentCABundleContent provides ca bundle byte content
+func (c unionCAContent) VerifyOptions() x509.VerifyOptions {
+	// TODO make more efficient.  This isn't actually used in any of our mainline paths.  It's called to build the TLSConfig
+	// TODO on file changes, but the actual authentication runs against the individual items, not the union.
+	ret, err := newCABundleAndVerifier(c.Name(), c.CurrentCABundleContent())
+	if err != nil {
+		// because we're made up of already vetted values, this indicates some kind of coding error
+		panic(err)
+	}
+
+	return ret.verifyOptions
+}
+
+// AddListener adds a listener to be notified when the CA content changes.
+func (c unionCAContent) AddListener(listener Listener) {
+	for _, curr := range c {
+		if notifier, ok := curr.(Notifier); ok {
+			notifier.AddListener(listener)
+		}
+	}
+}
+
+// AddListener adds a listener to be notified when the CA content changes.
+func (c unionCAContent) RunOnce() error {
+	errors := []error{}
+	for _, curr := range c {
+		if controller, ok := curr.(ControllerRunner); ok {
+			if err := controller.RunOnce(); err != nil {
+				errors = append(errors, err)
+			}
+		}
+	}
+
+	return utilerrors.NewAggregate(errors)
+}
+
+// Run runs the controller
+func (c unionCAContent) Run(workers int, stopCh <-chan struct{}) {
+	for _, curr := range c {
+		if controller, ok := curr.(ControllerRunner); ok {
+			go controller.Run(workers, stopCh)
+		}
+	}
+}

staging/src/k8s.io/apiserver/pkg/server/options/BUILD

@@ -50,7 +50,6 @@ go_library(
         "//staging/src/k8s.io/apiserver/pkg/audit/policy:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/authentication/request/headerrequest:go_default_library",
-        "//staging/src/k8s.io/apiserver/pkg/authentication/request/x509:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/authorization/path:go_default_library",

staging/src/k8s.io/apiserver/pkg/server/options/authentication.go

@@ -23,6 +23,8 @@ import (
 	"strings"
 	"time"
 
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+
 	"github.com/spf13/pflag"
 
 	v1 "k8s.io/api/core/v1"
@@ -30,12 +32,10 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
 	"k8s.io/apiserver/pkg/authentication/request/headerrequest"
-	"k8s.io/apiserver/pkg/authentication/request/x509"
 	"k8s.io/apiserver/pkg/server"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/util/cert"
 	"k8s.io/klog"
 	openapicommon "k8s.io/kube-openapi/pkg/common"
 )
@@ -112,7 +112,7 @@ func (s *RequestHeaderAuthenticationOptions) ToAuthenticationRequestHeaderConfig
 		return nil, nil
 	}
 
-	verifyFn, err := x509.NewStaticVerifierFromFile(s.ClientCAFile)
+	caBundleProvider, err := dynamiccertificates.NewDynamicCAContentFromFile("request-header", s.ClientCAFile)
 	if err != nil {
 		return nil, err
 	}
@@ -121,7 +121,7 @@ func (s *RequestHeaderAuthenticationOptions) ToAuthenticationRequestHeaderConfig
 		UsernameHeaders:     headerrequest.StaticStringSlice(s.UsernameHeaders),
 		GroupHeaders:        headerrequest.StaticStringSlice(s.GroupHeaders),
 		ExtraHeaderPrefixes: headerrequest.StaticStringSlice(s.ExtraHeaderPrefixes),
-		VerifyOptionFn:      verifyFn,
+		CAContentProvider:   caBundleProvider,
 		AllowedClientNames:  headerrequest.StaticStringSlice(s.AllowedNames),
 	}, nil
 }
@@ -132,23 +132,23 @@ type ClientCertAuthenticationOptions struct {
 	// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates
 	ClientCA string
 
-	// ClientVerifyOptionFn are the options for verifying incoming connections using mTLS and directly assigning to users.
+	// CAContentProvider are the options for verifying incoming connections using mTLS and directly assigning to users.
 	// Generally this is the CA bundle file used to authenticate client certificates
 	// If non-nil, this takes priority over the ClientCA file.
-	ClientVerifyOptionFn x509.VerifyOptionFunc
+	CAContentProvider dynamiccertificates.CAContentProvider
 }
 
 // GetClientVerifyOptionFn provides verify options for your authenticator while respecting the preferred order of verifiers.
-func (s *ClientCertAuthenticationOptions) GetClientVerifyOptionFn() (x509.VerifyOptionFunc, error) {
+func (s *ClientCertAuthenticationOptions) GetClientCAContentProvider() (dynamiccertificates.CAContentProvider, error) {
-	if s.ClientVerifyOptionFn != nil {
+	if s.CAContentProvider != nil {
-		return s.ClientVerifyOptionFn, nil
+		return s.CAContentProvider, nil
 	}
 
 	if len(s.ClientCA) == 0 {
 		return nil, nil
 	}
 
-	return x509.NewStaticVerifierFromFile(s.ClientCA)
+	return dynamiccertificates.NewDynamicCAContentFromFile("client-ca-bundle", s.ClientCA)
 }
 
 func (s *ClientCertAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
@@ -230,9 +230,9 @@ func (s *DelegatingAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
 		"Note that this can result in authentication that treats all requests as anonymous.")
 }
 
-func (s *DelegatingAuthenticationOptions) ApplyTo(c *server.AuthenticationInfo, servingInfo *server.SecureServingInfo, openAPIConfig *openapicommon.Config) error {
+func (s *DelegatingAuthenticationOptions) ApplyTo(authenticationInfo *server.AuthenticationInfo, servingInfo *server.SecureServingInfo, openAPIConfig *openapicommon.Config) error {
 	if s == nil {
-		c.Authenticator = nil
+		authenticationInfo.Authenticator = nil
 		return nil
 	}
 
@@ -266,32 +266,36 @@ func (s *DelegatingAuthenticationOptions) ApplyTo(c *server.AuthenticationInfo,
 	}
 
 	// configure AuthenticationInfo config
-	cfg.ClientVerifyOptionFn, err = s.ClientCert.GetClientVerifyOptionFn()
+	cfg.ClientCertificateCAContentProvider, err = s.ClientCert.GetClientCAContentProvider()
 	if err != nil {
 		return fmt.Errorf("unable to load client CA file: %v", err)
 	}
-	if err = c.ApplyClientCert(s.ClientCert.ClientCA, servingInfo); err != nil {
+	if cfg.ClientCertificateCAContentProvider != nil {
+		if err = authenticationInfo.ApplyClientCert(cfg.ClientCertificateCAContentProvider, servingInfo); err != nil {
 			return fmt.Errorf("unable to load client CA file: %v", err)
 		}
+	}
 
 	cfg.RequestHeaderConfig, err = s.RequestHeader.ToAuthenticationRequestHeaderConfig()
 	if err != nil {
 		return fmt.Errorf("unable to create request header authentication config: %v", err)
 	}
-	if err = c.ApplyClientCert(s.RequestHeader.ClientCAFile, servingInfo); err != nil {
+	if cfg.RequestHeaderConfig != nil {
+		if err = authenticationInfo.ApplyClientCert(cfg.RequestHeaderConfig.CAContentProvider, servingInfo); err != nil {
 			return fmt.Errorf("unable to load client CA file: %v", err)
 		}
+	}
 
 	// create authenticator
 	authenticator, securityDefinitions, err := cfg.New()
 	if err != nil {
 		return err
 	}
-	c.Authenticator = authenticator
+	authenticationInfo.Authenticator = authenticator
 	if openAPIConfig != nil {
 		openAPIConfig.SecurityDefinitions = securityDefinitions
 	}
-	c.SupportsBasicAuth = false
+	authenticationInfo.SupportsBasicAuth = false
 
 	return nil
 }
@@ -372,28 +376,14 @@ func inClusterClientCA(authConfigMap *v1.ConfigMap) (*ClientCertAuthenticationOp
 		// not having a client-ca is fine, return nil
 		return nil, nil
 	}
-
+	clientCAProvider, err := dynamiccertificates.NewStaticCAContent("client-ca-file", []byte(clientCA))
-	clientCAs, err := cert.NewPoolFromBytes([]byte(clientCA))
 	if err != nil {
-		return nil, fmt.Errorf("unable to load client CA from configmap: %v", err)
-	}
-	verifyOpts := x509.DefaultVerifyOptions()
-	verifyOpts.Roots = clientCAs
-
-	// we still need to write out the client-ca-file for now because it is used to plumb the options through the apiserver's
-	// configuration to hint clients.
-	// TODO deads2k this should eventually be made dynamic along with the authenticator.  I'm just wiring them one at at time.
-	f, err := ioutil.TempFile("", "client-ca-file")
-	if err != nil {
-		return nil, err
-	}
-	if err := ioutil.WriteFile(f.Name(), []byte(clientCA), 0600); err != nil {
 		return nil, err
 	}
 
 	return &ClientCertAuthenticationOptions{
-		ClientCA:             f.Name(),
+		ClientCA:          "",
-		ClientVerifyOptionFn: x509.StaticVerifierFn(verifyOpts),
+		CAContentProvider: clientCAProvider,
 	}, nil
 }
 

staging/src/k8s.io/apiserver/pkg/server/options/authentication_test.go

@@ -57,7 +57,7 @@ func TestToAuthenticationRequestHeaderConfig(t *testing.T) {
 				UsernameHeaders:     headerrequest.StaticStringSlice{"x-remote-user"},
 				GroupHeaders:        headerrequest.StaticStringSlice{"x-remote-group"},
 				ExtraHeaderPrefixes: headerrequest.StaticStringSlice{"x-remote-extra-"},
-				VerifyOptionFn:      nil, // this is nil because you can't compare functions
+				CAContentProvider:   nil, // this is nil because you can't compare functions
 				AllowedClientNames:  headerrequest.StaticStringSlice{"kube-aggregator"},
 			},
 		},
@@ -70,10 +70,10 @@ func TestToAuthenticationRequestHeaderConfig(t *testing.T) {
 				t.Fatal(err)
 			}
 			if resultConfig != nil {
-				if resultConfig.VerifyOptionFn == nil {
+				if resultConfig.CAContentProvider == nil {
 					t.Error("missing requestheader verify")
 				}
-				resultConfig.VerifyOptionFn = nil
+				resultConfig.CAContentProvider = nil
 			}
 
 			if !reflect.DeepEqual(resultConfig, testcase.expectConfig) {

staging/src/k8s.io/apiserver/pkg/server/secure_serving.go

@@ -72,6 +72,20 @@ func (s *SecureServingInfo) tlsConfig(stopCh <-chan struct{}) (*tls.Config, erro
 			s.SNICerts,
 			nil, // TODO see how to plumb an event recorder down in here. For now this results in simply klog messages.
 		)
+		// register if possible
+		if notifier, ok := s.ClientCA.(dynamiccertificates.Notifier); ok {
+			notifier.AddListener(dynamicCertificateController)
+		}
+		// start controllers if possible
+		if controller, ok := s.ClientCA.(dynamiccertificates.ControllerRunner); ok {
+			// runonce to be sure that we have a value.
+			if err := controller.RunOnce(); err != nil {
+				return nil, err
+			}
+
+			go controller.Run(1, stopCh)
+		}
+
 		// runonce to be sure that we have a value.
 		if err := dynamicCertificateController.RunOnce(); err != nil {
 			return nil, err

test/integration/apiserver/BUILD

@@ -89,6 +89,7 @@ filegroup(
         ":package-srcs",
         "//test/integration/apiserver/admissionwebhook:all-srcs",
         "//test/integration/apiserver/apply:all-srcs",
+        "//test/integration/apiserver/certreload:all-srcs",
         "//test/integration/apiserver/podlogs:all-srcs",
     ],
     tags = ["automanaged"],

test/integration/apiserver/certreload/BUILD

@@ -0,0 +1,29 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "certreload_test.go",
+        "main_test.go",
+    ],
+    tags = ["integration"],
+    deps = [
+        "//cmd/kube-apiserver/app/options:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library",
+        "//test/integration/framework:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

test/integration/apiserver/certreload/certreload_test.go

@@ -0,0 +1,131 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package podlogs
+
+import (
+	"crypto/tls"
+	"io/ioutil"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+
+	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
+	"k8s.io/kubernetes/test/integration/framework"
+)
+
+func TestClientCA(t *testing.T) {
+	stopCh := make(chan struct{})
+	defer close(stopCh)
+
+	// I have no idea what this cert is, but it doesn't matter, we just want something that always fails validation
+	differentClientCA := []byte(`-----BEGIN CERTIFICATE-----
+MIIDQDCCAiigAwIBAgIJANWw74P5KJk2MA0GCSqGSIb3DQEBCwUAMDQxMjAwBgNV
+BAMMKWdlbmVyaWNfd2ViaG9va19hZG1pc3Npb25fcGx1Z2luX3Rlc3RzX2NhMCAX
+DTE3MTExNjAwMDUzOVoYDzIyOTEwOTAxMDAwNTM5WjAjMSEwHwYDVQQDExh3ZWJo
+b29rLXRlc3QuZGVmYXVsdC5zdmMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQDXd/nQ89a5H8ifEsigmMd01Ib6NVR3bkJjtkvYnTbdfYEBj7UzqOQtHoLa
+dIVmefny5uIHvj93WD8WDVPB3jX2JHrXkDTXd/6o6jIXHcsUfFTVLp6/bZ+Anqe0
+r/7hAPkzA2A7APyTWM3ZbEeo1afXogXhOJ1u/wz0DflgcB21gNho4kKTONXO3NHD
+XLpspFqSkxfEfKVDJaYAoMnYZJtFNsa2OvsmLnhYF8bjeT3i07lfwrhUZvP+7Gsp
+7UgUwc06WuNHjfx1s5e6ySzH0QioMD1rjYneqOvk0pKrMIhuAEWXqq7jlXcDtx1E
+j+wnYbVqqVYheHZ8BCJoVAAQGs9/AgMBAAGjZDBiMAkGA1UdEwQCMAAwCwYDVR0P
+BAQDAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATApBgNVHREEIjAg
+hwR/AAABghh3ZWJob29rLXRlc3QuZGVmYXVsdC5zdmMwDQYJKoZIhvcNAQELBQAD
+ggEBAD/GKSPNyQuAOw/jsYZesb+RMedbkzs18sSwlxAJQMUrrXwlVdHrA8q5WhE6
+ABLqU1b8lQ8AWun07R8k5tqTmNvCARrAPRUqls/ryER+3Y9YEcxEaTc3jKNZFLbc
+T6YtcnkdhxsiO136wtiuatpYL91RgCmuSpR8+7jEHhuFU01iaASu7ypFrUzrKHTF
+bKwiLRQi1cMzVcLErq5CDEKiKhUkoDucyARFszrGt9vNIl/YCcBOkcNvM3c05Hn3
+M++C29JwS3Hwbubg6WO3wjFjoEhpCwU6qRYUz3MRp4tHO4kxKXx+oQnUiFnR7vW0
+YkNtGc1RUDHwecCTFpJtPb7Yu/E=
+-----END CERTIFICATE-----
+`)
+	differentFrontProxyCA := []byte(`-----BEGIN CERTIFICATE-----
+MIIBqDCCAU2gAwIBAgIUfbqeieihh/oERbfvRm38XvS/xHAwCgYIKoZIzj0EAwIw
+GjEYMBYGA1UEAxMPSW50ZXJtZWRpYXRlLUNBMCAXDTE2MTAxMTA1MDYwMFoYDzIx
+MTYwOTE3MDUwNjAwWjAUMRIwEAYDVQQDEwlNeSBDbGllbnQwWTATBgcqhkjOPQIB
+BggqhkjOPQMBBwNCAARv6N4R/sjMR65iMFGNLN1GC/vd7WhDW6J4X/iAjkRLLnNb
+KbRG/AtOUZ+7upJ3BWIRKYbOabbQGQe2BbKFiap4o3UwczAOBgNVHQ8BAf8EBAMC
+BaAwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU
+K/pZOWpNcYai6eHFpmJEeFpeQlEwHwYDVR0jBBgwFoAUX6nQlxjfWnP6aM1meO/Q
+a6b3a9kwCgYIKoZIzj0EAwIDSQAwRgIhAIWTKw/sjJITqeuNzJDAKU4xo1zL+xJ5
+MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps=
+-----END CERTIFICATE-----
+
+`)
+	clientCAFilename := ""
+	frontProxyCAFilename := ""
+
+	_, kubeconfig := framework.StartTestServer(t, stopCh, framework.TestServerSetup{
+		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
+			opts.GenericServerRunOptions.MaxRequestBodyBytes = 1024 * 1024
+			clientCAFilename = opts.Authentication.ClientCert.ClientCA
+			frontProxyCAFilename = opts.Authentication.RequestHeader.ClientCAFile
+			dynamiccertificates.FileRefreshDuration = 1 * time.Second
+		},
+	})
+	apiserverURL, err := url.Parse(kubeconfig.Host)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// when we run this the second time, we know which one we are expecting
+	acceptableCAs := []string{}
+	tlsConfig := &tls.Config{
+		InsecureSkipVerify: true,
+		GetClientCertificate: func(hello *tls.CertificateRequestInfo) (*tls.Certificate, error) {
+			acceptableCAs = []string{}
+			for _, curr := range hello.AcceptableCAs {
+				acceptableCAs = append(acceptableCAs, string(curr))
+			}
+			return &tls.Certificate{}, nil
+		},
+	}
+
+	conn, err := tls.Dial("tcp", apiserverURL.Host, tlsConfig)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer conn.Close()
+
+	if err := ioutil.WriteFile(clientCAFilename, differentClientCA, 0644); err != nil {
+		t.Fatal(err)
+	}
+	if err := ioutil.WriteFile(frontProxyCAFilename, differentFrontProxyCA, 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	time.Sleep(4 * time.Second)
+
+	conn2, err := tls.Dial("tcp", apiserverURL.Host, tlsConfig)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer conn2.Close()
+
+	expectedCAs := []string{"webhook-test.default.svc", "My Client"}
+	if len(expectedCAs) != len(acceptableCAs) {
+		t.Fatal(strings.Join(acceptableCAs, ":"))
+	}
+	for i := range expectedCAs {
+		if !strings.Contains(acceptableCAs[i], expectedCAs[i]) {
+			t.Errorf("expected %q, got %q", expectedCAs[i], acceptableCAs[i])
+		}
+	}
+}

test/integration/apiserver/certreload/main_test.go

@@ -0,0 +1,27 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package podlogs
+
+import (
+	"testing"
+
+	"k8s.io/kubernetes/test/integration/framework"
+)
+
+func TestMain(m *testing.M) {
+	framework.EtcdMain(m.Run)
+}