From 6beb96261e29754f2b7d0e44829eb6d15422cebf Mon Sep 17 00:00:00 2001 From: David Eads Date: Mon, 7 Oct 2019 14:06:42 -0400 Subject: [PATCH] wire up a means to dynamically reload ca bundles for kube-apiserver --- cmd/kubelet/app/BUILD | 2 +- cmd/kubelet/app/auth.go | 18 ++- pkg/kubeapiserver/authenticator/BUILD | 1 + pkg/kubeapiserver/authenticator/config.go | 11 +- pkg/kubeapiserver/options/authentication.go | 19 ++- .../options/authentication_test.go | 12 +- .../authenticatorfactory/delegating.go | 12 +- .../authenticatorfactory/requestheader.go | 19 ++- .../src/k8s.io/apiserver/pkg/server/config.go | 24 ++-- .../pkg/server/dynamiccertificates/BUILD | 1 + .../server/dynamiccertificates/client_ca.go | 3 + .../dynamicfile_content.go | 68 +++++++-- .../dynamiccertificates/static_content.go | 24 +++- .../server/dynamiccertificates/tlsconfig.go | 2 + .../dynamiccertificates/tlsconfig_test.go | 4 +- .../dynamiccertificates/union_content.go | 52 +++++++ .../k8s.io/apiserver/pkg/server/options/BUILD | 1 - .../pkg/server/options/authentication.go | 62 ++++----- .../pkg/server/options/authentication_test.go | 6 +- .../apiserver/pkg/server/secure_serving.go | 14 ++ test/integration/apiserver/BUILD | 1 + test/integration/apiserver/certreload/BUILD | 29 ++++ .../apiserver/certreload/certreload_test.go | 131 ++++++++++++++++++ .../apiserver/certreload/main_test.go | 27 ++++ 24 files changed, 438 insertions(+), 105 deletions(-) create mode 100644 test/integration/apiserver/certreload/BUILD create mode 100644 test/integration/apiserver/certreload/certreload_test.go create mode 100644 test/integration/apiserver/certreload/main_test.go diff --git a/cmd/kubelet/app/BUILD b/cmd/kubelet/app/BUILD index af51128482f..b9551f8ad93 100644 --- a/cmd/kubelet/app/BUILD +++ b/cmd/kubelet/app/BUILD @@ -118,10 +118,10 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/authentication/request/x509:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", diff --git a/cmd/kubelet/app/auth.go b/cmd/kubelet/app/auth.go index fdad2d5a0c6..eea8d2b1886 100644 --- a/cmd/kubelet/app/auth.go +++ b/cmd/kubelet/app/auth.go @@ -24,9 +24,9 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/authenticatorfactory" - "k8s.io/apiserver/pkg/authentication/request/x509" "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/authorization/authorizerfactory" + "k8s.io/apiserver/pkg/server/dynamiccertificates" clientset "k8s.io/client-go/kubernetes" authenticationclient "k8s.io/client-go/kubernetes/typed/authentication/v1beta1" authorizationclient "k8s.io/client-go/kubernetes/typed/authorization/v1beta1" @@ -64,15 +64,19 @@ func BuildAuth(nodeName types.NodeName, client clientset.Interface, config kubel // BuildAuthn creates an authenticator compatible with the kubelet's needs func BuildAuthn(client authenticationclient.TokenReviewInterface, authn kubeletconfig.KubeletAuthentication) (authenticator.Request, error) { - clientCertVerifier, err := x509.NewStaticVerifierFromFile(authn.X509.ClientCAFile) - if err != nil { - return nil, err + var clientCertificateCAContentProvider authenticatorfactory.CAContentProvider + var err error + if len(authn.X509.ClientCAFile) > 0 { + clientCertificateCAContentProvider, err = dynamiccertificates.NewDynamicCAContentFromFile("client-ca-bundle", authn.X509.ClientCAFile) + if err != nil { + return nil, err + } } authenticatorConfig := authenticatorfactory.DelegatingAuthenticatorConfig{ - Anonymous: authn.Anonymous.Enabled, - CacheTTL: authn.Webhook.CacheTTL.Duration, - ClientVerifyOptionFn: clientCertVerifier, + Anonymous: authn.Anonymous.Enabled, + CacheTTL: authn.Webhook.CacheTTL.Duration, + ClientCertificateCAContentProvider: clientCertificateCAContentProvider, } if authn.Webhook.Enabled { diff --git a/pkg/kubeapiserver/authenticator/BUILD b/pkg/kubeapiserver/authenticator/BUILD index e44953c04ca..68c7c30d862 100644 --- a/pkg/kubeapiserver/authenticator/BUILD +++ b/pkg/kubeapiserver/authenticator/BUILD @@ -24,6 +24,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/authentication/token/cache:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/token/tokenfile:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/token/union:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/apiserver/plugin/pkg/authenticator/password/passwordfile:go_default_library", "//staging/src/k8s.io/apiserver/plugin/pkg/authenticator/request/basicauth:go_default_library", diff --git a/pkg/kubeapiserver/authenticator/config.go b/pkg/kubeapiserver/authenticator/config.go index 6994116e9fc..de068e9f670 100644 --- a/pkg/kubeapiserver/authenticator/config.go +++ b/pkg/kubeapiserver/authenticator/config.go @@ -33,6 +33,7 @@ import ( tokencache "k8s.io/apiserver/pkg/authentication/token/cache" "k8s.io/apiserver/pkg/authentication/token/tokenfile" tokenunion "k8s.io/apiserver/pkg/authentication/token/union" + "k8s.io/apiserver/pkg/server/dynamiccertificates" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/apiserver/plugin/pkg/authenticator/password/passwordfile" "k8s.io/apiserver/plugin/pkg/authenticator/request/basicauth" @@ -77,10 +78,10 @@ type Config struct { // TODO, this is the only non-serializable part of the entire config. Factor it out into a clientconfig ServiceAccountTokenGetter serviceaccount.ServiceAccountTokenGetter BootstrapTokenAuthenticator authenticator.Token - // ClientVerifyOptionFn are the options for verifying incoming connections using mTLS and directly assigning to users. + // ClientCAContentProvider are the options for verifying incoming connections using mTLS and directly assigning to users. // Generally this is the CA bundle file used to authenticate client certificates // If this value is nil, then mutual TLS is disabled. - ClientVerifyOptionFn x509.VerifyOptionFunc + ClientCAContentProvider dynamiccertificates.CAContentProvider } // New returns an authenticator.Request or an error that supports the standard @@ -94,7 +95,7 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, er // Add the front proxy authenticator if requested if config.RequestHeaderConfig != nil { requestHeaderAuthenticator := headerrequest.NewDynamicVerifyOptionsSecure( - config.RequestHeaderConfig.VerifyOptionFn, + config.RequestHeaderConfig.CAContentProvider.VerifyOptions, config.RequestHeaderConfig.AllowedClientNames, config.RequestHeaderConfig.UsernameHeaders, config.RequestHeaderConfig.GroupHeaders, @@ -120,8 +121,8 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, er } // X509 methods - if config.ClientVerifyOptionFn != nil { - certAuth := x509.NewDynamic(config.ClientVerifyOptionFn, x509.CommonNameUserConversion) + if config.ClientCAContentProvider != nil { + certAuth := x509.NewDynamic(config.ClientCAContentProvider.VerifyOptions, x509.CommonNameUserConversion) authenticators = append(authenticators, certAuth) } diff --git a/pkg/kubeapiserver/options/authentication.go b/pkg/kubeapiserver/options/authentication.go index 6d3db368b04..1d4177802d8 100644 --- a/pkg/kubeapiserver/options/authentication.go +++ b/pkg/kubeapiserver/options/authentication.go @@ -324,7 +324,7 @@ func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() (kubeauthenticat if s.ClientCert != nil { var err error - ret.ClientVerifyOptionFn, err = s.ClientCert.GetClientVerifyOptionFn() + ret.ClientCAContentProvider, err = s.ClientCert.GetClientCAContentProvider() if err != nil { return kubeauthenticator.Config{}, err } @@ -390,15 +390,24 @@ func (o *BuiltInAuthenticationOptions) ApplyTo(c *genericapiserver.Config) error return nil } - var err error if o.ClientCert != nil { - if err = c.Authentication.ApplyClientCert(o.ClientCert.ClientCA, c.SecureServing); err != nil { + clientCertificateCAContentProvider, err := o.ClientCert.GetClientCAContentProvider() + if err != nil { + return fmt.Errorf("unable to load client CA file: %v", err) + } + if err = c.Authentication.ApplyClientCert(clientCertificateCAContentProvider, c.SecureServing); err != nil { return fmt.Errorf("unable to load client CA file: %v", err) } } if o.RequestHeader != nil { - if err = c.Authentication.ApplyClientCert(o.RequestHeader.ClientCAFile, c.SecureServing); err != nil { - return fmt.Errorf("unable to load client CA file: %v", err) + requestHeaderConfig, err := o.RequestHeader.ToAuthenticationRequestHeaderConfig() + if err != nil { + return fmt.Errorf("unable to create request header authentication config: %v", err) + } + if requestHeaderConfig != nil { + if err = c.Authentication.ApplyClientCert(requestHeaderConfig.CAContentProvider, c.SecureServing); err != nil { + return fmt.Errorf("unable to load client CA file: %v", err) + } } } diff --git a/pkg/kubeapiserver/options/authentication_test.go b/pkg/kubeapiserver/options/authentication_test.go index 2baa1877380..60e907a1e88 100644 --- a/pkg/kubeapiserver/options/authentication_test.go +++ b/pkg/kubeapiserver/options/authentication_test.go @@ -146,7 +146,7 @@ func TestToAuthenticationConfig(t *testing.T) { Anonymous: false, BasicAuthFile: "/testBasicAuthFile", BootstrapToken: false, - ClientVerifyOptionFn: nil, // this is nil because you can't compare functions + ClientCAContentProvider: nil, // this is nil because you can't compare functions TokenAuthFile: "/testTokenFile", OIDCIssuerURL: "testIssuerURL", OIDCClientID: "testClientID", @@ -165,7 +165,7 @@ func TestToAuthenticationConfig(t *testing.T) { UsernameHeaders: headerrequest.StaticStringSlice{"x-remote-user"}, GroupHeaders: headerrequest.StaticStringSlice{"x-remote-group"}, ExtraHeaderPrefixes: headerrequest.StaticStringSlice{"x-remote-extra-"}, - VerifyOptionFn: nil, // this is nil because you can't compare functions + CAContentProvider: nil, // this is nil because you can't compare functions AllowedClientNames: headerrequest.StaticStringSlice{"kube-aggregator"}, }, } @@ -176,14 +176,14 @@ func TestToAuthenticationConfig(t *testing.T) { } // nil these out because you cannot compare pointers. Ensure they are non-nil first - if resultConfig.ClientVerifyOptionFn == nil { + if resultConfig.ClientCAContentProvider == nil { t.Error("missing client verify") } - if resultConfig.RequestHeaderConfig.VerifyOptionFn == nil { + if resultConfig.RequestHeaderConfig.CAContentProvider == nil { t.Error("missing requestheader verify") } - resultConfig.ClientVerifyOptionFn = nil - resultConfig.RequestHeaderConfig.VerifyOptionFn = nil + resultConfig.ClientCAContentProvider = nil + resultConfig.RequestHeaderConfig.CAContentProvider = nil if !reflect.DeepEqual(resultConfig, expectConfig) { t.Error(cmp.Diff(resultConfig, expectConfig)) diff --git a/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/delegating.go b/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/delegating.go index ffd55de596d..61cc68988f3 100644 --- a/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/delegating.go +++ b/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/delegating.go @@ -30,7 +30,6 @@ import ( unionauth "k8s.io/apiserver/pkg/authentication/request/union" "k8s.io/apiserver/pkg/authentication/request/websocket" "k8s.io/apiserver/pkg/authentication/request/x509" - x509request "k8s.io/apiserver/pkg/authentication/request/x509" "k8s.io/apiserver/pkg/authentication/token/cache" webhooktoken "k8s.io/apiserver/plugin/pkg/authenticator/token/webhook" authenticationclient "k8s.io/client-go/kubernetes/typed/authentication/v1beta1" @@ -47,9 +46,10 @@ type DelegatingAuthenticatorConfig struct { // CacheTTL is the length of time that a token authentication answer will be cached. CacheTTL time.Duration - // ClientVerifyOptionFn are the options for verifying incoming connections using mTLS and directly assigning to users. + // CAContentProvider are the options for verifying incoming connections using mTLS and directly assigning to users. // Generally this is the CA bundle file used to authenticate client certificates - ClientVerifyOptionFn x509request.VerifyOptionFunc + // If this is nil, then mTLS will not be used. + ClientCertificateCAContentProvider CAContentProvider APIAudiences authenticator.Audiences @@ -64,7 +64,7 @@ func (c DelegatingAuthenticatorConfig) New() (authenticator.Request, *spec.Secur // Add the front proxy authenticator if requested if c.RequestHeaderConfig != nil { requestHeaderAuthenticator := headerrequest.NewDynamicVerifyOptionsSecure( - c.RequestHeaderConfig.VerifyOptionFn, + c.RequestHeaderConfig.CAContentProvider.VerifyOptions, c.RequestHeaderConfig.AllowedClientNames, c.RequestHeaderConfig.UsernameHeaders, c.RequestHeaderConfig.GroupHeaders, @@ -74,8 +74,8 @@ func (c DelegatingAuthenticatorConfig) New() (authenticator.Request, *spec.Secur } // x509 client cert auth - if c.ClientVerifyOptionFn != nil { - authenticators = append(authenticators, x509.NewDynamic(c.ClientVerifyOptionFn, x509.CommonNameUserConversion)) + if c.ClientCertificateCAContentProvider != nil { + authenticators = append(authenticators, x509.NewDynamic(c.ClientCertificateCAContentProvider.VerifyOptions, x509.CommonNameUserConversion)) } if c.TokenAccessReviewClient != nil { diff --git a/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/requestheader.go b/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/requestheader.go index 7108232fd22..eb39252cbe3 100644 --- a/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/requestheader.go +++ b/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/requestheader.go @@ -17,8 +17,9 @@ limitations under the License. package authenticatorfactory import ( + "crypto/x509" + "k8s.io/apiserver/pkg/authentication/request/headerrequest" - x509request "k8s.io/apiserver/pkg/authentication/request/x509" ) type RequestHeaderConfig struct { @@ -29,9 +30,19 @@ type RequestHeaderConfig struct { // ExtraHeaderPrefixes are the head prefixes to check (case-insentively) for filling in // the user.Info.Extra. All values of all matching headers will be added. ExtraHeaderPrefixes headerrequest.StringSliceProvider - // VerifyOptionFn are the options for verifying incoming connections using mTLS. Generally this points to CA bundle file which is used verify the identity of the front proxy. - // It may produce different options at will. - VerifyOptionFn x509request.VerifyOptionFunc + // CAContentProvider the options for verifying incoming connections using mTLS. Generally this points to CA bundle file which is used verify the identity of the front proxy. + // It may produce different options at will. + CAContentProvider CAContentProvider // AllowedClientNames is a list of common names that may be presented by the authenticating front proxy. Empty means: accept any. AllowedClientNames headerrequest.StringSliceProvider } + +// CAContentProvider provides ca bundle byte content +type CAContentProvider interface { + // Name is just an identifier + Name() string + // CurrentCABundleContent provides ca bundle byte content + CurrentCABundleContent() []byte + // VerifyOptions provides VerifyOptions for authenticators + VerifyOptions() x509.VerifyOptions +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index 45b277459f9..16208954a47 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -345,21 +345,19 @@ func DefaultOpenAPIConfig(getDefinitions openapicommon.GetOpenAPIDefinitions, de } } -func (c *AuthenticationInfo) ApplyClientCert(clientCAFile string, servingInfo *SecureServingInfo) error { - if servingInfo != nil { - if len(clientCAFile) > 0 { - clientCAProvider, err := dynamiccertificates.NewStaticCAContentFromFile(clientCAFile) - if err != nil { - return fmt.Errorf("unable to load client CA file: %v", err) - } - if servingInfo.ClientCA == nil { - servingInfo.ClientCA = clientCAProvider - } else { - servingInfo.ClientCA = dynamiccertificates.NewUnionCAContentProvider(servingInfo.ClientCA, clientCAProvider) - } - } +func (c *AuthenticationInfo) ApplyClientCert(clientCA dynamiccertificates.CAContentProvider, servingInfo *SecureServingInfo) error { + if servingInfo == nil { + return nil + } + if clientCA == nil { + return nil + } + if servingInfo.ClientCA == nil { + servingInfo.ClientCA = clientCA + return nil } + servingInfo.ClientCA = dynamiccertificates.NewUnionCAContentProvider(servingInfo.ClientCA, clientCA) return nil } diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD index 009f8e9de54..55d9aa65204 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/BUILD @@ -17,6 +17,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/client_ca.go b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/client_ca.go index f5c8cad4372..6a881aa4166 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/client_ca.go +++ b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/client_ca.go @@ -18,6 +18,7 @@ package dynamiccertificates import ( "bytes" + "crypto/x509" ) // CAContentProvider provides ca bundle byte content @@ -27,6 +28,8 @@ type CAContentProvider interface { // CurrentCABundleContent provides ca bundle byte content. Errors can be contained to the controllers initializing // the value. By the time you get here, you should always be returning a value that won't fail. CurrentCABundleContent() []byte + // VerifyOptions provides VerifyOptions for authenticators + VerifyOptions() x509.VerifyOptions } // dynamicCertificateContent holds the content that overrides the baseTLSConfig diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamicfile_content.go b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamicfile_content.go index a168eb68065..bb0ebd5537d 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamicfile_content.go +++ b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamicfile_content.go @@ -17,10 +17,10 @@ limitations under the License. package dynamiccertificates import ( + "bytes" "crypto/x509" "fmt" "io/ioutil" - "reflect" "sync/atomic" "time" @@ -32,10 +32,30 @@ import ( "k8s.io/klog" ) -type CAListener interface { +// FileRefreshDuration is exposed so that integration tests can crank up the reload speed. +var FileRefreshDuration = 1 * time.Minute + +// Listener is an interface to use to notify interested parties of a change. +type Listener interface { + // Enqueue should be called when an input may have changed Enqueue() } +// Notifier is a way to add listeners +type Notifier interface { + // AddListener is adds a listener to be notified of potential input changes + AddListener(listener Listener) +} + +// ControllerRunner is a generic interface for starting a controller +type ControllerRunner interface { + // RunOnce runs the sync loop a single time. This useful for synchronous priming + RunOnce() error + + // Run should be called a go .Run + Run(workers int, stopCh <-chan struct{}) +} + // DynamicFileCAContent provies a CAContentProvider that can dynamically react to new file content // It also fulfills the authenticator interface to provide verifyoptions type DynamicFileCAContent struct { @@ -47,18 +67,22 @@ type DynamicFileCAContent struct { // caBundle is a caBundleAndVerifier that contains the last read, non-zero length content of the file caBundle atomic.Value - listeners []CAListener + listeners []Listener // queue only ever has one item, but it has nice error handling backoff/retry semantics queue workqueue.RateLimitingInterface } +var _ Notifier = &DynamicFileCAContent{} +var _ CAContentProvider = &DynamicFileCAContent{} +var _ ControllerRunner = &DynamicFileCAContent{} + type caBundleAndVerifier struct { caBundle []byte verifyOptions x509.VerifyOptions } -// NewStaticCAContentFromFile returns a CAContentProvider based on a filename +// NewDynamicCAContentFromFile returns a CAContentProvider based on a filename that automatically reloads content func NewDynamicCAContentFromFile(purpose, filename string) (*DynamicFileCAContent, error) { if len(filename) == 0 { return nil, fmt.Errorf("missing filename for ca bundle") @@ -78,7 +102,7 @@ func NewDynamicCAContentFromFile(purpose, filename string) (*DynamicFileCAConten } // AddListener adds a listener to be notified when the CA content changes. -func (c *DynamicFileCAContent) AddListener(listener CAListener) { +func (c *DynamicFileCAContent) AddListener(listener Listener) { c.listeners = append(c.listeners, listener) } @@ -93,8 +117,7 @@ func (c *DynamicFileCAContent) loadCABundle() error { } // check to see if we have a change. If the values are the same, do nothing. - existing, ok := c.caBundle.Load().(*caBundleAndVerifier) - if ok && existing != nil && reflect.DeepEqual(existing.caBundle, caBundle) { + if !c.hasCAChanged(caBundle) { return nil } @@ -111,6 +134,30 @@ func (c *DynamicFileCAContent) loadCABundle() error { return nil } +// hasCAChanged returns true if the caBundle is different than the current. +func (c *DynamicFileCAContent) hasCAChanged(caBundle []byte) bool { + uncastExisting := c.caBundle.Load() + if uncastExisting == nil { + return true + } + + // check to see if we have a change. If the values are the same, do nothing. + existing, ok := uncastExisting.(*caBundleAndVerifier) + if !ok { + return true + } + if !bytes.Equal(existing.caBundle, caBundle) { + return true + } + + return false +} + +// RunOnce runs a single sync loop +func (c *DynamicFileCAContent) RunOnce() error { + return c.loadCABundle() +} + // Run starts the kube-apiserver and blocks until stopCh is closed. func (c *DynamicFileCAContent) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() @@ -123,7 +170,7 @@ func (c *DynamicFileCAContent) Run(workers int, stopCh <-chan struct{}) { go wait.Until(c.runWorker, time.Second, stopCh) // start timer that rechecks every minute, just in case. this also serves to prime the controller quickly. - _ = wait.PollImmediateUntil(1*time.Minute, func() (bool, error) { + _ = wait.PollImmediateUntil(FileRefreshDuration, func() (bool, error) { c.queue.Add(workItemKey) return false, nil }, stopCh) @@ -164,11 +211,12 @@ func (c *DynamicFileCAContent) Name() string { // CurrentCABundleContent provides ca bundle byte content func (c *DynamicFileCAContent) CurrentCABundleContent() (cabundle []byte) { - return c.caBundle.Load().(caBundleAndVerifier).caBundle + return c.caBundle.Load().(*caBundleAndVerifier).caBundle } +// VerifyOptions provides verifyoptions compatible with authenticators func (c *DynamicFileCAContent) VerifyOptions() x509.VerifyOptions { - return c.caBundle.Load().(caBundleAndVerifier).verifyOptions + return c.caBundle.Load().(*caBundleAndVerifier).verifyOptions } // newVerifyOptions creates a new verification func from a file. It reads the content and then fails. diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/static_content.go b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/static_content.go index 1d4b2208fc3..dffee8e51b9 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/static_content.go +++ b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/static_content.go @@ -18,15 +18,18 @@ package dynamiccertificates import ( "crypto/tls" + "crypto/x509" "fmt" "io/ioutil" ) type staticCAContent struct { name string - caBundle []byte + caBundle *caBundleAndVerifier } +var _ CAContentProvider = &staticCAContent{} + // NewStaticCAContentFromFile returns a CAContentProvider based on a filename func NewStaticCAContentFromFile(filename string) (CAContentProvider, error) { if len(filename) == 0 { @@ -37,15 +40,20 @@ func NewStaticCAContentFromFile(filename string) (CAContentProvider, error) { if err != nil { return nil, err } - return NewStaticCAContent(filename, caBundle), nil + return NewStaticCAContent(filename, caBundle) } // NewStaticCAContent returns a CAContentProvider that always returns the same value -func NewStaticCAContent(name string, caBundle []byte) CAContentProvider { +func NewStaticCAContent(name string, caBundle []byte) (CAContentProvider, error) { + caBundleAndVerifier, err := newCABundleAndVerifier(name, caBundle) + if err != nil { + return nil, err + } + return &staticCAContent{ name: name, - caBundle: caBundle, - } + caBundle: caBundleAndVerifier, + }, nil } // Name is just an identifier @@ -55,7 +63,11 @@ func (c *staticCAContent) Name() string { // CurrentCABundleContent provides ca bundle byte content func (c *staticCAContent) CurrentCABundleContent() (cabundle []byte) { - return c.caBundle + return c.caBundle.caBundle +} + +func (c *staticCAContent) VerifyOptions() x509.VerifyOptions { + return c.caBundle.verifyOptions } type staticCertKeyContent struct { diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig.go b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig.go index 194979743f0..0c428ad4aa7 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig.go +++ b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig.go @@ -60,6 +60,8 @@ type DynamicServingCertificateController struct { eventRecorder events.EventRecorder } +var _ Listener = &DynamicServingCertificateController{} + // NewDynamicServingCertificateController returns a controller that can be used to keep a TLSConfig up to date. func NewDynamicServingCertificateController( baseTLSConfig tls.Config, diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig_test.go b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig_test.go index 973a4d68362..5989321d081 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig_test.go @@ -89,7 +89,7 @@ func TestNewStaticCertKeyContent(t *testing.T) { }{ { name: "filled", - clientCA: NewStaticCAContent("test-ca", []byte("content-1")), + clientCA: &staticCAContent{name: "test-ca", caBundle: &caBundleAndVerifier{caBundle: []byte("content-1")}}, servingCert: testCertProvider, sniCerts: []SNICertKeyContentProvider{testCertProvider}, expected: &dynamicCertificateContent{ @@ -101,7 +101,7 @@ func TestNewStaticCertKeyContent(t *testing.T) { }, { name: "missingCA", - clientCA: NewStaticCAContent("test-ca", []byte("")), + clientCA: &staticCAContent{name: "test-ca", caBundle: &caBundleAndVerifier{caBundle: []byte("")}}, expected: nil, expectedErr: `not loading an empty client ca bundle from "test-ca"`, }, diff --git a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/union_content.go b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/union_content.go index 0e197b1bdb3..ef63219cfa5 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/union_content.go +++ b/staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates/union_content.go @@ -18,11 +18,18 @@ package dynamiccertificates import ( "bytes" + "crypto/x509" "strings" + + utilerrors "k8s.io/apimachinery/pkg/util/errors" ) type unionCAContent []CAContentProvider +var _ Notifier = &unionCAContent{} +var _ CAContentProvider = &unionCAContent{} +var _ ControllerRunner = &unionCAContent{} + // NewUnionCAContentProvider returns a CAContentProvider that is a union of other CAContentProviders func NewUnionCAContentProvider(caContentProviders ...CAContentProvider) CAContentProvider { return unionCAContent(caContentProviders) @@ -46,3 +53,48 @@ func (c unionCAContent) CurrentCABundleContent() []byte { return bytes.Join(caBundles, []byte("\n")) } + +// CurrentCABundleContent provides ca bundle byte content +func (c unionCAContent) VerifyOptions() x509.VerifyOptions { + // TODO make more efficient. This isn't actually used in any of our mainline paths. It's called to build the TLSConfig + // TODO on file changes, but the actual authentication runs against the individual items, not the union. + ret, err := newCABundleAndVerifier(c.Name(), c.CurrentCABundleContent()) + if err != nil { + // because we're made up of already vetted values, this indicates some kind of coding error + panic(err) + } + + return ret.verifyOptions +} + +// AddListener adds a listener to be notified when the CA content changes. +func (c unionCAContent) AddListener(listener Listener) { + for _, curr := range c { + if notifier, ok := curr.(Notifier); ok { + notifier.AddListener(listener) + } + } +} + +// AddListener adds a listener to be notified when the CA content changes. +func (c unionCAContent) RunOnce() error { + errors := []error{} + for _, curr := range c { + if controller, ok := curr.(ControllerRunner); ok { + if err := controller.RunOnce(); err != nil { + errors = append(errors, err) + } + } + } + + return utilerrors.NewAggregate(errors) +} + +// Run runs the controller +func (c unionCAContent) Run(workers int, stopCh <-chan struct{}) { + for _, curr := range c { + if controller, ok := curr.(ControllerRunner); ok { + go controller.Run(workers, stopCh) + } + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/BUILD b/staging/src/k8s.io/apiserver/pkg/server/options/BUILD index 7b3ff712a5f..c38cfd28ced 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/options/BUILD @@ -50,7 +50,6 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/audit/policy:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/request/headerrequest:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/authentication/request/x509:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authorization/path:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/authentication.go b/staging/src/k8s.io/apiserver/pkg/server/options/authentication.go index 25654cf113b..d90bbd66326 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/authentication.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/authentication.go @@ -23,6 +23,8 @@ import ( "strings" "time" + "k8s.io/apiserver/pkg/server/dynamiccertificates" + "github.com/spf13/pflag" v1 "k8s.io/api/core/v1" @@ -30,12 +32,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apiserver/pkg/authentication/authenticatorfactory" "k8s.io/apiserver/pkg/authentication/request/headerrequest" - "k8s.io/apiserver/pkg/authentication/request/x509" "k8s.io/apiserver/pkg/server" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/util/cert" "k8s.io/klog" openapicommon "k8s.io/kube-openapi/pkg/common" ) @@ -112,7 +112,7 @@ func (s *RequestHeaderAuthenticationOptions) ToAuthenticationRequestHeaderConfig return nil, nil } - verifyFn, err := x509.NewStaticVerifierFromFile(s.ClientCAFile) + caBundleProvider, err := dynamiccertificates.NewDynamicCAContentFromFile("request-header", s.ClientCAFile) if err != nil { return nil, err } @@ -121,7 +121,7 @@ func (s *RequestHeaderAuthenticationOptions) ToAuthenticationRequestHeaderConfig UsernameHeaders: headerrequest.StaticStringSlice(s.UsernameHeaders), GroupHeaders: headerrequest.StaticStringSlice(s.GroupHeaders), ExtraHeaderPrefixes: headerrequest.StaticStringSlice(s.ExtraHeaderPrefixes), - VerifyOptionFn: verifyFn, + CAContentProvider: caBundleProvider, AllowedClientNames: headerrequest.StaticStringSlice(s.AllowedNames), }, nil } @@ -132,23 +132,23 @@ type ClientCertAuthenticationOptions struct { // ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates ClientCA string - // ClientVerifyOptionFn are the options for verifying incoming connections using mTLS and directly assigning to users. + // CAContentProvider are the options for verifying incoming connections using mTLS and directly assigning to users. // Generally this is the CA bundle file used to authenticate client certificates // If non-nil, this takes priority over the ClientCA file. - ClientVerifyOptionFn x509.VerifyOptionFunc + CAContentProvider dynamiccertificates.CAContentProvider } // GetClientVerifyOptionFn provides verify options for your authenticator while respecting the preferred order of verifiers. -func (s *ClientCertAuthenticationOptions) GetClientVerifyOptionFn() (x509.VerifyOptionFunc, error) { - if s.ClientVerifyOptionFn != nil { - return s.ClientVerifyOptionFn, nil +func (s *ClientCertAuthenticationOptions) GetClientCAContentProvider() (dynamiccertificates.CAContentProvider, error) { + if s.CAContentProvider != nil { + return s.CAContentProvider, nil } if len(s.ClientCA) == 0 { return nil, nil } - return x509.NewStaticVerifierFromFile(s.ClientCA) + return dynamiccertificates.NewDynamicCAContentFromFile("client-ca-bundle", s.ClientCA) } func (s *ClientCertAuthenticationOptions) AddFlags(fs *pflag.FlagSet) { @@ -230,9 +230,9 @@ func (s *DelegatingAuthenticationOptions) AddFlags(fs *pflag.FlagSet) { "Note that this can result in authentication that treats all requests as anonymous.") } -func (s *DelegatingAuthenticationOptions) ApplyTo(c *server.AuthenticationInfo, servingInfo *server.SecureServingInfo, openAPIConfig *openapicommon.Config) error { +func (s *DelegatingAuthenticationOptions) ApplyTo(authenticationInfo *server.AuthenticationInfo, servingInfo *server.SecureServingInfo, openAPIConfig *openapicommon.Config) error { if s == nil { - c.Authenticator = nil + authenticationInfo.Authenticator = nil return nil } @@ -266,20 +266,24 @@ func (s *DelegatingAuthenticationOptions) ApplyTo(c *server.AuthenticationInfo, } // configure AuthenticationInfo config - cfg.ClientVerifyOptionFn, err = s.ClientCert.GetClientVerifyOptionFn() + cfg.ClientCertificateCAContentProvider, err = s.ClientCert.GetClientCAContentProvider() if err != nil { return fmt.Errorf("unable to load client CA file: %v", err) } - if err = c.ApplyClientCert(s.ClientCert.ClientCA, servingInfo); err != nil { - return fmt.Errorf("unable to load client CA file: %v", err) + if cfg.ClientCertificateCAContentProvider != nil { + if err = authenticationInfo.ApplyClientCert(cfg.ClientCertificateCAContentProvider, servingInfo); err != nil { + return fmt.Errorf("unable to load client CA file: %v", err) + } } cfg.RequestHeaderConfig, err = s.RequestHeader.ToAuthenticationRequestHeaderConfig() if err != nil { return fmt.Errorf("unable to create request header authentication config: %v", err) } - if err = c.ApplyClientCert(s.RequestHeader.ClientCAFile, servingInfo); err != nil { - return fmt.Errorf("unable to load client CA file: %v", err) + if cfg.RequestHeaderConfig != nil { + if err = authenticationInfo.ApplyClientCert(cfg.RequestHeaderConfig.CAContentProvider, servingInfo); err != nil { + return fmt.Errorf("unable to load client CA file: %v", err) + } } // create authenticator @@ -287,11 +291,11 @@ func (s *DelegatingAuthenticationOptions) ApplyTo(c *server.AuthenticationInfo, if err != nil { return err } - c.Authenticator = authenticator + authenticationInfo.Authenticator = authenticator if openAPIConfig != nil { openAPIConfig.SecurityDefinitions = securityDefinitions } - c.SupportsBasicAuth = false + authenticationInfo.SupportsBasicAuth = false return nil } @@ -372,28 +376,14 @@ func inClusterClientCA(authConfigMap *v1.ConfigMap) (*ClientCertAuthenticationOp // not having a client-ca is fine, return nil return nil, nil } - - clientCAs, err := cert.NewPoolFromBytes([]byte(clientCA)) + clientCAProvider, err := dynamiccertificates.NewStaticCAContent("client-ca-file", []byte(clientCA)) if err != nil { - return nil, fmt.Errorf("unable to load client CA from configmap: %v", err) - } - verifyOpts := x509.DefaultVerifyOptions() - verifyOpts.Roots = clientCAs - - // we still need to write out the client-ca-file for now because it is used to plumb the options through the apiserver's - // configuration to hint clients. - // TODO deads2k this should eventually be made dynamic along with the authenticator. I'm just wiring them one at at time. - f, err := ioutil.TempFile("", "client-ca-file") - if err != nil { - return nil, err - } - if err := ioutil.WriteFile(f.Name(), []byte(clientCA), 0600); err != nil { return nil, err } return &ClientCertAuthenticationOptions{ - ClientCA: f.Name(), - ClientVerifyOptionFn: x509.StaticVerifierFn(verifyOpts), + ClientCA: "", + CAContentProvider: clientCAProvider, }, nil } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/authentication_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/authentication_test.go index 22b5fb41da0..aa8de47cda8 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/authentication_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/authentication_test.go @@ -57,7 +57,7 @@ func TestToAuthenticationRequestHeaderConfig(t *testing.T) { UsernameHeaders: headerrequest.StaticStringSlice{"x-remote-user"}, GroupHeaders: headerrequest.StaticStringSlice{"x-remote-group"}, ExtraHeaderPrefixes: headerrequest.StaticStringSlice{"x-remote-extra-"}, - VerifyOptionFn: nil, // this is nil because you can't compare functions + CAContentProvider: nil, // this is nil because you can't compare functions AllowedClientNames: headerrequest.StaticStringSlice{"kube-aggregator"}, }, }, @@ -70,10 +70,10 @@ func TestToAuthenticationRequestHeaderConfig(t *testing.T) { t.Fatal(err) } if resultConfig != nil { - if resultConfig.VerifyOptionFn == nil { + if resultConfig.CAContentProvider == nil { t.Error("missing requestheader verify") } - resultConfig.VerifyOptionFn = nil + resultConfig.CAContentProvider = nil } if !reflect.DeepEqual(resultConfig, testcase.expectConfig) { diff --git a/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go b/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go index 7bbe77e7995..66c3bbd2968 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go +++ b/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go @@ -72,6 +72,20 @@ func (s *SecureServingInfo) tlsConfig(stopCh <-chan struct{}) (*tls.Config, erro s.SNICerts, nil, // TODO see how to plumb an event recorder down in here. For now this results in simply klog messages. ) + // register if possible + if notifier, ok := s.ClientCA.(dynamiccertificates.Notifier); ok { + notifier.AddListener(dynamicCertificateController) + } + // start controllers if possible + if controller, ok := s.ClientCA.(dynamiccertificates.ControllerRunner); ok { + // runonce to be sure that we have a value. + if err := controller.RunOnce(); err != nil { + return nil, err + } + + go controller.Run(1, stopCh) + } + // runonce to be sure that we have a value. if err := dynamicCertificateController.RunOnce(); err != nil { return nil, err diff --git a/test/integration/apiserver/BUILD b/test/integration/apiserver/BUILD index cdceb000fa4..d602f836869 100644 --- a/test/integration/apiserver/BUILD +++ b/test/integration/apiserver/BUILD @@ -89,6 +89,7 @@ filegroup( ":package-srcs", "//test/integration/apiserver/admissionwebhook:all-srcs", "//test/integration/apiserver/apply:all-srcs", + "//test/integration/apiserver/certreload:all-srcs", "//test/integration/apiserver/podlogs:all-srcs", ], tags = ["automanaged"], diff --git a/test/integration/apiserver/certreload/BUILD b/test/integration/apiserver/certreload/BUILD new file mode 100644 index 00000000000..35438779f95 --- /dev/null +++ b/test/integration/apiserver/certreload/BUILD @@ -0,0 +1,29 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_test") + +go_test( + name = "go_default_test", + srcs = [ + "certreload_test.go", + "main_test.go", + ], + tags = ["integration"], + deps = [ + "//cmd/kube-apiserver/app/options:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library", + "//test/integration/framework:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/test/integration/apiserver/certreload/certreload_test.go b/test/integration/apiserver/certreload/certreload_test.go new file mode 100644 index 00000000000..6c89abc6c8b --- /dev/null +++ b/test/integration/apiserver/certreload/certreload_test.go @@ -0,0 +1,131 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package podlogs + +import ( + "crypto/tls" + "io/ioutil" + "net/url" + "strings" + "testing" + "time" + + "k8s.io/apiserver/pkg/server/dynamiccertificates" + + "k8s.io/kubernetes/cmd/kube-apiserver/app/options" + "k8s.io/kubernetes/test/integration/framework" +) + +func TestClientCA(t *testing.T) { + stopCh := make(chan struct{}) + defer close(stopCh) + + // I have no idea what this cert is, but it doesn't matter, we just want something that always fails validation + differentClientCA := []byte(`-----BEGIN CERTIFICATE----- +MIIDQDCCAiigAwIBAgIJANWw74P5KJk2MA0GCSqGSIb3DQEBCwUAMDQxMjAwBgNV +BAMMKWdlbmVyaWNfd2ViaG9va19hZG1pc3Npb25fcGx1Z2luX3Rlc3RzX2NhMCAX +DTE3MTExNjAwMDUzOVoYDzIyOTEwOTAxMDAwNTM5WjAjMSEwHwYDVQQDExh3ZWJo +b29rLXRlc3QuZGVmYXVsdC5zdmMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQDXd/nQ89a5H8ifEsigmMd01Ib6NVR3bkJjtkvYnTbdfYEBj7UzqOQtHoLa +dIVmefny5uIHvj93WD8WDVPB3jX2JHrXkDTXd/6o6jIXHcsUfFTVLp6/bZ+Anqe0 +r/7hAPkzA2A7APyTWM3ZbEeo1afXogXhOJ1u/wz0DflgcB21gNho4kKTONXO3NHD +XLpspFqSkxfEfKVDJaYAoMnYZJtFNsa2OvsmLnhYF8bjeT3i07lfwrhUZvP+7Gsp +7UgUwc06WuNHjfx1s5e6ySzH0QioMD1rjYneqOvk0pKrMIhuAEWXqq7jlXcDtx1E +j+wnYbVqqVYheHZ8BCJoVAAQGs9/AgMBAAGjZDBiMAkGA1UdEwQCMAAwCwYDVR0P +BAQDAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATApBgNVHREEIjAg +hwR/AAABghh3ZWJob29rLXRlc3QuZGVmYXVsdC5zdmMwDQYJKoZIhvcNAQELBQAD +ggEBAD/GKSPNyQuAOw/jsYZesb+RMedbkzs18sSwlxAJQMUrrXwlVdHrA8q5WhE6 +ABLqU1b8lQ8AWun07R8k5tqTmNvCARrAPRUqls/ryER+3Y9YEcxEaTc3jKNZFLbc +T6YtcnkdhxsiO136wtiuatpYL91RgCmuSpR8+7jEHhuFU01iaASu7ypFrUzrKHTF +bKwiLRQi1cMzVcLErq5CDEKiKhUkoDucyARFszrGt9vNIl/YCcBOkcNvM3c05Hn3 +M++C29JwS3Hwbubg6WO3wjFjoEhpCwU6qRYUz3MRp4tHO4kxKXx+oQnUiFnR7vW0 +YkNtGc1RUDHwecCTFpJtPb7Yu/E= +-----END CERTIFICATE----- +`) + differentFrontProxyCA := []byte(`-----BEGIN CERTIFICATE----- +MIIBqDCCAU2gAwIBAgIUfbqeieihh/oERbfvRm38XvS/xHAwCgYIKoZIzj0EAwIw +GjEYMBYGA1UEAxMPSW50ZXJtZWRpYXRlLUNBMCAXDTE2MTAxMTA1MDYwMFoYDzIx +MTYwOTE3MDUwNjAwWjAUMRIwEAYDVQQDEwlNeSBDbGllbnQwWTATBgcqhkjOPQIB +BggqhkjOPQMBBwNCAARv6N4R/sjMR65iMFGNLN1GC/vd7WhDW6J4X/iAjkRLLnNb +KbRG/AtOUZ+7upJ3BWIRKYbOabbQGQe2BbKFiap4o3UwczAOBgNVHQ8BAf8EBAMC +BaAwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU +K/pZOWpNcYai6eHFpmJEeFpeQlEwHwYDVR0jBBgwFoAUX6nQlxjfWnP6aM1meO/Q +a6b3a9kwCgYIKoZIzj0EAwIDSQAwRgIhAIWTKw/sjJITqeuNzJDAKU4xo1zL+xJ5 +MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps= +-----END CERTIFICATE----- + +`) + clientCAFilename := "" + frontProxyCAFilename := "" + + _, kubeconfig := framework.StartTestServer(t, stopCh, framework.TestServerSetup{ + ModifyServerRunOptions: func(opts *options.ServerRunOptions) { + opts.GenericServerRunOptions.MaxRequestBodyBytes = 1024 * 1024 + clientCAFilename = opts.Authentication.ClientCert.ClientCA + frontProxyCAFilename = opts.Authentication.RequestHeader.ClientCAFile + dynamiccertificates.FileRefreshDuration = 1 * time.Second + }, + }) + apiserverURL, err := url.Parse(kubeconfig.Host) + if err != nil { + t.Fatal(err) + } + + // when we run this the second time, we know which one we are expecting + acceptableCAs := []string{} + tlsConfig := &tls.Config{ + InsecureSkipVerify: true, + GetClientCertificate: func(hello *tls.CertificateRequestInfo) (*tls.Certificate, error) { + acceptableCAs = []string{} + for _, curr := range hello.AcceptableCAs { + acceptableCAs = append(acceptableCAs, string(curr)) + } + return &tls.Certificate{}, nil + }, + } + + conn, err := tls.Dial("tcp", apiserverURL.Host, tlsConfig) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + if err := ioutil.WriteFile(clientCAFilename, differentClientCA, 0644); err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile(frontProxyCAFilename, differentFrontProxyCA, 0644); err != nil { + t.Fatal(err) + } + + time.Sleep(4 * time.Second) + + conn2, err := tls.Dial("tcp", apiserverURL.Host, tlsConfig) + if err != nil { + t.Fatal(err) + } + defer conn2.Close() + + expectedCAs := []string{"webhook-test.default.svc", "My Client"} + if len(expectedCAs) != len(acceptableCAs) { + t.Fatal(strings.Join(acceptableCAs, ":")) + } + for i := range expectedCAs { + if !strings.Contains(acceptableCAs[i], expectedCAs[i]) { + t.Errorf("expected %q, got %q", expectedCAs[i], acceptableCAs[i]) + } + } +} diff --git a/test/integration/apiserver/certreload/main_test.go b/test/integration/apiserver/certreload/main_test.go new file mode 100644 index 00000000000..379540cd44e --- /dev/null +++ b/test/integration/apiserver/certreload/main_test.go @@ -0,0 +1,27 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package podlogs + +import ( + "testing" + + "k8s.io/kubernetes/test/integration/framework" +) + +func TestMain(m *testing.M) { + framework.EtcdMain(m.Run) +}