From b22a170d4681da65673c6493fa03ee87fa2c5aab Mon Sep 17 00:00:00 2001 From: Tomas Nozicka Date: Wed, 29 Apr 2020 16:03:09 +0200 Subject: [PATCH] Fix client-ca dynamic reload in apiserver --- cmd/kube-apiserver/app/BUILD | 16 +- cmd/kube-apiserver/app/server.go | 42 +--- cmd/kubelet/app/auth.go | 33 ++- cmd/kubelet/app/server.go | 3 +- pkg/kubeapiserver/options/BUILD | 6 + pkg/kubeapiserver/options/authentication.go | 70 ++++-- test/integration/apiserver/certreload/BUILD | 3 + .../apiserver/certreload/certreload_test.go | 229 +++++++++++++++--- 8 files changed, 280 insertions(+), 122 deletions(-) diff --git a/cmd/kube-apiserver/app/BUILD b/cmd/kube-apiserver/app/BUILD index 9603737b8d3..c676bb00d1d 100644 --- a/cmd/kube-apiserver/app/BUILD +++ b/cmd/kube-apiserver/app/BUILD @@ -13,7 +13,6 @@ go_library( "//cmd/kube-apiserver/app/options:go_default_library", "//pkg/api/legacyscheme:go_default_library", "//pkg/capabilities:go_default_library", - "//pkg/controller/serviceaccount:go_default_library", "//pkg/features:go_default_library", "//pkg/generated/openapi:go_default_library", "//pkg/kubeapiserver:go_default_library", @@ -29,7 +28,6 @@ go_library( "//pkg/registry/cachesize:go_default_library", "//pkg/registry/rbac/rest:go_default_library", "//pkg/serviceaccount:go_default_library", - "//plugin/pkg/auth/authenticator/token/bootstrap:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver:go_default_library", @@ -43,7 +41,6 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", "//staging/src/k8s.io/apiserver/pkg/endpoints/openapi:go_default_library", "//staging/src/k8s.io/apiserver/pkg/features:go_default_library", @@ -76,12 +73,17 @@ go_library( "//staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/controllers/autoregister:go_default_library", - "//vendor/github.com/go-openapi/spec:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], ) +go_test( + name = "go_default_test", + srcs = ["server_test.go"], + embed = [":go_default_library"], +) + filegroup( name = "package-srcs", srcs = glob(["**"]), @@ -99,9 +101,3 @@ filegroup( tags = ["automanaged"], visibility = ["//visibility:public"], ) - -go_test( - name = "go_default_test", - srcs = ["server_test.go"], - embed = [":go_default_library"], -) diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 624368f539d..23e7f640944 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -30,17 +30,14 @@ import ( "strings" "time" - "github.com/go-openapi/spec" "github.com/spf13/cobra" extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/sets" utilwait "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/admission" - "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authorization/authorizer" openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" genericfeatures "k8s.io/apiserver/pkg/features" @@ -70,7 +67,6 @@ import ( "k8s.io/kubernetes/cmd/kube-apiserver/app/options" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/capabilities" - serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount" "k8s.io/kubernetes/pkg/features" generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi" "k8s.io/kubernetes/pkg/kubeapiserver" @@ -85,7 +81,6 @@ import ( "k8s.io/kubernetes/pkg/registry/cachesize" rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest" "k8s.io/kubernetes/pkg/serviceaccount" - "k8s.io/kubernetes/plugin/pkg/auth/authenticator/token/bootstrap" ) const ( @@ -440,9 +435,6 @@ func buildGenericConfig( if lastErr = s.SecureServing.ApplyTo(&genericConfig.SecureServing, &genericConfig.LoopbackClientConfig); lastErr != nil { return } - if lastErr = s.Authentication.ApplyTo(genericConfig); lastErr != nil { - return - } if lastErr = s.Features.ApplyTo(genericConfig); lastErr != nil { return } @@ -498,9 +490,8 @@ func buildGenericConfig( } versionedInformers = clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute) - genericConfig.Authentication.Authenticator, genericConfig.OpenAPIConfig.SecurityDefinitions, err = BuildAuthenticator(s, genericConfig.EgressSelector, clientgoExternalClient, versionedInformers) - if err != nil { - lastErr = fmt.Errorf("invalid authentication config: %v", err) + // Authentication.ApplyTo requires already applied OpenAPIConfig and EgressSelector if present + if lastErr = s.Authentication.ApplyTo(&genericConfig.Authentication, genericConfig.SecureServing, genericConfig.EgressSelector, genericConfig.OpenAPIConfig, clientgoExternalClient, versionedInformers); lastErr != nil { return } @@ -559,35 +550,6 @@ func buildGenericConfig( return } -// BuildAuthenticator constructs the authenticator -func BuildAuthenticator(s *options.ServerRunOptions, EgressSelector *egressselector.EgressSelector, extclient clientgoclientset.Interface, versionedInformer clientgoinformers.SharedInformerFactory) (authenticator.Request, *spec.SecurityDefinitions, error) { - authenticatorConfig, err := s.Authentication.ToAuthenticationConfig() - if err != nil { - return nil, nil, err - } - if s.Authentication.ServiceAccounts.Lookup || utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) { - authenticatorConfig.ServiceAccountTokenGetter = serviceaccountcontroller.NewGetterFromClient( - extclient, - versionedInformer.Core().V1().Secrets().Lister(), - versionedInformer.Core().V1().ServiceAccounts().Lister(), - versionedInformer.Core().V1().Pods().Lister(), - ) - } - authenticatorConfig.BootstrapTokenAuthenticator = bootstrap.NewTokenAuthenticator( - versionedInformer.Core().V1().Secrets().Lister().Secrets(v1.NamespaceSystem), - ) - - if EgressSelector != nil { - egressDialer, err := EgressSelector.Lookup(egressselector.Master.AsNetworkContext()) - if err != nil { - return nil, nil, err - } - authenticatorConfig.CustomDial = egressDialer - } - - return authenticatorConfig.New() -} - // BuildAuthorizer constructs the authorizer func BuildAuthorizer(s *options.ServerRunOptions, EgressSelector *egressselector.EgressSelector, versionedInformers clientgoinformers.SharedInformerFactory) (authorizer.Authorizer, authorizer.RuleResolver, error) { authorizationConfig := s.Authorization.ToAuthorizationConfig(versionedInformers) diff --git a/cmd/kubelet/app/auth.go b/cmd/kubelet/app/auth.go index 22a0285d8b9..6eadf29bb1b 100644 --- a/cmd/kubelet/app/auth.go +++ b/cmd/kubelet/app/auth.go @@ -36,7 +36,8 @@ import ( ) // BuildAuth creates an authenticator, an authorizer, and a matching authorizer attributes getter compatible with the kubelet's needs -func BuildAuth(nodeName types.NodeName, client clientset.Interface, config kubeletconfig.KubeletConfiguration) (server.AuthInterface, error) { +// It returns AuthInterface, a run method to start internal controllers (like cert reloading) and error. +func BuildAuth(nodeName types.NodeName, client clientset.Interface, config kubeletconfig.KubeletConfiguration) (server.AuthInterface, func(<-chan struct{}), error) { // Get clients, if provided var ( tokenClient authenticationclient.TokenReviewInterface @@ -47,47 +48,55 @@ func BuildAuth(nodeName types.NodeName, client clientset.Interface, config kubel sarClient = client.AuthorizationV1().SubjectAccessReviews() } - authenticator, err := BuildAuthn(tokenClient, config.Authentication) + authenticator, runAuthenticatorCAReload, err := BuildAuthn(tokenClient, config.Authentication) if err != nil { - return nil, err + return nil, nil, err } attributes := server.NewNodeAuthorizerAttributesGetter(nodeName) authorizer, err := BuildAuthz(sarClient, config.Authorization) if err != nil { - return nil, err + return nil, nil, err } - return server.NewKubeletAuth(authenticator, attributes, authorizer), nil + return server.NewKubeletAuth(authenticator, attributes, authorizer), runAuthenticatorCAReload, nil } // BuildAuthn creates an authenticator compatible with the kubelet's needs -func BuildAuthn(client authenticationclient.TokenReviewInterface, authn kubeletconfig.KubeletAuthentication) (authenticator.Request, error) { - var clientCertificateCAContentProvider authenticatorfactory.CAContentProvider +func BuildAuthn(client authenticationclient.TokenReviewInterface, authn kubeletconfig.KubeletAuthentication) (authenticator.Request, func(<-chan struct{}), error) { + var dynamicCAContentFromFile *dynamiccertificates.DynamicFileCAContent var err error if len(authn.X509.ClientCAFile) > 0 { - clientCertificateCAContentProvider, err = dynamiccertificates.NewDynamicCAContentFromFile("client-ca-bundle", authn.X509.ClientCAFile) + dynamicCAContentFromFile, err = dynamiccertificates.NewDynamicCAContentFromFile("client-ca-bundle", authn.X509.ClientCAFile) if err != nil { - return nil, err + return nil, nil, err } } authenticatorConfig := authenticatorfactory.DelegatingAuthenticatorConfig{ Anonymous: authn.Anonymous.Enabled, CacheTTL: authn.Webhook.CacheTTL.Duration, - ClientCertificateCAContentProvider: clientCertificateCAContentProvider, + ClientCertificateCAContentProvider: dynamicCAContentFromFile, } if authn.Webhook.Enabled { if client == nil { - return nil, errors.New("no client provided, cannot use webhook authentication") + return nil, nil, errors.New("no client provided, cannot use webhook authentication") } authenticatorConfig.TokenAccessReviewClient = client } authenticator, _, err := authenticatorConfig.New() - return authenticator, err + if err != nil { + return nil, nil, err + } + + return authenticator, func(stopCh <-chan struct{}) { + if dynamicCAContentFromFile != nil { + go dynamicCAContentFromFile.Run(1, stopCh) + } + }, err } // BuildAuthz creates an authorizer compatible with the kubelet's needs diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 35b9c8ffba2..531634fb69e 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -599,11 +599,12 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, featureGate f } if kubeDeps.Auth == nil { - auth, err := BuildAuth(nodeName, kubeDeps.KubeClient, s.KubeletConfiguration) + auth, runAuthenticatorCAReload, err := BuildAuth(nodeName, kubeDeps.KubeClient, s.KubeletConfiguration) if err != nil { return err } kubeDeps.Auth = auth + runAuthenticatorCAReload(stopCh) } var cgroupRoots []string diff --git a/pkg/kubeapiserver/options/BUILD b/pkg/kubeapiserver/options/BUILD index 7d8a7d2ac97..f0b9aabbeb8 100644 --- a/pkg/kubeapiserver/options/BUILD +++ b/pkg/kubeapiserver/options/BUILD @@ -14,6 +14,7 @@ go_library( importpath = "k8s.io/kubernetes/pkg/kubeapiserver/options", visibility = ["//visibility:public"], deps = [ + "//pkg/controller/serviceaccount:go_default_library", "//pkg/features:go_default_library", "//pkg/kubeapiserver/authenticator:go_default_library", "//pkg/kubeapiserver/authorizer:go_default_library", @@ -50,6 +51,8 @@ go_library( "//plugin/pkg/admission/storage/persistentvolume/resize:go_default_library", "//plugin/pkg/admission/storage/storageclass/setdefault:go_default_library", "//plugin/pkg/admission/storage/storageobjectinuseprotection:go_default_library", + "//plugin/pkg/auth/authenticator/token/bootstrap:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", @@ -58,14 +61,17 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/server/egressselector:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/component-base/cli/flag:go_default_library", "//staging/src/k8s.io/component-base/featuregate:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", "//vendor/k8s.io/klog:go_default_library", + "//vendor/k8s.io/kube-openapi/pkg/common:go_default_library", ], ) diff --git a/pkg/kubeapiserver/options/authentication.go b/pkg/kubeapiserver/options/authentication.go index 1a432a85c3c..8744c4e6506 100644 --- a/pkg/kubeapiserver/options/authentication.go +++ b/pkg/kubeapiserver/options/authentication.go @@ -24,17 +24,24 @@ import ( "time" "github.com/spf13/pflag" - "k8s.io/klog" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/authentication/authenticator" genericapiserver "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/egressselector" genericoptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" cliflag "k8s.io/component-base/cli/flag" + "k8s.io/klog" + openapicommon "k8s.io/kube-openapi/pkg/common" + serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount" "k8s.io/kubernetes/pkg/features" kubeauthenticator "k8s.io/kubernetes/pkg/kubeapiserver/authenticator" authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" + "k8s.io/kubernetes/plugin/pkg/auth/authenticator/token/bootstrap" ) type BuiltInAuthenticationOptions struct { @@ -406,35 +413,60 @@ func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() (kubeauthenticat return ret, nil } -func (o *BuiltInAuthenticationOptions) ApplyTo(c *genericapiserver.Config) error { +// ApplyTo requires already applied OpenAPIConfig and EgressSelector if present. +func (o *BuiltInAuthenticationOptions) ApplyTo(authInfo *genericapiserver.AuthenticationInfo, secureServing *genericapiserver.SecureServingInfo, egressSelector *egressselector.EgressSelector, openAPIConfig *openapicommon.Config, extclient kubernetes.Interface, versionedInformer informers.SharedInformerFactory) error { if o == nil { return nil } - if o.ClientCert != nil { - clientCertificateCAContentProvider, err := o.ClientCert.GetClientCAContentProvider() - if err != nil { - return fmt.Errorf("unable to load client CA file: %v", err) - } - if err = c.Authentication.ApplyClientCert(clientCertificateCAContentProvider, c.SecureServing); err != nil { + if openAPIConfig == nil { + return errors.New("uninitialized OpenAPIConfig") + } + + authenticatorConfig, err := o.ToAuthenticationConfig() + if err != nil { + return err + } + + if authenticatorConfig.ClientCAContentProvider != nil { + if err = authInfo.ApplyClientCert(authenticatorConfig.ClientCAContentProvider, secureServing); err != nil { return fmt.Errorf("unable to load client CA file: %v", err) } } - if o.RequestHeader != nil { - requestHeaderConfig, err := o.RequestHeader.ToAuthenticationRequestHeaderConfig() - if err != nil { - return fmt.Errorf("unable to create request header authentication config: %v", err) - } - if requestHeaderConfig != nil { - if err = c.Authentication.ApplyClientCert(requestHeaderConfig.CAContentProvider, c.SecureServing); err != nil { - return fmt.Errorf("unable to load client CA file: %v", err) - } + if authenticatorConfig.RequestHeaderConfig != nil && authenticatorConfig.RequestHeaderConfig.CAContentProvider != nil { + if err = authInfo.ApplyClientCert(authenticatorConfig.RequestHeaderConfig.CAContentProvider, secureServing); err != nil { + return fmt.Errorf("unable to load client CA file: %v", err) } } - c.Authentication.APIAudiences = o.APIAudiences + authInfo.APIAudiences = o.APIAudiences if o.ServiceAccounts != nil && o.ServiceAccounts.Issuer != "" && len(o.APIAudiences) == 0 { - c.Authentication.APIAudiences = authenticator.Audiences{o.ServiceAccounts.Issuer} + authInfo.APIAudiences = authenticator.Audiences{o.ServiceAccounts.Issuer} + } + + if o.ServiceAccounts.Lookup || utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) { + authenticatorConfig.ServiceAccountTokenGetter = serviceaccountcontroller.NewGetterFromClient( + extclient, + versionedInformer.Core().V1().Secrets().Lister(), + versionedInformer.Core().V1().ServiceAccounts().Lister(), + versionedInformer.Core().V1().Pods().Lister(), + ) + } + authenticatorConfig.BootstrapTokenAuthenticator = bootstrap.NewTokenAuthenticator( + versionedInformer.Core().V1().Secrets().Lister().Secrets(metav1.NamespaceSystem), + ) + + if egressSelector != nil { + egressDialer, err := egressSelector.Lookup(egressselector.Master.AsNetworkContext()) + if err != nil { + return err + } + authenticatorConfig.CustomDial = egressDialer + } + + authInfo.Authenticator, openAPIConfig.SecurityDefinitions, err = authenticatorConfig.New() + if err != nil { + return err } return nil diff --git a/test/integration/apiserver/certreload/BUILD b/test/integration/apiserver/certreload/BUILD index b1e7faa529d..287a1ad40a2 100644 --- a/test/integration/apiserver/certreload/BUILD +++ b/test/integration/apiserver/certreload/BUILD @@ -9,11 +9,14 @@ go_test( tags = ["integration"], deps = [ "//cmd/kube-apiserver/app/options:go_default_library", + "//staging/src/k8s.io/api/authorization/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", + "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/util/cert:go_default_library", "//staging/src/k8s.io/component-base/cli/flag:go_default_library", "//test/integration/framework:go_default_library", diff --git a/test/integration/apiserver/certreload/certreload_test.go b/test/integration/apiserver/certreload/certreload_test.go index f35d263203f..e12bb9e6ce6 100644 --- a/test/integration/apiserver/certreload/certreload_test.go +++ b/test/integration/apiserver/certreload/certreload_test.go @@ -19,63 +19,150 @@ package podlogs import ( "bytes" "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" "fmt" "io/ioutil" + "math/big" "path" "strings" "testing" "time" + authorizationv1 "k8s.io/api/authorization/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/server/dynamiccertificates" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" "k8s.io/client-go/util/cert" "k8s.io/component-base/cli/flag" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" "k8s.io/kubernetes/test/integration/framework" ) +type caWithClient struct { + CACert []byte + ClientCert []byte + ClientKey []byte +} + +func newTestCAWithClient(caSubject pkix.Name, caSerial *big.Int, clientSubject pkix.Name, subjectSerial *big.Int) (*caWithClient, error) { + ca := &x509.Certificate{ + SerialNumber: caSerial, + Subject: caSubject, + NotBefore: time.Now(), + NotAfter: time.Now().Add(24 * time.Hour), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + caPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return nil, err + } + + caBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, &caPrivateKey.PublicKey, caPrivateKey) + if err != nil { + return nil, err + } + + caPEM := new(bytes.Buffer) + err = pem.Encode(caPEM, &pem.Block{ + Type: "CERTIFICATE", + Bytes: caBytes, + }) + if err != nil { + return nil, err + } + + clientCert := &x509.Certificate{ + SerialNumber: subjectSerial, + Subject: clientSubject, + NotBefore: time.Now(), + NotAfter: time.Now().Add(24 * time.Hour), + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + KeyUsage: x509.KeyUsageDigitalSignature, + } + + clientCertPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return nil, err + } + + clientCertPrivateKeyPEM := new(bytes.Buffer) + err = pem.Encode(clientCertPrivateKeyPEM, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(clientCertPrivateKey), + }) + if err != nil { + return nil, err + } + + clientCertBytes, err := x509.CreateCertificate(rand.Reader, clientCert, ca, &clientCertPrivateKey.PublicKey, caPrivateKey) + if err != nil { + return nil, err + } + + clientCertPEM := new(bytes.Buffer) + err = pem.Encode(clientCertPEM, &pem.Block{ + Type: "CERTIFICATE", + Bytes: clientCertBytes, + }) + if err != nil { + return nil, err + } + + return &caWithClient{ + CACert: caPEM.Bytes(), + ClientCert: clientCertPEM.Bytes(), + ClientKey: clientCertPrivateKeyPEM.Bytes(), + }, nil +} + func TestClientCA(t *testing.T) { stopCh := make(chan struct{}) defer close(stopCh) - // I have no idea what this cert is, but it doesn't matter, we just want something that always fails validation - differentClientCA := []byte(`-----BEGIN CERTIFICATE----- -MIIDQDCCAiigAwIBAgIJANWw74P5KJk2MA0GCSqGSIb3DQEBCwUAMDQxMjAwBgNV -BAMMKWdlbmVyaWNfd2ViaG9va19hZG1pc3Npb25fcGx1Z2luX3Rlc3RzX2NhMCAX -DTE3MTExNjAwMDUzOVoYDzIyOTEwOTAxMDAwNTM5WjAjMSEwHwYDVQQDExh3ZWJo -b29rLXRlc3QuZGVmYXVsdC5zdmMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK -AoIBAQDXd/nQ89a5H8ifEsigmMd01Ib6NVR3bkJjtkvYnTbdfYEBj7UzqOQtHoLa -dIVmefny5uIHvj93WD8WDVPB3jX2JHrXkDTXd/6o6jIXHcsUfFTVLp6/bZ+Anqe0 -r/7hAPkzA2A7APyTWM3ZbEeo1afXogXhOJ1u/wz0DflgcB21gNho4kKTONXO3NHD -XLpspFqSkxfEfKVDJaYAoMnYZJtFNsa2OvsmLnhYF8bjeT3i07lfwrhUZvP+7Gsp -7UgUwc06WuNHjfx1s5e6ySzH0QioMD1rjYneqOvk0pKrMIhuAEWXqq7jlXcDtx1E -j+wnYbVqqVYheHZ8BCJoVAAQGs9/AgMBAAGjZDBiMAkGA1UdEwQCMAAwCwYDVR0P -BAQDAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATApBgNVHREEIjAg -hwR/AAABghh3ZWJob29rLXRlc3QuZGVmYXVsdC5zdmMwDQYJKoZIhvcNAQELBQAD -ggEBAD/GKSPNyQuAOw/jsYZesb+RMedbkzs18sSwlxAJQMUrrXwlVdHrA8q5WhE6 -ABLqU1b8lQ8AWun07R8k5tqTmNvCARrAPRUqls/ryER+3Y9YEcxEaTc3jKNZFLbc -T6YtcnkdhxsiO136wtiuatpYL91RgCmuSpR8+7jEHhuFU01iaASu7ypFrUzrKHTF -bKwiLRQi1cMzVcLErq5CDEKiKhUkoDucyARFszrGt9vNIl/YCcBOkcNvM3c05Hn3 -M++C29JwS3Hwbubg6WO3wjFjoEhpCwU6qRYUz3MRp4tHO4kxKXx+oQnUiFnR7vW0 -YkNtGc1RUDHwecCTFpJtPb7Yu/E= ------END CERTIFICATE----- -`) - differentFrontProxyCA := []byte(`-----BEGIN CERTIFICATE----- -MIIBqDCCAU2gAwIBAgIUfbqeieihh/oERbfvRm38XvS/xHAwCgYIKoZIzj0EAwIw -GjEYMBYGA1UEAxMPSW50ZXJtZWRpYXRlLUNBMCAXDTE2MTAxMTA1MDYwMFoYDzIx -MTYwOTE3MDUwNjAwWjAUMRIwEAYDVQQDEwlNeSBDbGllbnQwWTATBgcqhkjOPQIB -BggqhkjOPQMBBwNCAARv6N4R/sjMR65iMFGNLN1GC/vd7WhDW6J4X/iAjkRLLnNb -KbRG/AtOUZ+7upJ3BWIRKYbOabbQGQe2BbKFiap4o3UwczAOBgNVHQ8BAf8EBAMC -BaAwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU -K/pZOWpNcYai6eHFpmJEeFpeQlEwHwYDVR0jBBgwFoAUX6nQlxjfWnP6aM1meO/Q -a6b3a9kwCgYIKoZIzj0EAwIDSQAwRgIhAIWTKw/sjJITqeuNzJDAKU4xo1zL+xJ5 -MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps= ------END CERTIFICATE----- + frontProxyCA, err := newTestCAWithClient( + pkix.Name{ + CommonName: "test-front-proxy-ca", + }, + big.NewInt(43), + pkix.Name{ + CommonName: "test-aggregated-apiserver", + Organization: []string{"system:masters"}, + }, + big.NewInt(86), + ) + if err != nil { + t.Error(err) + return + } + + clientCA, err := newTestCAWithClient( + pkix.Name{ + CommonName: "test-client-ca", + }, + big.NewInt(42), + pkix.Name{ + CommonName: "system:admin", + Organization: []string{"system:masters"}, + }, + big.NewInt(84), + ) + if err != nil { + t.Error(err) + return + } -`) clientCAFilename := "" frontProxyCAFilename := "" @@ -84,12 +171,13 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps= opts.GenericServerRunOptions.MaxRequestBodyBytes = 1024 * 1024 clientCAFilename = opts.Authentication.ClientCert.ClientCA frontProxyCAFilename = opts.Authentication.RequestHeader.ClientCAFile + opts.Authentication.RequestHeader.AllowedNames = append(opts.Authentication.RequestHeader.AllowedNames, "test-aggregated-apiserver") dynamiccertificates.FileRefreshDuration = 1 * time.Second }, }) // wait for request header info - err := wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "requestheader-client-ca-file", "-----BEGIN CERTIFICATE-----", 1)) + err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "requestheader-client-ca-file", "-----BEGIN CERTIFICATE-----", 1)) if err != nil { t.Fatal(err) } @@ -100,10 +188,10 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps= } // when we run this the second time, we know which one we are expecting - if err := ioutil.WriteFile(clientCAFilename, differentClientCA, 0644); err != nil { + if err := ioutil.WriteFile(clientCAFilename, clientCA.CACert, 0644); err != nil { t.Fatal(err) } - if err := ioutil.WriteFile(frontProxyCAFilename, differentFrontProxyCA, 0644); err != nil { + if err := ioutil.WriteFile(frontProxyCAFilename, frontProxyCA.CACert, 0644); err != nil { t.Fatal(err) } @@ -114,7 +202,7 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps= t.Fatal(err) } - expectedCAs := []string{"webhook-test.default.svc", "My Client"} + expectedCAs := []string{"test-client-ca", "test-front-proxy-ca"} if len(expectedCAs) != len(acceptableCAs) { t.Fatal(strings.Join(acceptableCAs, ":")) } @@ -129,7 +217,7 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps= if err != nil { t.Error(err) } - err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "requestheader-client-ca-file", "MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps=", 1)) + err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "requestheader-client-ca-file", string(frontProxyCA.CACert), 1)) if err != nil { t.Error(err) } @@ -138,7 +226,68 @@ MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps= if err != nil { t.Error(err) } - err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "client-ca-file", "M++C29JwS3Hwbubg6WO3wjFjoEhpCwU6qRYUz3MRp4tHO4kxKXx+oQnUiFnR7vW0", 1)) + err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, waitForConfigMapCAContent(t, kubeClient, "client-ca-file", string(clientCA.CACert), 1)) + if err != nil { + t.Error(err) + } + + // Test an aggregated apiserver client (signed by the new front proxy CA) is authorized + extensionApiserverClient, err := kubernetes.NewForConfig(&rest.Config{ + Host: kubeconfig.Host, + TLSClientConfig: rest.TLSClientConfig{ + CAData: kubeconfig.TLSClientConfig.CAData, + CAFile: kubeconfig.TLSClientConfig.CAFile, + ServerName: kubeconfig.TLSClientConfig.ServerName, + KeyData: frontProxyCA.ClientKey, + CertData: frontProxyCA.ClientCert, + }, + }) + if err != nil { + t.Error(err) + return + } + + // Call an endpoint to make sure we are authenticated + err = extensionApiserverClient.AuthorizationV1().RESTClient(). + Post(). + Resource("subjectaccessreviews"). + VersionedParams(&metav1.CreateOptions{}, scheme.ParameterCodec). + Body(&authorizationv1.SubjectAccessReview{ + Spec: authorizationv1.SubjectAccessReviewSpec{ + ResourceAttributes: &authorizationv1.ResourceAttributes{ + Verb: "create", + Resource: "pods", + Namespace: "default", + }, + User: "deads2k", + }, + }). + SetHeader("X-Remote-User", "test-aggregated-apiserver"). + SetHeader("X-Remote-Group", "system:masters"). + Do(context.Background()). + Into(&authorizationv1.SubjectAccessReview{}) + if err != nil { + t.Error(err) + } + + // Test a client signed by the new ClientCA is authorized + testClient, err := kubernetes.NewForConfig(&rest.Config{ + Host: kubeconfig.Host, + TLSClientConfig: rest.TLSClientConfig{ + CAData: kubeconfig.TLSClientConfig.CAData, + CAFile: kubeconfig.TLSClientConfig.CAFile, + ServerName: kubeconfig.TLSClientConfig.ServerName, + KeyData: clientCA.ClientKey, + CertData: clientCA.ClientCert, + }, + }) + if err != nil { + t.Error(err) + return + } + + // Call an endpoint to make sure we are authenticated + _, err = testClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) if err != nil { t.Error(err) }