From 925900317e43e58435082f624f5969e3cfe25c67 Mon Sep 17 00:00:00 2001 From: Shihang Zhang Date: Thu, 15 Apr 2021 09:50:43 -0700 Subject: [PATCH] allow multiple of --service-account-issuer --- cmd/kube-apiserver/app/options/validation.go | 2 +- cmd/kube-apiserver/app/server.go | 6 +- cmd/kube-apiserver/app/testing/testserver.go | 25 +++++ pkg/kubeapiserver/authenticator/config.go | 12 +-- pkg/kubeapiserver/options/authentication.go | 41 ++++++--- .../options/authentication_test.go | 92 ++++++++++++------- pkg/serviceaccount/jwt.go | 16 ++-- pkg/serviceaccount/jwt_test.go | 24 ++++- .../authenticatorfactory/loopback.go | 4 +- .../src/k8s.io/apiserver/pkg/server/config.go | 2 +- test/e2e_node/services/apiserver.go | 31 ++++++- .../admissionwebhook/admission_test.go | 24 +++++ test/integration/auth/dynamic_client_test.go | 2 +- test/integration/auth/svcaccttoken_test.go | 3 +- test/integration/etcd/server.go | 25 +++++ test/integration/framework/master_utils.go | 2 +- test/integration/framework/test_server.go | 26 ++++++ .../serviceaccount/service_account_test.go | 2 +- 18 files changed, 267 insertions(+), 72 deletions(-) diff --git a/cmd/kube-apiserver/app/options/validation.go b/cmd/kube-apiserver/app/options/validation.go index f60d8b49c71..6bf7b9d8a87 100644 --- a/cmd/kube-apiserver/app/options/validation.go +++ b/cmd/kube-apiserver/app/options/validation.go @@ -113,7 +113,7 @@ func validateTokenRequest(options *ServerRunOptions) []error { var errs []error enableAttempted := options.ServiceAccountSigningKeyFile != "" || - options.Authentication.ServiceAccounts.Issuer != "" || + (len(options.Authentication.ServiceAccounts.Issuers) != 0 && options.Authentication.ServiceAccounts.Issuers[0] != "") || len(options.Authentication.APIAudiences) != 0 enableSucceeded := options.ServiceAccountIssuer != nil diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 4968a253759..52ebac843c8 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -416,7 +416,7 @@ func CreateKubeAPIServerConfig( pubKeys = append(pubKeys, keys...) } // Plumb the required metadata through ExtraConfig. - config.ExtraConfig.ServiceAccountIssuerURL = s.Authentication.ServiceAccounts.Issuer + config.ExtraConfig.ServiceAccountIssuerURL = s.Authentication.ServiceAccounts.Issuers[0] config.ExtraConfig.ServiceAccountJWKSURI = s.Authentication.ServiceAccounts.JWKSURI config.ExtraConfig.ServiceAccountPublicKeys = pubKeys @@ -633,7 +633,7 @@ func Complete(s *options.ServerRunOptions) (completedServerRunOptions, error) { } } - if s.ServiceAccountSigningKeyFile != "" && s.Authentication.ServiceAccounts.Issuer != "" { + if s.ServiceAccountSigningKeyFile != "" && len(s.Authentication.ServiceAccounts.Issuers) != 0 && s.Authentication.ServiceAccounts.Issuers[0] != "" { sk, err := keyutil.PrivateKeyFromFile(s.ServiceAccountSigningKeyFile) if err != nil { return options, fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err) @@ -655,7 +655,7 @@ func Complete(s *options.ServerRunOptions) (completedServerRunOptions, error) { } } - s.ServiceAccountIssuer, err = serviceaccount.JWTTokenGenerator(s.Authentication.ServiceAccounts.Issuer, sk) + s.ServiceAccountIssuer, err = serviceaccount.JWTTokenGenerator(s.Authentication.ServiceAccounts.Issuers[0], sk) if err != nil { return options, fmt.Errorf("failed to build token generator: %v", err) } diff --git a/cmd/kube-apiserver/app/testing/testserver.go b/cmd/kube-apiserver/app/testing/testserver.go index 89d370f8346..672031f3202 100644 --- a/cmd/kube-apiserver/app/testing/testserver.go +++ b/cmd/kube-apiserver/app/testing/testserver.go @@ -34,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/storage/storagebackend" @@ -47,6 +48,13 @@ import ( testutil "k8s.io/kubernetes/test/utils" ) +// This key is for testing purposes only and is not considered secure. +const ecdsaPrivateKey = `-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIEZmTmUhuanLjPA2CLquXivuwBDHTt5XYwgIr/kA1LtRoAoGCCqGSM49 +AwEHoUQDQgAEH6cuzP8XuD5wal6wf9M6xDljTOPLX2i8uIp/C/ASqiIGUeeKQtX0 +/IR3qCXyThP/dbCiHrF3v1cuhBOHY8CLVg== +-----END EC PRIVATE KEY-----` + // TearDownFunc is to be called to tear down a test server. type TearDownFunc func() @@ -182,11 +190,28 @@ func StartTestServer(t Logger, instanceOptions *TestServerInstanceOptions, custo if err := fs.Parse(customFlags); err != nil { return result, err } + + saSigningKeyFile, err := ioutil.TempFile("/tmp", "insecure_test_key") + if err != nil { + t.Fatalf("create temp file failed: %v", err) + } + defer os.RemoveAll(saSigningKeyFile.Name()) + if err = ioutil.WriteFile(saSigningKeyFile.Name(), []byte(ecdsaPrivateKey), 0666); err != nil { + t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err) + } + s.ServiceAccountSigningKeyFile = saSigningKeyFile.Name() + s.Authentication.ServiceAccounts.Issuers = []string{"https://foo.bar.example.com"} + s.Authentication.ServiceAccounts.KeyFiles = []string{saSigningKeyFile.Name()} + completedOptions, err := app.Complete(s) if err != nil { return result, fmt.Errorf("failed to set default ServerRunOptions: %v", err) } + if errs := completedOptions.Validate(); len(errs) != 0 { + return result, fmt.Errorf("failed to validate ServerRunOptions: %v", utilerrors.NewAggregate(errs)) + } + t.Logf("runtime-config=%v", completedOptions.APIEnablement.RuntimeConfig) t.Logf("Starting kube-apiserver on port %d...", s.SecureServing.BindPort) server, err := app.CreateServerChain(completedOptions, stopCh) diff --git a/pkg/kubeapiserver/authenticator/config.go b/pkg/kubeapiserver/authenticator/config.go index ce4be0fda3b..17b556ea48a 100644 --- a/pkg/kubeapiserver/authenticator/config.go +++ b/pkg/kubeapiserver/authenticator/config.go @@ -63,7 +63,7 @@ type Config struct { OIDCRequiredClaims map[string]string ServiceAccountKeyFiles []string ServiceAccountLookup bool - ServiceAccountIssuer string + ServiceAccountIssuers []string APIAudiences authenticator.Audiences WebhookTokenAuthnConfigFile string WebhookTokenAuthnVersion string @@ -131,8 +131,8 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, er } tokenAuthenticators = append(tokenAuthenticators, serviceAccountAuth) } - if config.ServiceAccountIssuer != "" { - serviceAccountAuth, err := newServiceAccountAuthenticator(config.ServiceAccountIssuer, config.ServiceAccountKeyFiles, config.APIAudiences, config.ServiceAccountTokenGetter) + if len(config.ServiceAccountIssuers) > 0 { + serviceAccountAuth, err := newServiceAccountAuthenticator(config.ServiceAccountIssuers, config.ServiceAccountKeyFiles, config.APIAudiences, config.ServiceAccountTokenGetter) if err != nil { return nil, nil, err } @@ -276,12 +276,12 @@ func newLegacyServiceAccountAuthenticator(keyfiles []string, lookup bool, apiAud allPublicKeys = append(allPublicKeys, publicKeys...) } - tokenAuthenticator := serviceaccount.JWTTokenAuthenticator(serviceaccount.LegacyIssuer, allPublicKeys, apiAudiences, serviceaccount.NewLegacyValidator(lookup, serviceAccountGetter)) + tokenAuthenticator := serviceaccount.JWTTokenAuthenticator([]string{serviceaccount.LegacyIssuer}, allPublicKeys, apiAudiences, serviceaccount.NewLegacyValidator(lookup, serviceAccountGetter)) return tokenAuthenticator, nil } // newServiceAccountAuthenticator returns an authenticator.Token or an error -func newServiceAccountAuthenticator(iss string, keyfiles []string, apiAudiences authenticator.Audiences, serviceAccountGetter serviceaccount.ServiceAccountTokenGetter) (authenticator.Token, error) { +func newServiceAccountAuthenticator(issuers []string, keyfiles []string, apiAudiences authenticator.Audiences, serviceAccountGetter serviceaccount.ServiceAccountTokenGetter) (authenticator.Token, error) { allPublicKeys := []interface{}{} for _, keyfile := range keyfiles { publicKeys, err := keyutil.PublicKeysFromFile(keyfile) @@ -291,7 +291,7 @@ func newServiceAccountAuthenticator(iss string, keyfiles []string, apiAudiences allPublicKeys = append(allPublicKeys, publicKeys...) } - tokenAuthenticator := serviceaccount.JWTTokenAuthenticator(iss, allPublicKeys, apiAudiences, serviceaccount.NewValidator(serviceAccountGetter)) + tokenAuthenticator := serviceaccount.JWTTokenAuthenticator(issuers, allPublicKeys, apiAudiences, serviceaccount.NewValidator(serviceAccountGetter)) return tokenAuthenticator, nil } diff --git a/pkg/kubeapiserver/options/authentication.go b/pkg/kubeapiserver/options/authentication.go index d28a3b94ea8..550f3d86e51 100644 --- a/pkg/kubeapiserver/options/authentication.go +++ b/pkg/kubeapiserver/options/authentication.go @@ -87,7 +87,7 @@ type OIDCAuthenticationOptions struct { type ServiceAccountAuthenticationOptions struct { KeyFiles []string Lookup bool - Issuer string + Issuers []string JWKSURI string MaxExpiration time.Duration ExtendExpiration bool @@ -191,14 +191,29 @@ func (o *BuiltInAuthenticationOptions) Validate() []error { allErrors = append(allErrors, fmt.Errorf("oidc-issuer-url and oidc-client-id should be specified together")) } - if o.ServiceAccounts != nil && len(o.ServiceAccounts.Issuer) > 0 && strings.Contains(o.ServiceAccounts.Issuer, ":") { - if _, err := url.Parse(o.ServiceAccounts.Issuer); err != nil { - allErrors = append(allErrors, fmt.Errorf("service-account-issuer contained a ':' but was not a valid URL: %v", err)) + if o.ServiceAccounts != nil && len(o.ServiceAccounts.Issuers) > 0 { + seen := make(map[string]bool) + for _, issuer := range o.ServiceAccounts.Issuers { + if strings.Contains(issuer, ":") { + if _, err := url.Parse(issuer); err != nil { + allErrors = append(allErrors, fmt.Errorf("service-account-issuer %q contained a ':' but was not a valid URL: %v", issuer, err)) + continue + } + } + if issuer == "" { + allErrors = append(allErrors, fmt.Errorf("service-account-issuer should not be an empty string")) + continue + } + if seen[issuer] { + allErrors = append(allErrors, fmt.Errorf("service-account-issuer %q is already specified", issuer)) + continue + } + seen[issuer] = true } } if o.ServiceAccounts != nil { - if len(o.ServiceAccounts.Issuer) == 0 { + if len(o.ServiceAccounts.Issuers) == 0 { allErrors = append(allErrors, errors.New("service-account-issuer is a required flag")) } if len(o.ServiceAccounts.KeyFiles) == 0 { @@ -308,7 +323,7 @@ func (o *BuiltInAuthenticationOptions) AddFlags(fs *pflag.FlagSet) { fs.BoolVar(&o.ServiceAccounts.Lookup, "service-account-lookup", o.ServiceAccounts.Lookup, "If true, validate ServiceAccount tokens exist in etcd as part of authentication.") - fs.StringVar(&o.ServiceAccounts.Issuer, "service-account-issuer", o.ServiceAccounts.Issuer, ""+ + fs.StringArrayVar(&o.ServiceAccounts.Issuers, "service-account-issuer", o.ServiceAccounts.Issuers, ""+ "Identifier of the service account token issuer. The issuer will assert this identifier "+ "in \"iss\" claim of issued tokens. This value is a string or URI. If this option is not "+ "a valid URI per the OpenID Discovery 1.0 spec, the ServiceAccountIssuerDiscovery feature "+ @@ -316,7 +331,9 @@ func (o *BuiltInAuthenticationOptions) AddFlags(fs *pflag.FlagSet) { "that this value comply with the OpenID spec: https://openid.net/specs/openid-connect-discovery-1_0.html. "+ "In practice, this means that service-account-issuer must be an https URL. It is also highly "+ "recommended that this URL be capable of serving OpenID discovery documents at "+ - "{service-account-issuer}/.well-known/openid-configuration.") + "{service-account-issuer}/.well-known/openid-configuration. "+ + "When this flag is specified multiple times, the first is used to generate tokens "+ + "and all are used to determine which issuers are accepted.") fs.StringVar(&o.ServiceAccounts.JWKSURI, "service-account-jwks-uri", o.ServiceAccounts.JWKSURI, ""+ "Overrides the URI for the JSON Web Key Set in the discovery doc served at "+ @@ -406,11 +423,11 @@ func (o *BuiltInAuthenticationOptions) ToAuthenticationConfig() (kubeauthenticat ret.APIAudiences = o.APIAudiences if o.ServiceAccounts != nil { - if o.ServiceAccounts.Issuer != "" && len(o.APIAudiences) == 0 { - ret.APIAudiences = authenticator.Audiences{o.ServiceAccounts.Issuer} + if len(o.ServiceAccounts.Issuers) != 0 && len(o.APIAudiences) == 0 { + ret.APIAudiences = authenticator.Audiences(o.ServiceAccounts.Issuers) } ret.ServiceAccountKeyFiles = o.ServiceAccounts.KeyFiles - ret.ServiceAccountIssuer = o.ServiceAccounts.Issuer + ret.ServiceAccountIssuers = o.ServiceAccounts.Issuers ret.ServiceAccountLookup = o.ServiceAccounts.Lookup } @@ -464,8 +481,8 @@ func (o *BuiltInAuthenticationOptions) ApplyTo(authInfo *genericapiserver.Authen } authInfo.APIAudiences = o.APIAudiences - if o.ServiceAccounts != nil && o.ServiceAccounts.Issuer != "" && len(o.APIAudiences) == 0 { - authInfo.APIAudiences = authenticator.Audiences{o.ServiceAccounts.Issuer} + if o.ServiceAccounts != nil && len(o.ServiceAccounts.Issuers) != 0 && len(o.APIAudiences) == 0 { + authInfo.APIAudiences = authenticator.Audiences(o.ServiceAccounts.Issuers) } authenticatorConfig.ServiceAccountTokenGetter = serviceaccountcontroller.NewGetterFromClient( diff --git a/pkg/kubeapiserver/options/authentication_test.go b/pkg/kubeapiserver/options/authentication_test.go index c50a69d339d..a431e3a8acf 100644 --- a/pkg/kubeapiserver/options/authentication_test.go +++ b/pkg/kubeapiserver/options/authentication_test.go @@ -51,34 +51,10 @@ func TestAuthenticationValidate(t *testing.T) { ClientID: "testClientID", }, testSA: &ServiceAccountAuthenticationOptions{ - Issuer: "http://foo.bar.com", + Issuers: []string{"http://foo.bar.com"}, KeyFiles: []string{"testkeyfile1", "testkeyfile2"}, }, }, - { - name: "test when OIDC and ServiceAccounts are invalid", - testOIDC: &OIDCAuthenticationOptions{ - UsernameClaim: "sub", - SigningAlgs: []string{"RS256"}, - IssuerURL: "testIssuerURL", - }, - testSA: &ServiceAccountAuthenticationOptions{ - Issuer: "http://foo.bar.com", - }, - expectErr: "oidc-issuer-url and oidc-client-id should be specified together", - }, - { - name: "test when OIDC and ServiceAccounts are invalid", - testOIDC: &OIDCAuthenticationOptions{ - UsernameClaim: "sub", - SigningAlgs: []string{"RS256"}, - IssuerURL: "testIssuerURL", - }, - testSA: &ServiceAccountAuthenticationOptions{ - Issuer: "http://foo.bar.com", - }, - expectErr: "service-account-key-file is a required flag", - }, { name: "test when OIDC is invalid", testOIDC: &OIDCAuthenticationOptions{ @@ -87,13 +63,13 @@ func TestAuthenticationValidate(t *testing.T) { IssuerURL: "testIssuerURL", }, testSA: &ServiceAccountAuthenticationOptions{ - Issuer: "http://foo.bar.com", + Issuers: []string{"http://foo.bar.com"}, KeyFiles: []string{"testkeyfile1", "testkeyfile2"}, }, expectErr: "oidc-issuer-url and oidc-client-id should be specified together", }, { - name: "test when ServiceAccount is invalid", + name: "test when ServiceAccounts doesn't have key file", testOIDC: &OIDCAuthenticationOptions{ UsernameClaim: "sub", SigningAlgs: []string{"RS256"}, @@ -101,9 +77,61 @@ func TestAuthenticationValidate(t *testing.T) { ClientID: "testClientID", }, testSA: &ServiceAccountAuthenticationOptions{ - Issuer: "http://[::1]:namedport", + Issuers: []string{"http://foo.bar.com"}, }, - expectErr: "service-account-issuer contained a ':' but was not a valid URL", + expectErr: "service-account-key-file is a required flag", + }, + { + name: "test when ServiceAccounts doesn't have issuer", + testOIDC: &OIDCAuthenticationOptions{ + UsernameClaim: "sub", + SigningAlgs: []string{"RS256"}, + IssuerURL: "testIssuerURL", + ClientID: "testClientID", + }, + testSA: &ServiceAccountAuthenticationOptions{ + Issuers: []string{}, + }, + expectErr: "service-account-issuer is a required flag", + }, + { + name: "test when ServiceAccounts has empty string as issuer", + testOIDC: &OIDCAuthenticationOptions{ + UsernameClaim: "sub", + SigningAlgs: []string{"RS256"}, + IssuerURL: "testIssuerURL", + ClientID: "testClientID", + }, + testSA: &ServiceAccountAuthenticationOptions{ + Issuers: []string{""}, + }, + expectErr: "service-account-issuer should not be an empty string", + }, + { + name: "test when ServiceAccounts has duplicate issuers", + testOIDC: &OIDCAuthenticationOptions{ + UsernameClaim: "sub", + SigningAlgs: []string{"RS256"}, + IssuerURL: "testIssuerURL", + ClientID: "testClientID", + }, + testSA: &ServiceAccountAuthenticationOptions{ + Issuers: []string{"http://foo.bar.com", "http://foo.bar.com"}, + }, + expectErr: "service-account-issuer \"http://foo.bar.com\" is already specified", + }, + { + name: "test when ServiceAccount has bad issuer", + testOIDC: &OIDCAuthenticationOptions{ + UsernameClaim: "sub", + SigningAlgs: []string{"RS256"}, + IssuerURL: "testIssuerURL", + ClientID: "testClientID", + }, + testSA: &ServiceAccountAuthenticationOptions{ + Issuers: []string{"http://[::1]:namedport"}, + }, + expectErr: "service-account-issuer \"http://[::1]:namedport\" contained a ':' but was not a valid URL", }, } @@ -154,8 +182,8 @@ func TestToAuthenticationConfig(t *testing.T) { AllowedNames: []string{"kube-aggregator"}, }, ServiceAccounts: &ServiceAccountAuthenticationOptions{ - Lookup: true, - Issuer: "http://foo.bar.com", + Lookup: true, + Issuers: []string{"http://foo.bar.com"}, }, TokenFile: &TokenFileAuthenticationOptions{ TokenFile: "/testTokenFile", @@ -176,7 +204,7 @@ func TestToAuthenticationConfig(t *testing.T) { OIDCUsernameClaim: "sub", OIDCSigningAlgs: []string{"RS256"}, ServiceAccountLookup: true, - ServiceAccountIssuer: "http://foo.bar.com", + ServiceAccountIssuers: []string{"http://foo.bar.com"}, WebhookTokenAuthnConfigFile: "/token-webhook-config", WebhookTokenAuthnCacheTTL: 180000000000, diff --git a/pkg/serviceaccount/jwt.go b/pkg/serviceaccount/jwt.go index c779957bdbe..6722b206d1d 100644 --- a/pkg/serviceaccount/jwt.go +++ b/pkg/serviceaccount/jwt.go @@ -224,9 +224,13 @@ func (j *jwtTokenGenerator) GenerateToken(claims *jwt.Claims, privateClaims inte // JWTTokenAuthenticator authenticates tokens as JWT tokens produced by JWTTokenGenerator // Token signatures are verified using each of the given public keys until one works (allowing key rotation) // If lookup is true, the service account and secret referenced as claims inside the token are retrieved and verified with the provided ServiceAccountTokenGetter -func JWTTokenAuthenticator(iss string, keys []interface{}, implicitAuds authenticator.Audiences, validator Validator) authenticator.Token { +func JWTTokenAuthenticator(issuers []string, keys []interface{}, implicitAuds authenticator.Audiences, validator Validator) authenticator.Token { + issuersMap := make(map[string]bool) + for _, issuer := range issuers { + issuersMap[issuer] = true + } return &jwtTokenAuthenticator{ - iss: iss, + issuers: issuersMap, keys: keys, implicitAuds: implicitAuds, validator: validator, @@ -234,7 +238,7 @@ func JWTTokenAuthenticator(iss string, keys []interface{}, implicitAuds authenti } type jwtTokenAuthenticator struct { - iss string + issuers map[string]bool keys []interface{} validator Validator implicitAuds authenticator.Audiences @@ -340,9 +344,5 @@ func (j *jwtTokenAuthenticator) hasCorrectIssuer(tokenData string) bool { if err := json.Unmarshal(payload, &claims); err != nil { return false } - if claims.Issuer != j.iss { - return false - } - return true - + return j.issuers[claims.Issuer] } diff --git a/pkg/serviceaccount/jwt_test.go b/pkg/serviceaccount/jwt_test.go index aecfb23a48d..ded059f412e 100644 --- a/pkg/serviceaccount/jwt_test.go +++ b/pkg/serviceaccount/jwt_test.go @@ -193,7 +193,7 @@ func TestTokenGenerateAndValidate(t *testing.T) { checkJSONWebSignatureHasKeyID(t, ecdsaToken, ecdsaKeyID) - // Generate signer with same keys as RSA signer but different issuer + // Generate signer with same keys as RSA signer but different unrecognized issuer badIssuerGenerator, err := serviceaccount.JWTTokenGenerator("foo", getPrivateKey(rsaPrivateKey)) if err != nil { t.Fatalf("error making generator: %v", err) @@ -203,6 +203,16 @@ func TestTokenGenerateAndValidate(t *testing.T) { t.Fatalf("error generating token: %v", err) } + // Generate signer with same keys as RSA signer but different recognized issuer + differentIssuerGenerator, err := serviceaccount.JWTTokenGenerator("bar", getPrivateKey(rsaPrivateKey)) + if err != nil { + t.Fatalf("error making generator: %v", err) + } + differentIssuerToken, err := differentIssuerGenerator.GenerateToken(serviceaccount.LegacyClaims(*serviceAccount, *rsaSecret)) + if err != nil { + t.Fatalf("error generating token: %v", err) + } + testCases := map[string]struct { Client clientset.Interface Keys []interface{} @@ -252,6 +262,16 @@ func TestTokenGenerateAndValidate(t *testing.T) { ExpectedErr: false, ExpectedOK: false, }, + "valid key, different issuer (rsa)": { + Token: differentIssuerToken, + Client: nil, + Keys: []interface{}{getPublicKey(rsaPublicKey)}, + ExpectedErr: false, + ExpectedOK: true, + ExpectedUserName: expectedUserName, + ExpectedUserUID: expectedUserUID, + ExpectedGroups: []string{"system:serviceaccounts", "system:serviceaccounts:test"}, + }, "valid key (ecdsa)": { Token: ecdsaToken, Client: nil, @@ -322,7 +342,7 @@ func TestTokenGenerateAndValidate(t *testing.T) { return tc.Client.CoreV1().Pods(namespace).Get(context.TODO(), name, metav1.GetOptions{}) })), ) - authn := serviceaccount.JWTTokenAuthenticator(serviceaccount.LegacyIssuer, tc.Keys, auds, serviceaccount.NewLegacyValidator(tc.Client != nil, getter)) + authn := serviceaccount.JWTTokenAuthenticator([]string{serviceaccount.LegacyIssuer, "bar"}, tc.Keys, auds, serviceaccount.NewLegacyValidator(tc.Client != nil, getter)) // An invalid, non-JWT token should always fail ctx := authenticator.WithAudiences(context.Background(), auds) diff --git a/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/loopback.go b/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/loopback.go index f31656529fe..fe51afcbc24 100644 --- a/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/loopback.go +++ b/staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/loopback.go @@ -24,6 +24,6 @@ import ( ) // NewFromTokens returns an authenticator.Request or an error -func NewFromTokens(tokens map[string]*user.DefaultInfo) authenticator.Request { - return bearertoken.New(tokenfile.New(tokens)) +func NewFromTokens(tokens map[string]*user.DefaultInfo, audiences authenticator.Audiences) authenticator.Request { + return bearertoken.New(authenticator.WrapAudienceAgnosticToken(audiences, tokenfile.New(tokens))) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index 12d18796e78..5536e8caccf 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -858,7 +858,7 @@ func AuthorizeClientBearerToken(loopback *restclient.Config, authn *Authenticati Groups: []string{user.SystemPrivilegedGroup}, } - tokenAuthenticator := authenticatorfactory.NewFromTokens(tokens) + tokenAuthenticator := authenticatorfactory.NewFromTokens(tokens, authn.APIAudiences) authn.Authenticator = authenticatorunion.New(tokenAuthenticator, authn.Authenticator) tokenAuthorizer := authorizerfactory.NewPrivilegedGroups(user.SystemPrivilegedGroup) diff --git a/test/e2e_node/services/apiserver.go b/test/e2e_node/services/apiserver.go index fe2ff8c69db..4a37dd18995 100644 --- a/test/e2e_node/services/apiserver.go +++ b/test/e2e_node/services/apiserver.go @@ -20,15 +20,25 @@ import ( "fmt" "io/ioutil" "net" + "os" "k8s.io/apiserver/pkg/storage/storagebackend" + utilerrors "k8s.io/apimachinery/pkg/util/errors" apiserver "k8s.io/kubernetes/cmd/kube-apiserver/app" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" "k8s.io/kubernetes/test/e2e/framework" ) -const clusterIPRange = "10.0.0.1/24" +const ( + clusterIPRange = "10.0.0.1/24" + // This key is for testing purposes only and is not considered secure. + ecdsaPrivateKey = `-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIEZmTmUhuanLjPA2CLquXivuwBDHTt5XYwgIr/kA1LtRoAoGCCqGSM49 +AwEHoUQDQgAEH6cuzP8XuD5wal6wf9M6xDljTOPLX2i8uIp/C/ASqiIGUeeKQtX0 +/IR3qCXyThP/dbCiHrF3v1cuhBOHY8CLVg== +-----END EC PRIVATE KEY-----` +) // APIServer is a server which manages apiserver. type APIServer struct { @@ -65,6 +75,20 @@ func (a *APIServer) Start() error { } o.Authentication.TokenFile.TokenFile = tokenFilePath o.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount", "TaintNodesByCondition"} + + saSigningKeyFile, err := ioutil.TempFile("/tmp", "insecure_test_key") + if err != nil { + return fmt.Errorf("create temp file failed: %v", err) + } + defer os.RemoveAll(saSigningKeyFile.Name()) + if err = ioutil.WriteFile(saSigningKeyFile.Name(), []byte(ecdsaPrivateKey), 0666); err != nil { + return fmt.Errorf("write file %s failed: %v", saSigningKeyFile.Name(), err) + } + o.ServiceAccountSigningKeyFile = saSigningKeyFile.Name() + o.Authentication.APIAudiences = []string{"https://foo.bar.example.com"} + o.Authentication.ServiceAccounts.Issuers = []string{"https://foo.bar.example.com"} + o.Authentication.ServiceAccounts.KeyFiles = []string{saSigningKeyFile.Name()} + errCh := make(chan error) go func() { defer close(errCh) @@ -73,6 +97,11 @@ func (a *APIServer) Start() error { errCh <- fmt.Errorf("set apiserver default options error: %v", err) return } + if errs := completedOptions.Validate(); len(errs) != 0 { + errCh <- fmt.Errorf("failed to validate ServerRunOptions: %v", utilerrors.NewAggregate(errs)) + return + } + err = apiserver.Run(completedOptions, a.stopCh) if err != nil { errCh <- fmt.Errorf("run apiserver error: %v", err) diff --git a/test/integration/apiserver/admissionwebhook/admission_test.go b/test/integration/apiserver/admissionwebhook/admission_test.go index 08b87f93b5c..eb40bc183d6 100644 --- a/test/integration/apiserver/admissionwebhook/admission_test.go +++ b/test/integration/apiserver/admissionwebhook/admission_test.go @@ -39,6 +39,7 @@ import ( admissionv1 "k8s.io/api/admissionregistration/v1" admissionv1beta1 "k8s.io/api/admissionregistration/v1beta1" appsv1beta1 "k8s.io/api/apps/v1beta1" + authenticationv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" extensionsv1beta1 "k8s.io/api/extensions/v1beta1" @@ -127,6 +128,8 @@ var ( gvr("", "v1", "pods/proxy"): {"*": testSubresourceProxy}, gvr("", "v1", "services/proxy"): {"*": testSubresourceProxy}, + gvr("", "v1", "serviceaccounts/token"): {"create": testTokenCreate}, + gvr("random.numbers.com", "v1", "integers"): {"create": testPruningRandomNumbers}, gvr("custom.fancy.com", "v2", "pants"): {"create": testNoPruningCustomFancy}, } @@ -882,6 +885,27 @@ func getParentGVR(gvr schema.GroupVersionResource) schema.GroupVersionResource { return parentGVR } +func testTokenCreate(c *testContext) { + saGVR := gvr("", "v1", "serviceaccounts") + sa, err := createOrGetResource(c.client, saGVR, c.resources[saGVR]) + if err != nil { + c.t.Error(err) + return + } + + c.admissionHolder.expect(c.gvr, gvk(c.resource.Group, c.resource.Version, c.resource.Kind), gvkCreateOptions, v1beta1.Create, sa.GetName(), sa.GetNamespace(), true, false, true) + if err = c.clientset.CoreV1().RESTClient().Post().Namespace(sa.GetNamespace()).Resource("serviceaccounts").Name(sa.GetName()).SubResource("token").Body(&authenticationv1.TokenRequest{ + ObjectMeta: metav1.ObjectMeta{Name: sa.GetName()}, + Spec: authenticationv1.TokenRequestSpec{ + Audiences: []string{"api"}, + }, + }).Do(context.TODO()).Error(); err != nil { + c.t.Error(err) + return + } + c.admissionHolder.verify(c.t) +} + func testSubresourceUpdate(c *testContext) { if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { parentGVR := getParentGVR(c.gvr) diff --git a/test/integration/auth/dynamic_client_test.go b/test/integration/auth/dynamic_client_test.go index 629a0b3a8d2..90b156881d2 100644 --- a/test/integration/auth/dynamic_client_test.go +++ b/test/integration/auth/dynamic_client_test.go @@ -69,7 +69,7 @@ func TestDynamicClientBuilder(t *testing.T) { if opts.Authentication.ServiceAccounts == nil { opts.Authentication.ServiceAccounts = &kubeoptions.ServiceAccountAuthenticationOptions{} } - opts.Authentication.ServiceAccounts.Issuer = iss + opts.Authentication.ServiceAccounts.Issuers = []string{iss} opts.Authentication.ServiceAccounts.KeyFiles = []string{tmpfile.Name()} }, ModifyServerConfig: func(config *controlplane.Config) { diff --git a/test/integration/auth/svcaccttoken_test.go b/test/integration/auth/svcaccttoken_test.go index 6499ba93424..68412d54e4d 100644 --- a/test/integration/auth/svcaccttoken_test.go +++ b/test/integration/auth/svcaccttoken_test.go @@ -54,6 +54,7 @@ import ( "k8s.io/kubernetes/test/integration/framework" ) +// This key is for testing purposes only and is not considered secure. const ecdsaPrivateKey = `-----BEGIN EC PRIVATE KEY----- MHcCAQEEIEZmTmUhuanLjPA2CLquXivuwBDHTt5XYwgIr/kA1LtRoAoGCCqGSM49 AwEHoUQDQgAEH6cuzP8XuD5wal6wf9M6xDljTOPLX2i8uIp/C/ASqiIGUeeKQtX0 @@ -87,7 +88,7 @@ func TestServiceAccountTokenCreate(t *testing.T) { masterConfig.GenericConfig.Authentication.APIAudiences = aud masterConfig.GenericConfig.Authentication.Authenticator = bearertoken.New( serviceaccount.JWTTokenAuthenticator( - iss, + []string{iss}, []interface{}{&pk}, aud, serviceaccount.NewValidator(serviceaccountgetter.NewGetterFromClient( diff --git a/test/integration/etcd/server.go b/test/integration/etcd/server.go index cd9b544f8ee..01fa0723d44 100644 --- a/test/integration/etcd/server.go +++ b/test/integration/etcd/server.go @@ -37,6 +37,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/wait" genericapiserveroptions "k8s.io/apiserver/pkg/server/options" cacheddiscovery "k8s.io/client-go/discovery/cached/memory" @@ -53,6 +54,13 @@ import ( _ "k8s.io/kubernetes/pkg/controlplane" ) +// This key is for testing purposes only and is not considered secure. +const ecdsaPrivateKey = `-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIEZmTmUhuanLjPA2CLquXivuwBDHTt5XYwgIr/kA1LtRoAoGCCqGSM49 +AwEHoUQDQgAEH6cuzP8XuD5wal6wf9M6xDljTOPLX2i8uIp/C/ASqiIGUeeKQtX0 +/IR3qCXyThP/dbCiHrF3v1cuhBOHY8CLVg== +-----END EC PRIVATE KEY-----` + // StartRealMasterOrDie starts an API master that is appropriate for use in tests that require one of every resource func StartRealMasterOrDie(t *testing.T, configFuncs ...func(*options.ServerRunOptions)) *Master { certDir, err := ioutil.TempDir("", t.Name()) @@ -70,12 +78,25 @@ func StartRealMasterOrDie(t *testing.T, configFuncs ...func(*options.ServerRunOp t.Fatal(err) } + saSigningKeyFile, err := ioutil.TempFile("/tmp", "insecure_test_key") + if err != nil { + t.Fatalf("create temp file failed: %v", err) + } + defer os.RemoveAll(saSigningKeyFile.Name()) + if err = ioutil.WriteFile(saSigningKeyFile.Name(), []byte(ecdsaPrivateKey), 0666); err != nil { + t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err) + } + kubeAPIServerOptions := options.NewServerRunOptions() kubeAPIServerOptions.SecureServing.Listener = listener kubeAPIServerOptions.SecureServing.ServerCert.CertDirectory = certDir + kubeAPIServerOptions.ServiceAccountSigningKeyFile = saSigningKeyFile.Name() kubeAPIServerOptions.Etcd.StorageConfig.Transport.ServerList = []string{framework.GetEtcdURL()} kubeAPIServerOptions.Etcd.DefaultStorageMediaType = runtime.ContentTypeJSON // force json we can easily interpret the result in etcd kubeAPIServerOptions.ServiceClusterIPRanges = defaultServiceClusterIPRange.String() + kubeAPIServerOptions.Authentication.APIAudiences = []string{"https://foo.bar.example.com"} + kubeAPIServerOptions.Authentication.ServiceAccounts.Issuers = []string{"https://foo.bar.example.com"} + kubeAPIServerOptions.Authentication.ServiceAccounts.KeyFiles = []string{saSigningKeyFile.Name()} kubeAPIServerOptions.Authorization.Modes = []string{"RBAC"} kubeAPIServerOptions.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"} kubeAPIServerOptions.APIEnablement.RuntimeConfig["api/all"] = "true" @@ -87,6 +108,10 @@ func StartRealMasterOrDie(t *testing.T, configFuncs ...func(*options.ServerRunOp t.Fatal(err) } + if errs := completedOptions.Validate(); len(errs) != 0 { + t.Fatalf("failed to validate ServerRunOptions: %v", utilerrors.NewAggregate(errs)) + } + // get etcd client before starting API server rawClient, kvClient, err := integration.GetEtcdClients(completedOptions.Etcd.StorageConfig.Transport) if err != nil { diff --git a/test/integration/framework/master_utils.go b/test/integration/framework/master_utils.go index 54609fa0296..e54d7ee87b5 100644 --- a/test/integration/framework/master_utils.go +++ b/test/integration/framework/master_utils.go @@ -172,7 +172,7 @@ func startApiserverOrDie(controlPlaneConfig *controlplane.Config, incomingServer Groups: []string{user.SystemPrivilegedGroup}, } - tokenAuthenticator := authenticatorfactory.NewFromTokens(tokens) + tokenAuthenticator := authenticatorfactory.NewFromTokens(tokens, controlPlaneConfig.GenericConfig.Authentication.APIAudiences) if controlPlaneConfig.GenericConfig.Authentication.Authenticator == nil { controlPlaneConfig.GenericConfig.Authentication.Authenticator = authenticatorunion.New(tokenAuthenticator, authauthenticator.RequestFunc(alwaysEmpty)) } else { diff --git a/test/integration/framework/test_server.go b/test/integration/framework/test_server.go index 2f36a64ae2f..68c0bc67390 100644 --- a/test/integration/framework/test_server.go +++ b/test/integration/framework/test_server.go @@ -29,6 +29,7 @@ import ( "github.com/google/uuid" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/wait" genericapiserver "k8s.io/apiserver/pkg/server" genericapiserveroptions "k8s.io/apiserver/pkg/server/options" @@ -41,6 +42,13 @@ import ( "k8s.io/kubernetes/test/utils" ) +// This key is for testing purposes only and is not considered secure. +const ecdsaPrivateKey = `-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIEZmTmUhuanLjPA2CLquXivuwBDHTt5XYwgIr/kA1LtRoAoGCCqGSM49 +AwEHoUQDQgAEH6cuzP8XuD5wal6wf9M6xDljTOPLX2i8uIp/C/ASqiIGUeeKQtX0 +/IR3qCXyThP/dbCiHrF3v1cuhBOHY8CLVg== +-----END EC PRIVATE KEY-----` + // TestServerSetup holds configuration information for a kube-apiserver test server. type TestServerSetup struct { ModifyServerRunOptions func(*options.ServerRunOptions) @@ -86,10 +94,20 @@ func StartTestServer(t *testing.T, stopCh <-chan struct{}, setup TestServerSetup t.Fatal(err) } + saSigningKeyFile, err := ioutil.TempFile("/tmp", "insecure_test_key") + if err != nil { + t.Fatalf("create temp file failed: %v", err) + } + defer os.RemoveAll(saSigningKeyFile.Name()) + if err = ioutil.WriteFile(saSigningKeyFile.Name(), []byte(ecdsaPrivateKey), 0666); err != nil { + t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err) + } + kubeAPIServerOptions := options.NewServerRunOptions() kubeAPIServerOptions.SecureServing.Listener = listener kubeAPIServerOptions.SecureServing.BindAddress = net.ParseIP("127.0.0.1") kubeAPIServerOptions.SecureServing.ServerCert.CertDirectory = certDir + kubeAPIServerOptions.ServiceAccountSigningKeyFile = saSigningKeyFile.Name() kubeAPIServerOptions.Etcd.StorageConfig.Prefix = path.Join("/", uuid.New().String(), "registry") kubeAPIServerOptions.Etcd.StorageConfig.Transport.ServerList = []string{GetEtcdURL()} kubeAPIServerOptions.ServiceClusterIPRanges = defaultServiceClusterIPRange.String() @@ -98,6 +116,9 @@ func StartTestServer(t *testing.T, stopCh <-chan struct{}, setup TestServerSetup kubeAPIServerOptions.Authentication.RequestHeader.ExtraHeaderPrefixes = []string{"X-Remote-Extra-"} kubeAPIServerOptions.Authentication.RequestHeader.AllowedNames = []string{"kube-aggregator"} kubeAPIServerOptions.Authentication.RequestHeader.ClientCAFile = proxyCACertFile.Name() + kubeAPIServerOptions.Authentication.APIAudiences = []string{"https://foo.bar.example.com"} + kubeAPIServerOptions.Authentication.ServiceAccounts.Issuers = []string{"https://foo.bar.example.com"} + kubeAPIServerOptions.Authentication.ServiceAccounts.KeyFiles = []string{saSigningKeyFile.Name()} kubeAPIServerOptions.Authentication.ClientCert.ClientCA = clientCACertFile.Name() kubeAPIServerOptions.Authorization.Modes = []string{"Node", "RBAC"} @@ -109,6 +130,11 @@ func StartTestServer(t *testing.T, stopCh <-chan struct{}, setup TestServerSetup if err != nil { t.Fatal(err) } + + if errs := completedOptions.Validate(); len(errs) != 0 { + t.Fatalf("failed to validate ServerRunOptions: %v", utilerrors.NewAggregate(errs)) + } + tunneler, proxyTransport, err := app.CreateNodeDialer(completedOptions) if err != nil { t.Fatal(err) diff --git a/test/integration/serviceaccount/service_account_test.go b/test/integration/serviceaccount/service_account_test.go index 34be3d0b306..fd189d4bf03 100644 --- a/test/integration/serviceaccount/service_account_test.go +++ b/test/integration/serviceaccount/service_account_test.go @@ -395,7 +395,7 @@ func startServiceAccountTestServer(t *testing.T) (*clientset.Clientset, restclie externalInformers.Core().V1().ServiceAccounts().Lister(), externalInformers.Core().V1().Pods().Lister(), ) - serviceAccountTokenAuth := serviceaccount.JWTTokenAuthenticator(serviceaccount.LegacyIssuer, []interface{}{&serviceAccountKey.PublicKey}, nil, serviceaccount.NewLegacyValidator(true, serviceAccountTokenGetter)) + serviceAccountTokenAuth := serviceaccount.JWTTokenAuthenticator([]string{serviceaccount.LegacyIssuer}, []interface{}{&serviceAccountKey.PublicKey}, nil, serviceaccount.NewLegacyValidator(true, serviceAccountTokenGetter)) authenticator := union.New( bearertoken.New(rootTokenAuth), bearertoken.New(serviceAccountTokenAuth),