From ed442cc3dd479dd383532a2e77c6c71087763bf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20Tyczy=C5=84ski?= Date: Fri, 10 Jun 2022 20:17:25 +0200 Subject: [PATCH] Clean(er) shutdown of auth integration tests --- test/integration/auth/accessreview_test.go | 53 ++++++----- test/integration/auth/bootstraptoken_test.go | 27 ++++-- test/integration/auth/rbac_test.go | 93 +++++++++++++------- 3 files changed, 107 insertions(+), 66 deletions(-) diff --git a/test/integration/auth/accessreview_test.go b/test/integration/auth/accessreview_test.go index 9b083509d30..e621821d630 100644 --- a/test/integration/auth/accessreview_test.go +++ b/test/integration/auth/accessreview_test.go @@ -28,9 +28,8 @@ import ( "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/authorization/authorizer" - clientset "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" api "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/kubernetes/pkg/controlplane" "k8s.io/kubernetes/test/integration/framework" ) @@ -57,13 +56,15 @@ func alwaysAlice(req *http.Request) (*authenticator.Response, bool, error) { } func TestSubjectAccessReview(t *testing.T) { - controlPlaneConfig := framework.NewIntegrationTestControlPlaneConfig() - controlPlaneConfig.GenericConfig.Authentication.Authenticator = authenticator.RequestFunc(alwaysAlice) - controlPlaneConfig.GenericConfig.Authorization.Authorizer = sarAuthorizer{} - _, s, closeFn := framework.RunAnAPIServer(controlPlaneConfig) - defer closeFn() - - clientset := clientset.NewForConfigOrDie(&restclient.Config{Host: s.URL}) + clientset, _, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{ + ModifyServerConfig: func(config *controlplane.Config) { + // Unset BearerToken to disable BearerToken authenticator. + config.GenericConfig.LoopbackClientConfig.BearerToken = "" + config.GenericConfig.Authentication.Authenticator = authenticator.RequestFunc(alwaysAlice) + config.GenericConfig.Authorization.Authorizer = sarAuthorizer{} + }, + }) + defer tearDownFn() tests := []struct { name string @@ -148,8 +149,7 @@ func TestSubjectAccessReview(t *testing.T) { func TestSelfSubjectAccessReview(t *testing.T) { username := "alice" - controlPlaneConfig := framework.NewIntegrationTestControlPlaneConfig() - controlPlaneConfig.GenericConfig.Authentication.Authenticator = authenticator.RequestFunc(func(req *http.Request) (*authenticator.Response, bool, error) { + authenticatorFunc := func(req *http.Request) (*authenticator.Response, bool, error) { return &authenticator.Response{ User: &user.DefaultInfo{ Name: username, @@ -157,12 +157,17 @@ func TestSelfSubjectAccessReview(t *testing.T) { Groups: []string{user.AllAuthenticated}, }, }, true, nil - }) - controlPlaneConfig.GenericConfig.Authorization.Authorizer = sarAuthorizer{} - _, s, closeFn := framework.RunAnAPIServer(controlPlaneConfig) - defer closeFn() + } - clientset := clientset.NewForConfigOrDie(&restclient.Config{Host: s.URL}) + clientset, _, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{ + ModifyServerConfig: func(config *controlplane.Config) { + // Unset BearerToken to disable BearerToken authenticator. + config.GenericConfig.LoopbackClientConfig.BearerToken = "" + config.GenericConfig.Authentication.Authenticator = authenticator.RequestFunc(authenticatorFunc) + config.GenericConfig.Authorization.Authorizer = sarAuthorizer{} + }, + }) + defer tearDownFn() tests := []struct { name string @@ -235,13 +240,15 @@ func TestSelfSubjectAccessReview(t *testing.T) { } func TestLocalSubjectAccessReview(t *testing.T) { - controlPlaneConfig := framework.NewIntegrationTestControlPlaneConfig() - controlPlaneConfig.GenericConfig.Authentication.Authenticator = authenticator.RequestFunc(alwaysAlice) - controlPlaneConfig.GenericConfig.Authorization.Authorizer = sarAuthorizer{} - _, s, closeFn := framework.RunAnAPIServer(controlPlaneConfig) - defer closeFn() - - clientset := clientset.NewForConfigOrDie(&restclient.Config{Host: s.URL}) + clientset, _, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{ + ModifyServerConfig: func(config *controlplane.Config) { + // Unset BearerToken to disable BearerToken authenticator. + config.GenericConfig.LoopbackClientConfig.BearerToken = "" + config.GenericConfig.Authentication.Authenticator = authenticator.RequestFunc(alwaysAlice) + config.GenericConfig.Authorization.Authorizer = sarAuthorizer{} + }, + }) + defer tearDownFn() tests := []struct { name string diff --git a/test/integration/auth/bootstraptoken_test.go b/test/integration/auth/bootstraptoken_test.go index 97d2d764392..c538cdeb1bb 100644 --- a/test/integration/auth/bootstraptoken_test.go +++ b/test/integration/auth/bootstraptoken_test.go @@ -29,7 +29,10 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apiserver/pkg/authentication/group" "k8s.io/apiserver/pkg/authentication/request/bearertoken" + "k8s.io/apiserver/pkg/authorization/authorizerfactory" + "k8s.io/client-go/rest" bootstrapapi "k8s.io/cluster-bootstrap/token/api" + "k8s.io/kubernetes/pkg/controlplane" "k8s.io/kubernetes/plugin/pkg/auth/authenticator/token/bootstrap" "k8s.io/kubernetes/test/integration" "k8s.io/kubernetes/test/integration/framework" @@ -117,17 +120,23 @@ func TestBootstrapTokenAuth(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { authenticator := group.NewAuthenticatedGroupAdder(bearertoken.New(bootstrap.NewTokenAuthenticator(bootstrapSecrets{test.secret}))) - // Set up an API server - controlPlaneConfig := framework.NewIntegrationTestControlPlaneConfig() - controlPlaneConfig.GenericConfig.Authentication.Authenticator = authenticator - _, s, closeFn := framework.RunAnAPIServer(controlPlaneConfig) - defer closeFn() - ns := framework.CreateTestingNamespace("auth-bootstrap-token", t) - defer framework.DeleteTestingNamespace(ns, t) + kubeClient, kubeConfig, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{ + ModifyServerConfig: func(config *controlplane.Config) { + config.GenericConfig.Authentication.Authenticator = authenticator + config.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer() + }, + }) + defer tearDownFn() + + ns := framework.CreateNamespaceOrDie(kubeClient, "auth-bootstrap-token", t) + defer framework.DeleteNamespaceOrDie(kubeClient, ns, t) previousResourceVersion := make(map[string]float64) - transport := http.DefaultTransport + transport, err := rest.TransportFor(kubeConfig) + if err != nil { + t.Fatal(err) + } token := validTokenID + "." + validSecret var bodyStr string @@ -144,7 +153,7 @@ func TestBootstrapTokenAuth(t *testing.T) { } test.request.body = bodyStr bodyBytes := bytes.NewReader([]byte(bodyStr)) - req, err := http.NewRequest(test.request.verb, s.URL+test.request.URL, bodyBytes) + req, err := http.NewRequest(test.request.verb, kubeConfig.Host+test.request.URL, bodyBytes) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/test/integration/auth/rbac_test.go b/test/integration/auth/rbac_test.go index 0cda9b5e149..74fa05cfad5 100644 --- a/test/integration/auth/rbac_test.go +++ b/test/integration/auth/rbac_test.go @@ -47,7 +47,7 @@ import ( "k8s.io/client-go/transport" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/klog/v2" - "k8s.io/kubernetes/pkg/api/legacyscheme" + "k8s.io/kubernetes/cmd/kube-apiserver/app/options" rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1" "k8s.io/kubernetes/pkg/controlplane" "k8s.io/kubernetes/pkg/registry/rbac/clusterrole" @@ -62,11 +62,11 @@ import ( "k8s.io/kubernetes/test/integration/framework" ) -func clientForToken(user string) *http.Client { +func clientForToken(user string, rt http.RoundTripper) *http.Client { return &http.Client{ Transport: transport.NewBearerAuthRoundTripper( user, - transport.DebugWrappers(http.DefaultTransport), + transport.DebugWrappers(rt), ), } } @@ -519,10 +519,7 @@ func TestRBAC(t *testing.T) { for i, tc := range tests { t.Run(fmt.Sprintf("case-%d", i), func(t *testing.T) { - // Create an API Server. - controlPlaneConfig := framework.NewIntegrationTestControlPlaneConfig() - controlPlaneConfig.GenericConfig.Authorization.Authorizer = newRBACAuthorizer(t, controlPlaneConfig) - controlPlaneConfig.GenericConfig.Authentication.Authenticator = group.NewAuthenticatedGroupAdder(bearertoken.New(tokenfile.New(map[string]*user.DefaultInfo{ + authenticator := group.NewAuthenticatedGroupAdder(bearertoken.New(tokenfile.New(map[string]*user.DefaultInfo{ superUser: {Name: "admin", Groups: []string{"system:masters"}}, "any-rolebinding-writer": {Name: "any-rolebinding-writer"}, "any-rolebinding-writer-namespace": {Name: "any-rolebinding-writer-namespace"}, @@ -535,14 +532,28 @@ func TestRBAC(t *testing.T) { "limitrange-patcher": {Name: "limitrange-patcher"}, "user-with-no-permissions": {Name: "user-with-no-permissions"}, }))) - controlPlaneConfig.GenericConfig.OpenAPIConfig = framework.DefaultOpenAPIConfig() - _, s, closeFn := framework.RunAnAPIServer(controlPlaneConfig) - defer closeFn() - clientConfig := &restclient.Config{Host: s.URL, ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}} + _, kubeConfig, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{ + ModifyServerRunOptions: func(opts *options.ServerRunOptions) { + // Disable ServiceAccount admission plugin as we don't have serviceaccount controller running. + // Also disable namespace lifecycle to workaroung the test limitation that first creates + // roles/rolebindings and only then creates corresponding namespaces. + opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount", "NamespaceLifecycle"} + }, + ModifyServerConfig: func(config *controlplane.Config) { + config.GenericConfig.Authentication.Authenticator = authenticator + config.GenericConfig.Authorization.Authorizer = newRBACAuthorizer(t, config) + }, + }) + defer tearDownFn() + + transport, err := restclient.TransportFor(kubeConfig) + if err != nil { + t.Fatal(err) + } // Bootstrap the API Server with the test case's initial roles. - superuserClient, _ := clientsetForToken(superUser, clientConfig) + superuserClient, _ := clientsetForToken(superUser, kubeConfig) if err := tc.bootstrapRoles.bootstrap(superuserClient); err != nil { t.Errorf("case %d: failed to apply initial roles: %v", i, err) return @@ -578,7 +589,7 @@ func TestRBAC(t *testing.T) { body = strings.NewReader(fmt.Sprintf(r.body, sub)) } - req, err := http.NewRequest(r.verb, s.URL+path, body) + req, err := http.NewRequest(r.verb, kubeConfig.Host+path, body) if r.verb == "PATCH" { // For patch operations, use the apply content type req.Header.Add("Content-Type", string(types.ApplyPatchType)) @@ -598,7 +609,7 @@ func TestRBAC(t *testing.T) { return } - resp, err := clientForToken(r.token).Do(req) + resp, err := clientForToken(r.token, transport).Do(req) if err != nil { t.Errorf("case %d, req %d: failed to make request: %v", i, j, err) return @@ -644,15 +655,15 @@ func TestRBAC(t *testing.T) { func TestBootstrapping(t *testing.T) { superUser := "admin/system:masters" - controlPlaneConfig := framework.NewIntegrationTestControlPlaneConfig() - controlPlaneConfig.GenericConfig.Authorization.Authorizer = newRBACAuthorizer(t, controlPlaneConfig) - controlPlaneConfig.GenericConfig.Authentication.Authenticator = bearertoken.New(tokenfile.New(map[string]*user.DefaultInfo{ - superUser: {Name: "admin", Groups: []string{"system:masters"}}, - })) - _, s, closeFn := framework.RunAnAPIServer(controlPlaneConfig) - defer closeFn() - - clientset := clientset.NewForConfigOrDie(&restclient.Config{BearerToken: superUser, Host: s.URL}) + clientset, _, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{ + ModifyServerConfig: func(config *controlplane.Config) { + config.GenericConfig.Authentication.Authenticator = bearertoken.New(tokenfile.New(map[string]*user.DefaultInfo{ + superUser: {Name: "admin", Groups: []string{"system:masters"}}, + })) + config.GenericConfig.Authorization.Authorizer = newRBACAuthorizer(t, config) + }, + }) + defer tearDownFn() watcher, err := clientset.RbacV1().ClusterRoles().Watch(context.TODO(), metav1.ListOptions{ResourceVersion: "0"}) if err != nil { @@ -705,14 +716,19 @@ func TestDiscoveryUpgradeBootstrapping(t *testing.T) { superUser := "admin/system:masters" - controlPlaneConfig := framework.NewIntegrationTestControlPlaneConfig() - controlPlaneConfig.GenericConfig.Authorization.Authorizer = newRBACAuthorizer(t, controlPlaneConfig) - controlPlaneConfig.GenericConfig.Authentication.Authenticator = bearertoken.New(tokenfile.New(map[string]*user.DefaultInfo{ - superUser: {Name: "admin", Groups: []string{"system:masters"}}, - })) - _, s, tearDownFn := framework.RunAnAPIServer(controlPlaneConfig) - - client := clientset.NewForConfigOrDie(&restclient.Config{BearerToken: superUser, Host: s.URL}) + etcdConfig := framework.SharedEtcd() + client, _, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{ + ModifyServerRunOptions: func(opts *options.ServerRunOptions) { + // Ensure we're using the same etcd across apiserver restarts. + opts.Etcd.StorageConfig = *etcdConfig + }, + ModifyServerConfig: func(config *controlplane.Config) { + config.GenericConfig.Authentication.Authenticator = bearertoken.New(tokenfile.New(map[string]*user.DefaultInfo{ + superUser: {Name: "admin", Groups: []string{"system:masters"}}, + })) + config.GenericConfig.Authorization.Authorizer = newRBACAuthorizer(t, config) + }, + }) // Modify the default RBAC discovery ClusterRoleBidnings to look more like the defaults that // existed prior to v1.14, but with user modifications. @@ -754,9 +770,18 @@ func TestDiscoveryUpgradeBootstrapping(t *testing.T) { // Check that upgraded API servers inherit `system:public-info-viewer` settings from // `system:discovery`, and respect auto-reconciliation annotations. - _, s, tearDownFn = framework.RunAnAPIServer(controlPlaneConfig) - - client = clientset.NewForConfigOrDie(&restclient.Config{BearerToken: superUser, Host: s.URL}) + client, _, tearDownFn = framework.StartTestServer(t, framework.TestServerSetup{ + ModifyServerRunOptions: func(opts *options.ServerRunOptions) { + // Ensure we're using the same etcd across apiserver restarts. + opts.Etcd.StorageConfig = *etcdConfig + }, + ModifyServerConfig: func(config *controlplane.Config) { + config.GenericConfig.Authentication.Authenticator = bearertoken.New(tokenfile.New(map[string]*user.DefaultInfo{ + superUser: {Name: "admin", Groups: []string{"system:masters"}}, + })) + config.GenericConfig.Authorization.Authorizer = newRBACAuthorizer(t, config) + }, + }) newDiscRoleBinding, err := client.RbacV1().ClusterRoleBindings().Get(context.TODO(), "system:discovery", metav1.GetOptions{}) if err != nil {