diff --git a/test/integration/auth/auth_test.go b/test/integration/auth/auth_test.go
index 1800e0e1f69..becba2d72e3 100644
--- a/test/integration/auth/auth_test.go
+++ b/test/integration/auth/auth_test.go
@@ -35,6 +35,7 @@ import (
 	"net/http/httptest"
 	"net/url"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"testing"
@@ -54,11 +55,9 @@ import (
 	"k8s.io/apiserver/pkg/authentication/request/bearertoken"
 	"k8s.io/apiserver/pkg/authentication/serviceaccount"
 	"k8s.io/apiserver/pkg/authentication/token/cache"
-	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
-	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
+	unionauthz "k8s.io/apiserver/pkg/authorization/union"
 	webhookutil "k8s.io/apiserver/pkg/util/webhook"
-	"k8s.io/apiserver/plugin/pkg/authenticator/token/tokentest"
 	"k8s.io/apiserver/plugin/pkg/authenticator/token/webhook"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -69,7 +68,6 @@ import (
 	"k8s.io/kubernetes/pkg/apis/autoscaling"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/apis/extensions"
-	"k8s.io/kubernetes/pkg/auth/authorizer/abac"
 	"k8s.io/kubernetes/pkg/controlplane"
 	"k8s.io/kubernetes/test/integration"
 	"k8s.io/kubernetes/test/integration/authutil"
@@ -82,13 +80,6 @@ const (
 	UnknownToken string = "qwerty" // Not present in token file.
 )
 
-func getTestTokenAuth() authenticator.Request {
-	tokenAuthenticator := tokentest.New()
-	tokenAuthenticator.Tokens[AliceToken] = &user.DefaultInfo{Name: "alice", UID: "1"}
-	tokenAuthenticator.Tokens[BobToken] = &user.DefaultInfo{Name: "bob", UID: "2"}
-	return group.NewGroupAdder(bearertoken.New(tokenAuthenticator), []string{user.AllAuthenticated})
-}
-
 func getTestWebhookTokenAuth(serverURL string, customDial utilnet.DialFunc) (authenticator.Request, error) {
 	kubecfgFile, err := os.CreateTemp("", "webhook-kubecfg")
 	if err != nil {
@@ -464,9 +455,7 @@ func TestAuthModeAlwaysAllow(t *testing.T) {
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
 			opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
-		},
-		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer()
+			opts.Authorization.Modes = []string{"AlwaysAllow"}
 		},
 	})
 	defer tearDownFn()
@@ -570,10 +559,8 @@ func TestAuthModeAlwaysDeny(t *testing.T) {
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
 			opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
-		},
-		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authentication.Authenticator = getTestTokenAuth()
-			config.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysDenyAuthorizer()
+			opts.Authorization.Modes = []string{"AlwaysDeny"}
+			opts.Authentication.TokenFile.TokenFile = "testdata/tokens.csv"
 		},
 	})
 	defer tearDownFn()
@@ -609,17 +596,6 @@ func TestAuthModeAlwaysDeny(t *testing.T) {
 	}
 }
 
-// Inject into control plane an authorizer that uses user info.
-// TODO(etune): remove this test once a more comprehensive built-in authorizer is implemented.
-type allowAliceAuthorizer struct{}
-
-func (allowAliceAuthorizer) Authorize(ctx context.Context, a authorizer.Attributes) (authorizer.Decision, string, error) {
-	if a.GetUser() != nil && a.GetUser().GetName() == "alice" {
-		return authorizer.DecisionAllow, "", nil
-	}
-	return authorizer.DecisionNoOpinion, "I can't allow that.  Go ask alice.", nil
-}
-
 // TestAliceNotForbiddenOrUnauthorized tests a user who is known to
 // the authentication system and authorized to do any actions.
 func TestAliceNotForbiddenOrUnauthorized(t *testing.T) {
@@ -627,10 +603,9 @@ func TestAliceNotForbiddenOrUnauthorized(t *testing.T) {
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
 			opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
-		},
-		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authentication.Authenticator = getTestTokenAuth()
-			config.GenericConfig.Authorization.Authorizer = allowAliceAuthorizer{}
+			opts.Authentication.TokenFile.TokenFile = "testdata/tokens.csv"
+			opts.Authorization.Modes = []string{"ABAC"}
+			opts.Authorization.PolicyFile = "testdata/allowalice.jsonl"
 		},
 	})
 	defer tearDownFn()
@@ -704,10 +679,9 @@ func TestBobIsForbidden(t *testing.T) {
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
 			opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
-		},
-		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authentication.Authenticator = getTestTokenAuth()
-			config.GenericConfig.Authorization.Authorizer = allowAliceAuthorizer{}
+			opts.Authentication.TokenFile.TokenFile = "testdata/tokens.csv"
+			opts.Authorization.Modes = []string{"ABAC"}
+			opts.Authorization.PolicyFile = "testdata/allowalice.jsonl"
 		},
 	})
 	defer tearDownFn()
@@ -754,10 +728,9 @@ func TestUnknownUserIsUnauthorized(t *testing.T) {
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
 			opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
-		},
-		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authentication.Authenticator = getTestTokenAuth()
-			config.GenericConfig.Authorization.Authorizer = allowAliceAuthorizer{}
+			opts.Authentication.TokenFile.TokenFile = "testdata/tokens.csv"
+			opts.Authorization.Modes = []string{"ABAC"}
+			opts.Authorization.PolicyFile = "testdata/allowalice.jsonl"
 		},
 	})
 	defer tearDownFn()
@@ -807,10 +780,13 @@ func (impersonateAuthorizer) Authorize(ctx context.Context, a authorizer.Attribu
 	if a.GetUser() != nil && a.GetUser().GetName() == "alice" && a.GetVerb() != "impersonate" {
 		return authorizer.DecisionAllow, "", nil
 	}
-	// bob can impersonate anyone, but that it
+	// bob can impersonate anyone, but that's it
 	if a.GetUser() != nil && a.GetUser().GetName() == "bob" && a.GetVerb() == "impersonate" {
 		return authorizer.DecisionAllow, "", nil
 	}
+	if a.GetUser() != nil && a.GetUser().GetName() == "bob" && a.GetVerb() != "impersonate" {
+		return authorizer.DecisionDeny, "", nil
+	}
 	// service accounts can do everything
 	if a.GetUser() != nil && strings.HasPrefix(a.GetUser().GetName(), serviceaccount.ServiceAccountUsernamePrefix) {
 		return authorizer.DecisionAllow, "", nil
@@ -824,10 +800,11 @@ func TestImpersonateIsForbidden(t *testing.T) {
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
 			opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
+			opts.Authentication.TokenFile.TokenFile = "testdata/tokens.csv"
 		},
 		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authentication.Authenticator = getTestTokenAuth()
-			config.GenericConfig.Authorization.Authorizer = impersonateAuthorizer{}
+			// Prepend an impersonation authorizer with specific opinions about alice and bob
+			config.GenericConfig.Authorization.Authorizer = unionauthz.New(impersonateAuthorizer{}, config.GenericConfig.Authorization.Authorizer)
 		},
 	})
 	defer tearDownFn()
@@ -860,7 +837,7 @@ func TestImpersonateIsForbidden(t *testing.T) {
 			// Expect all of bob's actions to return Forbidden
 			if resp.StatusCode != http.StatusForbidden {
 				t.Logf("case %v", r)
-				t.Errorf("Expected not status Forbidden, but got %s", resp.Status)
+				t.Errorf("Expected status Forbidden, but got %s", resp.Status)
 			}
 		}()
 	}
@@ -1101,23 +1078,13 @@ func csrPEM(t *testing.T) []byte {
 	return req
 }
 
-func newAuthorizerWithContents(t *testing.T, contents string) authorizer.Authorizer {
-	f, err := os.CreateTemp("", "auth_test")
-	if err != nil {
-		t.Fatalf("unexpected error creating policyfile: %v", err)
-	}
-	f.Close()
-	defer os.Remove(f.Name())
-
-	if err := os.WriteFile(f.Name(), []byte(contents), 0700); err != nil {
+func newABACFileWithContents(t *testing.T, contents string) string {
+	dir := t.TempDir()
+	file := filepath.Join(dir, "auth_test")
+	if err := os.WriteFile(file, []byte(contents), 0700); err != nil {
 		t.Fatalf("unexpected error writing policyfile: %v", err)
 	}
-
-	pl, err := abac.NewFromFile(f.Name())
-	if err != nil {
-		t.Fatalf("unexpected error creating authorizer from policyfile: %v", err)
-	}
-	return pl
+	return file
 }
 
 type trackingAuthorizer struct {
@@ -1137,10 +1104,10 @@ func TestAuthorizationAttributeDetermination(t *testing.T) {
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
 			opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
+			opts.Authentication.TokenFile.TokenFile = "testdata/tokens.csv"
 		},
 		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authentication.Authenticator = getTestTokenAuth()
-			config.GenericConfig.Authorization.Authorizer = trackingAuthorizer
+			config.GenericConfig.Authorization.Authorizer = unionauthz.New(config.GenericConfig.Authorization.Authorizer, trackingAuthorizer)
 		},
 	})
 	defer tearDownFn()
@@ -1203,18 +1170,13 @@ func TestAuthorizationAttributeDetermination(t *testing.T) {
 // TestNamespaceAuthorization tests that authorization can be controlled
 // by namespace.
 func TestNamespaceAuthorization(t *testing.T) {
-	// This file has alice and bob in it.
-	a := newAuthorizerWithContents(t, `{"namespace": "auth-namespace"}
-`)
-
 	kubeClient, kubeConfig, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
 			opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
-		},
-		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authentication.Authenticator = getTestTokenAuth()
-			config.GenericConfig.Authorization.Authorizer = a
+			opts.Authentication.TokenFile.TokenFile = "testdata/tokens.csv"
+			opts.Authorization.PolicyFile = newABACFileWithContents(t, `{"namespace": "auth-namespace"}`)
+			opts.Authorization.Modes = []string{"ABAC"}
 		},
 	})
 	defer tearDownFn()
@@ -1309,18 +1271,13 @@ func TestNamespaceAuthorization(t *testing.T) {
 // TestKindAuthorization tests that authorization can be controlled
 // by namespace.
 func TestKindAuthorization(t *testing.T) {
-	// This file has alice and bob in it.
-	a := newAuthorizerWithContents(t, `{"resource": "services"}
-`)
-
 	kubeClient, kubeConfig, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
 			opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
-		},
-		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authentication.Authenticator = getTestTokenAuth()
-			config.GenericConfig.Authorization.Authorizer = a
+			opts.Authentication.TokenFile.TokenFile = "testdata/tokens.csv"
+			opts.Authorization.PolicyFile = newABACFileWithContents(t, `{"resource": "services"}`)
+			opts.Authorization.Modes = []string{"ABAC"}
 		},
 	})
 	defer tearDownFn()
@@ -1397,17 +1354,13 @@ func TestKindAuthorization(t *testing.T) {
 // TestReadOnlyAuthorization tests that authorization can be controlled
 // by namespace.
 func TestReadOnlyAuthorization(t *testing.T) {
-	// This file has alice and bob in it.
-	a := newAuthorizerWithContents(t, `{"readonly": true}`)
-
 	kubeClient, kubeConfig, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
 			opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
-		},
-		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authentication.Authenticator = getTestTokenAuth()
-			config.GenericConfig.Authorization.Authorizer = a
+			opts.Authentication.TokenFile.TokenFile = "testdata/tokens.csv"
+			opts.Authorization.PolicyFile = newABACFileWithContents(t, `{"readonly": true}`)
+			opts.Authorization.Modes = []string{"ABAC"}
 		},
 	})
 	defer tearDownFn()
@@ -1484,12 +1437,13 @@ func testWebhookTokenAuthenticator(customDialer bool, t *testing.T) {
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
 			opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
+			opts.Authorization.Modes = []string{"ABAC"}
+			opts.Authorization.PolicyFile = "testdata/allowalice.jsonl"
 		},
 		ModifyServerConfig: func(config *controlplane.Config) {
 			config.GenericConfig.Authentication.Authenticator = group.NewAuthenticatedGroupAdder(authenticator)
 			// Disable checking API audiences that is set by testserver by default.
 			config.GenericConfig.Authentication.APIAudiences = nil
-			config.GenericConfig.Authorization.Authorizer = allowAliceAuthorizer{}
 		},
 	})
 	defer tearDownFn()
diff --git a/test/integration/auth/bootstraptoken_test.go b/test/integration/auth/bootstraptoken_test.go
index c538cdeb1bb..b513d10dfe7 100644
--- a/test/integration/auth/bootstraptoken_test.go
+++ b/test/integration/auth/bootstraptoken_test.go
@@ -29,9 +29,9 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apiserver/pkg/authentication/group"
 	"k8s.io/apiserver/pkg/authentication/request/bearertoken"
-	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
 	"k8s.io/client-go/rest"
 	bootstrapapi "k8s.io/cluster-bootstrap/token/api"
+	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 	"k8s.io/kubernetes/pkg/controlplane"
 	"k8s.io/kubernetes/plugin/pkg/auth/authenticator/token/bootstrap"
 	"k8s.io/kubernetes/test/integration"
@@ -122,9 +122,11 @@ func TestBootstrapTokenAuth(t *testing.T) {
 			authenticator := group.NewAuthenticatedGroupAdder(bearertoken.New(bootstrap.NewTokenAuthenticator(bootstrapSecrets{test.secret})))
 
 			kubeClient, kubeConfig, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{
+				ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
+					opts.Authorization.Modes = []string{"AlwaysAllow"}
+				},
 				ModifyServerConfig: func(config *controlplane.Config) {
 					config.GenericConfig.Authentication.Authenticator = authenticator
-					config.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer()
 				},
 			})
 			defer tearDownFn()
diff --git a/test/integration/auth/dynamic_client_test.go b/test/integration/auth/dynamic_client_test.go
index 823a7d0dd41..1192e46e656 100644
--- a/test/integration/auth/dynamic_client_test.go
+++ b/test/integration/auth/dynamic_client_test.go
@@ -24,12 +24,10 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
-	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
 	clientset "k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/controller-manager/pkg/clientbuilder"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
-	"k8s.io/kubernetes/pkg/controlplane"
 	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
 	"k8s.io/kubernetes/test/integration/framework"
 )
@@ -67,9 +65,7 @@ func TestDynamicClientBuilder(t *testing.T) {
 			}
 			opts.Authentication.ServiceAccounts.Issuers = []string{iss}
 			opts.Authentication.ServiceAccounts.KeyFiles = []string{tmpfile.Name()}
-		},
-		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer()
+			opts.Authorization.Modes = []string{"AlwaysAllow"}
 		},
 	})
 	defer tearDownFn()
diff --git a/test/integration/auth/rbac_test.go b/test/integration/auth/rbac_test.go
index 81aaa7d55cf..fbdcb975b14 100644
--- a/test/integration/auth/rbac_test.go
+++ b/test/integration/auth/rbac_test.go
@@ -35,9 +35,11 @@ import (
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/apiserver/pkg/authentication/group"
 	"k8s.io/apiserver/pkg/authentication/request/bearertoken"
+	unionauthn "k8s.io/apiserver/pkg/authentication/request/union"
 	"k8s.io/apiserver/pkg/authentication/token/tokenfile"
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
+	unionauthz "k8s.io/apiserver/pkg/authorization/union"
 	genericfeatures "k8s.io/apiserver/pkg/features"
 	"k8s.io/apiserver/pkg/registry/generic"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
@@ -552,10 +554,16 @@ func TestRBAC(t *testing.T) {
 					// Also disable namespace lifecycle to workaroung the test limitation that first creates
 					// roles/rolebindings and only then creates corresponding namespaces.
 					opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount", "NamespaceLifecycle"}
+					// Disable built-in authorizers
+					opts.Authorization.Modes = []string{"AlwaysDeny"}
 				},
 				ModifyServerConfig: func(config *controlplane.Config) {
-					config.GenericConfig.Authentication.Authenticator = authenticator
-					config.GenericConfig.Authorization.Authorizer, tearDownAuthorizerFn = newRBACAuthorizer(t, config)
+					// Append our custom test authenticator
+					config.GenericConfig.Authentication.Authenticator = unionauthn.New(config.GenericConfig.Authentication.Authenticator, authenticator)
+					// Append our custom test authorizer
+					var rbacAuthz authorizer.Authorizer
+					rbacAuthz, tearDownAuthorizerFn = newRBACAuthorizer(t, config)
+					config.GenericConfig.Authorization.Authorizer = unionauthz.New(config.GenericConfig.Authorization.Authorizer, rbacAuthz)
 				},
 			})
 			defer tearDownFn()
@@ -666,20 +674,9 @@ func TestRBAC(t *testing.T) {
 }
 
 func TestBootstrapping(t *testing.T) {
-	superUser := "admin/system:masters"
-
-	var tearDownAuthorizerFn func()
-	defer func() {
-		if tearDownAuthorizerFn != nil {
-			tearDownAuthorizerFn()
-		}
-	}()
 	clientset, _, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{
-		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authentication.Authenticator = bearertoken.New(tokenfile.New(map[string]*user.DefaultInfo{
-				superUser: {Name: "admin", Groups: []string{"system:masters"}},
-			}))
-			config.GenericConfig.Authorization.Authorizer, tearDownAuthorizerFn = newRBACAuthorizer(t, config)
+		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
+			opts.Authorization.Modes = []string{"RBAC"}
 		},
 	})
 	defer tearDownFn()
@@ -732,14 +729,6 @@ func TestDiscoveryUpgradeBootstrapping(t *testing.T) {
 			tearDownFn()
 		}
 	}()
-	var tearDownAuthorizerFn func()
-	defer func() {
-		if tearDownAuthorizerFn != nil {
-			tearDownAuthorizerFn()
-		}
-	}()
-
-	superUser := "admin/system:masters"
 
 	etcdConfig := framework.SharedEtcd()
 
@@ -747,12 +736,7 @@ func TestDiscoveryUpgradeBootstrapping(t *testing.T) {
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Ensure we're using the same etcd across apiserver restarts.
 			opts.Etcd.StorageConfig = *etcdConfig
-		},
-		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authentication.Authenticator = bearertoken.New(tokenfile.New(map[string]*user.DefaultInfo{
-				superUser: {Name: "admin", Groups: []string{"system:masters"}},
-			}))
-			config.GenericConfig.Authorization.Authorizer, tearDownAuthorizerFn = newRBACAuthorizer(t, config)
+			opts.Authorization.Modes = []string{"RBAC"}
 		},
 	})
 
@@ -793,8 +777,6 @@ func TestDiscoveryUpgradeBootstrapping(t *testing.T) {
 	// Stop the first API server.
 	tearDownFn()
 	tearDownFn = nil
-	tearDownAuthorizerFn()
-	tearDownAuthorizerFn = nil
 
 	// Check that upgraded API servers inherit `system:public-info-viewer` settings from
 	// `system:discovery`, and respect auto-reconciliation annotations.
@@ -802,12 +784,7 @@ func TestDiscoveryUpgradeBootstrapping(t *testing.T) {
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Ensure we're using the same etcd across apiserver restarts.
 			opts.Etcd.StorageConfig = *etcdConfig
-		},
-		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authentication.Authenticator = bearertoken.New(tokenfile.New(map[string]*user.DefaultInfo{
-				superUser: {Name: "admin", Groups: []string{"system:masters"}},
-			}))
-			config.GenericConfig.Authorization.Authorizer, tearDownAuthorizerFn = newRBACAuthorizer(t, config)
+			opts.Authorization.Modes = []string{"RBAC"}
 		},
 	})
 
diff --git a/test/integration/auth/selfsubjectreview_test.go b/test/integration/auth/selfsubjectreview_test.go
index c5e729366c2..bd511182349 100644
--- a/test/integration/auth/selfsubjectreview_test.go
+++ b/test/integration/auth/selfsubjectreview_test.go
@@ -30,7 +30,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/apiserver/pkg/authentication/user"
-	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
@@ -96,6 +95,10 @@ func TestGetsSelfAttributes(t *testing.T) {
 	}
 
 	kubeClient, _, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{
+		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
+			opts.APIEnablement.RuntimeConfig.Set("authentication.k8s.io/v1alpha1=true")
+			opts.Authorization.Modes = []string{"AlwaysAllow"}
+		},
 		ModifyServerConfig: func(config *controlplane.Config) {
 			// Unset BearerToken to disable BearerToken authenticator.
 			config.GenericConfig.LoopbackClientConfig.BearerToken = ""
@@ -104,10 +107,6 @@ func TestGetsSelfAttributes(t *testing.T) {
 				defer respMu.RUnlock()
 				return &authenticator.Response{User: response}, true, nil
 			})
-			config.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer()
-		},
-		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
-			opts.APIEnablement.RuntimeConfig.Set("authentication.k8s.io/v1alpha1=true")
 		},
 	})
 	defer tearDownFn()
@@ -156,6 +155,10 @@ func TestGetsSelfAttributesError(t *testing.T) {
 	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APISelfSubjectReview, true)()
 
 	kubeClient, _, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{
+		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
+			opts.APIEnablement.RuntimeConfig.Set("authentication.k8s.io/v1alpha1=true")
+			opts.Authorization.Modes = []string{"AlwaysAllow"}
+		},
 		ModifyServerConfig: func(config *controlplane.Config) {
 			// Unset BearerToken to disable BearerToken authenticator.
 			config.GenericConfig.LoopbackClientConfig.BearerToken = ""
@@ -170,10 +173,6 @@ func TestGetsSelfAttributesError(t *testing.T) {
 
 				return nil, false, fmt.Errorf("test error")
 			})
-			config.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer()
-		},
-		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
-			opts.APIEnablement.RuntimeConfig.Set("authentication.k8s.io/v1alpha1=true")
 		},
 	})
 	defer tearDownFn()
diff --git a/test/integration/auth/svcaccttoken_test.go b/test/integration/auth/svcaccttoken_test.go
index 11869ac7e6b..9daf6c1bc3d 100644
--- a/test/integration/auth/svcaccttoken_test.go
+++ b/test/integration/auth/svcaccttoken_test.go
@@ -19,12 +19,10 @@ package auth
 import (
 	"bytes"
 	"context"
-	"crypto/ecdsa"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"net/url"
 	"reflect"
@@ -42,18 +40,12 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
-	"k8s.io/apiserver/pkg/authentication/request/bearertoken"
 	apiserverserviceaccount "k8s.io/apiserver/pkg/authentication/serviceaccount"
-	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
-	v1listers "k8s.io/client-go/listers/core/v1"
 	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/cache"
-	"k8s.io/client-go/util/keyutil"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 	"k8s.io/kubernetes/pkg/apis/core"
-	serviceaccountgetter "k8s.io/kubernetes/pkg/controller/serviceaccount"
 	"k8s.io/kubernetes/pkg/controlplane"
 	"k8s.io/kubernetes/pkg/serviceaccount"
 	"k8s.io/kubernetes/test/integration/framework"
@@ -71,15 +63,6 @@ AwEHoUQDQgAEH6cuzP8XuD5wal6wf9M6xDljTOPLX2i8uIp/C/ASqiIGUeeKQtX0
 )
 
 func TestServiceAccountTokenCreate(t *testing.T) {
-
-	// Build client config, clientset, and informers
-	sk, err := keyutil.ParsePrivateKeyPEM([]byte(ecdsaPrivateKey))
-	if err != nil {
-		t.Fatalf("err: %v", err)
-	}
-
-	pk := sk.(*ecdsa.PrivateKey).PublicKey
-
 	const iss = "https://foo.bar.example.com"
 	aud := authenticator.Audiences{"api"}
 
@@ -89,12 +72,7 @@ func TestServiceAccountTokenCreate(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	gcs := &clientset.Clientset{}
-
-	tokenGenerator, err := serviceaccount.JWTTokenGenerator(iss, sk)
-	if err != nil {
-		t.Fatalf("err: %v", err)
-	}
+	var tokenGenerator serviceaccount.TokenGenerator
 
 	// Start the server
 	var serverAddress string
@@ -102,45 +80,21 @@ func TestServiceAccountTokenCreate(t *testing.T) {
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
 			opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
+			opts.Authorization.Modes = []string{"AlwaysAllow"}
+			// Disable token cache so we can check reaction to service account deletion quickly
+			opts.Authentication.TokenSuccessCacheTTL = 0
+			opts.Authentication.TokenFailureCacheTTL = 0
+			// Pin to fixed URLs for easier testing
+			opts.Authentication.ServiceAccounts.JWKSURI = "https:///openid/v1/jwks"
+			opts.Authentication.ServiceAccounts.Issuers = []string{iss}
+			opts.Authentication.APIAudiences = aud
 		},
 		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer()
-			config.GenericConfig.Authentication.APIAudiences = aud
-			config.GenericConfig.Authentication.Authenticator = bearertoken.New(
-				serviceaccount.JWTTokenAuthenticator(
-					[]string{iss},
-					[]interface{}{&pk},
-					aud,
-					serviceaccount.NewValidator(serviceaccountgetter.NewGetterFromClient(
-						gcs,
-						v1listers.NewSecretLister(newIndexer(func(namespace, name string) (interface{}, error) {
-							return gcs.CoreV1().Secrets(namespace).Get(context.TODO(), name, metav1.GetOptions{})
-						})),
-						v1listers.NewServiceAccountLister(newIndexer(func(namespace, name string) (interface{}, error) {
-							return gcs.CoreV1().ServiceAccounts(namespace).Get(context.TODO(), name, metav1.GetOptions{})
-						})),
-						v1listers.NewPodLister(newIndexer(func(namespace, name string) (interface{}, error) {
-							return gcs.CoreV1().Pods(namespace).Get(context.TODO(), name, metav1.GetOptions{})
-						})),
-					)),
-				),
-			)
+			// extract token generator
+			tokenGenerator = config.ExtraConfig.ServiceAccountIssuer
 
-			config.ExtraConfig.ServiceAccountIssuer = tokenGenerator
 			config.ExtraConfig.ServiceAccountMaxExpiration = maxExpirationDuration
 			config.ExtraConfig.ExtendExpiration = true
-
-			config.ExtraConfig.ServiceAccountIssuerURL = iss
-			config.ExtraConfig.ServiceAccountJWKSURI = ""
-			config.ExtraConfig.ServiceAccountPublicKeys = []interface{}{&pk}
-
-			// Compute the serverAddress.
-			serverAddress = config.GenericConfig.ExternalAddress
-			_, port, err := config.GenericConfig.SecureServing.HostPort()
-			if err != nil {
-				t.Fatalf("Couldn't get server port: %v", err)
-			}
-			serverAddress = net.JoinHostPort(serverAddress, strconv.Itoa(port))
 		},
 	})
 	defer tearDownFn()
@@ -156,7 +110,6 @@ func TestServiceAccountTokenCreate(t *testing.T) {
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
-	*gcs = *cs
 
 	kubeConfig.NegotiatedSerializer = scheme.Codecs.WithoutConversion()
 	rc, err := rest.UnversionedRESTClientFor(kubeConfig)
@@ -917,25 +870,36 @@ func TestServiceAccountTokenCreate(t *testing.T) {
 
 func doTokenReview(t *testing.T, cs clientset.Interface, treq *authenticationv1.TokenRequest, expectErr bool) authenticationv1.UserInfo {
 	t.Helper()
-	trev, err := cs.AuthenticationV1().TokenReviews().Create(context.TODO(), &authenticationv1.TokenReview{
-		Spec: authenticationv1.TokenReviewSpec{
-			Token: treq.Status.Token,
-		},
-	}, metav1.CreateOptions{})
-	if err != nil {
-		t.Fatalf("err: %v", err)
+	tries := 0
+	for {
+		trev, err := cs.AuthenticationV1().TokenReviews().Create(context.TODO(), &authenticationv1.TokenReview{
+			Spec: authenticationv1.TokenReviewSpec{
+				Token: treq.Status.Token,
+			},
+		}, metav1.CreateOptions{})
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		t.Logf("status: %+v", trev.Status)
+		if (trev.Status.Error != "") && !expectErr {
+			t.Fatalf("expected no error but got: %v", trev.Status.Error)
+		}
+		if (trev.Status.Error == "") && expectErr {
+			// if we expected an error and didn't get one, retry
+			// to let changes that invalidate the token percolate through informers
+			if tries < 10 {
+				tries++
+				time.Sleep(100 * time.Millisecond)
+				t.Logf("expected error but got: %+v, retrying", trev.Status)
+				continue
+			}
+			t.Fatalf("expected error but got: %+v", trev.Status)
+		}
+		if !trev.Status.Authenticated && !expectErr {
+			t.Fatal("expected token to be authenticated but it wasn't")
+		}
+		return trev.Status.User
 	}
-	t.Logf("status: %+v", trev.Status)
-	if (trev.Status.Error != "") && !expectErr {
-		t.Fatalf("expected no error but got: %v", trev.Status.Error)
-	}
-	if (trev.Status.Error == "") && expectErr {
-		t.Fatalf("expected error but got: %+v", trev.Status)
-	}
-	if !trev.Status.Authenticated && !expectErr {
-		t.Fatal("expected token to be authenticated but it wasn't")
-	}
-	return trev.Status.User
 }
 
 func checkPayload(t *testing.T, tok string, want string, parts ...string) {
@@ -1043,26 +1007,6 @@ func createDeleteSecret(t *testing.T, cs clientset.Interface, sec *v1.Secret) (*
 	}
 }
 
-func newIndexer(get func(namespace, name string) (interface{}, error)) cache.Indexer {
-	return &fakeIndexer{get: get}
-}
-
-type fakeIndexer struct {
-	cache.Indexer
-	get func(namespace, name string) (interface{}, error)
-}
-
-func (f *fakeIndexer) GetByKey(key string) (interface{}, bool, error) {
-	parts := strings.SplitN(key, "/", 2)
-	namespace := parts[0]
-	name := ""
-	if len(parts) == 2 {
-		name = parts[1]
-	}
-	obj, err := f.get(namespace, name)
-	return obj, err == nil, err
-}
-
 type recordingWarningHandler struct {
 	warnings []string
 
diff --git a/test/integration/auth/testdata/allowalice.jsonl b/test/integration/auth/testdata/allowalice.jsonl
new file mode 100644
index 00000000000..1093f210a97
--- /dev/null
+++ b/test/integration/auth/testdata/allowalice.jsonl
@@ -0,0 +1 @@
+{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"alice", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}}
\ No newline at end of file
diff --git a/test/integration/auth/testdata/tokens.csv b/test/integration/auth/testdata/tokens.csv
new file mode 100644
index 00000000000..5f009760c09
--- /dev/null
+++ b/test/integration/auth/testdata/tokens.csv
@@ -0,0 +1,2 @@
+abc123,alice,1
+xyz987,bob,2
diff --git a/test/integration/controlplane/synthetic_controlplane_test.go b/test/integration/controlplane/synthetic_controlplane_test.go
index 57d5351c249..c432351d8ef 100644
--- a/test/integration/controlplane/synthetic_controlplane_test.go
+++ b/test/integration/controlplane/synthetic_controlplane_test.go
@@ -39,20 +39,11 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
-	authauthenticator "k8s.io/apiserver/pkg/authentication/authenticator"
-	"k8s.io/apiserver/pkg/authentication/group"
-	"k8s.io/apiserver/pkg/authentication/request/bearertoken"
-	authenticatorunion "k8s.io/apiserver/pkg/authentication/request/union"
-	"k8s.io/apiserver/pkg/authentication/user"
-	"k8s.io/apiserver/pkg/authorization/authorizer"
-	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
-	"k8s.io/apiserver/plugin/pkg/authenticator/token/tokentest"
 	clientset "k8s.io/client-go/kubernetes"
 	clienttypedv1 "k8s.io/client-go/kubernetes/typed/core/v1"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
-	"k8s.io/kubernetes/pkg/controlplane"
 	"k8s.io/kubernetes/test/integration"
 	"k8s.io/kubernetes/test/integration/framework"
 )
@@ -63,15 +54,6 @@ const (
 	BobToken   string = "xyz987" // username: bob.  Present in token file.
 )
 
-type allowAliceAuthorizer struct{}
-
-func (allowAliceAuthorizer) Authorize(ctx context.Context, a authorizer.Attributes) (authorizer.Decision, string, error) {
-	if a.GetUser() != nil && a.GetUser().GetName() == "alice" {
-		return authorizer.DecisionAllow, "", nil
-	}
-	return authorizer.DecisionNoOpinion, "I can't allow that.  Go ask alice.", nil
-}
-
 func testPrefix(t *testing.T, prefix string) {
 	server := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd())
 	defer server.TearDownFn()
@@ -163,68 +145,54 @@ func TestEmptyList(t *testing.T) {
 	}
 }
 
-func initStatusForbiddenControlPlaneConfig(config *controlplane.Config) {
-	config.GenericConfig.Authentication.Authenticator = authenticatorunion.New(
-		authauthenticator.RequestFunc(func(req *http.Request) (*authauthenticator.Response, bool, error) {
-			return &authauthenticator.Response{
-				User: &user.DefaultInfo{
-					Name:   "unprivileged",
-					Groups: []string{user.AllAuthenticated},
-				},
-			}, true, nil
-		}))
-	config.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysDenyAuthorizer()
+func initStatusForbiddenControlPlaneConfig(options *options.ServerRunOptions) {
+	options.Authorization.Modes = []string{"AlwaysDeny"}
 }
 
-func initUnauthorizedControlPlaneConfig(config *controlplane.Config) {
-	tokenAuthenticator := tokentest.New()
-	tokenAuthenticator.Tokens[AliceToken] = &user.DefaultInfo{Name: "alice", UID: "1"}
-	tokenAuthenticator.Tokens[BobToken] = &user.DefaultInfo{Name: "bob", UID: "2"}
-	config.GenericConfig.Authentication.Authenticator = group.NewGroupAdder(bearertoken.New(tokenAuthenticator), []string{user.AllAuthenticated})
-	config.GenericConfig.Authorization.Authorizer = allowAliceAuthorizer{}
+func initUnauthorizedControlPlaneConfig(options *options.ServerRunOptions) {
+	options.Authentication.Anonymous.Allow = false
 }
 
 func TestStatus(t *testing.T) {
 	testCases := []struct {
-		name         string
-		modifyConfig func(*controlplane.Config)
-		statusCode   int
-		reqPath      string
-		reason       string
-		message      string
+		name          string
+		modifyOptions func(*options.ServerRunOptions)
+		statusCode    int
+		reqPath       string
+		reason        string
+		message       string
 	}{
 		{
-			name:         "404",
-			modifyConfig: nil,
-			statusCode:   http.StatusNotFound,
-			reqPath:      "/apis/batch/v1/namespaces/default/jobs/foo",
-			reason:       "NotFound",
-			message:      `jobs.batch "foo" not found`,
+			name:       "404",
+			statusCode: http.StatusNotFound,
+			reqPath:    "/apis/batch/v1/namespaces/default/jobs/foo",
+			reason:     "NotFound",
+			message:    `jobs.batch "foo" not found`,
 		},
 		{
-			name:         "403",
-			modifyConfig: initStatusForbiddenControlPlaneConfig,
-			statusCode:   http.StatusForbidden,
-			reqPath:      "/apis",
-			reason:       "Forbidden",
-			message:      `forbidden: User "unprivileged" cannot get path "/apis": Everything is forbidden.`,
+			name:          "403",
+			modifyOptions: initStatusForbiddenControlPlaneConfig,
+			statusCode:    http.StatusForbidden,
+			reqPath:       "/apis",
+			reason:        "Forbidden",
+			message:       `forbidden: User "system:anonymous" cannot get path "/apis": Everything is forbidden.`,
 		},
 		{
-			name:         "401",
-			modifyConfig: initUnauthorizedControlPlaneConfig,
-			statusCode:   http.StatusUnauthorized,
-			reqPath:      "/apis",
-			reason:       "Unauthorized",
-			message:      `Unauthorized`,
+			name:          "401",
+			modifyOptions: initUnauthorizedControlPlaneConfig,
+			statusCode:    http.StatusUnauthorized,
+			reqPath:       "/apis",
+			reason:        "Unauthorized",
+			message:       `Unauthorized`,
 		},
 	}
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			_, kubeConfig, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{
-				ModifyServerConfig: func(config *controlplane.Config) {
-					if tc.modifyConfig != nil {
-						tc.modifyConfig(config)
+				ModifyServerRunOptions: func(options *options.ServerRunOptions) {
+					if tc.modifyOptions != nil {
+						tc.modifyOptions(options)
 					}
 				},
 			})
@@ -232,7 +200,7 @@ func TestStatus(t *testing.T) {
 
 			// When modifying authenticator and authorizer, don't use
 			// bearer token than will be always authorized.
-			if tc.modifyConfig != nil {
+			if tc.modifyOptions != nil {
 				kubeConfig.BearerToken = ""
 			}
 			transport, err := restclient.TransportFor(kubeConfig)
diff --git a/test/integration/serviceaccount/service_account_test.go b/test/integration/serviceaccount/service_account_test.go
index f951599b442..2d34b00f722 100644
--- a/test/integration/serviceaccount/service_account_test.go
+++ b/test/integration/serviceaccount/service_account_test.go
@@ -22,8 +22,6 @@ package serviceaccount
 
 import (
 	"context"
-	"crypto/rand"
-	"crypto/rsa"
 	"fmt"
 	"testing"
 	"time"
@@ -33,18 +31,16 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/apiserver/pkg/authentication/authenticator"
-	"k8s.io/apiserver/pkg/authentication/group"
-	"k8s.io/apiserver/pkg/authentication/request/bearertoken"
-	"k8s.io/apiserver/pkg/authentication/request/union"
 	serviceaccountapiserver "k8s.io/apiserver/pkg/authentication/serviceaccount"
-	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
+	unionauthz "k8s.io/apiserver/pkg/authorization/union"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	clientinformers "k8s.io/client-go/informers"
 	clientset "k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/util/keyutil"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
+	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 	"k8s.io/kubernetes/pkg/controller"
 	serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount"
 	"k8s.io/kubernetes/pkg/controlplane"
@@ -55,9 +51,6 @@ import (
 )
 
 const (
-	rootUserName = "root"
-	rootToken    = "root-user-token" // Fake value for testing.
-
 	readOnlyServiceAccountName  = "ro"
 	readWriteServiceAccountName = "rw"
 )
@@ -319,50 +312,19 @@ func TestServiceAccountTokenAuthentication(t *testing.T) {
 
 // startServiceAccountTestServerAndWaitForCaches returns a started server
 // It is the responsibility of the caller to ensure the returned stopFunc is called
-func startServiceAccountTestServerAndWaitForCaches(t *testing.T) (*clientset.Clientset, *restclient.Config, func(), error) {
-	rootTokenAuth := authenticator.TokenFunc(func(ctx context.Context, token string) (*authenticator.Response, bool, error) {
-		if token == rootToken {
-			return &authenticator.Response{User: &user.DefaultInfo{Name: rootUserName}}, true, nil
-		}
-		return nil, false, nil
-	})
-	serviceAccountKey, _ := rsa.GenerateKey(rand.Reader, 2048)
-
-	var informers clientinformers.SharedInformerFactory
-	var externalInformers clientinformers.SharedInformerFactory
-	var rootClientset *clientset.Clientset
+func startServiceAccountTestServerAndWaitForCaches(t *testing.T) (clientset.Interface, *restclient.Config, func(), error) {
+	var serviceAccountKey interface{}
 
 	// Set up a API server
-	_, clientConfig, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{
+	rootClientset, clientConfig, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{
+		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
+			var err error
+			serviceAccountKey, err = keyutil.PrivateKeyFromFile(opts.ServiceAccountSigningKeyFile)
+			if err != nil {
+				t.Fatal(err)
+			}
+		},
 		ModifyServerConfig: func(config *controlplane.Config) {
-			rootConfig := restclient.CopyConfig(config.GenericConfig.LoopbackClientConfig)
-			rootConfig.BearerToken = rootToken
-			rootClientset = clientset.NewForConfigOrDie(rootConfig)
-			externalRootClientset := clientset.NewForConfigOrDie(rootConfig)
-
-			externalInformers = clientinformers.NewSharedInformerFactory(externalRootClientset, controller.NoResyncPeriodFunc())
-			informers = clientinformers.NewSharedInformerFactory(rootClientset, controller.NoResyncPeriodFunc())
-
-			// Set up two authenticators:
-			// 1. A token authenticator that maps the rootToken to the "root" user
-			// 2. A ServiceAccountToken authenticator that validates ServiceAccount tokens
-			serviceAccountTokenGetter := serviceaccountcontroller.NewGetterFromClient(
-				rootClientset,
-				externalInformers.Core().V1().Secrets().Lister(),
-				externalInformers.Core().V1().ServiceAccounts().Lister(),
-				externalInformers.Core().V1().Pods().Lister(),
-			)
-			serviceAccountTokenAuth := serviceaccount.JWTTokenAuthenticator(
-				[]string{serviceaccount.LegacyIssuer},
-				[]interface{}{&serviceAccountKey.PublicKey},
-				nil,
-				serviceaccount.NewLegacyValidator(true, serviceAccountTokenGetter),
-			)
-			authenticator := group.NewAuthenticatedGroupAdder(union.New(
-				bearertoken.New(rootTokenAuth),
-				bearertoken.New(serviceAccountTokenAuth),
-			))
-
 			// Set up a stub authorizer:
 			// 1. The "root" user is allowed to do anything
 			// 2. ServiceAccounts named "ro" are allowed read-only operations in their namespace
@@ -374,12 +336,6 @@ func startServiceAccountTestServerAndWaitForCaches(t *testing.T) (*clientset.Cli
 				}
 				ns := attrs.GetNamespace()
 
-				// If the user is "root"...
-				if username == rootUserName {
-					// allow them to do anything
-					return authorizer.DecisionAllow, "", nil
-				}
-
 				// If the user is a service account...
 				if serviceAccountNamespace, serviceAccountName, err := serviceaccountapiserver.SplitUsername(username); err == nil {
 					// Limit them to their own namespace
@@ -397,16 +353,7 @@ func startServiceAccountTestServerAndWaitForCaches(t *testing.T) (*clientset.Cli
 
 				return authorizer.DecisionNoOpinion, fmt.Sprintf("User %s is denied (ns=%s, readonly=%v, resource=%s)", username, ns, attrs.IsReadOnly(), attrs.GetResource()), nil
 			})
-
-			// Set up admission plugin to auto-assign serviceaccounts to pods
-			serviceAccountAdmission := serviceaccountadmission.NewServiceAccount()
-			serviceAccountAdmission.SetExternalKubeClientSet(externalRootClientset)
-			serviceAccountAdmission.SetExternalKubeInformerFactory(externalInformers)
-
-			config.GenericConfig.EnableIndex = true
-			config.GenericConfig.Authentication.Authenticator = authenticator
-			config.GenericConfig.Authorization.Authorizer = authorizer
-			config.GenericConfig.AdmissionControl = serviceAccountAdmission
+			config.GenericConfig.Authorization.Authorizer = unionauthz.New(config.GenericConfig.Authorization.Authorizer, authorizer)
 		},
 	})
 
@@ -416,6 +363,8 @@ func startServiceAccountTestServerAndWaitForCaches(t *testing.T) (*clientset.Cli
 		tearDownFn()
 	}
 
+	informers := clientinformers.NewSharedInformerFactory(rootClientset, controller.NoResyncPeriodFunc())
+
 	// Start the service account and service account token controllers
 	tokenGenerator, err := serviceaccount.JWTTokenGenerator(serviceaccount.LegacyIssuer, serviceAccountKey)
 	if err != nil {
@@ -445,7 +394,6 @@ func startServiceAccountTestServerAndWaitForCaches(t *testing.T) (*clientset.Cli
 		return rootClientset, clientConfig, stop, err
 	}
 	informers.Start(ctx.Done())
-	externalInformers.Start(ctx.Done())
 	go serviceAccountController.Run(ctx, 5)
 
 	// since this method starts the controllers in a separate goroutine
@@ -453,12 +401,11 @@ func startServiceAccountTestServerAndWaitForCaches(t *testing.T) (*clientset.Cli
 	// the tests can tell it is safe to call the server and requests won't be rejected
 	// thus we wait until caches have synced
 	informers.WaitForCacheSync(ctx.Done())
-	externalInformers.WaitForCacheSync(ctx.Done())
 
 	return rootClientset, clientConfig, stop, nil
 }
 
-func getServiceAccount(c *clientset.Clientset, ns string, name string, shouldWait bool) (*v1.ServiceAccount, error) {
+func getServiceAccount(c clientset.Interface, ns string, name string, shouldWait bool) (*v1.ServiceAccount, error) {
 	if !shouldWait {
 		return c.CoreV1().ServiceAccounts(ns).Get(context.TODO(), name, metav1.GetOptions{})
 	}
@@ -478,7 +425,7 @@ func getServiceAccount(c *clientset.Clientset, ns string, name string, shouldWai
 	return user, err
 }
 
-func createServiceAccountToken(c *clientset.Clientset, sa *v1.ServiceAccount, ns string, name string) (*v1.Secret, error) {
+func createServiceAccountToken(c clientset.Interface, sa *v1.ServiceAccount, ns string, name string) (*v1.Secret, error) {
 	secret := &v1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
@@ -508,7 +455,7 @@ func createServiceAccountToken(c *clientset.Clientset, sa *v1.ServiceAccount, ns
 	return secret, nil
 }
 
-func getReferencedServiceAccountToken(c *clientset.Clientset, ns string, name string, shouldWait bool) (string, string, error) {
+func getReferencedServiceAccountToken(c clientset.Interface, ns string, name string, shouldWait bool) (string, string, error) {
 	tokenName := ""
 	token := ""
 
@@ -564,7 +511,7 @@ func getReferencedServiceAccountToken(c *clientset.Clientset, ns string, name st
 
 type testOperation func() error
 
-func doServiceAccountAPIRequests(t *testing.T, c *clientset.Clientset, ns string, authenticated bool, canRead bool, canWrite bool) {
+func doServiceAccountAPIRequests(t *testing.T, c clientset.Interface, ns string, authenticated bool, canRead bool, canWrite bool) {
 	testSecret := &v1.Secret{
 		ObjectMeta: metav1.ObjectMeta{Name: "testSecret"},
 		Data:       map[string][]byte{"test": []byte("data")},
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 03859958f2a..9f92c0a7bdc 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -1625,7 +1625,6 @@ k8s.io/apiserver/plugin/pkg/audit/log
 k8s.io/apiserver/plugin/pkg/audit/truncate
 k8s.io/apiserver/plugin/pkg/audit/webhook
 k8s.io/apiserver/plugin/pkg/authenticator/token/oidc
-k8s.io/apiserver/plugin/pkg/authenticator/token/tokentest
 k8s.io/apiserver/plugin/pkg/authenticator/token/webhook
 k8s.io/apiserver/plugin/pkg/authorizer/webhook
 # k8s.io/cli-runtime v0.0.0 => ./staging/src/k8s.io/cli-runtime