diff --git a/test/integration/auth/auth_test.go b/test/integration/auth/auth_test.go
index 1800e0e1f69..becba2d72e3 100644
--- a/test/integration/auth/auth_test.go
+++ b/test/integration/auth/auth_test.go
@@ -35,6 +35,7 @@ import (
 	"net/http/httptest"
 	"net/url"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"testing"
@@ -54,11 +55,9 @@ import (
 	"k8s.io/apiserver/pkg/authentication/request/bearertoken"
 	"k8s.io/apiserver/pkg/authentication/serviceaccount"
 	"k8s.io/apiserver/pkg/authentication/token/cache"
-	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
-	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
+	unionauthz "k8s.io/apiserver/pkg/authorization/union"
 	webhookutil "k8s.io/apiserver/pkg/util/webhook"
-	"k8s.io/apiserver/plugin/pkg/authenticator/token/tokentest"
 	"k8s.io/apiserver/plugin/pkg/authenticator/token/webhook"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -69,7 +68,6 @@ import (
 	"k8s.io/kubernetes/pkg/apis/autoscaling"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/apis/extensions"
-	"k8s.io/kubernetes/pkg/auth/authorizer/abac"
 	"k8s.io/kubernetes/pkg/controlplane"
 	"k8s.io/kubernetes/test/integration"
 	"k8s.io/kubernetes/test/integration/authutil"
@@ -82,13 +80,6 @@ const (
 	UnknownToken string = "qwerty" // Not present in token file.
 )
 
-func getTestTokenAuth() authenticator.Request {
-	tokenAuthenticator := tokentest.New()
-	tokenAuthenticator.Tokens[AliceToken] = &user.DefaultInfo{Name: "alice", UID: "1"}
-	tokenAuthenticator.Tokens[BobToken] = &user.DefaultInfo{Name: "bob", UID: "2"}
-	return group.NewGroupAdder(bearertoken.New(tokenAuthenticator), []string{user.AllAuthenticated})
-}
-
 func getTestWebhookTokenAuth(serverURL string, customDial utilnet.DialFunc) (authenticator.Request, error) {
 	kubecfgFile, err := os.CreateTemp("", "webhook-kubecfg")
 	if err != nil {
@@ -464,9 +455,7 @@ func TestAuthModeAlwaysAllow(t *testing.T) {
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
 			opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
-		},
-		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer()
+			opts.Authorization.Modes = []string{"AlwaysAllow"}
 		},
 	})
 	defer tearDownFn()
@@ -570,10 +559,8 @@ func TestAuthModeAlwaysDeny(t *testing.T) {
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
 			opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
-		},
-		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authentication.Authenticator = getTestTokenAuth()
-			config.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysDenyAuthorizer()
+			opts.Authorization.Modes = []string{"AlwaysDeny"}
+			opts.Authentication.TokenFile.TokenFile = "testdata/tokens.csv"
 		},
 	})
 	defer tearDownFn()
@@ -609,17 +596,6 @@ func TestAuthModeAlwaysDeny(t *testing.T) {
 	}
 }
 
-// Inject into control plane an authorizer that uses user info.
-// TODO(etune): remove this test once a more comprehensive built-in authorizer is implemented.
-type allowAliceAuthorizer struct{}
-
-func (allowAliceAuthorizer) Authorize(ctx context.Context, a authorizer.Attributes) (authorizer.Decision, string, error) {
-	if a.GetUser() != nil && a.GetUser().GetName() == "alice" {
-		return authorizer.DecisionAllow, "", nil
-	}
-	return authorizer.DecisionNoOpinion, "I can't allow that.  Go ask alice.", nil
-}
-
 // TestAliceNotForbiddenOrUnauthorized tests a user who is known to
 // the authentication system and authorized to do any actions.
 func TestAliceNotForbiddenOrUnauthorized(t *testing.T) {
@@ -627,10 +603,9 @@ func TestAliceNotForbiddenOrUnauthorized(t *testing.T) {
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
 			opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
-		},
-		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authentication.Authenticator = getTestTokenAuth()
-			config.GenericConfig.Authorization.Authorizer = allowAliceAuthorizer{}
+			opts.Authentication.TokenFile.TokenFile = "testdata/tokens.csv"
+			opts.Authorization.Modes = []string{"ABAC"}
+			opts.Authorization.PolicyFile = "testdata/allowalice.jsonl"
 		},
 	})
 	defer tearDownFn()
@@ -704,10 +679,9 @@ func TestBobIsForbidden(t *testing.T) {
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
 			opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
-		},
-		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authentication.Authenticator = getTestTokenAuth()
-			config.GenericConfig.Authorization.Authorizer = allowAliceAuthorizer{}
+			opts.Authentication.TokenFile.TokenFile = "testdata/tokens.csv"
+			opts.Authorization.Modes = []string{"ABAC"}
+			opts.Authorization.PolicyFile = "testdata/allowalice.jsonl"
 		},
 	})
 	defer tearDownFn()
@@ -754,10 +728,9 @@ func TestUnknownUserIsUnauthorized(t *testing.T) {
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
 			opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
-		},
-		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authentication.Authenticator = getTestTokenAuth()
-			config.GenericConfig.Authorization.Authorizer = allowAliceAuthorizer{}
+			opts.Authentication.TokenFile.TokenFile = "testdata/tokens.csv"
+			opts.Authorization.Modes = []string{"ABAC"}
+			opts.Authorization.PolicyFile = "testdata/allowalice.jsonl"
 		},
 	})
 	defer tearDownFn()
@@ -807,10 +780,13 @@ func (impersonateAuthorizer) Authorize(ctx context.Context, a authorizer.Attribu
 	if a.GetUser() != nil && a.GetUser().GetName() == "alice" && a.GetVerb() != "impersonate" {
 		return authorizer.DecisionAllow, "", nil
 	}
-	// bob can impersonate anyone, but that it
+	// bob can impersonate anyone, but that's it
 	if a.GetUser() != nil && a.GetUser().GetName() == "bob" && a.GetVerb() == "impersonate" {
 		return authorizer.DecisionAllow, "", nil
 	}
+	if a.GetUser() != nil && a.GetUser().GetName() == "bob" && a.GetVerb() != "impersonate" {
+		return authorizer.DecisionDeny, "", nil
+	}
 	// service accounts can do everything
 	if a.GetUser() != nil && strings.HasPrefix(a.GetUser().GetName(), serviceaccount.ServiceAccountUsernamePrefix) {
 		return authorizer.DecisionAllow, "", nil
@@ -824,10 +800,11 @@ func TestImpersonateIsForbidden(t *testing.T) {
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
 			opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
+			opts.Authentication.TokenFile.TokenFile = "testdata/tokens.csv"
 		},
 		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authentication.Authenticator = getTestTokenAuth()
-			config.GenericConfig.Authorization.Authorizer = impersonateAuthorizer{}
+			// Prepend an impersonation authorizer with specific opinions about alice and bob
+			config.GenericConfig.Authorization.Authorizer = unionauthz.New(impersonateAuthorizer{}, config.GenericConfig.Authorization.Authorizer)
 		},
 	})
 	defer tearDownFn()
@@ -860,7 +837,7 @@ func TestImpersonateIsForbidden(t *testing.T) {
 			// Expect all of bob's actions to return Forbidden
 			if resp.StatusCode != http.StatusForbidden {
 				t.Logf("case %v", r)
-				t.Errorf("Expected not status Forbidden, but got %s", resp.Status)
+				t.Errorf("Expected status Forbidden, but got %s", resp.Status)
 			}
 		}()
 	}
@@ -1101,23 +1078,13 @@ func csrPEM(t *testing.T) []byte {
 	return req
 }
 
-func newAuthorizerWithContents(t *testing.T, contents string) authorizer.Authorizer {
-	f, err := os.CreateTemp("", "auth_test")
-	if err != nil {
-		t.Fatalf("unexpected error creating policyfile: %v", err)
-	}
-	f.Close()
-	defer os.Remove(f.Name())
-
-	if err := os.WriteFile(f.Name(), []byte(contents), 0700); err != nil {
+func newABACFileWithContents(t *testing.T, contents string) string {
+	dir := t.TempDir()
+	file := filepath.Join(dir, "auth_test")
+	if err := os.WriteFile(file, []byte(contents), 0700); err != nil {
 		t.Fatalf("unexpected error writing policyfile: %v", err)
 	}
-
-	pl, err := abac.NewFromFile(f.Name())
-	if err != nil {
-		t.Fatalf("unexpected error creating authorizer from policyfile: %v", err)
-	}
-	return pl
+	return file
 }
 
 type trackingAuthorizer struct {
@@ -1137,10 +1104,10 @@ func TestAuthorizationAttributeDetermination(t *testing.T) {
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
 			opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
+			opts.Authentication.TokenFile.TokenFile = "testdata/tokens.csv"
 		},
 		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authentication.Authenticator = getTestTokenAuth()
-			config.GenericConfig.Authorization.Authorizer = trackingAuthorizer
+			config.GenericConfig.Authorization.Authorizer = unionauthz.New(config.GenericConfig.Authorization.Authorizer, trackingAuthorizer)
 		},
 	})
 	defer tearDownFn()
@@ -1203,18 +1170,13 @@ func TestAuthorizationAttributeDetermination(t *testing.T) {
 // TestNamespaceAuthorization tests that authorization can be controlled
 // by namespace.
 func TestNamespaceAuthorization(t *testing.T) {
-	// This file has alice and bob in it.
-	a := newAuthorizerWithContents(t, `{"namespace": "auth-namespace"}
-`)
-
 	kubeClient, kubeConfig, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
 			opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
-		},
-		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authentication.Authenticator = getTestTokenAuth()
-			config.GenericConfig.Authorization.Authorizer = a
+			opts.Authentication.TokenFile.TokenFile = "testdata/tokens.csv"
+			opts.Authorization.PolicyFile = newABACFileWithContents(t, `{"namespace": "auth-namespace"}`)
+			opts.Authorization.Modes = []string{"ABAC"}
 		},
 	})
 	defer tearDownFn()
@@ -1309,18 +1271,13 @@ func TestNamespaceAuthorization(t *testing.T) {
 // TestKindAuthorization tests that authorization can be controlled
 // by namespace.
 func TestKindAuthorization(t *testing.T) {
-	// This file has alice and bob in it.
-	a := newAuthorizerWithContents(t, `{"resource": "services"}
-`)
-
 	kubeClient, kubeConfig, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
 			opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
-		},
-		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authentication.Authenticator = getTestTokenAuth()
-			config.GenericConfig.Authorization.Authorizer = a
+			opts.Authentication.TokenFile.TokenFile = "testdata/tokens.csv"
+			opts.Authorization.PolicyFile = newABACFileWithContents(t, `{"resource": "services"}`)
+			opts.Authorization.Modes = []string{"ABAC"}
 		},
 	})
 	defer tearDownFn()
@@ -1397,17 +1354,13 @@ func TestKindAuthorization(t *testing.T) {
 // TestReadOnlyAuthorization tests that authorization can be controlled
 // by namespace.
 func TestReadOnlyAuthorization(t *testing.T) {
-	// This file has alice and bob in it.
-	a := newAuthorizerWithContents(t, `{"readonly": true}`)
-
 	kubeClient, kubeConfig, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
 			opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
-		},
-		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authentication.Authenticator = getTestTokenAuth()
-			config.GenericConfig.Authorization.Authorizer = a
+			opts.Authentication.TokenFile.TokenFile = "testdata/tokens.csv"
+			opts.Authorization.PolicyFile = newABACFileWithContents(t, `{"readonly": true}`)
+			opts.Authorization.Modes = []string{"ABAC"}
 		},
 	})
 	defer tearDownFn()
@@ -1484,12 +1437,13 @@ func testWebhookTokenAuthenticator(customDialer bool, t *testing.T) {
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
 			opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
+			opts.Authorization.Modes = []string{"ABAC"}
+			opts.Authorization.PolicyFile = "testdata/allowalice.jsonl"
 		},
 		ModifyServerConfig: func(config *controlplane.Config) {
 			config.GenericConfig.Authentication.Authenticator = group.NewAuthenticatedGroupAdder(authenticator)
 			// Disable checking API audiences that is set by testserver by default.
 			config.GenericConfig.Authentication.APIAudiences = nil
-			config.GenericConfig.Authorization.Authorizer = allowAliceAuthorizer{}
 		},
 	})
 	defer tearDownFn()
diff --git a/test/integration/auth/bootstraptoken_test.go b/test/integration/auth/bootstraptoken_test.go
index c538cdeb1bb..b513d10dfe7 100644
--- a/test/integration/auth/bootstraptoken_test.go
+++ b/test/integration/auth/bootstraptoken_test.go
@@ -29,9 +29,9 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apiserver/pkg/authentication/group"
 	"k8s.io/apiserver/pkg/authentication/request/bearertoken"
-	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
 	"k8s.io/client-go/rest"
 	bootstrapapi "k8s.io/cluster-bootstrap/token/api"
+	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 	"k8s.io/kubernetes/pkg/controlplane"
 	"k8s.io/kubernetes/plugin/pkg/auth/authenticator/token/bootstrap"
 	"k8s.io/kubernetes/test/integration"
@@ -122,9 +122,11 @@ func TestBootstrapTokenAuth(t *testing.T) {
 			authenticator := group.NewAuthenticatedGroupAdder(bearertoken.New(bootstrap.NewTokenAuthenticator(bootstrapSecrets{test.secret})))
 
 			kubeClient, kubeConfig, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{
+				ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
+					opts.Authorization.Modes = []string{"AlwaysAllow"}
+				},
 				ModifyServerConfig: func(config *controlplane.Config) {
 					config.GenericConfig.Authentication.Authenticator = authenticator
-					config.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer()
 				},
 			})
 			defer tearDownFn()
diff --git a/test/integration/auth/dynamic_client_test.go b/test/integration/auth/dynamic_client_test.go
index 823a7d0dd41..1192e46e656 100644
--- a/test/integration/auth/dynamic_client_test.go
+++ b/test/integration/auth/dynamic_client_test.go
@@ -24,12 +24,10 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
-	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
 	clientset "k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/controller-manager/pkg/clientbuilder"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
-	"k8s.io/kubernetes/pkg/controlplane"
 	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
 	"k8s.io/kubernetes/test/integration/framework"
 )
@@ -67,9 +65,7 @@ func TestDynamicClientBuilder(t *testing.T) {
 			}
 			opts.Authentication.ServiceAccounts.Issuers = []string{iss}
 			opts.Authentication.ServiceAccounts.KeyFiles = []string{tmpfile.Name()}
-		},
-		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer()
+			opts.Authorization.Modes = []string{"AlwaysAllow"}
 		},
 	})
 	defer tearDownFn()
diff --git a/test/integration/auth/rbac_test.go b/test/integration/auth/rbac_test.go
index 81aaa7d55cf..fbdcb975b14 100644
--- a/test/integration/auth/rbac_test.go
+++ b/test/integration/auth/rbac_test.go
@@ -35,9 +35,11 @@ import (
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/apiserver/pkg/authentication/group"
 	"k8s.io/apiserver/pkg/authentication/request/bearertoken"
+	unionauthn "k8s.io/apiserver/pkg/authentication/request/union"
 	"k8s.io/apiserver/pkg/authentication/token/tokenfile"
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
+	unionauthz "k8s.io/apiserver/pkg/authorization/union"
 	genericfeatures "k8s.io/apiserver/pkg/features"
 	"k8s.io/apiserver/pkg/registry/generic"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
@@ -552,10 +554,16 @@ func TestRBAC(t *testing.T) {
 					// Also disable namespace lifecycle to workaroung the test limitation that first creates
 					// roles/rolebindings and only then creates corresponding namespaces.
 					opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount", "NamespaceLifecycle"}
+					// Disable built-in authorizers
+					opts.Authorization.Modes = []string{"AlwaysDeny"}
 				},
 				ModifyServerConfig: func(config *controlplane.Config) {
-					config.GenericConfig.Authentication.Authenticator = authenticator
-					config.GenericConfig.Authorization.Authorizer, tearDownAuthorizerFn = newRBACAuthorizer(t, config)
+					// Append our custom test authenticator
+					config.GenericConfig.Authentication.Authenticator = unionauthn.New(config.GenericConfig.Authentication.Authenticator, authenticator)
+					// Append our custom test authorizer
+					var rbacAuthz authorizer.Authorizer
+					rbacAuthz, tearDownAuthorizerFn = newRBACAuthorizer(t, config)
+					config.GenericConfig.Authorization.Authorizer = unionauthz.New(config.GenericConfig.Authorization.Authorizer, rbacAuthz)
 				},
 			})
 			defer tearDownFn()
@@ -666,20 +674,9 @@ func TestRBAC(t *testing.T) {
 }
 
 func TestBootstrapping(t *testing.T) {
-	superUser := "admin/system:masters"
-
-	var tearDownAuthorizerFn func()
-	defer func() {
-		if tearDownAuthorizerFn != nil {
-			tearDownAuthorizerFn()
-		}
-	}()
 	clientset, _, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{
-		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authentication.Authenticator = bearertoken.New(tokenfile.New(map[string]*user.DefaultInfo{
-				superUser: {Name: "admin", Groups: []string{"system:masters"}},
-			}))
-			config.GenericConfig.Authorization.Authorizer, tearDownAuthorizerFn = newRBACAuthorizer(t, config)
+		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
+			opts.Authorization.Modes = []string{"RBAC"}
 		},
 	})
 	defer tearDownFn()
@@ -732,14 +729,6 @@ func TestDiscoveryUpgradeBootstrapping(t *testing.T) {
 			tearDownFn()
 		}
 	}()
-	var tearDownAuthorizerFn func()
-	defer func() {
-		if tearDownAuthorizerFn != nil {
-			tearDownAuthorizerFn()
-		}
-	}()
-
-	superUser := "admin/system:masters"
 
 	etcdConfig := framework.SharedEtcd()
 
@@ -747,12 +736,7 @@ func TestDiscoveryUpgradeBootstrapping(t *testing.T) {
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Ensure we're using the same etcd across apiserver restarts.
 			opts.Etcd.StorageConfig = *etcdConfig
-		},
-		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authentication.Authenticator = bearertoken.New(tokenfile.New(map[string]*user.DefaultInfo{
-				superUser: {Name: "admin", Groups: []string{"system:masters"}},
-			}))
-			config.GenericConfig.Authorization.Authorizer, tearDownAuthorizerFn = newRBACAuthorizer(t, config)
+			opts.Authorization.Modes = []string{"RBAC"}
 		},
 	})
 
@@ -793,8 +777,6 @@ func TestDiscoveryUpgradeBootstrapping(t *testing.T) {
 	// Stop the first API server.
 	tearDownFn()
 	tearDownFn = nil
-	tearDownAuthorizerFn()
-	tearDownAuthorizerFn = nil
 
 	// Check that upgraded API servers inherit `system:public-info-viewer` settings from
 	// `system:discovery`, and respect auto-reconciliation annotations.
@@ -802,12 +784,7 @@ func TestDiscoveryUpgradeBootstrapping(t *testing.T) {
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Ensure we're using the same etcd across apiserver restarts.
 			opts.Etcd.StorageConfig = *etcdConfig
-		},
-		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authentication.Authenticator = bearertoken.New(tokenfile.New(map[string]*user.DefaultInfo{
-				superUser: {Name: "admin", Groups: []string{"system:masters"}},
-			}))
-			config.GenericConfig.Authorization.Authorizer, tearDownAuthorizerFn = newRBACAuthorizer(t, config)
+			opts.Authorization.Modes = []string{"RBAC"}
 		},
 	})
 
diff --git a/test/integration/auth/selfsubjectreview_test.go b/test/integration/auth/selfsubjectreview_test.go
index c5e729366c2..bd511182349 100644
--- a/test/integration/auth/selfsubjectreview_test.go
+++ b/test/integration/auth/selfsubjectreview_test.go
@@ -30,7 +30,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/apiserver/pkg/authentication/user"
-	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
@@ -96,6 +95,10 @@ func TestGetsSelfAttributes(t *testing.T) {
 	}
 
 	kubeClient, _, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{
+		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
+			opts.APIEnablement.RuntimeConfig.Set("authentication.k8s.io/v1alpha1=true")
+			opts.Authorization.Modes = []string{"AlwaysAllow"}
+		},
 		ModifyServerConfig: func(config *controlplane.Config) {
 			// Unset BearerToken to disable BearerToken authenticator.
 			config.GenericConfig.LoopbackClientConfig.BearerToken = ""
@@ -104,10 +107,6 @@ func TestGetsSelfAttributes(t *testing.T) {
 				defer respMu.RUnlock()
 				return &authenticator.Response{User: response}, true, nil
 			})
-			config.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer()
-		},
-		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
-			opts.APIEnablement.RuntimeConfig.Set("authentication.k8s.io/v1alpha1=true")
 		},
 	})
 	defer tearDownFn()
@@ -156,6 +155,10 @@ func TestGetsSelfAttributesError(t *testing.T) {
 	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APISelfSubjectReview, true)()
 
 	kubeClient, _, tearDownFn := framework.StartTestServer(t, framework.TestServerSetup{
+		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
+			opts.APIEnablement.RuntimeConfig.Set("authentication.k8s.io/v1alpha1=true")
+			opts.Authorization.Modes = []string{"AlwaysAllow"}
+		},
 		ModifyServerConfig: func(config *controlplane.Config) {
 			// Unset BearerToken to disable BearerToken authenticator.
 			config.GenericConfig.LoopbackClientConfig.BearerToken = ""
@@ -170,10 +173,6 @@ func TestGetsSelfAttributesError(t *testing.T) {
 
 				return nil, false, fmt.Errorf("test error")
 			})
-			config.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer()
-		},
-		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
-			opts.APIEnablement.RuntimeConfig.Set("authentication.k8s.io/v1alpha1=true")
 		},
 	})
 	defer tearDownFn()
diff --git a/test/integration/auth/svcaccttoken_test.go b/test/integration/auth/svcaccttoken_test.go
index 11869ac7e6b..9daf6c1bc3d 100644
--- a/test/integration/auth/svcaccttoken_test.go
+++ b/test/integration/auth/svcaccttoken_test.go
@@ -19,12 +19,10 @@ package auth
 import (
 	"bytes"
 	"context"
-	"crypto/ecdsa"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"net/url"
 	"reflect"
@@ -42,18 +40,12 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
-	"k8s.io/apiserver/pkg/authentication/request/bearertoken"
 	apiserverserviceaccount "k8s.io/apiserver/pkg/authentication/serviceaccount"
-	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
-	v1listers "k8s.io/client-go/listers/core/v1"
 	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/cache"
-	"k8s.io/client-go/util/keyutil"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 	"k8s.io/kubernetes/pkg/apis/core"
-	serviceaccountgetter "k8s.io/kubernetes/pkg/controller/serviceaccount"
 	"k8s.io/kubernetes/pkg/controlplane"
 	"k8s.io/kubernetes/pkg/serviceaccount"
 	"k8s.io/kubernetes/test/integration/framework"
@@ -71,15 +63,6 @@ AwEHoUQDQgAEH6cuzP8XuD5wal6wf9M6xDljTOPLX2i8uIp/C/ASqiIGUeeKQtX0
 )
 
 func TestServiceAccountTokenCreate(t *testing.T) {
-
-	// Build client config, clientset, and informers
-	sk, err := keyutil.ParsePrivateKeyPEM([]byte(ecdsaPrivateKey))
-	if err != nil {
-		t.Fatalf("err: %v", err)
-	}
-
-	pk := sk.(*ecdsa.PrivateKey).PublicKey
-
 	const iss = "https://foo.bar.example.com"
 	aud := authenticator.Audiences{"api"}
 
@@ -89,12 +72,7 @@ func TestServiceAccountTokenCreate(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	gcs := &clientset.Clientset{}
-
-	tokenGenerator, err := serviceaccount.JWTTokenGenerator(iss, sk)
-	if err != nil {
-		t.Fatalf("err: %v", err)
-	}
+	var tokenGenerator serviceaccount.TokenGenerator
 
 	// Start the server
 	var serverAddress string
@@ -102,45 +80,21 @@ func TestServiceAccountTokenCreate(t *testing.T) {
 		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
 			// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
 			opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"}
+			opts.Authorization.Modes = []string{"AlwaysAllow"}
+			// Disable token cache so we can check reaction to service account deletion quickly
+			opts.Authentication.TokenSuccessCacheTTL = 0
+			opts.Authentication.TokenFailureCacheTTL = 0
+			// Pin to fixed URLs for easier testing
+			opts.Authentication.ServiceAccounts.JWKSURI = "https:///openid/v1/jwks"
+			opts.Authentication.ServiceAccounts.Issuers = []string{iss}
+			opts.Authentication.APIAudiences = aud
 		},
 		ModifyServerConfig: func(config *controlplane.Config) {
-			config.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer()
-			config.GenericConfig.Authentication.APIAudiences = aud
-			config.GenericConfig.Authentication.Authenticator = bearertoken.New(
-				serviceaccount.JWTTokenAuthenticator(
-					[]string{iss},
-					[]interface{}{&pk},
-					aud,
-					serviceaccount.NewValidator(serviceaccountgetter.NewGetterFromClient(
-						gcs,
-						v1listers.NewSecretLister(newIndexer(func(namespace, name string) (interface{}, error) {
-							return gcs.CoreV1().Secrets(namespace).Get(context.TODO(), name, metav1.GetOptions{})
-						})),
-						v1listers.NewServiceAccountLister(newIndexer(func(namespace, name string) (interface{}, error) {
-							return gcs.CoreV1().ServiceAccounts(namespace).Get(context.TODO(), name, metav1.GetOptions{})
-						})),
-						v1listers.NewPodLister(newIndexer(func(namespace, name string) (interface{}, error) {
-							return gcs.CoreV1().Pods(namespace).Get(context.TODO(), name, metav1.GetOptions{})
-						})),
-					)),
-				),
-			)
+			// extract token generator
+			tokenGenerator = config.ExtraConfig.ServiceAccountIssuer
 
-			config.ExtraConfig.ServiceAccountIssuer = tokenGenerator
 			config.ExtraConfig.ServiceAccountMaxExpiration = maxExpirationDuration
 			config.ExtraConfig.ExtendExpiration = true
-
-			config.ExtraConfig.ServiceAccountIssuerURL = iss
-			config.ExtraConfig.ServiceAccountJWKSURI = ""
-			config.ExtraConfig.ServiceAccountPublicKeys = []interface{}{&pk}
-
-			// Compute the serverAddress.
-			serverAddress = config.GenericConfig.ExternalAddress
-			_, port, err := config.GenericConfig.SecureServing.HostPort()
-			if err != nil {
-				t.Fatalf("Couldn't get server port: %v", err)
-			}
-			serverAddress = net.JoinHostPort(serverAddress, strconv.Itoa(port))
 		},
 	})
 	defer tearDownFn()
@@ -156,7 +110,6 @@ func TestServiceAccountTokenCreate(t *testing.T) {
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
-	*gcs = *cs
 
 	kubeConfig.NegotiatedSerializer = scheme.Codecs.WithoutConversion()
 	rc, err := rest.UnversionedRESTClientFor(kubeConfig)
@@ -917,25 +870,36 @@ func TestServiceAccountTokenCreate(t *testing.T) {
 
 func doTokenReview(t *testing.T, cs clientset.Interface, treq *authenticationv1.TokenRequest, expectErr bool) authenticationv1.UserInfo {
 	t.Helper()
-	trev, err := cs.AuthenticationV1().TokenReviews().Create(context.TODO(), &authenticationv1.TokenReview{
-		Spec: authenticationv1.TokenReviewSpec{
-			Token: treq.Status.Token,
-		},
-	}, metav1.CreateOptions{})
-	if err != nil {
-		t.Fatalf("err: %v", err)
+	tries := 0
+	for {
+		trev, err := cs.AuthenticationV1().TokenReviews().Create(context.TODO(), &authenticationv1.TokenReview{
+			Spec: authenticationv1.TokenReviewSpec{
+				Token: treq.Status.Token,
+			},
+		}, metav1.CreateOptions{})
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		t.Logf("status: %+v", trev.Status)
+		if (trev.Status.Error != "") && !expectErr {
+			t.Fatalf("expected no error but got: %v", trev.Status.Error)
+		}
+		if (trev.Status.Error == "") && expectErr {
+			// if we expected an error and didn't get one, retry
+			// to let changes that invalidate the token percolate through informers
+			if tries < 10 {
+				tries++
+				time.Sleep(100 * time.Millisecond)
+				t.Logf("expected error but got: %+v, retrying", trev.Status)
+				continue
+			}
+			t.Fatalf("expected error but got: %+v", trev.Status)
+		}
+		if !trev.Status.Authenticated && !expectErr {
+			t.Fatal("expected token to be authenticated but it wasn't")
+		}
+		return trev.Status.User
 	}
-	t.Logf("status: %+v", trev.Status)
-	if (trev.Status.Error != "") && !expectErr {
-		t.Fatalf("expected no error but got: %v", trev.Status.Error)
-	}
-	if (trev.Status.Error == "") && expectErr {
-		t.Fatalf("expected error but got: %+v", trev.Status)
-	}
-	if !trev.Status.Authenticated && !expectErr {
-		t.Fatal("expected token to be authenticated but it wasn't")
-	}
-	return trev.Status.User
 }
 
 func checkPayload(t *testing.T, tok string, want string, parts ...string) {
@@ -1043,26 +1007,6 @@ func createDeleteSecret(t *testing.T, cs clientset.Interface, sec *v1.Secret) (*
 	}
 }
 
-func newIndexer(get func(namespace, name string) (interface{}, error)) cache.Indexer {
-	return &fakeIndexer{get: get}
-}
-
-type fakeIndexer struct {
-	cache.Indexer
-	get func(namespace, name string) (interface{}, error)
-}
-
-func (f *fakeIndexer) GetByKey(key string) (interface{}, bool, error) {
-	parts := strings.SplitN(key, "/", 2)
-	namespace := parts[0]
-	name := ""
-	if len(parts) == 2 {
-		name = parts[1]
-	}
-	obj, err := f.get(namespace, name)
-	return obj, err == nil, err
-}
-
 type recordingWarningHandler struct {
 	warnings []string
 
diff --git a/test/integration/auth/testdata/allowalice.jsonl b/test/integration/auth/testdata/allowalice.jsonl
new file mode 100644
index 00000000000..1093f210a97
--- /dev/null
+++ b/test/integration/auth/testdata/allowalice.jsonl
@@ -0,0 +1 @@
+{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"alice", "namespace": "*", "resource": "*", "apiGroup": "*", "nonResourcePath": "*"}}
\ No newline at end of file
diff --git a/test/integration/auth/testdata/tokens.csv b/test/integration/auth/testdata/tokens.csv
new file mode 100644
index 00000000000..5f009760c09
--- /dev/null
+++ b/test/integration/auth/testdata/tokens.csv
@@ -0,0 +1,2 @@
+abc123,alice,1
+xyz987,bob,2