Imperative api pls (#434)

* Add aggregation layer support

* prefer testing.Cleanup

* add sni certs to server opts

* test cleanup

* append snicerts instead of overwriting

---------

Co-authored-by: Tom Lebreux <tom.lebreux@suse.com>
Co-authored-by: joshmeranda <joshua.meranda@gmail.com>

pkg/ext/apiserver.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"strings"
 	"sync"
+	"time"
 
 	metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -20,6 +21,7 @@ import (
 	"k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/rest"
 	genericapiserver "k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 	genericoptions "k8s.io/apiserver/pkg/server/options"
 	utilversion "k8s.io/apiserver/pkg/util/version"
 	openapicommon "k8s.io/kube-openapi/pkg/common"
@@ -43,6 +45,15 @@ type ExtensionAPIServerOptions struct {
 
 	// Authenticator will be used to authenticate requests coming to the
 	// extension API server. Required.
+	//
+	// If the authenticator implements [dynamiccertificates.CAContentProvider], the
+	// ClientCA will be set on the underlying SecureServing struct. If the authenticator
+	// implements [dynamiccertificates.ControllerRunner] too, then Run() will be called so
+	// that the authenticators can run in the background. (See DefaultAuthenticator for
+	// example).
+	//
+	// Use a UnionAuthenticator to have multiple ways of authenticating requests. See
+	// [NewUnionAuthenticator] for an example.
 	Authenticator authenticator.Request
 
 	// Authorizer will be used to authorize requests based on the user,
@@ -64,6 +75,8 @@ type ExtensionAPIServerOptions struct {
 	// If nil, the default version is the version of the Kubernetes Go library
 	// compiled in the final binary.
 	EffectiveVersion utilversion.EffectiveVersion
+
+	SNICerts []dynamiccertificates.SNICertKeyContentProvider
 }
 
 // ExtensionAPIServer wraps a [genericapiserver.GenericAPIServer] to implement
@@ -157,8 +170,12 @@ func NewExtensionAPIServer(scheme *runtime.Scheme, codecs serializer.CodecFactor
 	if err := recommendedOpts.SecureServing.ApplyTo(&config.SecureServing, &config.LoopbackClientConfig); err != nil {
 		return nil, fmt.Errorf("applyto secureserving: %w", err)
 	}
+	config.SecureServing.SNICerts = append(config.SecureServing.SNICerts, opts.SNICerts...)
 
 	config.Authentication.Authenticator = opts.Authenticator
+	if caContentProvider, ok := opts.Authenticator.(dynamiccertificates.CAContentProvider); ok {
+		config.SecureServing.ClientCA = caContentProvider
+	}
 
 	completedConfig := config.Complete()
 	genericServer, err := completedConfig.New("imperative-api", genericapiserver.NewEmptyDelegate())
@@ -187,6 +204,11 @@ func (s *ExtensionAPIServer) Run(ctx context.Context) error {
 		}
 	}
 	prepared := s.genericAPIServer.PrepareRun()
+
+	if _, _, err := prepared.NonBlockingRunWithContext(ctx, time.Second*5); err != nil {
+		return err
+	}
+
 	s.handlerMu.Lock()
 	s.handler = prepared.Handler
 	s.handlerMu.Unlock()

pkg/ext/apiserver_authentication.go

@@ -0,0 +1,257 @@
+package ext
+
+import (
+	"context"
+	"crypto/x509"
+	"fmt"
+	"net/http"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apiserver/pkg/apis/apiserver"
+	"k8s.io/apiserver/pkg/authentication/authenticator"
+	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
+	"k8s.io/apiserver/pkg/authentication/request/headerrequest"
+	authenticatorunion "k8s.io/apiserver/pkg/authentication/request/union"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+	"k8s.io/apiserver/pkg/server/options"
+	"k8s.io/client-go/kubernetes"
+)
+
+var _ dynamiccertificates.ControllerRunner = &UnionAuthenticator{}
+var _ dynamiccertificates.CAContentProvider = &UnionAuthenticator{}
+
+// UnionAuthenticator chains authenticators together to allow many ways of authenticating
+// requests for the extension API server. For example, we might want to use Rancher's
+// token authentication and fallback to the default authentication (mTLS) defined
+// by Kubernetes.
+//
+// UnionAuthenticator is both a [dynamiccertificates.ControllerRunner] and a
+// [dynamiccertificates.CAContentProvider].
+type UnionAuthenticator struct {
+	unionAuthenticator     authenticator.Request
+	unionCAContentProvider dynamiccertificates.CAContentProvider
+}
+
+// NewUnionAuthenticator creates a [UnionAuthenticator].
+//
+// The authenticators will be tried one by one, in the order they are given, until
+// one succeed or all fails.
+//
+// Here's an example usage:
+//
+//	customAuth := authenticator.RequestFunc(func(req *http.Request) (*Response, bool, error) {
+//		// use request to determine what the user is, otherwise return false
+//	})
+//	default, err := NewDefaultAuthenticator(client)
+//	if err != nil {
+//		return err
+//	}
+//	auth := NewUnionAuthenticator(customAuth, default)
+//	err = auth.RunOnce(ctx)
+func NewUnionAuthenticator(authenticators ...authenticator.Request) *UnionAuthenticator {
+	caContentProviders := make([]dynamiccertificates.CAContentProvider, 0, len(authenticators))
+	for _, auth := range authenticators {
+		auth, ok := auth.(dynamiccertificates.CAContentProvider)
+		if !ok {
+			continue
+		}
+		caContentProviders = append(caContentProviders, auth)
+	}
+	return &UnionAuthenticator{
+		unionAuthenticator:     authenticatorunion.New(authenticators...),
+		unionCAContentProvider: dynamiccertificates.NewUnionCAContentProvider(caContentProviders...),
+	}
+}
+
+// AuthenticateRequest implements [authenticator.Request]
+func (u *UnionAuthenticator) AuthenticateRequest(req *http.Request) (*authenticator.Response, bool, error) {
+	return u.unionAuthenticator.AuthenticateRequest(req)
+}
+
+// AuthenticateRequest implements [dynamiccertificates.Notifier]
+// This is part of the [dynamiccertificates.CAContentProvider] interface.
+func (u *UnionAuthenticator) AddListener(listener dynamiccertificates.Listener) {
+	u.unionCAContentProvider.AddListener(listener)
+}
+
+// AuthenticateRequest implements [dynamiccertificates.CAContentProvider]
+func (u *UnionAuthenticator) Name() string {
+	return u.unionCAContentProvider.Name()
+}
+
+// AuthenticateRequest implements [dynamiccertificates.CAContentProvider]
+func (u *UnionAuthenticator) CurrentCABundleContent() []byte {
+	return u.unionCAContentProvider.CurrentCABundleContent()
+}
+
+// AuthenticateRequest implements [dynamiccertificates.CAContentProvider]
+func (u *UnionAuthenticator) VerifyOptions() (x509.VerifyOptions, bool) {
+	return u.unionCAContentProvider.VerifyOptions()
+}
+
+// AuthenticateRequest implements [dynamiccertificates.CAContentProvider]
+func (u *UnionAuthenticator) RunOnce(ctx context.Context) error {
+	runner, ok := u.unionCAContentProvider.(dynamiccertificates.ControllerRunner)
+	if !ok {
+		return nil
+	}
+	return runner.RunOnce(ctx)
+}
+
+// AuthenticateRequest implements [dynamiccertificates.CAContentProvider]
+func (u *UnionAuthenticator) Run(ctx context.Context, workers int) {
+	runner, ok := u.unionCAContentProvider.(dynamiccertificates.ControllerRunner)
+	if !ok {
+		return
+	}
+
+	runner.Run(ctx, workers)
+}
+
+const (
+	authenticationConfigMapNamespace = metav1.NamespaceSystem
+	authenticationConfigMapName      = "extension-apiserver-authentication"
+)
+
+var _ dynamiccertificates.ControllerRunner = &DefaultAuthenticator{}
+var _ dynamiccertificates.CAContentProvider = &DefaultAuthenticator{}
+
+// DefaultAuthenticator is an [authenticator.Request] that authenticates a user by:
+//   - making sure the client uses a certificate signed by the CA defined in the
+//     `extension-apiserver-authentication` configmap in the `kube-system` namespace and
+//   - making sure the CN of the cert is part of the allow list, also defined in the same configmap
+//
+// This authentication is better explained in https://kubernetes.io/docs/tasks/extend-kubernetes/configure-aggregation-layer/
+//
+// This authenticator is a [dynamiccertificates.ControllerRunner] which means
+// it will run in the background to dynamically watch the content of the configmap.
+//
+// When using the DefaultAuthenticator, it is suggested to call RunOnce() to initialize
+// the CA state. It is also possible to watch for changes to the CA bundle with the AddListener()
+// method.  Here's an example usage:
+//
+//	auth, err := NewDefaultAuthenticator(client)
+//	if err != nil {
+//	   return err
+//	}
+//	auth.AddListener(myListener{auth: auth}) // myListener should react to CA bundle changes
+//	err = auth.RunOnce(ctx)
+type DefaultAuthenticator struct {
+	requestHeaderConfig *authenticatorfactory.RequestHeaderConfig
+	authenticator       authenticator.Request
+}
+
+// NewDefaultAuthenticator creates a DefaultAuthenticator
+func NewDefaultAuthenticator(client kubernetes.Interface) (*DefaultAuthenticator, error) {
+	requestHeaderConfig, err := createRequestHeaderConfig(client)
+	if err != nil {
+		return nil, fmt.Errorf("requestheaderconfig: %w", err)
+	}
+
+	cfg := authenticatorfactory.DelegatingAuthenticatorConfig{
+		Anonymous: &apiserver.AnonymousAuthConfig{
+			Enabled: false,
+		},
+		RequestHeaderConfig: requestHeaderConfig,
+	}
+
+	authenticator, _, err := cfg.New()
+	if err != nil {
+		return nil, err
+	}
+
+	return &DefaultAuthenticator{
+		requestHeaderConfig: requestHeaderConfig,
+		authenticator:       authenticator,
+	}, nil
+}
+
+// AuthenticateRequest implements [authenticator.Request]
+func (b *DefaultAuthenticator) AuthenticateRequest(req *http.Request) (*authenticator.Response, bool, error) {
+	return b.authenticator.AuthenticateRequest(req)
+}
+
+// AuthenticateRequest implements [dynamiccertificates.Notifier]
+// This is part of the [dynamiccertificates.CAContentProvider] interface.
+func (b *DefaultAuthenticator) AddListener(listener dynamiccertificates.Listener) {
+	b.requestHeaderConfig.CAContentProvider.AddListener(listener)
+}
+
+// AuthenticateRequest implements [dynamiccertificates.CAContentProvider]
+func (b *DefaultAuthenticator) Name() string {
+	return b.requestHeaderConfig.CAContentProvider.Name()
+}
+
+// AuthenticateRequest implements [dynamiccertificates.CAContentProvider]
+func (b *DefaultAuthenticator) CurrentCABundleContent() []byte {
+	return b.requestHeaderConfig.CAContentProvider.CurrentCABundleContent()
+}
+
+// AuthenticateRequest implements [dynamiccertificates.CAContentProvider]
+func (b *DefaultAuthenticator) VerifyOptions() (x509.VerifyOptions, bool) {
+	return b.requestHeaderConfig.CAContentProvider.VerifyOptions()
+}
+
+// AuthenticateRequest implements [dynamiccertificates.ControllerRunner]
+func (b *DefaultAuthenticator) RunOnce(ctx context.Context) error {
+	runner, ok := b.requestHeaderConfig.CAContentProvider.(dynamiccertificates.ControllerRunner)
+	if !ok {
+		return nil
+	}
+	return runner.RunOnce(ctx)
+}
+
+// AuthenticateRequest implements [dynamiccertificates.ControllerRunner].
+//
+// It will be called by the "SecureServing" when starting the extension API server
+func (b *DefaultAuthenticator) Run(ctx context.Context, workers int) {
+	runner, ok := b.requestHeaderConfig.CAContentProvider.(dynamiccertificates.ControllerRunner)
+	if !ok {
+		return
+	}
+
+	runner.Run(ctx, workers)
+}
+
+// Copied from https://github.com/kubernetes/apiserver/blob/v0.30.1/pkg/server/options/authentication.go#L407
+func createRequestHeaderConfig(client kubernetes.Interface) (*authenticatorfactory.RequestHeaderConfig, error) {
+	dynamicRequestHeaderProvider, err := newDynamicRequestHeaderController(client)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create request header authentication config: %v", err)
+	}
+
+	return &authenticatorfactory.RequestHeaderConfig{
+		CAContentProvider:   dynamicRequestHeaderProvider,
+		UsernameHeaders:     headerrequest.StringSliceProvider(headerrequest.StringSliceProviderFunc(dynamicRequestHeaderProvider.UsernameHeaders)),
+		GroupHeaders:        headerrequest.StringSliceProvider(headerrequest.StringSliceProviderFunc(dynamicRequestHeaderProvider.GroupHeaders)),
+		ExtraHeaderPrefixes: headerrequest.StringSliceProvider(headerrequest.StringSliceProviderFunc(dynamicRequestHeaderProvider.ExtraHeaderPrefixes)),
+		AllowedClientNames:  headerrequest.StringSliceProvider(headerrequest.StringSliceProviderFunc(dynamicRequestHeaderProvider.AllowedClientNames)),
+	}, nil
+}
+
+// Copied from https://github.com/kubernetes/apiserver/blob/v0.30.1/pkg/server/options/authentication_dynamic_request_header.go#L42
+func newDynamicRequestHeaderController(client kubernetes.Interface) (*options.DynamicRequestHeaderController, error) {
+	requestHeaderCAController, err := dynamiccertificates.NewDynamicCAFromConfigMapController(
+		"client-ca",
+		authenticationConfigMapNamespace,
+		authenticationConfigMapName,
+		"requestheader-client-ca-file",
+		client)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create DynamicCAFromConfigMap controller: %v", err)
+	}
+
+	requestHeaderAuthRequestController := headerrequest.NewRequestHeaderAuthRequestController(
+		authenticationConfigMapName,
+		authenticationConfigMapNamespace,
+		client,
+		"requestheader-username-headers",
+		"requestheader-group-headers",
+		"requestheader-extra-headers-prefix",
+		"requestheader-allowed-names",
+	)
+	return &options.DynamicRequestHeaderController{
+		ConfigMapCAController:              requestHeaderCAController,
+		RequestHeaderAuthRequestController: requestHeaderAuthRequestController,
+	}, nil
+}

pkg/ext/apiserver_authentication_test.go

@@ -2,6 +2,7 @@ package ext
 
 import (
 	"context"
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -11,6 +12,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
@@ -64,7 +66,7 @@ func TestAuthenticationCustom(t *testing.T) {
 		testStore: newDefaultTestStore(),
 		userCh:    make(chan user.Info, 100),
 	}
-	extensionAPIServer, cleanup, err := setupExtensionAPIServer(t, scheme, store, func(opts *ExtensionAPIServerOptions) {
+	extensionAPIServer, err := setupExtensionAPIServer(t, scheme, store, func(opts *ExtensionAPIServerOptions) {
 		opts.Listener = ln
 		opts.Authorizer = authorizer.AuthorizerFunc(authzAllowAll)
 		opts.Authenticator = authenticator.RequestFunc(func(req *http.Request) (*authenticator.Response, bool, error) {
@@ -81,7 +83,6 @@ func TestAuthenticationCustom(t *testing.T) {
 		})
 	}, nil)
 	require.NoError(t, err)
-	defer cleanup()
 
 	unauthorized := apierrors.NewUnauthorized("Unauthorized")
 	unauthorized.ErrStatus.Kind = "Status"
@@ -179,3 +180,267 @@ func TestAuthenticationCustom(t *testing.T) {
 		})
 	}
 }
+
+func (s *ExtensionAPIServerSuite) TestAuthenticationDefault() {
+	t := s.T()
+
+	// Same CA but CN not in the list allowed
+	notAllowedCertPair, err := s.ca.NewClientCert("system:not-allowed")
+	require.NoError(t, err)
+	notAllowedCert, notAllowedKey, err := notAllowedCertPair.AsBytes()
+	require.NoError(t, err)
+
+	badCA, err := NewTinyCA()
+	require.NoError(t, err)
+	badCertPair, err := badCA.NewClientCert("system:auth-proxy")
+	require.NoError(t, err)
+	badCert, badKey, err := badCertPair.AsBytes()
+	require.NoError(t, err)
+
+	cert, key, err := s.cert.AsBytes()
+	require.NoError(t, err)
+	certificate, err := tls.X509KeyPair(cert, key)
+	require.NoError(t, err)
+
+	badCACertificate, err := tls.X509KeyPair(badCert, badKey)
+	require.NoError(t, err)
+
+	notAllowedCertificate, err := tls.X509KeyPair(notAllowedCert, notAllowedKey)
+	require.NoError(t, err)
+
+	scheme := runtime.NewScheme()
+	AddToScheme(scheme)
+
+	store := &authnTestStore{
+		testStore: newDefaultTestStore(),
+		userCh:    make(chan user.Info, 100),
+	}
+	defaultAuth, err := NewDefaultAuthenticator(s.client)
+	require.NoError(t, err)
+
+	func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
+		defer cancel()
+		err = defaultAuth.RunOnce(ctx)
+		require.NoError(t, err)
+	}()
+
+	ln, port, err := options.CreateListener("", ":0", net.ListenConfig{})
+	require.NoError(t, err)
+
+	_, err = setupExtensionAPIServer(t, scheme, store, func(opts *ExtensionAPIServerOptions) {
+		opts.Listener = ln
+		opts.Authenticator = defaultAuth
+		opts.Authorizer = authorizer.AuthorizerFunc(authzAllowAll)
+	}, nil)
+	require.NoError(t, err)
+
+	allPaths := []string{
+		"/",
+		"/openapi/v2",
+		"/openapi/v3",
+		"/openapi/v3/apis/ext.cattle.io/v1",
+		"/apis",
+		"/apis/ext.cattle.io",
+		"/apis/ext.cattle.io/v1",
+		"/apis/ext.cattle.io/v1/testtypes",
+		"/apis/ext.cattle.io/v1/testtypes/foo",
+	}
+
+	type test struct {
+		name   string
+		certs  []tls.Certificate
+		paths  []string
+		user   string
+		groups []string
+
+		expectedStatusCode int
+		expectedUser       *user.DefaultInfo
+	}
+	tests := []test{
+		{
+			name:   "authenticated request check user",
+			certs:  []tls.Certificate{certificate},
+			paths:  []string{"/apis/ext.cattle.io/v1/testtypes"},
+			user:   "my-user",
+			groups: []string{"my-group"},
+
+			expectedStatusCode: http.StatusOK,
+			expectedUser:       &user.DefaultInfo{Name: "my-user", Groups: []string{"my-group", "system:authenticated"}, Extra: map[string][]string{}},
+		},
+		{
+			name:   "authenticated request all paths",
+			certs:  []tls.Certificate{certificate},
+			paths:  allPaths,
+			user:   "my-user",
+			groups: []string{"my-group"},
+
+			expectedStatusCode: http.StatusOK,
+		},
+		{
+			name:   "authenticated request to unknown endpoint",
+			certs:  []tls.Certificate{certificate},
+			paths:  []string{"/unknown"},
+			user:   "my-user",
+			groups: []string{"my-group"},
+
+			expectedStatusCode: http.StatusNotFound,
+		},
+		{
+			name:   "no client certs",
+			paths:  append(allPaths, "/unknown"),
+			user:   "my-user",
+			groups: []string{"my-group"},
+
+			expectedStatusCode: http.StatusUnauthorized,
+		},
+		{
+			name:   "client certs from bad CA",
+			certs:  []tls.Certificate{badCACertificate},
+			paths:  append(allPaths, "/unknown"),
+			user:   "my-user",
+			groups: []string{"my-group"},
+
+			expectedStatusCode: http.StatusUnauthorized,
+		},
+		{
+			name:   "client certs with CN not allowed",
+			certs:  []tls.Certificate{notAllowedCertificate},
+			paths:  append(allPaths, "/unknown"),
+			user:   "my-user",
+			groups: []string{"my-group"},
+
+			expectedStatusCode: http.StatusUnauthorized,
+		},
+		{
+			name:   "no user",
+			paths:  append(allPaths, "/unknown"),
+			groups: []string{"my-group"},
+
+			expectedStatusCode: http.StatusUnauthorized,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			httpClient := http.Client{
+				Transport: &http.Transport{
+					TLSClientConfig: &tls.Config{
+						InsecureSkipVerify: true,
+						Certificates:       test.certs,
+					},
+				},
+			}
+
+			for _, path := range test.paths {
+				req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://127.0.0.1:%d%s", port, path), nil)
+				require.NoError(t, err)
+				if test.user != "" {
+					req.Header.Set("X-Remote-User", test.user)
+				}
+				for _, group := range test.groups {
+					req.Header.Add("X-Remote-Group", group)
+				}
+
+				// Eventually because the cache for auth might not be synced yet
+				require.EventuallyWithT(t, func(c *assert.CollectT) {
+					resp, err := httpClient.Do(req)
+					require.NoError(t, err)
+					defer resp.Body.Close()
+
+					require.Equal(t, test.expectedStatusCode, resp.StatusCode)
+				}, 5*time.Second, 110*time.Millisecond)
+
+				if test.expectedUser != nil {
+					authUser, found := store.getUser()
+					require.True(t, found)
+					require.Equal(t, test.expectedUser, authUser)
+				}
+			}
+		})
+	}
+}
+
+func (s *ExtensionAPIServerSuite) TestAuthenticationUnion() {
+	t := s.T()
+
+	scheme := runtime.NewScheme()
+	AddToScheme(scheme)
+
+	cert, key, err := s.cert.AsBytes()
+	require.NoError(t, err)
+	certificate, err := tls.X509KeyPair(cert, key)
+	require.NoError(t, err)
+
+	defaultAuth, err := NewDefaultAuthenticator(s.client)
+	require.NoError(t, err)
+
+	customAuth := authenticator.RequestFunc(func(req *http.Request) (*authenticator.Response, bool, error) {
+		user, ok := request.UserFrom(req.Context())
+		if !ok {
+			return nil, false, nil
+		}
+		if user.GetName() == "error" {
+			return nil, false, fmt.Errorf("fake error")
+		}
+		return &authenticator.Response{
+			User: user,
+		}, true, nil
+	})
+	auth := NewUnionAuthenticator(customAuth, defaultAuth)
+	func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
+		defer cancel()
+		err = auth.RunOnce(ctx)
+		require.NoError(t, err)
+	}()
+
+	ln, port, err := options.CreateListener("", ":0", net.ListenConfig{})
+	require.NoError(t, err)
+
+	store := &authnTestStore{
+		testStore: newDefaultTestStore(),
+		userCh:    make(chan user.Info, 100),
+	}
+	extensionAPIServer, err := setupExtensionAPIServer(t, scheme, store, func(opts *ExtensionAPIServerOptions) {
+		opts.Listener = ln
+		opts.Authorizer = authorizer.AuthorizerFunc(authzAllowAll)
+		opts.Authenticator = auth
+	}, nil)
+	require.NoError(t, err)
+
+	httpClient := http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true,
+				Certificates:       []tls.Certificate{certificate},
+			},
+		},
+	}
+	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://127.0.0.1:%d/openapi/v2", port), nil)
+	require.NoError(t, err)
+
+	userInfo := &user.DefaultInfo{
+		Name:   "my-user",
+		Groups: []string{"my-group"},
+	}
+
+	req.Header.Set("X-Remote-User", userInfo.GetName())
+	req.Header.Add("X-Remote-Group", userInfo.GetGroups()[0])
+	require.EventuallyWithT(t, func(c *assert.CollectT) {
+		resp, err := httpClient.Do(req)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		require.Equal(t, http.StatusOK, resp.StatusCode)
+	}, 5*time.Second, 110*time.Millisecond)
+
+	req = httptest.NewRequest(http.MethodGet, "/openapi/v2", nil)
+	w := httptest.NewRecorder()
+
+	ctx := request.WithUser(req.Context(), userInfo)
+	req = req.WithContext(ctx)
+
+	extensionAPIServer.ServeHTTP(w, req)
+	resp := w.Result()
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+}

pkg/ext/apiserver_authorization_test.go

@@ -145,7 +145,7 @@ func (s *ExtensionAPIServerSuite) TestAuthorization() {
 	ln, _, err := options.CreateListener("", ":0", net.ListenConfig{})
 	require.NoError(t, err)
 
-	extensionAPIServer, cleanup, err := setupExtensionAPIServerNoStore(t, scheme, func(opts *ExtensionAPIServerOptions) {
+	extensionAPIServer, err := setupExtensionAPIServerNoStore(t, scheme, func(opts *ExtensionAPIServerOptions) {
 		opts.Listener = ln
 		opts.Authorizer = authz
 		opts.Authenticator = authenticator.RequestFunc(func(req *http.Request) (*authenticator.Response, bool, error) {
@@ -169,7 +169,6 @@ func (s *ExtensionAPIServerSuite) TestAuthorization() {
 		return nil
 	})
 	require.NoError(t, err)
-	defer cleanup()
 
 	rbacBytes, err := os.ReadFile(filepath.Join("testdata", "rbac.yaml"))
 	require.NoError(t, err)

pkg/ext/apiserver_suite_test.go

@@ -2,14 +2,158 @@ package ext
 
 import (
 	"context"
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	crand "crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"fmt"
+	"math/big"
+	"os"
+	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/suite"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
+	certutil "k8s.io/client-go/util/cert"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 )
 
+// Copied and modified from envtest internal
+var (
+	ellipticCurve = elliptic.P256()
+	bigOne        = big.NewInt(1)
+)
+
+// CertPair is a private key and certificate for use for client auth, as a CA, or serving.
+type CertPair struct {
+	Key  crypto.Signer
+	Cert *x509.Certificate
+}
+
+// CertBytes returns the PEM-encoded version of the certificate for this pair.
+func (k CertPair) CertBytes() []byte {
+	return pem.EncodeToMemory(&pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: k.Cert.Raw,
+	})
+}
+
+// AsBytes encodes keypair in the appropriate formats for on-disk storage (PEM and
+// PKCS8, respectively).
+func (k CertPair) AsBytes() (cert []byte, key []byte, err error) {
+	cert = k.CertBytes()
+
+	rawKeyData, err := x509.MarshalPKCS8PrivateKey(k.Key)
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to encode private key: %w", err)
+	}
+
+	key = pem.EncodeToMemory(&pem.Block{
+		Type:  "PRIVATE KEY",
+		Bytes: rawKeyData,
+	})
+
+	return cert, key, nil
+}
+
+// TinyCA supports signing serving certs and client-certs,
+// and can be used as an auth mechanism with envtest.
+type TinyCA struct {
+	CA      CertPair
+	orgName string
+
+	nextSerial *big.Int
+}
+
+// newPrivateKey generates a new private key of a relatively sane size (see
+// rsaKeySize).
+func newPrivateKey() (crypto.Signer, error) {
+	return ecdsa.GenerateKey(ellipticCurve, crand.Reader)
+}
+
+// NewTinyCA creates a new a tiny CA utility for provisioning serving certs and client certs FOR TESTING ONLY.
+// Don't use this for anything else!
+func NewTinyCA() (*TinyCA, error) {
+	caPrivateKey, err := newPrivateKey()
+	if err != nil {
+		return nil, fmt.Errorf("unable to generate private key for CA: %w", err)
+	}
+	caCfg := certutil.Config{CommonName: "envtest-environment", Organization: []string{"envtest"}}
+	caCert, err := certutil.NewSelfSignedCACert(caCfg, caPrivateKey)
+	if err != nil {
+		return nil, fmt.Errorf("unable to generate certificate for CA: %w", err)
+	}
+
+	return &TinyCA{
+		CA:         CertPair{Key: caPrivateKey, Cert: caCert},
+		orgName:    "envtest",
+		nextSerial: big.NewInt(1),
+	}, nil
+}
+
+func (c *TinyCA) CertBytes() []byte {
+	return pem.EncodeToMemory(&pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: c.CA.Cert.Raw,
+	})
+}
+
+func (c *TinyCA) NewClientCert(name string) (CertPair, error) {
+	return c.makeCert(certutil.Config{
+		CommonName: name,
+		Usages:     []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+	})
+}
+
+func (c *TinyCA) makeCert(cfg certutil.Config) (CertPair, error) {
+	now := time.Now()
+
+	key, err := newPrivateKey()
+	if err != nil {
+		return CertPair{}, fmt.Errorf("unable to create private key: %w", err)
+	}
+
+	serial := new(big.Int).Set(c.nextSerial)
+	c.nextSerial.Add(c.nextSerial, bigOne)
+
+	template := x509.Certificate{
+		Subject:      pkix.Name{CommonName: cfg.CommonName, Organization: cfg.Organization},
+		DNSNames:     cfg.AltNames.DNSNames,
+		IPAddresses:  cfg.AltNames.IPs,
+		SerialNumber: serial,
+
+		KeyUsage:    x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage: cfg.Usages,
+
+		// technically not necessary for testing, but let's set anyway just in case.
+		NotBefore: now.UTC(),
+		// 1 week -- the default for cfssl, and just long enough for a
+		// long-term test, but not too long that anyone would try to use this
+		// seriously.
+		NotAfter: now.Add(168 * time.Hour).UTC(),
+	}
+
+	certRaw, err := x509.CreateCertificate(crand.Reader, &template, c.CA.Cert, key.Public(), c.CA.Key)
+	if err != nil {
+		return CertPair{}, fmt.Errorf("unable to create certificate: %w", err)
+	}
+
+	cert, err := x509.ParseCertificate(certRaw)
+	if err != nil {
+		return CertPair{}, fmt.Errorf("generated invalid certificate, could not parse: %w", err)
+	}
+
+	return CertPair{
+		Key:  key,
+		Cert: cert,
+	}, nil
+}
+
 type ExtensionAPIServerSuite struct {
 	suite.Suite
 
@@ -19,11 +163,42 @@ type ExtensionAPIServerSuite struct {
 	testEnv    envtest.Environment
 	client     *kubernetes.Clientset
 	restConfig *rest.Config
+
+	certTempPath string
+	ca           *TinyCA
+	cert         CertPair
 }
 
 func (s *ExtensionAPIServerSuite) SetupSuite() {
 	var err error
+	s.ca, err = NewTinyCA()
+	s.Require().NoError(err)
+	s.cert, err = s.ca.NewClientCert("system:auth-proxy")
+	s.Require().NoError(err)
+
+	cert, key, err := s.cert.AsBytes()
+	s.Require().NoError(err)
+
+	s.certTempPath = s.T().TempDir()
+
+	caFilepath := filepath.Join(s.certTempPath, "request-header-ca.crt")
+	certFilepath := filepath.Join(s.certTempPath, "client-auth-proxy.crt")
+	keyFilepath := filepath.Join(s.certTempPath, "client-auth-proxy.key")
+
+	os.WriteFile(caFilepath, s.ca.CertBytes(), 0644)
+	os.WriteFile(certFilepath, cert, 0644)
+	os.WriteFile(keyFilepath, key, 0644)
+
+	// Configures the aggregation layer according to
+	// https://kubernetes.io/docs/tasks/extend-kubernetes/configure-aggregation-layer/#enable-kubernetes-apiserver-flags
 	apiServer := &envtest.APIServer{}
+	apiServer.Configure().Append("requestheader-allowed-names", "system:auth-proxy")
+	apiServer.Configure().Append("requestheader-extra-headers-prefix", "X-Remote-Extra-")
+	apiServer.Configure().Append("requestheader-group-headers", "X-Remote-Group")
+	apiServer.Configure().Append("requestheader-username-headers", "X-Remote-User")
+	apiServer.Configure().Append("requestheader-client-ca-file", caFilepath)
+	apiServer.Configure().Append("proxy-client-cert-file", certFilepath)
+	apiServer.Configure().Append("proxy-client-key-file", keyFilepath)
 
 	s.testEnv = envtest.Environment{
 		ControlPlane: envtest.ControlPlane{
@@ -43,6 +218,7 @@ func (s *ExtensionAPIServerSuite) TearDownSuite() {
 	s.cancel()
 	err := s.testEnv.Stop()
 	s.Require().NoError(err)
+	os.RemoveAll(s.certTempPath)
 }
 
 func TestExtensionAPIServerSuite(t *testing.T) {

pkg/ext/apiserver_test.go

@@ -55,13 +55,12 @@ func TestStore(t *testing.T) {
 	store := newDefaultTestStore()
 	store.items = make(map[string]*TestType)
 
-	extensionAPIServer, cleanup, err := setupExtensionAPIServer(t, scheme, store, func(opts *ExtensionAPIServerOptions) {
+	extensionAPIServer, err := setupExtensionAPIServer(t, scheme, store, func(opts *ExtensionAPIServerOptions) {
 		opts.Listener = ln
 		opts.Authorizer = authorizer.AuthorizerFunc(authzAllowAll)
 		opts.Authenticator = authenticator.RequestFunc(authAsAdmin)
 	}, nil)
 	require.NoError(t, err)
-	defer cleanup()
 
 	ts := httptest.NewServer(extensionAPIServer)
 	defer ts.Close()
@@ -297,7 +296,7 @@ func TestDiscoveryAndOpenAPI(t *testing.T) {
 	require.NoError(t, err)
 
 	store := newDefaultTestStore()
-	extensionAPIServer, cleanup, err := setupExtensionAPIServer(t, scheme, store, func(opts *ExtensionAPIServerOptions) {
+	extensionAPIServer, err := setupExtensionAPIServer(t, scheme, store, func(opts *ExtensionAPIServerOptions) {
 		opts.Listener = ln
 		opts.Authorizer = authorizer.AuthorizerFunc(authzAllowAll)
 		opts.Authenticator = authenticator.RequestFunc(authAsAdmin)
@@ -345,7 +344,6 @@ func TestDiscoveryAndOpenAPI(t *testing.T) {
 		return nil
 	})
 	require.NoError(t, err)
-	defer cleanup()
 
 	tests := []struct {
 		path               string
@@ -674,7 +672,7 @@ func setupExtensionAPIServer(
 	store regrest.Storage,
 	optionSetter func(*ExtensionAPIServerOptions),
 	extensionAPIServerSetter func(*ExtensionAPIServer) error,
-) (*ExtensionAPIServer, func(), error) {
+) (*ExtensionAPIServer, error) {
 	fn := func(e *ExtensionAPIServer) error {
 		err := e.Install("testtypes", testTypeGV.WithKind("TestType"), store)
 		if err != nil {
@@ -693,7 +691,7 @@ func setupExtensionAPIServerNoStore(
 	scheme *runtime.Scheme,
 	optionSetter func(*ExtensionAPIServerOptions),
 	extensionAPIServerSetter func(*ExtensionAPIServer) error,
-) (*ExtensionAPIServer, func(), error) {
+) (*ExtensionAPIServer, error) {
 
 	addToSchemeTest(scheme)
 	codecs := serializer.NewCodecFactory(scheme)
@@ -709,26 +707,27 @@ func setupExtensionAPIServerNoStore(
 	}
 	extensionAPIServer, err := NewExtensionAPIServer(scheme, codecs, opts)
 	if err != nil {
-		return nil, func() {}, err
+		return nil, err
 	}
 
 	if extensionAPIServerSetter != nil {
 		err = extensionAPIServerSetter(extensionAPIServer)
 		if err != nil {
-			return nil, func() {}, fmt.Errorf("extensionAPIServerSetter: %w", err)
+			return nil, fmt.Errorf("extensionAPIServerSetter: %w", err)
 		}
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(func() {
+		cancel()
+	})
 
 	err = extensionAPIServer.Run(ctx)
-	require.NoError(t, err)
+	if err != nil {
-
+		return nil, err
-	cleanup := func() {
-		cancel()
 	}
 
-	return extensionAPIServer, cleanup, nil
+	return extensionAPIServer, nil
 }
 
 type recordingWatcher struct {
@@ -818,7 +817,7 @@ func TestCustomColumns(t *testing.T) {
 		testStore: newDefaultTestStore(),
 	}
 
-	extensionAPIServer, cleanup, err := setupExtensionAPIServerNoStore(t, scheme, func(opts *ExtensionAPIServerOptions) {
+	extensionAPIServer, err := setupExtensionAPIServerNoStore(t, scheme, func(opts *ExtensionAPIServerOptions) {
 		opts.Listener = ln
 		opts.Authorizer = authorizer.AuthorizerFunc(authzAllowAll)
 		opts.Authenticator = authenticator.RequestFunc(authAsAdmin)
@@ -831,7 +830,6 @@ func TestCustomColumns(t *testing.T) {
 	})
 
 	require.NoError(t, err)
-	defer cleanup()
 
 	ts := httptest.NewServer(extensionAPIServer)
 	defer ts.Close()