diff --git a/pkg/ext/apiserver.go b/pkg/ext/apiserver.go
index 55f5444d..30114ddd 100644
--- a/pkg/ext/apiserver.go
+++ b/pkg/ext/apiserver.go
@@ -108,6 +108,8 @@ type ExtensionAPIServer struct {
 
 	handlerMu sync.RWMutex
 	handler   http.Handler
+
+	registeredChan <-chan struct{}
 }
 
 type emptyAddresses struct{}
@@ -129,15 +131,38 @@ func NewExtensionAPIServer(scheme *runtime.Scheme, codecs serializer.CodecFactor
 		return nil, fmt.Errorf("listener must be provided")
 	}
 
+	if opts.GetOpenAPIDefinitions == nil {
+		return nil, fmt.Errorf("GetOpenAPIDefinitions must be provided")
+	}
+
+	if opts.Authorizer == nil {
+		// We need an authorizer to allow us to wrap it with a check for kube-apisevrer registration requests.
+		return nil, fmt.Errorf("authorizer must be provided")
+	}
+
 	recommendedOpts := genericoptions.NewRecommendedOptions("", codecs.LegacyCodec())
 	recommendedOpts.SecureServing.Listener = opts.Listener
 
+	registered := make(chan struct{})
+
+	var once sync.Once
+
 	resolver := &request.RequestInfoFactory{APIPrefixes: sets.NewString("apis", "api"), GrouplessAPIPrefixes: sets.NewString("api")}
 	config := genericapiserver.NewRecommendedConfig(codecs)
 	config.RequestInfoResolver = resolver
 	config.Authorization = genericapiserver.AuthorizationInfo{
-		Authorizer: opts.Authorizer,
+		Authorizer: authorizer.AuthorizerFunc(func(ctx context.Context, a authorizer.Attributes) (authorizer.Decision, string, error) {
+			switch a.GetUser().GetName() {
+			case "system:aggregator", "system:kube-aggregator":
+				once.Do(func() {
+					close(registered)
+				})
+			}
+
+			return opts.Authorizer.Authorize(ctx, a)
+		}),
 	}
+
 	// The default kube effective version ends up being the version of the
 	// library. (The value is hardcoded but it is kept up-to-date via some
 	// automation)
@@ -189,6 +214,7 @@ func NewExtensionAPIServer(scheme *runtime.Scheme, codecs serializer.CodecFactor
 		genericAPIServer: genericServer,
 		apiGroups:        make(map[string]genericapiserver.APIGroupInfo),
 		authorizer:       opts.Authorizer,
+		registeredChan:   registered,
 	}
 
 	return extensionAPIServer, nil
@@ -301,3 +327,7 @@ func getDefinitionName(scheme *runtime.Scheme, replacements map[string]string) f
 		return definitionName, defGVK
 	}
 }
+
+func (s *ExtensionAPIServer) Registered() <-chan struct{} {
+	return s.registeredChan
+}
diff --git a/pkg/server/server.go b/pkg/server/server.go
index 014bbb1f..830801de 100644
--- a/pkg/server/server.go
+++ b/pkg/server/server.go
@@ -44,6 +44,8 @@ type ExtensionAPIServer interface {
 	http.Handler
 	// Run configures the API server and make the HTTP handler available
 	Run(ctx context.Context) error
+	// Registered returns a channel that will be close once the registration requests are received from the kube-apiserver.
+	Registered() <-chan struct{}
 }
 
 type Server struct {
@@ -253,7 +255,16 @@ func setup(ctx context.Context, server *Server) error {
 		onSchemasHandler,
 		sf)
 
-	apiServer, handler, err := handler.New(server.RESTConfig, sf, server.authMiddleware, server.next, server.router, server.extensionAPIServer)
+	next := http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		select {
+		case <-server.extensionAPIServer.Registered():
+			server.next.ServeHTTP(rw, req)
+		default:
+			http.NotFoundHandler().ServeHTTP(rw, req)
+		}
+	})
+
+	apiServer, handler, err := handler.New(server.RESTConfig, sf, server.authMiddleware, next, server.router, server.extensionAPIServer)
 	if err != nil {
 		return err
 	}