From d8c5f71403ab39584528dd35eaf200370e081770 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Mon, 29 Apr 2024 22:06:01 +0200 Subject: [PATCH] kube-apiserver: split up config creation into generic and non-generic part Signed-off-by: Dr. Stefan Schimanski --- cmd/kube-apiserver/app/config.go | 21 ++- cmd/kube-apiserver/app/server.go | 159 +++------------------- pkg/controlplane/apiserver/config.go | 136 ++++++++++++++++++ test/integration/framework/test_server.go | 21 ++- 4 files changed, 191 insertions(+), 146 deletions(-) diff --git a/cmd/kube-apiserver/app/config.go b/cmd/kube-apiserver/app/config.go index 7a67875cb60..9f6874f067b 100644 --- a/cmd/kube-apiserver/app/config.go +++ b/cmd/kube-apiserver/app/config.go @@ -18,11 +18,16 @@ package app import ( apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/util/webhook" aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" + aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" + "k8s.io/kubernetes/cmd/kube-apiserver/app/options" + "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/controlplane" - "k8s.io/kubernetes/pkg/controlplane/apiserver" + controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver" + generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi" ) type Config struct { @@ -71,13 +76,23 @@ func NewConfig(opts options.CompletedOptions) (*Config, error) { Options: opts, } - kubeAPIs, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(opts) + genericConfig, versionedInformers, storageFactory, err := controlplaneapiserver.BuildGenericConfig( + opts.CompletedOptions, + []*runtime.Scheme{legacyscheme.Scheme, apiextensionsapiserver.Scheme, aggregatorscheme.Scheme}, + controlplane.DefaultAPIResourceConfigSource(), + generatedopenapi.GetOpenAPIDefinitions, + ) + if err != nil { + return nil, err + } + + kubeAPIs, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(opts, genericConfig, versionedInformers, storageFactory) if err != nil { return nil, err } c.KubeAPIs = kubeAPIs - apiExtensions, err := apiserver.CreateAPIExtensionsConfig(*kubeAPIs.ControlPlane.Generic, kubeAPIs.ControlPlane.VersionedInformers, pluginInitializer, opts.CompletedOptions, opts.MasterCount, + apiExtensions, err := controlplaneapiserver.CreateAPIExtensionsConfig(*kubeAPIs.ControlPlane.Generic, kubeAPIs.ControlPlane.VersionedInformers, pluginInitializer, opts.CompletedOptions, opts.MasterCount, serviceResolver, webhook.NewDefaultAuthenticationInfoResolverWrapper(kubeAPIs.ControlPlane.ProxyTransport, kubeAPIs.ControlPlane.Generic.EgressSelector, kubeAPIs.ControlPlane.Generic.LoopbackClientConfig, kubeAPIs.ControlPlane.Generic.TracerProvider)) if err != nil { return nil, err diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 543ed4d3ff4..ceb4401c620 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -21,31 +21,24 @@ package app import ( "context" - "crypto/tls" "fmt" - "net/http" "net/url" "os" "github.com/spf13/cobra" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" - "k8s.io/apimachinery/pkg/runtime" utilerrors "k8s.io/apimachinery/pkg/util/errors" - utilnet "k8s.io/apimachinery/pkg/util/net" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apiserver/pkg/admission" genericapifilters "k8s.io/apiserver/pkg/endpoints/filters" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/egressselector" + serverstorage "k8s.io/apiserver/pkg/server/storage" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/apiserver/pkg/util/notfoundhandler" "k8s.io/apiserver/pkg/util/webhook" - "k8s.io/client-go/dynamic" clientgoinformers "k8s.io/client-go/informers" - clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "k8s.io/client-go/util/keyutil" cliflag "k8s.io/component-base/cli/flag" "k8s.io/component-base/cli/globalflag" "k8s.io/component-base/logs" @@ -56,19 +49,12 @@ import ( "k8s.io/component-base/version/verflag" "k8s.io/klog/v2" aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" - aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" - controlplaneadmission "k8s.io/kubernetes/pkg/controlplane/apiserver/admission" - "k8s.io/kubernetes/cmd/kube-apiserver/app/options" - "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/capabilities" "k8s.io/kubernetes/pkg/controlplane" controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver" "k8s.io/kubernetes/pkg/controlplane/reconcilers" - "k8s.io/kubernetes/pkg/features" - generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi" kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" - "k8s.io/kubernetes/pkg/serviceaccount" ) func init() { @@ -196,49 +182,22 @@ func CreateServerChain(config CompletedConfig) (*aggregatorapiserver.APIAggregat return aggregatorServer, nil } -// CreateProxyTransport creates the dialer infrastructure to connect to the nodes. -func CreateProxyTransport() *http.Transport { - var proxyDialerFn utilnet.DialFunc - // Proxying to pods and services is IP-based... don't expect to be able to verify the hostname - proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true} - proxyTransport := utilnet.SetTransportDefaults(&http.Transport{ - DialContext: proxyDialerFn, - TLSClientConfig: proxyTLSClientConfig, - }) - return proxyTransport -} - // CreateKubeAPIServerConfig creates all the resources for running the API server, but runs none of them -func CreateKubeAPIServerConfig(opts options.CompletedOptions) ( +func CreateKubeAPIServerConfig( + opts options.CompletedOptions, + genericConfig *genericapiserver.Config, + versionedInformers clientgoinformers.SharedInformerFactory, + storageFactory *serverstorage.DefaultStorageFactory, +) ( *controlplane.Config, aggregatorapiserver.ServiceResolver, []admission.PluginInitializer, error, ) { - proxyTransport := CreateProxyTransport() + // global stuff + capabilities.Setup(opts.AllowPrivileged, opts.MaxConnectionBytesPerSec) - genericConfig, versionedInformers, storageFactory, err := controlplaneapiserver.BuildGenericConfig( - opts.CompletedOptions, - []*runtime.Scheme{legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme}, - controlplane.DefaultAPIResourceConfigSource(), - generatedopenapi.GetOpenAPIDefinitions, - ) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to create generic config: %w", err) - } - - // generic controlplane admission initializers - controlPlaneAdmissionConfig := &controlplaneadmission.Config{ - ExternalInformers: versionedInformers, - LoopbackClientConfig: genericConfig.LoopbackClientConfig, - } - serviceResolver := buildServiceResolver(opts.EnableAggregatorRouting, genericConfig.LoopbackClientConfig.Host, versionedInformers) - pluginInitializers, err := controlPlaneAdmissionConfig.New(proxyTransport, genericConfig.EgressSelector, serviceResolver, genericConfig.TracerProvider) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to create admission plugin initializer: %w", err) - } - - // additional kube admission initializers + // additional admission initializers kubeAdmissionConfig := &kubeapiserveradmission.Config{ CloudConfigFile: opts.CloudProvider.CloudConfigFile, } @@ -246,31 +205,15 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) ( if err != nil { return nil, nil, nil, fmt.Errorf("failed to create admission plugin initializer: %w", err) } - pluginInitializers = append(pluginInitializers, kubeInitializers...) - capabilities.Setup(opts.AllowPrivileged, opts.MaxConnectionBytesPerSec) - - opts.Metrics.Apply() - serviceaccount.RegisterMetrics() + serviceResolver := buildServiceResolver(opts.EnableAggregatorRouting, genericConfig.LoopbackClientConfig.Host, versionedInformers) + controlplaneConfig, admissionInitializers, err := controlplaneapiserver.CreateConfig(opts.CompletedOptions, genericConfig, versionedInformers, storageFactory, serviceResolver, kubeInitializers) + if err != nil { + return nil, nil, nil, err + } config := &controlplane.Config{ - ControlPlane: controlplaneapiserver.Config{ - Generic: genericConfig, - Extra: controlplaneapiserver.Extra{ - APIResourceConfigSource: storageFactory.APIResourceConfigSource, - StorageFactory: storageFactory, - EventTTL: opts.EventTTL, - EnableLogsSupport: opts.EnableLogsHandler, - ProxyTransport: proxyTransport, - SystemNamespaces: opts.SystemNamespaces, - - ServiceAccountIssuer: opts.ServiceAccountIssuer, - ServiceAccountMaxExpiration: opts.ServiceAccountTokenMaxExpiration, - ExtendExpiration: opts.Authentication.ServiceAccounts.ExtendExpiration, - - VersionedInformers: versionedInformers, - }, - }, + ControlPlane: *controlplaneConfig, Extra: controlplane.Extra{ KubeletClientConfig: opts.KubeletConfig, @@ -288,59 +231,6 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) ( }, } - if utilfeature.DefaultFeatureGate.Enabled(features.UnknownVersionInteroperabilityProxy) { - config.ControlPlane.PeerEndpointLeaseReconciler, err = controlplaneapiserver.CreatePeerEndpointLeaseReconciler(*genericConfig, storageFactory) - if err != nil { - return nil, nil, nil, err - } - // build peer proxy config only if peer ca file exists - if opts.PeerCAFile != "" { - config.ControlPlane.PeerProxy, err = controlplaneapiserver.BuildPeerProxy(versionedInformers, genericConfig.StorageVersionManager, opts.ProxyClientCertFile, - opts.ProxyClientKeyFile, opts.PeerCAFile, opts.PeerAdvertiseAddress, genericConfig.APIServerID, config.ControlPlane.Extra.PeerEndpointLeaseReconciler, config.ControlPlane.Generic.Serializer) - if err != nil { - return nil, nil, nil, err - } - } - } - - clientCAProvider, err := opts.Authentication.ClientCert.GetClientCAContentProvider() - if err != nil { - return nil, nil, nil, err - } - config.ControlPlane.ClusterAuthenticationInfo.ClientCA = clientCAProvider - - requestHeaderConfig, err := opts.Authentication.RequestHeader.ToAuthenticationRequestHeaderConfig() - if err != nil { - return nil, nil, nil, err - } - if requestHeaderConfig != nil { - config.ControlPlane.ClusterAuthenticationInfo.RequestHeaderCA = requestHeaderConfig.CAContentProvider - config.ControlPlane.ClusterAuthenticationInfo.RequestHeaderAllowedNames = requestHeaderConfig.AllowedClientNames - config.ControlPlane.ClusterAuthenticationInfo.RequestHeaderExtraHeaderPrefixes = requestHeaderConfig.ExtraHeaderPrefixes - config.ControlPlane.ClusterAuthenticationInfo.RequestHeaderGroupHeaders = requestHeaderConfig.GroupHeaders - config.ControlPlane.ClusterAuthenticationInfo.RequestHeaderUsernameHeaders = requestHeaderConfig.UsernameHeaders - } - - // setup admission - clientgoExternalClient, err := clientset.NewForConfig(genericConfig.LoopbackClientConfig) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to create real client-go external client: %w", err) - } - dynamicExternalClient, err := dynamic.NewForConfig(genericConfig.LoopbackClientConfig) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to create real dynamic external client: %w", err) - } - err = opts.Admission.ApplyTo( - genericConfig, - versionedInformers, - clientgoExternalClient, - dynamicExternalClient, - utilfeature.DefaultFeatureGate, - pluginInitializers...) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to apply admission: %w", err) - } - if config.ControlPlane.Generic.EgressSelector != nil { // Use the config.ControlPlane.Generic.EgressSelector lookup to find the dialer to connect to the kubelet config.Extra.KubeletClientConfig.Lookup = config.ControlPlane.Generic.EgressSelector.Lookup @@ -351,25 +241,12 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) ( if err != nil { return nil, nil, nil, err } - c := proxyTransport.Clone() + c := config.ControlPlane.Extra.ProxyTransport.Clone() c.DialContext = dialer config.ControlPlane.ProxyTransport = c } - // Load and set the public keys. - var pubKeys []interface{} - for _, f := range opts.Authentication.ServiceAccounts.KeyFiles { - keys, err := keyutil.PublicKeysFromFile(f) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to parse key file %q: %v", f, err) - } - pubKeys = append(pubKeys, keys...) - } - config.ControlPlane.ServiceAccountIssuerURL = opts.Authentication.ServiceAccounts.Issuers[0] - config.ControlPlane.ServiceAccountJWKSURI = opts.Authentication.ServiceAccounts.JWKSURI - config.ControlPlane.ServiceAccountPublicKeys = pubKeys - - return config, serviceResolver, pluginInitializers, nil + return config, serviceResolver, admissionInitializers, nil } var testServiceResolver webhook.ServiceResolver diff --git a/pkg/controlplane/apiserver/config.go b/pkg/controlplane/apiserver/config.go index d3da39d629a..924a3f4685d 100644 --- a/pkg/controlplane/apiserver/config.go +++ b/pkg/controlplane/apiserver/config.go @@ -18,6 +18,7 @@ package apiserver import ( "context" + "crypto/tls" "fmt" "net/http" "time" @@ -25,8 +26,10 @@ import ( oteltrace "go.opentelemetry.io/otel/trace" "k8s.io/apimachinery/pkg/runtime" + utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/endpoints/discovery/aggregated" openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" @@ -39,14 +42,19 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/apiserver/pkg/util/openapi" utilpeerproxy "k8s.io/apiserver/pkg/util/peerproxy" + "k8s.io/client-go/dynamic" clientgoinformers "k8s.io/client-go/informers" clientgoclientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/keyutil" "k8s.io/component-base/version" + aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" openapicommon "k8s.io/kube-openapi/pkg/common" "k8s.io/kubernetes/pkg/api/legacyscheme" + controlplaneadmission "k8s.io/kubernetes/pkg/controlplane/apiserver/admission" controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options" "k8s.io/kubernetes/pkg/controlplane/controller/clusterauthenticationtrust" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubeapiserver" "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest" @@ -254,3 +262,131 @@ func BuildAuthorizer(ctx context.Context, s controlplaneapiserver.CompletedOptio return authorizer, ruleResolver, enablesRBAC, err } + +// CreateConfig takes the generic controlplane apiserver options and +// creates a config for the generic Kube APIs out of it. +func CreateConfig( + opts controlplaneapiserver.CompletedOptions, + genericConfig *genericapiserver.Config, + versionedInformers clientgoinformers.SharedInformerFactory, + storageFactory *serverstorage.DefaultStorageFactory, + serviceResolver aggregatorapiserver.ServiceResolver, + additionalInitializers []admission.PluginInitializer, +) ( + *Config, + []admission.PluginInitializer, + error, +) { + proxyTransport := CreateProxyTransport() + + opts.Metrics.Apply() + serviceaccount.RegisterMetrics() + + config := &Config{ + Generic: genericConfig, + Extra: Extra{ + APIResourceConfigSource: storageFactory.APIResourceConfigSource, + StorageFactory: storageFactory, + EventTTL: opts.EventTTL, + EnableLogsSupport: opts.EnableLogsHandler, + ProxyTransport: proxyTransport, + SystemNamespaces: opts.SystemNamespaces, + + ServiceAccountIssuer: opts.ServiceAccountIssuer, + ServiceAccountMaxExpiration: opts.ServiceAccountTokenMaxExpiration, + ExtendExpiration: opts.Authentication.ServiceAccounts.ExtendExpiration, + + VersionedInformers: versionedInformers, + }, + } + + if utilfeature.DefaultFeatureGate.Enabled(features.UnknownVersionInteroperabilityProxy) { + var err error + config.PeerEndpointLeaseReconciler, err = CreatePeerEndpointLeaseReconciler(*genericConfig, storageFactory) + if err != nil { + return nil, nil, err + } + // build peer proxy config only if peer ca file exists + if opts.PeerCAFile != "" { + config.PeerProxy, err = BuildPeerProxy(versionedInformers, genericConfig.StorageVersionManager, opts.ProxyClientCertFile, + opts.ProxyClientKeyFile, opts.PeerCAFile, opts.PeerAdvertiseAddress, genericConfig.APIServerID, config.Extra.PeerEndpointLeaseReconciler, config.Generic.Serializer) + if err != nil { + return nil, nil, err + } + } + } + + clientCAProvider, err := opts.Authentication.ClientCert.GetClientCAContentProvider() + if err != nil { + return nil, nil, err + } + config.ClusterAuthenticationInfo.ClientCA = clientCAProvider + + requestHeaderConfig, err := opts.Authentication.RequestHeader.ToAuthenticationRequestHeaderConfig() + if err != nil { + return nil, nil, err + } + if requestHeaderConfig != nil { + config.ClusterAuthenticationInfo.RequestHeaderCA = requestHeaderConfig.CAContentProvider + config.ClusterAuthenticationInfo.RequestHeaderAllowedNames = requestHeaderConfig.AllowedClientNames + config.ClusterAuthenticationInfo.RequestHeaderExtraHeaderPrefixes = requestHeaderConfig.ExtraHeaderPrefixes + config.ClusterAuthenticationInfo.RequestHeaderGroupHeaders = requestHeaderConfig.GroupHeaders + config.ClusterAuthenticationInfo.RequestHeaderUsernameHeaders = requestHeaderConfig.UsernameHeaders + } + + // setup admission + genericAdmissionConfig := controlplaneadmission.Config{ + ExternalInformers: versionedInformers, + LoopbackClientConfig: genericConfig.LoopbackClientConfig, + } + genericInitializers, err := genericAdmissionConfig.New(proxyTransport, genericConfig.EgressSelector, serviceResolver, genericConfig.TracerProvider) + if err != nil { + return nil, nil, fmt.Errorf("failed to create admission plugin initializer: %w", err) + } + clientgoExternalClient, err := clientgoclientset.NewForConfig(genericConfig.LoopbackClientConfig) + if err != nil { + return nil, nil, fmt.Errorf("failed to create real client-go external client: %w", err) + } + dynamicExternalClient, err := dynamic.NewForConfig(genericConfig.LoopbackClientConfig) + if err != nil { + return nil, nil, fmt.Errorf("failed to create real dynamic external client: %w", err) + } + err = opts.Admission.ApplyTo( + genericConfig, + versionedInformers, + clientgoExternalClient, + dynamicExternalClient, + utilfeature.DefaultFeatureGate, + append(genericInitializers, additionalInitializers...)..., + ) + if err != nil { + return nil, nil, fmt.Errorf("failed to apply admission: %w", err) + } + + // Load and set the public keys. + var pubKeys []interface{} + for _, f := range opts.Authentication.ServiceAccounts.KeyFiles { + keys, err := keyutil.PublicKeysFromFile(f) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse key file %q: %w", f, err) + } + pubKeys = append(pubKeys, keys...) + } + config.ServiceAccountIssuerURL = opts.Authentication.ServiceAccounts.Issuers[0] + config.ServiceAccountJWKSURI = opts.Authentication.ServiceAccounts.JWKSURI + config.ServiceAccountPublicKeys = pubKeys + + return config, genericInitializers, nil +} + +// CreateProxyTransport creates the dialer infrastructure to connect to the nodes. +func CreateProxyTransport() *http.Transport { + var proxyDialerFn utilnet.DialFunc + // Proxying to pods and services is IP-based... don't expect to be able to verify the hostname + proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true} + proxyTransport := utilnet.SetTransportDefaults(&http.Transport{ + DialContext: proxyDialerFn, + TLSClientConfig: proxyTLSClientConfig, + }) + return proxyTransport +} diff --git a/test/integration/framework/test_server.go b/test/integration/framework/test_server.go index e1d2c64efa9..bb43850466d 100644 --- a/test/integration/framework/test_server.go +++ b/test/integration/framework/test_server.go @@ -28,7 +28,9 @@ import ( "github.com/google/uuid" + apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/wait" genericapiserver "k8s.io/apiserver/pkg/server" @@ -36,11 +38,16 @@ import ( client "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/util/cert" + aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" + netutils "k8s.io/utils/net" + "k8s.io/kubernetes/cmd/kube-apiserver/app" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" + "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/controlplane" + controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver" + generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi" "k8s.io/kubernetes/test/utils" - netutils "k8s.io/utils/net" ) // This key is for testing purposes only and is not considered secure. @@ -160,7 +167,17 @@ func StartTestServer(ctx context.Context, t testing.TB, setup TestServerSetup) ( t.Fatalf("failed to validate ServerRunOptions: %v", utilerrors.NewAggregate(errs)) } - kubeAPIServerConfig, _, _, err := app.CreateKubeAPIServerConfig(completedOptions) + genericConfig, versionedInformers, storageFactory, err := controlplaneapiserver.BuildGenericConfig( + completedOptions.CompletedOptions, + []*runtime.Scheme{legacyscheme.Scheme, apiextensionsapiserver.Scheme, aggregatorscheme.Scheme}, + controlplane.DefaultAPIResourceConfigSource(), + generatedopenapi.GetOpenAPIDefinitions, + ) + if err != nil { + t.Fatal(err) + } + + kubeAPIServerConfig, _, _, err := app.CreateKubeAPIServerConfig(completedOptions, genericConfig, versionedInformers, storageFactory) if err != nil { t.Fatal(err) }