diff --git a/cmd/kube-apiserver/app/config.go b/cmd/kube-apiserver/app/config.go index 4ffe3f3798a..931bfff5de0 100644 --- a/cmd/kube-apiserver/app/config.go +++ b/cmd/kube-apiserver/app/config.go @@ -29,7 +29,7 @@ type Config struct { Options options.CompletedOptions Aggregator *aggregatorapiserver.Config - ControlPlane *controlplane.Config + KubeAPIs *controlplane.Config ApiExtensions *apiextensionsapiserver.Config ExtraConfig @@ -42,7 +42,7 @@ type completedConfig struct { Options options.CompletedOptions Aggregator aggregatorapiserver.CompletedConfig - ControlPlane controlplane.CompletedConfig + KubeAPIs controlplane.CompletedConfig ApiExtensions apiextensionsapiserver.CompletedConfig ExtraConfig @@ -58,7 +58,7 @@ func (c *Config) Complete() (CompletedConfig, error) { Options: c.Options, Aggregator: c.Aggregator.Complete(), - ControlPlane: c.ControlPlane.Complete(), + KubeAPIs: c.KubeAPIs.Complete(), ApiExtensions: c.ApiExtensions.Complete(), ExtraConfig: c.ExtraConfig, @@ -71,20 +71,20 @@ func NewConfig(opts options.CompletedOptions) (*Config, error) { Options: opts, } - controlPlane, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(opts) + kubeAPIs, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(opts) if err != nil { return nil, err } - c.ControlPlane = controlPlane + c.KubeAPIs = kubeAPIs - apiExtensions, err := apiserver.CreateAPIExtensionsConfig(*controlPlane.GenericConfig, controlPlane.ExtraConfig.VersionedInformers, pluginInitializer, opts.CompletedOptions, opts.MasterCount, - serviceResolver, webhook.NewDefaultAuthenticationInfoResolverWrapper(controlPlane.ExtraConfig.ProxyTransport, controlPlane.GenericConfig.EgressSelector, controlPlane.GenericConfig.LoopbackClientConfig, controlPlane.GenericConfig.TracerProvider)) + apiExtensions, err := apiserver.CreateAPIExtensionsConfig(*kubeAPIs.ControlPlane.Generic, kubeAPIs.ControlPlane.VersionedInformers, pluginInitializer, opts.CompletedOptions, opts.MasterCount, + serviceResolver, webhook.NewDefaultAuthenticationInfoResolverWrapper(kubeAPIs.ControlPlane.ProxyTransport, kubeAPIs.ControlPlane.Generic.EgressSelector, kubeAPIs.ControlPlane.Generic.LoopbackClientConfig, kubeAPIs.ControlPlane.Generic.TracerProvider)) if err != nil { return nil, err } c.ApiExtensions = apiExtensions - aggregator, err := createAggregatorConfig(*controlPlane.GenericConfig, opts.CompletedOptions, controlPlane.ExtraConfig.VersionedInformers, serviceResolver, controlPlane.ExtraConfig.ProxyTransport, controlPlane.ExtraConfig.PeerProxy, pluginInitializer) + aggregator, err := createAggregatorConfig(*kubeAPIs.ControlPlane.Generic, opts.CompletedOptions, kubeAPIs.ControlPlane.VersionedInformers, serviceResolver, kubeAPIs.ControlPlane.ProxyTransport, kubeAPIs.Extra.PeerProxy, pluginInitializer) if err != nil { return nil, err } diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 3370d38889b..acd3f0b5104 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -172,14 +172,14 @@ func Run(opts options.CompletedOptions, stopCh <-chan struct{}) error { // CreateServerChain creates the apiservers connected via delegation. func CreateServerChain(config CompletedConfig) (*aggregatorapiserver.APIAggregator, error) { - notFoundHandler := notfoundhandler.New(config.ControlPlane.GenericConfig.Serializer, genericapifilters.NoMuxAndDiscoveryIncompleteKey) + notFoundHandler := notfoundhandler.New(config.KubeAPIs.ControlPlane.Generic.Serializer, genericapifilters.NoMuxAndDiscoveryIncompleteKey) apiExtensionsServer, err := config.ApiExtensions.New(genericapiserver.NewEmptyDelegateWithCustomHandler(notFoundHandler)) if err != nil { return nil, err } crdAPIEnabled := config.ApiExtensions.GenericConfig.MergedResourceConfig.ResourceEnabled(apiextensionsv1.SchemeGroupVersion.WithResource("customresourcedefinitions")) - kubeAPIServer, err := config.ControlPlane.New(apiExtensionsServer.GenericAPIServer) + kubeAPIServer, err := config.KubeAPIs.New(apiExtensionsServer.GenericAPIServer) if err != nil { return nil, err } @@ -218,6 +218,7 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) ( genericConfig, versionedInformers, storageFactory, err := controlplaneapiserver.BuildGenericConfig( opts.CompletedOptions, []*runtime.Scheme{legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme}, + controlplane.DefaultAPIResourceConfigSource(), generatedopenapi.GetOpenAPIDefinitions, ) if err != nil { @@ -230,14 +231,24 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) ( serviceaccount.RegisterMetrics() config := &controlplane.Config{ - GenericConfig: genericConfig, - ExtraConfig: controlplane.ExtraConfig{ - APIResourceConfigSource: storageFactory.APIResourceConfigSource, - StorageFactory: storageFactory, - EventTTL: opts.EventTTL, - KubeletClientConfig: opts.KubeletConfig, - EnableLogsSupport: opts.EnableLogsHandler, - ProxyTransport: proxyTransport, + ControlPlane: controlplaneapiserver.Config{ + Generic: genericConfig, + Extra: controlplaneapiserver.Extra{ + APIResourceConfigSource: storageFactory.APIResourceConfigSource, + StorageFactory: storageFactory, + EventTTL: opts.EventTTL, + EnableLogsSupport: opts.EnableLogsHandler, + ProxyTransport: proxyTransport, + + ServiceAccountIssuer: opts.ServiceAccountIssuer, + ServiceAccountMaxExpiration: opts.ServiceAccountTokenMaxExpiration, + ExtendExpiration: opts.Authentication.ServiceAccounts.ExtendExpiration, + + VersionedInformers: versionedInformers, + }, + }, + Extra: controlplane.Extra{ + KubeletClientConfig: opts.KubeletConfig, ServiceIPRange: opts.PrimaryServiceClusterIPRange, APIServerServiceIP: opts.APIServerServiceIP, @@ -250,24 +261,18 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) ( EndpointReconcilerType: reconcilers.Type(opts.EndpointReconcilerType), MasterCount: opts.MasterCount, - - ServiceAccountIssuer: opts.ServiceAccountIssuer, - ServiceAccountMaxExpiration: opts.ServiceAccountTokenMaxExpiration, - ExtendExpiration: opts.Authentication.ServiceAccounts.ExtendExpiration, - - VersionedInformers: versionedInformers, }, } if utilfeature.DefaultFeatureGate.Enabled(features.UnknownVersionInteroperabilityProxy) { - config.ExtraConfig.PeerEndpointLeaseReconciler, err = controlplaneapiserver.CreatePeerEndpointLeaseReconciler(*genericConfig, storageFactory) + config.Extra.PeerEndpointLeaseReconciler, err = controlplane.CreatePeerEndpointLeaseReconciler(*genericConfig, storageFactory) if err != nil { return nil, nil, nil, err } // build peer proxy config only if peer ca file exists if opts.PeerCAFile != "" { - config.ExtraConfig.PeerProxy, err = controlplaneapiserver.BuildPeerProxy(versionedInformers, genericConfig.StorageVersionManager, opts.ProxyClientCertFile, - opts.ProxyClientKeyFile, opts.PeerCAFile, opts.PeerAdvertiseAddress, genericConfig.APIServerID, config.ExtraConfig.PeerEndpointLeaseReconciler, config.GenericConfig.Serializer) + config.Extra.PeerProxy, err = controlplaneapiserver.BuildPeerProxy(versionedInformers, genericConfig.StorageVersionManager, opts.ProxyClientCertFile, + opts.ProxyClientKeyFile, opts.PeerCAFile, opts.PeerAdvertiseAddress, genericConfig.APIServerID, config.Extra.PeerEndpointLeaseReconciler, config.ControlPlane.Generic.Serializer) if err != nil { return nil, nil, nil, err } @@ -278,18 +283,18 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) ( if err != nil { return nil, nil, nil, err } - config.ExtraConfig.ClusterAuthenticationInfo.ClientCA = clientCAProvider + config.ControlPlane.ClusterAuthenticationInfo.ClientCA = clientCAProvider requestHeaderConfig, err := opts.Authentication.RequestHeader.ToAuthenticationRequestHeaderConfig() if err != nil { return nil, nil, nil, err } if requestHeaderConfig != nil { - config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderCA = requestHeaderConfig.CAContentProvider - config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderAllowedNames = requestHeaderConfig.AllowedClientNames - config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderExtraHeaderPrefixes = requestHeaderConfig.ExtraHeaderPrefixes - config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderGroupHeaders = requestHeaderConfig.GroupHeaders - config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderUsernameHeaders = requestHeaderConfig.UsernameHeaders + config.ControlPlane.ClusterAuthenticationInfo.RequestHeaderCA = requestHeaderConfig.CAContentProvider + config.ControlPlane.ClusterAuthenticationInfo.RequestHeaderAllowedNames = requestHeaderConfig.AllowedClientNames + config.ControlPlane.ClusterAuthenticationInfo.RequestHeaderExtraHeaderPrefixes = requestHeaderConfig.ExtraHeaderPrefixes + config.ControlPlane.ClusterAuthenticationInfo.RequestHeaderGroupHeaders = requestHeaderConfig.GroupHeaders + config.ControlPlane.ClusterAuthenticationInfo.RequestHeaderUsernameHeaders = requestHeaderConfig.UsernameHeaders } // setup admission @@ -322,19 +327,19 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) ( return nil, nil, nil, fmt.Errorf("failed to apply admission: %w", err) } - if config.GenericConfig.EgressSelector != nil { - // Use the config.GenericConfig.EgressSelector lookup to find the dialer to connect to the kubelet - config.ExtraConfig.KubeletClientConfig.Lookup = config.GenericConfig.EgressSelector.Lookup + if config.ControlPlane.Generic.EgressSelector != nil { + // Use the config.ControlPlane.Generic.EgressSelector lookup to find the dialer to connect to the kubelet + config.Extra.KubeletClientConfig.Lookup = config.ControlPlane.Generic.EgressSelector.Lookup - // Use the config.GenericConfig.EgressSelector lookup as the transport used by the "proxy" subresources. + // Use the config.ControlPlane.Generic.EgressSelector lookup as the transport used by the "proxy" subresources. networkContext := egressselector.Cluster.AsNetworkContext() - dialer, err := config.GenericConfig.EgressSelector.Lookup(networkContext) + dialer, err := config.ControlPlane.Generic.EgressSelector.Lookup(networkContext) if err != nil { return nil, nil, nil, err } c := proxyTransport.Clone() c.DialContext = dialer - config.ExtraConfig.ProxyTransport = c + config.ControlPlane.ProxyTransport = c } // Load and set the public keys. @@ -346,9 +351,9 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) ( } pubKeys = append(pubKeys, keys...) } - config.ExtraConfig.ServiceAccountIssuerURL = opts.Authentication.ServiceAccounts.Issuers[0] - config.ExtraConfig.ServiceAccountJWKSURI = opts.Authentication.ServiceAccounts.JWKSURI - config.ExtraConfig.ServiceAccountPublicKeys = pubKeys + config.ControlPlane.ServiceAccountIssuerURL = opts.Authentication.ServiceAccounts.Issuers[0] + config.ControlPlane.ServiceAccountJWKSURI = opts.Authentication.ServiceAccounts.JWKSURI + config.ControlPlane.ServiceAccountPublicKeys = pubKeys return config, serviceResolver, pluginInitializers, nil } diff --git a/pkg/controlplane/apiserver/completion.go b/pkg/controlplane/apiserver/completion.go new file mode 100644 index 00000000000..e679f3f2044 --- /dev/null +++ b/pkg/controlplane/apiserver/completion.go @@ -0,0 +1,44 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiserver + +import ( + "k8s.io/apiserver/pkg/endpoints/discovery" + genericapiserver "k8s.io/apiserver/pkg/server" +) + +type completedConfig struct { + Generic genericapiserver.CompletedConfig + *Extra +} + +// CompletedConfig embeds a private pointer that cannot be instantiated outside of this package +type CompletedConfig struct { + *completedConfig +} + +func (c *Config) Complete() CompletedConfig { + cfg := completedConfig{ + c.Generic.Complete(c.VersionedInformers), + &c.Extra, + } + + discoveryAddresses := discovery.DefaultAddresses{DefaultAddress: cfg.Generic.ExternalAddress} + cfg.Generic.DiscoveryAddresses = discoveryAddresses + + return CompletedConfig{&cfg} +} diff --git a/pkg/controlplane/apiserver/config.go b/pkg/controlplane/apiserver/config.go index 9b12fceb501..5ec3210dc03 100644 --- a/pkg/controlplane/apiserver/config.go +++ b/pkg/controlplane/apiserver/config.go @@ -19,6 +19,7 @@ package apiserver import ( "context" "fmt" + "net/http" "time" oteltrace "go.opentelemetry.io/otel/trace" @@ -47,28 +48,56 @@ import ( openapicommon "k8s.io/kube-openapi/pkg/common" "k8s.io/kubernetes/pkg/api/legacyscheme" - api "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/controlplane" controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options" + "k8s.io/kubernetes/pkg/controlplane/controller/clusterauthenticationtrust" "k8s.io/kubernetes/pkg/kubeapiserver" "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest" + "k8s.io/kubernetes/pkg/serviceaccount" ) +// Config defines configuration for the master +type Config struct { + Generic *genericapiserver.Config + Extra +} + +type Extra struct { + ClusterAuthenticationInfo clusterauthenticationtrust.ClusterAuthenticationInfo + + APIResourceConfigSource serverstorage.APIResourceConfigSource + StorageFactory serverstorage.StorageFactory + EventTTL time.Duration + + EnableLogsSupport bool + ProxyTransport *http.Transport + + ServiceAccountIssuer serviceaccount.TokenGenerator + ServiceAccountMaxExpiration time.Duration + ExtendExpiration bool + + // ServiceAccountIssuerDiscovery + ServiceAccountIssuerURL string + ServiceAccountJWKSURI string + ServiceAccountPublicKeys []interface{} + + VersionedInformers clientgoinformers.SharedInformerFactory +} + // BuildGenericConfig takes the master server options and produces the genericapiserver.Config associated with it func BuildGenericConfig( s controlplaneapiserver.CompletedOptions, schemes []*runtime.Scheme, + resourceConfig *serverstorage.ResourceConfig, getOpenAPIDefinitions func(ref openapicommon.ReferenceCallback) map[string]openapicommon.OpenAPIDefinition, ) ( genericConfig *genericapiserver.Config, versionedInformers clientgoinformers.SharedInformerFactory, storageFactory *serverstorage.DefaultStorageFactory, - lastErr error, ) { genericConfig = genericapiserver.NewConfig(legacyscheme.Codecs) - genericConfig.MergedResourceConfig = controlplane.DefaultAPIResourceConfigSource() + genericConfig.MergedResourceConfig = resourceConfig if lastErr = s.GenericServerRunOptions.ApplyTo(genericConfig); lastErr != nil { return @@ -98,7 +127,7 @@ func BuildGenericConfig( if lastErr = s.Features.ApplyTo(genericConfig, clientgoExternalClient, versionedInformers); lastErr != nil { return } - if lastErr = s.APIEnablement.ApplyTo(genericConfig, controlplane.DefaultAPIResourceConfigSource(), legacyscheme.Scheme); lastErr != nil { + if lastErr = s.APIEnablement.ApplyTo(genericConfig, resourceConfig, legacyscheme.Scheme); lastErr != nil { return } if lastErr = s.EgressSelector.ApplyTo(genericConfig); lastErr != nil { @@ -213,18 +242,6 @@ func BuildAuthorizer(ctx context.Context, s controlplaneapiserver.CompletedOptio return authorizer, ruleResolver, enablesRBAC, err } -// CreatePeerEndpointLeaseReconciler creates a apiserver endpoint lease reconciliation loop -// The peer endpoint leases are used to find network locations of apiservers for peer proxy -func CreatePeerEndpointLeaseReconciler(c genericapiserver.Config, storageFactory serverstorage.StorageFactory) (reconcilers.PeerEndpointLeaseReconciler, error) { - ttl := controlplane.DefaultEndpointReconcilerTTL - config, err := storageFactory.NewConfig(api.Resource("apiServerPeerIPInfo")) - if err != nil { - return nil, fmt.Errorf("error creating storage factory config: %w", err) - } - reconciler, err := reconcilers.NewPeerEndpointLeaseReconciler(config, "/peerserverleases/", ttl) - return reconciler, err -} - func BuildPeerProxy(versionedInformer clientgoinformers.SharedInformerFactory, svm storageversion.Manager, proxyClientCertFile string, proxyClientKeyFile string, peerCAFile string, peerAdvertiseAddress reconcilers.PeerAdvertiseAddress, apiServerID string, reconciler reconcilers.PeerEndpointLeaseReconciler, serializer runtime.NegotiatedSerializer) (utilpeerproxy.Interface, error) { diff --git a/pkg/controlplane/apiserver/config_test.go b/pkg/controlplane/apiserver/config_test.go index 9065fcf889d..80bff6f50a8 100644 --- a/pkg/controlplane/apiserver/config_test.go +++ b/pkg/controlplane/apiserver/config_test.go @@ -17,6 +17,9 @@ limitations under the License. package apiserver import ( + "net" + "testing" + extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -26,8 +29,6 @@ import ( "k8s.io/kubernetes/pkg/controlplane/apiserver/options" generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi" netutils "k8s.io/utils/net" - "net" - "testing" ) func TestBuildGenericConfig(t *testing.T) { @@ -52,6 +53,7 @@ func TestBuildGenericConfig(t *testing.T) { genericConfig, _, storageFactory, err := BuildGenericConfig( completedOptions, []*runtime.Scheme{legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme}, + nil, generatedopenapi.GetOpenAPIDefinitions, ) if err != nil { diff --git a/pkg/controlplane/instance.go b/pkg/controlplane/instance.go index 6f673dd1e3d..7bd4beec4d3 100644 --- a/pkg/controlplane/instance.go +++ b/pkg/controlplane/instance.go @@ -19,7 +19,6 @@ package controlplane import ( "fmt" "net" - "net/http" "os" "reflect" "strconv" @@ -69,7 +68,6 @@ import ( serverstorage "k8s.io/apiserver/pkg/server/storage" utilfeature "k8s.io/apiserver/pkg/util/feature" utilpeerproxy "k8s.io/apiserver/pkg/util/peerproxy" - "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" discoveryclient "k8s.io/client-go/kubernetes/typed/discovery/v1" @@ -80,6 +78,7 @@ import ( flowcontrolv1beta1 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta1" flowcontrolv1beta2 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta2" flowcontrolv1beta3 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta3" + controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver" "k8s.io/kubernetes/pkg/controlplane/apiserver/options" "k8s.io/kubernetes/pkg/controlplane/controller/apiserverleasegc" "k8s.io/kubernetes/pkg/controlplane/controller/clusterauthenticationtrust" @@ -125,6 +124,9 @@ const ( DefaultEndpointReconcilerInterval = 10 * time.Second // DefaultEndpointReconcilerTTL is the default TTL timeout for the storage layer DefaultEndpointReconcilerTTL = 15 * time.Second + // DefaultPeerEndpointReconcilerTTL is the default TTL timeout for peer endpoint + // leases on the storage layer + DefaultPeerEndpointReconcilerTTL = 15 * time.Second // IdentityLeaseComponentLabelKey is used to apply a component label to identity lease objects, indicating: // 1. the lease is an identity lease (different from leader election leases) // 2. which component owns this lease @@ -144,36 +146,25 @@ var ( // IdentityLeaseDurationSeconds is the duration of kube-apiserver lease in seconds // IdentityLeaseDurationSeconds is exposed so integration tests can tune this value. IdentityLeaseDurationSeconds = 3600 - // IdentityLeaseRenewIntervalSeconds is the interval of kube-apiserver renewing its lease in seconds - // IdentityLeaseRenewIntervalSeconds is exposed so integration tests can tune this value. + // IdentityLeaseRenewIntervalPeriod is the interval of kube-apiserver renewing its lease in seconds + // IdentityLeaseRenewIntervalPeriod is exposed so integration tests can tune this value. IdentityLeaseRenewIntervalPeriod = 10 * time.Second ) -// ExtraConfig defines extra configuration for the master -type ExtraConfig struct { - ClusterAuthenticationInfo clusterauthenticationtrust.ClusterAuthenticationInfo - - APIResourceConfigSource serverstorage.APIResourceConfigSource - StorageFactory serverstorage.StorageFactory +// Extra defines extra configuration for the master +type Extra struct { EndpointReconcilerConfig EndpointReconcilerConfig - EventTTL time.Duration KubeletClientConfig kubeletclient.KubeletClientConfig - EnableLogsSupport bool - ProxyTransport *http.Transport - // PeerProxy, if not nil, sets proxy transport between kube-apiserver peers for requests // that can not be served locally PeerProxy utilpeerproxy.Interface - // PeerEndpointLeaseReconciler updates the peer endpoint leases PeerEndpointLeaseReconciler peerreconcilers.PeerEndpointLeaseReconciler - // PeerCAFile is the ca bundle used by this kube-apiserver to verify peer apiservers' // serving certs when routing a request to the peer in the case the request can not be served // locally due to version skew. PeerCAFile string - // PeerAdvertiseAddress is the IP for this kube-apiserver which is used by peer apiservers to route a request // to this apiserver. This happens in cases where the peer is not able to serve the request due to // version skew. If unset, AdvertiseAddress/BindAddress will be used. @@ -219,17 +210,6 @@ type ExtraConfig struct { // Selects which reconciler to use EndpointReconcilerType reconcilers.Type - ServiceAccountIssuer serviceaccount.TokenGenerator - ServiceAccountMaxExpiration time.Duration - ExtendExpiration bool - - // ServiceAccountIssuerDiscovery - ServiceAccountIssuerURL string - ServiceAccountJWKSURI string - ServiceAccountPublicKeys []interface{} - - VersionedInformers informers.SharedInformerFactory - // RepairServicesInterval interval used by the repair loops for // the Services NodePort and ClusterIP resources RepairServicesInterval time.Duration @@ -237,13 +217,13 @@ type ExtraConfig struct { // Config defines configuration for the master type Config struct { - GenericConfig *genericapiserver.Config - ExtraConfig ExtraConfig + ControlPlane controlplaneapiserver.Config + Extra } type completedConfig struct { - GenericConfig genericapiserver.CompletedConfig - ExtraConfig *ExtraConfig + ControlPlane controlplaneapiserver.CompletedConfig + *Extra } // CompletedConfig embeds a private pointer that cannot be instantiated outside of this package @@ -266,11 +246,11 @@ type Instance struct { } func (c *Config) createMasterCountReconciler() reconcilers.EndpointReconciler { - endpointClient := corev1client.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig) - endpointSliceClient := discoveryclient.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig) + endpointClient := corev1client.NewForConfigOrDie(c.ControlPlane.Generic.LoopbackClientConfig) + endpointSliceClient := discoveryclient.NewForConfigOrDie(c.ControlPlane.Generic.LoopbackClientConfig) endpointsAdapter := reconcilers.NewEndpointsAdapter(endpointClient, endpointSliceClient) - return reconcilers.NewMasterCountEndpointReconciler(c.ExtraConfig.MasterCount, endpointsAdapter) + return reconcilers.NewMasterCountEndpointReconciler(c.Extra.MasterCount, endpointsAdapter) } func (c *Config) createNoneReconciler() reconcilers.EndpointReconciler { @@ -278,12 +258,12 @@ func (c *Config) createNoneReconciler() reconcilers.EndpointReconciler { } func (c *Config) createLeaseReconciler() reconcilers.EndpointReconciler { - endpointClient := corev1client.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig) - endpointSliceClient := discoveryclient.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig) + endpointClient := corev1client.NewForConfigOrDie(c.ControlPlane.Generic.LoopbackClientConfig) + endpointSliceClient := discoveryclient.NewForConfigOrDie(c.ControlPlane.Generic.LoopbackClientConfig) endpointsAdapter := reconcilers.NewEndpointsAdapter(endpointClient, endpointSliceClient) - ttl := c.ExtraConfig.MasterEndpointReconcileTTL - config, err := c.ExtraConfig.StorageFactory.NewConfig(api.Resource("apiServerIPInfo")) + ttl := c.Extra.MasterEndpointReconcileTTL + config, err := c.ControlPlane.StorageFactory.NewConfig(api.Resource("apiServerIPInfo")) if err != nil { klog.Fatalf("Error creating storage factory config: %v", err) } @@ -296,8 +276,8 @@ func (c *Config) createLeaseReconciler() reconcilers.EndpointReconciler { } func (c *Config) createEndpointReconciler() reconcilers.EndpointReconciler { - klog.Infof("Using reconciler: %v", c.ExtraConfig.EndpointReconcilerType) - switch c.ExtraConfig.EndpointReconcilerType { + klog.Infof("Using reconciler: %v", c.Extra.EndpointReconcilerType) + switch c.Extra.EndpointReconcilerType { // there are numerous test dependencies that depend on a default controller case reconcilers.MasterCountReconcilerType: return c.createMasterCountReconciler() @@ -306,7 +286,7 @@ func (c *Config) createEndpointReconciler() reconcilers.EndpointReconciler { case reconcilers.NoneEndpointReconcilerType: return c.createNoneReconciler() default: - klog.Fatalf("Reconciler not implemented: %v", c.ExtraConfig.EndpointReconcilerType) + klog.Fatalf("Reconciler not implemented: %v", c.Extra.EndpointReconcilerType) } return nil } @@ -314,49 +294,50 @@ func (c *Config) createEndpointReconciler() reconcilers.EndpointReconciler { // Complete fills in any fields not set that are required to have valid data. It's mutating the receiver. func (c *Config) Complete() CompletedConfig { cfg := completedConfig{ - c.GenericConfig.Complete(c.ExtraConfig.VersionedInformers), - &c.ExtraConfig, + c.ControlPlane.Complete(), + &c.Extra, } - serviceIPRange, apiServerServiceIP, err := options.ServiceIPRange(cfg.ExtraConfig.ServiceIPRange) + serviceIPRange, apiServerServiceIP, err := options.ServiceIPRange(cfg.Extra.ServiceIPRange) if err != nil { klog.Fatalf("Error determining service IP ranges: %v", err) } - if cfg.ExtraConfig.ServiceIPRange.IP == nil { - cfg.ExtraConfig.ServiceIPRange = serviceIPRange + if cfg.Extra.ServiceIPRange.IP == nil { + cfg.Extra.ServiceIPRange = serviceIPRange } - if cfg.ExtraConfig.APIServerServiceIP == nil { - cfg.ExtraConfig.APIServerServiceIP = apiServerServiceIP + if cfg.Extra.APIServerServiceIP == nil { + cfg.Extra.APIServerServiceIP = apiServerServiceIP } - discoveryAddresses := discovery.DefaultAddresses{DefaultAddress: cfg.GenericConfig.ExternalAddress} + // override the default discovery addresses in the generic controlplane adding service IP support + discoveryAddresses := discovery.DefaultAddresses{DefaultAddress: cfg.ControlPlane.Generic.ExternalAddress} discoveryAddresses.CIDRRules = append(discoveryAddresses.CIDRRules, - discovery.CIDRRule{IPRange: cfg.ExtraConfig.ServiceIPRange, Address: net.JoinHostPort(cfg.ExtraConfig.APIServerServiceIP.String(), strconv.Itoa(cfg.ExtraConfig.APIServerServicePort))}) - cfg.GenericConfig.DiscoveryAddresses = discoveryAddresses + discovery.CIDRRule{IPRange: cfg.Extra.ServiceIPRange, Address: net.JoinHostPort(cfg.Extra.APIServerServiceIP.String(), strconv.Itoa(cfg.Extra.APIServerServicePort))}) + cfg.ControlPlane.Generic.DiscoveryAddresses = discoveryAddresses - if cfg.ExtraConfig.ServiceNodePortRange.Size == 0 { + if cfg.Extra.ServiceNodePortRange.Size == 0 { // TODO: Currently no way to specify an empty range (do we need to allow this?) // We should probably allow this for clouds that don't require NodePort to do load-balancing (GCE) // but then that breaks the strict nestedness of ServiceType. // Review post-v1 - cfg.ExtraConfig.ServiceNodePortRange = kubeoptions.DefaultServiceNodePortRange - klog.Infof("Node port range unspecified. Defaulting to %v.", cfg.ExtraConfig.ServiceNodePortRange) + cfg.Extra.ServiceNodePortRange = kubeoptions.DefaultServiceNodePortRange + klog.Infof("Node port range unspecified. Defaulting to %v.", cfg.Extra.ServiceNodePortRange) } - if cfg.ExtraConfig.EndpointReconcilerConfig.Interval == 0 { - cfg.ExtraConfig.EndpointReconcilerConfig.Interval = DefaultEndpointReconcilerInterval + if cfg.Extra.EndpointReconcilerConfig.Interval == 0 { + cfg.Extra.EndpointReconcilerConfig.Interval = DefaultEndpointReconcilerInterval } - if cfg.ExtraConfig.MasterEndpointReconcileTTL == 0 { - cfg.ExtraConfig.MasterEndpointReconcileTTL = DefaultEndpointReconcilerTTL + if cfg.Extra.MasterEndpointReconcileTTL == 0 { + cfg.Extra.MasterEndpointReconcileTTL = DefaultEndpointReconcilerTTL } - if cfg.ExtraConfig.EndpointReconcilerConfig.Reconciler == nil { - cfg.ExtraConfig.EndpointReconcilerConfig.Reconciler = c.createEndpointReconciler() + if cfg.Extra.EndpointReconcilerConfig.Reconciler == nil { + cfg.Extra.EndpointReconcilerConfig.Reconciler = c.createEndpointReconciler() } - if cfg.ExtraConfig.RepairServicesInterval == 0 { - cfg.ExtraConfig.RepairServicesInterval = repairLoopInterval + if cfg.Extra.RepairServicesInterval == 0 { + cfg.Extra.RepairServicesInterval = repairLoopInterval } return CompletedConfig{&cfg} @@ -366,27 +347,27 @@ func (c *Config) Complete() CompletedConfig { // Certain config fields will be set to a default value if unset. // Certain config fields must be specified, including: // KubeletClientConfig -func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) (*Instance, error) { - if reflect.DeepEqual(c.ExtraConfig.KubeletClientConfig, kubeletclient.KubeletClientConfig{}) { +func (c CompletedConfig) New(delegationTarget genericapiserver.DelegationTarget) (*Instance, error) { + if reflect.DeepEqual(c.Extra.KubeletClientConfig, kubeletclient.KubeletClientConfig{}) { return nil, fmt.Errorf("Master.New() called with empty config.KubeletClientConfig") } - s, err := c.GenericConfig.New("kube-apiserver", delegationTarget) + s, err := c.ControlPlane.Generic.New("kube-apiserver", delegationTarget) if err != nil { return nil, err } - if c.ExtraConfig.EnableLogsSupport { + if c.ControlPlane.Extra.EnableLogsSupport { routes.Logs{}.Install(s.Handler.GoRestfulContainer) } // Metadata and keys are expected to only change across restarts at present, // so we just marshal immediately and serve the cached JSON bytes. md, err := serviceaccount.NewOpenIDMetadata( - c.ExtraConfig.ServiceAccountIssuerURL, - c.ExtraConfig.ServiceAccountJWKSURI, - c.GenericConfig.ExternalAddress, - c.ExtraConfig.ServiceAccountPublicKeys, + c.ControlPlane.Extra.ServiceAccountIssuerURL, + c.ControlPlane.Extra.ServiceAccountJWKSURI, + c.ControlPlane.Generic.ExternalAddress, + c.ControlPlane.Extra.ServiceAccountPublicKeys, ) if err != nil { // If there was an error, skip installing the endpoints and log the @@ -396,7 +377,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) msg := fmt.Sprintf("Could not construct pre-rendered responses for"+ " ServiceAccountIssuerDiscovery endpoints. Endpoints will not be"+ " enabled. Error: %v", err) - if c.ExtraConfig.ServiceAccountIssuerURL != "" { + if c.ControlPlane.Extra.ServiceAccountIssuerURL != "" { // The user likely expects this feature to be enabled if issuer URL is // set and the feature gate is enabled. In the future, if there is no // longer a feature gate and issuer URL is not set, the user may not @@ -413,37 +394,37 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) m := &Instance{ GenericAPIServer: s, - ClusterAuthenticationInfo: c.ExtraConfig.ClusterAuthenticationInfo, + ClusterAuthenticationInfo: c.ControlPlane.Extra.ClusterAuthenticationInfo, } - clientset, err := kubernetes.NewForConfig(c.GenericConfig.LoopbackClientConfig) + client, err := kubernetes.NewForConfig(c.ControlPlane.Generic.LoopbackClientConfig) if err != nil { return nil, err } // TODO: update to a version that caches success but will recheck on failure, unlike memcache discovery - discoveryClientForAdmissionRegistration := clientset.Discovery() + discoveryClientForAdmissionRegistration := client.Discovery() legacyRESTStorageProvider, err := corerest.New(corerest.Config{ GenericConfig: corerest.GenericConfig{ - StorageFactory: c.ExtraConfig.StorageFactory, - EventTTL: c.ExtraConfig.EventTTL, - LoopbackClientConfig: c.GenericConfig.LoopbackClientConfig, - ServiceAccountIssuer: c.ExtraConfig.ServiceAccountIssuer, - ExtendExpiration: c.ExtraConfig.ExtendExpiration, - ServiceAccountMaxExpiration: c.ExtraConfig.ServiceAccountMaxExpiration, - APIAudiences: c.GenericConfig.Authentication.APIAudiences, - Informers: c.ExtraConfig.VersionedInformers, + StorageFactory: c.ControlPlane.Extra.StorageFactory, + EventTTL: c.ControlPlane.Extra.EventTTL, + LoopbackClientConfig: c.ControlPlane.Generic.LoopbackClientConfig, + ServiceAccountIssuer: c.ControlPlane.Extra.ServiceAccountIssuer, + ExtendExpiration: c.ControlPlane.Extra.ExtendExpiration, + ServiceAccountMaxExpiration: c.ControlPlane.Extra.ServiceAccountMaxExpiration, + APIAudiences: c.ControlPlane.Generic.Authentication.APIAudiences, + Informers: c.ControlPlane.Extra.VersionedInformers, }, Proxy: corerest.ProxyConfig{ - Transport: c.ExtraConfig.ProxyTransport, - KubeletClientConfig: c.ExtraConfig.KubeletClientConfig, + Transport: c.ControlPlane.Extra.ProxyTransport, + KubeletClientConfig: c.Extra.KubeletClientConfig, }, Services: corerest.ServicesConfig{ - ClusterIPRange: c.ExtraConfig.ServiceIPRange, - SecondaryClusterIPRange: c.ExtraConfig.SecondaryServiceIPRange, - NodePortRange: c.ExtraConfig.ServiceNodePortRange, - IPRepairInterval: c.ExtraConfig.RepairServicesInterval, + ClusterIPRange: c.Extra.ServiceIPRange, + SecondaryClusterIPRange: c.Extra.SecondaryServiceIPRange, + NodePortRange: c.Extra.ServiceNodePortRange, + IPRepairInterval: c.Extra.RepairServicesInterval, }, }) if err != nil { @@ -460,8 +441,8 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) restStorageProviders := []RESTStorageProvider{ legacyRESTStorageProvider, apiserverinternalrest.StorageProvider{}, - authenticationrest.RESTStorageProvider{Authenticator: c.GenericConfig.Authentication.Authenticator, APIAudiences: c.GenericConfig.Authentication.APIAudiences}, - authorizationrest.RESTStorageProvider{Authorizer: c.GenericConfig.Authorization.Authorizer, RuleResolver: c.GenericConfig.RuleResolver}, + authenticationrest.RESTStorageProvider{Authenticator: c.ControlPlane.Generic.Authentication.Authenticator, APIAudiences: c.ControlPlane.Generic.Authentication.APIAudiences}, + authorizationrest.RESTStorageProvider{Authorizer: c.ControlPlane.Generic.Authorization.Authorizer, RuleResolver: c.ControlPlane.Generic.RuleResolver}, autoscalingrest.RESTStorageProvider{}, batchrest.RESTStorageProvider{}, certificatesrest.RESTStorageProvider{}, @@ -470,47 +451,47 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) networkingrest.RESTStorageProvider{}, noderest.RESTStorageProvider{}, policyrest.RESTStorageProvider{}, - rbacrest.RESTStorageProvider{Authorizer: c.GenericConfig.Authorization.Authorizer}, + rbacrest.RESTStorageProvider{Authorizer: c.ControlPlane.Generic.Authorization.Authorizer}, schedulingrest.RESTStorageProvider{}, storagerest.RESTStorageProvider{}, svmrest.RESTStorageProvider{}, - flowcontrolrest.RESTStorageProvider{InformerFactory: c.GenericConfig.SharedInformerFactory}, + flowcontrolrest.RESTStorageProvider{InformerFactory: c.ControlPlane.Generic.SharedInformerFactory}, // keep apps after extensions so legacy clients resolve the extensions versions of shared resource names. // See https://github.com/kubernetes/kubernetes/issues/42392 appsrest.StorageProvider{}, - admissionregistrationrest.RESTStorageProvider{Authorizer: c.GenericConfig.Authorization.Authorizer, DiscoveryClient: discoveryClientForAdmissionRegistration}, - eventsrest.RESTStorageProvider{TTL: c.ExtraConfig.EventTTL}, + admissionregistrationrest.RESTStorageProvider{Authorizer: c.ControlPlane.Generic.Authorization.Authorizer, DiscoveryClient: discoveryClientForAdmissionRegistration}, + eventsrest.RESTStorageProvider{TTL: c.ControlPlane.EventTTL}, resourcerest.RESTStorageProvider{}, } - if err := m.InstallAPIs(c.ExtraConfig.APIResourceConfigSource, c.GenericConfig.RESTOptionsGetter, restStorageProviders...); err != nil { + if err := m.InstallAPIs(c.ControlPlane.Extra.APIResourceConfigSource, c.ControlPlane.Generic.RESTOptionsGetter, restStorageProviders...); err != nil { return nil, err } m.GenericAPIServer.AddPostStartHookOrDie("start-system-namespaces-controller", func(hookContext genericapiserver.PostStartHookContext) error { - go systemnamespaces.NewController(clientset, c.ExtraConfig.VersionedInformers.Core().V1().Namespaces()).Run(hookContext.StopCh) + go systemnamespaces.NewController(client, c.ControlPlane.Extra.VersionedInformers.Core().V1().Namespaces()).Run(hookContext.StopCh) return nil }) - _, publicServicePort, err := c.GenericConfig.SecureServing.HostPort() + _, publicServicePort, err := c.ControlPlane.Generic.SecureServing.HostPort() if err != nil { return nil, fmt.Errorf("failed to get listener address: %w", err) } kubernetesServiceCtrl := kubernetesservice.New(kubernetesservice.Config{ - PublicIP: c.GenericConfig.PublicAddress, + PublicIP: c.ControlPlane.Generic.PublicAddress, - EndpointReconciler: c.ExtraConfig.EndpointReconcilerConfig.Reconciler, - EndpointInterval: c.ExtraConfig.EndpointReconcilerConfig.Interval, + EndpointReconciler: c.Extra.EndpointReconcilerConfig.Reconciler, + EndpointInterval: c.Extra.EndpointReconcilerConfig.Interval, - ServiceIP: c.ExtraConfig.APIServerServiceIP, - ServicePort: c.ExtraConfig.APIServerServicePort, + ServiceIP: c.Extra.APIServerServiceIP, + ServicePort: c.Extra.APIServerServicePort, PublicServicePort: publicServicePort, - KubernetesServiceNodePort: c.ExtraConfig.KubernetesServiceNodePort, - }, clientset, c.ExtraConfig.VersionedInformers.Core().V1().Services()) - m.GenericAPIServer.AddPostStartHookOrDie("bootstrap-controller", func(hookContext genericapiserver.PostStartHookContext) error { + KubernetesServiceNodePort: c.Extra.KubernetesServiceNodePort, + }, client, c.ControlPlane.Extra.VersionedInformers.Core().V1().Services()) + s.AddPostStartHookOrDie("bootstrap-controller", func(hookContext genericapiserver.PostStartHookContext) error { kubernetesServiceCtrl.Start(hookContext.StopCh) return nil }) - m.GenericAPIServer.AddPreShutdownHookOrDie("stop-kubernetes-service-controller", func() error { + s.AddPreShutdownHookOrDie("stop-kubernetes-service-controller", func() error { kubernetesServiceCtrl.Stop() return nil }) @@ -518,9 +499,9 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) if utilfeature.DefaultFeatureGate.Enabled(features.MultiCIDRServiceAllocator) { m.GenericAPIServer.AddPostStartHookOrDie("start-kubernetes-service-cidr-controller", func(hookContext genericapiserver.PostStartHookContext) error { controller := defaultservicecidr.NewController( - c.ExtraConfig.ServiceIPRange, - c.ExtraConfig.SecondaryServiceIPRange, - clientset, + c.Extra.ServiceIPRange, + c.Extra.SecondaryServiceIPRange, + client, ) // The default serviceCIDR must exist before the apiserver is healthy // otherwise the allocators for Services will not work. @@ -530,13 +511,13 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) } if utilfeature.DefaultFeatureGate.Enabled(features.UnknownVersionInteroperabilityProxy) { - peeraddress := getPeerAddress(c.ExtraConfig.PeerAdvertiseAddress, c.GenericConfig.PublicAddress, publicServicePort) + peeraddress := getPeerAddress(c.Extra.PeerAdvertiseAddress, c.ControlPlane.Generic.PublicAddress, publicServicePort) peerEndpointCtrl := peerreconcilers.New( - c.GenericConfig.APIServerID, + c.ControlPlane.Generic.APIServerID, peeraddress, - c.ExtraConfig.PeerEndpointLeaseReconciler, - c.ExtraConfig.EndpointReconcilerConfig.Interval, - clientset) + c.Extra.PeerEndpointLeaseReconciler, + c.Extra.EndpointReconcilerConfig.Interval, + client) if err != nil { return nil, fmt.Errorf("failed to create peer endpoint lease controller: %w", err) } @@ -551,16 +532,16 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) return nil }) // Add PostStartHooks for Unknown Version Proxy filter. - if c.ExtraConfig.PeerProxy != nil { + if c.Extra.PeerProxy != nil { m.GenericAPIServer.AddPostStartHookOrDie("unknown-version-proxy-filter", func(context genericapiserver.PostStartHookContext) error { - err := c.ExtraConfig.PeerProxy.WaitForCacheSync(context.StopCh) + err := c.Extra.PeerProxy.WaitForCacheSync(context.StopCh) return err }) } } m.GenericAPIServer.AddPostStartHookOrDie("start-cluster-authentication-info-controller", func(hookContext genericapiserver.PostStartHookContext) error { - controller := clusterauthenticationtrust.NewClusterAuthenticationTrustController(m.ClusterAuthenticationInfo, clientset) + controller := clusterauthenticationtrust.NewClusterAuthenticationTrustController(m.ClusterAuthenticationInfo, client) // generate a context from stopCh. This is to avoid modifying files which are relying on apiserver // TODO: See if we can pass ctx to the current method @@ -601,11 +582,11 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) leaseName := m.GenericAPIServer.APIServerID holderIdentity := m.GenericAPIServer.APIServerID + "_" + string(uuid.NewUUID()) - peeraddress := getPeerAddress(c.ExtraConfig.PeerAdvertiseAddress, c.GenericConfig.PublicAddress, publicServicePort) + peeraddress := getPeerAddress(c.Extra.PeerAdvertiseAddress, c.ControlPlane.Generic.PublicAddress, publicServicePort) // must replace ':,[]' in [ip:port] to be able to store this as a valid label value controller := lease.NewController( clock.RealClock{}, - clientset, + client, holderIdentity, int32(IdentityLeaseDurationSeconds), nil, @@ -620,7 +601,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) // TODO: move this into generic apiserver and make the lease identity value configurable m.GenericAPIServer.AddPostStartHookOrDie("start-kube-apiserver-identity-lease-garbage-collector", func(hookContext genericapiserver.PostStartHookContext) error { go apiserverleasegc.NewAPIServerLeaseGC( - clientset, + client, IdentityLeaseGCPeriod, metav1.NamespaceSystem, KubeAPIServerIdentityLeaseLabelSelector, @@ -630,7 +611,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) } m.GenericAPIServer.AddPostStartHookOrDie("start-legacy-token-tracking-controller", func(hookContext genericapiserver.PostStartHookContext) error { - go legacytokentracking.NewController(clientset).Run(hookContext.StopCh) + go legacytokentracking.NewController(client).Run(hookContext.StopCh) return nil }) @@ -804,6 +785,18 @@ func DefaultAPIResourceConfigSource() *serverstorage.ResourceConfig { return ret } +// CreatePeerEndpointLeaseReconciler creates a apiserver endpoint lease reconciliation loop +// The peer endpoint leases are used to find network locations of apiservers for peer proxy +func CreatePeerEndpointLeaseReconciler(c genericapiserver.Config, storageFactory serverstorage.StorageFactory) (peerreconcilers.PeerEndpointLeaseReconciler, error) { + ttl := DefaultPeerEndpointReconcilerTTL + config, err := storageFactory.NewConfig(api.Resource("apiServerPeerIPInfo")) + if err != nil { + return nil, fmt.Errorf("error creating storage factory config: %w", err) + } + reconciler, err := peerreconcilers.NewPeerEndpointLeaseReconciler(config, "/peerserverleases/", ttl) + return reconciler, err +} + // utility function to get the apiserver address that is used by peer apiservers to proxy // requests to this apiserver in case the peer is incapable of serving the request func getPeerAddress(peerAdvertiseAddress peerreconcilers.PeerAdvertiseAddress, publicAddress net.IP, publicServicePort int) string { diff --git a/pkg/controlplane/instance_test.go b/pkg/controlplane/instance_test.go index c921cf73c80..ff165c9f711 100644 --- a/pkg/controlplane/instance_test.go +++ b/pkg/controlplane/instance_test.go @@ -28,6 +28,8 @@ import ( "strings" "testing" + "github.com/stretchr/testify/assert" + autoscalingapiv2beta1 "k8s.io/api/autoscaling/v2beta1" autoscalingapiv2beta2 "k8s.io/api/autoscaling/v2beta2" batchapiv1beta1 "k8s.io/api/batch/v1beta1" @@ -57,8 +59,11 @@ import ( restclient "k8s.io/client-go/rest" kubeversion "k8s.io/component-base/version" aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" + netutils "k8s.io/utils/net" + "k8s.io/kubernetes/pkg/api/legacyscheme" flowcontrolv1bet3 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta3" + controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver" "k8s.io/kubernetes/pkg/controlplane/reconcilers" "k8s.io/kubernetes/pkg/controlplane/storageversionhashdata" generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi" @@ -67,9 +72,6 @@ import ( certificatesrest "k8s.io/kubernetes/pkg/registry/certificates/rest" corerest "k8s.io/kubernetes/pkg/registry/core/rest" "k8s.io/kubernetes/pkg/registry/registrytest" - netutils "k8s.io/utils/net" - - "github.com/stretchr/testify/assert" ) // setUp is a convenience function for setting up for (most) tests. @@ -77,53 +79,57 @@ func setUp(t *testing.T) (*etcd3testing.EtcdTestServer, Config, *assert.Assertio server, storageConfig := etcd3testing.NewUnsecuredEtcd3TestClientServer(t) config := &Config{ - GenericConfig: genericapiserver.NewConfig(legacyscheme.Codecs), - ExtraConfig: ExtraConfig{ - APIResourceConfigSource: DefaultAPIResourceConfigSource(), - APIServerServicePort: 443, - MasterCount: 1, - EndpointReconcilerType: reconcilers.MasterCountReconcilerType, - ServiceIPRange: net.IPNet{IP: netutils.ParseIPSloppy("10.0.0.0"), Mask: net.CIDRMask(24, 32)}, + ControlPlane: controlplaneapiserver.Config{ + Generic: genericapiserver.NewConfig(legacyscheme.Codecs), + Extra: controlplaneapiserver.Extra{ + APIResourceConfigSource: DefaultAPIResourceConfigSource(), + }, + }, + Extra: Extra{ + APIServerServicePort: 443, + MasterCount: 1, + EndpointReconcilerType: reconcilers.MasterCountReconcilerType, + ServiceIPRange: net.IPNet{IP: netutils.ParseIPSloppy("10.0.0.0"), Mask: net.CIDRMask(24, 32)}, }, } storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig() - storageConfig.StorageObjectCountTracker = config.GenericConfig.StorageObjectCountTracker + storageConfig.StorageObjectCountTracker = config.ControlPlane.Generic.StorageObjectCountTracker resourceEncoding := resourceconfig.MergeResourceEncodingConfigs(storageFactoryConfig.DefaultResourceEncoding, storageFactoryConfig.ResourceEncodingOverrides) storageFactory := serverstorage.NewDefaultStorageFactory(*storageConfig, "application/vnd.kubernetes.protobuf", storageFactoryConfig.Serializer, resourceEncoding, DefaultAPIResourceConfigSource(), nil) etcdOptions := options.NewEtcdOptions(storageConfig) // unit tests don't need watch cache and it leaks lots of goroutines with etcd testing functions during unit tests etcdOptions.EnableWatchCache = false - err := etcdOptions.ApplyWithStorageFactoryTo(storageFactory, config.GenericConfig) + err := etcdOptions.ApplyWithStorageFactoryTo(storageFactory, config.ControlPlane.Generic) if err != nil { t.Fatal(err) } kubeVersion := kubeversion.Get() - config.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer() - config.GenericConfig.Version = &kubeVersion - config.ExtraConfig.StorageFactory = storageFactory - config.GenericConfig.LoopbackClientConfig = &restclient.Config{APIPath: "/api", ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}} - config.GenericConfig.PublicAddress = netutils.ParseIPSloppy("192.168.10.4") - config.GenericConfig.LegacyAPIGroupPrefixes = sets.NewString("/api") - config.ExtraConfig.KubeletClientConfig = kubeletclient.KubeletClientConfig{Port: 10250} - config.ExtraConfig.ProxyTransport = utilnet.SetTransportDefaults(&http.Transport{ + config.ControlPlane.Generic.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer() + config.ControlPlane.Generic.Version = &kubeVersion + config.ControlPlane.StorageFactory = storageFactory + config.ControlPlane.Generic.LoopbackClientConfig = &restclient.Config{APIPath: "/api", ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}} + config.ControlPlane.Generic.PublicAddress = netutils.ParseIPSloppy("192.168.10.4") + config.ControlPlane.Generic.LegacyAPIGroupPrefixes = sets.NewString("/api") + config.Extra.KubeletClientConfig = kubeletclient.KubeletClientConfig{Port: 10250} + config.ControlPlane.ProxyTransport = utilnet.SetTransportDefaults(&http.Transport{ DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { return nil, nil }, TLSClientConfig: &tls.Config{}, }) // set fake SecureServingInfo because the listener port is needed for the kubernetes service - config.GenericConfig.SecureServing = &genericapiserver.SecureServingInfo{Listener: fakeLocalhost443Listener{}} + config.ControlPlane.Generic.SecureServing = &genericapiserver.SecureServingInfo{Listener: fakeLocalhost443Listener{}} getOpenAPIDefinitions := openapi.GetOpenAPIDefinitionsWithoutDisabledFeatures(generatedopenapi.GetOpenAPIDefinitions) namer := openapinamer.NewDefinitionNamer(legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme) - config.GenericConfig.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config(getOpenAPIDefinitions, namer) + config.ControlPlane.Generic.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config(getOpenAPIDefinitions, namer) - clientset, err := kubernetes.NewForConfig(config.GenericConfig.LoopbackClientConfig) + clientset, err := kubernetes.NewForConfig(config.ControlPlane.Generic.LoopbackClientConfig) if err != nil { t.Fatalf("unable to create client set due to %v", err) } - config.ExtraConfig.VersionedInformers = informers.NewSharedInformerFactory(clientset, config.GenericConfig.LoopbackClientConfig.Timeout) + config.ControlPlane.VersionedInformers = informers.NewSharedInformerFactory(clientset, config.ControlPlane.Generic.LoopbackClientConfig.Timeout) return server, *config, assert.New(t) } @@ -154,25 +160,25 @@ func TestLegacyRestStorageStrategies(t *testing.T) { storageProvider, err := corerest.New(corerest.Config{ GenericConfig: corerest.GenericConfig{ - StorageFactory: apiserverCfg.ExtraConfig.StorageFactory, - EventTTL: apiserverCfg.ExtraConfig.EventTTL, - LoopbackClientConfig: apiserverCfg.GenericConfig.LoopbackClientConfig, - Informers: apiserverCfg.ExtraConfig.VersionedInformers, + StorageFactory: apiserverCfg.ControlPlane.Extra.StorageFactory, + EventTTL: apiserverCfg.ControlPlane.Extra.EventTTL, + LoopbackClientConfig: apiserverCfg.ControlPlane.Generic.LoopbackClientConfig, + Informers: apiserverCfg.ControlPlane.Extra.VersionedInformers, }, Proxy: corerest.ProxyConfig{ - Transport: apiserverCfg.ExtraConfig.ProxyTransport, - KubeletClientConfig: apiserverCfg.ExtraConfig.KubeletClientConfig, + Transport: apiserverCfg.ControlPlane.Extra.ProxyTransport, + KubeletClientConfig: apiserverCfg.Extra.KubeletClientConfig, }, Services: corerest.ServicesConfig{ - ClusterIPRange: apiserverCfg.ExtraConfig.ServiceIPRange, - NodePortRange: apiserverCfg.ExtraConfig.ServiceNodePortRange, + ClusterIPRange: apiserverCfg.Extra.ServiceIPRange, + NodePortRange: apiserverCfg.Extra.ServiceNodePortRange, }, }) if err != nil { t.Fatalf("unexpected error from REST storage: %v", err) } - apiGroupInfo, err := storageProvider.NewRESTStorage(serverstorage.NewResourceConfig(), apiserverCfg.GenericConfig.RESTOptionsGetter) + apiGroupInfo, err := storageProvider.NewRESTStorage(serverstorage.NewResourceConfig(), apiserverCfg.ControlPlane.Generic.RESTOptionsGetter) if err != nil { t.Errorf("failed to create legacy REST storage: %v", err) } @@ -188,7 +194,7 @@ func TestCertificatesRestStorageStrategies(t *testing.T) { defer etcdserver.Terminate(t) certStorageProvider := certificatesrest.RESTStorageProvider{} - apiGroupInfo, err := certStorageProvider.NewRESTStorage(apiserverCfg.ExtraConfig.APIResourceConfigSource, apiserverCfg.GenericConfig.RESTOptionsGetter) + apiGroupInfo, err := certStorageProvider.NewRESTStorage(apiserverCfg.ControlPlane.APIResourceConfigSource, apiserverCfg.ControlPlane.Generic.RESTOptionsGetter) if err != nil { t.Fatalf("unexpected error from REST storage: %v", err) } diff --git a/test/integration/apiserver/apiserver_test.go b/test/integration/apiserver/apiserver_test.go index 1d93f755322..7341c71f430 100644 --- a/test/integration/apiserver/apiserver_test.go +++ b/test/integration/apiserver/apiserver_test.go @@ -84,7 +84,7 @@ func setupWithResources(t *testing.T, groupVersions []schema.GroupVersion, resou resourceConfig := controlplane.DefaultAPIResourceConfigSource() resourceConfig.EnableVersions(groupVersions...) resourceConfig.EnableResources(resources...) - config.ExtraConfig.APIResourceConfigSource = resourceConfig + config.ControlPlane.Extra.APIResourceConfigSource = resourceConfig } }, }) diff --git a/test/integration/apiserver/flowcontrol/concurrency_util_test.go b/test/integration/apiserver/flowcontrol/concurrency_util_test.go index 75d8adf1291..7d5fa348b37 100644 --- a/test/integration/apiserver/flowcontrol/concurrency_util_test.go +++ b/test/integration/apiserver/flowcontrol/concurrency_util_test.go @@ -155,7 +155,7 @@ func TestConcurrencyIsolation(t *testing.T) { }, ModifyServerConfig: func(config *controlplane.Config) { // Wrap default authorizer with one that delays requests from noxu clients - config.GenericConfig.Authorization.Authorizer = &noxuDelayingAuthorizer{config.GenericConfig.Authorization.Authorizer} + config.ControlPlane.Generic.Authorization.Authorizer = &noxuDelayingAuthorizer{config.ControlPlane.Generic.Authorization.Authorizer} }, }) defer closeFn() diff --git a/test/integration/apiserver/openapi/openapi_enum_test.go b/test/integration/apiserver/openapi/openapi_enum_test.go index 19285834105..931e9f02325 100644 --- a/test/integration/apiserver/openapi/openapi_enum_test.go +++ b/test/integration/apiserver/openapi/openapi_enum_test.go @@ -77,8 +77,8 @@ func TestEnablingOpenAPIEnumTypes(t *testing.T) { _, kubeConfig, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ ModifyServerConfig: func(config *controlplane.Config) { - config.GenericConfig.OpenAPIConfig = framework.DefaultOpenAPIConfig() - config.GenericConfig.OpenAPIConfig.GetDefinitions = getDefinitionsFn + config.ControlPlane.Generic.OpenAPIConfig = framework.DefaultOpenAPIConfig() + config.ControlPlane.Generic.OpenAPIConfig.GetDefinitions = getDefinitionsFn }, }) defer tearDownFn() diff --git a/test/integration/apiserver/watchcache_test.go b/test/integration/apiserver/watchcache_test.go index a1db723cbf8..3c280119fb2 100644 --- a/test/integration/apiserver/watchcache_test.go +++ b/test/integration/apiserver/watchcache_test.go @@ -61,7 +61,7 @@ func multiEtcdSetup(ctx context.Context, t *testing.T) (clientset.Interface, fra }, ModifyServerConfig: func(config *controlplane.Config) { // Switch off endpoints reconciler to avoid unnecessary operations. - config.ExtraConfig.EndpointReconcilerType = reconcilers.NoneEndpointReconcilerType + config.Extra.EndpointReconcilerType = reconcilers.NoneEndpointReconcilerType }, }) @@ -170,7 +170,7 @@ func BenchmarkListFromWatchCache(b *testing.B) { c, _, tearDownFn := framework.StartTestServer(tCtx, b, framework.TestServerSetup{ ModifyServerConfig: func(config *controlplane.Config) { // Switch off endpoints reconciler to avoid unnecessary operations. - config.ExtraConfig.EndpointReconcilerType = reconcilers.NoneEndpointReconcilerType + config.Extra.EndpointReconcilerType = reconcilers.NoneEndpointReconcilerType }, }) defer tearDownFn() diff --git a/test/integration/auth/accessreview_test.go b/test/integration/auth/accessreview_test.go index 89366d154d0..5a197076ab4 100644 --- a/test/integration/auth/accessreview_test.go +++ b/test/integration/auth/accessreview_test.go @@ -62,9 +62,9 @@ func TestSubjectAccessReview(t *testing.T) { clientset, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ ModifyServerConfig: func(config *controlplane.Config) { // Unset BearerToken to disable BearerToken authenticator. - config.GenericConfig.LoopbackClientConfig.BearerToken = "" - config.GenericConfig.Authentication.Authenticator = authenticator.RequestFunc(alwaysAlice) - config.GenericConfig.Authorization.Authorizer = sarAuthorizer{} + config.ControlPlane.Generic.LoopbackClientConfig.BearerToken = "" + config.ControlPlane.Generic.Authentication.Authenticator = authenticator.RequestFunc(alwaysAlice) + config.ControlPlane.Generic.Authorization.Authorizer = sarAuthorizer{} }, }) defer tearDownFn() @@ -172,9 +172,9 @@ func TestSelfSubjectAccessReview(t *testing.T) { clientset, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ ModifyServerConfig: func(config *controlplane.Config) { // Unset BearerToken to disable BearerToken authenticator. - config.GenericConfig.LoopbackClientConfig.BearerToken = "" - config.GenericConfig.Authentication.Authenticator = authenticator.RequestFunc(authenticatorFunc) - config.GenericConfig.Authorization.Authorizer = sarAuthorizer{} + config.ControlPlane.Generic.LoopbackClientConfig.BearerToken = "" + config.ControlPlane.Generic.Authentication.Authenticator = authenticator.RequestFunc(authenticatorFunc) + config.ControlPlane.Generic.Authorization.Authorizer = sarAuthorizer{} }, }) defer tearDownFn() @@ -256,9 +256,9 @@ func TestLocalSubjectAccessReview(t *testing.T) { clientset, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ ModifyServerConfig: func(config *controlplane.Config) { // Unset BearerToken to disable BearerToken authenticator. - config.GenericConfig.LoopbackClientConfig.BearerToken = "" - config.GenericConfig.Authentication.Authenticator = authenticator.RequestFunc(alwaysAlice) - config.GenericConfig.Authorization.Authorizer = sarAuthorizer{} + config.ControlPlane.Generic.LoopbackClientConfig.BearerToken = "" + config.ControlPlane.Generic.Authentication.Authenticator = authenticator.RequestFunc(alwaysAlice) + config.ControlPlane.Generic.Authorization.Authorizer = sarAuthorizer{} }, }) defer tearDownFn() diff --git a/test/integration/auth/auth_test.go b/test/integration/auth/auth_test.go index 818dc95176b..5569f11138f 100644 --- a/test/integration/auth/auth_test.go +++ b/test/integration/auth/auth_test.go @@ -813,7 +813,7 @@ func TestImpersonateIsForbidden(t *testing.T) { }, ModifyServerConfig: func(config *controlplane.Config) { // Prepend an impersonation authorizer with specific opinions about alice and bob - config.GenericConfig.Authorization.Authorizer = unionauthz.New(impersonateAuthorizer{}, config.GenericConfig.Authorization.Authorizer) + config.ControlPlane.Generic.Authorization.Authorizer = unionauthz.New(impersonateAuthorizer{}, config.ControlPlane.Generic.Authorization.Authorizer) }, }) defer tearDownFn() @@ -1118,7 +1118,7 @@ func TestAuthorizationAttributeDetermination(t *testing.T) { opts.Authentication.TokenFile.TokenFile = "testdata/tokens.csv" }, ModifyServerConfig: func(config *controlplane.Config) { - config.GenericConfig.Authorization.Authorizer = unionauthz.New(config.GenericConfig.Authorization.Authorizer, trackingAuthorizer) + config.ControlPlane.Generic.Authorization.Authorizer = unionauthz.New(config.ControlPlane.Generic.Authorization.Authorizer, trackingAuthorizer) }, }) defer tearDownFn() @@ -1458,9 +1458,9 @@ func testWebhookTokenAuthenticator(customDialer bool, t *testing.T) { opts.Authorization.PolicyFile = "testdata/allowalice.jsonl" }, ModifyServerConfig: func(config *controlplane.Config) { - config.GenericConfig.Authentication.Authenticator = group.NewAuthenticatedGroupAdder(authenticator) + config.ControlPlane.Generic.Authentication.Authenticator = group.NewAuthenticatedGroupAdder(authenticator) // Disable checking API audiences that is set by testserver by default. - config.GenericConfig.Authentication.APIAudiences = nil + config.ControlPlane.Generic.Authentication.APIAudiences = nil }, }) defer tearDownFn() diff --git a/test/integration/auth/bootstraptoken_test.go b/test/integration/auth/bootstraptoken_test.go index ef7914c575a..ff7521d9d1e 100644 --- a/test/integration/auth/bootstraptoken_test.go +++ b/test/integration/auth/bootstraptoken_test.go @@ -128,7 +128,7 @@ func TestBootstrapTokenAuth(t *testing.T) { opts.Authorization.Modes = []string{"AlwaysAllow"} }, ModifyServerConfig: func(config *controlplane.Config) { - config.GenericConfig.Authentication.Authenticator = authenticator + config.ControlPlane.Generic.Authentication.Authenticator = authenticator }, }) defer tearDownFn() diff --git a/test/integration/auth/rbac_test.go b/test/integration/auth/rbac_test.go index 7249ef942f8..f162eb6df2d 100644 --- a/test/integration/auth/rbac_test.go +++ b/test/integration/auth/rbac_test.go @@ -81,7 +81,7 @@ type testRESTOptionsGetter struct { } func (getter *testRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) { - storageConfig, err := getter.config.ExtraConfig.StorageFactory.NewConfig(resource) + storageConfig, err := getter.config.ControlPlane.Extra.StorageFactory.NewConfig(resource) if err != nil { return generic.RESTOptions{}, fmt.Errorf("failed to get storage: %v", err) } @@ -556,11 +556,11 @@ func TestRBAC(t *testing.T) { }, ModifyServerConfig: func(config *controlplane.Config) { // Append our custom test authenticator - config.GenericConfig.Authentication.Authenticator = unionauthn.New(config.GenericConfig.Authentication.Authenticator, authenticator) + config.ControlPlane.Generic.Authentication.Authenticator = unionauthn.New(config.ControlPlane.Generic.Authentication.Authenticator, authenticator) // Append our custom test authorizer var rbacAuthz authorizer.Authorizer rbacAuthz, tearDownAuthorizerFn = newRBACAuthorizer(t, config) - config.GenericConfig.Authorization.Authorizer = unionauthz.New(config.GenericConfig.Authorization.Authorizer, rbacAuthz) + config.ControlPlane.Generic.Authorization.Authorizer = unionauthz.New(config.ControlPlane.Generic.Authorization.Authorizer, rbacAuthz) }, }) defer tearDownFn() diff --git a/test/integration/auth/selfsubjectreview_test.go b/test/integration/auth/selfsubjectreview_test.go index 3677c0d92f5..de817c6bb11 100644 --- a/test/integration/auth/selfsubjectreview_test.go +++ b/test/integration/auth/selfsubjectreview_test.go @@ -101,8 +101,8 @@ func TestGetsSelfAttributes(t *testing.T) { }, ModifyServerConfig: func(config *controlplane.Config) { // Unset BearerToken to disable BearerToken authenticator. - config.GenericConfig.LoopbackClientConfig.BearerToken = "" - config.GenericConfig.Authentication.Authenticator = authenticator.RequestFunc(func(req *http.Request) (*authenticator.Response, bool, error) { + config.ControlPlane.Generic.LoopbackClientConfig.BearerToken = "" + config.ControlPlane.Generic.Authentication.Authenticator = authenticator.RequestFunc(func(req *http.Request) (*authenticator.Response, bool, error) { respMu.RLock() defer respMu.RUnlock() return &authenticator.Response{User: response}, true, nil @@ -215,8 +215,8 @@ func TestGetsSelfAttributesError(t *testing.T) { }, ModifyServerConfig: func(config *controlplane.Config) { // Unset BearerToken to disable BearerToken authenticator. - config.GenericConfig.LoopbackClientConfig.BearerToken = "" - config.GenericConfig.Authentication.Authenticator = authenticator.RequestFunc(func(req *http.Request) (*authenticator.Response, bool, error) { + config.ControlPlane.Generic.LoopbackClientConfig.BearerToken = "" + config.ControlPlane.Generic.Authentication.Authenticator = authenticator.RequestFunc(func(req *http.Request) (*authenticator.Response, bool, error) { if toggle.Load().(bool) { return &authenticator.Response{ User: &user.DefaultInfo{ diff --git a/test/integration/auth/svcaccttoken_test.go b/test/integration/auth/svcaccttoken_test.go index e4ec51e2fcb..c88e35a144e 100644 --- a/test/integration/auth/svcaccttoken_test.go +++ b/test/integration/auth/svcaccttoken_test.go @@ -104,10 +104,10 @@ func TestServiceAccountTokenCreate(t *testing.T) { }, ModifyServerConfig: func(config *controlplane.Config) { // extract token generator - tokenGenerator = config.ExtraConfig.ServiceAccountIssuer + tokenGenerator = config.ControlPlane.Extra.ServiceAccountIssuer - config.ExtraConfig.ServiceAccountMaxExpiration = maxExpirationDuration - config.ExtraConfig.ExtendExpiration = true + config.ControlPlane.Extra.ServiceAccountMaxExpiration = maxExpirationDuration + config.ControlPlane.Extra.ExtendExpiration = true }, }) defer tearDownFn() diff --git a/test/integration/daemonset/daemonset_test.go b/test/integration/daemonset/daemonset_test.go index 0cc6a1b097d..b42050410a3 100644 --- a/test/integration/daemonset/daemonset_test.go +++ b/test/integration/daemonset/daemonset_test.go @@ -1178,7 +1178,7 @@ func TestUpdateStatusDespitePodCreationFailure(t *testing.T) { limitedPodNumber := 2 ctx, closeFn, dc, informers, clientset := setupWithServerSetup(t, framework.TestServerSetup{ ModifyServerConfig: func(config *controlplane.Config) { - config.GenericConfig.AdmissionControl = &fakePodFailAdmission{ + config.ControlPlane.Generic.AdmissionControl = &fakePodFailAdmission{ limitedPodNumber: limitedPodNumber, } }, diff --git a/test/integration/defaulttolerationseconds/defaulttolerationseconds_test.go b/test/integration/defaulttolerationseconds/defaulttolerationseconds_test.go index 4a293a26184..2b88cec8672 100644 --- a/test/integration/defaulttolerationseconds/defaulttolerationseconds_test.go +++ b/test/integration/defaulttolerationseconds/defaulttolerationseconds_test.go @@ -32,8 +32,8 @@ func TestAdmission(t *testing.T) { tCtx := ktesting.Init(t) client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ ModifyServerConfig: func(cfg *controlplane.Config) { - cfg.GenericConfig.EnableProfiling = true - cfg.GenericConfig.AdmissionControl = defaulttolerationseconds.NewDefaultTolerationSeconds() + cfg.ControlPlane.Generic.EnableProfiling = true + cfg.ControlPlane.Generic.AdmissionControl = defaulttolerationseconds.NewDefaultTolerationSeconds() }, }) defer tearDownFn() diff --git a/test/integration/examples/webhook_test.go b/test/integration/examples/webhook_test.go index cb2e00fefba..e5fd7cee8e1 100644 --- a/test/integration/examples/webhook_test.go +++ b/test/integration/examples/webhook_test.go @@ -47,11 +47,11 @@ func TestWebhookLoopback(t *testing.T) { }, ModifyServerConfig: func(config *controlplane.Config) { // Avoid resolvable kubernetes service - config.ExtraConfig.EndpointReconcilerType = reconcilers.NoneEndpointReconcilerType + config.Extra.EndpointReconcilerType = reconcilers.NoneEndpointReconcilerType // Hook into audit to watch requests - config.GenericConfig.AuditBackend = auditSinkFunc(func(events ...*auditinternal.Event) {}) - config.GenericConfig.AuditPolicyRuleEvaluator = auditPolicyRuleEvaluator(func(attrs authorizer.Attributes) audit.RequestAuditConfig { + config.ControlPlane.Generic.AuditBackend = auditSinkFunc(func(events ...*auditinternal.Event) {}) + config.ControlPlane.Generic.AuditPolicyRuleEvaluator = auditPolicyRuleEvaluator(func(attrs authorizer.Attributes) audit.RequestAuditConfig { if attrs.GetPath() == webhookPath { if attrs.GetUser().GetName() != "system:apiserver" { t.Errorf("expected user %q, got %q", "system:apiserver", attrs.GetUser().GetName()) diff --git a/test/integration/framework/test_server.go b/test/integration/framework/test_server.go index f66b750b587..4913f1b53c9 100644 --- a/test/integration/framework/test_server.go +++ b/test/integration/framework/test_server.go @@ -182,7 +182,7 @@ func StartTestServer(ctx context.Context, t testing.TB, setup TestServerSetup) ( }() // Adjust the loopback config for external use (external server name and CA) - kubeAPIServerClientConfig := rest.CopyConfig(kubeAPIServerConfig.GenericConfig.LoopbackClientConfig) + kubeAPIServerClientConfig := rest.CopyConfig(kubeAPIServerConfig.ControlPlane.Generic.LoopbackClientConfig) kubeAPIServerClientConfig.CAFile = path.Join(certDir, "apiserver.crt") kubeAPIServerClientConfig.CAData = nil kubeAPIServerClientConfig.ServerName = "" diff --git a/test/integration/network/services_test.go b/test/integration/network/services_test.go index f01f051871f..076483cb3f8 100644 --- a/test/integration/network/services_test.go +++ b/test/integration/network/services_test.go @@ -50,7 +50,7 @@ func TestServicesFinalizersRepairLoop(t *testing.T) { opts.ServiceClusterIPRanges = serviceCIDR }, ModifyServerConfig: func(cfg *controlplane.Config) { - cfg.ExtraConfig.RepairServicesInterval = interval + cfg.Extra.RepairServicesInterval = interval }, }) defer tearDownFn() diff --git a/test/integration/openshift/openshift_test.go b/test/integration/openshift/openshift_test.go index 4a6c9959155..d90d0f16383 100644 --- a/test/integration/openshift/openshift_test.go +++ b/test/integration/openshift/openshift_test.go @@ -21,17 +21,20 @@ import ( genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/kubernetes/pkg/controlplane" + controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver" ) // This test references methods that OpenShift uses to customize the apiserver on startup, that // are not referenced directly by an instance. func TestApiserverExportsSymbols(t *testing.T) { _ = &controlplane.Config{ - GenericConfig: &genericapiserver.Config{ - EnableMetrics: true, - }, - ExtraConfig: controlplane.ExtraConfig{ - EnableLogsSupport: false, + ControlPlane: controlplaneapiserver.Config{ + Generic: &genericapiserver.Config{ + EnableMetrics: true, + }, + Extra: controlplaneapiserver.Extra{ + EnableLogsSupport: false, + }, }, } _ = &controlplane.Instance{ diff --git a/test/integration/serviceaccount/service_account_test.go b/test/integration/serviceaccount/service_account_test.go index 81511d56dd4..4c8656fb952 100644 --- a/test/integration/serviceaccount/service_account_test.go +++ b/test/integration/serviceaccount/service_account_test.go @@ -376,7 +376,7 @@ func startServiceAccountTestServerAndWaitForCaches(ctx context.Context, t *testi return authorizer.DecisionNoOpinion, fmt.Sprintf("User %s is denied (ns=%s, readonly=%v, resource=%s)", username, ns, attrs.IsReadOnly(), attrs.GetResource()), nil }) - config.GenericConfig.Authorization.Authorizer = unionauthz.New(config.GenericConfig.Authorization.Authorizer, authorizer) + config.ControlPlane.Generic.Authorization.Authorizer = unionauthz.New(config.ControlPlane.Generic.Authorization.Authorizer, authorizer) }, }) diff --git a/test/integration/statefulset/statefulset_test.go b/test/integration/statefulset/statefulset_test.go index d3d6f8dab91..f10e7a12722 100644 --- a/test/integration/statefulset/statefulset_test.go +++ b/test/integration/statefulset/statefulset_test.go @@ -381,7 +381,7 @@ func TestStatefulSetStatusWithPodFail(t *testing.T) { limitedPodNumber := 2 c, config, closeFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ ModifyServerConfig: func(config *controlplane.Config) { - config.GenericConfig.AdmissionControl = &fakePodFailAdmission{ + config.ControlPlane.Generic.AdmissionControl = &fakePodFailAdmission{ limitedPodNumber: limitedPodNumber, } }, diff --git a/test/integration/util/util.go b/test/integration/util/util.go index a945e11cb39..5cab9a144ef 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -518,7 +518,7 @@ func InitTestAPIServer(t *testing.T, nsPrefix string, admission admission.Interf }, ModifyServerConfig: func(config *controlplane.Config) { if admission != nil { - config.GenericConfig.AdmissionControl = admission + config.ControlPlane.Generic.AdmissionControl = admission } }, })