cmd/kube-apiserver: move options completion into options package

cmd/kube-apiserver/app/config.go

@@ -20,12 +20,13 @@ import (
 	apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
 	"k8s.io/apiserver/pkg/util/webhook"
 	aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
+	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 	"k8s.io/kubernetes/pkg/controlplane"
 	"k8s.io/kubernetes/pkg/controlplane/apiserver"
 )
 
 type Config struct {
-	Options completedServerRunOptions
+	Options options.CompletedOptions
 
 	Aggregator    *aggregatorapiserver.Config
 	ControlPlane  *controlplane.Config
@@ -38,7 +39,7 @@ type ExtraConfig struct {
 }
 
 type completedConfig struct {
-	Options completedServerRunOptions
+	Options options.CompletedOptions
 
 	Aggregator    aggregatorapiserver.CompletedConfig
 	ControlPlane  controlplane.CompletedConfig
@@ -65,7 +66,7 @@ func (c *Config) Complete() (CompletedConfig, error) {
 }
 
 // NewConfig creates all the resources for running kube-apiserver, but runs none of them.
-func NewConfig(opts completedServerRunOptions) (*Config, error) {
+func NewConfig(opts options.CompletedOptions) (*Config, error) {
 	c := &Config{
 		Options: opts,
 	}

cmd/kube-apiserver/app/options/completion.go

@@ -0,0 +1,199 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"strings"
+	"time"
+
+	serveroptions "k8s.io/apiserver/pkg/server/options"
+	"k8s.io/client-go/util/keyutil"
+	_ "k8s.io/component-base/metrics/prometheus/workqueue"
+	"k8s.io/klog/v2"
+	netutils "k8s.io/utils/net"
+
+	"k8s.io/kubernetes/pkg/controlplane"
+	"k8s.io/kubernetes/pkg/kubeapiserver"
+	kubeauthenticator "k8s.io/kubernetes/pkg/kubeapiserver/authenticator"
+	"k8s.io/kubernetes/pkg/serviceaccount"
+)
+
+// completedOptions is a private wrapper that enforces a call of Complete() before Run can be invoked.
+type completedOptions struct {
+	*ServerRunOptions
+}
+
+type CompletedOptions struct {
+	completedOptions
+}
+
+// Complete set default ServerRunOptions.
+// Should be called after kube-apiserver flags parsed.
+func Complete(opts *ServerRunOptions) (CompletedOptions, error) {
+	// set defaults
+	if err := opts.GenericServerRunOptions.DefaultAdvertiseAddress(opts.SecureServing.SecureServingOptions); err != nil {
+		return CompletedOptions{}, err
+	}
+
+	// process s.ServiceClusterIPRange from list to Primary and Secondary
+	// we process secondary only if provided by user
+	apiServerServiceIP, primaryServiceIPRange, secondaryServiceIPRange, err := getServiceIPAndRanges(opts.ServiceClusterIPRanges)
+	if err != nil {
+		return CompletedOptions{}, err
+	}
+	opts.PrimaryServiceClusterIPRange = primaryServiceIPRange
+	opts.SecondaryServiceClusterIPRange = secondaryServiceIPRange
+	opts.APIServerServiceIP = apiServerServiceIP
+
+	if err := opts.SecureServing.MaybeDefaultWithSelfSignedCerts(opts.GenericServerRunOptions.AdvertiseAddress.String(), []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP}); err != nil {
+		return CompletedOptions{}, fmt.Errorf("error creating self-signed certificates: %v", err)
+	}
+
+	if len(opts.GenericServerRunOptions.ExternalHost) == 0 {
+		if len(opts.GenericServerRunOptions.AdvertiseAddress) > 0 {
+			opts.GenericServerRunOptions.ExternalHost = opts.GenericServerRunOptions.AdvertiseAddress.String()
+		} else {
+			hostname, err := os.Hostname()
+			if err != nil {
+				return CompletedOptions{}, fmt.Errorf("error finding host name: %v", err)
+			}
+			opts.GenericServerRunOptions.ExternalHost = hostname
+		}
+		klog.Infof("external host was not specified, using %v", opts.GenericServerRunOptions.ExternalHost)
+	}
+
+	opts.Authentication.ApplyAuthorization(opts.Authorization)
+
+	// Use (ServiceAccountSigningKeyFile != "") as a proxy to the user enabling
+	// TokenRequest functionality. This defaulting was convenient, but messed up
+	// a lot of people when they rotated their serving cert with no idea it was
+	// connected to their service account keys. We are taking this opportunity to
+	// remove this problematic defaulting.
+	if opts.ServiceAccountSigningKeyFile == "" {
+		// Default to the private server key for service account token signing
+		if len(opts.Authentication.ServiceAccounts.KeyFiles) == 0 && opts.SecureServing.ServerCert.CertKey.KeyFile != "" {
+			if kubeauthenticator.IsValidServiceAccountKeyFile(opts.SecureServing.ServerCert.CertKey.KeyFile) {
+				opts.Authentication.ServiceAccounts.KeyFiles = []string{opts.SecureServing.ServerCert.CertKey.KeyFile}
+			} else {
+				klog.Warning("No TLS key provided, service account token authentication disabled")
+			}
+		}
+	}
+
+	if opts.ServiceAccountSigningKeyFile != "" && len(opts.Authentication.ServiceAccounts.Issuers) != 0 && opts.Authentication.ServiceAccounts.Issuers[0] != "" {
+		sk, err := keyutil.PrivateKeyFromFile(opts.ServiceAccountSigningKeyFile)
+		if err != nil {
+			return CompletedOptions{}, fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err)
+		}
+		if opts.Authentication.ServiceAccounts.MaxExpiration != 0 {
+			lowBound := time.Hour
+			upBound := time.Duration(1<<32) * time.Second
+			if opts.Authentication.ServiceAccounts.MaxExpiration < lowBound ||
+				opts.Authentication.ServiceAccounts.MaxExpiration > upBound {
+				return CompletedOptions{}, fmt.Errorf("the service-account-max-token-expiration must be between 1 hour and 2^32 seconds")
+			}
+			if opts.Authentication.ServiceAccounts.ExtendExpiration {
+				if opts.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.WarnOnlyBoundTokenExpirationSeconds*time.Second {
+					klog.Warningf("service-account-extend-token-expiration is true, in order to correctly trigger safe transition logic, service-account-max-token-expiration must be set longer than %d seconds (currently %s)", serviceaccount.WarnOnlyBoundTokenExpirationSeconds, opts.Authentication.ServiceAccounts.MaxExpiration)
+				}
+				if opts.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.ExpirationExtensionSeconds*time.Second {
+					klog.Warningf("service-account-extend-token-expiration is true, enabling tokens valid up to %d seconds, which is longer than service-account-max-token-expiration set to %s seconds", serviceaccount.ExpirationExtensionSeconds, opts.Authentication.ServiceAccounts.MaxExpiration)
+				}
+			}
+		}
+
+		opts.ServiceAccountIssuer, err = serviceaccount.JWTTokenGenerator(opts.Authentication.ServiceAccounts.Issuers[0], sk)
+		if err != nil {
+			return CompletedOptions{}, fmt.Errorf("failed to build token generator: %v", err)
+		}
+		opts.ServiceAccountTokenMaxExpiration = opts.Authentication.ServiceAccounts.MaxExpiration
+	}
+
+	if opts.Etcd.EnableWatchCache {
+		sizes := kubeapiserver.DefaultWatchCacheSizes()
+		// Ensure that overrides parse correctly.
+		userSpecified, err := serveroptions.ParseWatchCacheSizes(opts.Etcd.WatchCacheSizes)
+		if err != nil {
+			return CompletedOptions{}, err
+		}
+		for resource, size := range userSpecified {
+			sizes[resource] = size
+		}
+		opts.Etcd.WatchCacheSizes, err = serveroptions.WriteWatchCacheSizes(sizes)
+		if err != nil {
+			return CompletedOptions{}, err
+		}
+	}
+
+	for key, value := range opts.APIEnablement.RuntimeConfig {
+		if key == "v1" || strings.HasPrefix(key, "v1/") ||
+			key == "api/v1" || strings.HasPrefix(key, "api/v1/") {
+			delete(opts.APIEnablement.RuntimeConfig, key)
+			opts.APIEnablement.RuntimeConfig["/v1"] = value
+		}
+		if key == "api/legacy" {
+			delete(opts.APIEnablement.RuntimeConfig, key)
+		}
+	}
+
+	return CompletedOptions{completedOptions: completedOptions{ServerRunOptions: opts}}, nil
+}
+
+func getServiceIPAndRanges(serviceClusterIPRanges string) (net.IP, net.IPNet, net.IPNet, error) {
+	serviceClusterIPRangeList := []string{}
+	if serviceClusterIPRanges != "" {
+		serviceClusterIPRangeList = strings.Split(serviceClusterIPRanges, ",")
+	}
+
+	var apiServerServiceIP net.IP
+	var primaryServiceIPRange net.IPNet
+	var secondaryServiceIPRange net.IPNet
+	var err error
+	// nothing provided by user, use default range (only applies to the Primary)
+	if len(serviceClusterIPRangeList) == 0 {
+		var primaryServiceClusterCIDR net.IPNet
+		primaryServiceIPRange, apiServerServiceIP, err = controlplane.ServiceIPRange(primaryServiceClusterCIDR)
+		if err != nil {
+			return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("error determining service IP ranges: %v", err)
+		}
+		return apiServerServiceIP, primaryServiceIPRange, net.IPNet{}, nil
+	}
+
+	_, primaryServiceClusterCIDR, err := netutils.ParseCIDRSloppy(serviceClusterIPRangeList[0])
+	if err != nil {
+		return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("service-cluster-ip-range[0] is not a valid cidr")
+	}
+
+	primaryServiceIPRange, apiServerServiceIP, err = controlplane.ServiceIPRange(*primaryServiceClusterCIDR)
+	if err != nil {
+		return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("error determining service IP ranges for primary service cidr: %v", err)
+	}
+
+	// user provided at least two entries
+	// note: validation asserts that the list is max of two dual stack entries
+	if len(serviceClusterIPRangeList) > 1 {
+		_, secondaryServiceClusterCIDR, err := netutils.ParseCIDRSloppy(serviceClusterIPRangeList[1])
+		if err != nil {
+			return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("service-cluster-ip-range[1] is not an ip net")
+		}
+		secondaryServiceIPRange = *secondaryServiceClusterCIDR
+	}
+	return apiServerServiceIP, primaryServiceIPRange, secondaryServiceIPRange, nil
+}

cmd/kube-apiserver/app/options/completion_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package app
+package options
 
 import (
 	"testing"

cmd/kube-apiserver/app/server.go

@@ -22,12 +22,9 @@ package app
 import (
 	"crypto/tls"
 	"fmt"
-	"net"
 	"net/http"
 	"net/url"
 	"os"
-	"strings"
-	"time"
 
 	"github.com/spf13/cobra"
 
@@ -42,7 +39,6 @@ import (
 	genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/server/egressselector"
-	serveroptions "k8s.io/apiserver/pkg/server/options"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/apiserver/pkg/util/notfoundhandler"
 	"k8s.io/apiserver/pkg/util/webhook"
@@ -63,7 +59,6 @@ import (
 	"k8s.io/klog/v2"
 	aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
 	aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
-	netutils "k8s.io/utils/net"
 
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
@@ -72,9 +67,7 @@ import (
 	controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver"
 	"k8s.io/kubernetes/pkg/controlplane/reconcilers"
 	generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi"
-	"k8s.io/kubernetes/pkg/kubeapiserver"
 	kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission"
-	kubeauthenticator "k8s.io/kubernetes/pkg/kubeapiserver/authenticator"
 	"k8s.io/kubernetes/pkg/serviceaccount"
 )
 
@@ -112,7 +105,7 @@ cluster's shared state through which all other components interact.`,
 			cliflag.PrintFlags(fs)
 
 			// set default options
-			completedOptions, err := Complete(s)
+			completedOptions, err := options.Complete(s)
 			if err != nil {
 				return err
 			}
@@ -151,13 +144,13 @@ cluster's shared state through which all other components interact.`,
 }
 
 // Run runs the specified APIServer.  This should never exit.
-func Run(options completedServerRunOptions, stopCh <-chan struct{}) error {
+func Run(opts options.CompletedOptions, stopCh <-chan struct{}) error {
 	// To help debugging, immediately log version
 	klog.Infof("Version: %+v", version.Get())
 
 	klog.InfoS("Golang settings", "GOGC", os.Getenv("GOGC"), "GOMAXPROCS", os.Getenv("GOMAXPROCS"), "GOTRACEBACK", os.Getenv("GOTRACEBACK"))
 
-	config, err := NewConfig(options)
+	config, err := NewConfig(opts)
 	if err != nil {
 		return err
 	}
@@ -215,7 +208,7 @@ func CreateProxyTransport() *http.Transport {
 }
 
 // CreateKubeAPIServerConfig creates all the resources for running the API server, but runs none of them
-func CreateKubeAPIServerConfig(s completedServerRunOptions) (
+func CreateKubeAPIServerConfig(opts options.CompletedOptions) (
 	*controlplane.Config,
 	aggregatorapiserver.ServiceResolver,
 	[]admission.PluginInitializer,
@@ -224,7 +217,7 @@ func CreateKubeAPIServerConfig(s completedServerRunOptions) (
 	proxyTransport := CreateProxyTransport()
 
 	genericConfig, versionedInformers, storageFactory, err := controlplaneapiserver.BuildGenericConfig(
-		s.ServerRunOptions,
+		opts.ServerRunOptions,
 		[]*runtime.Scheme{legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme},
 		generatedopenapi.GetOpenAPIDefinitions,
 	)
@@ -232,9 +225,9 @@ func CreateKubeAPIServerConfig(s completedServerRunOptions) (
 		return nil, nil, nil, err
 	}
 
-	capabilities.Setup(s.AllowPrivileged, s.MaxConnectionBytesPerSec)
+	capabilities.Setup(opts.AllowPrivileged, opts.MaxConnectionBytesPerSec)
 
-	s.Metrics.Apply()
+	opts.Metrics.Apply()
 	serviceaccount.RegisterMetrics()
 
 	config := &controlplane.Config{
@@ -242,38 +235,38 @@ func CreateKubeAPIServerConfig(s completedServerRunOptions) (
 		ExtraConfig: controlplane.ExtraConfig{
 			APIResourceConfigSource: storageFactory.APIResourceConfigSource,
 			StorageFactory:          storageFactory,
-			EventTTL:                s.EventTTL,
+			EventTTL:                opts.EventTTL,
-			KubeletClientConfig:     s.KubeletConfig,
+			KubeletClientConfig:     opts.KubeletConfig,
-			EnableLogsSupport:       s.EnableLogsHandler,
+			EnableLogsSupport:       opts.EnableLogsHandler,
 			ProxyTransport:          proxyTransport,
 
-			ServiceIPRange:          s.PrimaryServiceClusterIPRange,
+			ServiceIPRange:          opts.PrimaryServiceClusterIPRange,
-			APIServerServiceIP:      s.APIServerServiceIP,
+			APIServerServiceIP:      opts.APIServerServiceIP,
-			SecondaryServiceIPRange: s.SecondaryServiceClusterIPRange,
+			SecondaryServiceIPRange: opts.SecondaryServiceClusterIPRange,
 
 			APIServerServicePort: 443,
 
-			ServiceNodePortRange:      s.ServiceNodePortRange,
+			ServiceNodePortRange:      opts.ServiceNodePortRange,
-			KubernetesServiceNodePort: s.KubernetesServiceNodePort,
+			KubernetesServiceNodePort: opts.KubernetesServiceNodePort,
 
-			EndpointReconcilerType: reconcilers.Type(s.EndpointReconcilerType),
+			EndpointReconcilerType: reconcilers.Type(opts.EndpointReconcilerType),
-			MasterCount:            s.MasterCount,
+			MasterCount:            opts.MasterCount,
 
-			ServiceAccountIssuer:        s.ServiceAccountIssuer,
+			ServiceAccountIssuer:        opts.ServiceAccountIssuer,
-			ServiceAccountMaxExpiration: s.ServiceAccountTokenMaxExpiration,
+			ServiceAccountMaxExpiration: opts.ServiceAccountTokenMaxExpiration,
-			ExtendExpiration:            s.Authentication.ServiceAccounts.ExtendExpiration,
+			ExtendExpiration:            opts.Authentication.ServiceAccounts.ExtendExpiration,
 
 			VersionedInformers: versionedInformers,
 		},
 	}
 
-	clientCAProvider, err := s.Authentication.ClientCert.GetClientCAContentProvider()
+	clientCAProvider, err := opts.Authentication.ClientCert.GetClientCAContentProvider()
 	if err != nil {
 		return nil, nil, nil, err
 	}
 	config.ExtraConfig.ClusterAuthenticationInfo.ClientCA = clientCAProvider
 
-	requestHeaderConfig, err := s.Authentication.RequestHeader.ToAuthenticationRequestHeaderConfig()
+	requestHeaderConfig, err := opts.Authentication.RequestHeader.ToAuthenticationRequestHeaderConfig()
 	if err != nil {
 		return nil, nil, nil, err
 	}
@@ -289,9 +282,9 @@ func CreateKubeAPIServerConfig(s completedServerRunOptions) (
 	admissionConfig := &kubeapiserveradmission.Config{
 		ExternalInformers:    versionedInformers,
 		LoopbackClientConfig: genericConfig.LoopbackClientConfig,
-		CloudConfigFile:      s.CloudProvider.CloudConfigFile,
+		CloudConfigFile:      opts.CloudProvider.CloudConfigFile,
 	}
-	serviceResolver := buildServiceResolver(s.EnableAggregatorRouting, genericConfig.LoopbackClientConfig.Host, versionedInformers)
+	serviceResolver := buildServiceResolver(opts.EnableAggregatorRouting, genericConfig.LoopbackClientConfig.Host, versionedInformers)
 	schemaResolver := resolver.NewDefinitionsSchemaResolver(k8sscheme.Scheme, genericConfig.OpenAPIConfig.GetDefinitions)
 	pluginInitializers, admissionPostStartHook, err := admissionConfig.New(proxyTransport, genericConfig.EgressSelector, serviceResolver, genericConfig.TracerProvider, schemaResolver)
 	if err != nil {
@@ -305,7 +298,7 @@ func CreateKubeAPIServerConfig(s completedServerRunOptions) (
 	if err != nil {
 		return nil, nil, nil, fmt.Errorf("failed to create real dynamic external client: %w", err)
 	}
-	err = s.Admission.ApplyTo(
+	err = opts.Admission.ApplyTo(
 		genericConfig,
 		versionedInformers,
 		clientgoExternalClient,
@@ -336,139 +329,20 @@ func CreateKubeAPIServerConfig(s completedServerRunOptions) (
 
 	// Load and set the public keys.
 	var pubKeys []interface{}
-	for _, f := range s.Authentication.ServiceAccounts.KeyFiles {
+	for _, f := range opts.Authentication.ServiceAccounts.KeyFiles {
 		keys, err := keyutil.PublicKeysFromFile(f)
 		if err != nil {
 			return nil, nil, nil, fmt.Errorf("failed to parse key file %q: %v", f, err)
 		}
 		pubKeys = append(pubKeys, keys...)
 	}
-	config.ExtraConfig.ServiceAccountIssuerURL = s.Authentication.ServiceAccounts.Issuers[0]
+	config.ExtraConfig.ServiceAccountIssuerURL = opts.Authentication.ServiceAccounts.Issuers[0]
-	config.ExtraConfig.ServiceAccountJWKSURI = s.Authentication.ServiceAccounts.JWKSURI
+	config.ExtraConfig.ServiceAccountJWKSURI = opts.Authentication.ServiceAccounts.JWKSURI
 	config.ExtraConfig.ServiceAccountPublicKeys = pubKeys
 
 	return config, serviceResolver, pluginInitializers, nil
 }
 
-// completedServerRunOptions is a private wrapper that enforces a call of Complete() before Run can be invoked.
-type completedServerRunOptions struct {
-	*options.ServerRunOptions
-}
-
-// Complete set default ServerRunOptions.
-// Should be called after kube-apiserver flags parsed.
-func Complete(s *options.ServerRunOptions) (completedServerRunOptions, error) {
-	var options completedServerRunOptions
-	// set defaults
-	if err := s.GenericServerRunOptions.DefaultAdvertiseAddress(s.SecureServing.SecureServingOptions); err != nil {
-		return options, err
-	}
-
-	// process s.ServiceClusterIPRange from list to Primary and Secondary
-	// we process secondary only if provided by user
-	apiServerServiceIP, primaryServiceIPRange, secondaryServiceIPRange, err := getServiceIPAndRanges(s.ServiceClusterIPRanges)
-	if err != nil {
-		return options, err
-	}
-	s.PrimaryServiceClusterIPRange = primaryServiceIPRange
-	s.SecondaryServiceClusterIPRange = secondaryServiceIPRange
-	s.APIServerServiceIP = apiServerServiceIP
-
-	if err := s.SecureServing.MaybeDefaultWithSelfSignedCerts(s.GenericServerRunOptions.AdvertiseAddress.String(), []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP}); err != nil {
-		return options, fmt.Errorf("error creating self-signed certificates: %v", err)
-	}
-
-	if len(s.GenericServerRunOptions.ExternalHost) == 0 {
-		if len(s.GenericServerRunOptions.AdvertiseAddress) > 0 {
-			s.GenericServerRunOptions.ExternalHost = s.GenericServerRunOptions.AdvertiseAddress.String()
-		} else {
-			hostname, err := os.Hostname()
-			if err != nil {
-				return options, fmt.Errorf("error finding host name: %v", err)
-			}
-			s.GenericServerRunOptions.ExternalHost = hostname
-		}
-		klog.Infof("external host was not specified, using %v", s.GenericServerRunOptions.ExternalHost)
-	}
-
-	s.Authentication.ApplyAuthorization(s.Authorization)
-
-	// Use (ServiceAccountSigningKeyFile != "") as a proxy to the user enabling
-	// TokenRequest functionality. This defaulting was convenient, but messed up
-	// a lot of people when they rotated their serving cert with no idea it was
-	// connected to their service account keys. We are taking this opportunity to
-	// remove this problematic defaulting.
-	if s.ServiceAccountSigningKeyFile == "" {
-		// Default to the private server key for service account token signing
-		if len(s.Authentication.ServiceAccounts.KeyFiles) == 0 && s.SecureServing.ServerCert.CertKey.KeyFile != "" {
-			if kubeauthenticator.IsValidServiceAccountKeyFile(s.SecureServing.ServerCert.CertKey.KeyFile) {
-				s.Authentication.ServiceAccounts.KeyFiles = []string{s.SecureServing.ServerCert.CertKey.KeyFile}
-			} else {
-				klog.Warning("No TLS key provided, service account token authentication disabled")
-			}
-		}
-	}
-
-	if s.ServiceAccountSigningKeyFile != "" && len(s.Authentication.ServiceAccounts.Issuers) != 0 && s.Authentication.ServiceAccounts.Issuers[0] != "" {
-		sk, err := keyutil.PrivateKeyFromFile(s.ServiceAccountSigningKeyFile)
-		if err != nil {
-			return options, fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err)
-		}
-		if s.Authentication.ServiceAccounts.MaxExpiration != 0 {
-			lowBound := time.Hour
-			upBound := time.Duration(1<<32) * time.Second
-			if s.Authentication.ServiceAccounts.MaxExpiration < lowBound ||
-				s.Authentication.ServiceAccounts.MaxExpiration > upBound {
-				return options, fmt.Errorf("the service-account-max-token-expiration must be between 1 hour and 2^32 seconds")
-			}
-			if s.Authentication.ServiceAccounts.ExtendExpiration {
-				if s.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.WarnOnlyBoundTokenExpirationSeconds*time.Second {
-					klog.Warningf("service-account-extend-token-expiration is true, in order to correctly trigger safe transition logic, service-account-max-token-expiration must be set longer than %d seconds (currently %s)", serviceaccount.WarnOnlyBoundTokenExpirationSeconds, s.Authentication.ServiceAccounts.MaxExpiration)
-				}
-				if s.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.ExpirationExtensionSeconds*time.Second {
-					klog.Warningf("service-account-extend-token-expiration is true, enabling tokens valid up to %d seconds, which is longer than service-account-max-token-expiration set to %s seconds", serviceaccount.ExpirationExtensionSeconds, s.Authentication.ServiceAccounts.MaxExpiration)
-				}
-			}
-		}
-
-		s.ServiceAccountIssuer, err = serviceaccount.JWTTokenGenerator(s.Authentication.ServiceAccounts.Issuers[0], sk)
-		if err != nil {
-			return options, fmt.Errorf("failed to build token generator: %v", err)
-		}
-		s.ServiceAccountTokenMaxExpiration = s.Authentication.ServiceAccounts.MaxExpiration
-	}
-
-	if s.Etcd.EnableWatchCache {
-		sizes := kubeapiserver.DefaultWatchCacheSizes()
-		// Ensure that overrides parse correctly.
-		userSpecified, err := serveroptions.ParseWatchCacheSizes(s.Etcd.WatchCacheSizes)
-		if err != nil {
-			return options, err
-		}
-		for resource, size := range userSpecified {
-			sizes[resource] = size
-		}
-		s.Etcd.WatchCacheSizes, err = serveroptions.WriteWatchCacheSizes(sizes)
-		if err != nil {
-			return options, err
-		}
-	}
-
-	for key, value := range s.APIEnablement.RuntimeConfig {
-		if key == "v1" || strings.HasPrefix(key, "v1/") ||
-			key == "api/v1" || strings.HasPrefix(key, "api/v1/") {
-			delete(s.APIEnablement.RuntimeConfig, key)
-			s.APIEnablement.RuntimeConfig["/v1"] = value
-		}
-		if key == "api/legacy" {
-			delete(s.APIEnablement.RuntimeConfig, key)
-		}
-	}
-
-	options.ServerRunOptions = s
-	return options, nil
-}
-
 var testServiceResolver webhook.ServiceResolver
 
 // SetServiceResolverForTests allows the service resolver to be overridden during tests.
@@ -508,45 +382,3 @@ func buildServiceResolver(enabledAggregatorRouting bool, hostname string, inform
 	}
 	return serviceResolver
 }
-
-func getServiceIPAndRanges(serviceClusterIPRanges string) (net.IP, net.IPNet, net.IPNet, error) {
-	serviceClusterIPRangeList := []string{}
-	if serviceClusterIPRanges != "" {
-		serviceClusterIPRangeList = strings.Split(serviceClusterIPRanges, ",")
-	}
-
-	var apiServerServiceIP net.IP
-	var primaryServiceIPRange net.IPNet
-	var secondaryServiceIPRange net.IPNet
-	var err error
-	// nothing provided by user, use default range (only applies to the Primary)
-	if len(serviceClusterIPRangeList) == 0 {
-		var primaryServiceClusterCIDR net.IPNet
-		primaryServiceIPRange, apiServerServiceIP, err = controlplane.ServiceIPRange(primaryServiceClusterCIDR)
-		if err != nil {
-			return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("error determining service IP ranges: %v", err)
-		}
-		return apiServerServiceIP, primaryServiceIPRange, net.IPNet{}, nil
-	}
-
-	_, primaryServiceClusterCIDR, err := netutils.ParseCIDRSloppy(serviceClusterIPRangeList[0])
-	if err != nil {
-		return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("service-cluster-ip-range[0] is not a valid cidr")
-	}
-
-	primaryServiceIPRange, apiServerServiceIP, err = controlplane.ServiceIPRange(*primaryServiceClusterCIDR)
-	if err != nil {
-		return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("error determining service IP ranges for primary service cidr: %v", err)
-	}
-
-	// user provided at least two entries
-	// note: validation asserts that the list is max of two dual stack entries
-	if len(serviceClusterIPRangeList) > 1 {
-		_, secondaryServiceClusterCIDR, err := netutils.ParseCIDRSloppy(serviceClusterIPRangeList[1])
-		if err != nil {
-			return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("service-cluster-ip-range[1] is not an ip net")
-		}
-		secondaryServiceIPRange = *secondaryServiceClusterCIDR
-	}
-	return apiServerServiceIP, primaryServiceIPRange, secondaryServiceIPRange, nil
-}

cmd/kube-apiserver/app/testing/testserver.go

@@ -44,6 +44,7 @@ import (
 	"k8s.io/client-go/util/cert"
 	"k8s.io/klog/v2"
 	"k8s.io/kube-aggregator/pkg/apiserver"
+
 	"k8s.io/kubernetes/cmd/kube-apiserver/app"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil"
@@ -235,7 +236,7 @@ func StartTestServer(t Logger, instanceOptions *TestServerInstanceOptions, custo
 	s.Authentication.ServiceAccounts.Issuers = []string{"https://foo.bar.example.com"}
 	s.Authentication.ServiceAccounts.KeyFiles = []string{saSigningKeyFile.Name()}
 
-	completedOptions, err := app.Complete(s)
+	completedOptions, err := options.Complete(s)
 	if err != nil {
 		return result, fmt.Errorf("failed to set default ServerRunOptions: %v", err)
 	}

test/e2e_node/services/apiserver.go

@@ -96,7 +96,7 @@ func (a *APIServer) Start() error {
 	errCh := make(chan error)
 	go func() {
 		defer close(errCh)
-		completedOptions, err := apiserver.Complete(o)
+		completedOptions, err := options.Complete(o)
 		if err != nil {
 			errCh <- fmt.Errorf("set apiserver default options error: %w", err)
 			return

test/integration/etcd/server.go

@@ -104,7 +104,7 @@ func StartRealAPIServerOrDie(t *testing.T, configFuncs ...func(*options.ServerRu
 	for _, f := range configFuncs {
 		f(kubeAPIServerOptions)
 	}
-	completedOptions, err := app.Complete(kubeAPIServerOptions)
+	completedOptions, err := options.Complete(kubeAPIServerOptions)
 	if err != nil {
 		t.Fatal(err)
 	}

test/integration/framework/test_server.go

@@ -150,7 +150,7 @@ func StartTestServer(ctx context.Context, t testing.TB, setup TestServerSetup) (
 		setup.ModifyServerRunOptions(kubeAPIServerOptions)
 	}
 
-	completedOptions, err := app.Complete(kubeAPIServerOptions)
+	completedOptions, err := options.Complete(kubeAPIServerOptions)
 	if err != nil {
 		t.Fatal(err)
 	}