diff --git a/cmd/kube-apiserver/app/config.go b/cmd/kube-apiserver/app/config.go index b305f503af7..651ebd5ae78 100644 --- a/cmd/kube-apiserver/app/config.go +++ b/cmd/kube-apiserver/app/config.go @@ -20,12 +20,13 @@ import ( apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" "k8s.io/apiserver/pkg/util/webhook" aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" + "k8s.io/kubernetes/cmd/kube-apiserver/app/options" "k8s.io/kubernetes/pkg/controlplane" "k8s.io/kubernetes/pkg/controlplane/apiserver" ) type Config struct { - Options completedServerRunOptions + Options options.CompletedOptions Aggregator *aggregatorapiserver.Config ControlPlane *controlplane.Config @@ -38,7 +39,7 @@ type ExtraConfig struct { } type completedConfig struct { - Options completedServerRunOptions + Options options.CompletedOptions Aggregator aggregatorapiserver.CompletedConfig ControlPlane controlplane.CompletedConfig @@ -65,7 +66,7 @@ func (c *Config) Complete() (CompletedConfig, error) { } // NewConfig creates all the resources for running kube-apiserver, but runs none of them. -func NewConfig(opts completedServerRunOptions) (*Config, error) { +func NewConfig(opts options.CompletedOptions) (*Config, error) { c := &Config{ Options: opts, } diff --git a/cmd/kube-apiserver/app/options/completion.go b/cmd/kube-apiserver/app/options/completion.go new file mode 100644 index 00000000000..5cb9d589e19 --- /dev/null +++ b/cmd/kube-apiserver/app/options/completion.go @@ -0,0 +1,199 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "fmt" + "net" + "os" + "strings" + "time" + + serveroptions "k8s.io/apiserver/pkg/server/options" + "k8s.io/client-go/util/keyutil" + _ "k8s.io/component-base/metrics/prometheus/workqueue" + "k8s.io/klog/v2" + netutils "k8s.io/utils/net" + + "k8s.io/kubernetes/pkg/controlplane" + "k8s.io/kubernetes/pkg/kubeapiserver" + kubeauthenticator "k8s.io/kubernetes/pkg/kubeapiserver/authenticator" + "k8s.io/kubernetes/pkg/serviceaccount" +) + +// completedOptions is a private wrapper that enforces a call of Complete() before Run can be invoked. +type completedOptions struct { + *ServerRunOptions +} + +type CompletedOptions struct { + completedOptions +} + +// Complete set default ServerRunOptions. +// Should be called after kube-apiserver flags parsed. +func Complete(opts *ServerRunOptions) (CompletedOptions, error) { + // set defaults + if err := opts.GenericServerRunOptions.DefaultAdvertiseAddress(opts.SecureServing.SecureServingOptions); err != nil { + return CompletedOptions{}, err + } + + // process s.ServiceClusterIPRange from list to Primary and Secondary + // we process secondary only if provided by user + apiServerServiceIP, primaryServiceIPRange, secondaryServiceIPRange, err := getServiceIPAndRanges(opts.ServiceClusterIPRanges) + if err != nil { + return CompletedOptions{}, err + } + opts.PrimaryServiceClusterIPRange = primaryServiceIPRange + opts.SecondaryServiceClusterIPRange = secondaryServiceIPRange + opts.APIServerServiceIP = apiServerServiceIP + + if err := opts.SecureServing.MaybeDefaultWithSelfSignedCerts(opts.GenericServerRunOptions.AdvertiseAddress.String(), []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP}); err != nil { + return CompletedOptions{}, fmt.Errorf("error creating self-signed certificates: %v", err) + } + + if len(opts.GenericServerRunOptions.ExternalHost) == 0 { + if len(opts.GenericServerRunOptions.AdvertiseAddress) > 0 { + opts.GenericServerRunOptions.ExternalHost = opts.GenericServerRunOptions.AdvertiseAddress.String() + } else { + hostname, err := os.Hostname() + if err != nil { + return CompletedOptions{}, fmt.Errorf("error finding host name: %v", err) + } + opts.GenericServerRunOptions.ExternalHost = hostname + } + klog.Infof("external host was not specified, using %v", opts.GenericServerRunOptions.ExternalHost) + } + + opts.Authentication.ApplyAuthorization(opts.Authorization) + + // Use (ServiceAccountSigningKeyFile != "") as a proxy to the user enabling + // TokenRequest functionality. This defaulting was convenient, but messed up + // a lot of people when they rotated their serving cert with no idea it was + // connected to their service account keys. We are taking this opportunity to + // remove this problematic defaulting. + if opts.ServiceAccountSigningKeyFile == "" { + // Default to the private server key for service account token signing + if len(opts.Authentication.ServiceAccounts.KeyFiles) == 0 && opts.SecureServing.ServerCert.CertKey.KeyFile != "" { + if kubeauthenticator.IsValidServiceAccountKeyFile(opts.SecureServing.ServerCert.CertKey.KeyFile) { + opts.Authentication.ServiceAccounts.KeyFiles = []string{opts.SecureServing.ServerCert.CertKey.KeyFile} + } else { + klog.Warning("No TLS key provided, service account token authentication disabled") + } + } + } + + if opts.ServiceAccountSigningKeyFile != "" && len(opts.Authentication.ServiceAccounts.Issuers) != 0 && opts.Authentication.ServiceAccounts.Issuers[0] != "" { + sk, err := keyutil.PrivateKeyFromFile(opts.ServiceAccountSigningKeyFile) + if err != nil { + return CompletedOptions{}, fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err) + } + if opts.Authentication.ServiceAccounts.MaxExpiration != 0 { + lowBound := time.Hour + upBound := time.Duration(1<<32) * time.Second + if opts.Authentication.ServiceAccounts.MaxExpiration < lowBound || + opts.Authentication.ServiceAccounts.MaxExpiration > upBound { + return CompletedOptions{}, fmt.Errorf("the service-account-max-token-expiration must be between 1 hour and 2^32 seconds") + } + if opts.Authentication.ServiceAccounts.ExtendExpiration { + if opts.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.WarnOnlyBoundTokenExpirationSeconds*time.Second { + klog.Warningf("service-account-extend-token-expiration is true, in order to correctly trigger safe transition logic, service-account-max-token-expiration must be set longer than %d seconds (currently %s)", serviceaccount.WarnOnlyBoundTokenExpirationSeconds, opts.Authentication.ServiceAccounts.MaxExpiration) + } + if opts.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.ExpirationExtensionSeconds*time.Second { + klog.Warningf("service-account-extend-token-expiration is true, enabling tokens valid up to %d seconds, which is longer than service-account-max-token-expiration set to %s seconds", serviceaccount.ExpirationExtensionSeconds, opts.Authentication.ServiceAccounts.MaxExpiration) + } + } + } + + opts.ServiceAccountIssuer, err = serviceaccount.JWTTokenGenerator(opts.Authentication.ServiceAccounts.Issuers[0], sk) + if err != nil { + return CompletedOptions{}, fmt.Errorf("failed to build token generator: %v", err) + } + opts.ServiceAccountTokenMaxExpiration = opts.Authentication.ServiceAccounts.MaxExpiration + } + + if opts.Etcd.EnableWatchCache { + sizes := kubeapiserver.DefaultWatchCacheSizes() + // Ensure that overrides parse correctly. + userSpecified, err := serveroptions.ParseWatchCacheSizes(opts.Etcd.WatchCacheSizes) + if err != nil { + return CompletedOptions{}, err + } + for resource, size := range userSpecified { + sizes[resource] = size + } + opts.Etcd.WatchCacheSizes, err = serveroptions.WriteWatchCacheSizes(sizes) + if err != nil { + return CompletedOptions{}, err + } + } + + for key, value := range opts.APIEnablement.RuntimeConfig { + if key == "v1" || strings.HasPrefix(key, "v1/") || + key == "api/v1" || strings.HasPrefix(key, "api/v1/") { + delete(opts.APIEnablement.RuntimeConfig, key) + opts.APIEnablement.RuntimeConfig["/v1"] = value + } + if key == "api/legacy" { + delete(opts.APIEnablement.RuntimeConfig, key) + } + } + + return CompletedOptions{completedOptions: completedOptions{ServerRunOptions: opts}}, nil +} + +func getServiceIPAndRanges(serviceClusterIPRanges string) (net.IP, net.IPNet, net.IPNet, error) { + serviceClusterIPRangeList := []string{} + if serviceClusterIPRanges != "" { + serviceClusterIPRangeList = strings.Split(serviceClusterIPRanges, ",") + } + + var apiServerServiceIP net.IP + var primaryServiceIPRange net.IPNet + var secondaryServiceIPRange net.IPNet + var err error + // nothing provided by user, use default range (only applies to the Primary) + if len(serviceClusterIPRangeList) == 0 { + var primaryServiceClusterCIDR net.IPNet + primaryServiceIPRange, apiServerServiceIP, err = controlplane.ServiceIPRange(primaryServiceClusterCIDR) + if err != nil { + return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("error determining service IP ranges: %v", err) + } + return apiServerServiceIP, primaryServiceIPRange, net.IPNet{}, nil + } + + _, primaryServiceClusterCIDR, err := netutils.ParseCIDRSloppy(serviceClusterIPRangeList[0]) + if err != nil { + return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("service-cluster-ip-range[0] is not a valid cidr") + } + + primaryServiceIPRange, apiServerServiceIP, err = controlplane.ServiceIPRange(*primaryServiceClusterCIDR) + if err != nil { + return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("error determining service IP ranges for primary service cidr: %v", err) + } + + // user provided at least two entries + // note: validation asserts that the list is max of two dual stack entries + if len(serviceClusterIPRangeList) > 1 { + _, secondaryServiceClusterCIDR, err := netutils.ParseCIDRSloppy(serviceClusterIPRangeList[1]) + if err != nil { + return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("service-cluster-ip-range[1] is not an ip net") + } + secondaryServiceIPRange = *secondaryServiceClusterCIDR + } + return apiServerServiceIP, primaryServiceIPRange, secondaryServiceIPRange, nil +} diff --git a/cmd/kube-apiserver/app/server_test.go b/cmd/kube-apiserver/app/options/completion_test.go similarity index 99% rename from cmd/kube-apiserver/app/server_test.go rename to cmd/kube-apiserver/app/options/completion_test.go index cf53239d8c6..dc9449c91cd 100644 --- a/cmd/kube-apiserver/app/server_test.go +++ b/cmd/kube-apiserver/app/options/completion_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package app +package options import ( "testing" diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index bc5deb924c7..9a9983819c8 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -22,12 +22,9 @@ package app import ( "crypto/tls" "fmt" - "net" "net/http" "net/url" "os" - "strings" - "time" "github.com/spf13/cobra" @@ -42,7 +39,6 @@ import ( genericapifilters "k8s.io/apiserver/pkg/endpoints/filters" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/egressselector" - serveroptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/apiserver/pkg/util/notfoundhandler" "k8s.io/apiserver/pkg/util/webhook" @@ -63,7 +59,6 @@ import ( "k8s.io/klog/v2" aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" - netutils "k8s.io/utils/net" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" "k8s.io/kubernetes/pkg/api/legacyscheme" @@ -72,9 +67,7 @@ import ( controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver" "k8s.io/kubernetes/pkg/controlplane/reconcilers" generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi" - "k8s.io/kubernetes/pkg/kubeapiserver" kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" - kubeauthenticator "k8s.io/kubernetes/pkg/kubeapiserver/authenticator" "k8s.io/kubernetes/pkg/serviceaccount" ) @@ -112,7 +105,7 @@ cluster's shared state through which all other components interact.`, cliflag.PrintFlags(fs) // set default options - completedOptions, err := Complete(s) + completedOptions, err := options.Complete(s) if err != nil { return err } @@ -151,13 +144,13 @@ cluster's shared state through which all other components interact.`, } // Run runs the specified APIServer. This should never exit. -func Run(options completedServerRunOptions, stopCh <-chan struct{}) error { +func Run(opts options.CompletedOptions, stopCh <-chan struct{}) error { // To help debugging, immediately log version klog.Infof("Version: %+v", version.Get()) klog.InfoS("Golang settings", "GOGC", os.Getenv("GOGC"), "GOMAXPROCS", os.Getenv("GOMAXPROCS"), "GOTRACEBACK", os.Getenv("GOTRACEBACK")) - config, err := NewConfig(options) + config, err := NewConfig(opts) if err != nil { return err } @@ -215,7 +208,7 @@ func CreateProxyTransport() *http.Transport { } // CreateKubeAPIServerConfig creates all the resources for running the API server, but runs none of them -func CreateKubeAPIServerConfig(s completedServerRunOptions) ( +func CreateKubeAPIServerConfig(opts options.CompletedOptions) ( *controlplane.Config, aggregatorapiserver.ServiceResolver, []admission.PluginInitializer, @@ -224,7 +217,7 @@ func CreateKubeAPIServerConfig(s completedServerRunOptions) ( proxyTransport := CreateProxyTransport() genericConfig, versionedInformers, storageFactory, err := controlplaneapiserver.BuildGenericConfig( - s.ServerRunOptions, + opts.ServerRunOptions, []*runtime.Scheme{legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme}, generatedopenapi.GetOpenAPIDefinitions, ) @@ -232,9 +225,9 @@ func CreateKubeAPIServerConfig(s completedServerRunOptions) ( return nil, nil, nil, err } - capabilities.Setup(s.AllowPrivileged, s.MaxConnectionBytesPerSec) + capabilities.Setup(opts.AllowPrivileged, opts.MaxConnectionBytesPerSec) - s.Metrics.Apply() + opts.Metrics.Apply() serviceaccount.RegisterMetrics() config := &controlplane.Config{ @@ -242,38 +235,38 @@ func CreateKubeAPIServerConfig(s completedServerRunOptions) ( ExtraConfig: controlplane.ExtraConfig{ APIResourceConfigSource: storageFactory.APIResourceConfigSource, StorageFactory: storageFactory, - EventTTL: s.EventTTL, - KubeletClientConfig: s.KubeletConfig, - EnableLogsSupport: s.EnableLogsHandler, + EventTTL: opts.EventTTL, + KubeletClientConfig: opts.KubeletConfig, + EnableLogsSupport: opts.EnableLogsHandler, ProxyTransport: proxyTransport, - ServiceIPRange: s.PrimaryServiceClusterIPRange, - APIServerServiceIP: s.APIServerServiceIP, - SecondaryServiceIPRange: s.SecondaryServiceClusterIPRange, + ServiceIPRange: opts.PrimaryServiceClusterIPRange, + APIServerServiceIP: opts.APIServerServiceIP, + SecondaryServiceIPRange: opts.SecondaryServiceClusterIPRange, APIServerServicePort: 443, - ServiceNodePortRange: s.ServiceNodePortRange, - KubernetesServiceNodePort: s.KubernetesServiceNodePort, + ServiceNodePortRange: opts.ServiceNodePortRange, + KubernetesServiceNodePort: opts.KubernetesServiceNodePort, - EndpointReconcilerType: reconcilers.Type(s.EndpointReconcilerType), - MasterCount: s.MasterCount, + EndpointReconcilerType: reconcilers.Type(opts.EndpointReconcilerType), + MasterCount: opts.MasterCount, - ServiceAccountIssuer: s.ServiceAccountIssuer, - ServiceAccountMaxExpiration: s.ServiceAccountTokenMaxExpiration, - ExtendExpiration: s.Authentication.ServiceAccounts.ExtendExpiration, + ServiceAccountIssuer: opts.ServiceAccountIssuer, + ServiceAccountMaxExpiration: opts.ServiceAccountTokenMaxExpiration, + ExtendExpiration: opts.Authentication.ServiceAccounts.ExtendExpiration, VersionedInformers: versionedInformers, }, } - clientCAProvider, err := s.Authentication.ClientCert.GetClientCAContentProvider() + clientCAProvider, err := opts.Authentication.ClientCert.GetClientCAContentProvider() if err != nil { return nil, nil, nil, err } config.ExtraConfig.ClusterAuthenticationInfo.ClientCA = clientCAProvider - requestHeaderConfig, err := s.Authentication.RequestHeader.ToAuthenticationRequestHeaderConfig() + requestHeaderConfig, err := opts.Authentication.RequestHeader.ToAuthenticationRequestHeaderConfig() if err != nil { return nil, nil, nil, err } @@ -289,9 +282,9 @@ func CreateKubeAPIServerConfig(s completedServerRunOptions) ( admissionConfig := &kubeapiserveradmission.Config{ ExternalInformers: versionedInformers, LoopbackClientConfig: genericConfig.LoopbackClientConfig, - CloudConfigFile: s.CloudProvider.CloudConfigFile, + CloudConfigFile: opts.CloudProvider.CloudConfigFile, } - serviceResolver := buildServiceResolver(s.EnableAggregatorRouting, genericConfig.LoopbackClientConfig.Host, versionedInformers) + serviceResolver := buildServiceResolver(opts.EnableAggregatorRouting, genericConfig.LoopbackClientConfig.Host, versionedInformers) schemaResolver := resolver.NewDefinitionsSchemaResolver(k8sscheme.Scheme, genericConfig.OpenAPIConfig.GetDefinitions) pluginInitializers, admissionPostStartHook, err := admissionConfig.New(proxyTransport, genericConfig.EgressSelector, serviceResolver, genericConfig.TracerProvider, schemaResolver) if err != nil { @@ -305,7 +298,7 @@ func CreateKubeAPIServerConfig(s completedServerRunOptions) ( if err != nil { return nil, nil, nil, fmt.Errorf("failed to create real dynamic external client: %w", err) } - err = s.Admission.ApplyTo( + err = opts.Admission.ApplyTo( genericConfig, versionedInformers, clientgoExternalClient, @@ -336,139 +329,20 @@ func CreateKubeAPIServerConfig(s completedServerRunOptions) ( // Load and set the public keys. var pubKeys []interface{} - for _, f := range s.Authentication.ServiceAccounts.KeyFiles { + for _, f := range opts.Authentication.ServiceAccounts.KeyFiles { keys, err := keyutil.PublicKeysFromFile(f) if err != nil { return nil, nil, nil, fmt.Errorf("failed to parse key file %q: %v", f, err) } pubKeys = append(pubKeys, keys...) } - config.ExtraConfig.ServiceAccountIssuerURL = s.Authentication.ServiceAccounts.Issuers[0] - config.ExtraConfig.ServiceAccountJWKSURI = s.Authentication.ServiceAccounts.JWKSURI + config.ExtraConfig.ServiceAccountIssuerURL = opts.Authentication.ServiceAccounts.Issuers[0] + config.ExtraConfig.ServiceAccountJWKSURI = opts.Authentication.ServiceAccounts.JWKSURI config.ExtraConfig.ServiceAccountPublicKeys = pubKeys return config, serviceResolver, pluginInitializers, nil } -// completedServerRunOptions is a private wrapper that enforces a call of Complete() before Run can be invoked. -type completedServerRunOptions struct { - *options.ServerRunOptions -} - -// Complete set default ServerRunOptions. -// Should be called after kube-apiserver flags parsed. -func Complete(s *options.ServerRunOptions) (completedServerRunOptions, error) { - var options completedServerRunOptions - // set defaults - if err := s.GenericServerRunOptions.DefaultAdvertiseAddress(s.SecureServing.SecureServingOptions); err != nil { - return options, err - } - - // process s.ServiceClusterIPRange from list to Primary and Secondary - // we process secondary only if provided by user - apiServerServiceIP, primaryServiceIPRange, secondaryServiceIPRange, err := getServiceIPAndRanges(s.ServiceClusterIPRanges) - if err != nil { - return options, err - } - s.PrimaryServiceClusterIPRange = primaryServiceIPRange - s.SecondaryServiceClusterIPRange = secondaryServiceIPRange - s.APIServerServiceIP = apiServerServiceIP - - if err := s.SecureServing.MaybeDefaultWithSelfSignedCerts(s.GenericServerRunOptions.AdvertiseAddress.String(), []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP}); err != nil { - return options, fmt.Errorf("error creating self-signed certificates: %v", err) - } - - if len(s.GenericServerRunOptions.ExternalHost) == 0 { - if len(s.GenericServerRunOptions.AdvertiseAddress) > 0 { - s.GenericServerRunOptions.ExternalHost = s.GenericServerRunOptions.AdvertiseAddress.String() - } else { - hostname, err := os.Hostname() - if err != nil { - return options, fmt.Errorf("error finding host name: %v", err) - } - s.GenericServerRunOptions.ExternalHost = hostname - } - klog.Infof("external host was not specified, using %v", s.GenericServerRunOptions.ExternalHost) - } - - s.Authentication.ApplyAuthorization(s.Authorization) - - // Use (ServiceAccountSigningKeyFile != "") as a proxy to the user enabling - // TokenRequest functionality. This defaulting was convenient, but messed up - // a lot of people when they rotated their serving cert with no idea it was - // connected to their service account keys. We are taking this opportunity to - // remove this problematic defaulting. - if s.ServiceAccountSigningKeyFile == "" { - // Default to the private server key for service account token signing - if len(s.Authentication.ServiceAccounts.KeyFiles) == 0 && s.SecureServing.ServerCert.CertKey.KeyFile != "" { - if kubeauthenticator.IsValidServiceAccountKeyFile(s.SecureServing.ServerCert.CertKey.KeyFile) { - s.Authentication.ServiceAccounts.KeyFiles = []string{s.SecureServing.ServerCert.CertKey.KeyFile} - } else { - klog.Warning("No TLS key provided, service account token authentication disabled") - } - } - } - - if s.ServiceAccountSigningKeyFile != "" && len(s.Authentication.ServiceAccounts.Issuers) != 0 && s.Authentication.ServiceAccounts.Issuers[0] != "" { - sk, err := keyutil.PrivateKeyFromFile(s.ServiceAccountSigningKeyFile) - if err != nil { - return options, fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err) - } - if s.Authentication.ServiceAccounts.MaxExpiration != 0 { - lowBound := time.Hour - upBound := time.Duration(1<<32) * time.Second - if s.Authentication.ServiceAccounts.MaxExpiration < lowBound || - s.Authentication.ServiceAccounts.MaxExpiration > upBound { - return options, fmt.Errorf("the service-account-max-token-expiration must be between 1 hour and 2^32 seconds") - } - if s.Authentication.ServiceAccounts.ExtendExpiration { - if s.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.WarnOnlyBoundTokenExpirationSeconds*time.Second { - klog.Warningf("service-account-extend-token-expiration is true, in order to correctly trigger safe transition logic, service-account-max-token-expiration must be set longer than %d seconds (currently %s)", serviceaccount.WarnOnlyBoundTokenExpirationSeconds, s.Authentication.ServiceAccounts.MaxExpiration) - } - if s.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.ExpirationExtensionSeconds*time.Second { - klog.Warningf("service-account-extend-token-expiration is true, enabling tokens valid up to %d seconds, which is longer than service-account-max-token-expiration set to %s seconds", serviceaccount.ExpirationExtensionSeconds, s.Authentication.ServiceAccounts.MaxExpiration) - } - } - } - - s.ServiceAccountIssuer, err = serviceaccount.JWTTokenGenerator(s.Authentication.ServiceAccounts.Issuers[0], sk) - if err != nil { - return options, fmt.Errorf("failed to build token generator: %v", err) - } - s.ServiceAccountTokenMaxExpiration = s.Authentication.ServiceAccounts.MaxExpiration - } - - if s.Etcd.EnableWatchCache { - sizes := kubeapiserver.DefaultWatchCacheSizes() - // Ensure that overrides parse correctly. - userSpecified, err := serveroptions.ParseWatchCacheSizes(s.Etcd.WatchCacheSizes) - if err != nil { - return options, err - } - for resource, size := range userSpecified { - sizes[resource] = size - } - s.Etcd.WatchCacheSizes, err = serveroptions.WriteWatchCacheSizes(sizes) - if err != nil { - return options, err - } - } - - for key, value := range s.APIEnablement.RuntimeConfig { - if key == "v1" || strings.HasPrefix(key, "v1/") || - key == "api/v1" || strings.HasPrefix(key, "api/v1/") { - delete(s.APIEnablement.RuntimeConfig, key) - s.APIEnablement.RuntimeConfig["/v1"] = value - } - if key == "api/legacy" { - delete(s.APIEnablement.RuntimeConfig, key) - } - } - - options.ServerRunOptions = s - return options, nil -} - var testServiceResolver webhook.ServiceResolver // SetServiceResolverForTests allows the service resolver to be overridden during tests. @@ -508,45 +382,3 @@ func buildServiceResolver(enabledAggregatorRouting bool, hostname string, inform } return serviceResolver } - -func getServiceIPAndRanges(serviceClusterIPRanges string) (net.IP, net.IPNet, net.IPNet, error) { - serviceClusterIPRangeList := []string{} - if serviceClusterIPRanges != "" { - serviceClusterIPRangeList = strings.Split(serviceClusterIPRanges, ",") - } - - var apiServerServiceIP net.IP - var primaryServiceIPRange net.IPNet - var secondaryServiceIPRange net.IPNet - var err error - // nothing provided by user, use default range (only applies to the Primary) - if len(serviceClusterIPRangeList) == 0 { - var primaryServiceClusterCIDR net.IPNet - primaryServiceIPRange, apiServerServiceIP, err = controlplane.ServiceIPRange(primaryServiceClusterCIDR) - if err != nil { - return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("error determining service IP ranges: %v", err) - } - return apiServerServiceIP, primaryServiceIPRange, net.IPNet{}, nil - } - - _, primaryServiceClusterCIDR, err := netutils.ParseCIDRSloppy(serviceClusterIPRangeList[0]) - if err != nil { - return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("service-cluster-ip-range[0] is not a valid cidr") - } - - primaryServiceIPRange, apiServerServiceIP, err = controlplane.ServiceIPRange(*primaryServiceClusterCIDR) - if err != nil { - return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("error determining service IP ranges for primary service cidr: %v", err) - } - - // user provided at least two entries - // note: validation asserts that the list is max of two dual stack entries - if len(serviceClusterIPRangeList) > 1 { - _, secondaryServiceClusterCIDR, err := netutils.ParseCIDRSloppy(serviceClusterIPRangeList[1]) - if err != nil { - return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("service-cluster-ip-range[1] is not an ip net") - } - secondaryServiceIPRange = *secondaryServiceClusterCIDR - } - return apiServerServiceIP, primaryServiceIPRange, secondaryServiceIPRange, nil -} diff --git a/cmd/kube-apiserver/app/testing/testserver.go b/cmd/kube-apiserver/app/testing/testserver.go index b625d38d301..0de34d08968 100644 --- a/cmd/kube-apiserver/app/testing/testserver.go +++ b/cmd/kube-apiserver/app/testing/testserver.go @@ -44,6 +44,7 @@ import ( "k8s.io/client-go/util/cert" "k8s.io/klog/v2" "k8s.io/kube-aggregator/pkg/apiserver" + "k8s.io/kubernetes/cmd/kube-apiserver/app" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" @@ -235,7 +236,7 @@ func StartTestServer(t Logger, instanceOptions *TestServerInstanceOptions, custo s.Authentication.ServiceAccounts.Issuers = []string{"https://foo.bar.example.com"} s.Authentication.ServiceAccounts.KeyFiles = []string{saSigningKeyFile.Name()} - completedOptions, err := app.Complete(s) + completedOptions, err := options.Complete(s) if err != nil { return result, fmt.Errorf("failed to set default ServerRunOptions: %v", err) } diff --git a/test/e2e_node/services/apiserver.go b/test/e2e_node/services/apiserver.go index 957f594df21..769b5cfd8e7 100644 --- a/test/e2e_node/services/apiserver.go +++ b/test/e2e_node/services/apiserver.go @@ -96,7 +96,7 @@ func (a *APIServer) Start() error { errCh := make(chan error) go func() { defer close(errCh) - completedOptions, err := apiserver.Complete(o) + completedOptions, err := options.Complete(o) if err != nil { errCh <- fmt.Errorf("set apiserver default options error: %w", err) return diff --git a/test/integration/etcd/server.go b/test/integration/etcd/server.go index 530834d7833..cdb4e2587cd 100644 --- a/test/integration/etcd/server.go +++ b/test/integration/etcd/server.go @@ -104,7 +104,7 @@ func StartRealAPIServerOrDie(t *testing.T, configFuncs ...func(*options.ServerRu for _, f := range configFuncs { f(kubeAPIServerOptions) } - completedOptions, err := app.Complete(kubeAPIServerOptions) + completedOptions, err := options.Complete(kubeAPIServerOptions) if err != nil { t.Fatal(err) } diff --git a/test/integration/framework/test_server.go b/test/integration/framework/test_server.go index 123193c1c04..13b1d1a0577 100644 --- a/test/integration/framework/test_server.go +++ b/test/integration/framework/test_server.go @@ -150,7 +150,7 @@ func StartTestServer(ctx context.Context, t testing.TB, setup TestServerSetup) ( setup.ModifyServerRunOptions(kubeAPIServerOptions) } - completedOptions, err := app.Complete(kubeAPIServerOptions) + completedOptions, err := options.Complete(kubeAPIServerOptions) if err != nil { t.Fatal(err) }