diff --git a/cmd/kube-apiserver/app/aggregator.go b/cmd/kube-apiserver/app/aggregator.go index e03cdf9dccd..7aa12d12b87 100644 --- a/cmd/kube-apiserver/app/aggregator.go +++ b/cmd/kube-apiserver/app/aggregator.go @@ -47,13 +47,13 @@ import ( apiregistrationclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1" informers "k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1" "k8s.io/kube-aggregator/pkg/controllers/autoregister" - "k8s.io/kubernetes/cmd/kube-apiserver/app/options" + controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options" "k8s.io/kubernetes/pkg/controlplane/controller/crdregistration" ) func createAggregatorConfig( kubeAPIServerConfig genericapiserver.Config, - commandOptions *options.ServerRunOptions, + commandOptions controlplaneapiserver.CompletedOptions, externalInformers kubeexternalinformers.SharedInformerFactory, serviceResolver aggregatorapiserver.ServiceResolver, proxyTransport *http.Transport, diff --git a/cmd/kube-apiserver/app/config.go b/cmd/kube-apiserver/app/config.go index 651ebd5ae78..ef4671ab3fe 100644 --- a/cmd/kube-apiserver/app/config.go +++ b/cmd/kube-apiserver/app/config.go @@ -77,14 +77,14 @@ func NewConfig(opts options.CompletedOptions) (*Config, error) { } c.ControlPlane = controlPlane - apiExtensions, err := apiserver.CreateAPIExtensionsConfig(*controlPlane.GenericConfig, controlPlane.ExtraConfig.VersionedInformers, pluginInitializer, opts.ServerRunOptions, opts.MasterCount, + apiExtensions, err := apiserver.CreateAPIExtensionsConfig(*controlPlane.GenericConfig, controlPlane.ExtraConfig.VersionedInformers, pluginInitializer, opts.CompletedOptions, opts.MasterCount, serviceResolver, webhook.NewDefaultAuthenticationInfoResolverWrapper(controlPlane.ExtraConfig.ProxyTransport, controlPlane.GenericConfig.EgressSelector, controlPlane.GenericConfig.LoopbackClientConfig, controlPlane.GenericConfig.TracerProvider)) if err != nil { return nil, err } c.ApiExtensions = apiExtensions - aggregator, err := createAggregatorConfig(*controlPlane.GenericConfig, opts.ServerRunOptions, controlPlane.ExtraConfig.VersionedInformers, serviceResolver, controlPlane.ExtraConfig.ProxyTransport, pluginInitializer) + aggregator, err := createAggregatorConfig(*controlPlane.GenericConfig, opts.CompletedOptions, controlPlane.ExtraConfig.VersionedInformers, serviceResolver, controlPlane.ExtraConfig.ProxyTransport, pluginInitializer) if err != nil { return nil, err } diff --git a/cmd/kube-apiserver/app/options/completion.go b/cmd/kube-apiserver/app/options/completion.go index 00327414566..32018eda75f 100644 --- a/cmd/kube-apiserver/app/options/completion.go +++ b/cmd/kube-apiserver/app/options/completion.go @@ -45,7 +45,7 @@ type CompletedOptions struct { // Complete set default ServerRunOptions. // Should be called after kube-apiserver flags parsed. -func Complete(opts *ServerRunOptions) (CompletedOptions, error) { +func (opts *ServerRunOptions) Complete() (CompletedOptions, error) { if opts == nil { return CompletedOptions{completedOptions: &completedOptions{}}, nil } diff --git a/cmd/kube-apiserver/app/options/options_test.go b/cmd/kube-apiserver/app/options/options_test.go index 2a27cc508e0..8366b566a64 100644 --- a/cmd/kube-apiserver/app/options/options_test.go +++ b/cmd/kube-apiserver/app/options/options_test.go @@ -45,7 +45,7 @@ import ( ) func TestAddFlags(t *testing.T) { - fs := pflag.NewFlagSet("addflagstest", pflag.ContinueOnError) + fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError) s := NewServerRunOptions() for _, f := range s.Flags().FlagSets { fs.AddFlagSet(f) diff --git a/cmd/kube-apiserver/app/options/validation.go b/cmd/kube-apiserver/app/options/validation.go index 221efe7d245..4767ea5cabe 100644 --- a/cmd/kube-apiserver/app/options/validation.go +++ b/cmd/kube-apiserver/app/options/validation.go @@ -29,7 +29,7 @@ import ( // TODO: Longer term we should read this from some config store, rather than a flag. // validateClusterIPFlags is expected to be called after Complete() -func validateClusterIPFlags(options *ServerRunOptions) []error { +func validateClusterIPFlags(options Extra) []error { var errs []error // maxCIDRBits is used to define the maximum CIDR size for the cluster ip(s) maxCIDRBits := 20 @@ -89,7 +89,7 @@ func validateMaxCIDRRange(cidr net.IPNet, maxCIDRBits int, cidrFlag string) erro return nil } -func validateServiceNodePort(options *ServerRunOptions) []error { +func validateServiceNodePort(options Extra) []error { var errs []error if options.KubernetesServiceNodePort < 0 || options.KubernetesServiceNodePort > 65535 { @@ -103,12 +103,12 @@ func validateServiceNodePort(options *ServerRunOptions) []error { } // Validate checks ServerRunOptions and return a slice of found errs. -func (s *ServerRunOptions) Validate() []error { +func (s CompletedOptions) Validate() []error { var errs []error - errs = append(errs, s.Options.Validate()...) - errs = append(errs, validateClusterIPFlags(s)...) - errs = append(errs, validateServiceNodePort(s)...) + errs = append(errs, s.CompletedOptions.Validate()...) + errs = append(errs, validateClusterIPFlags(s.Extra)...) + errs = append(errs, validateServiceNodePort(s.Extra)...) return errs } diff --git a/cmd/kube-apiserver/app/options/validation_test.go b/cmd/kube-apiserver/app/options/validation_test.go index 3b2281888ca..71843eca1be 100644 --- a/cmd/kube-apiserver/app/options/validation_test.go +++ b/cmd/kube-apiserver/app/options/validation_test.go @@ -143,7 +143,7 @@ func TestClusterServiceIPRange(t *testing.T) { t.Run(tc.name, func(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.MultiCIDRServiceAllocator, tc.gate)() - errs := validateClusterIPFlags(tc.options) + errs := validateClusterIPFlags(tc.options.Extra) if len(errs) > 0 && !tc.expectErrors { t.Errorf("expected no errors, errors found %+v", errs) } @@ -200,7 +200,7 @@ func TestValidateServiceNodePort(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - errs := validateServiceNodePort(tc.options) + errs := validateServiceNodePort(tc.options.Extra) if errs != nil && !tc.expectErrors { t.Errorf("expected no errors, error found %+v", errs) } diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 9a9983819c8..b021bac2574 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -105,7 +105,7 @@ cluster's shared state through which all other components interact.`, cliflag.PrintFlags(fs) // set default options - completedOptions, err := options.Complete(s) + completedOptions, err := s.Complete() if err != nil { return err } @@ -217,7 +217,7 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) ( proxyTransport := CreateProxyTransport() genericConfig, versionedInformers, storageFactory, err := controlplaneapiserver.BuildGenericConfig( - opts.ServerRunOptions, + opts.CompletedOptions, []*runtime.Scheme{legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme}, generatedopenapi.GetOpenAPIDefinitions, ) diff --git a/cmd/kube-apiserver/app/testing/testserver.go b/cmd/kube-apiserver/app/testing/testserver.go index 0de34d08968..570efca524a 100644 --- a/cmd/kube-apiserver/app/testing/testserver.go +++ b/cmd/kube-apiserver/app/testing/testserver.go @@ -236,7 +236,7 @@ func StartTestServer(t Logger, instanceOptions *TestServerInstanceOptions, custo s.Authentication.ServiceAccounts.Issuers = []string{"https://foo.bar.example.com"} s.Authentication.ServiceAccounts.KeyFiles = []string{saSigningKeyFile.Name()} - completedOptions, err := options.Complete(s) + completedOptions, err := s.Complete() if err != nil { return result, fmt.Errorf("failed to set default ServerRunOptions: %v", err) } diff --git a/pkg/controlplane/apiserver/apiextensions.go b/pkg/controlplane/apiserver/apiextensions.go index 6571b661e80..67c5ee83a58 100644 --- a/pkg/controlplane/apiserver/apiextensions.go +++ b/pkg/controlplane/apiserver/apiextensions.go @@ -30,14 +30,14 @@ import ( "k8s.io/apiserver/pkg/util/webhook" "k8s.io/client-go/informers" - "k8s.io/kubernetes/cmd/kube-apiserver/app/options" + controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options" ) func CreateAPIExtensionsConfig( kubeAPIServerConfig server.Config, kubeInformers informers.SharedInformerFactory, pluginInitializers []admission.PluginInitializer, - commandOptions *options.ServerRunOptions, + commandOptions controlplaneapiserver.CompletedOptions, masterCount int, serviceResolver webhook.ServiceResolver, authResolverWrapper webhook.AuthenticationInfoResolverWrapper, diff --git a/pkg/controlplane/apiserver/config.go b/pkg/controlplane/apiserver/config.go index 553697dc00f..af4b82b2657 100644 --- a/pkg/controlplane/apiserver/config.go +++ b/pkg/controlplane/apiserver/config.go @@ -40,9 +40,9 @@ import ( "k8s.io/component-base/version" openapicommon "k8s.io/kube-openapi/pkg/common" - "k8s.io/kubernetes/cmd/kube-apiserver/app/options" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/controlplane" + controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options" "k8s.io/kubernetes/pkg/kubeapiserver" "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest" @@ -50,7 +50,7 @@ import ( // BuildGenericConfig takes the master server options and produces the genericapiserver.Config associated with it func BuildGenericConfig( - s *options.ServerRunOptions, + s controlplaneapiserver.CompletedOptions, schemes []*runtime.Scheme, getOpenAPIDefinitions func(ref openapicommon.ReferenceCallback) map[string]openapicommon.OpenAPIDefinition, ) ( @@ -167,7 +167,7 @@ func BuildGenericConfig( } // BuildAuthorizer constructs the authorizer -func BuildAuthorizer(s *options.ServerRunOptions, EgressSelector *egressselector.EgressSelector, versionedInformers clientgoinformers.SharedInformerFactory) (authorizer.Authorizer, authorizer.RuleResolver, error) { +func BuildAuthorizer(s controlplaneapiserver.CompletedOptions, EgressSelector *egressselector.EgressSelector, versionedInformers clientgoinformers.SharedInformerFactory) (authorizer.Authorizer, authorizer.RuleResolver, error) { authorizationConfig := s.Authorization.ToAuthorizationConfig(versionedInformers) if EgressSelector != nil { @@ -182,7 +182,7 @@ func BuildAuthorizer(s *options.ServerRunOptions, EgressSelector *egressselector } // BuildPriorityAndFairness constructs the guts of the API Priority and Fairness filter -func BuildPriorityAndFairness(s *options.ServerRunOptions, extclient clientgoclientset.Interface, versionedInformer clientgoinformers.SharedInformerFactory) (utilflowcontrol.Interface, error) { +func BuildPriorityAndFairness(s controlplaneapiserver.CompletedOptions, extclient clientgoclientset.Interface, versionedInformer clientgoinformers.SharedInformerFactory) (utilflowcontrol.Interface, error) { if s.GenericServerRunOptions.MaxRequestsInFlight+s.GenericServerRunOptions.MaxMutatingRequestsInFlight <= 0 { return nil, fmt.Errorf("invalid configuration: MaxRequestsInFlight=%d and MaxMutatingRequestsInFlight=%d; they must add up to something positive", s.GenericServerRunOptions.MaxRequestsInFlight, s.GenericServerRunOptions.MaxMutatingRequestsInFlight) } diff --git a/pkg/controlplane/instance.go b/pkg/controlplane/instance.go index aaa27854477..5ddd3fe7ca9 100644 --- a/pkg/controlplane/instance.go +++ b/pkg/controlplane/instance.go @@ -77,6 +77,7 @@ import ( flowcontrolv1beta1 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta1" flowcontrolv1beta2 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta2" flowcontrolv1beta3 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta3" + "k8s.io/kubernetes/pkg/controlplane/apiserver/options" "k8s.io/kubernetes/pkg/controlplane/controller/apiserverleasegc" "k8s.io/kubernetes/pkg/controlplane/controller/clusterauthenticationtrust" "k8s.io/kubernetes/pkg/controlplane/controller/legacytokentracking" @@ -295,7 +296,7 @@ func (c *Config) Complete() CompletedConfig { &c.ExtraConfig, } - serviceIPRange, apiServerServiceIP, err := ServiceIPRange(cfg.ExtraConfig.ServiceIPRange) + serviceIPRange, apiServerServiceIP, err := options.ServiceIPRange(cfg.ExtraConfig.ServiceIPRange) if err != nil { klog.Fatalf("Error determining service IP ranges: %v", err) } diff --git a/test/e2e_node/services/apiserver.go b/test/e2e_node/services/apiserver.go index 769b5cfd8e7..f8fe58bcea0 100644 --- a/test/e2e_node/services/apiserver.go +++ b/test/e2e_node/services/apiserver.go @@ -96,7 +96,7 @@ func (a *APIServer) Start() error { errCh := make(chan error) go func() { defer close(errCh) - completedOptions, err := options.Complete(o) + completedOptions, err := o.Complete() if err != nil { errCh <- fmt.Errorf("set apiserver default options error: %w", err) return diff --git a/test/integration/etcd/server.go b/test/integration/etcd/server.go index cdb4e2587cd..2a70e79919e 100644 --- a/test/integration/etcd/server.go +++ b/test/integration/etcd/server.go @@ -88,23 +88,23 @@ func StartRealAPIServerOrDie(t *testing.T, configFuncs ...func(*options.ServerRu t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err) } - kubeAPIServerOptions := options.NewServerRunOptions() - kubeAPIServerOptions.SecureServing.Listener = listener - kubeAPIServerOptions.SecureServing.ServerCert.CertDirectory = certDir - kubeAPIServerOptions.ServiceAccountSigningKeyFile = saSigningKeyFile.Name() - kubeAPIServerOptions.Etcd.StorageConfig.Transport.ServerList = []string{framework.GetEtcdURL()} - kubeAPIServerOptions.Etcd.DefaultStorageMediaType = runtime.ContentTypeJSON // force json we can easily interpret the result in etcd - kubeAPIServerOptions.ServiceClusterIPRanges = defaultServiceClusterIPRange.String() - kubeAPIServerOptions.Authentication.APIAudiences = []string{"https://foo.bar.example.com"} - kubeAPIServerOptions.Authentication.ServiceAccounts.Issuers = []string{"https://foo.bar.example.com"} - kubeAPIServerOptions.Authentication.ServiceAccounts.KeyFiles = []string{saSigningKeyFile.Name()} - kubeAPIServerOptions.Authorization.Modes = []string{"RBAC"} - kubeAPIServerOptions.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"} - kubeAPIServerOptions.APIEnablement.RuntimeConfig["api/all"] = "true" + opts := options.NewServerRunOptions() + opts.Options.SecureServing.Listener = listener + opts.Options.SecureServing.ServerCert.CertDirectory = certDir + opts.Options.ServiceAccountSigningKeyFile = saSigningKeyFile.Name() + opts.Options.Etcd.StorageConfig.Transport.ServerList = []string{framework.GetEtcdURL()} + opts.Options.Etcd.DefaultStorageMediaType = runtime.ContentTypeJSON // force json we can easily interpret the result in etcd + opts.ServiceClusterIPRanges = defaultServiceClusterIPRange.String() + opts.Options.Authentication.APIAudiences = []string{"https://foo.bar.example.com"} + opts.Options.Authentication.ServiceAccounts.Issuers = []string{"https://foo.bar.example.com"} + opts.Options.Authentication.ServiceAccounts.KeyFiles = []string{saSigningKeyFile.Name()} + opts.Options.Authorization.Modes = []string{"RBAC"} + opts.Options.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"} + opts.Options.APIEnablement.RuntimeConfig["api/all"] = "true" for _, f := range configFuncs { - f(kubeAPIServerOptions) + f(opts) } - completedOptions, err := options.Complete(kubeAPIServerOptions) + completedOptions, err := opts.Complete() if err != nil { t.Fatal(err) } diff --git a/test/integration/framework/test_server.go b/test/integration/framework/test_server.go index 13b1d1a0577..6b15b09b2d8 100644 --- a/test/integration/framework/test_server.go +++ b/test/integration/framework/test_server.go @@ -127,30 +127,30 @@ func StartTestServer(ctx context.Context, t testing.TB, setup TestServerSetup) ( t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err) } - kubeAPIServerOptions := options.NewServerRunOptions() - kubeAPIServerOptions.SecureServing.Listener = listener - kubeAPIServerOptions.SecureServing.BindAddress = netutils.ParseIPSloppy("127.0.0.1") - kubeAPIServerOptions.SecureServing.ServerCert.CertDirectory = certDir - kubeAPIServerOptions.ServiceAccountSigningKeyFile = saSigningKeyFile.Name() - kubeAPIServerOptions.Etcd.StorageConfig.Prefix = path.Join("/", uuid.New().String(), "registry") - kubeAPIServerOptions.Etcd.StorageConfig.Transport.ServerList = []string{GetEtcdURL()} - kubeAPIServerOptions.ServiceClusterIPRanges = defaultServiceClusterIPRange.String() - kubeAPIServerOptions.Authentication.RequestHeader.UsernameHeaders = []string{"X-Remote-User"} - kubeAPIServerOptions.Authentication.RequestHeader.GroupHeaders = []string{"X-Remote-Group"} - kubeAPIServerOptions.Authentication.RequestHeader.ExtraHeaderPrefixes = []string{"X-Remote-Extra-"} - kubeAPIServerOptions.Authentication.RequestHeader.AllowedNames = []string{"kube-aggregator"} - kubeAPIServerOptions.Authentication.RequestHeader.ClientCAFile = proxyCACertFile.Name() - kubeAPIServerOptions.Authentication.APIAudiences = []string{"https://foo.bar.example.com"} - kubeAPIServerOptions.Authentication.ServiceAccounts.Issuers = []string{"https://foo.bar.example.com"} - kubeAPIServerOptions.Authentication.ServiceAccounts.KeyFiles = []string{saSigningKeyFile.Name()} - kubeAPIServerOptions.Authentication.ClientCert.ClientCA = clientCACertFile.Name() - kubeAPIServerOptions.Authorization.Modes = []string{"Node", "RBAC"} + opts := options.NewServerRunOptions() + opts.SecureServing.Listener = listener + opts.SecureServing.BindAddress = netutils.ParseIPSloppy("127.0.0.1") + opts.SecureServing.ServerCert.CertDirectory = certDir + opts.ServiceAccountSigningKeyFile = saSigningKeyFile.Name() + opts.Etcd.StorageConfig.Prefix = path.Join("/", uuid.New().String(), "registry") + opts.Etcd.StorageConfig.Transport.ServerList = []string{GetEtcdURL()} + opts.ServiceClusterIPRanges = defaultServiceClusterIPRange.String() + opts.Authentication.RequestHeader.UsernameHeaders = []string{"X-Remote-User"} + opts.Authentication.RequestHeader.GroupHeaders = []string{"X-Remote-Group"} + opts.Authentication.RequestHeader.ExtraHeaderPrefixes = []string{"X-Remote-Extra-"} + opts.Authentication.RequestHeader.AllowedNames = []string{"kube-aggregator"} + opts.Authentication.RequestHeader.ClientCAFile = proxyCACertFile.Name() + opts.Authentication.APIAudiences = []string{"https://foo.bar.example.com"} + opts.Authentication.ServiceAccounts.Issuers = []string{"https://foo.bar.example.com"} + opts.Authentication.ServiceAccounts.KeyFiles = []string{saSigningKeyFile.Name()} + opts.Authentication.ClientCert.ClientCA = clientCACertFile.Name() + opts.Authorization.Modes = []string{"Node", "RBAC"} if setup.ModifyServerRunOptions != nil { - setup.ModifyServerRunOptions(kubeAPIServerOptions) + setup.ModifyServerRunOptions(opts) } - completedOptions, err := options.Complete(kubeAPIServerOptions) + completedOptions, err := opts.Complete() if err != nil { t.Fatal(err) }