diff --git a/cmd/kube-apiserver/app/options/validation.go b/cmd/kube-apiserver/app/options/validation.go index 66936baa4dd..9d825e8fe65 100644 --- a/cmd/kube-apiserver/app/options/validation.go +++ b/cmd/kube-apiserver/app/options/validation.go @@ -24,9 +24,9 @@ import ( genericoptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" netutils "k8s.io/utils/net" - "k8s.io/apimachinery/pkg/util/version" controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options" "k8s.io/kubernetes/pkg/controlplane/reconcilers" "k8s.io/kubernetes/pkg/features" @@ -142,13 +142,10 @@ func (s CompletedOptions) Validate() []error { errs = append(errs, fmt.Errorf("--apiserver-count should be a positive number, but value '%d' provided", s.MasterCount)) } - // TODO: remove in 1.32 - // emulationVersion is introduced in 1.31, so it is only allowed to be equal to the binary version at 1.31. + // TODO(#125980): remove in 1.32 effectiveVersion := s.GenericServerRunOptions.ComponentGlobalsRegistry.EffectiveVersionFor(s.GenericServerRunOptions.ComponentName) - binaryVersion := version.MajorMinor(effectiveVersion.BinaryVersion().Major(), effectiveVersion.BinaryVersion().Minor()) - if binaryVersion.EqualTo(version.MajorMinor(1, 31)) && !effectiveVersion.EmulationVersion().EqualTo(binaryVersion) { - errs = append(errs, fmt.Errorf("emulation version needs to be equal to binary version(%s) in compatibility-version alpha, got %s", - binaryVersion.String(), effectiveVersion.EmulationVersion().String())) + if err := utilversion.ValidateKubeEffectiveVersion(effectiveVersion); err != nil { + errs = append(errs, err) } return errs diff --git a/cmd/kube-scheduler/app/options/options.go b/cmd/kube-scheduler/app/options/options.go index e039b0d05c9..c4dd5df2cf2 100644 --- a/cmd/kube-scheduler/app/options/options.go +++ b/cmd/kube-scheduler/app/options/options.go @@ -25,9 +25,11 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/uuid" apiserveroptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/dynamic" "k8s.io/client-go/dynamic/dynamicinformer" clientset "k8s.io/client-go/kubernetes" @@ -72,12 +74,21 @@ type Options struct { Master string + // ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored. + ComponentGlobalsRegistry utilversion.ComponentGlobalsRegistry + // Flags hold the parsed CLI flags. Flags *cliflag.NamedFlagSets } // NewOptions returns default scheduler app options. func NewOptions() *Options { + // make sure DefaultKubeComponent is registered in the DefaultComponentGlobalsRegistry. + if utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent) == nil { + featureGate := utilfeature.DefaultMutableFeatureGate + effectiveVersion := utilversion.DefaultKubeEffectiveVersion() + utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate)) + } o := &Options{ SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(), Authentication: apiserveroptions.NewDelegatingAuthenticationOptions(), @@ -94,8 +105,9 @@ func NewOptions() *Options { ResourceName: "kube-scheduler", ResourceNamespace: "kube-system", }, - Metrics: metrics.NewOptions(), - Logs: logs.NewOptions(), + Metrics: metrics.NewOptions(), + Logs: logs.NewOptions(), + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, } o.Authentication.TolerateInClusterLookupFailure = true @@ -189,7 +201,7 @@ func (o *Options) initFlags() { o.Authorization.AddFlags(nfs.FlagSet("authorization")) o.Deprecated.AddFlags(nfs.FlagSet("deprecated")) options.BindLeaderElectionFlags(o.LeaderElection, nfs.FlagSet("leader election")) - utilfeature.DefaultMutableFeatureGate.AddFlag(nfs.FlagSet("feature gate")) + o.ComponentGlobalsRegistry.AddFlags(nfs.FlagSet("feature gate")) o.Metrics.AddFlags(nfs.FlagSet("metrics")) logsapi.AddFlags(o.Logs, nfs.FlagSet("logs")) @@ -198,6 +210,9 @@ func (o *Options) initFlags() { // ApplyTo applies the scheduler options to the given scheduler app configuration. func (o *Options) ApplyTo(logger klog.Logger, c *schedulerappconfig.Config) error { + if err := o.ComponentGlobalsRegistry.SetFallback(); err != nil { + return err + } if len(o.ConfigFile) == 0 { // If the --config arg is not specified, honor the deprecated as well as leader election CLI args. o.ApplyDeprecated() @@ -251,7 +266,11 @@ func (o *Options) ApplyTo(logger klog.Logger, c *schedulerappconfig.Config) erro // Validate validates all the required options. func (o *Options) Validate() []error { var errs []error - + if err := o.ComponentGlobalsRegistry.SetFallback(); err != nil { + errs = append(errs, err) + } else { + errs = append(errs, o.ComponentGlobalsRegistry.Validate()...) + } if err := validation.ValidateKubeSchedulerConfiguration(o.ComponentConfig); err != nil { errs = append(errs, err.Errors()...) } @@ -260,6 +279,12 @@ func (o *Options) Validate() []error { errs = append(errs, o.Authorization.Validate()...) errs = append(errs, o.Metrics.Validate()...) + // TODO(#125980): remove in 1.32 + effectiveVersion := o.ComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent) + if err := utilversion.ValidateKubeEffectiveVersion(effectiveVersion); err != nil { + errs = append(errs, err) + } + return errs } diff --git a/cmd/kube-scheduler/app/options/options_test.go b/cmd/kube-scheduler/app/options/options_test.go index cb721e39b83..40206db23d3 100644 --- a/cmd/kube-scheduler/app/options/options_test.go +++ b/cmd/kube-scheduler/app/options/options_test.go @@ -32,6 +32,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" apiserveroptions "k8s.io/apiserver/pkg/server/options" + utilversion "k8s.io/apiserver/pkg/util/version" componentbaseconfig "k8s.io/component-base/config" "k8s.io/component-base/logs" "k8s.io/klog/v2/ktesting" @@ -321,7 +322,8 @@ profiles: AlwaysAllowPaths: []string{"/healthz", "/readyz", "/livez"}, // note: this does not match /healthz/ or /healthz/* AlwaysAllowGroups: []string{"system:masters"}, }, - Logs: logs.NewOptions(), + Logs: logs.NewOptions(), + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectedUsername: "config", expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ @@ -372,23 +374,26 @@ profiles: } return cfg }(), - Logs: logs.NewOptions(), + Logs: logs.NewOptions(), + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectedError: "no kind \"KubeSchedulerConfiguration\" is registered for version \"componentconfig/v1alpha1\"", }, { name: "unknown version kubescheduler.config.k8s.io/unknown", options: &Options{ - ConfigFile: unknownVersionConfig, - Logs: logs.NewOptions(), + ConfigFile: unknownVersionConfig, + Logs: logs.NewOptions(), + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectedError: "no kind \"KubeSchedulerConfiguration\" is registered for version \"kubescheduler.config.k8s.io/unknown\"", }, { name: "config file with no version", options: &Options{ - ConfigFile: noVersionConfig, - Logs: logs.NewOptions(), + ConfigFile: noVersionConfig, + Logs: logs.NewOptions(), + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectedError: "Object 'apiVersion' is missing", }, @@ -424,7 +429,8 @@ profiles: AlwaysAllowPaths: []string{"/healthz", "/readyz", "/livez"}, // note: this does not match /healthz/ or /healthz/* AlwaysAllowGroups: []string{"system:masters"}, }, - Logs: logs.NewOptions(), + Logs: logs.NewOptions(), + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectedUsername: "flag", expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ @@ -496,7 +502,8 @@ profiles: AlwaysAllowPaths: []string{"/healthz", "/readyz", "/livez"}, // note: this does not match /healthz/ or /healthz/* AlwaysAllowGroups: []string{"system:masters"}, }, - Logs: logs.NewOptions(), + Logs: logs.NewOptions(), + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ TypeMeta: metav1.TypeMeta{ @@ -539,8 +546,9 @@ profiles: { name: "plugin config", options: &Options{ - ConfigFile: pluginConfigFile, - Logs: logs.NewOptions(), + ConfigFile: pluginConfigFile, + Logs: logs.NewOptions(), + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectedUsername: "config", expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ @@ -659,8 +667,9 @@ profiles: { name: "multiple profiles", options: &Options{ - ConfigFile: multiProfilesConfig, - Logs: logs.NewOptions(), + ConfigFile: multiProfilesConfig, + Logs: logs.NewOptions(), + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectedUsername: "config", expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ @@ -774,15 +783,17 @@ profiles: { name: "no config", options: &Options{ - Logs: logs.NewOptions(), + Logs: logs.NewOptions(), + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectedError: "no configuration has been provided", }, { name: "unknown field", options: &Options{ - ConfigFile: unknownFieldConfig, - Logs: logs.NewOptions(), + ConfigFile: unknownFieldConfig, + Logs: logs.NewOptions(), + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectedError: `unknown field "foo"`, checkErrFn: runtime.IsStrictDecodingError, @@ -790,8 +801,9 @@ profiles: { name: "duplicate fields", options: &Options{ - ConfigFile: duplicateFieldConfig, - Logs: logs.NewOptions(), + ConfigFile: duplicateFieldConfig, + Logs: logs.NewOptions(), + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectedError: `key "leaderElect" already set`, checkErrFn: runtime.IsStrictDecodingError, @@ -799,8 +811,9 @@ profiles: { name: "high throughput profile", options: &Options{ - ConfigFile: highThroughputProfileConfig, - Logs: logs.NewOptions(), + ConfigFile: highThroughputProfileConfig, + Logs: logs.NewOptions(), + ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry, }, expectedUsername: "config", expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index dc9f8638f0f..9cbc044f111 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -38,6 +38,7 @@ import ( "k8s.io/apiserver/pkg/server/mux" "k8s.io/apiserver/pkg/server/routes" utilfeature "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/events" @@ -45,6 +46,7 @@ import ( cliflag "k8s.io/component-base/cli/flag" "k8s.io/component-base/cli/globalflag" "k8s.io/component-base/configz" + "k8s.io/component-base/featuregate" "k8s.io/component-base/logs" logsapi "k8s.io/component-base/logs/api/v1" "k8s.io/component-base/metrics/features" @@ -74,6 +76,10 @@ type Option func(runtime.Registry) error // NewSchedulerCommand creates a *cobra.Command object with default parameters and registryOptions func NewSchedulerCommand(registryOptions ...Option) *cobra.Command { + // explicitly register (if not already registered) the kube effective version and feature gate in DefaultComponentGlobalsRegistry, + // which will be used in NewOptions. + _, _ = utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( + utilversion.DefaultKubeComponent, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) opts := options.NewOptions() cmd := &cobra.Command{ @@ -86,6 +92,10 @@ suitable Node. Multiple different schedulers may be used within a cluster; kube-scheduler is the reference implementation. See [scheduling](https://kubernetes.io/docs/concepts/scheduling-eviction/) for more information about scheduling and the kube-scheduler component.`, + PersistentPreRunE: func(*cobra.Command, []string) error { + // makes sure feature gates are set before RunE. + return opts.ComponentGlobalsRegistry.Set() + }, RunE: func(cmd *cobra.Command, args []string) error { return runCommand(cmd, opts, registryOptions...) }, @@ -120,10 +130,10 @@ for more information about scheduling and the kube-scheduler component.`, // runCommand runs the scheduler. func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Option) error { verflag.PrintAndExitIfRequested() - + fg := opts.ComponentGlobalsRegistry.FeatureGateFor(utilversion.DefaultKubeComponent) // Activate logging as soon as possible, after that // show flags with the final logging configuration. - if err := logsapi.ValidateAndApply(opts.Logs, utilfeature.DefaultFeatureGate); err != nil { + if err := logsapi.ValidateAndApply(opts.Logs, fg); err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) } @@ -142,7 +152,7 @@ func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Op return err } // add feature enablement metrics - utilfeature.DefaultMutableFeatureGate.AddMetrics() + fg.(featuregate.MutableFeatureGate).AddMetrics() return Run(ctx, cc, sched) } diff --git a/cmd/kube-scheduler/app/server_test.go b/cmd/kube-scheduler/app/server_test.go index 5935fc87465..8b23584ec9e 100644 --- a/cmd/kube-scheduler/app/server_test.go +++ b/cmd/kube-scheduler/app/server_test.go @@ -32,7 +32,10 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/version" "k8s.io/apiserver/pkg/util/feature" + utilversion "k8s.io/apiserver/pkg/util/version" componentbaseconfig "k8s.io/component-base/config" "k8s.io/component-base/featuregate" featuregatetesting "k8s.io/component-base/featuregate/testing" @@ -215,6 +218,8 @@ leaderElection: wantPlugins map[string]*config.Plugins wantLeaderElection *componentbaseconfig.LeaderElectionConfiguration wantClientConnection *componentbaseconfig.ClientConnectionConfiguration + wantErr bool + wantFeaturesGates map[string]bool }{ { name: "default config with an alpha feature enabled", @@ -376,6 +381,46 @@ leaderElection: ResourceNamespace: configv1.SchedulerDefaultLockObjectNamespace, }, }, + { + name: "emulated version out of range", + flags: []string{ + "--kubeconfig", configKubeconfig, + "--emulated-version=1.28", + }, + wantErr: true, + }, + { + name: "default feature gates at binary version", + flags: []string{ + "--kubeconfig", configKubeconfig, + }, + wantFeaturesGates: map[string]bool{"kubeA": true, "kubeB": false}, + }, + { + name: "default feature gates at emulated version", + flags: []string{ + "--kubeconfig", configKubeconfig, + "--emulated-version=1.31", + }, + wantFeaturesGates: map[string]bool{"kubeA": false, "kubeB": false}, + }, + { + name: "set feature gates at emulated version", + flags: []string{ + "--kubeconfig", configKubeconfig, + "--emulated-version=1.31", + "--feature-gates=kubeA=false,kubeB=true", + }, + wantFeaturesGates: map[string]bool{"kubeA": false, "kubeB": true}, + }, + { + name: "cannot set locked feature gate", + flags: []string{ + "--kubeconfig", configKubeconfig, + "--feature-gates=kubeA=false,kubeB=true", + }, + wantErr: true, + }, } makeListener := func(t *testing.T) net.Listener { @@ -392,6 +437,23 @@ leaderElection: for k, v := range tc.restoreFeatures { featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, k, v) } + componentGlobalsRegistry := utilversion.DefaultComponentGlobalsRegistry + t.Cleanup(func() { + componentGlobalsRegistry.Reset() + }) + componentGlobalsRegistry.Reset() + verKube := utilversion.NewEffectiveVersion("1.32") + fg := feature.DefaultFeatureGate.DeepCopy() + utilruntime.Must(fg.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{ + "kubeA": { + {Version: version.MustParse("1.32"), Default: true, LockToDefault: true, PreRelease: featuregate.GA}, + {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Beta}, + }, + "kubeB": { + {Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Alpha}, + }, + })) + utilruntime.Must(componentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, verKube, fg)) fs := pflag.NewFlagSet("test", pflag.PanicOnError) opts := options.NewOptions() @@ -415,6 +477,12 @@ leaderElection: ctx, cancel := context.WithCancel(context.Background()) defer cancel() _, sched, err := Setup(ctx, opts, tc.registryOptions...) + if tc.wantErr { + if err == nil { + t.Fatal("expected Setup error, got nil") + } + return + } if err != nil { t.Fatal(err) } @@ -443,6 +511,12 @@ leaderElection: t.Errorf("Unexpected clientConnection diff (-want, +got): %s", diff) } } + for f, v := range tc.wantFeaturesGates { + enabled := fg.Enabled(featuregate.Feature(f)) + if enabled != v { + t.Errorf("expected featuregate.Enabled(%s)=%v, got %v", f, v, enabled) + } + } }) } } diff --git a/cmd/kube-scheduler/app/testing/testserver.go b/cmd/kube-scheduler/app/testing/testserver.go index 857b3097db2..cc0eb62fdb2 100644 --- a/cmd/kube-scheduler/app/testing/testserver.go +++ b/cmd/kube-scheduler/app/testing/testserver.go @@ -103,6 +103,9 @@ func StartTestServer(ctx context.Context, customFlags []string) (result TestServ fs.AddFlagSet(f) } fs.Parse(customFlags) + if err := opts.ComponentGlobalsRegistry.Set(); err != nil { + return result, err + } if opts.SecureServing.BindPort != 0 { opts.SecureServing.Listener, opts.SecureServing.BindPort, err = createListenerOnFreePort() diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/version.go b/staging/src/k8s.io/apiserver/pkg/util/version/version.go index a7a5fda87c7..bedf85d4d88 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/version/version.go +++ b/staging/src/k8s.io/apiserver/pkg/util/version/version.go @@ -155,3 +155,15 @@ func DefaultKubeEffectiveVersion() MutableEffectiveVersion { binaryVersion := version.MustParse(baseversion.DefaultKubeBinaryVersion).WithInfo(baseversion.Get()) return newEffectiveVersion(binaryVersion) } + +// ValidateKubeEffectiveVersion validates the EmulationVersion is equal to the binary version at 1.31 for kube components. +// TODO: remove in 1.32 +// emulationVersion is introduced in 1.31, so it is only allowed to be equal to the binary version at 1.31. +func ValidateKubeEffectiveVersion(effectiveVersion EffectiveVersion) error { + binaryVersion := version.MajorMinor(effectiveVersion.BinaryVersion().Major(), effectiveVersion.BinaryVersion().Minor()) + if binaryVersion.EqualTo(version.MajorMinor(1, 31)) && !effectiveVersion.EmulationVersion().EqualTo(binaryVersion) { + return fmt.Errorf("emulation version needs to be equal to binary version(%s) in compatibility-version alpha, got %s", + binaryVersion.String(), effectiveVersion.EmulationVersion().String()) + } + return nil +}