diff --git a/cmd/kube-apiserver/app/options/options_test.go b/cmd/kube-apiserver/app/options/options_test.go index 8b22349924d..33fe7785b89 100644 --- a/cmd/kube-apiserver/app/options/options_test.go +++ b/cmd/kube-apiserver/app/options/options_test.go @@ -35,10 +35,10 @@ import ( auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered" audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate" cliflag "k8s.io/component-base/cli/flag" + basecompatibility "k8s.io/component-base/compatibility" "k8s.io/component-base/featuregate" "k8s.io/component-base/logs" "k8s.io/component-base/metrics" - utilversion "k8s.io/component-base/version" kapi "k8s.io/kubernetes/pkg/apis/core" controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options" "k8s.io/kubernetes/pkg/controlplane/reconcilers" @@ -49,14 +49,12 @@ import ( ) func TestAddFlags(t *testing.T) { - componentGlobalsRegistry := featuregate.DefaultComponentGlobalsRegistry - t.Cleanup(func() { - componentGlobalsRegistry.Reset() - }) + componentGlobalsRegistry := basecompatibility.NewComponentGlobalsRegistry() fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError) - utilruntime.Must(componentGlobalsRegistry.Register("test", utilversion.NewEffectiveVersion("1.32"), featuregate.NewFeatureGate())) + utilruntime.Must(componentGlobalsRegistry.Register("test", basecompatibility.NewEffectiveVersionFromString("1.32", "1.31", "1.31"), featuregate.NewFeatureGate())) s := NewServerRunOptions() + s.GenericServerRunOptions.ComponentGlobalsRegistry = componentGlobalsRegistry for _, f := range s.Flags().FlagSets { fs.AddFlagSet(f) } @@ -150,7 +148,7 @@ func TestAddFlags(t *testing.T) { JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024), MaxRequestBodyBytes: int64(3 * 1024 * 1024), ComponentGlobalsRegistry: componentGlobalsRegistry, - ComponentName: featuregate.DefaultKubeComponent, + ComponentName: basecompatibility.DefaultKubeComponent, }, Admission: &kubeoptions.AdmissionOptions{ GenericAdmission: &apiserveroptions.AdmissionOptions{ diff --git a/cmd/kube-apiserver/app/options/validation.go b/cmd/kube-apiserver/app/options/validation.go index f8780610e67..63bc6fd6566 100644 --- a/cmd/kube-apiserver/app/options/validation.go +++ b/cmd/kube-apiserver/app/options/validation.go @@ -24,7 +24,6 @@ import ( genericoptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" - utilversion "k8s.io/component-base/version" netutils "k8s.io/utils/net" controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options" @@ -142,10 +141,5 @@ func (s CompletedOptions) Validate() []error { errs = append(errs, fmt.Errorf("--apiserver-count should be a positive number, but value '%d' provided", s.MasterCount)) } - effectiveVersion := s.GenericServerRunOptions.ComponentGlobalsRegistry.EffectiveVersionFor(s.GenericServerRunOptions.ComponentName) - if err := utilversion.ValidateKubeEffectiveVersion(effectiveVersion); err != nil { - errs = append(errs, err) - } - return errs } diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 8aa05a4d8f8..06a8ca81801 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -41,6 +41,7 @@ import ( "k8s.io/client-go/rest" cliflag "k8s.io/component-base/cli/flag" "k8s.io/component-base/cli/globalflag" + basecompatibility "k8s.io/component-base/compatibility" "k8s.io/component-base/featuregate" "k8s.io/component-base/logs" logsapi "k8s.io/component-base/logs/api/v1" @@ -64,10 +65,9 @@ func init() { // NewAPIServerCommand creates a *cobra.Command object with default parameters func NewAPIServerCommand() *cobra.Command { - _, featureGate := featuregate.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( - featuregate.DefaultKubeComponent, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) s := options.NewServerRunOptions() ctx := genericapiserver.SetupSignalContext() + featureGate := s.GenericServerRunOptions.ComponentGlobalsRegistry.FeatureGateFor(basecompatibility.DefaultKubeComponent) cmd := &cobra.Command{ Use: "kube-apiserver", @@ -79,7 +79,7 @@ cluster's shared state through which all other components interact.`, // stop printing usage when the command errors SilenceUsage: true, PersistentPreRunE: func(*cobra.Command, []string) error { - if err := featuregate.DefaultComponentGlobalsRegistry.Set(); err != nil { + if err := s.GenericServerRunOptions.ComponentGlobalsRegistry.Set(); err != nil { return err } // silence client-go warnings. @@ -108,7 +108,7 @@ cluster's shared state through which all other components interact.`, return utilerrors.NewAggregate(errs) } // add feature enablement metrics - featureGate.AddMetrics() + featureGate.(featuregate.MutableFeatureGate).AddMetrics() return Run(ctx, completedOptions) }, Args: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/kube-apiserver/app/testing/testserver.go b/cmd/kube-apiserver/app/testing/testserver.go index 685047ac10e..d7be4ad537e 100644 --- a/cmd/kube-apiserver/app/testing/testserver.go +++ b/cmd/kube-apiserver/app/testing/testserver.go @@ -43,22 +43,20 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/util/wait" serveroptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/pkg/storage/storagebackend" "k8s.io/apiserver/pkg/storageversion" + "k8s.io/apiserver/pkg/util/compatibility" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" clientgotransport "k8s.io/client-go/transport" "k8s.io/client-go/util/cert" "k8s.io/client-go/util/keyutil" - "k8s.io/component-base/featuregate" + basecompatibility "k8s.io/component-base/compatibility" featuregatetesting "k8s.io/component-base/featuregate/testing" logsapi "k8s.io/component-base/logs/api/v1" - utilversion "k8s.io/component-base/version" "k8s.io/klog/v2" "k8s.io/kube-aggregator/pkg/apiserver" "k8s.io/kubernetes/pkg/features" @@ -104,11 +102,8 @@ type TestServerInstanceOptions struct { // an apiserver version skew scenario where all apiservers use the same proxyCA to verify client connections. ProxyCA *ProxyCA // Set the BinaryVersion of server effective version. - // If empty, effective version will default to version.DefaultKubeBinaryVersion. + // If empty, effective version will default to DefaultKubeEffectiveVersion. BinaryVersion string - // Set the EmulationVersion of server effective version. - // If empty, emulation version will default to the effective version. - EmulationVersion string // Set non-default request timeout in the server. RequestTimeout time.Duration } @@ -194,21 +189,20 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions, fs := pflag.NewFlagSet("test", pflag.PanicOnError) - featureGate := utilfeature.DefaultMutableFeatureGate - featureGate.AddMetrics() - effectiveVersion := utilversion.DefaultKubeEffectiveVersion() + featureGate := utilfeature.DefaultMutableFeatureGate.DeepCopy() + effectiveVersion := compatibility.DefaultKubeEffectiveVersionForTest() if instanceOptions.BinaryVersion != "" { - effectiveVersion = utilversion.NewEffectiveVersion(instanceOptions.BinaryVersion) + effectiveVersion = basecompatibility.NewEffectiveVersionFromString(instanceOptions.BinaryVersion, "", "") } - if instanceOptions.EmulationVersion != "" { - effectiveVersion.SetEmulationVersion(version.MustParse(instanceOptions.EmulationVersion)) + effectiveVersion.SetEmulationVersion(featureGate.EmulationVersion()) + componentGlobalsRegistry := basecompatibility.NewComponentGlobalsRegistry() + if err := componentGlobalsRegistry.Register(basecompatibility.DefaultKubeComponent, effectiveVersion, featureGate); err != nil { + return result, err } - // need to call SetFeatureGateEmulationVersionDuringTest to reset the feature gate emulation version at the end of the test. - featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, featureGate, effectiveVersion.EmulationVersion()) - featuregate.DefaultComponentGlobalsRegistry.Reset() - utilruntime.Must(featuregate.DefaultComponentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, effectiveVersion, featureGate)) s := options.NewServerRunOptions() + // set up new instance of ComponentGlobalsRegistry instead of using the DefaultComponentGlobalsRegistry to avoid contention in parallel tests. + s.Options.GenericServerRunOptions.ComponentGlobalsRegistry = componentGlobalsRegistry if instanceOptions.RequestTimeout > 0 { s.GenericServerRunOptions.RequestTimeout = instanceOptions.RequestTimeout } @@ -330,15 +324,6 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions, return result, err } s.Authentication.ClientCert.ClientCA = clientCACertFile - if utilfeature.DefaultFeatureGate.Enabled(features.UnknownVersionInteroperabilityProxy) { - // TODO: set up a general clean up for testserver - if clientgotransport.DialerStopCh == wait.NeverStop { - ctx, cancel := context.WithTimeout(context.Background(), time.Hour) - t.Cleanup(cancel) - clientgotransport.DialerStopCh = ctx.Done() - } - s.PeerCAFile = filepath.Join(s.SecureServing.ServerCert.CertDirectory, s.SecureServing.ServerCert.PairName+".crt") - } } s.SecureServing.ExternalAddress = s.SecureServing.Listener.Addr().(*net.TCPAddr).IP // use listener addr although it is a loopback device @@ -374,8 +359,32 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions, s.Authentication.RequestHeader.ExtraHeaderPrefixes = extraHeaders } - if err := featuregate.DefaultComponentGlobalsRegistry.Set(); err != nil { - return result, err + if err := componentGlobalsRegistry.Set(); err != nil { + return result, fmt.Errorf("%w\nIf you are using SetFeatureGate*DuringTest, try using --emulated-version and --feature-gates flags instead", err) + } + // If the local ComponentGlobalsRegistry is changed by the flags, + // we need to copy the new feature values back to the DefaultFeatureGate because most feature checks still use the DefaultFeatureGate. + // We cannot directly use DefaultFeatureGate in ComponentGlobalsRegistry because the changes done by ComponentGlobalsRegistry.Set() will not be undone at the end of the test. + if !featureGate.EmulationVersion().EqualTo(utilfeature.DefaultMutableFeatureGate.EmulationVersion()) { + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultMutableFeatureGate, effectiveVersion.EmulationVersion()) + } + for f := range utilfeature.DefaultMutableFeatureGate.GetAll() { + if featureGate.Enabled(f) != utilfeature.DefaultFeatureGate.Enabled(f) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, f, featureGate.Enabled(f)) + } + } + utilfeature.DefaultMutableFeatureGate.AddMetrics() + + if instanceOptions.EnableCertAuth { + if featureGate.Enabled(features.UnknownVersionInteroperabilityProxy) { + // TODO: set up a general clean up for testserver + if clientgotransport.DialerStopCh == wait.NeverStop { + ctx, cancel := context.WithTimeout(context.Background(), time.Hour) + t.Cleanup(cancel) + clientgotransport.DialerStopCh = ctx.Done() + } + s.PeerCAFile = filepath.Join(s.SecureServing.ServerCert.CertDirectory, s.SecureServing.ServerCert.PairName+".crt") + } } saSigningKeyFile, err := os.CreateTemp("/tmp", "insecure_test_key") diff --git a/cmd/kube-controller-manager/app/config/config.go b/cmd/kube-controller-manager/app/config/config.go index 3034f288d91..feb25bfc7d5 100644 --- a/cmd/kube-controller-manager/app/config/config.go +++ b/cmd/kube-controller-manager/app/config/config.go @@ -21,6 +21,7 @@ import ( clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" + basecompatibility "k8s.io/component-base/compatibility" kubectrlmgrconfig "k8s.io/kubernetes/pkg/controller/apis/config" ) @@ -43,6 +44,9 @@ type Config struct { EventBroadcaster record.EventBroadcaster EventRecorder record.EventRecorder + + // ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored. + ComponentGlobalsRegistry basecompatibility.ComponentGlobalsRegistry } type completedConfig struct { diff --git a/cmd/kube-controller-manager/app/controllermanager.go b/cmd/kube-controller-manager/app/controllermanager.go index 733157a80d1..d209b057136 100644 --- a/cmd/kube-controller-manager/app/controllermanager.go +++ b/cmd/kube-controller-manager/app/controllermanager.go @@ -54,6 +54,7 @@ import ( "k8s.io/client-go/util/keyutil" cliflag "k8s.io/component-base/cli/flag" "k8s.io/component-base/cli/globalflag" + basecompatibility "k8s.io/component-base/compatibility" "k8s.io/component-base/configz" "k8s.io/component-base/featuregate" "k8s.io/component-base/logs" @@ -99,9 +100,6 @@ const ( // NewControllerManagerCommand creates a *cobra.Command object with default parameters func NewControllerManagerCommand() *cobra.Command { - _, _ = featuregate.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( - featuregate.DefaultKubeComponent, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) - s, err := options.NewKubeControllerManagerOptions() if err != nil { klog.Background().Error(err, "Unable to initialize command options") @@ -142,7 +140,7 @@ controller, and serviceaccounts controller.`, } // add feature enablement metrics - fg := s.ComponentGlobalsRegistry.FeatureGateFor(featuregate.DefaultKubeComponent) + fg := s.ComponentGlobalsRegistry.FeatureGateFor(basecompatibility.DefaultKubeComponent) fg.(featuregate.MutableFeatureGate).AddMetrics() return Run(context.Background(), c.Complete()) }, @@ -218,7 +216,7 @@ func Run(ctx context.Context, c *config.CompletedConfig) error { slis.SLIMetricsWithReset{}.Install(unsecuredMux) if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentStatusz) { - statusz.Install(unsecuredMux, kubeControllerManager, statusz.NewRegistry()) + statusz.Install(unsecuredMux, kubeControllerManager, statusz.NewRegistry(c.ComponentGlobalsRegistry.EffectiveVersionFor(basecompatibility.DefaultKubeComponent))) } handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, &c.Authorization, &c.Authentication) @@ -289,11 +287,11 @@ func Run(ctx context.Context, c *config.CompletedConfig) error { } if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.CoordinatedLeaderElection) { - binaryVersion, err := semver.ParseTolerant(featuregate.DefaultComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent).BinaryVersion().String()) + binaryVersion, err := semver.ParseTolerant(c.ComponentGlobalsRegistry.EffectiveVersionFor(basecompatibility.DefaultKubeComponent).BinaryVersion().String()) if err != nil { return err } - emulationVersion, err := semver.ParseTolerant(featuregate.DefaultComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent).EmulationVersion().String()) + emulationVersion, err := semver.ParseTolerant(c.ComponentGlobalsRegistry.EffectiveVersionFor(basecompatibility.DefaultKubeComponent).EmulationVersion().String()) if err != nil { return err } diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index 7556d946922..e046fb4a200 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -25,6 +25,7 @@ import ( utilerrors "k8s.io/apimachinery/pkg/util/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" apiserveroptions "k8s.io/apiserver/pkg/server/options" + "k8s.io/apiserver/pkg/util/compatibility" utilfeature "k8s.io/apiserver/pkg/util/feature" clientgofeaturegate "k8s.io/client-go/features" clientset "k8s.io/client-go/kubernetes" @@ -36,11 +37,11 @@ import ( cpnames "k8s.io/cloud-provider/names" cpoptions "k8s.io/cloud-provider/options" cliflag "k8s.io/component-base/cli/flag" + basecompatibility "k8s.io/component-base/compatibility" "k8s.io/component-base/featuregate" "k8s.io/component-base/logs" logsapi "k8s.io/component-base/logs/api/v1" "k8s.io/component-base/metrics" - utilversion "k8s.io/component-base/version" cmoptions "k8s.io/controller-manager/options" "k8s.io/klog/v2" kubectrlmgrconfigv1alpha1 "k8s.io/kube-controller-manager/config/v1alpha1" @@ -106,7 +107,7 @@ type KubeControllerManagerOptions struct { ShowHiddenMetricsForVersion string // ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored. - ComponentGlobalsRegistry featuregate.ComponentGlobalsRegistry + ComponentGlobalsRegistry basecompatibility.ComponentGlobalsRegistry } // NewKubeControllerManagerOptions creates a new KubeControllerManagerOptions with a default config. @@ -116,10 +117,12 @@ func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) { return nil, err } - if featuregate.DefaultComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent) == nil { + componentGlobalsRegistry := compatibility.DefaultComponentGlobalsRegistry + + if componentGlobalsRegistry.EffectiveVersionFor(basecompatibility.DefaultKubeComponent) == nil { featureGate := utilfeature.DefaultMutableFeatureGate - effectiveVersion := utilversion.DefaultKubeEffectiveVersion() - utilruntime.Must(featuregate.DefaultComponentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, effectiveVersion, featureGate)) + effectiveVersion := compatibility.DefaultBuildEffectiveVersion() + utilruntime.Must(componentGlobalsRegistry.Register(basecompatibility.DefaultKubeComponent, effectiveVersion, featureGate)) } s := KubeControllerManagerOptions{ @@ -211,7 +214,7 @@ func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) { Authorization: apiserveroptions.NewDelegatingAuthorizationOptions(), Metrics: metrics.NewOptions(), Logs: logs.NewOptions(), - ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry, + ComponentGlobalsRegistry: componentGlobalsRegistry, } s.Authentication.RemoteKubeConfigFileOptional = true @@ -230,7 +233,6 @@ func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) { s.GarbageCollectorController.GCIgnoredResources = gcIgnoredResources s.Generic.LeaderElection.ResourceName = "kube-controller-manager" s.Generic.LeaderElection.ResourceNamespace = "kube-system" - return &s, nil } @@ -452,7 +454,6 @@ func (s *KubeControllerManagerOptions) Validate(allControllers []string, disable errs = append(errs, s.Authentication.Validate()...) errs = append(errs, s.Authorization.Validate()...) errs = append(errs, s.Metrics.Validate()...) - errs = append(errs, utilversion.ValidateKubeEffectiveVersion(s.ComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent))) // in-tree cloud providers are disabled since v1.31 (KEP-2395) if len(s.KubeCloudShared.CloudProvider.Name) > 0 && !cloudprovider.IsExternal(s.KubeCloudShared.CloudProvider.Name) { @@ -498,10 +499,11 @@ func (s KubeControllerManagerOptions) Config(allControllers []string, disabledBy eventRecorder := eventBroadcaster.NewRecorder(clientgokubescheme.Scheme, v1.EventSource{Component: KubeControllerManagerUserAgent}) c := &kubecontrollerconfig.Config{ - Client: client, - Kubeconfig: kubeconfig, - EventBroadcaster: eventBroadcaster, - EventRecorder: eventRecorder, + Client: client, + Kubeconfig: kubeconfig, + EventBroadcaster: eventBroadcaster, + EventRecorder: eventRecorder, + ComponentGlobalsRegistry: s.ComponentGlobalsRegistry, } if err := s.ApplyTo(c, allControllers, disabledByDefaultControllers, controllerAliases); err != nil { return nil, err diff --git a/cmd/kube-controller-manager/app/options/options_test.go b/cmd/kube-controller-manager/app/options/options_test.go index 33ba650e603..3c1ce4e39f4 100644 --- a/cmd/kube-controller-manager/app/options/options_test.go +++ b/cmd/kube-controller-manager/app/options/options_test.go @@ -34,7 +34,7 @@ import ( "k8s.io/apiserver/pkg/apis/apiserver" apiserveroptions "k8s.io/apiserver/pkg/server/options" - utilversion "k8s.io/component-base/version" + basecompatibility "k8s.io/component-base/compatibility" componentbaseconfig "k8s.io/component-base/config" "k8s.io/component-base/featuregate" @@ -447,10 +447,11 @@ func TestAddFlags(t *testing.T) { AlwaysAllowPaths: []string{"/healthz", "/readyz", "/livez"}, // note: this does not match /healthz/ or /healthz/* AlwaysAllowGroups: []string{"system:masters"}, }, - Master: "192.168.4.20", - Metrics: &metrics.Options{}, - Logs: logs.NewOptions(), - ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry, + Master: "192.168.4.20", + Metrics: &metrics.Options{}, + Logs: logs.NewOptions(), + // ignores comparing ComponentGlobalsRegistry in this test. + ComponentGlobalsRegistry: s.ComponentGlobalsRegistry, } // Sort GCIgnoredResources because it's built from a map, which means the @@ -736,27 +737,6 @@ func TestApplyTo(t *testing.T) { } func TestEmulatedVersion(t *testing.T) { - var cleanupAndSetupFunc = func() featuregate.FeatureGate { - componentGlobalsRegistry := featuregate.DefaultComponentGlobalsRegistry - componentGlobalsRegistry.Reset() // make sure this test have a clean state - t.Cleanup(func() { - componentGlobalsRegistry.Reset() // make sure this test doesn't leak a dirty state - }) - - verKube := utilversion.NewEffectiveVersion("1.32") - fg := featuregate.NewVersionedFeatureGate(version.MustParse("1.32")) - utilruntime.Must(fg.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{ - "kubeA": { - {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Beta}, - {Version: version.MustParse("1.32"), Default: true, LockToDefault: true, PreRelease: featuregate.GA}, - }, - "kubeB": { - {Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Alpha}, - }, - })) - utilruntime.Must(componentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, verKube, fg)) - return fg - } testcases := []struct { name string @@ -808,9 +788,8 @@ func TestEmulatedVersion(t *testing.T) { for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - fg := cleanupAndSetupFunc() - fs, s := setupControllerManagerFlagSet(t) + fg := s.ComponentGlobalsRegistry.FeatureGateFor(basecompatibility.DefaultKubeComponent) err := fs.Parse(tc.flags) checkTestError(t, err, false, "") err = s.Validate([]string{""}, []string{""}, nil) @@ -1558,6 +1537,22 @@ func setupControllerManagerFlagSet(t *testing.T) (*pflag.FlagSet, *KubeControlle t.Fatal(fmt.Errorf("NewKubeControllerManagerOptions failed with %w", err)) } + componentGlobalsRegistry := basecompatibility.NewComponentGlobalsRegistry() + + verKube := basecompatibility.NewEffectiveVersionFromString("1.32", "1.31", "1.31") + fg := featuregate.NewVersionedFeatureGate(version.MustParse("1.32")) + utilruntime.Must(fg.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{ + "kubeA": { + {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.32"), Default: true, LockToDefault: true, PreRelease: featuregate.GA}, + }, + "kubeB": { + {Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Alpha}, + }, + })) + utilruntime.Must(componentGlobalsRegistry.Register(basecompatibility.DefaultKubeComponent, verKube, fg)) + s.ComponentGlobalsRegistry = componentGlobalsRegistry + for _, f := range s.Flags([]string{""}, []string{""}, nil).FlagSets { fs.AddFlagSet(f) } diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index 789a7a7cb09..02234ae5916 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -42,6 +42,7 @@ import ( "k8s.io/apiserver/pkg/server/healthz" "k8s.io/apiserver/pkg/server/mux" "k8s.io/apiserver/pkg/server/routes" + "k8s.io/apiserver/pkg/util/compatibility" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" @@ -474,7 +475,7 @@ func serveMetrics(ctx context.Context, bindAddress string, proxyMode kubeproxyco } if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentStatusz) { - statusz.Install(proxyMux, kubeProxy, statusz.NewRegistry()) + statusz.Install(proxyMux, kubeProxy, statusz.NewRegistry(compatibility.DefaultBuildEffectiveVersion())) } fn := func() { diff --git a/cmd/kube-scheduler/app/config/config.go b/cmd/kube-scheduler/app/config/config.go index 398349d70fe..6e4b23d11dc 100644 --- a/cmd/kube-scheduler/app/config/config.go +++ b/cmd/kube-scheduler/app/config/config.go @@ -26,6 +26,7 @@ import ( restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/events" "k8s.io/client-go/tools/leaderelection" + basecompatibility "k8s.io/component-base/compatibility" "k8s.io/component-base/zpages/flagz" kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" ) @@ -61,6 +62,9 @@ type Config struct { // value, the pod will be moved from unschedulablePods to backoffQ or activeQ. // If this value is empty, the default value (5min) will be used. PodMaxInUnschedulablePodsDuration time.Duration + + // ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored. + ComponentGlobalsRegistry basecompatibility.ComponentGlobalsRegistry } type completedConfig struct { diff --git a/cmd/kube-scheduler/app/options/options.go b/cmd/kube-scheduler/app/options/options.go index eb92adbeef0..70cc1ed4110 100644 --- a/cmd/kube-scheduler/app/options/options.go +++ b/cmd/kube-scheduler/app/options/options.go @@ -28,6 +28,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/uuid" apiserveroptions "k8s.io/apiserver/pkg/server/options" + "k8s.io/apiserver/pkg/util/compatibility" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/dynamic" "k8s.io/client-go/dynamic/dynamicinformer" @@ -39,13 +40,12 @@ import ( "k8s.io/client-go/tools/leaderelection/resourcelock" "k8s.io/client-go/tools/record" cliflag "k8s.io/component-base/cli/flag" + basecompatibility "k8s.io/component-base/compatibility" componentbaseconfig "k8s.io/component-base/config" "k8s.io/component-base/config/options" - "k8s.io/component-base/featuregate" "k8s.io/component-base/logs" logsapi "k8s.io/component-base/logs/api/v1" "k8s.io/component-base/metrics" - utilversion "k8s.io/component-base/version" zpagesfeatures "k8s.io/component-base/zpages/features" "k8s.io/component-base/zpages/flagz" "k8s.io/klog/v2" @@ -78,7 +78,7 @@ type Options struct { Master string // ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored. - ComponentGlobalsRegistry featuregate.ComponentGlobalsRegistry + ComponentGlobalsRegistry basecompatibility.ComponentGlobalsRegistry // Flags hold the parsed CLI flags. Flags *cliflag.NamedFlagSets @@ -86,12 +86,17 @@ type Options struct { // NewOptions returns default scheduler app options. func NewOptions() *Options { + componentGlobalsRegistry := compatibility.DefaultComponentGlobalsRegistry // make sure DefaultKubeComponent is registered in the DefaultComponentGlobalsRegistry. - if featuregate.DefaultComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent) == nil { + if componentGlobalsRegistry.EffectiveVersionFor(basecompatibility.DefaultKubeComponent) == nil { featureGate := utilfeature.DefaultMutableFeatureGate - effectiveVersion := utilversion.DefaultKubeEffectiveVersion() - utilruntime.Must(featuregate.DefaultComponentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, effectiveVersion, featureGate)) + effectiveVersion := compatibility.DefaultBuildEffectiveVersion() + utilruntime.Must(componentGlobalsRegistry.Register(basecompatibility.DefaultKubeComponent, effectiveVersion, featureGate)) } + return NewOptionsWithComponentGlobalsRegistry(componentGlobalsRegistry) +} + +func NewOptionsWithComponentGlobalsRegistry(componentGlobalsRegistry basecompatibility.ComponentGlobalsRegistry) *Options { o := &Options{ SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(), Authentication: apiserveroptions.NewDelegatingAuthenticationOptions(), @@ -110,7 +115,7 @@ func NewOptions() *Options { }, Metrics: metrics.NewOptions(), Logs: logs.NewOptions(), - ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry, + ComponentGlobalsRegistry: componentGlobalsRegistry, } o.Authentication.TolerateInClusterLookupFailure = true @@ -287,11 +292,6 @@ func (o *Options) Validate() []error { errs = append(errs, o.Authorization.Validate()...) errs = append(errs, o.Metrics.Validate()...) - effectiveVersion := o.ComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent) - if err := utilversion.ValidateKubeEffectiveVersion(effectiveVersion); err != nil { - errs = append(errs, err) - } - return errs } @@ -337,6 +337,7 @@ func (o *Options) Config(ctx context.Context) (*schedulerappconfig.Config, error dynClient := dynamic.NewForConfigOrDie(c.KubeConfig) c.DynInformerFactory = dynamicinformer.NewFilteredDynamicSharedInformerFactory(dynClient, 0, corev1.NamespaceAll, nil) c.LeaderElection = leaderElectionConfig + c.ComponentGlobalsRegistry = o.ComponentGlobalsRegistry return c, nil } diff --git a/cmd/kube-scheduler/app/options/options_test.go b/cmd/kube-scheduler/app/options/options_test.go index ee60e90e2c5..aa56926a1a5 100644 --- a/cmd/kube-scheduler/app/options/options_test.go +++ b/cmd/kube-scheduler/app/options/options_test.go @@ -32,8 +32,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" apiserveroptions "k8s.io/apiserver/pkg/server/options" + basecompatibility "k8s.io/component-base/compatibility" componentbaseconfig "k8s.io/component-base/config" - "k8s.io/component-base/featuregate" "k8s.io/component-base/logs" "k8s.io/klog/v2/ktesting" v1 "k8s.io/kube-scheduler/config/v1" @@ -282,6 +282,8 @@ profiles: defaultPodMaxBackoffSeconds := int64(10) defaultPercentageOfNodesToScore := ptr.To[int32](0) + componentGlobalsRegistry := basecompatibility.NewComponentGlobalsRegistry() + testcases := []struct { name string options *Options @@ -323,7 +325,7 @@ profiles: AlwaysAllowGroups: []string{"system:masters"}, }, Logs: logs.NewOptions(), - ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry, + ComponentGlobalsRegistry: componentGlobalsRegistry, }, expectedUsername: "config", expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ @@ -375,7 +377,7 @@ profiles: return cfg }(), Logs: logs.NewOptions(), - ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry, + ComponentGlobalsRegistry: componentGlobalsRegistry, }, expectedError: "no kind \"KubeSchedulerConfiguration\" is registered for version \"componentconfig/v1alpha1\"", }, @@ -384,7 +386,7 @@ profiles: options: &Options{ ConfigFile: unknownVersionConfig, Logs: logs.NewOptions(), - ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry, + ComponentGlobalsRegistry: componentGlobalsRegistry, }, expectedError: "no kind \"KubeSchedulerConfiguration\" is registered for version \"kubescheduler.config.k8s.io/unknown\"", }, @@ -393,7 +395,7 @@ profiles: options: &Options{ ConfigFile: noVersionConfig, Logs: logs.NewOptions(), - ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry, + ComponentGlobalsRegistry: componentGlobalsRegistry, }, expectedError: "Object 'apiVersion' is missing", }, @@ -430,7 +432,7 @@ profiles: AlwaysAllowGroups: []string{"system:masters"}, }, Logs: logs.NewOptions(), - ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry, + ComponentGlobalsRegistry: componentGlobalsRegistry, }, expectedUsername: "flag", expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ @@ -503,7 +505,7 @@ profiles: AlwaysAllowGroups: []string{"system:masters"}, }, Logs: logs.NewOptions(), - ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry, + ComponentGlobalsRegistry: componentGlobalsRegistry, }, expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ TypeMeta: metav1.TypeMeta{ @@ -548,7 +550,7 @@ profiles: options: &Options{ ConfigFile: pluginConfigFile, Logs: logs.NewOptions(), - ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry, + ComponentGlobalsRegistry: componentGlobalsRegistry, }, expectedUsername: "config", expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ @@ -669,7 +671,7 @@ profiles: options: &Options{ ConfigFile: multiProfilesConfig, Logs: logs.NewOptions(), - ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry, + ComponentGlobalsRegistry: componentGlobalsRegistry, }, expectedUsername: "config", expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ @@ -784,7 +786,7 @@ profiles: name: "no config", options: &Options{ Logs: logs.NewOptions(), - ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry, + ComponentGlobalsRegistry: componentGlobalsRegistry, }, expectedError: "no configuration has been provided", }, @@ -793,7 +795,7 @@ profiles: options: &Options{ ConfigFile: unknownFieldConfig, Logs: logs.NewOptions(), - ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry, + ComponentGlobalsRegistry: componentGlobalsRegistry, }, expectedError: `unknown field "foo"`, checkErrFn: runtime.IsStrictDecodingError, @@ -803,7 +805,7 @@ profiles: options: &Options{ ConfigFile: duplicateFieldConfig, Logs: logs.NewOptions(), - ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry, + ComponentGlobalsRegistry: componentGlobalsRegistry, }, expectedError: `key "leaderElect" already set`, checkErrFn: runtime.IsStrictDecodingError, @@ -813,7 +815,7 @@ profiles: options: &Options{ ConfigFile: highThroughputProfileConfig, Logs: logs.NewOptions(), - ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry, + ComponentGlobalsRegistry: componentGlobalsRegistry, }, expectedUsername: "config", expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index 0a1809504c8..a7922dd684c 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -46,6 +46,7 @@ import ( "k8s.io/client-go/tools/leaderelection" cliflag "k8s.io/component-base/cli/flag" "k8s.io/component-base/cli/globalflag" + basecompatibility "k8s.io/component-base/compatibility" "k8s.io/component-base/configz" "k8s.io/component-base/featuregate" "k8s.io/component-base/logs" @@ -84,10 +85,6 @@ type Option func(runtime.Registry) error // NewSchedulerCommand creates a *cobra.Command object with default parameters and registryOptions func NewSchedulerCommand(registryOptions ...Option) *cobra.Command { - // explicitly register (if not already registered) the kube effective version and feature gate in DefaultComponentGlobalsRegistry, - // which will be used in NewOptions. - _, _ = featuregate.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( - featuregate.DefaultKubeComponent, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) opts := options.NewOptions() cmd := &cobra.Command{ @@ -138,7 +135,7 @@ for more information about scheduling and the kube-scheduler component.`, // runCommand runs the scheduler. func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Option) error { verflag.PrintAndExitIfRequested() - fg := opts.ComponentGlobalsRegistry.FeatureGateFor(featuregate.DefaultKubeComponent) + fg := opts.ComponentGlobalsRegistry.FeatureGateFor(basecompatibility.DefaultKubeComponent) // Activate logging as soon as possible, after that // show flags with the final logging configuration. if err := logsapi.ValidateAndApply(opts.Logs, fg); err != nil { @@ -216,11 +213,11 @@ func Run(ctx context.Context, cc *schedulerserverconfig.CompletedConfig, sched * readyzChecks = append(readyzChecks, handlerSyncCheck) if cc.LeaderElection != nil && utilfeature.DefaultFeatureGate.Enabled(kubefeatures.CoordinatedLeaderElection) { - binaryVersion, err := semver.ParseTolerant(featuregate.DefaultComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent).BinaryVersion().String()) + binaryVersion, err := semver.ParseTolerant(cc.ComponentGlobalsRegistry.EffectiveVersionFor(basecompatibility.DefaultKubeComponent).BinaryVersion().String()) if err != nil { return err } - emulationVersion, err := semver.ParseTolerant(featuregate.DefaultComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent).EmulationVersion().String()) + emulationVersion, err := semver.ParseTolerant(cc.ComponentGlobalsRegistry.EffectiveVersionFor(basecompatibility.DefaultKubeComponent).EmulationVersion().String()) if err != nil { return err } diff --git a/cmd/kube-scheduler/app/server_test.go b/cmd/kube-scheduler/app/server_test.go index 98bc99c6a72..aa5f5e50bfb 100644 --- a/cmd/kube-scheduler/app/server_test.go +++ b/cmd/kube-scheduler/app/server_test.go @@ -36,10 +36,10 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/version" "k8s.io/apiserver/pkg/util/feature" + basecompatibility "k8s.io/component-base/compatibility" componentbaseconfig "k8s.io/component-base/config" "k8s.io/component-base/featuregate" featuregatetesting "k8s.io/component-base/featuregate/testing" - utilversion "k8s.io/component-base/version" configv1 "k8s.io/kube-scheduler/config/v1" "k8s.io/kubernetes/cmd/kube-scheduler/app/options" "k8s.io/kubernetes/pkg/features" @@ -438,12 +438,8 @@ leaderElection: for k, v := range tc.restoreFeatures { featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, k, v) } - componentGlobalsRegistry := featuregate.DefaultComponentGlobalsRegistry - t.Cleanup(func() { - componentGlobalsRegistry.Reset() - }) - componentGlobalsRegistry.Reset() - verKube := utilversion.NewEffectiveVersion("1.32") + componentGlobalsRegistry := basecompatibility.NewComponentGlobalsRegistry() + verKube := basecompatibility.NewEffectiveVersionFromString("1.32", "1.31", "1.31") fg := feature.DefaultFeatureGate.DeepCopy() utilruntime.Must(fg.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{ "kubeA": { @@ -454,10 +450,10 @@ leaderElection: {Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Alpha}, }, })) - utilruntime.Must(componentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, verKube, fg)) + utilruntime.Must(componentGlobalsRegistry.Register(basecompatibility.DefaultKubeComponent, verKube, fg)) fs := pflag.NewFlagSet("test", pflag.PanicOnError) - opts := options.NewOptions() + opts := options.NewOptionsWithComponentGlobalsRegistry(componentGlobalsRegistry) // use listeners instead of static ports so parallel test runs don't conflict opts.SecureServing.Listener = makeListener(t) diff --git a/pkg/controlplane/apiserver/config.go b/pkg/controlplane/apiserver/config.go index 1dab17042fa..7a1b42ee127 100644 --- a/pkg/controlplane/apiserver/config.go +++ b/pkg/controlplane/apiserver/config.go @@ -189,8 +189,7 @@ func BuildGenericConfig( s.Etcd.StorageConfig.Transport.TracerProvider = noopoteltrace.NewTracerProvider() } - storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig() - storageFactoryConfig.CurrentVersion = genericConfig.EffectiveVersion + storageFactoryConfig := kubeapiserver.NewStorageFactoryConfigEffectiveVersion(genericConfig.EffectiveVersion) storageFactoryConfig.APIResourceConfig = genericConfig.MergedResourceConfig storageFactoryConfig.DefaultResourceEncoding.SetEffectiveVersion(genericConfig.EffectiveVersion) storageFactory, lastErr = storageFactoryConfig.Complete(s.Etcd).New() diff --git a/pkg/controlplane/apiserver/options/options_test.go b/pkg/controlplane/apiserver/options/options_test.go index 55e30fafcce..4f202860842 100644 --- a/pkg/controlplane/apiserver/options/options_test.go +++ b/pkg/controlplane/apiserver/options/options_test.go @@ -40,10 +40,10 @@ import ( auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered" audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate" cliflag "k8s.io/component-base/cli/flag" + basecompatibility "k8s.io/component-base/compatibility" "k8s.io/component-base/featuregate" "k8s.io/component-base/logs" "k8s.io/component-base/metrics" - utilversion "k8s.io/component-base/version" kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" "k8s.io/kubernetes/pkg/serviceaccount" v1alpha1testing "k8s.io/kubernetes/pkg/serviceaccount/externaljwt/plugin/testing/v1alpha1" @@ -51,13 +51,11 @@ import ( ) func TestAddFlags(t *testing.T) { - componentGlobalsRegistry := featuregate.DefaultComponentGlobalsRegistry - t.Cleanup(func() { - componentGlobalsRegistry.Reset() - }) + componentGlobalsRegistry := basecompatibility.NewComponentGlobalsRegistry() fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError) - utilruntime.Must(componentGlobalsRegistry.Register("test", utilversion.NewEffectiveVersion("1.32"), featuregate.NewFeatureGate())) + utilruntime.Must(componentGlobalsRegistry.Register("test", basecompatibility.NewEffectiveVersionFromString("1.32", "1.31", "1.31"), featuregate.NewFeatureGate())) s := NewOptions() + s.GenericServerRunOptions.ComponentGlobalsRegistry = componentGlobalsRegistry var fss cliflag.NamedFlagSets s.AddFlags(&fss) for _, f := range fss.FlagSets { @@ -141,7 +139,7 @@ func TestAddFlags(t *testing.T) { JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024), MaxRequestBodyBytes: int64(3 * 1024 * 1024), ComponentGlobalsRegistry: componentGlobalsRegistry, - ComponentName: featuregate.DefaultKubeComponent, + ComponentName: basecompatibility.DefaultKubeComponent, }, Admission: &kubeoptions.AdmissionOptions{ GenericAdmission: &apiserveroptions.AdmissionOptions{ diff --git a/pkg/controlplane/apiserver/options/validation_test.go b/pkg/controlplane/apiserver/options/validation_test.go index 3ce550eb498..2ff582b6e77 100644 --- a/pkg/controlplane/apiserver/options/validation_test.go +++ b/pkg/controlplane/apiserver/options/validation_test.go @@ -25,6 +25,7 @@ import ( kubeapiserveradmission "k8s.io/apiserver/pkg/admission" genericoptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" + basecompatibility "k8s.io/component-base/compatibility" "k8s.io/component-base/featuregate" basemetrics "k8s.io/component-base/metrics" "k8s.io/kubernetes/pkg/features" @@ -202,7 +203,7 @@ func TestValidateOptions(t *testing.T) { name: "validate master count equal 0", expectErrors: true, options: &Options{ - GenericServerRunOptions: &genericoptions.ServerRunOptions{ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry}, + GenericServerRunOptions: &genericoptions.ServerRunOptions{ComponentGlobalsRegistry: basecompatibility.NewComponentGlobalsRegistry()}, Etcd: &genericoptions.EtcdOptions{}, SecureServing: &genericoptions.SecureServingOptionsWithLoopback{}, Audit: &genericoptions.AuditOptions{}, @@ -229,7 +230,7 @@ func TestValidateOptions(t *testing.T) { name: "validate token request enable not attempted", expectErrors: true, options: &Options{ - GenericServerRunOptions: &genericoptions.ServerRunOptions{ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry}, + GenericServerRunOptions: &genericoptions.ServerRunOptions{ComponentGlobalsRegistry: basecompatibility.NewComponentGlobalsRegistry()}, Etcd: &genericoptions.EtcdOptions{}, SecureServing: &genericoptions.SecureServingOptionsWithLoopback{}, Audit: &genericoptions.AuditOptions{}, diff --git a/pkg/controlplane/apiserver/server.go b/pkg/controlplane/apiserver/server.go index dfeea62cf8b..dd36d9977f7 100644 --- a/pkg/controlplane/apiserver/server.go +++ b/pkg/controlplane/apiserver/server.go @@ -161,7 +161,7 @@ func (c completedConfig) New(name string, delegationTarget genericapiserver.Dele } if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentStatusz) { - statusz.Install(s.GenericAPIServer.Handler.NonGoRestfulMux, name, statusz.NewRegistry()) + statusz.Install(s.GenericAPIServer.Handler.NonGoRestfulMux, name, statusz.NewRegistry(c.Generic.EffectiveVersion)) } if utilfeature.DefaultFeatureGate.Enabled(apiserverfeatures.CoordinatedLeaderElection) { diff --git a/pkg/controlplane/instance_test.go b/pkg/controlplane/instance_test.go index 06ac492d0c9..81437588298 100644 --- a/pkg/controlplane/instance_test.go +++ b/pkg/controlplane/instance_test.go @@ -55,6 +55,7 @@ import ( "k8s.io/apiserver/pkg/server/resourceconfig" serverstorage "k8s.io/apiserver/pkg/server/storage" etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing" + "k8s.io/apiserver/pkg/util/compatibility" "k8s.io/apiserver/pkg/util/openapi" "k8s.io/client-go/discovery" "k8s.io/client-go/informers" @@ -103,7 +104,7 @@ func setUp(t *testing.T) (*etcd3testing.EtcdTestServer, Config, *assert.Assertio }, } - config.ControlPlane.Generic.EffectiveVersion = utilversion.DefaultKubeEffectiveVersion() + config.ControlPlane.Generic.EffectiveVersion = compatibility.DefaultKubeEffectiveVersionForTest() storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig() storageFactoryConfig.DefaultResourceEncoding.SetEffectiveVersion(config.ControlPlane.Generic.EffectiveVersion) storageConfig.StorageObjectCountTracker = config.ControlPlane.Generic.StorageObjectCountTracker @@ -241,7 +242,7 @@ func TestVersion(t *testing.T) { t.Errorf("unexpected error: %v", err) } expectedInfo := utilversion.Get() - kubeVersion := utilversion.DefaultKubeEffectiveVersion().BinaryVersion() + kubeVersion := compatibility.DefaultKubeEffectiveVersionForTest().BinaryVersion() expectedInfo.Major = fmt.Sprintf("%d", kubeVersion.Major()) expectedInfo.Minor = fmt.Sprintf("%d", kubeVersion.Minor()) diff --git a/pkg/kubeapiserver/default_storage_factory_builder.go b/pkg/kubeapiserver/default_storage_factory_builder.go index 044164ea158..e9d57598f8c 100644 --- a/pkg/kubeapiserver/default_storage_factory_builder.go +++ b/pkg/kubeapiserver/default_storage_factory_builder.go @@ -25,7 +25,8 @@ import ( "k8s.io/apiserver/pkg/server/resourceconfig" serverstorage "k8s.io/apiserver/pkg/server/storage" "k8s.io/apiserver/pkg/storage/storagebackend" - version "k8s.io/component-base/version" + "k8s.io/apiserver/pkg/util/compatibility" + basecompatibility "k8s.io/component-base/compatibility" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/admissionregistration" "k8s.io/kubernetes/pkg/apis/apps" @@ -61,6 +62,11 @@ func DefaultWatchCacheSizes() map[schema.GroupResource]int { // NewStorageFactoryConfig returns a new StorageFactoryConfig set up with necessary resource overrides. func NewStorageFactoryConfig() *StorageFactoryConfig { + return NewStorageFactoryConfigEffectiveVersion(compatibility.DefaultComponentGlobalsRegistry.EffectiveVersionFor(basecompatibility.DefaultKubeComponent)) +} + +// NewStorageFactoryConfigEffectiveVersion returns a new StorageFactoryConfig set up with necessary resource overrides for a given EffectiveVersion. +func NewStorageFactoryConfigEffectiveVersion(effectiveVersion basecompatibility.EffectiveVersion) *StorageFactoryConfig { resources := []schema.GroupVersionResource{ // If a resource has to be stored in a version that is not the // latest, then it can be listed here. Usually this is the case @@ -87,7 +93,7 @@ func NewStorageFactoryConfig() *StorageFactoryConfig { return &StorageFactoryConfig{ Serializer: legacyscheme.Codecs, - DefaultResourceEncoding: serverstorage.NewDefaultResourceEncodingConfig(legacyscheme.Scheme), + DefaultResourceEncoding: serverstorage.NewDefaultResourceEncodingConfigForEffectiveVersion(legacyscheme.Scheme, effectiveVersion), ResourceEncodingOverrides: resources, } } @@ -101,7 +107,6 @@ type StorageFactoryConfig struct { Serializer runtime.StorageSerializer ResourceEncodingOverrides []schema.GroupVersionResource EtcdServersOverrides []string - CurrentVersion version.EffectiveVersion } // Complete completes the StorageFactoryConfig with provided etcdOptions returning completedStorageFactoryConfig. diff --git a/pkg/kubelet/server/server.go b/pkg/kubelet/server/server.go index 8ba83872c7f..73a36b5b56b 100644 --- a/pkg/kubelet/server/server.go +++ b/pkg/kubelet/server/server.go @@ -59,6 +59,7 @@ import ( "k8s.io/apiserver/pkg/server/healthz" "k8s.io/apiserver/pkg/server/httplog" "k8s.io/apiserver/pkg/server/routes" + "k8s.io/apiserver/pkg/util/compatibility" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/apiserver/pkg/util/flushwriter" "k8s.io/component-base/configz" @@ -565,7 +566,7 @@ func (s *Server) InstallDebuggingHandlers() { if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentStatusz) { s.addMetricsBucketMatcher("statusz") - statusz.Install(s.restfulCont, ComponentKubelet, statusz.NewRegistry()) + statusz.Install(s.restfulCont, ComponentKubelet, statusz.NewRegistry(compatibility.DefaultBuildEffectiveVersion())) } // The /runningpods endpoint is used for testing only. diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go index 054f16ac622..4f35b127af0 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go @@ -24,7 +24,6 @@ import ( "k8s.io/apiextensions-apiserver/pkg/cmd/server/options" genericapiserver "k8s.io/apiserver/pkg/server" - "k8s.io/component-base/featuregate" ) func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command { @@ -34,7 +33,7 @@ func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command Short: "Launch an API extensions API server", Long: "Launch an API extensions API server", PersistentPreRunE: func(*cobra.Command, []string) error { - return featuregate.DefaultComponentGlobalsRegistry.Set() + return o.ServerRunOptions.ComponentGlobalsRegistry.Set() }, RunE: func(c *cobra.Command, args []string) error { if err := o.Complete(); err != nil { diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go index 6bc05f9fc9d..46a68e29da2 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go @@ -31,18 +31,18 @@ import ( extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" "k8s.io/apiextensions-apiserver/pkg/cmd/server/options" generatedopenapi "k8s.io/apiextensions-apiserver/pkg/generated/openapi" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/storage/storagebackend" + "k8s.io/apiserver/pkg/util/compatibility" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/apiserver/pkg/util/openapi" "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" - "k8s.io/component-base/featuregate" + basecompatibility "k8s.io/component-base/compatibility" + featuregatetesting "k8s.io/component-base/featuregate/testing" logsapi "k8s.io/component-base/logs/api/v1" - utilversion "k8s.io/component-base/version" "k8s.io/klog/v2" ) @@ -124,15 +124,17 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin fs := pflag.NewFlagSet("test", pflag.PanicOnError) - featureGate := utilfeature.DefaultMutableFeatureGate - - // Configure the effective version. - effectiveVersion := utilversion.DefaultKubeEffectiveVersion() - effectiveVersion.SetEmulationVersion(featureGate.EmulationVersion()) - - featuregate.DefaultComponentGlobalsRegistry.Reset() - utilruntime.Must(featuregate.DefaultComponentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, effectiveVersion, featureGate)) s := options.NewCustomResourceDefinitionsServerOptions(os.Stdout, os.Stderr) + + // set up new instance of ComponentGlobalsRegistry instead of using the DefaultComponentGlobalsRegistry to avoid contention in parallel tests. + featureGate := utilfeature.DefaultMutableFeatureGate.DeepCopy() + effectiveVersion := compatibility.DefaultKubeEffectiveVersionForTest() + effectiveVersion.SetEmulationVersion(featureGate.EmulationVersion()) + componentGlobalsRegistry := basecompatibility.NewComponentGlobalsRegistry() + if err := componentGlobalsRegistry.Register(basecompatibility.DefaultKubeComponent, effectiveVersion, featureGate); err != nil { + return result, err + } + s.ServerRunOptions.ComponentGlobalsRegistry = componentGlobalsRegistry s.AddFlags(fs) s.RecommendedOptions.SecureServing.Listener, s.RecommendedOptions.SecureServing.BindPort, err = createLocalhostListenerOnFreePort() @@ -155,8 +157,18 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin fs.Parse(customFlags) - if err := featuregate.DefaultComponentGlobalsRegistry.Set(); err != nil { - return result, err + if err := componentGlobalsRegistry.Set(); err != nil { + return result, fmt.Errorf("%w\nIf you are using SetFeatureGate*DuringTest, try using --emulated-version and --feature-gates flags instead", err) + } + // If the local ComponentGlobalsRegistry is changed by the flags, + // we need to copy the new feature values back to the DefaultFeatureGate because most feature checks still use the DefaultFeatureGate. + if !featureGate.EmulationVersion().EqualTo(utilfeature.DefaultMutableFeatureGate.EmulationVersion()) { + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t.(featuregatetesting.TB), utilfeature.DefaultMutableFeatureGate, effectiveVersion.EmulationVersion()) + } + for f := range utilfeature.DefaultMutableFeatureGate.GetAll() { + if featureGate.Enabled(f) != utilfeature.DefaultFeatureGate.Enabled(f) { + featuregatetesting.SetFeatureGateDuringTest(t.(featuregatetesting.TB), utilfeature.DefaultFeatureGate, f, featureGate.Enabled(f)) + } } if err := s.Complete(); err != nil { diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/fieldselector_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/fieldselector_test.go index 83aba5ad9a2..edf5fe7a7c4 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/fieldselector_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/fieldselector_test.go @@ -41,7 +41,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -593,9 +592,7 @@ func TestFieldSelectorOpenAPI(t *testing.T) { func TestFieldSelectorDropFields(t *testing.T) { _, ctx := ktesting.NewTestContext(t) - featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31")) - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceFieldSelectors, false) - tearDown, apiExtensionClient, _, err := fixtures.StartDefaultServerWithClients(t) + tearDown, apiExtensionClient, _, err := fixtures.StartDefaultServerWithClients(t, "--emulated-version=1.31", "--feature-gates=CustomResourceFieldSelectors=false") if err != nil { t.Fatal(err) } @@ -675,9 +672,8 @@ func TestFieldSelectorDropFields(t *testing.T) { } func TestFieldSelectorDisablement(t *testing.T) { - featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31")) _, ctx := ktesting.NewTestContext(t) - tearDown, config, _, err := fixtures.StartDefaultServer(t) + tearDown, config, _, err := fixtures.StartDefaultServer(t, "--emulated-version=1.31") if err != nil { t.Fatal(err) } diff --git a/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/x509_test.go b/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/x509_test.go index 48fe6798e47..01cf24aa643 100644 --- a/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/x509_test.go +++ b/staging/src/k8s.io/apiserver/pkg/authentication/request/x509/x509_test.go @@ -34,7 +34,6 @@ import ( "github.com/stretchr/testify/assert" asn1util "k8s.io/apimachinery/pkg/apis/asn1" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/version" "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/features" @@ -799,9 +798,6 @@ func TestX509(t *testing.T) { ExpectErr: false, setupFunc: func(t *testing.T) { t.Helper() - // This statement can be removed once utilversion.DefaultKubeEffectiveVersion() is >= 1.33 - featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, feature.DefaultFeatureGate, version.MustParse("1.33")) - }, }, "common name and empty UID with feature gate disabled": { @@ -822,8 +818,6 @@ func TestX509(t *testing.T) { ExpectErr: false, setupFunc: func(t *testing.T) { t.Helper() - // This statement can be removed once utilversion.DefaultKubeEffectiveVersion() is >= 1.33 - featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, feature.DefaultFeatureGate, version.MustParse("1.33")) featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.AllowParsingUserUIDFromCertAuth, false) }, }, @@ -836,9 +830,6 @@ func TestX509(t *testing.T) { ExpectErrMsg: regexp.MustCompile("UID cannot be an empty string"), setupFunc: func(t *testing.T) { t.Helper() - // This statement can be removed once utilversion.DefaultKubeEffectiveVersion() is >= 1.33 - featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, feature.DefaultFeatureGate, version.MustParse("1.33")) - }, }, "ca with non-string UID": { @@ -850,9 +841,6 @@ func TestX509(t *testing.T) { ExpectErrMsg: regexp.MustCompile("unable to parse UID into a string"), setupFunc: func(t *testing.T) { t.Helper() - // This statement can be removed once utilversion.DefaultKubeEffectiveVersion() is >= 1.33 - featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, feature.DefaultFeatureGate, version.MustParse("1.33")) - }, }, "ca with multiple UIDs": { @@ -866,9 +854,6 @@ func TestX509(t *testing.T) { ExpectErrMsg: regexp.MustCompile("expected 1 UID, but found multiple"), setupFunc: func(t *testing.T) { t.Helper() - // This statement can be removed once utilversion.DefaultKubeEffectiveVersion() is >= 1.33 - featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, feature.DefaultFeatureGate, version.MustParse("1.33")) - }, }, "ca with multiple organizations": { diff --git a/staging/src/k8s.io/apiserver/pkg/cel/environment/base.go b/staging/src/k8s.io/apiserver/pkg/cel/environment/base.go index 3f4197b40ab..aa053e52e06 100644 --- a/staging/src/k8s.io/apiserver/pkg/cel/environment/base.go +++ b/staging/src/k8s.io/apiserver/pkg/cel/environment/base.go @@ -32,9 +32,9 @@ import ( celconfig "k8s.io/apiserver/pkg/apis/cel" "k8s.io/apiserver/pkg/cel/library" genericfeatures "k8s.io/apiserver/pkg/features" + "k8s.io/apiserver/pkg/util/compatibility" utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/component-base/featuregate" - utilversion "k8s.io/component-base/version" + basecompatibility "k8s.io/component-base/compatibility" ) // DefaultCompatibilityVersion returns a default compatibility version for use with EnvSet @@ -50,9 +50,9 @@ import ( // A default version number equal to the current Kubernetes major.minor version // indicates fast forward CEL features that can be used when rollback is no longer needed. func DefaultCompatibilityVersion() *version.Version { - effectiveVer := featuregate.DefaultComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent) + effectiveVer := compatibility.DefaultComponentGlobalsRegistry.EffectiveVersionFor(basecompatibility.DefaultKubeComponent) if effectiveVer == nil { - effectiveVer = utilversion.DefaultKubeEffectiveVersion() + effectiveVer = compatibility.DefaultBuildEffectiveVersion() } return effectiveVer.MinCompatibilityVersion() } diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index ee037aefed2..9e9bf4e3d20 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -73,12 +73,12 @@ import ( flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request" "k8s.io/client-go/informers" restclient "k8s.io/client-go/rest" + basecompatibility "k8s.io/component-base/compatibility" "k8s.io/component-base/featuregate" "k8s.io/component-base/logs" "k8s.io/component-base/metrics/features" "k8s.io/component-base/metrics/prometheus/slis" "k8s.io/component-base/tracing" - utilversion "k8s.io/component-base/version" "k8s.io/component-base/zpages/flagz" "k8s.io/klog/v2" openapicommon "k8s.io/kube-openapi/pkg/common" @@ -153,7 +153,7 @@ type Config struct { // EffectiveVersion determines which apis and features are available // based on when the api/feature lifecyle. - EffectiveVersion utilversion.EffectiveVersion + EffectiveVersion basecompatibility.EffectiveVersion // FeatureGate is a way to plumb feature gate through if you have them. FeatureGate featuregate.FeatureGate // AuditBackend is where audit events are sent to. diff --git a/staging/src/k8s.io/apiserver/pkg/server/config_test.go b/staging/src/k8s.io/apiserver/pkg/server/config_test.go index 1f30143a977..a4c2b7359a5 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config_test.go @@ -47,9 +47,9 @@ import ( "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/rest" + basecompatibility "k8s.io/component-base/compatibility" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/component-base/tracing" - utilversion "k8s.io/component-base/version" "k8s.io/klog/v2/ktesting" netutils "k8s.io/utils/net" ) @@ -124,7 +124,7 @@ func TestNewWithDelegate(t *testing.T) { delegateConfig.PublicAddress = netutils.ParseIPSloppy("192.168.10.4") delegateConfig.LegacyAPIGroupPrefixes = sets.NewString("/api") delegateConfig.LoopbackClientConfig = &rest.Config{} - delegateConfig.EffectiveVersion = utilversion.NewEffectiveVersion("") + delegateConfig.EffectiveVersion = basecompatibility.NewEffectiveVersionFromString("", "", "") clientset := fake.NewSimpleClientset() if clientset == nil { t.Fatal("unable to create fake client set") @@ -157,7 +157,7 @@ func TestNewWithDelegate(t *testing.T) { wrappingConfig.PublicAddress = netutils.ParseIPSloppy("192.168.10.4") wrappingConfig.LegacyAPIGroupPrefixes = sets.NewString("/api") wrappingConfig.LoopbackClientConfig = &rest.Config{} - wrappingConfig.EffectiveVersion = utilversion.NewEffectiveVersion("") + wrappingConfig.EffectiveVersion = basecompatibility.NewEffectiveVersionFromString("", "", "") wrappingConfig.HealthzChecks = append(wrappingConfig.HealthzChecks, healthz.NamedCheck("wrapping-health", func(r *http.Request) error { return fmt.Errorf("wrapping failed healthcheck") @@ -434,7 +434,7 @@ func TestNewFeatureGatedSerializer(t *testing.T) { } }))) config.ExternalAddress = "192.168.10.4:443" - config.EffectiveVersion = utilversion.NewEffectiveVersion("") + config.EffectiveVersion = basecompatibility.NewEffectiveVersionFromString("", "", "") config.LoopbackClientConfig = &rest.Config{} if _, err := config.Complete(nil).New("test", NewEmptyDelegate()); err != nil { diff --git a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go index e810a460879..43456f2f1e4 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go +++ b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go @@ -54,8 +54,8 @@ import ( "k8s.io/apiserver/pkg/storageversion" utilfeature "k8s.io/apiserver/pkg/util/feature" restclient "k8s.io/client-go/rest" + basecompatibility "k8s.io/component-base/compatibility" "k8s.io/component-base/featuregate" - utilversion "k8s.io/component-base/version" "k8s.io/klog/v2" openapibuilder3 "k8s.io/kube-openapi/pkg/builder3" openapicommon "k8s.io/kube-openapi/pkg/common" @@ -244,7 +244,7 @@ type GenericAPIServer struct { // EffectiveVersion determines which apis and features are available // based on when the api/feature lifecyle. - EffectiveVersion utilversion.EffectiveVersion + EffectiveVersion basecompatibility.EffectiveVersion // FeatureGate is a way to plumb feature gate through if you have them. FeatureGate featuregate.FeatureGate diff --git a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go index 25c66b8d1b6..d3457cf34a1 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go @@ -38,6 +38,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" + utilversion "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/version" "k8s.io/apiserver/pkg/apis/example" @@ -52,7 +53,7 @@ import ( "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" restclient "k8s.io/client-go/rest" - utilversion "k8s.io/component-base/version" + basecompatibility "k8s.io/component-base/compatibility" "k8s.io/klog/v2/ktesting" kubeopenapi "k8s.io/kube-openapi/pkg/common" "k8s.io/kube-openapi/pkg/validation/spec" @@ -138,7 +139,7 @@ func setUp(t *testing.T) (Config, *assert.Assertions) { if clientset == nil { t.Fatal("unable to create fake client set") } - config.EffectiveVersion = utilversion.NewEffectiveVersion("") + config.EffectiveVersion = basecompatibility.NewEffectiveVersionFromString("", "", "") config.OpenAPIConfig = DefaultOpenAPIConfig(testGetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(runtime.NewScheme())) config.OpenAPIConfig.Info.Version = "unversioned" config.OpenAPIV3Config = DefaultOpenAPIV3Config(testGetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(runtime.NewScheme())) @@ -460,8 +461,8 @@ func TestNotRestRoutesHaveAuth(t *testing.T) { config.EnableProfiling = true kubeVersion := fakeVersion() - effectiveVersion := utilversion.NewEffectiveVersion(kubeVersion.String()) - effectiveVersion.Set(effectiveVersion.BinaryVersion().WithInfo(kubeVersion), effectiveVersion.EmulationVersion(), effectiveVersion.MinCompatibilityVersion()) + binaryVersion := utilversion.MustParse(kubeVersion.String()) + effectiveVersion := basecompatibility.NewEffectiveVersion(binaryVersion, false, binaryVersion, binaryVersion.SubtractMinor(1)) config.EffectiveVersion = effectiveVersion s, err := config.Complete(nil).New("test", NewEmptyDelegate()) diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go b/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go index a4d31ef926e..1e3bd0060e2 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go @@ -27,9 +27,9 @@ import ( "k8s.io/apimachinery/pkg/util/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/util/compatibility" utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/component-base/featuregate" - utilversion "k8s.io/component-base/version" + basecompatibility "k8s.io/component-base/compatibility" "github.com/spf13/pflag" ) @@ -95,22 +95,22 @@ type ServerRunOptions struct { ShutdownWatchTerminationGracePeriod time.Duration // ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored. - ComponentGlobalsRegistry featuregate.ComponentGlobalsRegistry + ComponentGlobalsRegistry basecompatibility.ComponentGlobalsRegistry // ComponentName is name under which the server's global variabled are registered in the ComponentGlobalsRegistry. ComponentName string } func NewServerRunOptions() *ServerRunOptions { - if featuregate.DefaultComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent) == nil { + if compatibility.DefaultComponentGlobalsRegistry.EffectiveVersionFor(basecompatibility.DefaultKubeComponent) == nil { featureGate := utilfeature.DefaultMutableFeatureGate - effectiveVersion := utilversion.DefaultKubeEffectiveVersion() - utilruntime.Must(featuregate.DefaultComponentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, effectiveVersion, featureGate)) + effectiveVersion := compatibility.DefaultBuildEffectiveVersion() + utilruntime.Must(compatibility.DefaultComponentGlobalsRegistry.Register(basecompatibility.DefaultKubeComponent, effectiveVersion, featureGate)) } - return NewServerRunOptionsForComponent(featuregate.DefaultKubeComponent, featuregate.DefaultComponentGlobalsRegistry) + return NewServerRunOptionsForComponent(basecompatibility.DefaultKubeComponent, compatibility.DefaultComponentGlobalsRegistry) } -func NewServerRunOptionsForComponent(componentName string, componentGlobalsRegistry featuregate.ComponentGlobalsRegistry) *ServerRunOptions { +func NewServerRunOptionsForComponent(componentName string, componentGlobalsRegistry basecompatibility.ComponentGlobalsRegistry) *ServerRunOptions { defaults := server.NewConfig(serializer.CodecFactory{}) return &ServerRunOptions{ MaxRequestsInFlight: defaults.MaxRequestsInFlight, diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options_test.go index 69c09ccc4c7..462a733365d 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options_test.go @@ -26,15 +26,15 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/version" utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/component-base/featuregate" - utilversion "k8s.io/component-base/version" + basecompatibility "k8s.io/component-base/compatibility" netutils "k8s.io/utils/net" ) func TestServerRunOptionsValidate(t *testing.T) { - testRegistry := featuregate.NewComponentGlobalsRegistry() + defaultComponentGlobalsRegistry := basecompatibility.NewComponentGlobalsRegistry() + testRegistry := basecompatibility.NewComponentGlobalsRegistry() featureGate := utilfeature.DefaultFeatureGate.DeepCopy() - effectiveVersion := utilversion.NewEffectiveVersion("1.35") + effectiveVersion := basecompatibility.NewEffectiveVersionFromString("1.35", "1.32", "1.32") effectiveVersion.SetEmulationVersion(version.MajorMinor(1, 31)) testComponent := "test" utilruntime.Must(testRegistry.Register(testComponent, effectiveVersion, featureGate)) @@ -55,7 +55,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry, + ComponentGlobalsRegistry: defaultComponentGlobalsRegistry, }, expectErr: "--max-requests-inflight can not be negative value", }, @@ -70,7 +70,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry, + ComponentGlobalsRegistry: defaultComponentGlobalsRegistry, }, expectErr: "--max-mutating-requests-inflight can not be negative value", }, @@ -85,7 +85,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry, + ComponentGlobalsRegistry: defaultComponentGlobalsRegistry, }, expectErr: "--request-timeout can not be negative value", }, @@ -100,7 +100,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: -1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry, + ComponentGlobalsRegistry: defaultComponentGlobalsRegistry, }, expectErr: "--min-request-timeout can not be negative value", }, @@ -115,7 +115,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: -10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry, + ComponentGlobalsRegistry: defaultComponentGlobalsRegistry, }, expectErr: "ServerRunOptions.JSONPatchMaxCopyBytes can not be negative value", }, @@ -130,7 +130,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: -10 * 1024 * 1024, - ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry, + ComponentGlobalsRegistry: defaultComponentGlobalsRegistry, }, expectErr: "ServerRunOptions.MaxRequestBodyBytes can not be negative value", }, @@ -146,7 +146,7 @@ func TestServerRunOptionsValidate(t *testing.T) { JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, LivezGracePeriod: -time.Second, - ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry, + ComponentGlobalsRegistry: defaultComponentGlobalsRegistry, }, expectErr: "--livez-grace-period can not be a negative value", }, @@ -162,7 +162,7 @@ func TestServerRunOptionsValidate(t *testing.T) { JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, ShutdownDelayDuration: -time.Second, - ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry, + ComponentGlobalsRegistry: defaultComponentGlobalsRegistry, }, expectErr: "--shutdown-delay-duration can not be negative value", }, @@ -178,7 +178,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry, + ComponentGlobalsRegistry: defaultComponentGlobalsRegistry, }, expectErr: "--strict-transport-security-directives invalid, allowed values: max-age=expireTime, includeSubDomains, preload. see https://tools.ietf.org/html/rfc6797#section-6.1 for more information", }, @@ -211,7 +211,7 @@ func TestServerRunOptionsValidate(t *testing.T) { MinRequestTimeout: 1800, JSONPatchMaxCopyBytes: 10 * 1024 * 1024, MaxRequestBodyBytes: 10 * 1024 * 1024, - ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry, + ComponentGlobalsRegistry: defaultComponentGlobalsRegistry, }, }, } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go index 3d9c4c3fc0f..898996837be 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go @@ -46,7 +46,7 @@ import ( "k8s.io/client-go/discovery" restclient "k8s.io/client-go/rest" cliflag "k8s.io/component-base/cli/flag" - utilversion "k8s.io/component-base/version" + basecompatibility "k8s.io/component-base/compatibility" "k8s.io/klog/v2/ktesting" netutils "k8s.io/utils/net" ) @@ -278,7 +278,7 @@ func TestServerRunWithSNI(t *testing.T) { // launch server config := setUp(t) v := fakeVersion() - config.EffectiveVersion = utilversion.NewEffectiveVersion(v.String()) + config.EffectiveVersion = basecompatibility.NewEffectiveVersionFromString(v.String(), "", "") config.EnableIndex = true secureOptions := (&SecureServingOptions{ diff --git a/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go b/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go index ce1bec6763a..612983cca91 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go @@ -22,7 +22,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" apimachineryversion "k8s.io/apimachinery/pkg/util/version" - version "k8s.io/component-base/version" + "k8s.io/apiserver/pkg/util/compatibility" + basecompatibility "k8s.io/component-base/compatibility" ) type ResourceEncodingConfig interface { @@ -43,7 +44,7 @@ type DefaultResourceEncodingConfig struct { // resources records the overriding encoding configs for individual resources. resources map[schema.GroupResource]*OverridingResourceEncoding scheme *runtime.Scheme - effectiveVersion version.EffectiveVersion + effectiveVersion basecompatibility.EffectiveVersion } type OverridingResourceEncoding struct { @@ -54,7 +55,11 @@ type OverridingResourceEncoding struct { var _ ResourceEncodingConfig = &DefaultResourceEncodingConfig{} func NewDefaultResourceEncodingConfig(scheme *runtime.Scheme) *DefaultResourceEncodingConfig { - return &DefaultResourceEncodingConfig{resources: map[schema.GroupResource]*OverridingResourceEncoding{}, scheme: scheme, effectiveVersion: version.DefaultKubeEffectiveVersion()} + return NewDefaultResourceEncodingConfigForEffectiveVersion(scheme, compatibility.DefaultComponentGlobalsRegistry.EffectiveVersionFor(basecompatibility.DefaultKubeComponent)) +} + +func NewDefaultResourceEncodingConfigForEffectiveVersion(scheme *runtime.Scheme, effectiveVersion basecompatibility.EffectiveVersion) *DefaultResourceEncodingConfig { + return &DefaultResourceEncodingConfig{resources: map[schema.GroupResource]*OverridingResourceEncoding{}, scheme: scheme, effectiveVersion: effectiveVersion} } func (o *DefaultResourceEncodingConfig) SetResourceEncoding(resourceBeingStored schema.GroupResource, externalEncodingVersion, internalVersion schema.GroupVersion) { @@ -64,7 +69,7 @@ func (o *DefaultResourceEncodingConfig) SetResourceEncoding(resourceBeingStored } } -func (o *DefaultResourceEncodingConfig) SetEffectiveVersion(effectiveVersion version.EffectiveVersion) { +func (o *DefaultResourceEncodingConfig) SetEffectiveVersion(effectiveVersion basecompatibility.EffectiveVersion) { o.effectiveVersion = effectiveVersion } @@ -121,7 +126,7 @@ type replacementInterface interface { APILifecycleReplacement() schema.GroupVersionKind } -func emulatedStorageVersion(binaryVersionOfResource schema.GroupVersion, example runtime.Object, effectiveVersion version.EffectiveVersion, scheme *runtime.Scheme) (schema.GroupVersion, error) { +func emulatedStorageVersion(binaryVersionOfResource schema.GroupVersion, example runtime.Object, effectiveVersion basecompatibility.EffectiveVersion, scheme *runtime.Scheme) (schema.GroupVersion, error) { if example == nil || effectiveVersion == nil { return binaryVersionOfResource, nil } @@ -172,7 +177,7 @@ func emulatedStorageVersion(binaryVersionOfResource schema.GroupVersion, example } // If it was introduced after current compatibility version, don't use it - // skip the introduced check for test when currentVersion is 0.0 to test all apis + // skip the introduced check for test when current compatibility version is 0.0 to test all apis if introduced, hasIntroduced := exampleOfGVK.(introducedInterface); hasIntroduced && (compatibilityVersion.Major() > 0 || compatibilityVersion.Minor() > 0) { // Skip versions that have a replacement. diff --git a/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config_test.go b/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config_test.go index a96e7e321d8..a6984deed26 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config_test.go @@ -25,7 +25,7 @@ import ( "k8s.io/apimachinery/pkg/test" utilruntime "k8s.io/apimachinery/pkg/util/runtime" utilversion "k8s.io/apimachinery/pkg/util/version" - "k8s.io/component-base/version" + basecompatibility "k8s.io/component-base/compatibility" ) func TestEmulatedStorageVersion(t *testing.T) { @@ -33,21 +33,21 @@ func TestEmulatedStorageVersion(t *testing.T) { name string scheme *runtime.Scheme binaryVersion schema.GroupVersion - effectiveVersion version.EffectiveVersion + effectiveVersion basecompatibility.EffectiveVersion want schema.GroupVersion }{ { name: "pick compatible", scheme: AlphaBetaScheme(utilversion.MustParse("1.31"), utilversion.MustParse("1.32")), binaryVersion: v1beta1, - effectiveVersion: version.NewEffectiveVersion("1.32"), + effectiveVersion: basecompatibility.NewEffectiveVersionFromString("1.32", "", ""), want: v1alpha1, }, { name: "alpha has been replaced, pick binary version", scheme: AlphaReplacedBetaScheme(utilversion.MustParse("1.31"), utilversion.MustParse("1.32")), binaryVersion: v1beta1, - effectiveVersion: version.NewEffectiveVersion("1.32"), + effectiveVersion: basecompatibility.NewEffectiveVersionFromString("1.32", "", ""), want: v1beta1, }, } diff --git a/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory_test.go b/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory_test.go index fb79da75ba0..72d69a4b334 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory_test.go @@ -33,7 +33,7 @@ import ( "k8s.io/apiserver/pkg/apis/example2" example2install "k8s.io/apiserver/pkg/apis/example2/install" "k8s.io/apiserver/pkg/storage/storagebackend" - version "k8s.io/component-base/version" + basecompatibility "k8s.io/component-base/compatibility" ) var ( @@ -569,8 +569,7 @@ func TestStorageFactoryCompatibilityVersion(t *testing.T) { gvk := gvks[0] t.Run(gvk.GroupKind().String()+"@"+tc.effectiveVersion, func(t *testing.T) { - config := NewDefaultResourceEncodingConfig(sch) - config.SetEffectiveVersion(version.NewEffectiveVersion(tc.effectiveVersion)) + config := NewDefaultResourceEncodingConfigForEffectiveVersion(sch, basecompatibility.NewEffectiveVersionFromString(tc.effectiveVersion, "", "")) f := NewDefaultStorageFactory( storagebackend.Config{}, "", diff --git a/staging/src/k8s.io/apiserver/pkg/util/compatibility/registry.go b/staging/src/k8s.io/apiserver/pkg/util/compatibility/registry.go new file mode 100644 index 00000000000..4110db43d3c --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/util/compatibility/registry.go @@ -0,0 +1,53 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package compatibility + +import ( + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + utilfeature "k8s.io/apiserver/pkg/util/feature" + basecompatibility "k8s.io/component-base/compatibility" +) + +// DefaultComponentGlobalsRegistry is the global var to store the effective versions and feature gates for all components for easy access. +// Example usage: +// // register the component effective version and feature gate first +// wardleEffectiveVersion := basecompatibility.NewEffectiveVersion("1.2") +// wardleFeatureGate := featuregate.NewFeatureGate() +// utilruntime.Must(compatibility.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, wardleFeatureGate, false)) +// +// cmd := &cobra.Command{ +// ... +// // call DefaultComponentGlobalsRegistry.Set() in PersistentPreRunE to ensure the feature gates are set based on emulation version right after parsing the flags. +// PersistentPreRunE: func(*cobra.Command, []string) error { +// if err := compatibility.DefaultComponentGlobalsRegistry.Set(); err != nil { +// return err +// } +// ... +// }, +// RunE: func(c *cobra.Command, args []string) error { +// // call compatibility.DefaultComponentGlobalsRegistry.Validate() somewhere +// }, +// } +// +// flags := cmd.Flags() +// // add flags +// compatibility.DefaultComponentGlobalsRegistry.AddFlags(flags) +var DefaultComponentGlobalsRegistry basecompatibility.ComponentGlobalsRegistry = basecompatibility.NewComponentGlobalsRegistry() + +func init() { + utilruntime.Must(DefaultComponentGlobalsRegistry.Register(basecompatibility.DefaultKubeComponent, DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)) +} diff --git a/staging/src/k8s.io/apiserver/pkg/util/compatibility/version.go b/staging/src/k8s.io/apiserver/pkg/util/compatibility/version.go new file mode 100644 index 00000000000..9ceaf3de65d --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/util/compatibility/version.go @@ -0,0 +1,65 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package compatibility + +import ( + "k8s.io/apimachinery/pkg/util/version" + basecompatibility "k8s.io/component-base/compatibility" + baseversion "k8s.io/component-base/version" +) + +// minimumKubeEmulationVersion is the first release emulation version is introduced, +// so the emulation version cannot go lower than that. +var minimumKubeEmulationVersion *version.Version = version.MajorMinor(1, 31) + +// DefaultBuildEffectiveVersion returns the MutableEffectiveVersion based on the +// current build information. +func DefaultBuildEffectiveVersion() basecompatibility.MutableEffectiveVersion { + binaryVersion := defaultBuildBinaryVersion() + useDefaultBuildBinaryVersion := true + // fall back to the hard coded kube version only when the git tag is not available for local unit tests. + if binaryVersion.Major() == 0 && binaryVersion.Minor() == 0 { + useDefaultBuildBinaryVersion = false + binaryVersion = version.MustParse(baseversion.DefaultKubeBinaryVersion) + } + versionFloor := kubeEffectiveVersionFloors(binaryVersion) + return basecompatibility.NewEffectiveVersion(binaryVersion, useDefaultBuildBinaryVersion, versionFloor, versionFloor) +} + +func kubeEffectiveVersionFloors(binaryVersion *version.Version) *version.Version { + // both emulationVersion and minCompatibilityVersion can be set to binaryVersion - 3 + versionFloor := binaryVersion.WithPatch(0).SubtractMinor(3) + if versionFloor.LessThan(minimumKubeEmulationVersion) { + versionFloor = minimumKubeEmulationVersion + } + return versionFloor +} + +// DefaultKubeEffectiveVersionForTest returns the MutableEffectiveVersion based on the +// latest K8s release hardcoded in DefaultKubeBinaryVersion. +// DefaultKubeBinaryVersion is hard coded because defaultBuildBinaryVersion would return 0.0 when test is run without a git tag. +// We do not enforce the N-3..N emulation version range in tests so that the tests would not automatically fail when there is a version bump. +// Only used in tests. +func DefaultKubeEffectiveVersionForTest() basecompatibility.MutableEffectiveVersion { + binaryVersion := version.MustParse(baseversion.DefaultKubeBinaryVersion).WithInfo(baseversion.Get()) + return basecompatibility.NewEffectiveVersion(binaryVersion, false, version.MustParse("0.0"), version.MustParse("0.0")) +} + +func defaultBuildBinaryVersion() *version.Version { + verInfo := baseversion.Get() + return version.MustParse(verInfo.String()).WithInfo(verInfo) +} diff --git a/staging/src/k8s.io/apiserver/pkg/util/compatibility/version_test.go b/staging/src/k8s.io/apiserver/pkg/util/compatibility/version_test.go new file mode 100644 index 00000000000..ed10f937136 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/util/compatibility/version_test.go @@ -0,0 +1,76 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package compatibility + +import ( + "testing" + + "k8s.io/apimachinery/pkg/util/version" + basecompatibility "k8s.io/component-base/compatibility" +) + +func TestValidateKubeEffectiveVersion(t *testing.T) { + tests := []struct { + name string + emulationVersion string + minCompatibilityVersion string + expectErr bool + }{ + { + name: "valid versions", + emulationVersion: "v1.32.0", + minCompatibilityVersion: "v1.31.0", + expectErr: false, + }, + { + name: "emulationVersion too low", + emulationVersion: "v1.30.0", + minCompatibilityVersion: "v1.31.0", + expectErr: true, + }, + { + name: "minCompatibilityVersion too low", + emulationVersion: "v1.31.0", + minCompatibilityVersion: "v1.30.0", + expectErr: true, + }, + { + name: "both versions too low", + emulationVersion: "v1.30.0", + minCompatibilityVersion: "v1.29.0", + expectErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + binaryVersion := version.MustParseGeneric("1.33") + versionFloor := kubeEffectiveVersionFloors(binaryVersion) + effective := basecompatibility.NewEffectiveVersion(binaryVersion, false, versionFloor, versionFloor) + effective.SetEmulationVersion(version.MustParseGeneric(test.emulationVersion)) + effective.SetMinCompatibilityVersion(version.MustParseGeneric(test.minCompatibilityVersion)) + + err := effective.Validate() + if test.expectErr && err == nil { + t.Error("expected error, but got nil") + } + if !test.expectErr && err != nil { + t.Errorf("unexpected error: %v", err) + } + }) + } +} diff --git a/staging/src/k8s.io/component-base/compatibility/OWNERS b/staging/src/k8s.io/component-base/compatibility/OWNERS new file mode 100644 index 00000000000..80e9c52364c --- /dev/null +++ b/staging/src/k8s.io/component-base/compatibility/OWNERS @@ -0,0 +1,12 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +# Currently assigned this directory to sig-api-machinery since this is +# an interface to the version definition in "k8s.io/apiserver/pkg/util/compatibility". + +approvers: + - sig-api-machinery-api-approvers +reviewers: + - sig-api-machinery-api-reviewers + - siyuanfoundation +labels: + - sig/api-machinery diff --git a/staging/src/k8s.io/component-base/featuregate/registry.go b/staging/src/k8s.io/component-base/compatibility/registry.go similarity index 82% rename from staging/src/k8s.io/component-base/featuregate/registry.go rename to staging/src/k8s.io/component-base/compatibility/registry.go index cf35403da4a..24d620efd00 100644 --- a/staging/src/k8s.io/component-base/featuregate/registry.go +++ b/staging/src/k8s.io/component-base/compatibility/registry.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package featuregate +package compatibility import ( "fmt" @@ -26,38 +26,12 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/version" cliflag "k8s.io/component-base/cli/flag" - baseversion "k8s.io/component-base/version" + "k8s.io/component-base/featuregate" "k8s.io/klog/v2" ) -// DefaultComponentGlobalsRegistry is the global var to store the effective versions and feature gates for all components for easy access. -// Example usage: -// // register the component effective version and feature gate first -// _, _ = utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) -// wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2") -// wardleFeatureGate := featuregate.NewFeatureGate() -// utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, wardleFeatureGate, false)) -// -// cmd := &cobra.Command{ -// ... -// // call DefaultComponentGlobalsRegistry.Set() in PersistentPreRunE -// PersistentPreRunE: func(*cobra.Command, []string) error { -// if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil { -// return err -// } -// ... -// }, -// RunE: func(c *cobra.Command, args []string) error { -// // call utilversion.DefaultComponentGlobalsRegistry.Validate() somewhere -// }, -// } -// -// flags := cmd.Flags() -// // add flags -// utilversion.DefaultComponentGlobalsRegistry.AddFlags(flags) -var DefaultComponentGlobalsRegistry ComponentGlobalsRegistry = NewComponentGlobalsRegistry() - const ( + // DefaultKubeComponent is the component name for k8s control plane components. DefaultKubeComponent = "kube" klogLevel = 2 @@ -65,10 +39,10 @@ const ( type VersionMapping func(from *version.Version) *version.Version -// ComponentGlobals stores the global variables for a component for easy access. +// ComponentGlobals stores the global variables for a component for easy access, including feature gate and effective version. type ComponentGlobals struct { - effectiveVersion baseversion.MutableEffectiveVersion - featureGate MutableVersionedFeatureGate + effectiveVersion MutableEffectiveVersion + featureGate featuregate.MutableVersionedFeatureGate // emulationVersionMapping contains the mapping from the emulation version of this component // to the emulation version of another component. @@ -84,22 +58,24 @@ type ComponentGlobals struct { dependentMinCompatibilityVersion bool } +// ComponentGlobalsRegistry stores the global variables for different components for easy access, including feature gate and effective version of each component. type ComponentGlobalsRegistry interface { // EffectiveVersionFor returns the EffectiveVersion registered under the component. // Returns nil if the component is not registered. - EffectiveVersionFor(component string) baseversion.EffectiveVersion + EffectiveVersionFor(component string) EffectiveVersion // FeatureGateFor returns the FeatureGate registered under the component. // Returns nil if the component is not registered. - FeatureGateFor(component string) FeatureGate + FeatureGateFor(component string) featuregate.FeatureGate // Register registers the EffectiveVersion and FeatureGate for a component. // returns error if the component is already registered. - Register(component string, effectiveVersion baseversion.MutableEffectiveVersion, featureGate MutableVersionedFeatureGate) error + Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error // ComponentGlobalsOrRegister would return the registered global variables for the component if it already exists in the registry. // Otherwise, the provided variables would be registered under the component, and the same variables would be returned. - ComponentGlobalsOrRegister(component string, effectiveVersion baseversion.MutableEffectiveVersion, featureGate MutableVersionedFeatureGate) (baseversion.MutableEffectiveVersion, MutableVersionedFeatureGate) + ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) // AddFlags adds flags of "--emulated-version" and "--feature-gates" AddFlags(fs *pflag.FlagSet) // Set sets the flags for all global variables for all components registered. + // A component's feature gate and effective version would not be updated until Set() is called. Set() error // SetFallback calls Set() if it has never been called. SetFallback() error @@ -118,9 +94,13 @@ type ComponentGlobalsRegistry interface { type componentGlobalsRegistry struct { componentGlobals map[string]*ComponentGlobals mutex sync.RWMutex - // list of component name to emulation version set from the flag. + // emulationVersionConfig stores the list of component name to emulation version set from the flag. + // When the `--emulated-version` flag is parsed, it would not take effect until Set() is called, + // because the emulation version needs to be set before the feature gate is set. emulationVersionConfig []string - // map of component name to the list of feature gates set from the flag. + // featureGatesConfig stores the map of component name to the list of feature gates set from the flag. + // When the `--feature-gates` flag is parsed, it would not take effect until Set() is called, + // because the emulation version needs to be set before the feature gate is set. featureGatesConfig map[string][]string // set stores if the Set() function for the registry is already called. set bool @@ -143,7 +123,7 @@ func (r *componentGlobalsRegistry) Reset() { r.set = false } -func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) baseversion.EffectiveVersion { +func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) EffectiveVersion { r.mutex.RLock() defer r.mutex.RUnlock() globals, ok := r.componentGlobals[component] @@ -153,7 +133,7 @@ func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) basever return globals.effectiveVersion } -func (r *componentGlobalsRegistry) FeatureGateFor(component string) FeatureGate { +func (r *componentGlobalsRegistry) FeatureGateFor(component string) featuregate.FeatureGate { r.mutex.RLock() defer r.mutex.RUnlock() globals, ok := r.componentGlobals[component] @@ -163,7 +143,7 @@ func (r *componentGlobalsRegistry) FeatureGateFor(component string) FeatureGate return globals.featureGate } -func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVersion baseversion.MutableEffectiveVersion, featureGate MutableVersionedFeatureGate) error { +func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error { if _, ok := r.componentGlobals[component]; ok { return fmt.Errorf("component globals of %s already registered", component) } @@ -182,7 +162,7 @@ func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVer return nil } -func (r *componentGlobalsRegistry) Register(component string, effectiveVersion baseversion.MutableEffectiveVersion, featureGate MutableVersionedFeatureGate) error { +func (r *componentGlobalsRegistry) Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error { if effectiveVersion == nil { return fmt.Errorf("cannot register nil effectiveVersion") } @@ -191,7 +171,7 @@ func (r *componentGlobalsRegistry) Register(component string, effectiveVersion b return r.unsafeRegister(component, effectiveVersion, featureGate) } -func (r *componentGlobalsRegistry) ComponentGlobalsOrRegister(component string, effectiveVersion baseversion.MutableEffectiveVersion, featureGate MutableVersionedFeatureGate) (baseversion.MutableEffectiveVersion, MutableVersionedFeatureGate) { +func (r *componentGlobalsRegistry) ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) { r.mutex.Lock() defer r.mutex.Unlock() globals, ok := r.componentGlobals[component] @@ -219,22 +199,16 @@ func (r *componentGlobalsRegistry) unsafeKnownFeatures() []string { func (r *componentGlobalsRegistry) unsafeVersionFlagOptions(isEmulation bool) []string { var vs []string for component, globals := range r.componentGlobals { - binaryVer := globals.effectiveVersion.BinaryVersion() if isEmulation { if globals.dependentEmulationVersion { continue } - // emulated version could be between binaryMajor.{binaryMinor} and binaryMajor.{binaryMinor} - // TODO: change to binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor} in 1.32 - vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component, - binaryVer.SubtractMinor(0).String(), binaryVer.String(), globals.effectiveVersion.EmulationVersion().String())) + vs = append(vs, fmt.Sprintf("%s=%s", component, globals.effectiveVersion.AllowedEmulationVersionRange())) } else { if globals.dependentMinCompatibilityVersion { continue } - // min compatibility version could be between binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor} - vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component, - binaryVer.SubtractMinor(1).String(), binaryVer.String(), globals.effectiveVersion.MinCompatibilityVersion().String())) + vs = append(vs, fmt.Sprintf("%s=%s", component, globals.effectiveVersion.AllowedMinCompatibilityVersionRange())) } } sort.Strings(vs) diff --git a/staging/src/k8s.io/component-base/featuregate/registry_test.go b/staging/src/k8s.io/component-base/compatibility/registry_test.go similarity index 87% rename from staging/src/k8s.io/component-base/featuregate/registry_test.go rename to staging/src/k8s.io/component-base/compatibility/registry_test.go index b825fd3d758..28a456e6227 100644 --- a/staging/src/k8s.io/component-base/featuregate/registry_test.go +++ b/staging/src/k8s.io/component-base/compatibility/registry_test.go @@ -1,5 +1,5 @@ /* -Copyright 2024 The Kubernetes Authors. +Copyright 2025 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package featuregate +package compatibility import ( "fmt" @@ -25,7 +25,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/version" - baseversion "k8s.io/component-base/version" + "k8s.io/component-base/featuregate" ) const ( @@ -34,8 +34,8 @@ const ( func TestEffectiveVersionRegistry(t *testing.T) { r := NewComponentGlobalsRegistry() - ver1 := baseversion.NewEffectiveVersion("1.31") - ver2 := baseversion.NewEffectiveVersion("1.28") + ver1 := NewEffectiveVersionFromString("1.31", "", "") + ver2 := NewEffectiveVersionFromString("1.28", "", "") if r.EffectiveVersionFor(testComponent) != nil { t.Fatalf("expected nil EffectiveVersion initially") @@ -57,40 +57,40 @@ func TestEffectiveVersionRegistry(t *testing.T) { func testRegistry(t *testing.T) *componentGlobalsRegistry { r := NewComponentGlobalsRegistry() - verKube := baseversion.NewEffectiveVersion("1.31") - fgKube := NewVersionedFeatureGate(version.MustParse("0.0")) - err := fgKube.AddVersioned(map[Feature]VersionedSpecs{ + verKube := NewEffectiveVersionFromString("1.31", "1.31", "1.30") + fgKube := featuregate.NewVersionedFeatureGate(version.MustParse("0.0")) + err := fgKube.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{ "kubeA": { - {Version: version.MustParse("1.27"), Default: false, PreRelease: Alpha}, - {Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, - {Version: version.MustParse("1.31"), Default: true, LockToDefault: true, PreRelease: GA}, + {Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("1.28"), Default: false, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.31"), Default: true, LockToDefault: true, PreRelease: featuregate.GA}, }, "kubeB": { - {Version: version.MustParse("1.30"), Default: false, PreRelease: Alpha}, + {Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha}, }, "commonC": { - {Version: version.MustParse("1.27"), Default: false, PreRelease: Alpha}, - {Version: version.MustParse("1.29"), Default: true, PreRelease: Beta}, + {Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("1.29"), Default: true, PreRelease: featuregate.Beta}, }, }) if err != nil { t.Fatal(err) } - verTest := baseversion.NewEffectiveVersion("2.8") - fgTest := NewVersionedFeatureGate(version.MustParse("0.0")) - err = fgTest.AddVersioned(map[Feature]VersionedSpecs{ + verTest := NewEffectiveVersionFromString("2.8", "2.8", "2.7") + fgTest := featuregate.NewVersionedFeatureGate(version.MustParse("0.0")) + err = fgTest.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{ "testA": { - {Version: version.MustParse("2.7"), Default: false, PreRelease: Alpha}, - {Version: version.MustParse("2.8"), Default: false, PreRelease: Beta}, - {Version: version.MustParse("2.10"), Default: true, PreRelease: GA}, + {Version: version.MustParse("2.7"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("2.8"), Default: false, PreRelease: featuregate.Beta}, + {Version: version.MustParse("2.10"), Default: true, PreRelease: featuregate.GA}, }, "testB": { - {Version: version.MustParse("2.9"), Default: false, PreRelease: Alpha}, + {Version: version.MustParse("2.9"), Default: false, PreRelease: featuregate.Alpha}, }, "commonC": { - {Version: version.MustParse("2.7"), Default: false, PreRelease: Alpha}, - {Version: version.MustParse("2.9"), Default: true, PreRelease: Beta}, + {Version: version.MustParse("2.7"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("2.9"), Default: true, PreRelease: featuregate.Beta}, }, }) if err != nil { @@ -154,8 +154,8 @@ func TestFlags(t *testing.T) { parseError string expectedKubeEmulationVersion string expectedTestEmulationVersion string - expectedKubeFeatureValues map[Feature]bool - expectedTestFeatureValues map[Feature]bool + expectedKubeFeatureValues map[featuregate.Feature]bool + expectedTestFeatureValues map[featuregate.Feature]bool }{ { name: "setting kube emulation version", @@ -214,8 +214,8 @@ func TestFlags(t *testing.T) { }, expectedKubeEmulationVersion: "1.31", expectedTestEmulationVersion: "2.7", - expectedKubeFeatureValues: map[Feature]bool{"kubeA": true, "kubeB": false, "commonC": true}, - expectedTestFeatureValues: map[Feature]bool{"testA": true, "testB": false, "commonC": false}, + expectedKubeFeatureValues: map[featuregate.Feature]bool{"kubeA": true, "kubeB": false, "commonC": true}, + expectedTestFeatureValues: map[featuregate.Feature]bool{"testA": true, "testB": false, "commonC": false}, }, { name: "setting future test feature flag", @@ -235,8 +235,8 @@ func TestFlags(t *testing.T) { }, expectedKubeEmulationVersion: "1.30", expectedTestEmulationVersion: "2.7", - expectedKubeFeatureValues: map[Feature]bool{"kubeA": false, "kubeB": true, "commonC": false}, - expectedTestFeatureValues: map[Feature]bool{"testA": false, "testB": false, "commonC": true}, + expectedKubeFeatureValues: map[featuregate.Feature]bool{"kubeA": false, "kubeB": true, "commonC": false}, + expectedTestFeatureValues: map[featuregate.Feature]bool{"testA": false, "testB": false, "commonC": true}, }, { name: "setting kube feature flag with different prefix", @@ -313,9 +313,9 @@ func TestFlags(t *testing.T) { func TestVersionMapping(t *testing.T) { r := NewComponentGlobalsRegistry() - ver1 := baseversion.NewEffectiveVersion("0.58") - ver2 := baseversion.NewEffectiveVersion("1.28") - ver3 := baseversion.NewEffectiveVersion("2.10") + ver1 := NewEffectiveVersionFromString("0.58", "", "") + ver2 := NewEffectiveVersionFromString("1.28", "", "") + ver3 := NewEffectiveVersionFromString("2.10", "", "") utilruntime.Must(r.Register("test1", ver1, nil)) utilruntime.Must(r.Register("test2", ver2, nil)) @@ -355,9 +355,9 @@ func TestVersionMapping(t *testing.T) { func TestVersionMappingWithMultipleDependency(t *testing.T) { r := NewComponentGlobalsRegistry() - ver1 := baseversion.NewEffectiveVersion("0.58") - ver2 := baseversion.NewEffectiveVersion("1.28") - ver3 := baseversion.NewEffectiveVersion("2.10") + ver1 := NewEffectiveVersionFromString("0.58", "", "") + ver2 := NewEffectiveVersionFromString("1.28", "", "") + ver3 := NewEffectiveVersionFromString("2.10", "", "") utilruntime.Must(r.Register("test1", ver1, nil)) utilruntime.Must(r.Register("test2", ver2, nil)) @@ -382,9 +382,9 @@ func TestVersionMappingWithMultipleDependency(t *testing.T) { func TestVersionMappingWithCyclicDependency(t *testing.T) { r := NewComponentGlobalsRegistry() - ver1 := baseversion.NewEffectiveVersion("0.58") - ver2 := baseversion.NewEffectiveVersion("1.28") - ver3 := baseversion.NewEffectiveVersion("2.10") + ver1 := NewEffectiveVersionFromString("0.58", "", "") + ver2 := NewEffectiveVersionFromString("1.28", "", "") + ver3 := NewEffectiveVersionFromString("2.10", "", "") utilruntime.Must(r.Register("test1", ver1, nil)) utilruntime.Must(r.Register("test2", ver2, nil)) diff --git a/staging/src/k8s.io/component-base/compatibility/version.go b/staging/src/k8s.io/component-base/compatibility/version.go new file mode 100644 index 00000000000..1230e8774ef --- /dev/null +++ b/staging/src/k8s.io/component-base/compatibility/version.go @@ -0,0 +1,213 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package compatibility + +import ( + "fmt" + "sync/atomic" + + "k8s.io/apimachinery/pkg/util/version" + baseversion "k8s.io/component-base/version" +) + +// EffectiveVersion stores all the version information of a component. +type EffectiveVersion interface { + // BinaryVersion is the binary version of a component. Tied to a particular binary release. + BinaryVersion() *version.Version + // EmulationVersion is the version a component emulate its capabilities (APIs, features, ...) of. + // If EmulationVersion is set to be different from BinaryVersion, the component will emulate the behavior of this version instead of the underlying binary version. + EmulationVersion() *version.Version + // MinCompatibilityVersion is the minimum version a component is compatible with (in terms of storage versions, validation rules, ...). + MinCompatibilityVersion() *version.Version + EqualTo(other EffectiveVersion) bool + String() string + Validate() []error + // AllowedEmulationVersionRange returns the string of the allowed range of emulation version. + // Used only for docs/help. + AllowedEmulationVersionRange() string + // AllowedMinCompatibilityVersionRange returns the string of the allowed range of min compatibility version. + // Used only for docs/help. + AllowedMinCompatibilityVersionRange() string +} + +type MutableEffectiveVersion interface { + EffectiveVersion + SetEmulationVersion(emulationVersion *version.Version) + SetMinCompatibilityVersion(minCompatibilityVersion *version.Version) +} + +type effectiveVersion struct { + // When true, BinaryVersion() returns the current binary version + useDefaultBuildBinaryVersion atomic.Bool + // Holds the last binary version stored in Set() + binaryVersion atomic.Pointer[version.Version] + // If the emulationVersion is set by the users, it could only contain major and minor versions. + // In tests, emulationVersion could be the same as the binary version, or set directly, + // which can have "alpha" as pre-release to continue serving expired apis while we clean up the test. + emulationVersion atomic.Pointer[version.Version] + // minCompatibilityVersion could only contain major and minor versions. + minCompatibilityVersion atomic.Pointer[version.Version] + // emulationVersionFloor is the minimum emulationVersion allowed. No limit if nil. + emulationVersionFloor *version.Version + // minCompatibilityVersionFloor is the minimum minCompatibilityVersionFloor allowed. No limit if nil. + minCompatibilityVersionFloor *version.Version +} + +func (m *effectiveVersion) BinaryVersion() *version.Version { + if m.useDefaultBuildBinaryVersion.Load() { + return defaultBuildBinaryVersion() + } + return m.binaryVersion.Load() +} + +func (m *effectiveVersion) EmulationVersion() *version.Version { + ver := m.emulationVersion.Load() + if ver != nil { + // Emulation version can have "alpha" as pre-release to continue serving expired apis while we clean up the test. + // The pre-release should not be accessible to the users. + return ver.WithPreRelease(m.BinaryVersion().PreRelease()) + } + return ver +} + +func (m *effectiveVersion) MinCompatibilityVersion() *version.Version { + return m.minCompatibilityVersion.Load() +} + +func (m *effectiveVersion) EqualTo(other EffectiveVersion) bool { + return m.BinaryVersion().EqualTo(other.BinaryVersion()) && m.EmulationVersion().EqualTo(other.EmulationVersion()) && m.MinCompatibilityVersion().EqualTo(other.MinCompatibilityVersion()) +} + +func (m *effectiveVersion) String() string { + if m == nil { + return "" + } + return fmt.Sprintf("{BinaryVersion: %s, EmulationVersion: %s, MinCompatibilityVersion: %s}", + m.BinaryVersion().String(), m.EmulationVersion().String(), m.MinCompatibilityVersion().String()) +} + +func majorMinor(ver *version.Version) *version.Version { + if ver == nil { + return ver + } + return version.MajorMinor(ver.Major(), ver.Minor()) +} + +func (m *effectiveVersion) SetEmulationVersion(emulationVersion *version.Version) { + m.emulationVersion.Store(majorMinor(emulationVersion)) + // set the default minCompatibilityVersion to be emulationVersion - 1 if possible + minCompatibilityVersion := majorMinor(emulationVersion.SubtractMinor(1)) + if minCompatibilityVersion.LessThan(m.minCompatibilityVersionFloor) { + minCompatibilityVersion = m.minCompatibilityVersionFloor + } + m.minCompatibilityVersion.Store(minCompatibilityVersion) +} + +// SetMinCompatibilityVersion should be called after SetEmulationVersion +func (m *effectiveVersion) SetMinCompatibilityVersion(minCompatibilityVersion *version.Version) { + m.minCompatibilityVersion.Store(majorMinor(minCompatibilityVersion)) +} + +func (m *effectiveVersion) AllowedEmulationVersionRange() string { + binaryVersion := m.BinaryVersion() + if binaryVersion == nil { + return "" + } + + // Consider patch version to be 0. + binaryVersion = version.MajorMinor(binaryVersion.Major(), binaryVersion.Minor()) + + floor := m.emulationVersionFloor + if floor == nil { + floor = version.MajorMinor(0, 0) + } + + return fmt.Sprintf("%s..%s (default=%s)", floor.String(), binaryVersion.String(), m.EmulationVersion().String()) +} + +func (m *effectiveVersion) AllowedMinCompatibilityVersionRange() string { + binaryVersion := m.BinaryVersion() + if binaryVersion == nil { + return "" + } + + // Consider patch version to be 0. + binaryVersion = version.MajorMinor(binaryVersion.Major(), binaryVersion.Minor()) + + floor := m.minCompatibilityVersionFloor + if floor == nil { + floor = version.MajorMinor(0, 0) + } + + return fmt.Sprintf("%s..%s (default=%s)", floor.String(), binaryVersion.String(), m.MinCompatibilityVersion().String()) +} + +func (m *effectiveVersion) Validate() []error { + var errs []error + // Validate only checks the major and minor versions. + binaryVersion := m.BinaryVersion().WithPatch(0) + emulationVersion := m.emulationVersion.Load() + minCompatibilityVersion := m.minCompatibilityVersion.Load() + // emulationVersion can only be between emulationVersionFloor and binaryVersion + if emulationVersion.GreaterThan(binaryVersion) || emulationVersion.LessThan(m.emulationVersionFloor) { + errs = append(errs, fmt.Errorf("emulation version %s is not between [%s, %s]", emulationVersion.String(), m.emulationVersionFloor.String(), binaryVersion.String())) + } + // minCompatibilityVersion can only be between minCompatibilityVersionFloor and emulationVersion + if minCompatibilityVersion.GreaterThan(emulationVersion) || minCompatibilityVersion.LessThan(m.minCompatibilityVersionFloor) { + errs = append(errs, fmt.Errorf("minCompatibilityVersion version %s is not between [%s, %s]", minCompatibilityVersion.String(), m.minCompatibilityVersionFloor.String(), emulationVersion.String())) + } + return errs +} + +// NewEffectiveVersion creates a MutableEffectiveVersion from the binaryVersion. +// If useDefaultBuildBinaryVersion is true, the call of BinaryVersion() will always return the current binary version. +// NewEffectiveVersion(binaryVersion, true) should only be used if the binary version is dynamic. +// Otherwise, use NewEffectiveVersion(binaryVersion, false) or NewEffectiveVersionFromString. +func NewEffectiveVersion(binaryVersion *version.Version, useDefaultBuildBinaryVersion bool, emulationVersionFloor, minCompatibilityVersionFloor *version.Version) MutableEffectiveVersion { + effective := &effectiveVersion{ + emulationVersionFloor: emulationVersionFloor, + minCompatibilityVersionFloor: minCompatibilityVersionFloor, + } + compatVersion := binaryVersion.SubtractMinor(1) + effective.binaryVersion.Store(binaryVersion) + effective.useDefaultBuildBinaryVersion.Store(useDefaultBuildBinaryVersion) + effective.SetEmulationVersion(binaryVersion) + effective.SetMinCompatibilityVersion(compatVersion) + return effective +} + +// NewEffectiveVersionFromString creates a MutableEffectiveVersion from the binaryVersion string. +func NewEffectiveVersionFromString(binaryVer, emulationVerFloor, minCompatibilityVerFloor string) MutableEffectiveVersion { + if binaryVer == "" { + return &effectiveVersion{} + } + binaryVersion := version.MustParse(binaryVer) + emulationVersionFloor := version.MajorMinor(0, 0) + if emulationVerFloor != "" { + emulationVersionFloor = version.MustParse(emulationVerFloor) + } + minCompatibilityVersionFloor := version.MajorMinor(0, 0) + if minCompatibilityVerFloor != "" { + minCompatibilityVersionFloor = version.MustParse(minCompatibilityVerFloor) + } + return NewEffectiveVersion(binaryVersion, false, emulationVersionFloor, minCompatibilityVersionFloor) +} + +func defaultBuildBinaryVersion() *version.Version { + verInfo := baseversion.Get() + return version.MustParse(verInfo.String()).WithInfo(verInfo) +} diff --git a/staging/src/k8s.io/component-base/compatibility/version_test.go b/staging/src/k8s.io/component-base/compatibility/version_test.go new file mode 100644 index 00000000000..ce25a13b7a0 --- /dev/null +++ b/staging/src/k8s.io/component-base/compatibility/version_test.go @@ -0,0 +1,148 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package compatibility + +import ( + "testing" + + "k8s.io/apimachinery/pkg/util/version" +) + +func TestValidate(t *testing.T) { + tests := []struct { + name string + binaryVersion string + emulationVersion string + minCompatibilityVersion string + emulationVersionFloor string + minCompatibilityVersionFloor string + expectErrors bool + }{ + { + name: "patch version diff ok", + binaryVersion: "v1.32.1", + emulationVersion: "v1.32.2", + minCompatibilityVersion: "v1.32.5", + }, + { + name: "emulation version greater than binary not ok", + binaryVersion: "v1.32.2", + emulationVersion: "v1.33.0", + minCompatibilityVersion: "v1.31.0", + expectErrors: true, + }, + { + name: "min compatibility version greater than emulation version not ok", + binaryVersion: "v1.32.2", + emulationVersion: "v1.31.0", + minCompatibilityVersion: "v1.32.0", + expectErrors: true, + }, + { + name: "between floor and binary ok", + binaryVersion: "v1.32.1", + emulationVersion: "v1.31.0", + minCompatibilityVersion: "v1.30.0", + emulationVersionFloor: "v1.31.0", + minCompatibilityVersionFloor: "v1.30.0", + }, + { + name: "emulation version less than floor not ok", + binaryVersion: "v1.32.1", + emulationVersion: "v1.30.0", + minCompatibilityVersion: "v1.30.0", + emulationVersionFloor: "v1.31.0", + minCompatibilityVersionFloor: "v1.30.0", + expectErrors: true, + }, + { + name: "min compatibility version less than floor not ok", + binaryVersion: "v1.32.1", + emulationVersion: "v1.31.0", + minCompatibilityVersion: "v1.29.0", + emulationVersionFloor: "v1.31.0", + minCompatibilityVersionFloor: "v1.30.0", + expectErrors: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + effective := NewEffectiveVersionFromString(test.binaryVersion, test.emulationVersionFloor, test.minCompatibilityVersionFloor) + emulationVersion := version.MustParseGeneric(test.emulationVersion) + minCompatibilityVersion := version.MustParseGeneric(test.minCompatibilityVersion) + effective.SetEmulationVersion(emulationVersion) + effective.SetMinCompatibilityVersion(minCompatibilityVersion) + + errs := effective.Validate() + if len(errs) > 0 && !test.expectErrors { + t.Errorf("expected no errors, errors found %+v", errs) + } + + if len(errs) == 0 && test.expectErrors { + t.Errorf("expected errors, no errors found") + } + }) + } +} + +func TestSetEmulationVersion(t *testing.T) { + tests := []struct { + name string + binaryVersion string + emulationVersion string + expectMinCompatibilityVersion string + emulationVersionFloor string + minCompatibilityVersionFloor string + }{ + { + name: "minCompatibilityVersion default to 1 minor less than emulationVersion", + binaryVersion: "v1.34", + emulationVersion: "v1.32", + expectMinCompatibilityVersion: "v1.31", + emulationVersionFloor: "v1.31", + minCompatibilityVersionFloor: "v1.31", + }, + { + name: "minCompatibilityVersion default to emulationVersion when hitting the floor", + binaryVersion: "v1.34", + emulationVersion: "v1.31", + expectMinCompatibilityVersion: "v1.31", + emulationVersionFloor: "v1.31", + minCompatibilityVersionFloor: "v1.31", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + effective := NewEffectiveVersionFromString(test.binaryVersion, test.emulationVersionFloor, test.minCompatibilityVersionFloor) + + emulationVersion := version.MustParseGeneric(test.emulationVersion) + effective.SetEmulationVersion(emulationVersion) + errs := effective.Validate() + if len(errs) > 0 { + t.Fatalf("expected no Validate errors, errors found %+v", errs) + return + } + + expectMinCompatibilityVersion := version.MustParseGeneric(test.expectMinCompatibilityVersion) + if !effective.MinCompatibilityVersion().EqualTo(expectMinCompatibilityVersion) { + t.Errorf("expected minCompatibilityVersion %s, got %s", expectMinCompatibilityVersion.String(), effective.MinCompatibilityVersion().String()) + } + }) + } +} diff --git a/staging/src/k8s.io/component-base/version/version.go b/staging/src/k8s.io/component-base/version/version.go index 0c62305839c..1d268d4c680 100644 --- a/staging/src/k8s.io/component-base/version/version.go +++ b/staging/src/k8s.io/component-base/version/version.go @@ -19,43 +19,10 @@ package version import ( "fmt" "runtime" - "sync/atomic" - "k8s.io/apimachinery/pkg/util/version" apimachineryversion "k8s.io/apimachinery/pkg/version" ) -var minimumKubeEmulationVersion *version.Version = version.MajorMinor(1, 31) - -type EffectiveVersion interface { - BinaryVersion() *version.Version - EmulationVersion() *version.Version - MinCompatibilityVersion() *version.Version - EqualTo(other EffectiveVersion) bool - String() string - Validate() []error -} - -type MutableEffectiveVersion interface { - EffectiveVersion - Set(binaryVersion, emulationVersion, minCompatibilityVersion *version.Version) - SetEmulationVersion(emulationVersion *version.Version) - SetMinCompatibilityVersion(minCompatibilityVersion *version.Version) -} - -type effectiveVersion struct { - // When true, BinaryVersion() returns the current binary version - useDefaultBuildBinaryVersion atomic.Bool - // Holds the last binary version stored in Set() - binaryVersion atomic.Pointer[version.Version] - // If the emulationVersion is set by the users, it could only contain major and minor versions. - // In tests, emulationVersion could be the same as the binary version, or set directly, - // which can have "alpha" as pre-release to continue serving expired apis while we clean up the test. - emulationVersion atomic.Pointer[version.Version] - // minCompatibilityVersion could only contain major and minor versions. - minCompatibilityVersion atomic.Pointer[version.Version] -} - // Get returns the overall codebase version. It's for detecting // what code a binary was built from. func Get() apimachineryversion.Info { @@ -73,134 +40,3 @@ func Get() apimachineryversion.Info { Platform: fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH), } } - -func (m *effectiveVersion) BinaryVersion() *version.Version { - if m.useDefaultBuildBinaryVersion.Load() { - return defaultBuildBinaryVersion() - } - return m.binaryVersion.Load() -} - -func (m *effectiveVersion) EmulationVersion() *version.Version { - ver := m.emulationVersion.Load() - if ver != nil { - // Emulation version can have "alpha" as pre-release to continue serving expired apis while we clean up the test. - // The pre-release should not be accessible to the users. - return ver.WithPreRelease(m.BinaryVersion().PreRelease()) - } - return ver -} - -func (m *effectiveVersion) MinCompatibilityVersion() *version.Version { - return m.minCompatibilityVersion.Load() -} - -func (m *effectiveVersion) EqualTo(other EffectiveVersion) bool { - return m.BinaryVersion().EqualTo(other.BinaryVersion()) && m.EmulationVersion().EqualTo(other.EmulationVersion()) && m.MinCompatibilityVersion().EqualTo(other.MinCompatibilityVersion()) -} - -func (m *effectiveVersion) String() string { - if m == nil { - return "" - } - return fmt.Sprintf("{BinaryVersion: %s, EmulationVersion: %s, MinCompatibilityVersion: %s}", - m.BinaryVersion().String(), m.EmulationVersion().String(), m.MinCompatibilityVersion().String()) -} - -func majorMinor(ver *version.Version) *version.Version { - if ver == nil { - return ver - } - return version.MajorMinor(ver.Major(), ver.Minor()) -} - -func (m *effectiveVersion) Set(binaryVersion, emulationVersion, minCompatibilityVersion *version.Version) { - m.binaryVersion.Store(binaryVersion) - m.useDefaultBuildBinaryVersion.Store(false) - m.emulationVersion.Store(majorMinor(emulationVersion)) - m.minCompatibilityVersion.Store(majorMinor(minCompatibilityVersion)) -} - -func (m *effectiveVersion) SetEmulationVersion(emulationVersion *version.Version) { - m.emulationVersion.Store(majorMinor(emulationVersion)) - // set the default minCompatibilityVersion to be emulationVersion - 1 - m.minCompatibilityVersion.Store(majorMinor(emulationVersion.SubtractMinor(1))) -} - -// SetMinCompatibilityVersion should be called after SetEmulationVersion -func (m *effectiveVersion) SetMinCompatibilityVersion(minCompatibilityVersion *version.Version) { - m.minCompatibilityVersion.Store(majorMinor(minCompatibilityVersion)) -} - -func (m *effectiveVersion) Validate() []error { - var errs []error - // Validate only checks the major and minor versions. - binaryVersion := m.BinaryVersion().WithPatch(0) - emulationVersion := m.emulationVersion.Load() - minCompatibilityVersion := m.minCompatibilityVersion.Load() - - // emulationVersion can only be 1.{binaryMinor-3}...1.{binaryMinor} - maxEmuVer := binaryVersion - minEmuVer := binaryVersion.SubtractMinor(3) - if emulationVersion.GreaterThan(maxEmuVer) || emulationVersion.LessThan(minEmuVer) { - errs = append(errs, fmt.Errorf("emulation version %s is not between [%s, %s]", emulationVersion.String(), minEmuVer.String(), maxEmuVer.String())) - } - // minCompatibilityVersion can only be 1.{binaryMinor-3} to 1.{binaryMinor} - maxCompVer := emulationVersion - minCompVer := binaryVersion.SubtractMinor(4) - if minCompatibilityVersion.GreaterThan(maxCompVer) || minCompatibilityVersion.LessThan(minCompVer) { - errs = append(errs, fmt.Errorf("minCompatibilityVersion version %s is not between [%s, %s]", minCompatibilityVersion.String(), minCompVer.String(), maxCompVer.String())) - } - return errs -} - -func newEffectiveVersion(binaryVersion *version.Version, useDefaultBuildBinaryVersion bool) MutableEffectiveVersion { - effective := &effectiveVersion{} - compatVersion := binaryVersion.SubtractMinor(1) - effective.Set(binaryVersion, binaryVersion, compatVersion) - effective.useDefaultBuildBinaryVersion.Store(useDefaultBuildBinaryVersion) - return effective -} - -func NewEffectiveVersion(binaryVer string) MutableEffectiveVersion { - if binaryVer == "" { - return &effectiveVersion{} - } - binaryVersion := version.MustParse(binaryVer) - return newEffectiveVersion(binaryVersion, false) -} - -func defaultBuildBinaryVersion() *version.Version { - verInfo := Get() - return version.MustParse(verInfo.String()).WithInfo(verInfo) -} - -// DefaultBuildEffectiveVersion returns the MutableEffectiveVersion based on the -// current build information. -func DefaultBuildEffectiveVersion() MutableEffectiveVersion { - binaryVersion := defaultBuildBinaryVersion() - if binaryVersion.Major() == 0 && binaryVersion.Minor() == 0 { - return DefaultKubeEffectiveVersion() - } - return newEffectiveVersion(binaryVersion, true) -} - -// DefaultKubeEffectiveVersion returns the MutableEffectiveVersion based on the -// latest K8s release. -func DefaultKubeEffectiveVersion() MutableEffectiveVersion { - binaryVersion := version.MustParse(DefaultKubeBinaryVersion).WithInfo(Get()) - return newEffectiveVersion(binaryVersion, false) -} - -// ValidateKubeEffectiveVersion validates the EmulationVersion is at least 1.31 and MinCompatibilityVersion -// is at least 1.30 for kube components. -func ValidateKubeEffectiveVersion(effectiveVersion EffectiveVersion) error { - if !effectiveVersion.EmulationVersion().AtLeast(minimumKubeEmulationVersion) { - return fmt.Errorf("emulation version needs to be greater or equal to 1.31, got %s", effectiveVersion.EmulationVersion().String()) - } - if !effectiveVersion.MinCompatibilityVersion().AtLeast(minimumKubeEmulationVersion.SubtractMinor(1)) { - return fmt.Errorf("minCompatibilityVersion version needs to be greater or equal to 1.30, got %s", effectiveVersion.MinCompatibilityVersion().String()) - } - - return nil -} diff --git a/staging/src/k8s.io/component-base/version/version_test.go b/staging/src/k8s.io/component-base/version/version_test.go deleted file mode 100644 index 3f6c34bd46e..00000000000 --- a/staging/src/k8s.io/component-base/version/version_test.go +++ /dev/null @@ -1,185 +0,0 @@ -/* -Copyright 2024 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package version - -import ( - "testing" - - "k8s.io/apimachinery/pkg/util/version" -) - -func TestValidate(t *testing.T) { - tests := []struct { - name string - binaryVersion string - emulationVersion string - minCompatibilityVersion string - expectErrors bool - }{ - { - name: "patch version diff ok", - binaryVersion: "v1.32.2", - emulationVersion: "v1.32.1", - minCompatibilityVersion: "v1.31.5", - }, - { - name: "emulation version one minor lower than binary ok", - binaryVersion: "v1.32.2", - emulationVersion: "v1.31.0", - minCompatibilityVersion: "v1.31.0", - }, - { - name: "emulation version two minor lower than binary ok", - binaryVersion: "v1.33.2", - emulationVersion: "v1.31.0", - minCompatibilityVersion: "v1.31.0", - expectErrors: false, - }, - { - name: "emulation version three minor lower than binary ok", - binaryVersion: "v1.35.0", - emulationVersion: "v1.32.0", - minCompatibilityVersion: "v1.32.0", - }, - { - name: "emulation version four minor lower than binary not ok", - binaryVersion: "v1.36.0", - emulationVersion: "v1.32.0", - minCompatibilityVersion: "v1.32.0", - expectErrors: true, - }, - { - name: "emulation version one minor higher than binary not ok", - binaryVersion: "v1.32.2", - emulationVersion: "v1.33.0", - minCompatibilityVersion: "v1.31.0", - expectErrors: true, - }, - { - name: "emulation version two minor higher than binary not ok", - binaryVersion: "v1.32.2", - emulationVersion: "v1.34.0", - minCompatibilityVersion: "v1.31.0", - expectErrors: true, - }, - { - name: "compatibility version same as binary ok", - binaryVersion: "v1.32.2", - emulationVersion: "v1.32.0", - minCompatibilityVersion: "v1.32.0", - expectErrors: false, - }, - { - name: "compatibility version two minor lower than binary ok", - binaryVersion: "v1.32.2", - emulationVersion: "v1.32.0", - minCompatibilityVersion: "v1.30.0", - expectErrors: false, - }, - { - name: "compatibility version three minor lower than binary ok", - binaryVersion: "v1.34.2", - emulationVersion: "v1.33.0", - minCompatibilityVersion: "v1.31.0", - expectErrors: false, - }, - { - name: "compatibility version one minor higher than binary not ok", - binaryVersion: "v1.32.2", - emulationVersion: "v1.32.0", - minCompatibilityVersion: "v1.33.0", - expectErrors: true, - }, - { - name: "emulation version lower than compatibility version not ok", - binaryVersion: "v1.34.2", - emulationVersion: "v1.32.0", - minCompatibilityVersion: "v1.33.0", - expectErrors: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - binaryVersion := version.MustParseGeneric(test.binaryVersion) - effective := &effectiveVersion{} - emulationVersion := version.MustParseGeneric(test.emulationVersion) - minCompatibilityVersion := version.MustParseGeneric(test.minCompatibilityVersion) - effective.Set(binaryVersion, emulationVersion, minCompatibilityVersion) - - errs := effective.Validate() - if len(errs) > 0 && !test.expectErrors { - t.Errorf("expected no errors, errors found %+v", errs) - } - - if len(errs) == 0 && test.expectErrors { - t.Errorf("expected errors, no errors found") - } - }) - } -} - -func TestValidateKubeEffectiveVersion(t *testing.T) { - tests := []struct { - name string - emulationVersion string - minCompatibilityVersion string - expectErr bool - }{ - { - name: "valid versions", - emulationVersion: "v1.31.0", - minCompatibilityVersion: "v1.31.0", - expectErr: false, - }, - { - name: "emulationVersion too low", - emulationVersion: "v1.30.0", - minCompatibilityVersion: "v1.31.0", - expectErr: true, - }, - { - name: "minCompatibilityVersion too low", - emulationVersion: "v1.31.0", - minCompatibilityVersion: "v1.29.0", - expectErr: true, - }, - { - name: "both versions too low", - emulationVersion: "v1.30.0", - minCompatibilityVersion: "v1.30.0", - expectErr: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - - effective := NewEffectiveVersion("1.32") - effective.SetEmulationVersion(version.MustParseGeneric(test.emulationVersion)) - effective.SetMinCompatibilityVersion(version.MustParseGeneric(test.minCompatibilityVersion)) - - err := ValidateKubeEffectiveVersion(effective) - if test.expectErr && err == nil { - t.Error("expected error, but got nil") - } - if !test.expectErr && err != nil { - t.Errorf("unexpected error: %v", err) - } - }) - } -} diff --git a/staging/src/k8s.io/component-base/zpages/statusz/registry.go b/staging/src/k8s.io/component-base/zpages/statusz/registry.go index 1165de9d3d8..226e7fa0abf 100644 --- a/staging/src/k8s.io/component-base/zpages/statusz/registry.go +++ b/staging/src/k8s.io/component-base/zpages/statusz/registry.go @@ -20,9 +20,9 @@ import ( "time" "k8s.io/apimachinery/pkg/util/version" - "k8s.io/component-base/featuregate" "k8s.io/klog/v2" + "k8s.io/component-base/compatibility" compbasemetrics "k8s.io/component-base/metrics" utilversion "k8s.io/component-base/version" ) @@ -34,9 +34,12 @@ type statuszRegistry interface { emulationVersion() *version.Version } -type registry struct{} +type registry struct { + // componentGlobalsRegistry compatibility.ComponentGlobalsRegistry + effectiveVersion compatibility.EffectiveVersion +} -func (registry) processStartTime() time.Time { +func (*registry) processStartTime() time.Time { start, err := compbasemetrics.GetProcessStart() if err != nil { klog.Errorf("Could not get process start time, %v", err) @@ -45,23 +48,20 @@ func (registry) processStartTime() time.Time { return time.Unix(int64(start), 0) } -func (registry) goVersion() string { +func (*registry) goVersion() string { return utilversion.Get().GoVersion } -func (registry) binaryVersion() *version.Version { - effectiveVer := featuregate.DefaultComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent) - if effectiveVer != nil { - return effectiveVer.BinaryVersion() +func (r *registry) binaryVersion() *version.Version { + if r.effectiveVersion != nil { + return r.effectiveVersion.BinaryVersion() } - - return utilversion.DefaultKubeEffectiveVersion().BinaryVersion() + return version.MustParse(utilversion.Get().String()) } -func (registry) emulationVersion() *version.Version { - effectiveVer := featuregate.DefaultComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent) - if effectiveVer != nil { - return effectiveVer.EmulationVersion() +func (r *registry) emulationVersion() *version.Version { + if r.effectiveVersion != nil { + return r.effectiveVersion.EmulationVersion() } return nil diff --git a/staging/src/k8s.io/component-base/zpages/statusz/registry_test.go b/staging/src/k8s.io/component-base/zpages/statusz/registry_test.go index 738e2aba85c..3d866953fdb 100644 --- a/staging/src/k8s.io/component-base/zpages/statusz/registry_test.go +++ b/staging/src/k8s.io/component-base/zpages/statusz/registry_test.go @@ -20,14 +20,12 @@ import ( "testing" "github.com/stretchr/testify/assert" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/version" - "k8s.io/component-base/featuregate" + "k8s.io/component-base/compatibility" utilversion "k8s.io/component-base/version" ) func TestBinaryVersion(t *testing.T) { - componentGlobalsRegistry := featuregate.DefaultComponentGlobalsRegistry tests := []struct { name string setFakeEffectiveVersion bool @@ -42,20 +40,18 @@ func TestBinaryVersion(t *testing.T) { }, { name: "binaryVersion without effective version", - wantBinaryVersion: utilversion.DefaultKubeEffectiveVersion().BinaryVersion(), + wantBinaryVersion: version.MustParse(utilversion.Get().String()), }, } for _, tt := range tests { - componentGlobalsRegistry.Reset() t.Run(tt.name, func(t *testing.T) { + registry := ®istry{} if tt.setFakeEffectiveVersion { - verKube := utilversion.NewEffectiveVersion(tt.fakeVersion) - fg := featuregate.NewVersionedFeatureGate(version.MustParse(tt.fakeVersion)) - utilruntime.Must(componentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, verKube, fg)) + verKube := compatibility.NewEffectiveVersionFromString(tt.fakeVersion, "", "") + registry.effectiveVersion = verKube } - registry := ®istry{} got := registry.binaryVersion() assert.Equal(t, tt.wantBinaryVersion, got) }) @@ -63,7 +59,6 @@ func TestBinaryVersion(t *testing.T) { } func TestEmulationVersion(t *testing.T) { - componentGlobalsRegistry := featuregate.DefaultComponentGlobalsRegistry tests := []struct { name string setFakeEffectiveVersion bool @@ -83,16 +78,14 @@ func TestEmulationVersion(t *testing.T) { } for _, tt := range tests { - componentGlobalsRegistry.Reset() t.Run(tt.name, func(t *testing.T) { + registry := ®istry{} if tt.setFakeEffectiveVersion { - verKube := utilversion.NewEffectiveVersion("0.0.0") + verKube := compatibility.NewEffectiveVersionFromString("0.0.0", "", "") verKube.SetEmulationVersion(version.MustParse(tt.fakeEmulVer)) - fg := featuregate.NewVersionedFeatureGate(version.MustParse(tt.fakeEmulVer)) - utilruntime.Must(componentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, verKube, fg)) + registry.effectiveVersion = verKube } - registry := ®istry{} got := registry.emulationVersion() if tt.wantEmul != nil && got != nil { assert.Equal(t, tt.wantEmul.Major(), got.Major()) diff --git a/staging/src/k8s.io/component-base/zpages/statusz/statusz.go b/staging/src/k8s.io/component-base/zpages/statusz/statusz.go index 7b284b81038..ef0814e3600 100644 --- a/staging/src/k8s.io/component-base/zpages/statusz/statusz.go +++ b/staging/src/k8s.io/component-base/zpages/statusz/statusz.go @@ -24,6 +24,7 @@ import ( "net/http" "time" + "k8s.io/component-base/compatibility" "k8s.io/component-base/zpages/httputil" "k8s.io/klog/v2" ) @@ -62,8 +63,8 @@ type mux interface { Handle(path string, handler http.Handler) } -func NewRegistry() statuszRegistry { - return registry{} +func NewRegistry(effectiveVersion compatibility.EffectiveVersion) statuszRegistry { + return ®istry{effectiveVersion: effectiveVersion} } func Install(m mux, componentName string, reg statuszRegistry) { diff --git a/staging/src/k8s.io/cri-client/go.mod b/staging/src/k8s.io/cri-client/go.mod index 05c2d59e095..86b8907af2b 100644 --- a/staging/src/k8s.io/cri-client/go.mod +++ b/staging/src/k8s.io/cri-client/go.mod @@ -45,7 +45,6 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -57,7 +56,6 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect - github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect diff --git a/staging/src/k8s.io/cri-client/go.sum b/staging/src/k8s.io/cri-client/go.sum index c2a286f9322..6517078ed96 100644 --- a/staging/src/k8s.io/cri-client/go.sum +++ b/staging/src/k8s.io/cri-client/go.sum @@ -18,7 +18,6 @@ github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -70,7 +69,6 @@ github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -117,9 +115,7 @@ github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoG github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= diff --git a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go index 32f6368ef57..89fda03982d 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go @@ -31,7 +31,6 @@ import ( genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/filters" genericoptions "k8s.io/apiserver/pkg/server/options" - "k8s.io/component-base/featuregate" "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1" "k8s.io/kube-aggregator/pkg/apiserver" aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" @@ -63,7 +62,7 @@ func NewCommandStartAggregator(ctx context.Context, defaults *AggregatorOptions) Short: "Launch a API aggregator and proxy server", Long: "Launch a API aggregator and proxy server", PersistentPreRunE: func(*cobra.Command, []string) error { - return featuregate.DefaultComponentGlobalsRegistry.Set() + return o.ServerRunOptions.ComponentGlobalsRegistry.Set() }, RunE: func(c *cobra.Command, args []string) error { if err := o.Complete(); err != nil { diff --git a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go index f40c44552da..90f48b49478 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go @@ -33,9 +33,11 @@ import ( "k8s.io/apiserver/pkg/endpoints/openapi" genericapiserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" + "k8s.io/apiserver/pkg/util/compatibility" utilfeature "k8s.io/apiserver/pkg/util/feature" + basecompatibility "k8s.io/component-base/compatibility" "k8s.io/component-base/featuregate" - utilversion "k8s.io/component-base/version" + baseversion "k8s.io/component-base/version" "k8s.io/sample-apiserver/pkg/admission/plugin/banflunder" "k8s.io/sample-apiserver/pkg/admission/wardleinitializer" "k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1" @@ -51,6 +53,8 @@ const defaultEtcdPathPrefix = "/registry/wardle.example.com" // WardleServerOptions contains state for master/api server type WardleServerOptions struct { RecommendedOptions *genericoptions.RecommendedOptions + // ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored. + ComponentGlobalsRegistry basecompatibility.ComponentGlobalsRegistry SharedInformerFactory informers.SharedInformerFactory StdOut io.Writer @@ -63,7 +67,7 @@ func WardleVersionToKubeVersion(ver *version.Version) *version.Version { if ver.Major() != 1 { return nil } - kubeVer := utilversion.DefaultKubeEffectiveVersion().BinaryVersion() + kubeVer := version.MustParse(baseversion.DefaultKubeBinaryVersion) // "1.2" maps to kubeVer offset := int(ver.Minor()) - 2 mappedVer := kubeVer.OffsetMinor(offset) @@ -80,6 +84,7 @@ func NewWardleServerOptions(out, errOut io.Writer) *WardleServerOptions { defaultEtcdPathPrefix, apiserver.Codecs.LegacyCodec(v1alpha1.SchemeGroupVersion), ), + ComponentGlobalsRegistry: compatibility.DefaultComponentGlobalsRegistry, StdOut: out, StdErr: errOut, @@ -99,7 +104,7 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti if skipDefaultComponentGlobalsRegistrySet { return nil } - return featuregate.DefaultComponentGlobalsRegistry.Set() + return defaults.ComponentGlobalsRegistry.Set() }, RunE: func(c *cobra.Command, args []string) error { if err := o.Complete(); err != nil { @@ -135,8 +140,8 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti // Register the "Wardle" component with the global component registry, // associating it with its effective version and feature gate configuration. // Will skip if the component has been registered, like in the integration test. - _, wardleFeatureGate := featuregate.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( - apiserver.WardleComponentName, utilversion.NewEffectiveVersion(defaultWardleVersion), + _, wardleFeatureGate := defaults.ComponentGlobalsRegistry.ComponentGlobalsOrRegister( + apiserver.WardleComponentName, basecompatibility.NewEffectiveVersionFromString(defaultWardleVersion, "", ""), featuregate.NewVersionedFeatureGate(version.MustParse(defaultWardleVersion))) // Add versioned feature specifications for the "BanFlunder" feature. @@ -150,14 +155,14 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti })) // Register the default kube component if not already present in the global registry. - _, _ = featuregate.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(featuregate.DefaultKubeComponent, - utilversion.NewEffectiveVersion(utilversion.DefaultKubeBinaryVersion), utilfeature.DefaultMutableFeatureGate) + _, _ = defaults.ComponentGlobalsRegistry.ComponentGlobalsOrRegister(basecompatibility.DefaultKubeComponent, + basecompatibility.NewEffectiveVersionFromString(baseversion.DefaultKubeBinaryVersion, "", ""), utilfeature.DefaultMutableFeatureGate) // Set the emulation version mapping from the "Wardle" component to the kube component. // This ensures that the emulation version of the latter is determined by the emulation version of the former. - utilruntime.Must(featuregate.DefaultComponentGlobalsRegistry.SetEmulationVersionMapping(apiserver.WardleComponentName, featuregate.DefaultKubeComponent, WardleVersionToKubeVersion)) + utilruntime.Must(defaults.ComponentGlobalsRegistry.SetEmulationVersionMapping(apiserver.WardleComponentName, basecompatibility.DefaultKubeComponent, WardleVersionToKubeVersion)) - featuregate.DefaultComponentGlobalsRegistry.AddFlags(flags) + defaults.ComponentGlobalsRegistry.AddFlags(flags) return cmd } @@ -166,13 +171,13 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti func (o WardleServerOptions) Validate(args []string) error { errors := []error{} errors = append(errors, o.RecommendedOptions.Validate()...) - errors = append(errors, featuregate.DefaultComponentGlobalsRegistry.Validate()...) + errors = append(errors, o.ComponentGlobalsRegistry.Validate()...) return utilerrors.NewAggregate(errors) } // Complete fills in fields required to have valid data func (o *WardleServerOptions) Complete() error { - if featuregate.DefaultComponentGlobalsRegistry.FeatureGateFor(apiserver.WardleComponentName).Enabled("BanFlunder") { + if o.ComponentGlobalsRegistry.FeatureGateFor(apiserver.WardleComponentName).Enabled("BanFlunder") { // register admission plugins banflunder.Register(o.RecommendedOptions.Admission.Plugins) @@ -209,8 +214,8 @@ func (o *WardleServerOptions) Config() (*apiserver.Config, error) { serverConfig.OpenAPIV3Config.Info.Title = "Wardle" serverConfig.OpenAPIV3Config.Info.Version = "0.1" - serverConfig.FeatureGate = featuregate.DefaultComponentGlobalsRegistry.FeatureGateFor(featuregate.DefaultKubeComponent) - serverConfig.EffectiveVersion = featuregate.DefaultComponentGlobalsRegistry.EffectiveVersionFor(apiserver.WardleComponentName) + serverConfig.FeatureGate = o.ComponentGlobalsRegistry.FeatureGateFor(basecompatibility.DefaultKubeComponent) + serverConfig.EffectiveVersion = o.ComponentGlobalsRegistry.EffectiveVersionFor(apiserver.WardleComponentName) if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil { return nil, err diff --git a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go index cfd5672291c..aff528e5a8e 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go @@ -20,13 +20,13 @@ import ( "testing" "k8s.io/apimachinery/pkg/util/version" - utilversion "k8s.io/component-base/version" + "k8s.io/apiserver/pkg/util/compatibility" "github.com/stretchr/testify/assert" ) func TestWardleEmulationVersionToKubeEmulationVersion(t *testing.T) { - defaultKubeEffectiveVersion := utilversion.DefaultKubeEffectiveVersion() + defaultKubeEffectiveVersion := compatibility.DefaultKubeEffectiveVersionForTest() testCases := []struct { desc string diff --git a/test/integration/apiserver/admissionwebhook/match_conditions_test.go b/test/integration/apiserver/admissionwebhook/match_conditions_test.go index f35076dd59b..f79f158863d 100644 --- a/test/integration/apiserver/admissionwebhook/match_conditions_test.go +++ b/test/integration/apiserver/admissionwebhook/match_conditions_test.go @@ -22,7 +22,6 @@ import ( "crypto/x509" "encoding/json" "io" - "k8s.io/apimachinery/pkg/util/version" "net/http" "net/http/httptest" "strconv" @@ -634,9 +633,9 @@ func TestMatchConditionsWithoutStrictCostEnforcement(t *testing.T) { for _, testcase := range testcases { t.Run(testcase.name, func(t *testing.T) { upCh := recorder.Reset() - featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31")) - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.StrictCostEnforcementForWebhooks, false) - server, err := apiservertesting.StartTestServer(t, &apiservertesting.TestServerInstanceOptions{EmulationVersion: "1.31"}, []string{ + server, err := apiservertesting.StartTestServer(t, nil, []string{ + "--emulated-version", "1.31", + "--feature-gates", "StrictCostEnforcementForWebhooks=false", "--disable-admission-plugins=ServiceAccount", }, framework.SharedEtcd()) if err != nil { diff --git a/test/integration/apiserver/apiserver_test.go b/test/integration/apiserver/apiserver_test.go index 4e0e0205038..3eeb0bc0e45 100644 --- a/test/integration/apiserver/apiserver_test.go +++ b/test/integration/apiserver/apiserver_test.go @@ -56,10 +56,12 @@ import ( "k8s.io/apimachinery/pkg/types" utiljson "k8s.io/apimachinery/pkg/util/json" "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/endpoints/handlers" "k8s.io/apiserver/pkg/storage/storagebackend" + "k8s.io/apiserver/pkg/util/compatibility" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/discovery/cached/memory" "k8s.io/client-go/dynamic" @@ -70,7 +72,6 @@ import ( "k8s.io/client-go/restmapper" "k8s.io/client-go/tools/pager" featuregatetesting "k8s.io/component-base/featuregate/testing" - utilversion "k8s.io/component-base/version" "k8s.io/klog/v2" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" @@ -3198,8 +3199,7 @@ func TestEmulatedStorageVersion(t *testing.T) { for emulatedVersion, cases := range groupedCases { t.Run(emulatedVersion, func(t *testing.T) { server := kubeapiservertesting.StartTestServerOrDie( - t, &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: emulatedVersion}, - []string{"--emulated-version=kube=" + emulatedVersion, `--storage-media-type=application/json`}, framework.SharedEtcd()) + t, nil, []string{"--emulated-version=kube=" + emulatedVersion, `--storage-media-type=application/json`}, framework.SharedEtcd()) defer server.TearDownFn() client := clientset.NewForConfigOrDie(server.ClientConfig) @@ -3302,7 +3302,7 @@ func TestAllowedEmulationVersions(t *testing.T) { }{ { name: "default", - emulationVersion: utilversion.DefaultKubeEffectiveVersion().EmulationVersion().String(), + emulationVersion: compatibility.DefaultKubeEffectiveVersionForTest().EmulationVersion().String(), }, } @@ -3337,6 +3337,7 @@ func TestAllowedEmulationVersions(t *testing.T) { } func TestEnableEmulationVersion(t *testing.T) { + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.32")) server := kubeapiservertesting.StartTestServerOrDie(t, &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "1.32"}, []string{"--emulated-version=kube=1.31"}, framework.SharedEtcd()) @@ -3398,6 +3399,7 @@ func TestEnableEmulationVersion(t *testing.T) { } func TestDisableEmulationVersion(t *testing.T) { + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.32")) server := kubeapiservertesting.StartTestServerOrDie(t, &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "1.32"}, []string{}, framework.SharedEtcd()) diff --git a/test/integration/apiserver/cel/validatingadmissionpolicy_test.go b/test/integration/apiserver/cel/validatingadmissionpolicy_test.go index 72e2fac9422..3e588bcf01d 100644 --- a/test/integration/apiserver/cel/validatingadmissionpolicy_test.go +++ b/test/integration/apiserver/cel/validatingadmissionpolicy_test.go @@ -20,7 +20,6 @@ import ( "context" "encoding/json" "fmt" - "k8s.io/apimachinery/pkg/util/version" "net/http" "net/http/httptest" "os" @@ -2235,9 +2234,9 @@ func Test_CostLimitForValidation(t *testing.T) { func Test_CostLimitForValidationWithFeatureDisabled(t *testing.T) { resetPolicyRefreshInterval := generic.SetPolicyRefreshIntervalForTests(policyRefreshInterval) defer resetPolicyRefreshInterval() - featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31")) - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.StrictCostEnforcementForVAP, false) - server, err := apiservertesting.StartTestServer(t, &apiservertesting.TestServerInstanceOptions{EmulationVersion: "1.31"}, []string{ + server, err := apiservertesting.StartTestServer(t, nil, []string{ + "--emulated-version", "1.31", + "--feature-gates", "StrictCostEnforcementForVAP=false", "--enable-admission-plugins", "ValidatingAdmissionPolicy", }, framework.SharedEtcd()) if err != nil { diff --git a/test/integration/client/client_test.go b/test/integration/client/client_test.go index 7f51c941798..3314b631c46 100644 --- a/test/integration/client/client_test.go +++ b/test/integration/client/client_test.go @@ -47,6 +47,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/features" + "k8s.io/apiserver/pkg/util/compatibility" utilfeature "k8s.io/apiserver/pkg/util/feature" appsv1ac "k8s.io/client-go/applyconfigurations/apps/v1" autoscalingv1ac "k8s.io/client-go/applyconfigurations/autoscaling/v1" @@ -83,7 +84,7 @@ func TestClient(t *testing.T) { t.Fatalf("unexpected error: %v", err) } expectedInfo := utilversion.Get() - kubeVersion := utilversion.DefaultKubeEffectiveVersion().BinaryVersion() + kubeVersion := compatibility.DefaultKubeEffectiveVersionForTest().BinaryVersion() expectedInfo.Major = fmt.Sprintf("%d", kubeVersion.Major()) expectedInfo.Minor = fmt.Sprintf("%d", kubeVersion.Minor()) diff --git a/test/integration/controlplane/transformation/transformation_test.go b/test/integration/controlplane/transformation/transformation_test.go index 1bd859f7483..c0c95ce230a 100644 --- a/test/integration/controlplane/transformation/transformation_test.go +++ b/test/integration/controlplane/transformation/transformation_test.go @@ -25,7 +25,6 @@ import ( "path/filepath" "strconv" "strings" - "sync" "testing" "time" @@ -117,7 +116,7 @@ func newTransformTest(tb testing.TB, transformerConfigYAML string, reload bool, return nil, fmt.Errorf("failed to read config file: %w", err) } - if e.kubeAPIServer, err = startTestServerLocked( + if e.kubeAPIServer, err = kubeapiservertesting.StartTestServer( tb, nil, e.getEncryptionOptions(reload), e.storageConfig); err != nil { e.cleanUp() @@ -148,15 +147,6 @@ func newTransformTest(tb testing.TB, transformerConfigYAML string, reload bool, return &e, nil } -var startTestServerLock sync.Mutex - -// startTestServerLocked prevents parallel calls to kubeapiservertesting.StartTestServer because it messes with global state. -func startTestServerLocked(t ktesting.TB, instanceOptions *kubeapiservertesting.TestServerInstanceOptions, customFlags []string, storageConfig *storagebackend.Config) (result kubeapiservertesting.TestServer, err error) { - startTestServerLock.Lock() - defer startTestServerLock.Unlock() - return kubeapiservertesting.StartTestServer(t, instanceOptions, customFlags, storageConfig) -} - func (e *transformTest) cleanUp() { if e.configDir != "" { os.RemoveAll(e.configDir) diff --git a/test/integration/etcd/data.go b/test/integration/etcd/data.go index 03ad87ad582..8b8031d0f55 100644 --- a/test/integration/etcd/data.go +++ b/test/integration/etcd/data.go @@ -25,6 +25,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/version" + "k8s.io/apiserver/pkg/util/compatibility" utilversion "k8s.io/component-base/version" "k8s.io/kubernetes/test/utils/image" @@ -34,9 +35,9 @@ import ( // Tests aiming for full coverage of versions should test fixtures of all supported versions. func GetSupportedEmulatedVersions() []string { return []string{ - utilversion.DefaultKubeEffectiveVersion().BinaryVersion().SubtractMinor(2).String(), - utilversion.DefaultKubeEffectiveVersion().BinaryVersion().SubtractMinor(1).String(), - utilversion.DefaultKubeEffectiveVersion().BinaryVersion().String(), + compatibility.DefaultKubeEffectiveVersionForTest().BinaryVersion().SubtractMinor(2).String(), + compatibility.DefaultKubeEffectiveVersionForTest().BinaryVersion().SubtractMinor(1).String(), + compatibility.DefaultKubeEffectiveVersionForTest().BinaryVersion().String(), } } diff --git a/test/integration/etcd/etcd_storage_path_test.go b/test/integration/etcd/etcd_storage_path_test.go index 768d9a540e5..c5738509a69 100644 --- a/test/integration/etcd/etcd_storage_path_test.go +++ b/test/integration/etcd/etcd_storage_path_test.go @@ -104,7 +104,6 @@ func testEtcdStoragePathWithVersion(t *testing.T, v string) { // only understand v1beta1. featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.MultiCIDRServiceAllocator, false) } - registerEffectiveEmulationVersion(t) apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) { // Disable alphas when emulating previous versions. diff --git a/test/integration/etcd/server.go b/test/integration/etcd/server.go index cb8d71a335b..fff4882c739 100644 --- a/test/integration/etcd/server.go +++ b/test/integration/etcd/server.go @@ -36,9 +36,9 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/json" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" genericapiserveroptions "k8s.io/apiserver/pkg/server/options" + "k8s.io/apiserver/pkg/util/compatibility" "k8s.io/apiserver/pkg/util/feature" cacheddiscovery "k8s.io/client-go/discovery/cached/memory" "k8s.io/client-go/dynamic" @@ -46,9 +46,8 @@ import ( restclient "k8s.io/client-go/rest" "k8s.io/client-go/restmapper" utiltesting "k8s.io/client-go/util/testing" - "k8s.io/component-base/featuregate" + basecompatibility "k8s.io/component-base/compatibility" featuregatetesting "k8s.io/component-base/featuregate/testing" - utilversion "k8s.io/component-base/version" "k8s.io/kubernetes/cmd/kube-apiserver/app" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" "k8s.io/kubernetes/test/integration" @@ -67,17 +66,6 @@ AwEHoUQDQgAEH6cuzP8XuD5wal6wf9M6xDljTOPLX2i8uIp/C/ASqiIGUeeKQtX0 /IR3qCXyThP/dbCiHrF3v1cuhBOHY8CLVg== -----END EC PRIVATE KEY-----` -func registerEffectiveEmulationVersion(t *testing.T) { - featureGate := feature.DefaultMutableFeatureGate - featureGate.AddMetrics() - - effectiveVersion := utilversion.DefaultKubeEffectiveVersion() - effectiveVersion.SetEmulationVersion(featureGate.EmulationVersion()) - featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, featureGate, effectiveVersion.EmulationVersion()) - featuregate.DefaultComponentGlobalsRegistry.Reset() - utilruntime.Must(featuregate.DefaultComponentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, effectiveVersion, featureGate)) -} - // StartRealAPIServerOrDie starts an API server that is appropriate for use in tests that require one of every resource func StartRealAPIServerOrDie(t *testing.T, configFuncs ...func(*options.ServerRunOptions)) *APIServer { tCtx := ktesting.Init(t) @@ -108,6 +96,17 @@ func StartRealAPIServerOrDie(t *testing.T, configFuncs ...func(*options.ServerRu } opts := options.NewServerRunOptions() + // If EmulationVersion of DefaultFeatureGate is set during test, we need to propagate it to the apiserver ComponentGlobalsRegistry. + featureGate := feature.DefaultMutableFeatureGate.DeepCopy() + effectiveVersion := compatibility.DefaultKubeEffectiveVersionForTest() + effectiveVersion.SetEmulationVersion(featureGate.EmulationVersion()) + // set up new instance of ComponentGlobalsRegistry instead of using the DefaultComponentGlobalsRegistry to avoid contention in parallel tests. + componentGlobalsRegistry := basecompatibility.NewComponentGlobalsRegistry() + if err := componentGlobalsRegistry.Register(basecompatibility.DefaultKubeComponent, effectiveVersion, featureGate); err != nil { + t.Fatal(err) + } + opts.GenericServerRunOptions.ComponentGlobalsRegistry = componentGlobalsRegistry + opts.Options.SecureServing.Listener = listener opts.Options.SecureServing.ServerCert.CertDirectory = certDir opts.Options.ServiceAccountSigningKeyFile = saSigningKeyFile.Name() @@ -123,6 +122,19 @@ func StartRealAPIServerOrDie(t *testing.T, configFuncs ...func(*options.ServerRu for _, f := range configFuncs { f(opts) } + + // If the local ComponentGlobalsRegistry is changed by configFuncs, + // we need to copy the new feature values back to the DefaultFeatureGate because most feature checks still use the DefaultFeatureGate. + if !featureGate.EmulationVersion().EqualTo(feature.DefaultMutableFeatureGate.EmulationVersion()) { + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, feature.DefaultMutableFeatureGate, effectiveVersion.EmulationVersion()) + } + for f := range feature.DefaultMutableFeatureGate.GetAll() { + if featureGate.Enabled(f) != feature.DefaultFeatureGate.Enabled(f) { + featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, f, featureGate.Enabled(f)) + } + } + feature.DefaultMutableFeatureGate.AddMetrics() + completedOptions, err := opts.Complete(tCtx) if err != nil { t.Fatal(err) diff --git a/test/integration/examples/apiserver_test.go b/test/integration/examples/apiserver_test.go index a50a8a846a5..efaa6da8f9f 100644 --- a/test/integration/examples/apiserver_test.go +++ b/test/integration/examples/apiserver_test.go @@ -55,9 +55,9 @@ import ( clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "k8s.io/client-go/transport" "k8s.io/client-go/util/cert" + basecompatibility "k8s.io/component-base/compatibility" "k8s.io/component-base/featuregate" featuregatetesting "k8s.io/component-base/featuregate/testing" - utilversion "k8s.io/component-base/version" apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/cmd/kube-apiserver/app" @@ -282,11 +282,8 @@ func testFrontProxyConfig(t *testing.T, withUID bool) { extraKASFlags = []string{"--requestheader-uid-headers=x-remote-uid"} } - // each wardle binary is bundled with a specific kube binary. - kubeBinaryVersion := sampleserver.WardleVersionToKubeVersion(version.MustParse(wardleBinaryVersion)).String() - // start up the KAS and prepare the options for the wardle API server - testKAS, wardleOptions, wardlePort := prepareAggregatedWardleAPIServer(ctx, t, testNamespace, kubeBinaryVersion, wardleBinaryVersion, extraKASFlags, withUID) + testKAS, wardleOptions, wardlePort := prepareAggregatedWardleAPIServer(ctx, t, testNamespace, wardleBinaryVersion, extraKASFlags, withUID) kubeConfig := getKubeConfig(testKAS) // create the SA that we will use to query the aggregated API @@ -402,10 +399,7 @@ func testAggregatedAPIServer(t *testing.T, setWardleFeatureGate, banFlunder bool ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) t.Cleanup(cancel) - // each wardle binary is bundled with a specific kube binary. - kubeBinaryVersion := sampleserver.WardleVersionToKubeVersion(version.MustParse(wardleBinaryVersion)).String() - - testKAS, wardleOptions, wardlePort := prepareAggregatedWardleAPIServer(ctx, t, testNamespace, kubeBinaryVersion, wardleBinaryVersion, nil, false) + testKAS, wardleOptions, wardlePort := prepareAggregatedWardleAPIServer(ctx, t, testNamespace, wardleBinaryVersion, nil, false) kubeClientConfig := getKubeConfig(testKAS) wardleCertDir, _ := os.MkdirTemp("", "test-integration-wardle-server") @@ -685,7 +679,7 @@ func TestAggregatedAPIServerRejectRedirectResponse(t *testing.T) { } } -func prepareAggregatedWardleAPIServer(ctx context.Context, t *testing.T, namespace, kubebinaryVersion, wardleBinaryVersion string, kubeAPIServerFlags []string, withUID bool) (*kastesting.TestServer, *sampleserver.WardleServerOptions, int) { +func prepareAggregatedWardleAPIServer(ctx context.Context, t *testing.T, namespace, wardleBinaryVersion string, kubeAPIServerFlags []string, withUID bool) (*kastesting.TestServer, *sampleserver.WardleServerOptions, int) { // makes the kube-apiserver very responsive. it's normally a minute dynamiccertificates.FileRefreshDuration = 1 * time.Second @@ -697,22 +691,17 @@ func prepareAggregatedWardleAPIServer(ctx context.Context, t *testing.T, namespa // endpoints cannot have loopback IPs so we need to override the resolver itself t.Cleanup(app.SetServiceResolverForTests(staticURLServiceResolver(fmt.Sprintf("https://127.0.0.1:%d", wardlePort)))) - // TODO figure out how to actually make BinaryVersion/EmulationVersion work with Wardle and KAS at the same time when Alpha FG are being set - if withUID { - kubebinaryVersion = "" - } - testServer := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{ EnableCertAuth: true, - BinaryVersion: kubebinaryVersion, }, kubeAPIServerFlags, framework.SharedEtcd()) t.Cleanup(func() { testServer.TearDownFn() }) - _, _ = featuregate.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( - apiserver.WardleComponentName, utilversion.NewEffectiveVersion(wardleBinaryVersion), + componentGlobalsRegistry := testServer.ServerOpts.Options.GenericServerRunOptions.ComponentGlobalsRegistry + _, _ = componentGlobalsRegistry.ComponentGlobalsOrRegister( + apiserver.WardleComponentName, basecompatibility.NewEffectiveVersionFromString(wardleBinaryVersion, "", ""), featuregate.NewVersionedFeatureGate(version.MustParse(wardleBinaryVersion))) kubeClient := client.NewForConfigOrDie(getKubeConfig(testServer)) @@ -740,6 +729,7 @@ func prepareAggregatedWardleAPIServer(ctx context.Context, t *testing.T, namespa } wardleOptions := sampleserver.NewWardleServerOptions(os.Stdout, os.Stderr) + wardleOptions.ComponentGlobalsRegistry = componentGlobalsRegistry // ensure this is a SAN on the generated cert for service FQDN wardleOptions.AlternateDNS = []string{ fmt.Sprintf("api.%s.svc", namespace), diff --git a/test/integration/framework/test_server.go b/test/integration/framework/test_server.go index 8afe46e9ebd..223c1704eca 100644 --- a/test/integration/framework/test_server.go +++ b/test/integration/framework/test_server.go @@ -35,9 +35,13 @@ import ( "k8s.io/apimachinery/pkg/util/wait" genericapiserver "k8s.io/apiserver/pkg/server" genericapiserveroptions "k8s.io/apiserver/pkg/server/options" + "k8s.io/apiserver/pkg/util/compatibility" + utilfeature "k8s.io/apiserver/pkg/util/feature" client "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/util/cert" + basecompatibility "k8s.io/component-base/compatibility" + featuregatetesting "k8s.io/component-base/featuregate/testing" aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" netutils "k8s.io/utils/net" @@ -136,6 +140,17 @@ func StartTestServer(ctx context.Context, t testing.TB, setup TestServerSetup) ( } opts := options.NewServerRunOptions() + // If EmulationVersion of DefaultFeatureGate is set during test, we need to propagate it to the apiserver ComponentGlobalsRegistry. + featureGate := utilfeature.DefaultMutableFeatureGate.DeepCopy() + effectiveVersion := compatibility.DefaultKubeEffectiveVersionForTest() + effectiveVersion.SetEmulationVersion(featureGate.EmulationVersion()) + // set up new instance of ComponentGlobalsRegistry instead of using the DefaultComponentGlobalsRegistry to avoid contention in parallel tests. + componentGlobalsRegistry := basecompatibility.NewComponentGlobalsRegistry() + if err := componentGlobalsRegistry.Register(basecompatibility.DefaultKubeComponent, effectiveVersion, featureGate); err != nil { + t.Fatal(err) + } + opts.GenericServerRunOptions.ComponentGlobalsRegistry = componentGlobalsRegistry + opts.SecureServing.Listener = listener opts.SecureServing.BindAddress = netutils.ParseIPSloppy("127.0.0.1") opts.SecureServing.ServerCert.CertDirectory = certDir @@ -158,6 +173,18 @@ func StartTestServer(ctx context.Context, t testing.TB, setup TestServerSetup) ( setup.ModifyServerRunOptions(opts) } + // If the local ComponentGlobalsRegistry is changed by ModifyServerRunOptions, + // we need to copy the new feature values back to the DefaultFeatureGate because most feature checks still use the DefaultFeatureGate. + if !featureGate.EmulationVersion().EqualTo(utilfeature.DefaultMutableFeatureGate.EmulationVersion()) { + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultMutableFeatureGate, effectiveVersion.EmulationVersion()) + } + for f := range utilfeature.DefaultMutableFeatureGate.GetAll() { + if featureGate.Enabled(f) != utilfeature.DefaultFeatureGate.Enabled(f) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, f, featureGate.Enabled(f)) + } + } + utilfeature.DefaultMutableFeatureGate.AddMetrics() + completedOptions, err := opts.Complete(ctx) if err != nil { t.Fatal(err) diff --git a/test/integration/pods/pods_test.go b/test/integration/pods/pods_test.go index c137a3311cb..c02d3890d6d 100644 --- a/test/integration/pods/pods_test.go +++ b/test/integration/pods/pods_test.go @@ -1230,9 +1230,8 @@ func TestMutablePodSchedulingDirectives(t *testing.T) { // Test disabling of RelaxedDNSSearchValidation after a Pod has been created func TestRelaxedDNSSearchValidation(t *testing.T) { // Disable ServiceAccount admission plugin as we don't have serviceaccount controller running. - server := kubeapiservertesting.StartTestServerOrDie(t, - &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "1.32"}, - framework.DefaultTestServerFlags(), framework.SharedEtcd()) + server := kubeapiservertesting.StartTestServerOrDie(t, nil, + append(framework.DefaultTestServerFlags(), "--emulated-version=1.32"), framework.SharedEtcd()) defer server.TearDownFn() client := clientset.NewForConfigOrDie(server.ClientConfig) diff --git a/test/integration/service/loadbalancer_test.go b/test/integration/service/loadbalancer_test.go index ed5830fc2f8..8a183865394 100644 --- a/test/integration/service/loadbalancer_test.go +++ b/test/integration/service/loadbalancer_test.go @@ -19,7 +19,7 @@ package service import ( "context" "encoding/json" - "k8s.io/apimachinery/pkg/util/version" + "fmt" "testing" "time" @@ -34,10 +34,8 @@ import ( clientset "k8s.io/client-go/kubernetes" servicecontroller "k8s.io/cloud-provider/controllers/service" fakecloud "k8s.io/cloud-provider/fake" - featuregatetesting "k8s.io/component-base/featuregate/testing" controllersmetrics "k8s.io/component-base/metrics/prometheus/controllers" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/test/integration/framework" "k8s.io/utils/net" utilpointer "k8s.io/utils/pointer" @@ -708,13 +706,12 @@ func Test_ServiceLoadBalancerIPMode(t *testing.T) { for _, tc := range testCases { t.Run("", func(t *testing.T) { - var testServerOptions *kubeapiservertesting.TestServerInstanceOptions + serverFlags := framework.DefaultTestServerFlags() if !tc.ipModeEnabled { - testServerOptions = &kubeapiservertesting.TestServerInstanceOptions{EmulationVersion: "1.31"} - featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31")) + serverFlags = append(serverFlags, "--emulated-version=1.31") } - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, tc.ipModeEnabled) - server := kubeapiservertesting.StartTestServerOrDie(t, testServerOptions, framework.DefaultTestServerFlags(), framework.SharedEtcd()) + serverFlags = append(serverFlags, fmt.Sprintf("--feature-gates=LoadBalancerIPMode=%v", tc.ipModeEnabled)) + server := kubeapiservertesting.StartTestServerOrDie(t, nil, serverFlags, framework.SharedEtcd()) defer server.TearDownFn() client, err := clientset.NewForConfig(server.ClientConfig)