diff --git a/cmd/kube-apiserver/app/options/options_test.go b/cmd/kube-apiserver/app/options/options_test.go index 1e4d5a679f8..fa53175c9ad 100644 --- a/cmd/kube-apiserver/app/options/options_test.go +++ b/cmd/kube-apiserver/app/options/options_test.go @@ -53,7 +53,7 @@ func TestAddFlags(t *testing.T) { featureGate := featuregate.NewFeatureGate() componentRegistry := utilversion.NewComponentGlobalsRegistry() effectiveVersion := utilversion.NewEffectiveVersion("1.32") - _ = componentRegistry.Register("test", effectiveVersion, featureGate, true) + utilruntime.Must(componentRegistry.Register("test", effectiveVersion, featureGate)) s := NewServerRunOptions(featureGate, effectiveVersion) for _, f := range s.Flags().FlagSets { fs.AddFlagSet(f) diff --git a/cmd/kube-apiserver/app/testing/testserver.go b/cmd/kube-apiserver/app/testing/testserver.go index f4268b23a6d..0e6411fe869 100644 --- a/cmd/kube-apiserver/app/testing/testserver.go +++ b/cmd/kube-apiserver/app/testing/testserver.go @@ -43,6 +43,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" serveroptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/pkg/storage/storagebackend" @@ -182,12 +183,12 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions, fs := pflag.NewFlagSet("test", pflag.PanicOnError) featureGate := utilfeature.DefaultMutableFeatureGate - binaryVersion := utilversion.DefaultKubeEffectiveVersion().BinaryVersion().String() + effectiveVersion := utilversion.DefaultKubeEffectiveVersion() if instanceOptions.BinaryVersion != "" { - binaryVersion = instanceOptions.BinaryVersion + effectiveVersion = utilversion.NewEffectiveVersion(instanceOptions.BinaryVersion) } - effectiveVersion := utilversion.NewEffectiveVersion(binaryVersion) - _ = utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate, true) + utilversion.DefaultComponentGlobalsRegistry.Reset() + utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate)) s := options.NewServerRunOptions(featureGate, effectiveVersion) diff --git a/pkg/controlplane/apiserver/config.go b/pkg/controlplane/apiserver/config.go index 7933f076ec6..1fc862c34c9 100644 --- a/pkg/controlplane/apiserver/config.go +++ b/pkg/controlplane/apiserver/config.go @@ -46,7 +46,6 @@ import ( clientgoinformers "k8s.io/client-go/informers" clientgoclientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/util/keyutil" - "k8s.io/component-base/version" aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" openapicommon "k8s.io/kube-openapi/pkg/common" @@ -172,9 +171,6 @@ func BuildGenericConfig( sets.NewString("attach", "exec", "proxy", "log", "portforward"), ) - kubeVersion := version.Get() - genericConfig.Version = &kubeVersion - if genericConfig.EgressSelector != nil { s.Etcd.StorageConfig.Transport.EgressLookup = genericConfig.EgressSelector.Lookup } diff --git a/pkg/controlplane/apiserver/options/options_test.go b/pkg/controlplane/apiserver/options/options_test.go index a3f65ebde7f..557983c3679 100644 --- a/pkg/controlplane/apiserver/options/options_test.go +++ b/pkg/controlplane/apiserver/options/options_test.go @@ -48,7 +48,7 @@ func TestAddFlags(t *testing.T) { featureGate := featuregate.NewFeatureGate() effectiveVersion := utilversion.NewEffectiveVersion("1.32") componentRegistry := utilversion.NewComponentGlobalsRegistry() - _ = componentRegistry.Register("test", effectiveVersion, featureGate, true) + utilruntime.Must(componentRegistry.Register("test", effectiveVersion, featureGate)) s := NewOptions(featureGate, effectiveVersion) var fss cliflag.NamedFlagSets s.AddFlags(&fss) diff --git a/pkg/controlplane/instance_test.go b/pkg/controlplane/instance_test.go index 32d1811d82c..524a529520a 100644 --- a/pkg/controlplane/instance_test.go +++ b/pkg/controlplane/instance_test.go @@ -20,6 +20,7 @@ import ( "context" "crypto/tls" "encoding/json" + "fmt" "io" "net" "net/http" @@ -118,9 +119,7 @@ func setUp(t *testing.T) (*etcd3testing.EtcdTestServer, Config, *assert.Assertio t.Fatal(err) } - kubeVersion := kubeversion.Get() config.ControlPlane.Generic.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer() - config.ControlPlane.Generic.Version = &kubeVersion config.ControlPlane.StorageFactory = storageFactory config.ControlPlane.Generic.LoopbackClientConfig = &restclient.Config{APIPath: "/api", ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}} config.ControlPlane.Generic.PublicAddress = netutils.ParseIPSloppy("192.168.10.4") @@ -243,9 +242,13 @@ func TestVersion(t *testing.T) { if err != nil { t.Errorf("unexpected error: %v", err) } + expectedInfo := kubeversion.Get() + kubeVersion := utilversion.DefaultKubeEffectiveVersion().BinaryVersion() + expectedInfo.Major = fmt.Sprintf("%d", kubeVersion.Major()) + expectedInfo.Minor = fmt.Sprintf("%d", kubeVersion.Minor()) - if !reflect.DeepEqual(kubeversion.Get(), info) { - t.Errorf("Expected %#v, Got %#v", kubeversion.Get(), info) + if !reflect.DeepEqual(expectedInfo, info) { + t.Errorf("Expected %#v, Got %#v", expectedInfo, info) } } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go index 517fc9e531a..a6d6213a08b 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go @@ -40,7 +40,6 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/apimachinery/pkg/version" "k8s.io/apiserver/pkg/endpoints/discovery" "k8s.io/apiserver/pkg/endpoints/discovery/aggregated" genericregistry "k8s.io/apiserver/pkg/registry/generic" @@ -118,12 +117,6 @@ func (cfg *Config) Complete() CompletedConfig { } c.GenericConfig.EnableDiscovery = false - if c.GenericConfig.Version == nil { - c.GenericConfig.Version = &version.Info{ - Major: "0", - Minor: "1", - } - } return CompletedConfig{&c} } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go index 547a0ba7712..ef5fce2fead 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go @@ -31,6 +31,7 @@ import ( extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" "k8s.io/apiextensions-apiserver/pkg/cmd/server/options" generatedopenapi "k8s.io/apiextensions-apiserver/pkg/generated/openapi" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" genericapiserver "k8s.io/apiserver/pkg/server" @@ -124,7 +125,8 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin featureGate := utilfeature.DefaultMutableFeatureGate effectiveVersion := utilversion.DefaultKubeEffectiveVersion() - _ = utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate, true) + utilversion.DefaultComponentGlobalsRegistry.Reset() + utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate)) s := options.NewCustomResourceDefinitionsServerOptions(os.Stdout, os.Stderr, featureGate, effectiveVersion) utilversion.DefaultComponentGlobalsRegistry.AddFlags(fs) diff --git a/staging/src/k8s.io/apimachinery/pkg/util/version/version.go b/staging/src/k8s.io/apimachinery/pkg/util/version/version.go index d327f00b3be..b7812ff2d15 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/version/version.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/version/version.go @@ -23,6 +23,8 @@ import ( "regexp" "strconv" "strings" + + apimachineryversion "k8s.io/apimachinery/pkg/version" ) // Version is an opaque representation of a version number @@ -31,6 +33,7 @@ type Version struct { semver bool preRelease string buildMetadata string + info apimachineryversion.Info } var ( @@ -252,19 +255,30 @@ func (v *Version) WithMinor(minor uint) *Version { return &result } -// SubtractMinor returns the version diff minor versions back, with the same major and no patch. -// If diff >= current minor, the minor would be 0. -func (v *Version) SubtractMinor(diff uint) *Version { +// SubtractMinor returns the version with offset from the original minor, with the same major and no patch. +// If -offset >= current minor, the minor would be 0. +func (v *Version) OffsetMinor(offset int) *Version { var minor uint - if diff < v.Minor() { - minor = v.Minor() - diff + if offset >= 0 { + minor = v.Minor() + uint(offset) + } else { + diff := uint(-offset) + if diff < v.Minor() { + minor = v.Minor() - diff + } } return MajorMinor(v.Major(), minor) } +// SubtractMinor returns the version diff minor versions back, with the same major and no patch. +// If diff >= current minor, the minor would be 0. +func (v *Version) SubtractMinor(diff uint) *Version { + return v.OffsetMinor(-int(diff)) +} + // AddMinor returns the version diff minor versions forward, with the same major and no patch. func (v *Version) AddMinor(diff uint) *Version { - return MajorMinor(v.Major(), v.Minor()+diff) + return v.OffsetMinor(int(diff)) } // WithPatch returns copy of the version object with requested patch number @@ -441,3 +455,30 @@ func (v *Version) Compare(other string) (int, error) { } return v.compareInternal(ov), nil } + +// WithInfo returns copy of the version object with requested info +func (v *Version) WithInfo(info apimachineryversion.Info) *Version { + result := *v + result.info = info + return &result +} + +func (v *Version) Info() *apimachineryversion.Info { + if v == nil { + return nil + } + // in case info is empty, or the major and minor in info is different from the actual major and minor + v.info.Major = itoa(v.Major()) + v.info.Minor = itoa(v.Minor()) + if v.info.GitVersion == "" { + v.info.GitVersion = v.String() + } + return &v.info +} + +func itoa(i uint) string { + if i == 0 { + return "" + } + return strconv.Itoa(int(i)) +} diff --git a/staging/src/k8s.io/apimachinery/pkg/util/version/version_test.go b/staging/src/k8s.io/apimachinery/pkg/util/version/version_test.go index c8625fb8091..a3345ddfdde 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/version/version_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/version/version_test.go @@ -453,37 +453,42 @@ func TestHighestSupportedVersion(t *testing.T) { } } -func TestSubtractMinor(t *testing.T) { +func TestOffsetMinor(t *testing.T) { var tests = []struct { version string - diff uint + diff int expectedComponents []uint }{ { version: "1.0.2", - diff: 3, + diff: -3, expectedComponents: []uint{1, 0}, }, { version: "1.3.2-alpha+001", - diff: 2, + diff: -2, expectedComponents: []uint{1, 1}, }, { version: "1.3.2-alpha+001", - diff: 3, + diff: -3, expectedComponents: []uint{1, 0}, }, { version: "1.20", - diff: 5, + diff: -5, expectedComponents: []uint{1, 15}, }, + { + version: "1.20", + diff: 5, + expectedComponents: []uint{1, 25}, + }, } for _, test := range tests { version, _ := ParseGeneric(test.version) - if !reflect.DeepEqual(test.expectedComponents, version.SubtractMinor(test.diff).Components()) { + if !reflect.DeepEqual(test.expectedComponents, version.OffsetMinor(test.diff).Components()) { t.Error("parse returned un'expected components") } } diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index b266cb2e494..931226bb678 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -44,7 +44,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/version" utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup" - apimachineryversion "k8s.io/apimachinery/pkg/version" "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/audit" "k8s.io/apiserver/pkg/authentication/authenticator" @@ -150,8 +149,6 @@ type Config struct { // done values in this values for this map are ignored. PostStartHooks map[string]PostStartHookConfigEntry - // Version will enable the /version endpoint if non-nil - Version *apimachineryversion.Info // EffectiveVersion determines which apis and features are available // based on when the api/feature lifecyle. EffectiveVersion utilversion.EffectiveVersion @@ -702,12 +699,8 @@ func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedCo } c.ExternalAddress = net.JoinHostPort(c.ExternalAddress, strconv.Itoa(port)) } - var ver *version.Version - if c.EffectiveVersion != nil { - ver = c.EffectiveVersion.EmulationVersion() - } - completeOpenAPI(c.OpenAPIConfig, ver) - completeOpenAPIV3(c.OpenAPIV3Config, ver) + completeOpenAPI(c.OpenAPIConfig, c.EffectiveVersion.EmulationVersion()) + completeOpenAPIV3(c.OpenAPIV3Config, c.EffectiveVersion.EmulationVersion()) if c.DiscoveryAddresses == nil { c.DiscoveryAddresses = discovery.DefaultAddresses{DefaultAddress: c.ExternalAddress} @@ -834,7 +827,6 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G StorageVersionManager: c.StorageVersionManager, EffectiveVersion: c.EffectiveVersion, - Version: c.Version, FeatureGate: c.FeatureGate, muxAndDiscoveryCompleteSignals: map[string]<-chan struct{}{}, @@ -1103,7 +1095,7 @@ func installAPI(s *GenericAPIServer, c *Config) { } } - routes.Version{Version: c.Version}.Install(s.Handler.GoRestfulContainer) + routes.Version{Version: c.EffectiveVersion.BinaryVersion().Info()}.Install(s.Handler.GoRestfulContainer) if c.EnableDiscovery { if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) { diff --git a/staging/src/k8s.io/apiserver/pkg/server/config_test.go b/staging/src/k8s.io/apiserver/pkg/server/config_test.go index f58f3bf9c2b..6d25272b0a8 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config_test.go @@ -40,6 +40,7 @@ import ( "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/server/healthz" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/rest" @@ -90,6 +91,7 @@ func TestNewWithDelegate(t *testing.T) { delegateConfig.PublicAddress = netutils.ParseIPSloppy("192.168.10.4") delegateConfig.LegacyAPIGroupPrefixes = sets.NewString("/api") delegateConfig.LoopbackClientConfig = &rest.Config{} + delegateConfig.EffectiveVersion = utilversion.NewEffectiveVersion("") clientset := fake.NewSimpleClientset() if clientset == nil { t.Fatal("unable to create fake client set") @@ -122,6 +124,7 @@ func TestNewWithDelegate(t *testing.T) { wrappingConfig.PublicAddress = netutils.ParseIPSloppy("192.168.10.4") wrappingConfig.LegacyAPIGroupPrefixes = sets.NewString("/api") wrappingConfig.LoopbackClientConfig = &rest.Config{} + wrappingConfig.EffectiveVersion = utilversion.NewEffectiveVersion("") wrappingConfig.HealthzChecks = append(wrappingConfig.HealthzChecks, healthz.NamedCheck("wrapping-health", func(r *http.Request) error { return fmt.Errorf("wrapping failed healthcheck") diff --git a/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go b/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go index 841c1f5729c..564b5bc4352 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go +++ b/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go @@ -100,7 +100,7 @@ func (e *resourceExpirationEvaluator) shouldServe(gv schema.GroupVersion, versio } introduced, ok := versionedPtr.(introducedInterface) - // skip the introduced check for test where currentVersion is 0.0 + // skip the introduced check for test when currentVersion is 0.0 to test all apis if ok && (e.currentVersion.Major() > 0 || e.currentVersion.Minor() > 0) { majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced() verIntroduced := apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced)) diff --git a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go index c47fc5b23f0..7b8e13da256 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go +++ b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go @@ -40,7 +40,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup" - "k8s.io/apimachinery/pkg/version" "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/audit" "k8s.io/apiserver/pkg/authorization/authorizer" @@ -238,8 +237,6 @@ type GenericAPIServer struct { // StorageVersionManager holds the storage versions of the API resources installed by this server. StorageVersionManager storageversion.Manager - // Version will enable the /version endpoint if non-nil - Version *version.Info // EffectiveVersion determines which apis and features are available // based on when the api/feature lifecyle. EffectiveVersion utilversion.EffectiveVersion diff --git a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go index c2bfef4dda4..c6a7cbdcff0 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go @@ -138,7 +138,7 @@ func setUp(t *testing.T) (Config, *assert.Assertions) { if clientset == nil { t.Fatal("unable to create fake client set") } - + config.EffectiveVersion = utilversion.NewEffectiveVersion("") config.OpenAPIConfig = DefaultOpenAPIConfig(testGetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(runtime.NewScheme())) config.OpenAPIConfig.Info.Version = "unversioned" config.OpenAPIV3Config = DefaultOpenAPIV3Config(testGetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(runtime.NewScheme())) @@ -460,8 +460,9 @@ func TestNotRestRoutesHaveAuth(t *testing.T) { config.EnableProfiling = true kubeVersion := fakeVersion() - config.Version = &kubeVersion - config.EffectiveVersion = utilversion.NewEffectiveVersion(kubeVersion.String()) + effectiveVersion := utilversion.NewEffectiveVersion(kubeVersion.String()) + effectiveVersion.Set(effectiveVersion.BinaryVersion().WithInfo(kubeVersion), effectiveVersion.EmulationVersion(), effectiveVersion.MinCompatibilityVersion()) + config.EffectiveVersion = effectiveVersion s, err := config.Complete(nil).New("test", NewEmptyDelegate()) if err != nil { diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go index 40feeaf62c1..dc87524efcf 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go @@ -278,7 +278,6 @@ func TestServerRunWithSNI(t *testing.T) { // launch server config := setUp(t) v := fakeVersion() - config.Version = &v config.EffectiveVersion = utilversion.NewEffectiveVersion(v.String()) config.EnableIndex = true diff --git a/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go b/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go index 5d5cfccc7fd..7339d17df50 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go @@ -157,6 +157,7 @@ func emulatedStorageVersion(binaryVersionOfResource schema.GroupVersion, example } // If it was introduced after current compatibility version, don't use it + // skip the introduced check for test when currentVersion is 0.0 to test all apis if introduced, hasIntroduced := exampleOfGVK.(introducedInterface); hasIntroduced && (compatibilityVersion.Major() > 0 || compatibilityVersion.Minor() > 0) { // API resource lifecycles should be relative to k8s api version majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced() diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/registry.go b/staging/src/k8s.io/apiserver/pkg/util/version/registry.go index 589d42a1ade..170ac28045e 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/version/registry.go +++ b/staging/src/k8s.io/apiserver/pkg/util/version/registry.go @@ -59,12 +59,29 @@ var DefaultComponentGlobalsRegistry ComponentGlobalsRegistry = NewComponentGloba const ( DefaultKubeComponent = "kube" + + klogLevel = 2 ) +type VersionMapping func(from *version.Version) *version.Version + // ComponentGlobals stores the global variables for a component for easy access. type ComponentGlobals struct { effectiveVersion MutableEffectiveVersion featureGate featuregate.MutableVersionedFeatureGate + + // emulationVersionMapping contains the mapping from the emulation version of this component + // to the emulation version of another component. + emulationVersionMapping map[string]VersionMapping + // dependentEmulationVersion stores whether or not this component's EmulationVersion is dependent through mapping on another component. + // If true, the emulation version cannot be set from the flag, or version mapping from another component. + dependentEmulationVersion bool + // minCompatibilityVersionMapping contains the mapping from the min compatibility version of this component + // to the min compatibility version of another component. + minCompatibilityVersionMapping map[string]VersionMapping + // dependentMinCompatibilityVersion stores whether or not this component's MinCompatibilityVersion is dependent through mapping on another component + // If true, the min compatibility version cannot be set from the flag, or version mapping from another component. + dependentMinCompatibilityVersion bool } type ComponentGlobalsRegistry interface { @@ -75,9 +92,8 @@ type ComponentGlobalsRegistry interface { // Returns nil if the component is not registered. FeatureGateFor(component string) featuregate.FeatureGate // Register registers the EffectiveVersion and FeatureGate for a component. - // Overrides existing ComponentGlobals if it is already in the registry if override is true, - // otherwise returns error if the component is already registered. - Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error + // returns error if the component is already registered. + Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error // ComponentGlobalsOrRegister would return the registered global variables for the component if it already exists in the registry. // Otherwise, the provided variables would be registered under the component, and the same variables would be returned. ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) @@ -85,25 +101,43 @@ type ComponentGlobalsRegistry interface { AddFlags(fs *pflag.FlagSet) // Set sets the flags for all global variables for all components registered. Set() error - // SetAllComponents calls the Validate() function for all the global variables for all components registered. + // Validate calls the Validate() function for all the global variables for all components registered. Validate() []error + // Reset removes all stored ComponentGlobals, configurations, and version mappings. + Reset() + // SetEmulationVersionMapping sets the mapping from the emulation version of one component + // to the emulation version of another component. + // Once set, the emulation version of the toComponent will be determined by the emulation version of the fromComponent, + // and cannot be set from cmd flags anymore. + // For a given component, its emulation version can only depend on one other component, no multiple dependency is allowed. + SetEmulationVersionMapping(fromComponent, toComponent string, f VersionMapping) error } type componentGlobalsRegistry struct { - componentGlobals map[string]ComponentGlobals + componentGlobals map[string]*ComponentGlobals mutex sync.RWMutex - // map of component name to emulation version set from the flag. - emulationVersionConfig cliflag.ConfigurationMap + // list of component name to emulation version set from the flag. + emulationVersionConfig []string // map of component name to the list of feature gates set from the flag. featureGatesConfig map[string][]string } -func NewComponentGlobalsRegistry() ComponentGlobalsRegistry { +func NewComponentGlobalsRegistry() *componentGlobalsRegistry { return &componentGlobalsRegistry{ - componentGlobals: make(map[string]ComponentGlobals), + componentGlobals: make(map[string]*ComponentGlobals), + emulationVersionConfig: nil, + featureGatesConfig: nil, } } +func (r *componentGlobalsRegistry) Reset() { + r.mutex.RLock() + defer r.mutex.RUnlock() + r.componentGlobals = make(map[string]*ComponentGlobals) + r.emulationVersionConfig = nil + r.featureGatesConfig = nil +} + func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) EffectiveVersion { r.mutex.RLock() defer r.mutex.RUnlock() @@ -124,8 +158,8 @@ func (r *componentGlobalsRegistry) FeatureGateFor(component string) featuregate. return globals.featureGate } -func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error { - if _, ok := r.componentGlobals[component]; ok && !override { +func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error { + if _, ok := r.componentGlobals[component]; ok { return fmt.Errorf("component globals of %s already registered", component) } if featureGate != nil { @@ -133,18 +167,23 @@ func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVer return err } } - c := ComponentGlobals{effectiveVersion: effectiveVersion, featureGate: featureGate} - r.componentGlobals[component] = c + c := ComponentGlobals{ + effectiveVersion: effectiveVersion, + featureGate: featureGate, + emulationVersionMapping: make(map[string]VersionMapping), + minCompatibilityVersionMapping: make(map[string]VersionMapping), + } + r.componentGlobals[component] = &c return nil } -func (r *componentGlobalsRegistry) Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate, override bool) error { +func (r *componentGlobalsRegistry) Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error { if effectiveVersion == nil { return fmt.Errorf("cannot register nil effectiveVersion") } r.mutex.Lock() defer r.mutex.Unlock() - return r.unsafeRegister(component, effectiveVersion, featureGate, override) + return r.unsafeRegister(component, effectiveVersion, featureGate) } func (r *componentGlobalsRegistry) ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) { @@ -154,13 +193,11 @@ func (r *componentGlobalsRegistry) ComponentGlobalsOrRegister(component string, if ok { return globals.effectiveVersion, globals.featureGate } - utilruntime.Must(r.unsafeRegister(component, effectiveVersion, featureGate, false)) + utilruntime.Must(r.unsafeRegister(component, effectiveVersion, featureGate)) return effectiveVersion, featureGate } -func (r *componentGlobalsRegistry) knownFeatures() []string { - r.mutex.Lock() - defer r.mutex.Unlock() +func (r *componentGlobalsRegistry) unsafeKnownFeatures() []string { var known []string for component, globals := range r.componentGlobals { if globals.featureGate == nil { @@ -174,18 +211,22 @@ func (r *componentGlobalsRegistry) knownFeatures() []string { return known } -func (r *componentGlobalsRegistry) versionFlagOptions(isEmulation bool) []string { - r.mutex.Lock() - defer r.mutex.Unlock() +func (r *componentGlobalsRegistry) unsafeVersionFlagOptions(isEmulation bool) []string { var vs []string for component, globals := range r.componentGlobals { binaryVer := globals.effectiveVersion.BinaryVersion() if isEmulation { + if globals.dependentEmulationVersion { + continue + } // emulated version could be between binaryMajor.{binaryMinor} and binaryMajor.{binaryMinor} // TODO: change to binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor} in 1.32 vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component, binaryVer.SubtractMinor(0).String(), binaryVer.String(), globals.effectiveVersion.EmulationVersion().String())) } else { + if globals.dependentMinCompatibilityVersion { + continue + } // min compatibility version could be between binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor} vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component, binaryVer.SubtractMinor(1).String(), binaryVer.String(), globals.effectiveVersion.MinCompatibilityVersion().String())) @@ -200,51 +241,133 @@ func (r *componentGlobalsRegistry) AddFlags(fs *pflag.FlagSet) { return } r.mutex.Lock() + defer r.mutex.Unlock() for _, globals := range r.componentGlobals { if globals.featureGate != nil { globals.featureGate.Close() } } - r.emulationVersionConfig = make(cliflag.ConfigurationMap) + if r.emulationVersionConfig != nil || r.featureGatesConfig != nil { + klog.Warning("calling componentGlobalsRegistry.AddFlags more than once, the registry will be set by the latest flags") + } + r.emulationVersionConfig = []string{} r.featureGatesConfig = make(map[string][]string) - r.mutex.Unlock() - fs.Var(&r.emulationVersionConfig, "emulated-version", ""+ + fs.StringSliceVar(&r.emulationVersionConfig, "emulated-version", r.emulationVersionConfig, ""+ "The versions different components emulate their capabilities (APIs, features, ...) of.\n"+ "If set, the component will emulate the behavior of this version instead of the underlying binary version.\n"+ - "Version format could only be major.minor, for example: '--emulated-version=wardle=1.2,kube=1.31'. Options are:\n"+strings.Join(r.versionFlagOptions(true), "\n")) + "Version format could only be major.minor, for example: '--emulated-version=wardle=1.2,kube=1.31'. Options are:\n"+strings.Join(r.unsafeVersionFlagOptions(true), "\n")+ + "If the component is not specified, defaults to \"kube\"") fs.Var(cliflag.NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey(&r.featureGatesConfig), "feature-gates", "Comma-separated list of component:key=value pairs that describe feature gates for alpha/experimental features of different components.\n"+ "If the component is not specified, defaults to \"kube\". This flag can be repeatedly invoked. For example: --feature-gates 'wardle:featureA=true,wardle:featureB=false' --feature-gates 'kube:featureC=true'"+ - "Options are:\n"+strings.Join(r.knownFeatures(), "\n")) + "Options are:\n"+strings.Join(r.unsafeKnownFeatures(), "\n")) +} + +type componentVersion struct { + component string + ver *version.Version +} + +// getFullEmulationVersionConfig expands the given version config with version registered version mapping, +// and returns the map of component to Version. +func (r *componentGlobalsRegistry) getFullEmulationVersionConfig( + versionConfigMap map[string]*version.Version) (map[string]*version.Version, error) { + result := map[string]*version.Version{} + setQueue := []componentVersion{} + for comp, ver := range versionConfigMap { + if _, ok := r.componentGlobals[comp]; !ok { + return result, fmt.Errorf("component not registered: %s", comp) + } + klog.V(klogLevel).Infof("setting version %s=%s", comp, ver.String()) + setQueue = append(setQueue, componentVersion{comp, ver}) + } + for len(setQueue) > 0 { + cv := setQueue[0] + if _, visited := result[cv.component]; visited { + return result, fmt.Errorf("setting version of %s more than once, probably version mapping loop", cv.component) + } + setQueue = setQueue[1:] + result[cv.component] = cv.ver + for toComp, f := range r.componentGlobals[cv.component].emulationVersionMapping { + toVer := f(cv.ver) + if toVer == nil { + return result, fmt.Errorf("got nil version from mapping of %s=%s to component:%s", cv.component, cv.ver.String(), toComp) + } + klog.V(klogLevel).Infof("setting version %s=%s from version mapping of %s=%s", toComp, toVer.String(), cv.component, cv.ver.String()) + setQueue = append(setQueue, componentVersion{toComp, toVer}) + } + } + return result, nil +} + +func toVersionMap(versionConfig []string) (map[string]*version.Version, error) { + m := map[string]*version.Version{} + for _, compVer := range versionConfig { + // default to "kube" of component is not specified + k := "kube" + v := compVer + if strings.Contains(compVer, "=") { + arr := strings.SplitN(compVer, "=", 2) + if len(arr) != 2 { + return m, fmt.Errorf("malformed pair, expect string=string") + } + k = strings.TrimSpace(arr[0]) + v = strings.TrimSpace(arr[1]) + } + ver, err := version.Parse(v) + if err != nil { + return m, err + } + if ver.Patch() != 0 { + return m, fmt.Errorf("patch version not allowed, got: %s=%s", k, ver.String()) + } + if existingVer, ok := m[k]; ok { + return m, fmt.Errorf("duplicate version flag, %s=%s and %s=%s", k, existingVer.String(), k, ver.String()) + } + m[k] = ver + } + return m, nil } func (r *componentGlobalsRegistry) Set() error { r.mutex.Lock() defer r.mutex.Unlock() - for comp, emuVer := range r.emulationVersionConfig { + emulationVersionConfigMap, err := toVersionMap(r.emulationVersionConfig) + if err != nil { + return err + } + for comp := range emulationVersionConfigMap { if _, ok := r.componentGlobals[comp]; !ok { return fmt.Errorf("component not registered: %s", comp) } - klog.V(2).Infof("setting %s:emulation version to %s\n", comp, emuVer) - v, err := version.Parse(emuVer) - if err != nil { - return err + // only components without any dependencies can be set from the flag. + if r.componentGlobals[comp].dependentEmulationVersion { + return fmt.Errorf("EmulationVersion of %s is set by mapping, cannot set it by flag", comp) + } + } + if emulationVersions, err := r.getFullEmulationVersionConfig(emulationVersionConfigMap); err != nil { + return err + } else { + for comp, ver := range emulationVersions { + r.componentGlobals[comp].effectiveVersion.SetEmulationVersion(ver) } - r.componentGlobals[comp].effectiveVersion.SetEmulationVersion(v) } // Set feature gate emulation version before setting feature gate flag values. for comp, globals := range r.componentGlobals { if globals.featureGate == nil { continue } - klog.V(2).Infof("setting %s:feature gate emulation version to %s\n", comp, globals.effectiveVersion.EmulationVersion().String()) + klog.V(klogLevel).Infof("setting %s:feature gate emulation version to %s", comp, globals.effectiveVersion.EmulationVersion().String()) if err := globals.featureGate.SetEmulationVersion(globals.effectiveVersion.EmulationVersion()); err != nil { return err } } for comp, fg := range r.featureGatesConfig { if comp == "" { + if _, ok := r.featureGatesConfig[DefaultKubeComponent]; ok { + return fmt.Errorf("set kube feature gates with default empty prefix or kube: prefix consistently, do not mix use") + } comp = DefaultKubeComponent } if _, ok := r.componentGlobals[comp]; !ok { @@ -255,7 +378,7 @@ func (r *componentGlobalsRegistry) Set() error { return fmt.Errorf("component featureGate not registered: %s", comp) } flagVal := strings.Join(fg, ",") - klog.V(2).Infof("setting %s:feature-gates=%s\n", comp, flagVal) + klog.V(klogLevel).Infof("setting %s:feature-gates=%s", comp, flagVal) if err := featureGate.Set(flagVal); err != nil { return err } @@ -275,3 +398,39 @@ func (r *componentGlobalsRegistry) Validate() []error { } return errs } + +func (r *componentGlobalsRegistry) SetEmulationVersionMapping(fromComponent, toComponent string, f VersionMapping) error { + if f == nil { + return nil + } + klog.V(klogLevel).Infof("setting EmulationVersion mapping from %s to %s", fromComponent, toComponent) + r.mutex.Lock() + defer r.mutex.Unlock() + if _, ok := r.componentGlobals[fromComponent]; !ok { + return fmt.Errorf("component not registered: %s", fromComponent) + } + if _, ok := r.componentGlobals[toComponent]; !ok { + return fmt.Errorf("component not registered: %s", toComponent) + } + // check multiple dependency + if r.componentGlobals[toComponent].dependentEmulationVersion { + return fmt.Errorf("mapping of %s already exists from another component", toComponent) + } + r.componentGlobals[toComponent].dependentEmulationVersion = true + + versionMapping := r.componentGlobals[fromComponent].emulationVersionMapping + if _, ok := versionMapping[toComponent]; ok { + return fmt.Errorf("EmulationVersion from %s to %s already exists", fromComponent, toComponent) + } + versionMapping[toComponent] = f + klog.V(klogLevel).Infof("setting the default EmulationVersion of %s based on mapping from the default EmulationVersion of %s", fromComponent, toComponent) + defaultFromVersion := r.componentGlobals[fromComponent].effectiveVersion.EmulationVersion() + emulationVersions, err := r.getFullEmulationVersionConfig(map[string]*version.Version{fromComponent: defaultFromVersion}) + if err != nil { + return err + } + for comp, ver := range emulationVersions { + r.componentGlobals[comp].effectiveVersion.SetEmulationVersion(ver) + } + return nil +} diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/registry_test.go b/staging/src/k8s.io/apiserver/pkg/util/version/registry_test.go index 68625e526b0..1badd5344d2 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/version/registry_test.go +++ b/staging/src/k8s.io/apiserver/pkg/util/version/registry_test.go @@ -22,8 +22,8 @@ import ( "testing" "github.com/spf13/pflag" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/version" - cliflag "k8s.io/component-base/cli/flag" "k8s.io/component-base/featuregate" ) @@ -39,30 +39,23 @@ func TestEffectiveVersionRegistry(t *testing.T) { if r.EffectiveVersionFor(testComponent) != nil { t.Fatalf("expected nil EffectiveVersion initially") } - if err := r.Register(testComponent, ver1, nil, false); err != nil { + if err := r.Register(testComponent, ver1, nil); err != nil { t.Fatalf("expected no error to register new component, but got err: %v", err) } if !r.EffectiveVersionFor(testComponent).EqualTo(ver1) { t.Fatalf("expected EffectiveVersionFor to return the version registered") } // overwrite - if err := r.Register(testComponent, ver2, nil, false); err == nil { + if err := r.Register(testComponent, ver2, nil); err == nil { t.Fatalf("expected error to register existing component when override is false") } - if err := r.Register(testComponent, ver2, nil, true); err != nil { - t.Fatalf("expected no error to overriding existing component, but got err: %v", err) - } - if !r.EffectiveVersionFor(testComponent).EqualTo(ver2) { + if !r.EffectiveVersionFor(testComponent).EqualTo(ver1) { t.Fatalf("expected EffectiveVersionFor to return the version overridden") } } func testRegistry(t *testing.T) *componentGlobalsRegistry { - r := componentGlobalsRegistry{ - componentGlobals: map[string]ComponentGlobals{}, - emulationVersionConfig: make(cliflag.ConfigurationMap), - featureGatesConfig: make(map[string][]string), - } + r := NewComponentGlobalsRegistry() verKube := NewEffectiveVersion("1.31") fgKube := featuregate.NewVersionedFeatureGate(version.MustParse("0.0")) err := fgKube.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{ @@ -102,19 +95,35 @@ func testRegistry(t *testing.T) *componentGlobalsRegistry { if err != nil { t.Fatal(err) } - _ = r.Register(DefaultKubeComponent, verKube, fgKube, true) - _ = r.Register(testComponent, verTest, fgTest, true) - return &r + utilruntime.Must(r.Register(DefaultKubeComponent, verKube, fgKube)) + utilruntime.Must(r.Register(testComponent, verTest, fgTest)) + return r } func TestVersionFlagOptions(t *testing.T) { r := testRegistry(t) - emuVers := strings.Join(r.versionFlagOptions(true), "\n") + emuVers := strings.Join(r.unsafeVersionFlagOptions(true), "\n") expectedEmuVers := "kube=1.31..1.31 (default=1.31)\ntest=2.8..2.8 (default=2.8)" if emuVers != expectedEmuVers { t.Errorf("wanted emulation version flag options to be: %s, got %s", expectedEmuVers, emuVers) } - minCompVers := strings.Join(r.versionFlagOptions(false), "\n") + minCompVers := strings.Join(r.unsafeVersionFlagOptions(false), "\n") + expectedMinCompVers := "kube=1.30..1.31 (default=1.30)\ntest=2.7..2.8 (default=2.7)" + if minCompVers != expectedMinCompVers { + t.Errorf("wanted min compatibility version flag options to be: %s, got %s", expectedMinCompVers, minCompVers) + } +} + +func TestVersionFlagOptionsWithMapping(t *testing.T) { + r := testRegistry(t) + utilruntime.Must(r.SetEmulationVersionMapping(testComponent, DefaultKubeComponent, + func(from *version.Version) *version.Version { return from.OffsetMinor(3) })) + emuVers := strings.Join(r.unsafeVersionFlagOptions(true), "\n") + expectedEmuVers := "test=2.8..2.8 (default=2.8)" + if emuVers != expectedEmuVers { + t.Errorf("wanted emulation version flag options to be: %s, got %s", expectedEmuVers, emuVers) + } + minCompVers := strings.Join(r.unsafeVersionFlagOptions(false), "\n") expectedMinCompVers := "kube=1.30..1.31 (default=1.30)\ntest=2.7..2.8 (default=2.7)" if minCompVers != expectedMinCompVers { t.Errorf("wanted min compatibility version flag options to be: %s, got %s", expectedMinCompVers, minCompVers) @@ -123,7 +132,7 @@ func TestVersionFlagOptions(t *testing.T) { func TestVersionedFeatureGateFlag(t *testing.T) { r := testRegistry(t) - known := strings.Join(r.knownFeatures(), "\n") + known := strings.Join(r.unsafeKnownFeatures(), "\n") expectedKnown := "kube:AllAlpha=true|false (ALPHA - default=false)\n" + "kube:AllBeta=true|false (BETA - default=false)\n" + "kube:commonC=true|false (BETA - default=true)\n" + @@ -140,86 +149,127 @@ func TestVersionedFeatureGateFlag(t *testing.T) { func TestFlags(t *testing.T) { tests := []struct { name string - emulationVersionFlag string - featureGatesFlag string + flags []string parseError string - expectedKubeEmulationVersion *version.Version - expectedTestEmulationVersion *version.Version + expectedKubeEmulationVersion string + expectedTestEmulationVersion string expectedKubeFeatureValues map[featuregate.Feature]bool expectedTestFeatureValues map[featuregate.Feature]bool }{ { name: "setting kube emulation version", - emulationVersionFlag: "kube=1.30", - expectedKubeEmulationVersion: version.MajorMinor(1, 30), + flags: []string{"--emulated-version=kube=1.30"}, + expectedKubeEmulationVersion: "1.30", }, { - name: "setting kube emulation version, prefix v ok", - emulationVersionFlag: "kube=v1.30", - expectedKubeEmulationVersion: version.MajorMinor(1, 30), + name: "setting kube emulation version twice", + flags: []string{ + "--emulated-version=kube=1.30", + "--emulated-version=kube=1.32", + }, + parseError: "duplicate version flag, kube=1.30 and kube=1.32", + }, + { + name: "prefix v ok", + flags: []string{"--emulated-version=kube=v1.30"}, + expectedKubeEmulationVersion: "1.30", + }, + { + name: "patch version not ok", + flags: []string{"--emulated-version=kube=1.30.2"}, + parseError: "patch version not allowed, got: kube=1.30.2", }, { name: "setting test emulation version", - emulationVersionFlag: "test=2.7", - expectedKubeEmulationVersion: version.MajorMinor(1, 31), - expectedTestEmulationVersion: version.MajorMinor(2, 7), + flags: []string{"--emulated-version=test=2.7"}, + expectedKubeEmulationVersion: "1.31", + expectedTestEmulationVersion: "2.7", }, { - name: "version missing component", - emulationVersionFlag: "1.31", - parseError: "component not registered: 1.31", + name: "version missing component default to kube", + flags: []string{"--emulated-version=1.30"}, + expectedKubeEmulationVersion: "1.30", }, { - name: "version unregistered component", - emulationVersionFlag: "test3=1.31", - parseError: "component not registered: test3", + name: "version missing component default to kube with duplicate", + flags: []string{"--emulated-version=1.30", "--emulated-version=kube=1.30"}, + parseError: "duplicate version flag, kube=1.30 and kube=1.30", }, { - name: "invalid version", - emulationVersionFlag: "test=1.foo", - parseError: "illegal version string \"1.foo\"", + name: "version unregistered component", + flags: []string{"--emulated-version=test3=1.31"}, + parseError: "component not registered: test3", }, { - name: "setting test feature flag", - emulationVersionFlag: "test=2.7", - featureGatesFlag: "test:testA=true", - expectedKubeEmulationVersion: version.MajorMinor(1, 31), - expectedTestEmulationVersion: version.MajorMinor(2, 7), + name: "invalid version", + flags: []string{"--emulated-version=test=1.foo"}, + parseError: "illegal version string \"1.foo\"", + }, + { + name: "setting test feature flag", + flags: []string{ + "--emulated-version=test=2.7", + "--feature-gates=test:testA=true", + }, + expectedKubeEmulationVersion: "1.31", + expectedTestEmulationVersion: "2.7", expectedKubeFeatureValues: map[featuregate.Feature]bool{"kubeA": true, "kubeB": false, "commonC": true}, expectedTestFeatureValues: map[featuregate.Feature]bool{"testA": true, "testB": false, "commonC": false}, }, { - name: "setting future test feature flag", - emulationVersionFlag: "test=2.7", - featureGatesFlag: "test:testA=true,test:testB=true", - parseError: "cannot set feature gate testB to true, feature is PreAlpha at emulated version 2.7", + name: "setting future test feature flag", + flags: []string{ + "--emulated-version=test=2.7", + "--feature-gates=test:testA=true,test:testB=true", + }, + parseError: "cannot set feature gate testB to true, feature is PreAlpha at emulated version 2.7", }, { - name: "setting kube feature flag", - emulationVersionFlag: "test=2.7,kube=1.30", - featureGatesFlag: "test:commonC=true,commonC=false,kube:kubeB=true", - expectedKubeEmulationVersion: version.MajorMinor(1, 30), - expectedTestEmulationVersion: version.MajorMinor(2, 7), + name: "setting kube feature flag", + flags: []string{ + "--emulated-version=test=2.7", + "--emulated-version=kube=1.30", + "--feature-gates=kubeB=false,test:commonC=true", + "--feature-gates=commonC=false,kubeB=true", + }, + expectedKubeEmulationVersion: "1.30", + expectedTestEmulationVersion: "2.7", expectedKubeFeatureValues: map[featuregate.Feature]bool{"kubeA": false, "kubeB": true, "commonC": false}, expectedTestFeatureValues: map[featuregate.Feature]bool{"testA": false, "testB": false, "commonC": true}, }, { - name: "setting locked kube feature flag", - emulationVersionFlag: "test=2.7", - featureGatesFlag: "kubeA=false", - parseError: "cannot set feature gate kubeA to false, feature is locked to true", + name: "setting kube feature flag with different prefix", + flags: []string{ + "--emulated-version=test=2.7", + "--emulated-version=kube=1.30", + "--feature-gates=kube:kubeB=false,test:commonC=true", + "--feature-gates=commonC=false,kubeB=true", + }, + parseError: "set kube feature gates with default empty prefix or kube: prefix consistently, do not mix use", }, { - name: "setting unknown test feature flag", - emulationVersionFlag: "test=2.7", - featureGatesFlag: "test:testD=true", - parseError: "unrecognized feature gate: testD", + name: "setting locked kube feature flag", + flags: []string{ + "--emulated-version=test=2.7", + "--feature-gates=kubeA=false", + }, + parseError: "cannot set feature gate kubeA to false, feature is locked to true", }, { - name: "setting unknown component feature flag", - emulationVersionFlag: "test=2.7", - featureGatesFlag: "test3:commonC=true", - parseError: "component not registered: test3", + name: "setting unknown test feature flag", + flags: []string{ + "--emulated-version=test=2.7", + "--feature-gates=test:testD=true", + }, + parseError: "unrecognized feature gate: testD", + }, + { + name: "setting unknown component feature flag", + flags: []string{ + "--emulated-version=test=2.7", + "--feature-gates=test3:commonC=true", + }, + parseError: "component not registered: test3", }, } for i, test := range tests { @@ -227,9 +277,7 @@ func TestFlags(t *testing.T) { fs := pflag.NewFlagSet("testflag", pflag.ContinueOnError) r := testRegistry(t) r.AddFlags(fs) - - err := fs.Parse([]string{fmt.Sprintf("--emulated-version=%s", test.emulationVersionFlag), - fmt.Sprintf("--feature-gates=%s", test.featureGatesFlag)}) + err := fs.Parse(test.flags) if err == nil { err = r.Set() } @@ -242,19 +290,11 @@ func TestFlags(t *testing.T) { if err != nil { t.Fatalf("%d: Parse() expected: nil, got: %v", i, err) } - if test.expectedKubeEmulationVersion != nil { - v := r.EffectiveVersionFor("kube").EmulationVersion() - if !v.EqualTo(test.expectedKubeEmulationVersion) { - t.Fatalf("%d: EmulationVersion expected: %s, got: %s", i, test.expectedKubeEmulationVersion.String(), v.String()) - return - } + if len(test.expectedKubeEmulationVersion) > 0 { + assertVersionEqualTo(t, r.EffectiveVersionFor(DefaultKubeComponent).EmulationVersion(), test.expectedKubeEmulationVersion) } - if test.expectedTestEmulationVersion != nil { - v := r.EffectiveVersionFor("test").EmulationVersion() - if !v.EqualTo(test.expectedTestEmulationVersion) { - t.Fatalf("%d: EmulationVersion expected: %s, got: %s", i, test.expectedTestEmulationVersion.String(), v.String()) - return - } + if len(test.expectedTestEmulationVersion) > 0 { + assertVersionEqualTo(t, r.EffectiveVersionFor(testComponent).EmulationVersion(), test.expectedTestEmulationVersion) } for f, v := range test.expectedKubeFeatureValues { if r.FeatureGateFor(DefaultKubeComponent).Enabled(f) != v { @@ -269,3 +309,110 @@ func TestFlags(t *testing.T) { }) } } + +func TestVersionMapping(t *testing.T) { + r := NewComponentGlobalsRegistry() + ver1 := NewEffectiveVersion("0.58") + ver2 := NewEffectiveVersion("1.28") + ver3 := NewEffectiveVersion("2.10") + + utilruntime.Must(r.Register("test1", ver1, nil)) + utilruntime.Must(r.Register("test2", ver2, nil)) + utilruntime.Must(r.Register("test3", ver3, nil)) + + assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.58") + assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.28") + assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.10") + + utilruntime.Must(r.SetEmulationVersionMapping("test2", "test3", + func(from *version.Version) *version.Version { + return version.MajorMinor(from.Major()+1, from.Minor()-19) + })) + utilruntime.Must(r.SetEmulationVersionMapping("test1", "test2", + func(from *version.Version) *version.Version { + return version.MajorMinor(from.Major()+1, from.Minor()-28) + })) + assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.58") + assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.30") + assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.11") + + fs := pflag.NewFlagSet("testflag", pflag.ContinueOnError) + r.AddFlags(fs) + + if err := fs.Parse([]string{fmt.Sprintf("--emulated-version=%s", "test1=0.56")}); err != nil { + t.Fatal(err) + return + } + if err := r.Set(); err != nil { + t.Fatal(err) + return + } + assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.56") + assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.28") + assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.09") +} + +func TestVersionMappingWithMultipleDependency(t *testing.T) { + r := NewComponentGlobalsRegistry() + ver1 := NewEffectiveVersion("0.58") + ver2 := NewEffectiveVersion("1.28") + ver3 := NewEffectiveVersion("2.10") + + utilruntime.Must(r.Register("test1", ver1, nil)) + utilruntime.Must(r.Register("test2", ver2, nil)) + utilruntime.Must(r.Register("test3", ver3, nil)) + + assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.58") + assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.28") + assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.10") + + utilruntime.Must(r.SetEmulationVersionMapping("test1", "test2", + func(from *version.Version) *version.Version { + return version.MajorMinor(from.Major()+1, from.Minor()-28) + })) + err := r.SetEmulationVersionMapping("test3", "test2", + func(from *version.Version) *version.Version { + return version.MajorMinor(from.Major()-1, from.Minor()+19) + }) + if err == nil { + t.Errorf("expect error when setting 2nd mapping to test2") + } +} + +func TestVersionMappingWithCyclicDependency(t *testing.T) { + r := NewComponentGlobalsRegistry() + ver1 := NewEffectiveVersion("0.58") + ver2 := NewEffectiveVersion("1.28") + ver3 := NewEffectiveVersion("2.10") + + utilruntime.Must(r.Register("test1", ver1, nil)) + utilruntime.Must(r.Register("test2", ver2, nil)) + utilruntime.Must(r.Register("test3", ver3, nil)) + + assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.58") + assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.28") + assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.10") + + utilruntime.Must(r.SetEmulationVersionMapping("test1", "test2", + func(from *version.Version) *version.Version { + return version.MajorMinor(from.Major()+1, from.Minor()-28) + })) + utilruntime.Must(r.SetEmulationVersionMapping("test2", "test3", + func(from *version.Version) *version.Version { + return version.MajorMinor(from.Major()+1, from.Minor()-19) + })) + err := r.SetEmulationVersionMapping("test3", "test1", + func(from *version.Version) *version.Version { + return version.MajorMinor(from.Major()-2, from.Minor()+48) + }) + if err == nil { + t.Errorf("expect cyclic version mapping error") + } +} + +func assertVersionEqualTo(t *testing.T, ver *version.Version, expectedVer string) { + if ver.EqualTo(version.MustParse(expectedVer)) { + return + } + t.Errorf("expected: %s, got %s", expectedVer, ver.String()) +} diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/version.go b/staging/src/k8s.io/apiserver/pkg/util/version/version.go index fce927c6c61..1596aef389b 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/version/version.go +++ b/staging/src/k8s.io/apiserver/pkg/util/version/version.go @@ -18,10 +18,8 @@ package version import ( "fmt" - "strings" "sync/atomic" - "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/util/version" baseversion "k8s.io/component-base/version" ) @@ -40,37 +38,6 @@ type MutableEffectiveVersion interface { Set(binaryVersion, emulationVersion, minCompatibilityVersion *version.Version) SetEmulationVersion(emulationVersion *version.Version) SetMinCompatibilityVersion(minCompatibilityVersion *version.Version) - // AddFlags adds the "{prefix}-emulated-version" to the flagset. - AddFlags(fs *pflag.FlagSet, prefix string) -} - -type VersionVar struct { - val atomic.Pointer[version.Version] -} - -// Set sets the flag value -func (v *VersionVar) Set(s string) error { - components := strings.Split(s, ".") - if len(components) != 2 { - return fmt.Errorf("version %s is not in the format of major.minor", s) - } - ver, err := version.ParseGeneric(s) - if err != nil { - return err - } - v.val.Store(ver) - return nil -} - -// String returns the flag value -func (v *VersionVar) String() string { - ver := v.val.Load() - return ver.String() -} - -// Type gets the flag type -func (v *VersionVar) Type() string { - return "version" } type effectiveVersion struct { @@ -78,9 +45,9 @@ type effectiveVersion struct { // If the emulationVersion is set by the users, it could only contain major and minor versions. // In tests, emulationVersion could be the same as the binary version, or set directly, // which can have "alpha" as pre-release to continue serving expired apis while we clean up the test. - emulationVersion VersionVar + emulationVersion atomic.Pointer[version.Version] // minCompatibilityVersion could only contain major and minor versions. - minCompatibilityVersion VersionVar + minCompatibilityVersion atomic.Pointer[version.Version] } func (m *effectiveVersion) BinaryVersion() *version.Version { @@ -88,13 +55,17 @@ func (m *effectiveVersion) BinaryVersion() *version.Version { } func (m *effectiveVersion) EmulationVersion() *version.Version { - // Emulation version can have "alpha" as pre-release to continue serving expired apis while we clean up the test. - // The pre-release should not be accessible to the users. - return m.emulationVersion.val.Load().WithPreRelease(m.BinaryVersion().PreRelease()) + ver := m.emulationVersion.Load() + if ver != nil { + // Emulation version can have "alpha" as pre-release to continue serving expired apis while we clean up the test. + // The pre-release should not be accessible to the users. + return ver.WithPreRelease(m.BinaryVersion().PreRelease()) + } + return ver } func (m *effectiveVersion) MinCompatibilityVersion() *version.Version { - return m.minCompatibilityVersion.val.Load() + return m.minCompatibilityVersion.Load() } func (m *effectiveVersion) EqualTo(other EffectiveVersion) bool { @@ -109,26 +80,33 @@ func (m *effectiveVersion) String() string { m.BinaryVersion().String(), m.EmulationVersion().String(), m.MinCompatibilityVersion().String()) } +func majorMinor(ver *version.Version) *version.Version { + if ver == nil { + return ver + } + return version.MajorMinor(ver.Major(), ver.Minor()) +} + func (m *effectiveVersion) Set(binaryVersion, emulationVersion, minCompatibilityVersion *version.Version) { m.binaryVersion.Store(binaryVersion) - m.emulationVersion.val.Store(version.MajorMinor(emulationVersion.Major(), emulationVersion.Minor())) - m.minCompatibilityVersion.val.Store(version.MajorMinor(minCompatibilityVersion.Major(), minCompatibilityVersion.Minor())) + m.emulationVersion.Store(majorMinor(emulationVersion)) + m.minCompatibilityVersion.Store(majorMinor(minCompatibilityVersion)) } func (m *effectiveVersion) SetEmulationVersion(emulationVersion *version.Version) { - m.emulationVersion.val.Store(version.MajorMinor(emulationVersion.Major(), emulationVersion.Minor())) + m.emulationVersion.Store(majorMinor(emulationVersion)) } func (m *effectiveVersion) SetMinCompatibilityVersion(minCompatibilityVersion *version.Version) { - m.minCompatibilityVersion.val.Store(version.MajorMinor(minCompatibilityVersion.Major(), minCompatibilityVersion.Minor())) + m.minCompatibilityVersion.Store(majorMinor(minCompatibilityVersion)) } func (m *effectiveVersion) Validate() []error { var errs []error // Validate only checks the major and minor versions. binaryVersion := m.binaryVersion.Load().WithPatch(0) - emulationVersion := m.emulationVersion.val.Load() - minCompatibilityVersion := m.minCompatibilityVersion.val.Load() + emulationVersion := m.emulationVersion.Load() + minCompatibilityVersion := m.minCompatibilityVersion.Load() // emulationVersion can only be 1.{binaryMinor-1}...1.{binaryMinor}. maxEmuVer := binaryVersion @@ -151,45 +129,36 @@ func (m *effectiveVersion) Validate() []error { return errs } -// AddFlags adds the "{prefix}-emulated-version" to the flagset. -func (m *effectiveVersion) AddFlags(fs *pflag.FlagSet, prefix string) { - if m == nil { - return - } - if len(prefix) > 0 && !strings.HasSuffix(prefix, "-") { - prefix += "-" - } - fs.Var(&m.emulationVersion, prefix+"emulated-version", ""+ - "The version the K8s component emulates its capabilities (APIs, features, ...) of.\n"+ - "If set, the component will emulate the behavior of this version instead of the underlying binary version.\n"+ - "Any capabilities present in the binary version that were introduced after the emulated version will be unavailable and any capabilities removed after the emulated version will be available.\n"+ - "This flag applies only to component capabilities, and does not disable bug fixes and performance improvements present in the binary version.\n"+ - "Defaults to the binary version. The value should be between 1.{binaryMinorVersion-1} and 1.{binaryMinorVersion}.\n"+ - "Format could only be major.minor") -} - -func NewEffectiveVersion(binaryVer string) MutableEffectiveVersion { +func newEffectiveVersion(binaryVersion *version.Version) MutableEffectiveVersion { effective := &effectiveVersion{} - binaryVersion := version.MustParse(binaryVer) compatVersion := binaryVersion.SubtractMinor(1) effective.Set(binaryVersion, binaryVersion, compatVersion) return effective } +func NewEffectiveVersion(binaryVer string) MutableEffectiveVersion { + if binaryVer == "" { + return &effectiveVersion{} + } + binaryVersion := version.MustParse(binaryVer) + return newEffectiveVersion(binaryVersion) +} + // DefaultBuildEffectiveVersion returns the MutableEffectiveVersion based on the // current build information. func DefaultBuildEffectiveVersion() MutableEffectiveVersion { verInfo := baseversion.Get() - ver := NewEffectiveVersion(verInfo.String()) - if ver.BinaryVersion().Major() == 0 && ver.BinaryVersion().Minor() == 0 { - ver = DefaultKubeEffectiveVersion() + binaryVersion := version.MustParse(verInfo.String()).WithInfo(verInfo) + if binaryVersion.Major() == 0 && binaryVersion.Minor() == 0 { + return DefaultKubeEffectiveVersion() } - return ver + return newEffectiveVersion(binaryVersion) } // DefaultKubeEffectiveVersion returns the MutableEffectiveVersion based on the // latest K8s release. // Should update for each minor release! func DefaultKubeEffectiveVersion() MutableEffectiveVersion { - return NewEffectiveVersion("1.31") + binaryVersion := version.MustParse("1.31").WithInfo(baseversion.Get()) + return newEffectiveVersion(binaryVersion) } diff --git a/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go b/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go index 24db0318f25..784e3e4e186 100644 --- a/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go +++ b/staging/src/k8s.io/apiserver/pkg/util/version/version_test.go @@ -17,11 +17,8 @@ limitations under the License. package version import ( - "fmt" - "strings" "testing" - "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/util/version" ) @@ -127,54 +124,3 @@ func TestValidate(t *testing.T) { }) } } - -func TestEffectiveVersionsFlag(t *testing.T) { - tests := []struct { - name string - emulationVersion string - expectedEmulationVersion *version.Version - parseError string - }{ - { - name: "major.minor ok", - emulationVersion: "1.30", - expectedEmulationVersion: version.MajorMinor(1, 30), - }, - { - name: "v prefix ok", - emulationVersion: "v1.30", - expectedEmulationVersion: version.MajorMinor(1, 30), - }, - { - name: "semantic version not ok", - emulationVersion: "1.30.1", - parseError: "version 1.30.1 is not in the format of major.minor", - }, - { - name: "invalid version", - emulationVersion: "1.foo", - parseError: "illegal version string", - }, - } - for i, test := range tests { - t.Run(test.name, func(t *testing.T) { - fs := pflag.NewFlagSet("testflag", pflag.ContinueOnError) - effective := NewEffectiveVersion("1.30") - effective.AddFlags(fs, "test") - - err := fs.Parse([]string{fmt.Sprintf("--test-emulated-version=%s", test.emulationVersion)}) - if test.parseError != "" { - if !strings.Contains(err.Error(), test.parseError) { - t.Fatalf("%d: Parse() Expected %v, Got %v", i, test.parseError, err) - } - return - } - if err != nil { - t.Fatalf("%d: Parse() Expected nil, Got %v", i, err) - } - if !effective.EmulationVersion().EqualTo(test.expectedEmulationVersion) { - t.Errorf("%d: EmulationVersion Expected %s, Got %s", i, test.expectedEmulationVersion.String(), effective.EmulationVersion().String()) - } - }) - } -} diff --git a/staging/src/k8s.io/component-base/featuregate/feature_gate.go b/staging/src/k8s.io/component-base/featuregate/feature_gate.go index d164baf3878..680d92d3d83 100644 --- a/staging/src/k8s.io/component-base/featuregate/feature_gate.go +++ b/staging/src/k8s.io/component-base/featuregate/feature_gate.go @@ -127,6 +127,9 @@ type MutableFeatureGate interface { AddFlag(fs *pflag.FlagSet) // Close sets closed to true, and prevents subsequent calls to Add Close() + // OpenForModification sets closedForModification to false, and allows subsequent calls to SetEmulationVersion to change enabled features + // before the next Enabled is called. + OpenForModification() // Set parses and stores flag gates for known features // from a string like feature1=true,feature2=false,... Set(value string) error @@ -163,11 +166,25 @@ type MutableVersionedFeatureGate interface { // SetEmulationVersion overrides the emulationVersion of the feature gate. // Otherwise, the emulationVersion will be the same as the binary version. // If set, the feature defaults and availability will be as if the binary is at the emulated version. + // Returns error if the new emulationVersion will change the enablement state of a feature that has already been queried. + // If you have to use featureGate.Enabled before parsing the flags, call featureGate.OpenForModification following featureGate.Enabled. SetEmulationVersion(emulationVersion *version.Version) error // GetAll returns a copy of the map of known feature names to versioned feature specs. GetAllVersioned() map[Feature]VersionedSpecs // AddVersioned adds versioned feature specs to the featureGate. AddVersioned(features map[Feature]VersionedSpecs) error + // OverrideDefaultAtVersion sets a local override for the registered default value of a named + // feature for the prerelease lifecycle the given version is at. + // If the feature has not been previously registered (e.g. by a call to Add), + // has a locked default, or if the gate has already registered itself with a FlagSet, a non-nil + // error is returned. + // + // When two or more components consume a common feature, one component can override its + // default at runtime in order to adopt new defaults before or after the other + // components. For example, a new feature can be evaluated with a limited blast radius by + // overriding its default to true for a limited number of components without simultaneously + // changing its default for all consuming components. + OverrideDefaultAtVersion(name Feature, override bool, ver *version.Version) error } // MutableVersionedFeatureGateForTests is a feature gate interface that should only be used in tests. @@ -196,7 +213,14 @@ type featureGate struct { // while enabled keeps the values of all resolved features. enabledRaw atomic.Value // closed is set to true when AddFlag is called, and prevents subsequent calls to Add - closed bool + closed bool + // closedForModification is set to true when Enabled is called, and prevents subsequent calls to SetEmulationVersion to change the enabled features. + // TODO: after all feature gates have migrated to versioned feature gates, + // closedForModification should also prevents subsequent calls to Set and SetFromMap to change the enabled features + closedForModification atomic.Bool + // queriedFeatures stores all the features that have been queried through the Enabled interface. + // It is reset when closedForModification is reset. + queriedFeatures atomic.Value emulationVersion atomic.Pointer[version.Version] } @@ -205,8 +229,8 @@ func setUnsetAlphaGates(known map[Feature]VersionedSpecs, enabled map[Feature]bo if k == "AllAlpha" || k == "AllBeta" { continue } - currentVersion := getCurrentVersion(v, cVer) - if currentVersion.PreRelease == Alpha { + featureSpec := featureSpecAtEmulationVersion(v, cVer) + if featureSpec.PreRelease == Alpha { if _, found := enabled[k]; !found { enabled[k] = val } @@ -219,8 +243,8 @@ func setUnsetBetaGates(known map[Feature]VersionedSpecs, enabled map[Feature]boo if k == "AllAlpha" || k == "AllBeta" { continue } - currentVersion := getCurrentVersion(v, cVer) - if currentVersion.PreRelease == Beta { + featureSpec := featureSpecAtEmulationVersion(v, cVer) + if featureSpec.PreRelease == Beta { if _, found := enabled[k]; !found { enabled[k] = val } @@ -251,6 +275,7 @@ func NewVersionedFeatureGate(emulationVersion *version.Version) *featureGate { f.enabled.Store(map[Feature]bool{}) f.enabledRaw.Store(map[string]bool{}) f.emulationVersion.Store(emulationVersion) + f.queriedFeatures.Store(map[Feature]struct{}{}) klog.V(1).Infof("new feature gate with emulationVersion=%s", f.emulationVersion.Load().String()) return f } @@ -291,11 +316,11 @@ func (f *featureGate) Validate() []error { return []error{fmt.Errorf("cannot cast enabledRaw to map[string]bool")} } enabled := map[Feature]bool{} - return f.unsafeSetFromMap(enabled, m) + return f.unsafeSetFromMap(enabled, m, f.EmulationVersion()) } // unsafeSetFromMap stores flag gates for known features from a map[string]bool into an enabled map. -func (f *featureGate) unsafeSetFromMap(enabled map[Feature]bool, m map[string]bool) []error { +func (f *featureGate) unsafeSetFromMap(enabled map[Feature]bool, m map[string]bool, emulationVersion *version.Version) []error { var errs []error // Copy existing state known := map[Feature]VersionedSpecs{} @@ -312,26 +337,26 @@ func (f *featureGate) unsafeSetFromMap(enabled map[Feature]bool, m map[string]bo errs = append(errs, fmt.Errorf("unrecognized feature gate: %s", k)) return errs } - currentVersion := f.getCurrentVersion(versionedSpecs) - if currentVersion.LockToDefault && currentVersion.Default != v { - errs = append(errs, fmt.Errorf("cannot set feature gate %v to %v, feature is locked to %v", k, v, currentVersion.Default)) + featureSpec := featureSpecAtEmulationVersion(versionedSpecs, emulationVersion) + if featureSpec.LockToDefault && featureSpec.Default != v { + errs = append(errs, fmt.Errorf("cannot set feature gate %v to %v, feature is locked to %v", k, v, featureSpec.Default)) continue } // Handle "special" features like "all alpha gates" if fn, found := f.special[key]; found { - fn(known, enabled, v, f.emulationVersion.Load()) + fn(known, enabled, v, emulationVersion) enabled[key] = v continue } - if currentVersion.PreRelease == PreAlpha { - errs = append(errs, fmt.Errorf("cannot set feature gate %v to %v, feature is PreAlpha at emulated version %s", k, v, f.EmulationVersion().String())) + if featureSpec.PreRelease == PreAlpha { + errs = append(errs, fmt.Errorf("cannot set feature gate %v to %v, feature is PreAlpha at emulated version %s", k, v, emulationVersion.String())) continue } enabled[key] = v - if currentVersion.PreRelease == Deprecated { + if featureSpec.PreRelease == Deprecated { klog.Warningf("Setting deprecated feature gate %s=%t. It will be removed in a future release.", k, v) - } else if currentVersion.PreRelease == GA { + } else if featureSpec.PreRelease == GA { klog.Warningf("Setting GA feature gate %s=%t. It will be removed in a future release.", k, v) } } @@ -361,7 +386,7 @@ func (f *featureGate) SetFromMap(m map[string]bool) error { } f.enabledRaw.Store(enabledRaw) - errs := f.unsafeSetFromMap(enabled, enabledRaw) + errs := f.unsafeSetFromMap(enabled, enabledRaw, f.EmulationVersion()) if len(errs) == 0 { // Persist changes f.enabled.Store(enabled) @@ -429,6 +454,10 @@ func (f *featureGate) AddVersioned(features map[Feature]VersionedSpecs) error { } func (f *featureGate) OverrideDefault(name Feature, override bool) error { + return f.OverrideDefaultAtVersion(name, override, f.EmulationVersion()) +} + +func (f *featureGate) OverrideDefaultAtVersion(name Feature, override bool, ver *version.Version) error { f.lock.Lock() defer f.lock.Unlock() @@ -446,12 +475,12 @@ func (f *featureGate) OverrideDefault(name Feature, override bool) error { if !ok { return fmt.Errorf("cannot override default: feature %q is not registered", name) } - spec := f.getCurrentVersion(specs) + spec := featureSpecAtEmulationVersion(specs, ver) switch { case spec.LockToDefault: return fmt.Errorf("cannot override default: feature %q default is locked to %t", name, spec.Default) case spec.PreRelease == PreAlpha: - return fmt.Errorf("cannot override default: feature %q is not available before emulation version %s", name, f.EmulationVersion().String()) + return fmt.Errorf("cannot override default: feature %q is not available before version %s", name, ver.String()) case spec.PreRelease == Deprecated: klog.Warningf("Overriding default of deprecated feature gate %s=%t. It will be removed in a future release.", name, override) case spec.PreRelease == GA: @@ -469,12 +498,12 @@ func (f *featureGate) OverrideDefault(name Feature, override bool) error { func (f *featureGate) GetAll() map[Feature]FeatureSpec { retval := map[Feature]FeatureSpec{} for k, v := range f.GetAllVersioned() { - spec := f.getCurrentVersion(v) + spec := f.featureSpecAtEmulationVersion(v) if spec.PreRelease == PreAlpha { // The feature is not available at the emulation version. continue } - retval[k] = *f.getCurrentVersion(v) + retval[k] = *f.featureSpecAtEmulationVersion(v) } return retval } @@ -492,7 +521,6 @@ func (f *featureGate) SetEmulationVersion(emulationVersion *version.Version) err f.lock.Lock() defer f.lock.Unlock() klog.V(1).Infof("set feature gate emulationVersion to %s", emulationVersion.String()) - f.emulationVersion.Store(emulationVersion) // Copy existing state enabledRaw := map[string]bool{} @@ -501,10 +529,24 @@ func (f *featureGate) SetEmulationVersion(emulationVersion *version.Version) err } // enabled map should be reset whenever emulationVersion is changed. enabled := map[Feature]bool{} - errs := f.unsafeSetFromMap(enabled, enabledRaw) + errs := f.unsafeSetFromMap(enabled, enabledRaw, emulationVersion) + + if f.closedForModification.Load() { + queriedFeatures := f.queriedFeatures.Load().(map[Feature]struct{}) + known := f.known.Load().(map[Feature]VersionedSpecs) + for feature := range queriedFeatures { + newVal := featureEnabled(feature, enabled, known, emulationVersion) + oldVal := f.Enabled(feature) + // it is ok to modify emulation version if it does not result in feature enablemennt change for features that have already been queried. + if newVal != oldVal { + errs = append(errs, fmt.Errorf("SetEmulationVersion will change already queried feature:%s from %v to %v\ncall featureGate.OpenForModification() first to override", feature, oldVal, newVal)) + } + } + } if len(errs) == 0 { // Persist changes f.enabled.Store(enabled) + f.emulationVersion.Store(emulationVersion) } return utilerrors.NewAggregate(errs) } @@ -514,32 +556,49 @@ func (f *featureGate) EmulationVersion() *version.Version { } // FeatureSpec returns the FeatureSpec at the EmulationVersion if the key exists, an error otherwise. +// This is useful to keep multiple implementations of a feature based on the PreRelease or Version info. func (f *featureGate) FeatureSpec(key Feature) (FeatureSpec, error) { if v, ok := f.known.Load().(map[Feature]VersionedSpecs)[key]; ok { - currentVersion := f.getCurrentVersion(v) - return *currentVersion, nil + featureSpec := f.featureSpecAtEmulationVersion(v) + return *featureSpec, nil } return FeatureSpec{}, fmt.Errorf("feature %q is not registered in FeatureGate %q", key, f.featureGateName) } -// Enabled returns true if the key is enabled. If the key is not known, this call will panic. -func (f *featureGate) Enabled(key Feature) bool { - // fallback to default behavior, since we don't have emulation version set - if v, ok := f.enabled.Load().(map[Feature]bool)[key]; ok { +func (f *featureGate) recordQueried(key Feature) { + queriedFeatures := map[Feature]struct{}{} + for k := range f.queriedFeatures.Load().(map[Feature]struct{}) { + queriedFeatures[k] = struct{}{} + } + queriedFeatures[key] = struct{}{} + f.queriedFeatures.Store(queriedFeatures) + f.closedForModification.Store(true) +} + +func featureEnabled(key Feature, enabled map[Feature]bool, known map[Feature]VersionedSpecs, emulationVersion *version.Version) bool { + // check explicitly set enabled list + if v, ok := enabled[key]; ok { return v } - if v, ok := f.known.Load().(map[Feature]VersionedSpecs)[key]; ok { - return f.getCurrentVersion(v).Default + if v, ok := known[key]; ok { + return featureSpecAtEmulationVersion(v, emulationVersion).Default } - panic(fmt.Errorf("feature %q is not registered in FeatureGate %q", key, f.featureGateName)) + panic(fmt.Errorf("feature %q is not registered in FeatureGate", key)) } -func (f *featureGate) getCurrentVersion(v VersionedSpecs) *FeatureSpec { - return getCurrentVersion(v, f.EmulationVersion()) +// Enabled returns true if the key is enabled. If the key is not known, this call will panic. +func (f *featureGate) Enabled(key Feature) bool { + v := featureEnabled(key, f.enabled.Load().(map[Feature]bool), f.known.Load().(map[Feature]VersionedSpecs), f.EmulationVersion()) + f.recordQueried(key) + return v } -func getCurrentVersion(v VersionedSpecs, emulationVersion *version.Version) *FeatureSpec { +func (f *featureGate) featureSpecAtEmulationVersion(v VersionedSpecs) *FeatureSpec { + return featureSpecAtEmulationVersion(v, f.EmulationVersion()) +} + +func featureSpecAtEmulationVersion(v VersionedSpecs, emulationVersion *version.Version) *FeatureSpec { i := len(v) - 1 for ; i >= 0; i-- { if v[i].Version.GreaterThan(emulationVersion) { @@ -561,6 +620,18 @@ func (f *featureGate) Close() { f.lock.Unlock() } +// OpenForModification sets closedForModification to false, and allows subsequent calls to SetEmulationVersion to change enabled features +// before the next Enabled is called. +func (f *featureGate) OpenForModification() { + queriedFeatures := []Feature{} + for feature := range f.queriedFeatures.Load().(map[Feature]struct{}) { + queriedFeatures = append(queriedFeatures, feature) + } + klog.Warningf("open feature gate for modification after querying features: %v.", queriedFeatures) + f.closedForModification.Store(false) + f.queriedFeatures.Store(map[Feature]struct{}{}) +} + // AddFlag adds a flag for setting global feature gates to the specified FlagSet. func (f *featureGate) AddFlag(fs *pflag.FlagSet) { // TODO(mtaufen): Shouldn't we just close it on the first Set/SetFromMap instead? @@ -590,11 +661,11 @@ func (f *featureGate) KnownFeatures() []string { known = append(known, fmt.Sprintf("%s=true|false (%s - default=%t)", k, v[0].PreRelease, v[0].Default)) continue } - currentV := f.getCurrentVersion(v) - if currentV.PreRelease == GA || currentV.PreRelease == Deprecated || currentV.PreRelease == PreAlpha { + featureSpec := f.featureSpecAtEmulationVersion(v) + if featureSpec.PreRelease == GA || featureSpec.PreRelease == Deprecated || featureSpec.PreRelease == PreAlpha { continue } - known = append(known, fmt.Sprintf("%s=true|false (%s - default=%t)", k, currentV.PreRelease, currentV.Default)) + known = append(known, fmt.Sprintf("%s=true|false (%s - default=%t)", k, featureSpec.PreRelease, featureSpec.Default)) } sort.Strings(known) return known @@ -620,7 +691,7 @@ func (f *featureGate) DeepCopy() MutableVersionedFeatureGate { // Construct a new featureGate around the copied state. // Note that specialFeatures is treated as immutable by convention, - // and we maintain the value of f.closed across the copy. + // and we maintain the value of f.closed across the copy, but resets closedForModification. fg := &featureGate{ special: specialFeatures, closed: f.closed, @@ -629,6 +700,7 @@ func (f *featureGate) DeepCopy() MutableVersionedFeatureGate { fg.known.Store(known) fg.enabled.Store(enabled) fg.enabledRaw.Store(enabledRaw) + fg.queriedFeatures.Store(map[Feature]struct{}{}) return fg } @@ -636,9 +708,15 @@ func (f *featureGate) DeepCopy() MutableVersionedFeatureGate { func (f *featureGate) Reset(m map[string]bool) { enabled := map[Feature]bool{} enabledRaw := map[string]bool{} + queriedFeatures := map[Feature]struct{}{} f.enabled.Store(enabled) f.enabledRaw.Store(enabledRaw) _ = f.SetFromMap(m) + f.closedForModification.Store(false) + f.queriedFeatures.Store(queriedFeatures) + f.lock.Lock() + defer f.lock.Unlock() + f.closed = false } func (f *featureGate) EnabledRawMap() map[string]bool { diff --git a/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go b/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go index 9156a024abe..85f99d25787 100644 --- a/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go +++ b/staging/src/k8s.io/component-base/featuregate/feature_gate_test.go @@ -1280,6 +1280,48 @@ func TestVersionedFeatureGateOverrideDefault(t *testing.T) { } }) + t.Run("overrides at specific version take effect", func(t *testing.T) { + f := NewVersionedFeatureGate(version.MustParse("1.29")) + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + if err := f.AddVersioned(map[Feature]VersionedSpecs{ + "TestFeature1": { + {Version: version.MustParse("1.28"), Default: true}, + }, + "TestFeature2": { + {Version: version.MustParse("1.26"), Default: false}, + {Version: version.MustParse("1.29"), Default: false}, + }, + }); err != nil { + t.Fatal(err) + } + if f.OverrideDefaultAtVersion("TestFeature1", false, version.MustParse("1.27")) == nil { + t.Error("expected error when attempting to override the default for a feature not available at given version") + } + require.NoError(t, f.OverrideDefaultAtVersion("TestFeature2", true, version.MustParse("1.27"))) + if !f.Enabled("TestFeature1") { + t.Error("expected TestFeature1 to have effective default of true") + } + if !f.Enabled("TestFeature2") { + t.Error("expected TestFeature2 to have effective default of true") + } + f.OpenForModification() + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.29"))) + if !f.Enabled("TestFeature1") { + t.Error("expected TestFeature1 to have effective default of true") + } + if f.Enabled("TestFeature2") { + t.Error("expected TestFeature2 to have effective default of false") + } + f.OpenForModification() + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.26"))) + if f.Enabled("TestFeature1") { + t.Error("expected TestFeature1 to have effective default of false") + } + if !f.Enabled("TestFeature2") { + t.Error("expected TestFeature2 to have effective default of true") + } + }) + t.Run("overrides are preserved across deep copies", func(t *testing.T) { f := NewVersionedFeatureGate(version.MustParse("1.29")) require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) @@ -1433,7 +1475,7 @@ func TestVersionedFeatureGateOverrideDefault(t *testing.T) { }) } -func TestGetCurrentVersion(t *testing.T) { +func TestFeatureSpecAtEmulationVersion(t *testing.T) { specs := VersionedSpecs{{Version: version.MustParse("1.29"), Default: true, PreRelease: GA}, {Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, {Version: version.MustParse("1.25"), Default: false, PreRelease: Alpha}, @@ -1469,11 +1511,41 @@ func TestGetCurrentVersion(t *testing.T) { }, } for i, test := range tests { - t.Run(fmt.Sprintf("getCurrentVersion for emulationVersion %s", test.cVersion), func(t *testing.T) { - result := getCurrentVersion(specs, version.MustParse(test.cVersion)) + t.Run(fmt.Sprintf("featureSpecAtEmulationVersion for emulationVersion %s", test.cVersion), func(t *testing.T) { + result := featureSpecAtEmulationVersion(specs, version.MustParse(test.cVersion)) if !reflect.DeepEqual(*result, test.expect) { - t.Errorf("%d: getCurrentVersion(, %s) Expected %v, Got %v", i, test.cVersion, test.expect, result) + t.Errorf("%d: featureSpecAtEmulationVersion(, %s) Expected %v, Got %v", i, test.cVersion, test.expect, result) } }) } } + +func TestOpenForModification(t *testing.T) { + const testBetaGate Feature = "testBetaGate" + f := NewVersionedFeatureGate(version.MustParse("1.29")) + + err := f.AddVersioned(map[Feature]VersionedSpecs{ + testBetaGate: { + {Version: version.MustParse("1.29"), Default: true, PreRelease: Beta}, + {Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.26"), Default: false, PreRelease: Alpha}, + }, + }) + require.NoError(t, err) + + if f.Enabled(testBetaGate) != true { + t.Errorf("Expected true") + } + err = f.SetEmulationVersion(version.MustParse("1.28")) + if err == nil { + t.Fatalf("Expected error when SetEmulationVersion after querying features") + } + if f.Enabled(testBetaGate) != true { + t.Errorf("Expected true") + } + f.OpenForModification() + require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) + if f.Enabled(testBetaGate) != false { + t.Errorf("Expected false at 1.28") + } +} diff --git a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go index 93ac2b14b31..8b2499a8b6c 100644 --- a/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go @@ -38,7 +38,6 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/transport" "k8s.io/component-base/tracing" - "k8s.io/component-base/version" v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" v1helper "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper" "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1" @@ -185,8 +184,6 @@ func (cfg *Config) Complete() CompletedConfig { // the kube aggregator wires its own discovery mechanism // TODO eventually collapse this by extracting all of the discovery out c.GenericConfig.EnableDiscovery = false - version := version.Get() - c.GenericConfig.Version = &version return CompletedConfig{&c} } diff --git a/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go b/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go index 6558efda6f1..e8307e72f04 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/apiserver/apiserver.go @@ -17,17 +17,13 @@ limitations under the License. package apiserver import ( - "strconv" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/apimachinery/pkg/version" "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" - utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/sample-apiserver/pkg/apis/wardle" "k8s.io/sample-apiserver/pkg/apis/wardle/install" wardleregistry "k8s.io/sample-apiserver/pkg/registry" @@ -94,11 +90,6 @@ func (cfg *Config) Complete() CompletedConfig { cfg.GenericConfig.Complete(), &cfg.ExtraConfig, } - wardleEffectiveVersion := utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(WardleComponentName) - c.GenericConfig.Version = &version.Info{ - Major: strconv.Itoa(int(wardleEffectiveVersion.BinaryVersion().Major())), - Minor: strconv.Itoa(int(wardleEffectiveVersion.BinaryVersion().Minor())), - } return CompletedConfig{&c} } diff --git a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go index df497e450f2..6152d0b84ca 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go @@ -59,22 +59,14 @@ type WardleServerOptions struct { AlternateDNS []string } -func mapWardleEffectiveVersionToKubeEffectiveVersion(registry utilversion.ComponentGlobalsRegistry) error { - wardleVer := registry.EffectiveVersionFor(apiserver.WardleComponentName) - kubeVer := registry.EffectiveVersionFor(utilversion.DefaultKubeComponent).(utilversion.MutableEffectiveVersion) - // map from wardle emulation version to kube emulation version. - emulationVersionMap := map[string]string{ - "1.2": kubeVer.BinaryVersion().AddMinor(1).String(), - "1.1": kubeVer.BinaryVersion().String(), - "1.0": kubeVer.BinaryVersion().SubtractMinor(1).String(), +func wardleEmulationVersionToKubeEmulationVersion(ver *version.Version) *version.Version { + if ver.Major() != 1 { + return nil } - wardleEmulationVer := wardleVer.EmulationVersion() - if kubeEmulationVer, ok := emulationVersionMap[wardleEmulationVer.String()]; ok { - kubeVer.SetEmulationVersion(version.MustParse(kubeEmulationVer)) - } else { - return fmt.Errorf("cannot find mapping from wardle emulation version: %s to kube version", wardleVer.EmulationVersion().String()) - } - return nil + kubeVer := utilversion.DefaultKubeEffectiveVersion().BinaryVersion() + // "1.1" maps to kubeVer + offset := int(ver.Minor()) - 1 + return kubeVer.OffsetMinor(offset) } // NewWardleServerOptions returns a new WardleServerOptions @@ -100,11 +92,7 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti Short: "Launch a wardle API server", Long: "Launch a wardle API server", PersistentPreRunE: func(*cobra.Command, []string) error { - if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil { - return err - } - // convert wardle effective version to kube effective version to be used in generic api server, and set the generic api server feature gate. - return mapWardleEffectiveVersionToKubeEffectiveVersion(utilversion.DefaultComponentGlobalsRegistry) + return utilversion.DefaultComponentGlobalsRegistry.Set() }, RunE: func(c *cobra.Command, args []string) error { if err := o.Complete(); err != nil { @@ -133,10 +121,10 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti {Version: version.MustParse("1.0"), Default: false, PreRelease: featuregate.Alpha}, }, })) - utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, wardleFeatureGate, false)) + utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, wardleFeatureGate)) _, _ = utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister( utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate) - + utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.SetEmulationVersionMapping(apiserver.WardleComponentName, utilversion.DefaultKubeComponent, wardleEmulationVersionToKubeEmulationVersion)) utilversion.DefaultComponentGlobalsRegistry.AddFlags(flags) return cmd @@ -189,8 +177,8 @@ func (o *WardleServerOptions) Config() (*apiserver.Config, error) { serverConfig.OpenAPIV3Config.Info.Title = "Wardle" serverConfig.OpenAPIV3Config.Info.Version = "0.1" - serverConfig.FeatureGate = utilversion.DefaultComponentGlobalsRegistry.FeatureGateFor(utilversion.DefaultKubeComponent) - serverConfig.EffectiveVersion = utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent) + serverConfig.FeatureGate = utilversion.DefaultComponentGlobalsRegistry.FeatureGateFor(apiserver.WardleComponentName) + serverConfig.EffectiveVersion = utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(apiserver.WardleComponentName) if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil { return nil, err diff --git a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go index 7d7a0bc74e7..5a803318c12 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go @@ -21,13 +21,11 @@ import ( "k8s.io/apimachinery/pkg/util/version" utilversion "k8s.io/apiserver/pkg/util/version" - "k8s.io/sample-apiserver/pkg/apiserver" "github.com/stretchr/testify/assert" ) -func TestMapBinaryEffectiveVersionToKubeEffectiveVersion(t *testing.T) { - wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2") +func TestWardleEmulationVersionToKubeEmulationVersion(t *testing.T) { defaultKubeEffectiveVersion := utilversion.DefaultKubeEffectiveVersion() testCases := []struct { @@ -35,32 +33,26 @@ func TestMapBinaryEffectiveVersionToKubeEffectiveVersion(t *testing.T) { wardleEmulationVer *version.Version expectedKubeEmulationVer *version.Version }{ + { + desc: "same version as than kube binary", + wardleEmulationVer: version.MajorMinor(1, 1), + expectedKubeEmulationVer: defaultKubeEffectiveVersion.BinaryVersion(), + }, { desc: "1 version higher than kube binary", wardleEmulationVer: version.MajorMinor(1, 2), - expectedKubeEmulationVer: defaultKubeEffectiveVersion.BinaryVersion().AddMinor(1), + expectedKubeEmulationVer: defaultKubeEffectiveVersion.BinaryVersion().OffsetMinor(1), }, { desc: "no mapping", - wardleEmulationVer: version.MajorMinor(1, 10), + wardleEmulationVer: version.MajorMinor(2, 10), }, } for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - registry := utilversion.NewComponentGlobalsRegistry() - _ = registry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, nil, true) - _ = registry.Register(utilversion.DefaultKubeComponent, defaultKubeEffectiveVersion, nil, true) - - wardleEffectiveVersion.SetEmulationVersion(tc.wardleEmulationVer) - err := mapWardleEffectiveVersionToKubeEffectiveVersion(registry) - if tc.expectedKubeEmulationVer == nil { - if err == nil { - t.Fatal("expected error, no error found") - } - } else { - assert.True(t, registry.EffectiveVersionFor(utilversion.DefaultKubeComponent).EmulationVersion().EqualTo(tc.expectedKubeEmulationVer)) - } + mappedKubeEmulationVer := wardleEmulationVersionToKubeEmulationVersion(tc.wardleEmulationVer) + assert.True(t, mappedKubeEmulationVer.EqualTo(tc.expectedKubeEmulationVer)) }) } } diff --git a/test/integration/apiserver/apiserver_test.go b/test/integration/apiserver/apiserver_test.go index eba370d6e91..5c4152915a5 100644 --- a/test/integration/apiserver/apiserver_test.go +++ b/test/integration/apiserver/apiserver_test.go @@ -56,6 +56,7 @@ import ( "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/endpoints/handlers" "k8s.io/apiserver/pkg/storage/storagebackend" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/discovery/cached/memory" "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" @@ -3103,6 +3104,48 @@ func TestEmulatedStorageVersion(t *testing.T) { } } +// TestAllowedEmulationVersions tests the TestServer can start without problem for all allowed emulation versions. +func TestAllowedEmulationVersions(t *testing.T) { + tcs := []struct { + name string + emulationVersion string + }{ + { + name: "default", + emulationVersion: utilversion.DefaultKubeEffectiveVersion().EmulationVersion().String(), + }, + } + + for _, tc := range tcs { + t.Run(tc.emulationVersion, func(t *testing.T) { + server := kubeapiservertesting.StartTestServerOrDie(t, nil, + []string{fmt.Sprintf("--emulated-version=kube=%s", tc.emulationVersion)}, framework.SharedEtcd()) + defer server.TearDownFn() + + rt, err := restclient.TransportFor(server.ClientConfig) + if err != nil { + t.Fatal(err) + } + + req, err := http.NewRequest("GET", server.ClientConfig.Host+"/", nil) + if err != nil { + t.Fatal(err) + } + resp, err := rt.RoundTrip(req) + if err != nil { + t.Fatal(err) + } + expectedStatusCode := 200 + if resp.StatusCode != expectedStatusCode { + t.Errorf("expect status code: %d, got : %d\n", expectedStatusCode, resp.StatusCode) + } + defer func() { + _ = resp.Body.Close() + }() + }) + } +} + func TestEnableEmulationVersion(t *testing.T) { server := kubeapiservertesting.StartTestServerOrDie(t, &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "1.32"}, diff --git a/test/integration/client/client_test.go b/test/integration/client/client_test.go index 4802ca1e750..02b56c6a64f 100644 --- a/test/integration/client/client_test.go +++ b/test/integration/client/client_test.go @@ -48,6 +48,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/utils/pointer" + utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/component-base/version" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" "k8s.io/kubernetes/pkg/api/legacyscheme" @@ -65,7 +66,12 @@ func TestClient(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - if e, a := version.Get(), *info; !reflect.DeepEqual(e, a) { + expectedInfo := version.Get() + kubeVersion := utilversion.DefaultKubeEffectiveVersion().BinaryVersion() + expectedInfo.Major = fmt.Sprintf("%d", kubeVersion.Major()) + expectedInfo.Minor = fmt.Sprintf("%d", kubeVersion.Minor()) + + if e, a := expectedInfo, *info; !reflect.DeepEqual(e, a) { t.Errorf("expected %#v, got %#v", e, a) } diff --git a/test/integration/etcd/crd_overlap_storage_test.go b/test/integration/etcd/crd_overlap_storage_test.go index ebe1e042dc3..7aa898a0bf2 100644 --- a/test/integration/etcd/crd_overlap_storage_test.go +++ b/test/integration/etcd/crd_overlap_storage_test.go @@ -32,11 +32,9 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" - utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/dynamic" apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" apiregistrationclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1" - "k8s.io/kubernetes/cmd/kube-apiserver/app/options" ) // TestOverlappingBuiltInResources ensures the list of group-resources the custom resource finalizer should skip is up to date @@ -71,9 +69,7 @@ func TestOverlappingBuiltInResources(t *testing.T) { // TestOverlappingCustomResourceAPIService ensures creating and deleting a custom resource overlapping with APIServices does not destroy APIService data func TestOverlappingCustomResourceAPIService(t *testing.T) { - apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) { - opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("1.30") - }) + apiServer := StartRealAPIServerOrDie(t) defer apiServer.Cleanup() apiServiceClient, err := apiregistrationclient.NewForConfig(apiServer.Config) @@ -235,9 +231,7 @@ func TestOverlappingCustomResourceAPIService(t *testing.T) { // TestOverlappingCustomResourceCustomResourceDefinition ensures creating and deleting a custom resource overlapping with CustomResourceDefinition does not destroy CustomResourceDefinition data func TestOverlappingCustomResourceCustomResourceDefinition(t *testing.T) { - apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) { - opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("1.30") - }) + apiServer := StartRealAPIServerOrDie(t) defer apiServer.Cleanup() crdClient, err := crdclient.NewForConfig(apiServer.Config) diff --git a/test/integration/etcd/etcd_cross_group_test.go b/test/integration/etcd/etcd_cross_group_test.go index f24f0ee39b4..c2592fc0a49 100644 --- a/test/integration/etcd/etcd_cross_group_test.go +++ b/test/integration/etcd/etcd_cross_group_test.go @@ -39,7 +39,7 @@ import ( func TestCrossGroupStorage(t *testing.T) { apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) { // force enable all resources so we can check storage. - opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("1.30") + opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("0.0") }) defer apiServer.Cleanup() diff --git a/test/integration/examples/apiserver_test.go b/test/integration/examples/apiserver_test.go index 8d4164ea9b5..55cc7f61d2e 100644 --- a/test/integration/examples/apiserver_test.go +++ b/test/integration/examples/apiserver_test.go @@ -226,6 +226,18 @@ func TestAPIServiceWaitOnStart(t *testing.T) { } func TestAggregatedAPIServer(t *testing.T) { + t.Run("WithoutWardleFeatureGateAtV1.2", func(t *testing.T) { + testAggregatedAPIServer(t, false, "1.2") + }) + t.Run("WithoutWardleFeatureGateAtV1.1", func(t *testing.T) { + testAggregatedAPIServer(t, false, "1.1") + }) + t.Run("WithWardleFeatureGateAtV1.1", func(t *testing.T) { + testAggregatedAPIServer(t, true, "1.1") + }) +} + +func testAggregatedAPIServer(t *testing.T, enableWardleFeatureGate bool, emulationVersion string) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) t.Cleanup(cancel) @@ -240,7 +252,7 @@ func TestAggregatedAPIServer(t *testing.T) { // endpoints cannot have loopback IPs so we need to override the resolver itself t.Cleanup(app.SetServiceResolverForTests(staticURLServiceResolver(fmt.Sprintf("https://127.0.0.1:%d", wardlePort)))) - testServer := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{EnableCertAuth: true}, nil, framework.SharedEtcd()) + testServer := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{EnableCertAuth: true, BinaryVersion: "1.32"}, nil, framework.SharedEtcd()) defer testServer.TearDownFn() kubeClientConfig := rest.CopyConfig(testServer.ClientConfig) // force json because everything speaks it @@ -286,15 +298,18 @@ func TestAggregatedAPIServer(t *testing.T) { o.RecommendedOptions.SecureServing.Listener = listener o.RecommendedOptions.SecureServing.BindAddress = netutils.ParseIPSloppy("127.0.0.1") wardleCmd := sampleserver.NewCommandStartWardleServer(ctx, o) - wardleCmd.SetArgs([]string{ + args := []string{ "--authentication-kubeconfig", wardleToKASKubeConfigFile, "--authorization-kubeconfig", wardleToKASKubeConfigFile, "--etcd-servers", framework.GetEtcdURL(), "--cert-dir", wardleCertDir, "--kubeconfig", wardleToKASKubeConfigFile, - "--emulated-version", "wardle=1.1", - "--feature-gates", "wardle:BanFlunder=true", - }) + "--emulated-version", fmt.Sprintf("wardle=%s", emulationVersion), + } + if enableWardleFeatureGate { + args = append(args, "--feature-gates", "wardle:BanFlunder=true") + } + wardleCmd.SetArgs(args) if err := wardleCmd.Execute(); err != nil { t.Error(err) } @@ -393,6 +408,8 @@ func TestAggregatedAPIServer(t *testing.T) { if err != nil { t.Fatal(err) } + // clean up data after test is done + defer wardleClient.Fischers().Delete(ctx, "panda", metav1.DeleteOptions{}) fischersList, err := wardleClient.Fischers().List(ctx, metav1.ListOptions{}) if err != nil { t.Fatal(err) @@ -409,8 +426,16 @@ func TestAggregatedAPIServer(t *testing.T) { Name: "badname", }, }, metav1.CreateOptions{}) - if err == nil { - t.Fatal("expect flunder:badname not admitted") + banFlunder := enableWardleFeatureGate || emulationVersion == "1.2" + if banFlunder && err == nil { + t.Fatal("expect flunder:badname not admitted when wardle feature gates are specified") + } + if !banFlunder { + if err != nil { + t.Fatal("expect flunder:badname admitted when wardle feature gates are not specified") + } else { + defer wardleClient.Flunders(metav1.NamespaceSystem).Delete(ctx, "badname", metav1.DeleteOptions{}) + } } _, err = wardleClient.Flunders(metav1.NamespaceSystem).Create(ctx, &wardlev1alpha1.Flunder{ ObjectMeta: metav1.ObjectMeta{ @@ -420,12 +445,17 @@ func TestAggregatedAPIServer(t *testing.T) { if err != nil { t.Fatal(err) } + defer wardleClient.Flunders(metav1.NamespaceSystem).Delete(ctx, "panda", metav1.DeleteOptions{}) flunderList, err := wardleClient.Flunders(metav1.NamespaceSystem).List(ctx, metav1.ListOptions{}) if err != nil { t.Fatal(err) } - if len(flunderList.Items) != 1 { - t.Errorf("expected one flunder: %#v", flunderList.Items) + expectedFlunderCount := 2 + if banFlunder { + expectedFlunderCount = 1 + } + if len(flunderList.Items) != expectedFlunderCount { + t.Errorf("expected %d flunder: %#v", expectedFlunderCount, flunderList.Items) } if len(flunderList.ResourceVersion) == 0 { t.Error("expected non-empty resource version for flunder list") diff --git a/test/integration/servicecidr/allocator_test.go b/test/integration/servicecidr/allocator_test.go index ad6d6280dc2..dd4b45cc521 100644 --- a/test/integration/servicecidr/allocator_test.go +++ b/test/integration/servicecidr/allocator_test.go @@ -29,7 +29,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" - utilversion "k8s.io/apiserver/pkg/util/version" "k8s.io/client-go/kubernetes" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" @@ -123,7 +122,6 @@ func TestServiceAllocIPAddress(t *testing.T) { ModifyServerRunOptions: func(opts *options.ServerRunOptions) { opts.ServiceClusterIPRanges = serviceCIDR opts.GenericServerRunOptions.AdvertiseAddress = netutils.ParseIPSloppy("2001:db8::10") - opts.GenericServerRunOptions.EffectiveVersion = utilversion.NewEffectiveVersion("1.31") opts.APIEnablement.RuntimeConfig.Set("networking.k8s.io/v1alpha1=true") }, })