diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index f2086239d65..3e8a6b8c068 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -156,8 +156,9 @@ type Config struct { EffectiveVersion basecompatibility.EffectiveVersion // EmulationForwardCompatible is an option to implicitly enable all APIs which are introduced after the emulation version and // have higher priority than APIs of the same group resource enabled at the emulation version. - // If true, all APIs that have higher priority than the APIs of the same group resource enabled at the emulation version will be installed. + // If true, all APIs that have higher priority than the APIs(beta+) of the same group resource enabled at the emulation version will be installed. // This is needed when a controller implementation migrates to newer API versions, for the binary version, and also uses the newer API versions even when emulation version is set. + // Not applicable to alpha APIs. EmulationForwardCompatible bool // RuntimeConfigEmulationForwardCompatible is an option to explicitly enable specific APIs introduced after the emulation version through the runtime-config. // If true, APIs identified by group/version that are enabled in the --runtime-config flag will be installed even if it is introduced after the emulation version. --runtime-config flag values that identify multiple APIs, such as api/all,api/ga,api/beta, are not influenced by this flag and will only enable APIs available at the current emulation version. diff --git a/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go b/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go index dd2368f91c1..1b94b561356 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go +++ b/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go @@ -20,6 +20,7 @@ import ( "fmt" "os" "regexp" + "sort" "strconv" "strings" @@ -28,6 +29,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" apimachineryversion "k8s.io/apimachinery/pkg/util/version" + "k8s.io/apimachinery/pkg/version" "k8s.io/apiserver/pkg/registry/rest" serverstorage "k8s.io/apiserver/pkg/server/storage" "k8s.io/klog/v2" @@ -71,6 +73,7 @@ type ResourceExpirationEvaluatorOptions struct { Prerelease string // EmulationForwardCompatible indicates whether the apiserver should serve resources that are introduced after the current version, // when resources of the same group and resource name but with lower priority are served. + // Not applicable to alpha APIs. EmulationForwardCompatible bool // RuntimeConfigEmulationForwardCompatible indicates whether the apiserver should serve resources that are introduced after the current version, // when the resource is explicitly enabled in runtime-config. @@ -222,6 +225,9 @@ func (e *resourceExpirationEvaluator) RemoveUnavailableKinds(groupName string, v func (e *resourceExpirationEvaluator) removeUnintroducedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage, apiResourceConfigSource serverstorage.APIResourceConfigSource) error { versionsToRemove := sets.NewString() prioritizedVersions := versioner.PrioritizedVersionsForGroup(groupName) + sort.Slice(prioritizedVersions, func(i, j int) bool { + return version.CompareKubeAwareVersionStrings(prioritizedVersions[i].Version, prioritizedVersions[j].Version) > 0 + }) enabledResources := sets.NewString() // iterate from the end to the front, so that we remove the lower priority versions first. diff --git a/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds_test.go b/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds_test.go index 2923b5db658..20f38a98c95 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/deleted_kinds_test.go @@ -801,13 +801,272 @@ func Test_removeUnIntroducedKinds(t *testing.T) { }, }, }, + { + name: "both-runtime-config-and-emulation-forward-compatible-runtime-config-alpha-api", + resourceExpirationEvaluator: resourceExpirationEvaluator{ + currentVersion: apimachineryversion.MajorMinor(1, 19), + runtimeConfigEmulationForwardCompatible: true, + emulationForwardCompatible: true, + }, + runtimeConfig: map[string]string{ + groupName + "/v2alpha1": "true", + }, + versionedResourcesStorageMap: map[string]map[string]rest.Storage{ + "v1": { + resource1: storageIntroducedIn(1, 19), + }, + "v2alpha1": { + resource1: storageIntroducedAndRemovedIn(1, 20, 1, 21), + resource2: storageIntroducedAndRemovedIn(1, 20, 1, 21), + }, + "v2beta1": { + resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22), + }, + "v2beta2": { + resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23), + resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23), + }, + "v2": { + resource1: storageIntroducedIn(1, 23), + resource2: storageIntroducedIn(1, 23), + }, + }, + expectedStorage: map[string]map[string]rest.Storage{ + "v1": { + resource1: storageIntroducedIn(1, 19), + }, + "v2alpha1": { + resource1: storageIntroducedAndRemovedIn(1, 20, 1, 21), + resource2: storageIntroducedAndRemovedIn(1, 20, 1, 21), + }, + "v2": { + resource1: storageIntroducedIn(1, 23), + }, + }, + }, + { + name: "both-runtime-config-and-emulation-forward-compatible-runtime-config-alpha-resource", + resourceExpirationEvaluator: resourceExpirationEvaluator{ + currentVersion: apimachineryversion.MajorMinor(1, 19), + runtimeConfigEmulationForwardCompatible: true, + emulationForwardCompatible: true, + }, + runtimeConfig: map[string]string{ + groupName + "/v2alpha1/resource2": "true", + }, + versionedResourcesStorageMap: map[string]map[string]rest.Storage{ + "v1": { + resource1: storageIntroducedIn(1, 19), + }, + "v2alpha1": { + resource1: storageIntroducedAndRemovedIn(1, 20, 1, 21), + resource2: storageIntroducedAndRemovedIn(1, 20, 1, 21), + }, + "v2beta1": { + resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22), + }, + "v2beta2": { + resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23), + resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23), + }, + "v2": { + resource1: storageIntroducedIn(1, 23), + resource2: storageIntroducedIn(1, 23), + }, + }, + expectedStorage: map[string]map[string]rest.Storage{ + "v1": { + resource1: storageIntroducedIn(1, 19), + }, + "v2alpha1": { + resource2: storageIntroducedAndRemovedIn(1, 20, 1, 21), + }, + "v2": { + resource1: storageIntroducedIn(1, 23), + }, + }, + }, + { + name: "emulation-forward-compatible-beta1-api", + resourceExpirationEvaluator: resourceExpirationEvaluator{ + currentVersion: apimachineryversion.MajorMinor(1, 21), + emulationForwardCompatible: true, + }, + runtimeConfig: map[string]string{ + groupName + "/v2beta1": "true", + }, + versionedResourcesStorageMap: map[string]map[string]rest.Storage{ + "v1": { + resource1: storageIntroducedIn(1, 20), + }, + "v2beta1": { + resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22), + resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22), + }, + "v2beta2": { + resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23), + resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23), + }, + "v2": { + resource1: storageIntroducedIn(1, 23), + resource2: storageIntroducedIn(1, 23), + }, + }, + expectedStorage: map[string]map[string]rest.Storage{ + "v1": { + resource1: storageIntroducedIn(1, 20), + }, + "v2beta1": { + resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22), + resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22), + }, + "v2beta2": { + resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23), + resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23), + }, + "v2": { + resource1: storageIntroducedIn(1, 23), + resource2: storageIntroducedIn(1, 23), + }, + }, + }, + { + name: "emulation-forward-compatible-beta1-resource", + resourceExpirationEvaluator: resourceExpirationEvaluator{ + currentVersion: apimachineryversion.MajorMinor(1, 21), + emulationForwardCompatible: true, + }, + runtimeConfig: map[string]string{ + groupName + "/v2beta1/resource2": "true", + }, + versionedResourcesStorageMap: map[string]map[string]rest.Storage{ + "v1": { + resource1: storageIntroducedIn(1, 20), + }, + "v2beta1": { + resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22), + }, + "v2beta2": { + resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23), + resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23), + }, + "v2": { + resource1: storageIntroducedIn(1, 23), + resource2: storageIntroducedIn(1, 23), + }, + }, + expectedStorage: map[string]map[string]rest.Storage{ + "v1": { + resource1: storageIntroducedIn(1, 20), + }, + "v2beta1": { + resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22), + }, + "v2beta2": { + resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23), + }, + "v2": { + resource1: storageIntroducedIn(1, 23), + resource2: storageIntroducedIn(1, 23), + }, + }, + }, + { + name: "both-runtime-config-and-emulation-forward-compatible-runtime-config-beta1-api", + resourceExpirationEvaluator: resourceExpirationEvaluator{ + currentVersion: apimachineryversion.MajorMinor(1, 20), + emulationForwardCompatible: true, + runtimeConfigEmulationForwardCompatible: true, + }, + runtimeConfig: map[string]string{ + groupName + "/v2beta1": "true", + }, + versionedResourcesStorageMap: map[string]map[string]rest.Storage{ + "v1": { + resource1: storageIntroducedIn(1, 20), + }, + "v2beta1": { + resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22), + resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22), + }, + "v2beta2": { + resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23), + resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23), + }, + "v2": { + resource1: storageIntroducedIn(1, 23), + resource2: storageIntroducedIn(1, 23), + }, + }, + expectedStorage: map[string]map[string]rest.Storage{ + "v1": { + resource1: storageIntroducedIn(1, 20), + }, + "v2beta1": { + resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22), + resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22), + }, + "v2beta2": { + resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23), + resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23), + }, + "v2": { + resource1: storageIntroducedIn(1, 23), + resource2: storageIntroducedIn(1, 23), + }, + }, + }, + { + name: "both-runtime-config-and-emulation-forward-compatible-runtime-config-beta1-resource", + resourceExpirationEvaluator: resourceExpirationEvaluator{ + currentVersion: apimachineryversion.MajorMinor(1, 20), + emulationForwardCompatible: true, + runtimeConfigEmulationForwardCompatible: true, + }, + runtimeConfig: map[string]string{ + groupName + "/v2beta1/resource2": "true", + }, + versionedResourcesStorageMap: map[string]map[string]rest.Storage{ + "v1": { + resource1: storageIntroducedIn(1, 20), + }, + "v2beta1": { + resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22), + resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22), + }, + "v2beta2": { + resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23), + resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23), + }, + "v2": { + resource1: storageIntroducedIn(1, 23), + resource2: storageIntroducedIn(1, 23), + }, + }, + expectedStorage: map[string]map[string]rest.Storage{ + "v1": { + resource1: storageIntroducedIn(1, 20), + }, + "v2beta1": { + resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22), + }, + "v2beta2": { + resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23), + }, + "v2": { + resource1: storageIntroducedIn(1, 23), + resource2: storageIntroducedIn(1, 23), + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { resourceConfig := serverstorage.NewResourceConfig() convertor := &dummyConvertor{prioritizedVersions: []schema.GroupVersion{ {Group: groupName, Version: "v2"}, {Group: groupName, Version: "v1"}, - {Group: groupName, Version: "v2beta2"}, {Group: groupName, Version: "v2beta1"}, + {Group: groupName, Version: "v2beta1"}, + {Group: groupName, Version: "v2beta2"}, {Group: groupName, Version: "v2alpha1"}}} resourceConfig.EnableVersions(convertor.PrioritizedVersionsForGroup(groupName)...) resourceConfig, err := resourceconfig.MergeAPIResourceConfigs(resourceConfig, tt.runtimeConfig, convertor) diff --git a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go index 6060b8ca4c2..a0ff71b9b8e 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go +++ b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go @@ -247,8 +247,9 @@ type GenericAPIServer struct { EffectiveVersion basecompatibility.EffectiveVersion // EmulationForwardCompatible is an option to implicitly enable all APIs which are introduced after the emulation version and // have higher priority than APIs of the same group resource enabled at the emulation version. - // If true, all APIs that have higher priority than the APIs of the same group resource enabled at the emulation version will be installed. + // If true, all APIs that have higher priority than the APIs(beta+) of the same group resource enabled at the emulation version will be installed. // This is needed when a controller implementation migrates to newer API versions, for the binary version, and also uses the newer API versions even when emulation version is set. + // Not applicable to alpha APIs. EmulationForwardCompatible bool // RuntimeConfigEmulationForwardCompatible is an option to explicitly enable specific APIs introduced after the emulation version through the runtime-config. // If true, APIs identified by group/version that are enabled in the --runtime-config flag will be installed even if it is introduced after the emulation version. --runtime-config flag values that identify multiple APIs, such as api/all,api/ga,api/beta, are not influenced by this flag and will only enable APIs available at the current emulation version. diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go b/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go index a1bf4fa932e..d4032fd2135 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go @@ -100,8 +100,9 @@ type ServerRunOptions struct { ComponentName string // EmulationForwardCompatible is an option to implicitly enable all APIs which are introduced after the emulation version and // have higher priority than APIs of the same group resource enabled at the emulation version. - // If true, all APIs that have higher priority than the APIs of the same group resource enabled at the emulation version will be installed. + // If true, all APIs that have higher priority than the APIs(beta+) of the same group resource enabled at the emulation version will be installed. // This is needed when a controller implementation migrates to newer API versions, for the binary version, and also uses the newer API versions even when emulation version is set. + // Not applicable to alpha APIs. EmulationForwardCompatible bool // RuntimeConfigEmulationForwardCompatible is an option to explicitly enable specific APIs introduced after the emulation version through the runtime-config. // If true, APIs identified by group/version that are enabled in the --runtime-config flag will be installed even if it is introduced after the emulation version. --runtime-config flag values that identify multiple APIs, such as api/all,api/ga,api/beta, are not influenced by this flag and will only enable APIs available at the current emulation version. @@ -399,10 +400,11 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) { s.ComponentGlobalsRegistry.AddFlags(fs) fs.BoolVar(&s.EmulationForwardCompatible, "emulation-forward-compatible", s.EmulationForwardCompatible, ""+ - "If true all APIs that have higher priority than the APIs enabled at the emulation version of the same group resource will be installed. "+ + "If true, for any beta+ APIs enabled by default or by --runtime-config at the emulation version, their future versions with higher priority/stability will be auto enabled even if they introduced after the emulation version. "+ "Can only be set to true if the emulation version is lower than the binary version.") fs.BoolVar(&s.RuntimeConfigEmulationForwardCompatible, "runtime-config-emulation-forward-compatible", s.RuntimeConfigEmulationForwardCompatible, ""+ "If true, APIs identified by group/version that are enabled in the --runtime-config flag will be installed even if it is introduced after the emulation version. "+ + "If false, server would fail to start if any APIs identified by group/version that are enabled in the --runtime-config flag are introduced after the emulation version. "+ "Can only be set to true if the emulation version is lower than the binary version.") } diff --git a/staging/src/k8s.io/apiserver/pkg/server/resourceconfig/helpers.go b/staging/src/k8s.io/apiserver/pkg/server/resourceconfig/helpers.go index b67ec20e711..c546112ec9c 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/resourceconfig/helpers.go +++ b/staging/src/k8s.io/apiserver/pkg/server/resourceconfig/helpers.go @@ -19,11 +19,13 @@ package resourceconfig import ( "fmt" "regexp" + "sort" "strconv" "strings" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/version" serverstore "k8s.io/apiserver/pkg/server/storage" cliflag "k8s.io/component-base/cli/flag" ) @@ -258,7 +260,11 @@ func EmulationForwardCompatibleResourceConfig( continue } // if a gv is enabled, all the versions with higher priority (all the versions before gv in PrioritizedVersionsForGroup) are also implicitly enabled for emulation forward compatibility. - for _, pgv := range registry.PrioritizedVersionsForGroup(gv.Group) { + prioritizedVersions := registry.PrioritizedVersionsForGroup(gv.Group) + sort.Slice(prioritizedVersions, func(i, j int) bool { + return version.CompareKubeAwareVersionStrings(prioritizedVersions[i].Version, prioritizedVersions[j].Version) > 0 + }) + for _, pgv := range prioritizedVersions { if pgv.Version == gv.Version { break } @@ -275,7 +281,11 @@ func EmulationForwardCompatibleResourceConfig( continue } // if a gvr is enabled, all the versions with the same resource name and higher priority (all the versions before gv in PrioritizedVersionsForGroup) are also implicitly enabled for emulation forward compatibility. - for _, pgv := range registry.PrioritizedVersionsForGroup(gvr.Group) { + prioritizedVersions := registry.PrioritizedVersionsForGroup(gvr.Group) + sort.Slice(prioritizedVersions, func(i, j int) bool { + return version.CompareKubeAwareVersionStrings(prioritizedVersions[i].Version, prioritizedVersions[j].Version) > 0 + }) + for _, pgv := range prioritizedVersions { if pgv.Version == gvr.Version { break } diff --git a/staging/src/k8s.io/apiserver/pkg/server/resourceconfig/helpers_test.go b/staging/src/k8s.io/apiserver/pkg/server/resourceconfig/helpers_test.go index ec43e84d8d6..549b2696290 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/resourceconfig/helpers_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/resourceconfig/helpers_test.go @@ -802,5 +802,5 @@ func addTestGVs(t *testing.T, s *runtime.Scheme) { s.AddKnownTypes(v2, &runtimetesting.TestType1{}, &runtimetesting.TestType2{}) require.NoError(t, runtimetesting.RegisterConversions(s)) - require.NoError(t, s.SetVersionPriority(v2, v1, v2beta2, v2beta1)) + require.NoError(t, s.SetVersionPriority(v2, v1, v2beta1, v2beta2)) } diff --git a/test/integration/etcd/data.go b/test/integration/etcd/data.go index b9c57a5d9b4..75f06e47913 100644 --- a/test/integration/etcd/data.go +++ b/test/integration/etcd/data.go @@ -294,18 +294,16 @@ func GetEtcdStorageDataForNamespaceServedAt(namespace string, v string, removeAl IntroducedVersion: "1.7", }, gvr("networking.k8s.io", "v1", "ipaddresses"): { - Stub: `{"metadata": {"name": "192.168.2.3"}, "spec": {"parentRef": {"resource": "services","name": "test", "namespace": "ns"}}}`, - ExpectedEtcdPath: "/registry/ipaddresses/192.168.2.3", - ExpectedGVK: gvkP("networking.k8s.io", "v1beta1", "IPAddress"), - IntroducedVersion: "1.33", - EmulationForwardCompatibleSinceVersion: "1.31", + Stub: `{"metadata": {"name": "192.168.2.3"}, "spec": {"parentRef": {"resource": "services","name": "test", "namespace": "ns"}}}`, + ExpectedEtcdPath: "/registry/ipaddresses/192.168.2.3", + ExpectedGVK: gvkP("networking.k8s.io", "v1beta1", "IPAddress"), + IntroducedVersion: "1.33", }, gvr("networking.k8s.io", "v1", "servicecidrs"): { - Stub: `{"metadata": {"name": "range-b2"}, "spec": {"cidrs": ["192.168.0.0/16","fd00:1::/120"]}}`, - ExpectedEtcdPath: "/registry/servicecidrs/range-b2", - ExpectedGVK: gvkP("networking.k8s.io", "v1beta1", "ServiceCIDR"), - IntroducedVersion: "1.33", - EmulationForwardCompatibleSinceVersion: "1.31", + Stub: `{"metadata": {"name": "range-b2"}, "spec": {"cidrs": ["192.168.0.0/16","fd00:1::/120"]}}`, + ExpectedEtcdPath: "/registry/servicecidrs/range-b2", + ExpectedGVK: gvkP("networking.k8s.io", "v1beta1", "ServiceCIDR"), + IntroducedVersion: "1.33", }, // -- @@ -680,6 +678,22 @@ func GetEtcdStorageDataForNamespaceServedAt(namespace string, v string, removeAl // -- } + // get the min version the Beta api of a group is introduced, and it would be used to determine emulation forward compatibility. + minBetaVersions := map[schema.GroupResource]*version.Version{} + for key, data := range etcdStorageData { + if !strings.Contains(key.Version, "beta") { + continue + } + introduced := version.MustParse(data.IntroducedVersion) + if ver, ok := minBetaVersions[key.GroupResource()]; ok { + if introduced.LessThan(ver) { + minBetaVersions[key.GroupResource()] = introduced + } + } else { + minBetaVersions[key.GroupResource()] = introduced + } + } + // Delete types no longer served or not yet added at a particular emulated version. for key, data := range etcdStorageData { if data.RemovedVersion != "" && version.MustParse(v).AtLeast(version.MustParse(data.RemovedVersion)) { @@ -688,7 +702,8 @@ func GetEtcdStorageDataForNamespaceServedAt(namespace string, v string, removeAl if data.IntroducedVersion == "" || version.MustParse(v).AtLeast(version.MustParse(data.IntroducedVersion)) { continue } - if data.EmulationForwardCompatibleSinceVersion != "" && version.MustParse(v).AtLeast(version.MustParse(data.EmulationForwardCompatibleSinceVersion)) { + minBetaVersion, ok := minBetaVersions[key.GroupResource()] + if ok && version.MustParse(v).AtLeast(minBetaVersion) { continue } delete(etcdStorageData, key) @@ -735,10 +750,6 @@ type StorageData struct { ExpectedGVK *schema.GroupVersionKind // The GVK that we expect this object to be stored as - leave this nil to use the default IntroducedVersion string // The version that this type is introduced RemovedVersion string // The version that this type is removed. May be empty for stable resources - // EmulationForwardCompatibleSinceVersion indicates the api should be kept if the emulation version is >= this version, even when the api is introduced after the emulation version. - // Only needed for some Beta (Beta2+) and GA APIs, and is the version when the lowest Beta api is introduced. - // This is needed to enable some Beta features where the api used in the corresponding controller has GAed. - EmulationForwardCompatibleSinceVersion string } // Prerequisite contains information required to create a resource (but not verify it)