github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-23 19:56:01 +00:00
Code Issues Projects Releases Wiki Activity

bug fix: fix version order in emulation forward compatibility.

Browse Source 
Signed-off-by: Siyuan Zhang <sizhang@google.com>
This commit is contained in:
Siyuan Zhang 2025-04-07 18:38:43 -07:00
parent 195803cde5
commit b1a9cc3473
8 changed files with 313 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
staging/src/k8s.io/apiserver/pkg/server/config.go
View File

@ -156,8 +156,9 @@ type Config struct {
EffectiveVersion basecompatibility.EffectiveVersion
// EmulationForwardCompatible is an option to implicitly enable all APIs which are introduced after the emulation version and
// have higher priority than APIs of the same group resource enabled at the emulation version.
// If true, all APIs that have higher priority than the APIs of the same group resource enabled at the emulation version will be installed.
// If true, all APIs that have higher priority than the APIs(beta+) of the same group resource enabled at the emulation version will be installed.
// This is needed when a controller implementation migrates to newer API versions, for the binary version, and also uses the newer API versions even when emulation version is set.
// Not applicable to alpha APIs.
EmulationForwardCompatible bool
// RuntimeConfigEmulationForwardCompatible is an option to explicitly enable specific APIs introduced after the emulation version through the runtime-config.
// If true, APIs identified by group/version that are enabled in the --runtime-config flag will be installed even if it is introduced after the emulation version. --runtime-config flag values that identify multiple APIs, such as api/all,api/ga,api/beta, are not influenced by this flag and will only enable APIs available at the current emulation version.

6
staging/src/k8s.io/apiserver/pkg/server/deleted_kinds.go
View File

@ -20,6 +20,7 @@ import (
"fmt"
"os"
"regexp"
"sort"
"strconv"
"strings"
@ -28,6 +29,7 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
apimachineryversion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/apimachinery/pkg/version"
"k8s.io/apiserver/pkg/registry/rest"
serverstorage "k8s.io/apiserver/pkg/server/storage"
"k8s.io/klog/v2"
@ -71,6 +73,7 @@ type ResourceExpirationEvaluatorOptions struct {
Prerelease string
// EmulationForwardCompatible indicates whether the apiserver should serve resources that are introduced after the current version,
// when resources of the same group and resource name but with lower priority are served.
// Not applicable to alpha APIs.
EmulationForwardCompatible bool
// RuntimeConfigEmulationForwardCompatible indicates whether the apiserver should serve resources that are introduced after the current version,
// when the resource is explicitly enabled in runtime-config.
@ -222,6 +225,9 @@ func (e *resourceExpirationEvaluator) RemoveUnavailableKinds(groupName string, v
func (e *resourceExpirationEvaluator) removeUnintroducedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage, apiResourceConfigSource serverstorage.APIResourceConfigSource) error {
versionsToRemove := sets.NewString()
prioritizedVersions := versioner.PrioritizedVersionsForGroup(groupName)
sort.Slice(prioritizedVersions, func(i, j int) bool {
return version.CompareKubeAwareVersionStrings(prioritizedVersions[i].Version, prioritizedVersions[j].Version) > 0
})
enabledResources := sets.NewString()
// iterate from the end to the front, so that we remove the lower priority versions first.

261
staging/src/k8s.io/apiserver/pkg/server/deleted_kinds_test.go
View File

@ -801,13 +801,272 @@ func Test_removeUnIntroducedKinds(t *testing.T) {
},
},
},
{
name: "both-runtime-config-and-emulation-forward-compatible-runtime-config-alpha-api",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 19),
runtimeConfigEmulationForwardCompatible: true,
emulationForwardCompatible: true,
},
runtimeConfig: map[string]string{
groupName + "/v2alpha1": "true",
},
versionedResourcesStorageMap: map[string]map[string]rest.Storage{
"v1": {
resource1: storageIntroducedIn(1, 19),
},
"v2alpha1": {
resource1: storageIntroducedAndRemovedIn(1, 20, 1, 21),
resource2: storageIntroducedAndRemovedIn(1, 20, 1, 21),
},
"v2beta1": {
resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22),
},
"v2beta2": {
resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23),
resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
},
"v2": {
resource1: storageIntroducedIn(1, 23),
resource2: storageIntroducedIn(1, 23),
},
},
expectedStorage: map[string]map[string]rest.Storage{
"v1": {
resource1: storageIntroducedIn(1, 19),
},
"v2alpha1": {
resource1: storageIntroducedAndRemovedIn(1, 20, 1, 21),
resource2: storageIntroducedAndRemovedIn(1, 20, 1, 21),
},
"v2": {
resource1: storageIntroducedIn(1, 23),
},
},
},
{
name: "both-runtime-config-and-emulation-forward-compatible-runtime-config-alpha-resource",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 19),
runtimeConfigEmulationForwardCompatible: true,
emulationForwardCompatible: true,
},
runtimeConfig: map[string]string{
groupName + "/v2alpha1/resource2": "true",
},
versionedResourcesStorageMap: map[string]map[string]rest.Storage{
"v1": {
resource1: storageIntroducedIn(1, 19),
},
"v2alpha1": {
resource1: storageIntroducedAndRemovedIn(1, 20, 1, 21),
resource2: storageIntroducedAndRemovedIn(1, 20, 1, 21),
},
"v2beta1": {
resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22),
},
"v2beta2": {
resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23),
resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
},
"v2": {
resource1: storageIntroducedIn(1, 23),
resource2: storageIntroducedIn(1, 23),
},
},
expectedStorage: map[string]map[string]rest.Storage{
"v1": {
resource1: storageIntroducedIn(1, 19),
},
"v2alpha1": {
resource2: storageIntroducedAndRemovedIn(1, 20, 1, 21),
},
"v2": {
resource1: storageIntroducedIn(1, 23),
},
},
},
{
name: "emulation-forward-compatible-beta1-api",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 21),
emulationForwardCompatible: true,
},
runtimeConfig: map[string]string{
groupName + "/v2beta1": "true",
},
versionedResourcesStorageMap: map[string]map[string]rest.Storage{
"v1": {
resource1: storageIntroducedIn(1, 20),
},
"v2beta1": {
resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22),
resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22),
},
"v2beta2": {
resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23),
resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
},
"v2": {
resource1: storageIntroducedIn(1, 23),
resource2: storageIntroducedIn(1, 23),
},
},
expectedStorage: map[string]map[string]rest.Storage{
"v1": {
resource1: storageIntroducedIn(1, 20),
},
"v2beta1": {
resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22),
resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22),
},
"v2beta2": {
resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23),
resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
},
"v2": {
resource1: storageIntroducedIn(1, 23),
resource2: storageIntroducedIn(1, 23),
},
},
},
{
name: "emulation-forward-compatible-beta1-resource",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 21),
emulationForwardCompatible: true,
},
runtimeConfig: map[string]string{
groupName + "/v2beta1/resource2": "true",
},
versionedResourcesStorageMap: map[string]map[string]rest.Storage{
"v1": {
resource1: storageIntroducedIn(1, 20),
},
"v2beta1": {
resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22),
},
"v2beta2": {
resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23),
resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
},
"v2": {
resource1: storageIntroducedIn(1, 23),
resource2: storageIntroducedIn(1, 23),
},
},
expectedStorage: map[string]map[string]rest.Storage{
"v1": {
resource1: storageIntroducedIn(1, 20),
},
"v2beta1": {
resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22),
},
"v2beta2": {
resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
},
"v2": {
resource1: storageIntroducedIn(1, 23),
resource2: storageIntroducedIn(1, 23),
},
},
},
{
name: "both-runtime-config-and-emulation-forward-compatible-runtime-config-beta1-api",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 20),
emulationForwardCompatible: true,
runtimeConfigEmulationForwardCompatible: true,
},
runtimeConfig: map[string]string{
groupName + "/v2beta1": "true",
},
versionedResourcesStorageMap: map[string]map[string]rest.Storage{
"v1": {
resource1: storageIntroducedIn(1, 20),
},
"v2beta1": {
resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22),
resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22),
},
"v2beta2": {
resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23),
resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
},
"v2": {
resource1: storageIntroducedIn(1, 23),
resource2: storageIntroducedIn(1, 23),
},
},
expectedStorage: map[string]map[string]rest.Storage{
"v1": {
resource1: storageIntroducedIn(1, 20),
},
"v2beta1": {
resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22),
resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22),
},
"v2beta2": {
resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23),
resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
},
"v2": {
resource1: storageIntroducedIn(1, 23),
resource2: storageIntroducedIn(1, 23),
},
},
},
{
name: "both-runtime-config-and-emulation-forward-compatible-runtime-config-beta1-resource",
resourceExpirationEvaluator: resourceExpirationEvaluator{
currentVersion: apimachineryversion.MajorMinor(1, 20),
emulationForwardCompatible: true,
runtimeConfigEmulationForwardCompatible: true,
},
runtimeConfig: map[string]string{
groupName + "/v2beta1/resource2": "true",
},
versionedResourcesStorageMap: map[string]map[string]rest.Storage{
"v1": {
resource1: storageIntroducedIn(1, 20),
},
"v2beta1": {
resource1: storageIntroducedAndRemovedIn(1, 21, 1, 22),
resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22),
},
"v2beta2": {
resource1: storageIntroducedAndRemovedIn(1, 22, 1, 23),
resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
},
"v2": {
resource1: storageIntroducedIn(1, 23),
resource2: storageIntroducedIn(1, 23),
},
},
expectedStorage: map[string]map[string]rest.Storage{
"v1": {
resource1: storageIntroducedIn(1, 20),
},
"v2beta1": {
resource2: storageIntroducedAndRemovedIn(1, 21, 1, 22),
},
"v2beta2": {
resource2: storageIntroducedAndRemovedIn(1, 22, 1, 23),
},
"v2": {
resource1: storageIntroducedIn(1, 23),
resource2: storageIntroducedIn(1, 23),
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
resourceConfig := serverstorage.NewResourceConfig()
convertor := &dummyConvertor{prioritizedVersions: []schema.GroupVersion{
{Group: groupName, Version: "v2"}, {Group: groupName, Version: "v1"},
{Group: groupName, Version: "v2beta2"}, {Group: groupName, Version: "v2beta1"},
{Group: groupName, Version: "v2beta1"},
{Group: groupName, Version: "v2beta2"},
{Group: groupName, Version: "v2alpha1"}}}
resourceConfig.EnableVersions(convertor.PrioritizedVersionsForGroup(groupName)...)
resourceConfig, err := resourceconfig.MergeAPIResourceConfigs(resourceConfig, tt.runtimeConfig, convertor)

3
staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go
View File

@ -247,8 +247,9 @@ type GenericAPIServer struct {
EffectiveVersion basecompatibility.EffectiveVersion
// EmulationForwardCompatible is an option to implicitly enable all APIs which are introduced after the emulation version and
// have higher priority than APIs of the same group resource enabled at the emulation version.
// If true, all APIs that have higher priority than the APIs of the same group resource enabled at the emulation version will be installed.
// If true, all APIs that have higher priority than the APIs(beta+) of the same group resource enabled at the emulation version will be installed.
// This is needed when a controller implementation migrates to newer API versions, for the binary version, and also uses the newer API versions even when emulation version is set.
// Not applicable to alpha APIs.
EmulationForwardCompatible bool
// RuntimeConfigEmulationForwardCompatible is an option to explicitly enable specific APIs introduced after the emulation version through the runtime-config.
// If true, APIs identified by group/version that are enabled in the --runtime-config flag will be installed even if it is introduced after the emulation version. --runtime-config flag values that identify multiple APIs, such as api/all,api/ga,api/beta, are not influenced by this flag and will only enable APIs available at the current emulation version.

6
staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go
View File

@ -100,8 +100,9 @@ type ServerRunOptions struct {
ComponentName string
// EmulationForwardCompatible is an option to implicitly enable all APIs which are introduced after the emulation version and
// have higher priority than APIs of the same group resource enabled at the emulation version.
// If true, all APIs that have higher priority than the APIs of the same group resource enabled at the emulation version will be installed.
// If true, all APIs that have higher priority than the APIs(beta+) of the same group resource enabled at the emulation version will be installed.
// This is needed when a controller implementation migrates to newer API versions, for the binary version, and also uses the newer API versions even when emulation version is set.
// Not applicable to alpha APIs.
EmulationForwardCompatible bool
// RuntimeConfigEmulationForwardCompatible is an option to explicitly enable specific APIs introduced after the emulation version through the runtime-config.
// If true, APIs identified by group/version that are enabled in the --runtime-config flag will be installed even if it is introduced after the emulation version. --runtime-config flag values that identify multiple APIs, such as api/all,api/ga,api/beta, are not influenced by this flag and will only enable APIs available at the current emulation version.
@ -399,10 +400,11 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
s.ComponentGlobalsRegistry.AddFlags(fs)
fs.BoolVar(&s.EmulationForwardCompatible, "emulation-forward-compatible", s.EmulationForwardCompatible, ""+
"If true all APIs that have higher priority than the APIs enabled at the emulation version of the same group resource will be installed. "+
"If true, for any beta+ APIs enabled by default or by --runtime-config at the emulation version, their future versions with higher priority/stability will be auto enabled even if they introduced after the emulation version. "+
"Can only be set to true if the emulation version is lower than the binary version.")
fs.BoolVar(&s.RuntimeConfigEmulationForwardCompatible, "runtime-config-emulation-forward-compatible", s.RuntimeConfigEmulationForwardCompatible, ""+
"If true, APIs identified by group/version that are enabled in the --runtime-config flag will be installed even if it is introduced after the emulation version. "+
"If false, server would fail to start if any APIs identified by group/version that are enabled in the --runtime-config flag are introduced after the emulation version. "+
"Can only be set to true if the emulation version is lower than the binary version.")
}

14
staging/src/k8s.io/apiserver/pkg/server/resourceconfig/helpers.go
View File

@ -19,11 +19,13 @@ package resourceconfig
import (
"fmt"
"regexp"
"sort"
"strconv"
"strings"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/version"
serverstore "k8s.io/apiserver/pkg/server/storage"
cliflag "k8s.io/component-base/cli/flag"
)
@ -258,7 +260,11 @@ func EmulationForwardCompatibleResourceConfig(
continue
}
// if a gv is enabled, all the versions with higher priority (all the versions before gv in PrioritizedVersionsForGroup) are also implicitly enabled for emulation forward compatibility.
for _, pgv := range registry.PrioritizedVersionsForGroup(gv.Group) {
prioritizedVersions := registry.PrioritizedVersionsForGroup(gv.Group)
sort.Slice(prioritizedVersions, func(i, j int) bool {
return version.CompareKubeAwareVersionStrings(prioritizedVersions[i].Version, prioritizedVersions[j].Version) > 0
})
for _, pgv := range prioritizedVersions {
if pgv.Version == gv.Version {
break
}
@ -275,7 +281,11 @@ func EmulationForwardCompatibleResourceConfig(
continue
}
// if a gvr is enabled, all the versions with the same resource name and higher priority (all the versions before gv in PrioritizedVersionsForGroup) are also implicitly enabled for emulation forward compatibility.
for _, pgv := range registry.PrioritizedVersionsForGroup(gvr.Group) {
prioritizedVersions := registry.PrioritizedVersionsForGroup(gvr.Group)
sort.Slice(prioritizedVersions, func(i, j int) bool {
return version.CompareKubeAwareVersionStrings(prioritizedVersions[i].Version, prioritizedVersions[j].Version) > 0
})
for _, pgv := range prioritizedVersions {
if pgv.Version == gvr.Version {
break
}

2
staging/src/k8s.io/apiserver/pkg/server/resourceconfig/helpers_test.go
View File

@ -802,5 +802,5 @@ func addTestGVs(t *testing.T, s *runtime.Scheme) {
s.AddKnownTypes(v2, &runtimetesting.TestType1{}, &runtimetesting.TestType2{})
require.NoError(t, runtimetesting.RegisterConversions(s))
require.NoError(t, s.SetVersionPriority(v2, v1, v2beta2, v2beta1))
require.NoError(t, s.SetVersionPriority(v2, v1, v2beta1, v2beta2))
}

41
test/integration/etcd/data.go
View File

@ -294,18 +294,16 @@ func GetEtcdStorageDataForNamespaceServedAt(namespace string, v string, removeAl
IntroducedVersion: "1.7",
},
gvr("networking.k8s.io", "v1", "ipaddresses"): {
Stub: `{"metadata": {"name": "192.168.2.3"}, "spec": {"parentRef": {"resource": "services","name": "test", "namespace": "ns"}}}`,
ExpectedEtcdPath: "/registry/ipaddresses/192.168.2.3",
ExpectedGVK: gvkP("networking.k8s.io", "v1beta1", "IPAddress"),
IntroducedVersion: "1.33",
EmulationForwardCompatibleSinceVersion: "1.31",
Stub: `{"metadata": {"name": "192.168.2.3"}, "spec": {"parentRef": {"resource": "services","name": "test", "namespace": "ns"}}}`,
ExpectedEtcdPath: "/registry/ipaddresses/192.168.2.3",
ExpectedGVK: gvkP("networking.k8s.io", "v1beta1", "IPAddress"),
IntroducedVersion: "1.33",
},
gvr("networking.k8s.io", "v1", "servicecidrs"): {
Stub: `{"metadata": {"name": "range-b2"}, "spec": {"cidrs": ["192.168.0.0/16","fd00:1::/120"]}}`,
ExpectedEtcdPath: "/registry/servicecidrs/range-b2",
ExpectedGVK: gvkP("networking.k8s.io", "v1beta1", "ServiceCIDR"),
IntroducedVersion: "1.33",
EmulationForwardCompatibleSinceVersion: "1.31",
Stub: `{"metadata": {"name": "range-b2"}, "spec": {"cidrs": ["192.168.0.0/16","fd00:1::/120"]}}`,
ExpectedEtcdPath: "/registry/servicecidrs/range-b2",
ExpectedGVK: gvkP("networking.k8s.io", "v1beta1", "ServiceCIDR"),
IntroducedVersion: "1.33",
},
// --
@ -680,6 +678,22 @@ func GetEtcdStorageDataForNamespaceServedAt(namespace string, v string, removeAl
// --
}
// get the min version the Beta api of a group is introduced, and it would be used to determine emulation forward compatibility.
minBetaVersions := map[schema.GroupResource]*version.Version{}
for key, data := range etcdStorageData {
if !strings.Contains(key.Version, "beta") {
continue
}
introduced := version.MustParse(data.IntroducedVersion)
if ver, ok := minBetaVersions[key.GroupResource()]; ok {
if introduced.LessThan(ver) {
minBetaVersions[key.GroupResource()] = introduced
}
} else {
minBetaVersions[key.GroupResource()] = introduced
}
}
// Delete types no longer served or not yet added at a particular emulated version.
for key, data := range etcdStorageData {
if data.RemovedVersion != "" && version.MustParse(v).AtLeast(version.MustParse(data.RemovedVersion)) {
@ -688,7 +702,8 @@ func GetEtcdStorageDataForNamespaceServedAt(namespace string, v string, removeAl
if data.IntroducedVersion == "" || version.MustParse(v).AtLeast(version.MustParse(data.IntroducedVersion)) {
continue
}
if data.EmulationForwardCompatibleSinceVersion != "" && version.MustParse(v).AtLeast(version.MustParse(data.EmulationForwardCompatibleSinceVersion)) {
minBetaVersion, ok := minBetaVersions[key.GroupResource()]
if ok && version.MustParse(v).AtLeast(minBetaVersion) {
continue
}
delete(etcdStorageData, key)
@ -735,10 +750,6 @@ type StorageData struct {
ExpectedGVK *schema.GroupVersionKind // The GVK that we expect this object to be stored as - leave this nil to use the default
IntroducedVersion string // The version that this type is introduced
RemovedVersion string // The version that this type is removed. May be empty for stable resources
// EmulationForwardCompatibleSinceVersion indicates the api should be kept if the emulation version is >= this version, even when the api is introduced after the emulation version.
// Only needed for some Beta (Beta2+) and GA APIs, and is the version when the lowest Beta api is introduced.
// This is needed to enable some Beta features where the api used in the corresponding controller has GAed.
EmulationForwardCompatibleSinceVersion string
}
// Prerequisite contains information required to create a resource (but not verify it)