From b47e80a319bcc002d00e78080cbbfa85e5889de7 Mon Sep 17 00:00:00 2001 From: Aohan Yang Date: Fri, 13 Sep 2024 22:49:40 +0800 Subject: [PATCH 1/2] Promote LoadBalancerIPMode to GA --- pkg/features/kube_features.go | 1 + pkg/features/versioned_kube_features.go | 1 + .../featuregates_linter/test_data/versioned_feature_list.yaml | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index a024ec5c835..f3d90e79493 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -895,6 +895,7 @@ const ( // kep: http://kep.k8s.io/1860 // alpha: v1.29 // beta: v1.30 + // GA: v1.32 // LoadBalancerIPMode enables the IPMode field in the LoadBalancerIngress status of a Service LoadBalancerIPMode featuregate.Feature = "LoadBalancerIPMode" diff --git a/pkg/features/versioned_kube_features.go b/pkg/features/versioned_kube_features.go index ec491e17279..d544a7ea9ce 100644 --- a/pkg/features/versioned_kube_features.go +++ b/pkg/features/versioned_kube_features.go @@ -447,6 +447,7 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate LoadBalancerIPMode: { {Version: version.MustParse("1.29"), Default: false, PreRelease: featuregate.Alpha}, {Version: version.MustParse("1.30"), Default: true, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.32"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, }, LocalStorageCapacityIsolationFSQuotaMonitoring: { diff --git a/test/featuregates_linter/test_data/versioned_feature_list.yaml b/test/featuregates_linter/test_data/versioned_feature_list.yaml index ba9819c01f1..3d21e927931 100644 --- a/test/featuregates_linter/test_data/versioned_feature_list.yaml +++ b/test/featuregates_linter/test_data/versioned_feature_list.yaml @@ -616,6 +616,10 @@ lockToDefault: false preRelease: Beta version: "1.30" + - default: true + lockToDefault: true + preRelease: GA + version: "1.32" - name: LocalStorageCapacityIsolationFSQuotaMonitoring versionedSpecs: - default: false From da5738d9aa9ab8e9616eae8fccd1994e14cf8b05 Mon Sep 17 00:00:00 2001 From: Aohan Yang Date: Tue, 8 Oct 2024 23:44:24 +0800 Subject: [PATCH 2/2] Set feature gate emulation version during test --- pkg/apis/core/v1/defaults_test.go | 4 ++++ pkg/apis/core/validation/validation_test.go | 4 ++++ pkg/proxy/iptables/proxier_test.go | 4 ++++ pkg/proxy/ipvs/proxier_test.go | 4 ++++ pkg/proxy/nftables/proxier_test.go | 4 ++++ pkg/proxy/servicechangetracker_test.go | 4 ++++ pkg/registry/core/service/storage/storage_test.go | 5 +++++ pkg/registry/core/service/strategy_test.go | 4 ++++ test/integration/service/loadbalancer_test.go | 8 +++++++- 9 files changed, 40 insertions(+), 1 deletion(-) diff --git a/pkg/apis/core/v1/defaults_test.go b/pkg/apis/core/v1/defaults_test.go index 8ad20568720..e43e34dd5e3 100644 --- a/pkg/apis/core/v1/defaults_test.go +++ b/pkg/apis/core/v1/defaults_test.go @@ -29,6 +29,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/version" utilfeature "k8s.io/apiserver/pkg/util/feature" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/api/legacyscheme" @@ -1260,6 +1261,9 @@ func TestSetDefaultServiceLoadbalancerIPMode(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + if !tc.ipModeEnabled { + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31")) + } featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, tc.ipModeEnabled) obj := roundTrip(t, runtime.Object(tc.svc)) svc := obj.(*v1.Service) diff --git a/pkg/apis/core/validation/validation_test.go b/pkg/apis/core/validation/validation_test.go index 3bce4261ac6..e7eb7804d96 100644 --- a/pkg/apis/core/validation/validation_test.go +++ b/pkg/apis/core/validation/validation_test.go @@ -38,6 +38,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/apimachinery/pkg/util/version" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/component-base/featuregate" featuregatetesting "k8s.io/component-base/featuregate/testing" @@ -24119,6 +24120,9 @@ func TestValidateLoadBalancerStatus(t *testing.T) { } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + if !tc.ipModeEnabled { + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31")) + } featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, tc.ipModeEnabled) status := core.LoadBalancerStatus{} tc.tweakLBStatus(&status) diff --git a/pkg/proxy/iptables/proxier_test.go b/pkg/proxy/iptables/proxier_test.go index 3f67fa66930..a9d3a9e2269 100644 --- a/pkg/proxy/iptables/proxier_test.go +++ b/pkg/proxy/iptables/proxier_test.go @@ -41,6 +41,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/version" utilfeature "k8s.io/apiserver/pkg/util/feature" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/component-base/metrics/legacyregistry" @@ -6726,6 +6727,9 @@ func TestLoadBalancerIngressRouteTypeProxy(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { + if !testCase.ipModeEnabled { + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31")) + } featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, testCase.ipModeEnabled) ipt := iptablestest.NewFake() fp := NewFakeProxier(ipt) diff --git a/pkg/proxy/ipvs/proxier_test.go b/pkg/proxy/ipvs/proxier_test.go index 5091868ba32..cf6eb6451e7 100644 --- a/pkg/proxy/ipvs/proxier_test.go +++ b/pkg/proxy/ipvs/proxier_test.go @@ -37,6 +37,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/version" utilfeature "k8s.io/apiserver/pkg/util/feature" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/component-base/metrics/testutil" @@ -5885,6 +5886,9 @@ func TestLoadBalancerIngressRouteTypeProxy(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { + if !testCase.ipModeEnabled { + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31")) + } featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, testCase.ipModeEnabled) _, fp := buildFakeProxier(t) makeServiceMap(fp, diff --git a/pkg/proxy/nftables/proxier_test.go b/pkg/proxy/nftables/proxier_test.go index bc2b7ded68a..379070e871e 100644 --- a/pkg/proxy/nftables/proxier_test.go +++ b/pkg/proxy/nftables/proxier_test.go @@ -35,6 +35,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/version" utilfeature "k8s.io/apiserver/pkg/util/feature" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/component-base/metrics/testutil" @@ -4677,6 +4678,9 @@ func TestLoadBalancerIngressRouteTypeProxy(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { + if !testCase.ipModeEnabled { + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31")) + } featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, testCase.ipModeEnabled) nft, fp := NewFakeProxier(v1.IPv4Protocol) makeServiceMap(fp, diff --git a/pkg/proxy/servicechangetracker_test.go b/pkg/proxy/servicechangetracker_test.go index 1ce32bbfd26..a989d097da5 100644 --- a/pkg/proxy/servicechangetracker_test.go +++ b/pkg/proxy/servicechangetracker_test.go @@ -27,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/dump" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/version" utilfeature "k8s.io/apiserver/pkg/util/feature" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/features" @@ -572,6 +573,9 @@ func TestServiceToServiceMap(t *testing.T) { for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { + if !tc.ipModeEnabled { + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31")) + } featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, tc.ipModeEnabled) svcTracker := NewServiceChangeTracker(nil, tc.ipFamily, nil, nil) // outputs diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go index 531b0497784..c0b4ebc40a8 100644 --- a/pkg/registry/core/service/storage/storage_test.go +++ b/pkg/registry/core/service/storage/storage_test.go @@ -35,6 +35,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" machineryutilnet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/apimachinery/pkg/util/version" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/generic" genericregistrytest "k8s.io/apiserver/pkg/registry/generic/testing" @@ -12000,6 +12001,7 @@ func TestUpdateServiceLoadBalancerStatus(t *testing.T) { // Test here is negative, because starting with v1.30 the feature gate is enabled by default, so we should // now disable it to do the proper test if !loadbalancerIPModeInUse(tc.statusBeforeUpdate) { + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31")) featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, false) } oldSvc := obj.(*api.Service).DeepCopy() @@ -12009,6 +12011,9 @@ func TestUpdateServiceLoadBalancerStatus(t *testing.T) { t.Errorf("updated status: %s", err) } + if !tc.ipModeEnabled { + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31")) + } featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, tc.ipModeEnabled) newSvc := obj.(*api.Service).DeepCopy() newSvc.Status = tc.newStatus diff --git a/pkg/registry/core/service/strategy_test.go b/pkg/registry/core/service/strategy_test.go index f57c8b875cb..549e827f8c8 100644 --- a/pkg/registry/core/service/strategy_test.go +++ b/pkg/registry/core/service/strategy_test.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/version" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/rest" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -475,6 +476,9 @@ func TestDropServiceStatusDisabledFields(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + if !tc.ipModeEnabled { + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31")) + } featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, tc.ipModeEnabled) dropServiceStatusDisabledFields(tc.svc, tc.oldSvc) diff --git a/test/integration/service/loadbalancer_test.go b/test/integration/service/loadbalancer_test.go index 6efd85753b1..ed5830fc2f8 100644 --- a/test/integration/service/loadbalancer_test.go +++ b/test/integration/service/loadbalancer_test.go @@ -19,6 +19,7 @@ package service import ( "context" "encoding/json" + "k8s.io/apimachinery/pkg/util/version" "testing" "time" @@ -707,8 +708,13 @@ func Test_ServiceLoadBalancerIPMode(t *testing.T) { for _, tc := range testCases { t.Run("", func(t *testing.T) { + var testServerOptions *kubeapiservertesting.TestServerInstanceOptions + if !tc.ipModeEnabled { + testServerOptions = &kubeapiservertesting.TestServerInstanceOptions{EmulationVersion: "1.31"} + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31")) + } featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, tc.ipModeEnabled) - server := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.DefaultTestServerFlags(), framework.SharedEtcd()) + server := kubeapiservertesting.StartTestServerOrDie(t, testServerOptions, framework.DefaultTestServerFlags(), framework.SharedEtcd()) defer server.TearDownFn() client, err := clientset.NewForConfig(server.ClientConfig)