diff --git a/pkg/apis/core/types.go b/pkg/apis/core/types.go index ebe7f0a297b..22fee46ce22 100644 --- a/pkg/apis/core/types.go +++ b/pkg/apis/core/types.go @@ -4018,6 +4018,15 @@ type LoadBalancerIngress struct { // +optional Hostname string + // IPMode specifies how the load-balancer IP behaves, and may only be specified when the ip field is specified. + // Setting this to "VIP" indicates that traffic is delivered to the node with + // the destination set to the load-balancer's IP and port. + // Setting this to "Proxy" indicates that traffic is delivered to the node or pod with + // the destination set to the node's IP and node port or the pod's IP and port. + // Service implementations may use this information to adjust traffic routing. + // +optional + IPMode *LoadBalancerIPMode + // Ports is a list of records of service ports // If used, every port defined in the service should have an entry in it // +optional @@ -6090,3 +6099,15 @@ type PortStatus struct { // +kubebuilder:validation:MaxLength=316 Error *string } + +// LoadBalancerIPMode represents the mode of the LoadBalancer ingress IP +type LoadBalancerIPMode string + +const ( + // LoadBalancerIPModeVIP indicates that traffic is delivered to the node with + // the destination set to the load-balancer's IP and port. + LoadBalancerIPModeVIP LoadBalancerIPMode = "VIP" + // LoadBalancerIPModeProxy indicates that traffic is delivered to the node or pod with + // the destination set to the node's IP and port or the pod's IP and port. + LoadBalancerIPModeProxy LoadBalancerIPMode = "Proxy" +) diff --git a/pkg/apis/core/v1/defaults.go b/pkg/apis/core/v1/defaults.go index 51337fe169c..e3a43b1fbd7 100644 --- a/pkg/apis/core/v1/defaults.go +++ b/pkg/apis/core/v1/defaults.go @@ -142,6 +142,19 @@ func SetDefaults_Service(obj *v1.Service) { obj.Spec.AllocateLoadBalancerNodePorts = pointer.Bool(true) } } + + if obj.Spec.Type == v1.ServiceTypeLoadBalancer { + if utilfeature.DefaultFeatureGate.Enabled(features.LoadBalancerIPMode) { + ipMode := v1.LoadBalancerIPModeVIP + + for i, ing := range obj.Status.LoadBalancer.Ingress { + if ing.IP != "" && ing.IPMode == nil { + obj.Status.LoadBalancer.Ingress[i].IPMode = &ipMode + } + } + } + } + } func SetDefaults_Pod(obj *v1.Pod) { // If limits are specified, but requests are not, default requests to limits diff --git a/pkg/apis/core/v1/defaults_test.go b/pkg/apis/core/v1/defaults_test.go index e5ff6404bc5..c6db824c620 100644 --- a/pkg/apis/core/v1/defaults_test.go +++ b/pkg/apis/core/v1/defaults_test.go @@ -1221,6 +1221,74 @@ func TestSetDefaultServiceSessionAffinityConfig(t *testing.T) { } } +func TestSetDefaultServiceLoadbalancerIPMode(t *testing.T) { + modeVIP := v1.LoadBalancerIPModeVIP + modeProxy := v1.LoadBalancerIPModeProxy + testCases := []struct { + name string + ipModeEnabled bool + svc *v1.Service + expectedIPMode []*v1.LoadBalancerIPMode + }{ + { + name: "Set IP but not set IPMode with LoadbalancerIPMode disabled", + ipModeEnabled: false, + svc: &v1.Service{ + Spec: v1.ServiceSpec{Type: v1.ServiceTypeLoadBalancer}, + Status: v1.ServiceStatus{ + LoadBalancer: v1.LoadBalancerStatus{ + Ingress: []v1.LoadBalancerIngress{{ + IP: "1.2.3.4", + }}, + }, + }}, + expectedIPMode: []*v1.LoadBalancerIPMode{nil}, + }, { + name: "Set IP but bot set IPMode with LoadbalancerIPMode enabled", + ipModeEnabled: true, + svc: &v1.Service{ + Spec: v1.ServiceSpec{Type: v1.ServiceTypeLoadBalancer}, + Status: v1.ServiceStatus{ + LoadBalancer: v1.LoadBalancerStatus{ + Ingress: []v1.LoadBalancerIngress{{ + IP: "1.2.3.4", + }}, + }, + }}, + expectedIPMode: []*v1.LoadBalancerIPMode{&modeVIP}, + }, { + name: "Both IP and IPMode are set with LoadbalancerIPMode enabled", + ipModeEnabled: true, + svc: &v1.Service{ + Spec: v1.ServiceSpec{Type: v1.ServiceTypeLoadBalancer}, + Status: v1.ServiceStatus{ + LoadBalancer: v1.LoadBalancerStatus{ + Ingress: []v1.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &modeProxy, + }}, + }, + }}, + expectedIPMode: []*v1.LoadBalancerIPMode{&modeProxy}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, tc.ipModeEnabled)() + obj := roundTrip(t, runtime.Object(tc.svc)) + svc := obj.(*v1.Service) + for i, s := range svc.Status.LoadBalancer.Ingress { + got := s.IPMode + expected := tc.expectedIPMode[i] + if !reflect.DeepEqual(got, expected) { + t.Errorf("Expected IPMode %v, got %v", tc.expectedIPMode[i], s.IPMode) + } + } + }) + } +} + func TestSetDefaultSecretVolumeSource(t *testing.T) { s := v1.PodSpec{} s.Volumes = []v1.Volume{ diff --git a/pkg/apis/core/validation/validation.go b/pkg/apis/core/validation/validation.go index 3ee8cd2f8de..5c7b20adf6e 100644 --- a/pkg/apis/core/validation/validation.go +++ b/pkg/apis/core/validation/validation.go @@ -6997,6 +6997,10 @@ func ValidatePodLogOptions(opts *core.PodLogOptions) field.ErrorList { return allErrs } +var ( + supportedLoadBalancerIPMode = sets.NewString(string(core.LoadBalancerIPModeVIP), string(core.LoadBalancerIPModeProxy)) +) + // ValidateLoadBalancerStatus validates required fields on a LoadBalancerStatus func ValidateLoadBalancerStatus(status *core.LoadBalancerStatus, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} @@ -7007,6 +7011,17 @@ func ValidateLoadBalancerStatus(status *core.LoadBalancerStatus, fldPath *field. allErrs = append(allErrs, field.Invalid(idxPath.Child("ip"), ingress.IP, "must be a valid IP address")) } } + + if utilfeature.DefaultFeatureGate.Enabled(features.LoadBalancerIPMode) && ingress.IPMode == nil { + if len(ingress.IP) > 0 { + allErrs = append(allErrs, field.Required(idxPath.Child("ipMode"), "must be specified when `ip` is set")) + } + } else if ingress.IPMode != nil && len(ingress.IP) == 0 { + allErrs = append(allErrs, field.Forbidden(idxPath.Child("ipMode"), "may not be specified when `ip` is not set")) + } else if ingress.IPMode != nil && !supportedLoadBalancerIPMode.Has(string(*ingress.IPMode)) { + allErrs = append(allErrs, field.NotSupported(idxPath.Child("ipMode"), ingress.IPMode, supportedLoadBalancerIPMode.List())) + } + if len(ingress.Hostname) > 0 { for _, msg := range validation.IsDNS1123Subdomain(ingress.Hostname) { allErrs = append(allErrs, field.Invalid(idxPath.Child("hostname"), ingress.Hostname, msg)) diff --git a/pkg/apis/core/validation/validation_test.go b/pkg/apis/core/validation/validation_test.go index 24df46aa32d..9d9029d03fe 100644 --- a/pkg/apis/core/validation/validation_test.go +++ b/pkg/apis/core/validation/validation_test.go @@ -22997,3 +22997,87 @@ func TestValidateDynamicResourceAllocation(t *testing.T) { } }) } + +func TestValidateLoadBalancerStatus(t *testing.T) { + ipModeVIP := core.LoadBalancerIPModeVIP + ipModeProxy := core.LoadBalancerIPModeProxy + ipModeDummy := core.LoadBalancerIPMode("dummy") + + testCases := []struct { + name string + ipModeEnabled bool + tweakLBStatus func(s *core.LoadBalancerStatus) + numErrs int + }{ + /* LoadBalancerIPMode*/ + { + name: "valid vip ipMode", + ipModeEnabled: true, + tweakLBStatus: func(s *core.LoadBalancerStatus) { + s.Ingress = []core.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeVIP, + }} + }, + numErrs: 0, + }, { + name: "valid proxy ipMode", + ipModeEnabled: true, + tweakLBStatus: func(s *core.LoadBalancerStatus) { + s.Ingress = []core.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeProxy, + }} + }, + numErrs: 0, + }, { + name: "invalid ipMode", + ipModeEnabled: true, + tweakLBStatus: func(s *core.LoadBalancerStatus) { + s.Ingress = []core.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeDummy, + }} + }, + numErrs: 1, + }, { + name: "missing ipMode with LoadbalancerIPMode enabled", + ipModeEnabled: true, + tweakLBStatus: func(s *core.LoadBalancerStatus) { + s.Ingress = []core.LoadBalancerIngress{{ + IP: "1.2.3.4", + }} + }, + numErrs: 1, + }, { + name: "missing ipMode with LoadbalancerIPMode disabled", + ipModeEnabled: false, + tweakLBStatus: func(s *core.LoadBalancerStatus) { + s.Ingress = []core.LoadBalancerIngress{{ + IP: "1.2.3.4", + }} + }, + numErrs: 0, + }, { + name: "missing ip with ipMode present", + ipModeEnabled: true, + tweakLBStatus: func(s *core.LoadBalancerStatus) { + s.Ingress = []core.LoadBalancerIngress{{ + IPMode: &ipModeProxy, + }} + }, + numErrs: 1, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, tc.ipModeEnabled)() + s := core.LoadBalancerStatus{} + tc.tweakLBStatus(&s) + errs := ValidateLoadBalancerStatus(&s, field.NewPath("status")) + if len(errs) != tc.numErrs { + t.Errorf("Unexpected error list for case %q(expected:%v got %v) - Errors:\n %v", tc.name, tc.numErrs, len(errs), errs.ToAggregate()) + } + }) + } +} diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 18e15c86886..80890e5d6a4 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -885,6 +885,12 @@ const ( // // Enables In-Place Pod Vertical Scaling InPlacePodVerticalScaling featuregate.Feature = "InPlacePodVerticalScaling" + + // owner: @Sh4d1,@RyanAoh + // kep: http://kep.k8s.io/1860 + // alpha: v1.29 + // LoadBalancerIPMode enables the IPMode field in the LoadBalancerIngress status of a Service + LoadBalancerIPMode featuregate.Feature = "LoadBalancerIPMode" ) func init() { @@ -1124,6 +1130,8 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS PodIndexLabel: {Default: true, PreRelease: featuregate.Beta}, + LoadBalancerIPMode: {Default: false, PreRelease: featuregate.Alpha}, + // inherited features from generic apiserver, relisted here to get a conflict if it is changed // unintentionally on either side: diff --git a/pkg/registry/core/service/storage/storage_test.go b/pkg/registry/core/service/storage/storage_test.go index 60fd585cab0..73292a30ba5 100644 --- a/pkg/registry/core/service/storage/storage_test.go +++ b/pkg/registry/core/service/storage/storage_test.go @@ -27,6 +27,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" @@ -39,9 +40,12 @@ import ( genericregistrytest "k8s.io/apiserver/pkg/registry/generic/testing" "k8s.io/apiserver/pkg/registry/rest" etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" epstest "k8s.io/kubernetes/pkg/api/endpoints/testing" svctest "k8s.io/kubernetes/pkg/api/service/testing" api "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/kubernetes/pkg/features" endpointstore "k8s.io/kubernetes/pkg/registry/core/endpoint/storage" podstore "k8s.io/kubernetes/pkg/registry/core/pod/storage" "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" @@ -11681,3 +11685,288 @@ func TestServiceRegistryResourceLocation(t *testing.T) { }) } } + +func TestUpdateServiceLoadBalancerStatus(t *testing.T) { + storage, statusStorage, server := newStorage(t, []api.IPFamily{api.IPv4Protocol}) + defer server.Terminate(t) + defer storage.Store.DestroyFunc() + defer statusStorage.store.DestroyFunc() + + ipModeVIP := api.LoadBalancerIPModeVIP + ipModeProxy := api.LoadBalancerIPModeProxy + ipModeDummy := api.LoadBalancerIPMode("dummy") + + testCases := []struct { + name string + ipModeEnabled bool + statusBeforeUpdate api.ServiceStatus + newStatus api.ServiceStatus + expectedStatus api.ServiceStatus + expectErr bool + expectedReasonForError metav1.StatusReason + }{ + /*LoadBalancerIPMode disabled*/ + { + name: "LoadBalancerIPMode disabled, ipMode not used in old, not used in new", + ipModeEnabled: false, + statusBeforeUpdate: api.ServiceStatus{}, + newStatus: api.ServiceStatus{ + LoadBalancer: api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + }}, + }, + }, + expectedStatus: api.ServiceStatus{ + LoadBalancer: api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + }}, + }, + }, + expectErr: false, + }, { + name: "LoadBalancerIPMode disabled, ipMode used in old and in new", + ipModeEnabled: false, + statusBeforeUpdate: api.ServiceStatus{ + LoadBalancer: api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeVIP, + }}, + }, + }, + newStatus: api.ServiceStatus{ + LoadBalancer: api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeProxy, + }}, + }, + }, + expectedStatus: api.ServiceStatus{ + LoadBalancer: api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeProxy, + }}, + }, + }, + expectErr: false, + }, { + name: "LoadBalancerIPMode disabled, ipMode not used in old, used in new", + ipModeEnabled: false, + statusBeforeUpdate: api.ServiceStatus{ + LoadBalancer: api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + }}, + }, + }, + newStatus: api.ServiceStatus{ + LoadBalancer: api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeProxy, + }}, + }, + }, + expectedStatus: api.ServiceStatus{ + LoadBalancer: api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + }}, + }, + }, + expectErr: false, + }, { + name: "LoadBalancerIPMode disabled, ipMode used in old, not used in new", + ipModeEnabled: false, + statusBeforeUpdate: api.ServiceStatus{ + LoadBalancer: api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeVIP, + }}, + }, + }, + newStatus: api.ServiceStatus{ + LoadBalancer: api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + }}, + }, + }, + expectedStatus: api.ServiceStatus{ + LoadBalancer: api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + }}, + }, + }, + expectErr: false, + }, + /*LoadBalancerIPMode enabled*/ + { + name: "LoadBalancerIPMode enabled, ipMode not used in old, not used in new", + ipModeEnabled: true, + statusBeforeUpdate: api.ServiceStatus{}, + newStatus: api.ServiceStatus{ + LoadBalancer: api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + }}, + }, + }, + expectedStatus: api.ServiceStatus{}, + expectErr: true, + expectedReasonForError: metav1.StatusReasonInvalid, + }, { + name: "LoadBalancerIPMode enabled, ipMode used in old and in new", + ipModeEnabled: true, + statusBeforeUpdate: api.ServiceStatus{ + LoadBalancer: api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeProxy, + }}, + }, + }, + newStatus: api.ServiceStatus{ + LoadBalancer: api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeVIP, + }}, + }, + }, + expectedStatus: api.ServiceStatus{ + LoadBalancer: api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeVIP, + }}, + }, + }, + expectErr: false, + }, { + name: "LoadBalancerIPMode enabled, ipMode not used in old, used in new", + ipModeEnabled: true, + statusBeforeUpdate: api.ServiceStatus{ + LoadBalancer: api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + }}, + }, + }, + newStatus: api.ServiceStatus{ + LoadBalancer: api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeProxy, + }}, + }, + }, + expectedStatus: api.ServiceStatus{ + LoadBalancer: api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeProxy, + }}, + }, + }, + expectErr: false, + }, { + name: "LoadBalancerIPMode enabled, ipMode used in old, not used in new", + ipModeEnabled: true, + statusBeforeUpdate: api.ServiceStatus{ + LoadBalancer: api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeVIP, + }}, + }, + }, + newStatus: api.ServiceStatus{ + LoadBalancer: api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + }}, + }, + }, + expectedStatus: api.ServiceStatus{}, + expectErr: true, + expectedReasonForError: metav1.StatusReasonInvalid, + }, { + name: "LoadBalancerIPMode enabled, ipMode not used in old, invalid value used in new", + ipModeEnabled: true, + statusBeforeUpdate: api.ServiceStatus{ + LoadBalancer: api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + }}, + }, + }, + newStatus: api.ServiceStatus{ + LoadBalancer: api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeDummy, + }}, + }, + }, + expectedStatus: api.ServiceStatus{}, + expectErr: true, + expectedReasonForError: metav1.StatusReasonInvalid, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + + svc := svctest.MakeService("foo", svctest.SetTypeLoadBalancer) + ctx := genericapirequest.NewDefaultContext() + obj, err := storage.Create(ctx, svc, rest.ValidateAllObjectFunc, &metav1.CreateOptions{}) + if err != nil { + t.Errorf("created svc: %s", err) + } + defer storage.Delete(ctx, svc.Name, rest.ValidateAllObjectFunc, &metav1.DeleteOptions{}) + + // prepare status + if loadbalancerIPModeInUse(tc.statusBeforeUpdate) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, true)() + } + oldSvc := obj.(*api.Service).DeepCopy() + oldSvc.Status = tc.statusBeforeUpdate + obj, _, err = statusStorage.Update(ctx, oldSvc.Name, rest.DefaultUpdatedObjectInfo(oldSvc), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{}) + if err != nil { + t.Errorf("updated status: %s", err) + } + + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, tc.ipModeEnabled)() + newSvc := obj.(*api.Service).DeepCopy() + newSvc.Status = tc.newStatus + obj, _, err = statusStorage.Update(ctx, newSvc.Name, rest.DefaultUpdatedObjectInfo(newSvc), rest.ValidateAllObjectFunc, rest.ValidateAllObjectUpdateFunc, false, &metav1.UpdateOptions{}) + if err != nil { + if tc.expectErr && tc.expectedReasonForError == errors.ReasonForError(err) { + return + } + t.Errorf("updated status: %s", err) + } + + updated := obj.(*api.Service) + if !reflect.DeepEqual(tc.expectedStatus, updated.Status) { + t.Errorf("%v: unexpected svc status: %v", tc.name, cmp.Diff(tc.expectedStatus, updated.Status)) + } + }) + } +} + +func loadbalancerIPModeInUse(status api.ServiceStatus) bool { + for _, ing := range status.LoadBalancer.Ingress { + if ing.IPMode != nil { + return true + } + } + return false +} diff --git a/pkg/registry/core/service/strategy.go b/pkg/registry/core/service/strategy.go index acb3f3d66a2..b11a6329834 100644 --- a/pkg/registry/core/service/strategy.go +++ b/pkg/registry/core/service/strategy.go @@ -24,10 +24,12 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apiserver/pkg/storage/names" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/api/legacyscheme" serviceapi "k8s.io/kubernetes/pkg/api/service" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/core/validation" + "k8s.io/kubernetes/pkg/features" "sigs.k8s.io/structured-merge-diff/v4/fieldpath" ) @@ -144,6 +146,8 @@ func (serviceStatusStrategy) GetResetFields() map[fieldpath.APIVersion]*fieldpat func (serviceStatusStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) { newService := obj.(*api.Service) oldService := old.(*api.Service) + + dropServiceStatusDisabledFields(newService, oldService) // status changes are not allowed to update spec newService.Spec = oldService.Spec } @@ -158,6 +162,33 @@ func (serviceStatusStrategy) WarningsOnUpdate(ctx context.Context, obj, old runt return nil } +// dropServiceStatusDisabledFields drops fields that are not used if their associated feature gates +// are not enabled. The typical pattern is: +// +// if !utilfeature.DefaultFeatureGate.Enabled(features.MyFeature) && !myFeatureInUse(oldSvc) { +// newSvc.Status.MyFeature = nil +// } +func dropServiceStatusDisabledFields(newSvc *api.Service, oldSvc *api.Service) { + if !utilfeature.DefaultFeatureGate.Enabled(features.LoadBalancerIPMode) && !loadbalancerIPModeInUse(oldSvc) { + for i := range newSvc.Status.LoadBalancer.Ingress { + newSvc.Status.LoadBalancer.Ingress[i].IPMode = nil + } + } +} + +// returns true when the LoadBalancer Ingress IPMode fields are in use. +func loadbalancerIPModeInUse(svc *api.Service) bool { + if svc == nil { + return false + } + for _, ing := range svc.Status.LoadBalancer.Ingress { + if ing.IPMode != nil { + return true + } + } + return false +} + func sameStringSlice(a []string, b []string) bool { if len(a) != len(b) { return false diff --git a/pkg/registry/core/service/strategy_test.go b/pkg/registry/core/service/strategy_test.go index 8908615c344..3a35b5860ba 100644 --- a/pkg/registry/core/service/strategy_test.go +++ b/pkg/registry/core/service/strategy_test.go @@ -26,8 +26,11 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/rest" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" api "k8s.io/kubernetes/pkg/apis/core" _ "k8s.io/kubernetes/pkg/apis/core/install" + "k8s.io/kubernetes/pkg/features" utilpointer "k8s.io/utils/pointer" ) @@ -229,6 +232,251 @@ func TestDropDisabledField(t *testing.T) { } +func TestDropServiceStatusDisabledFields(t *testing.T) { + ipModeVIP := api.LoadBalancerIPModeVIP + ipModeProxy := api.LoadBalancerIPModeProxy + + testCases := []struct { + name string + ipModeEnabled bool + svc *api.Service + oldSvc *api.Service + compareSvc *api.Service + }{ + /*LoadBalancerIPMode disabled*/ + { + name: "LoadBalancerIPMode disabled, ipMode not used in old, not used in new", + ipModeEnabled: false, + svc: makeValidServiceCustom(func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Status.LoadBalancer = api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + }}, + } + }), + oldSvc: makeValidServiceCustom(func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Status.LoadBalancer = api.LoadBalancerStatus{} + }), + compareSvc: makeValidServiceCustom(func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Status.LoadBalancer = api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + }}, + } + }), + }, { + name: "LoadBalancerIPMode disabled, ipMode used in old and in new", + ipModeEnabled: false, + svc: makeValidServiceCustom(func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Status.LoadBalancer = api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeProxy, + }}, + } + }), + oldSvc: makeValidServiceCustom(func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Status.LoadBalancer = api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeVIP, + }}, + } + }), + compareSvc: makeValidServiceCustom(func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Status.LoadBalancer = api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeProxy, + }}, + } + }), + }, { + name: "LoadBalancerIPMode disabled, ipMode not used in old, used in new", + ipModeEnabled: false, + svc: makeValidServiceCustom(func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Status.LoadBalancer = api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeVIP, + }}, + } + }), + oldSvc: makeValidServiceCustom(func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Status.LoadBalancer = api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + }}, + } + }), + compareSvc: makeValidServiceCustom(func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Status.LoadBalancer = api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + }}, + } + }), + }, { + name: "LoadBalancerIPMode disabled, ipMode used in old, not used in new", + ipModeEnabled: false, + svc: makeValidServiceCustom(func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Status.LoadBalancer = api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + }}, + } + }), + oldSvc: makeValidServiceCustom(func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Status.LoadBalancer = api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeProxy, + }}, + } + }), + compareSvc: makeValidServiceCustom(func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Status.LoadBalancer = api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + }}, + } + }), + }, + /*LoadBalancerIPMode enabled*/ + { + name: "LoadBalancerIPMode enabled, ipMode not used in old, not used in new", + ipModeEnabled: true, + svc: makeValidServiceCustom(func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Status.LoadBalancer = api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + }}, + } + }), + oldSvc: nil, + compareSvc: makeValidServiceCustom(func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Status.LoadBalancer = api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + }}, + } + }), + }, { + name: "LoadBalancerIPMode enabled, ipMode used in old and in new", + ipModeEnabled: true, + svc: makeValidServiceCustom(func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Status.LoadBalancer = api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeProxy, + }}, + } + }), + oldSvc: makeValidServiceCustom(func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Status.LoadBalancer = api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeVIP, + }}, + } + }), + compareSvc: makeValidServiceCustom(func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Status.LoadBalancer = api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeProxy, + }}, + } + }), + }, { + name: "LoadBalancerIPMode enabled, ipMode not used in old, used in new", + ipModeEnabled: true, + svc: makeValidServiceCustom(func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Status.LoadBalancer = api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeVIP, + }}, + } + }), + oldSvc: makeValidServiceCustom(func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Status.LoadBalancer = api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + }}, + } + }), + compareSvc: makeValidServiceCustom(func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Status.LoadBalancer = api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeVIP, + }}, + } + }), + }, { + name: "LoadBalancerIPMode enabled, ipMode used in old, not used in new", + ipModeEnabled: true, + svc: makeValidServiceCustom(func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Status.LoadBalancer = api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + }}, + } + }), + oldSvc: makeValidServiceCustom(func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Status.LoadBalancer = api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + IPMode: &ipModeProxy, + }}, + } + }), + compareSvc: makeValidServiceCustom(func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Status.LoadBalancer = api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{{ + IP: "1.2.3.4", + }}, + } + }), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, tc.ipModeEnabled)() + dropServiceStatusDisabledFields(tc.svc, tc.oldSvc) + + if !reflect.DeepEqual(tc.svc, tc.compareSvc) { + t.Errorf("%v: unexpected svc spec: %v", tc.name, cmp.Diff(tc.svc, tc.compareSvc)) + } + }) + } +} + func TestDropTypeDependentFields(t *testing.T) { // Tweaks used below. setTypeExternalName := func(svc *api.Service) { diff --git a/staging/src/k8s.io/api/core/v1/types.go b/staging/src/k8s.io/api/core/v1/types.go index 5dd5d09ce26..0c9248307ca 100644 --- a/staging/src/k8s.io/api/core/v1/types.go +++ b/staging/src/k8s.io/api/core/v1/types.go @@ -4632,6 +4632,15 @@ type LoadBalancerIngress struct { // +optional Hostname string `json:"hostname,omitempty" protobuf:"bytes,2,opt,name=hostname"` + // IPMode specifies how the load-balancer IP behaves, and may only be specified when the ip field is specified. + // Setting this to "VIP" indicates that traffic is delivered to the node with + // the destination set to the load-balancer's IP and port. + // Setting this to "Proxy" indicates that traffic is delivered to the node or pod with + // the destination set to the node's IP and node port or the pod's IP and port. + // Service implementations may use this information to adjust traffic routing. + // +optional + IPMode *LoadBalancerIPMode `json:"ipMode,omitempty" protobuf:"bytes,3,opt,name=ipMode"` + // Ports is a list of records of service ports // If used, every port defined in the service should have an entry in it // +listType=atomic @@ -6994,3 +7003,15 @@ type PortStatus struct { // +kubebuilder:validation:MaxLength=316 Error *string `json:"error,omitempty" protobuf:"bytes,3,opt,name=error"` } + +// LoadBalancerIPMode represents the mode of the LoadBalancer ingress IP +type LoadBalancerIPMode string + +const ( + // LoadBalancerIPModeVIP indicates that traffic is delivered to the node with + // the destination set to the load-balancer's IP and port. + LoadBalancerIPModeVIP LoadBalancerIPMode = "VIP" + // LoadBalancerIPModeProxy indicates that traffic is delivered to the node or pod with + // the destination set to the node's IP and port or the pod's IP and port. + LoadBalancerIPModeProxy LoadBalancerIPMode = "Proxy" +) diff --git a/staging/src/k8s.io/cloud-provider/service/helpers/helper.go b/staging/src/k8s.io/cloud-provider/service/helpers/helper.go index e363c7db2c5..fd436c3d37d 100644 --- a/staging/src/k8s.io/cloud-provider/service/helpers/helper.go +++ b/staging/src/k8s.io/cloud-provider/service/helpers/helper.go @@ -180,5 +180,8 @@ func ingressEqual(lhs, rhs *v1.LoadBalancerIngress) bool { if lhs.Hostname != rhs.Hostname { return false } + if lhs.IPMode != rhs.IPMode { + return false + } return true }