Merge pull request #92312 from Sh4d1/kep_1860

Make Kubernetes aware of the LoadBalancer behaviour

api/openapi-spec/swagger.json

@@ -7337,6 +7337,10 @@
         "ip": {
           "description": "IP is set for load-balancer ingress points that are IP based (typically GCE or OpenStack load-balancers)",
           "type": "string"
+        },
+        "ipMode": {
+          "description": "IPMode specifies how the load-balancer's IP behaves. Setting this to \"VIP\" indicates that the traffic passing through this load-balancer is delivered with the destination IP and port set to the load-balancer's IP and port. Setting this to \"Proxy\" indicates that the load-balancer acts like a proxy, delivering traffic with the destination IP and port set to the node's IP and nodePort or to the pod's IP and targetPort. This field can only be set when the ip field is also set, and defaults to \"VIP\" if not specified.",
+          "type": "string"
         }
       },
       "type": "object"

pkg/apis/core/types.go

@@ -3508,6 +3508,15 @@ type LoadBalancerIngress struct {
 	// (typically AWS load-balancers)
 	// +optional
 	Hostname string
+
+	// IPMode specifies how the load-balancer's IP behaves.
+	// Setting this to "VIP" indicates that the traffic passing through
+	// this load-balancer is delivered with the destination IP and port set to the load-balancer's IP and port.
+	// Setting this to "Proxy" indicates that the load-balancer acts like a proxy,
+	// delivering traffic with the destination IP and port set to the node's IP and nodePort or to the pod's IP and targetPort.
+	// This field can only be set when the ip field is also set, and defaults to "VIP" if not specified.
+	// +optional
+	IPMode *LoadBalancerIPMode
 }
 
 const (
@@ -3515,6 +3524,18 @@ const (
 	MaxServiceTopologyKeys = 16
 )
 
+// LoadBalancerIPMode represents the mode of the LoadBalancer ingress IP
+type LoadBalancerIPMode string
+
+const (
+	// LoadBalancerIPModeVIP indicates that the traffic passing through this LoadBalancer
+	// is delivered with the destination IP set to the specified LoadBalancer IP
+	LoadBalancerIPModeVIP LoadBalancerIPMode = "VIP"
+	// LoadBalancerIPModeProxy indicates that the specified LoadBalancer acts like a proxy,
+	// changing the destination IP to the node IP and the source IP to the LoadBalancer (mostly private) IP
+	LoadBalancerIPModeProxy LoadBalancerIPMode = "Proxy"
+)
+
 // IPFamily represents the IP Family (IPv4 or IPv6). This type is used
 // to express the family of an IP expressed by a type (e.g. service.spec.ipFamilies).
 type IPFamily string

pkg/apis/core/v1/defaults.go

@@ -19,7 +19,7 @@ package v1
 import (
 	"time"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/kubernetes/pkg/util/parsers"
@@ -166,6 +166,17 @@ func SetDefaults_Service(obj *v1.Service) {
 		// further IPFamilies, IPFamilyPolicy defaulting is in ClusterIP alloc/reserve logic
 		// note: conversion logic handles cases where ClusterIPs is used (but not ClusterIP).
 	}
+
+	if utilfeature.DefaultFeatureGate.Enabled(features.LoadBalancerIPMode) &&
+		obj.Spec.Type == v1.ServiceTypeLoadBalancer {
+		ipMode := v1.LoadBalancerIPModeVIP
+
+		for i, ing := range obj.Status.LoadBalancer.Ingress {
+			if ing.IP != "" && ing.IPMode == nil {
+				obj.Status.LoadBalancer.Ingress[i].IPMode = &ipMode
+			}
+		}
+	}
 }
 func SetDefaults_Pod(obj *v1.Pod) {
 	// If limits are specified, but requests are not, default requests to limits

pkg/apis/core/v1/zz_generated.conversion.go

@@ -4291,6 +4291,7 @@ func Convert_core_List_To_v1_List(in *core.List, out *v1.List, s conversion.Scop
 func autoConvert_v1_LoadBalancerIngress_To_core_LoadBalancerIngress(in *v1.LoadBalancerIngress, out *core.LoadBalancerIngress, s conversion.Scope) error {
 	out.IP = in.IP
 	out.Hostname = in.Hostname
+	out.IPMode = (*core.LoadBalancerIPMode)(unsafe.Pointer(in.IPMode))
 	return nil
 }
 
@@ -4302,6 +4303,7 @@ func Convert_v1_LoadBalancerIngress_To_core_LoadBalancerIngress(in *v1.LoadBalan
 func autoConvert_core_LoadBalancerIngress_To_v1_LoadBalancerIngress(in *core.LoadBalancerIngress, out *v1.LoadBalancerIngress, s conversion.Scope) error {
 	out.IP = in.IP
 	out.Hostname = in.Hostname
+	out.IPMode = (*v1.LoadBalancerIPMode)(unsafe.Pointer(in.IPMode))
 	return nil
 }
 

pkg/apis/core/validation/validation.go

@@ -5973,6 +5973,10 @@ func ValidatePodLogOptions(opts *core.PodLogOptions) field.ErrorList {
 	return allErrs
 }
 
+var (
+	supportedLoadBalancerIPMode = sets.NewString(string(core.LoadBalancerIPModeVIP), string(core.LoadBalancerIPModeProxy))
+)
+
 // ValidateLoadBalancerStatus validates required fields on a LoadBalancerStatus
 func ValidateLoadBalancerStatus(status *core.LoadBalancerStatus, fldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}
@@ -5983,6 +5987,23 @@ func ValidateLoadBalancerStatus(status *core.LoadBalancerStatus, fldPath *field.
 				allErrs = append(allErrs, field.Invalid(idxPath.Child("ip"), ingress.IP, "must be a valid IP address"))
 			}
 		}
+
+		if utilfeature.DefaultFeatureGate.Enabled(features.LoadBalancerIPMode) {
+			if len(ingress.IP) > 0 && ingress.IPMode == nil {
+				allErrs = append(allErrs, field.Required(idxPath.Child("ipMode"), "must be specified when `ip` is set"))
+			}
+		}
+
+		if ingress.IPMode != nil {
+			if len(ingress.IP) == 0 {
+				allErrs = append(allErrs, field.Forbidden(idxPath.Child("ipMode"), "may not be used when `ip` is not set"))
+			}
+
+			if !supportedLoadBalancerIPMode.Has(string(*ingress.IPMode)) {
+				allErrs = append(allErrs, field.NotSupported(idxPath.Child("ipMode"), ingress.IPMode, supportedLoadBalancerIPMode.List()))
+			}
+		}
+
 		if len(ingress.Hostname) > 0 {
 			for _, msg := range validation.IsDNS1123Subdomain(ingress.Hostname) {
 				allErrs = append(allErrs, field.Invalid(idxPath.Child("hostname"), ingress.Hostname, msg))

pkg/apis/core/validation/validation_test.go

@@ -11071,6 +11071,90 @@ func TestValidateServiceCreate(t *testing.T) {
 	}
 }
 
+func TestValidateLoadBalancerStatus(t *testing.T) {
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, true)()
+
+	ipModeVIP := core.LoadBalancerIPModeVIP
+	ipModeProxy := core.LoadBalancerIPModeProxy
+	ipModeDummy := core.LoadBalancerIPMode("dummy")
+
+	testCases := []struct {
+		name          string
+		tweakLBStatus func(s *core.LoadBalancerStatus)
+		numErrs       int
+	}{
+		/* LoadBalancerIPMode*/
+		{
+			name: `valid vip ipMode`,
+			tweakLBStatus: func(s *core.LoadBalancerStatus) {
+				s.Ingress = []core.LoadBalancerIngress{
+					{
+						IP:     "1.2.3.4",
+						IPMode: &ipModeVIP,
+					},
+				}
+			},
+			numErrs: 0,
+		},
+		{
+			name: `valid proxy ipMode`,
+			tweakLBStatus: func(s *core.LoadBalancerStatus) {
+				s.Ingress = []core.LoadBalancerIngress{
+					{
+						IP:     "1.2.3.4",
+						IPMode: &ipModeProxy,
+					},
+				}
+			},
+			numErrs: 0,
+		},
+		{
+			name: `invalid ipMode`,
+			tweakLBStatus: func(s *core.LoadBalancerStatus) {
+				s.Ingress = []core.LoadBalancerIngress{
+					{
+						IP:     "1.2.3.4",
+						IPMode: &ipModeDummy,
+					},
+				}
+			},
+			numErrs: 1,
+		},
+		{
+			name: `missing ipMode`,
+			tweakLBStatus: func(s *core.LoadBalancerStatus) {
+				s.Ingress = []core.LoadBalancerIngress{
+					{
+						IP: "1.2.3.4",
+					},
+				}
+			},
+			numErrs: 1,
+		},
+		{
+			name: `missing ip with ipMode present`,
+			tweakLBStatus: func(s *core.LoadBalancerStatus) {
+				s.Ingress = []core.LoadBalancerIngress{
+					{
+						IPMode: &ipModeProxy,
+					},
+				}
+			},
+			numErrs: 1,
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			s := core.LoadBalancerStatus{}
+			tc.tweakLBStatus(&s)
+			errs := ValidateLoadBalancerStatus(&s, field.NewPath("status"))
+			if len(errs) != tc.numErrs {
+				t.Errorf("Unexpected error list for case %q(expected:%v got %v) - Errors:\n %v", tc.name, tc.numErrs, len(errs), errs.ToAggregate())
+			}
+		})
+	}
+}
+
 func TestValidateServiceExternalTrafficFieldsCombination(t *testing.T) {
 	testCases := []struct {
 		name     string

pkg/apis/core/zz_generated.deepcopy.go

@@ -2146,6 +2146,11 @@ func (in *List) DeepCopyObject() runtime.Object {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *LoadBalancerIngress) DeepCopyInto(out *LoadBalancerIngress) {
 	*out = *in
+	if in.IPMode != nil {
+		in, out := &in.IPMode, &out.IPMode
+		*out = new(LoadBalancerIPMode)
+		**out = **in
+	}
 	return
 }
 
@@ -2165,7 +2170,9 @@ func (in *LoadBalancerStatus) DeepCopyInto(out *LoadBalancerStatus) {
 	if in.Ingress != nil {
 		in, out := &in.Ingress, &out.Ingress
 		*out = make([]LoadBalancerIngress, len(*in))
-		copy(*out, *in)
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
 	}
 	return
 }

pkg/features/kube_features.go

@@ -663,6 +663,11 @@ const (
 	//
 	// Enables kubelet support to size memory backed volumes
 	SizeMemoryBackedVolumes featuregate.Feature = "SizeMemoryBackedVolumes"
+
+	// owner: @Sh4d1
+	// alpha: v1.21
+	// LoadBalancerIPMode enables the IPMode field in the LoadBalancerIngress status of a Service
+	LoadBalancerIPMode featuregate.Feature = "LoadBalancerIPMode"
 )
 
 func init() {
@@ -763,6 +768,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 	HPAContainerMetrics:                            {Default: false, PreRelease: featuregate.Alpha},
 	RootCAConfigMap:                                {Default: true, PreRelease: featuregate.Beta},
 	SizeMemoryBackedVolumes:                        {Default: false, PreRelease: featuregate.Alpha},
+	LoadBalancerIPMode:                             {Default: false, PreRelease: featuregate.Alpha},
 
 	// inherited features from generic apiserver, relisted here to get a conflict if it is changed
 	// unintentionally on either side:

pkg/proxy/iptables/BUILD

@@ -39,6 +39,7 @@ go_test(
     srcs = ["proxier_test.go"],
     embed = [":go_default_library"],
     deps = [
+        "//pkg/features:go_default_library",
         "//pkg/proxy:go_default_library",
         "//pkg/proxy/healthcheck:go_default_library",
         "//pkg/proxy/util:go_default_library",
@@ -53,6 +54,8 @@ go_test(
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
         "//vendor/k8s.io/klog/v2:go_default_library",
         "//vendor/k8s.io/utils/exec:go_default_library",

pkg/proxy/iptables/proxier.go

@@ -782,10 +782,10 @@ func (proxier *Proxier) deleteEndpointConnections(connectionMap []proxy.ServiceE
 					klog.Errorf("Failed to delete %s endpoint connections for externalIP %s, error: %v", epSvcPair.ServicePortName.String(), extIP, err)
 				}
 			}
-			for _, lbIP := range svcInfo.LoadBalancerIPStrings() {
-				err := conntrack.ClearEntriesForNAT(proxier.exec, lbIP, endpointIP, svcProto)
+			for _, lbIngress := range svcInfo.LoadBalancerIngress() {
+				err := conntrack.ClearEntriesForNAT(proxier.exec, lbIngress.IP, endpointIP, svcProto)
 				if err != nil {
-					klog.Errorf("Failed to delete %s endpoint connections for LoabBalancerIP %s, error: %v", epSvcPair.ServicePortName.String(), lbIP, err)
+					klog.Errorf("Failed to delete %s endpoint connections for LoabBalancerIP %s, error: %v", epSvcPair.ServicePortName.String(), lbIngress.IP, err)
 				}
 			}
 		}
@@ -1161,8 +1161,8 @@ func (proxier *Proxier) syncProxyRules() {
 
 		// Capture load-balancer ingress.
 		fwChain := svcInfo.serviceFirewallChainName
-		for _, ingress := range svcInfo.LoadBalancerIPStrings() {
-			if ingress != "" {
+		for _, ingress := range svcInfo.LoadBalancerIngress() {
+			if ingress.IP != "" {
 				if hasEndpoints {
 					// create service firewall chain
 					if chain, ok := existingNATChains[fwChain]; ok {
@@ -1175,15 +1175,17 @@ func (proxier *Proxier) syncProxyRules() {
 					// This currently works for loadbalancers that preserves source ips.
 					// For loadbalancers which direct traffic to service NodePort, the firewall rules will not apply.
 
+					if !utilfeature.DefaultFeatureGate.Enabled(features.LoadBalancerIPMode) || *ingress.IPMode == v1.LoadBalancerIPModeVIP {
 						args = append(args[:0],
 							"-A", string(kubeServicesChain),
 							"-m", "comment", "--comment", fmt.Sprintf(`"%s loadbalancer IP"`, svcNameString),
 							"-m", protocol, "-p", protocol,
-						"-d", utilproxy.ToCIDR(net.ParseIP(ingress)),
+							"-d", utilproxy.ToCIDR(net.ParseIP(ingress.IP)),
 							"--dport", strconv.Itoa(svcInfo.Port()),
 						)
 						// jump to service firewall chain
 						writeLine(proxier.natRules, append(args, "-j", string(fwChain))...)
+					}
 
 					args = append(args[:0],
 						"-A", string(fwChain),
@@ -1218,7 +1220,7 @@ func (proxier *Proxier) syncProxyRules() {
 						// loadbalancer's backend hosts. In this case, request will not hit the loadbalancer but loop back directly.
 						// Need to add the following rule to allow request on host.
 						if allowFromNode {
-							writeLine(proxier.natRules, append(args, "-s", utilproxy.ToCIDR(net.ParseIP(ingress)), "-j", string(chosenChain))...)
+							writeLine(proxier.natRules, append(args, "-s", utilproxy.ToCIDR(net.ParseIP(ingress.IP)), "-j", string(chosenChain))...)
 						}
 					}
 
@@ -1231,7 +1233,7 @@ func (proxier *Proxier) syncProxyRules() {
 						"-A", string(kubeExternalServicesChain),
 						"-m", "comment", "--comment", fmt.Sprintf(`"%s has no endpoints"`, svcNameString),
 						"-m", protocol, "-p", protocol,
-						"-d", utilproxy.ToCIDR(net.ParseIP(ingress)),
+						"-d", utilproxy.ToCIDR(net.ParseIP(ingress.IP)),
 						"--dport", strconv.Itoa(svcInfo.Port()),
 						"-j", "REJECT",
 					)

pkg/proxy/iptables/proxier_test.go

@@ -26,6 +26,7 @@ import (
 	"testing"
 	"time"
 
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/klog/v2"
 
 	"github.com/stretchr/testify/assert"
@@ -34,6 +35,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/proxy"
 	"k8s.io/kubernetes/pkg/proxy/healthcheck"
 	utilproxy "k8s.io/kubernetes/pkg/proxy/util"
@@ -46,6 +48,8 @@ import (
 	"k8s.io/utils/exec"
 	fakeexec "k8s.io/utils/exec/testing"
 	utilpointer "k8s.io/utils/pointer"
+
+	featuregatetesting "k8s.io/component-base/featuregate/testing"
 )
 
 func checkAllLines(t *testing.T, table utiliptables.Table, save []byte, expectedLines map[utiliptables.Chain]string) {
@@ -2682,4 +2686,85 @@ COMMIT
 	assert.NotEqual(t, expectedIPTablesWithSlice, fp.iptablesData.String())
 }
 
+func TestLoadBalancerIngressRouteTypeProxy(t *testing.T) {
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, true)()
+	ipModeProxy := v1.LoadBalancerIPModeProxy
+	ipModeVIP := v1.LoadBalancerIPModeVIP
+
+	testCases := []struct {
+		svcIP        string
+		svcLBIP      string
+		ipMode       *v1.LoadBalancerIPMode
+		expectedRule bool
+	}{
+		{
+			svcIP:        "10.20.30.41",
+			svcLBIP:      "1.2.3.4",
+			ipMode:       &ipModeProxy,
+			expectedRule: false,
+		},
+		{
+			svcIP:        "10.20.30.42",
+			svcLBIP:      "1.2.3.5",
+			ipMode:       &ipModeVIP,
+			expectedRule: true,
+		},
+	}
+
+	svcPort := 80
+	svcNodePort := 3001
+	svcPortName := proxy.ServicePortName{
+		NamespacedName: makeNSN("ns1", "svc1"),
+		Port:           "p80",
+		Protocol:       v1.ProtocolTCP,
+	}
+
+	for _, testCase := range testCases {
+		ipt := iptablestest.NewFake()
+		fp := NewFakeProxier(ipt, false)
+		makeServiceMap(fp,
+			makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) {
+				svc.Spec.Type = "LoadBalancer"
+				svc.Spec.ClusterIP = testCase.svcIP
+				svc.Spec.Ports = []v1.ServicePort{{
+					Name:     svcPortName.Port,
+					Port:     int32(svcPort),
+					Protocol: v1.ProtocolTCP,
+					NodePort: int32(svcNodePort),
+				}}
+				svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
+					IP:     testCase.svcLBIP,
+					IPMode: testCase.ipMode,
+				}}
+			}),
+		)
+
+		epIP := "10.180.0.1"
+		makeEndpointsMap(fp,
+			makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *v1.Endpoints) {
+				ept.Subsets = []v1.EndpointSubset{{
+					Addresses: []v1.EndpointAddress{{
+						IP: epIP,
+					}},
+					Ports: []v1.EndpointPort{{
+						Name:     svcPortName.Port,
+						Port:     int32(svcPort),
+						Protocol: v1.ProtocolTCP,
+					}},
+				}}
+			}),
+		)
+
+		fp.syncProxyRules()
+
+		proto := strings.ToLower(string(v1.ProtocolTCP))
+		fwChain := string(serviceFirewallChainName(svcPortName.String(), proto))
+
+		kubeSvcRules := ipt.GetRules(string(kubeServicesChain))
+		if hasJump(kubeSvcRules, fwChain, testCase.svcLBIP, svcPort) != testCase.expectedRule {
+			errorf(fmt.Sprintf("Found jump to firewall chain %v", fwChain), kubeSvcRules, t)
+		}
+	}
+}
+
 // TODO(thockin): add *more* tests for syncProxyRules() or break it down further and test the pieces.

pkg/proxy/ipvs/BUILD

@@ -15,6 +15,7 @@ go_test(
     ],
     embed = [":go_default_library"],
     deps = [
+        "//pkg/features:go_default_library",
         "//pkg/proxy:go_default_library",
         "//pkg/proxy/healthcheck:go_default_library",
         "//pkg/proxy/ipvs/testing:go_default_library",
@@ -34,6 +35,8 @@ go_test(
         "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
         "//vendor/k8s.io/utils/exec:go_default_library",
         "//vendor/k8s.io/utils/exec/testing:go_default_library",

pkg/proxy/ipvs/proxier.go

@@ -1332,11 +1332,11 @@ func (proxier *Proxier) syncProxyRules() {
 		}
 
 		// Capture load-balancer ingress.
-		for _, ingress := range svcInfo.LoadBalancerIPStrings() {
-			if ingress != "" {
+		for _, ingress := range svcInfo.LoadBalancerIngress() {
+			if ingress.IP != "" && (!utilfeature.DefaultFeatureGate.Enabled(features.LoadBalancerIPMode) || *ingress.IPMode == v1.LoadBalancerIPModeVIP) {
 				// ipset call
 				entry = &utilipset.Entry{
-					IP:       ingress,
+					IP:       ingress.IP,
 					Port:     svcInfo.Port(),
 					Protocol: protocol,
 					SetType:  utilipset.HashIPPort,
@@ -1371,7 +1371,7 @@ func (proxier *Proxier) syncProxyRules() {
 					for _, src := range svcInfo.LoadBalancerSourceRanges() {
 						// ipset call
 						entry = &utilipset.Entry{
-							IP:       ingress,
+							IP:       ingress.IP,
 							Port:     svcInfo.Port(),
 							Protocol: protocol,
 							Net:      src,
@@ -1395,10 +1395,10 @@ func (proxier *Proxier) syncProxyRules() {
 					// Need to add the following rule to allow request on host.
 					if allowFromNode {
 						entry = &utilipset.Entry{
-							IP:       ingress,
+							IP:       ingress.IP,
 							Port:     svcInfo.Port(),
 							Protocol: protocol,
-							IP2:      ingress,
+							IP2:      ingress.IP,
 							SetType:  utilipset.HashIPPortIP,
 						}
 						// enumerate all white list source ip
@@ -1412,7 +1412,7 @@ func (proxier *Proxier) syncProxyRules() {
 
 				// ipvs call
 				serv := &utilipvs.VirtualServer{
-					Address:   net.ParseIP(ingress),
+					Address:   net.ParseIP(ingress.IP),
 					Port:      uint16(svcInfo.Port()),
 					Protocol:  string(svcInfo.Protocol()),
 					Scheduler: proxier.ipvsScheduler,
@@ -1957,10 +1957,10 @@ func (proxier *Proxier) deleteEndpointConnections(connectionMap []proxy.ServiceE
 					klog.Errorf("Failed to delete %s endpoint connections for externalIP %s, error: %v", epSvcPair.ServicePortName.String(), extIP, err)
 				}
 			}
-			for _, lbIP := range svcInfo.LoadBalancerIPStrings() {
-				err := conntrack.ClearEntriesForNAT(proxier.exec, lbIP, endpointIP, svcProto)
+			for _, lbIngress := range svcInfo.LoadBalancerIngress() {
+				err := conntrack.ClearEntriesForNAT(proxier.exec, lbIngress.IP, endpointIP, svcProto)
 				if err != nil {
-					klog.Errorf("Failed to delete %s endpoint connections for LoadBalancerIP %s, error: %v", epSvcPair.ServicePortName.String(), lbIP, err)
+					klog.Errorf("Failed to delete %s endpoint connections for LoadBalancerIP %s, error: %v", epSvcPair.ServicePortName.String(), lbIngress.IP, err)
 				}
 			}
 		}

pkg/proxy/ipvs/proxier_test.go

@@ -33,6 +33,8 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/sets"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/proxy"
 	"k8s.io/kubernetes/pkg/proxy/healthcheck"
 	netlinktest "k8s.io/kubernetes/pkg/proxy/ipvs/testing"
@@ -51,6 +53,8 @@ import (
 	utilpointer "k8s.io/utils/pointer"
 
 	utilnet "k8s.io/utils/net"
+
+	featuregatetesting "k8s.io/component-base/featuregate/testing"
 )
 
 const testHostname = "test-hostname"
@@ -4320,3 +4324,81 @@ func TestFilterCIDRs(t *testing.T) {
 		t.Errorf("cidrs %v is not expected %v", cidrs, expected)
 	}
 }
+
+func TestLoadBalancerIngressRouteTypeProxy(t *testing.T) {
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, true)()
+	ipModeProxy := v1.LoadBalancerIPModeProxy
+	ipModeVIP := v1.LoadBalancerIPModeVIP
+
+	testCases := []struct {
+		svcIP            string
+		svcLBIP          string
+		ipMode           *v1.LoadBalancerIPMode
+		expectedServices int
+	}{
+		{
+			svcIP:            "10.20.30.41",
+			svcLBIP:          "1.2.3.4",
+			ipMode:           &ipModeProxy,
+			expectedServices: 1,
+		},
+		{
+			svcIP:            "10.20.30.42",
+			svcLBIP:          "1.2.3.5",
+			ipMode:           &ipModeVIP,
+			expectedServices: 2,
+		},
+	}
+
+	svcPort := 80
+	svcNodePort := 3001
+	svcPortName := proxy.ServicePortName{
+		NamespacedName: makeNSN("ns1", "svc1"),
+		Port:           "p80",
+	}
+
+	for _, testCase := range testCases {
+		_, fp := buildFakeProxier()
+		makeServiceMap(fp,
+			makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) {
+				svc.Spec.Type = "LoadBalancer"
+				svc.Spec.ClusterIP = testCase.svcIP
+				svc.Spec.Ports = []v1.ServicePort{{
+					Name:     svcPortName.Port,
+					Port:     int32(svcPort),
+					Protocol: v1.ProtocolTCP,
+					NodePort: int32(svcNodePort),
+				}}
+				svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
+					IP:     testCase.svcLBIP,
+					IPMode: testCase.ipMode,
+				}}
+			}),
+		)
+
+		epIP := "10.180.0.1"
+		makeEndpointsMap(fp,
+			makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *v1.Endpoints) {
+				ept.Subsets = []v1.EndpointSubset{{
+					Addresses: []v1.EndpointAddress{{
+						IP: epIP,
+					}},
+					Ports: []v1.EndpointPort{{
+						Name: svcPortName.Port,
+						Port: int32(svcPort),
+					}},
+				}}
+			}),
+		)
+
+		fp.syncProxyRules()
+
+		services, err := fp.ipvs.GetVirtualServers()
+		if err != nil {
+			t.Errorf("Failed to get ipvs services, err: %v", err)
+		}
+		if len(services) != testCase.expectedServices {
+			t.Errorf("Expected %d ipvs services, got %d", testCase.expectedServices, len(services))
+		}
+	}
+}

pkg/proxy/service.go

@@ -105,13 +105,9 @@ func (info *BaseServiceInfo) ExternalIPStrings() []string {
 	return info.externalIPs
 }
 
-// LoadBalancerIPStrings is part of ServicePort interface.
-func (info *BaseServiceInfo) LoadBalancerIPStrings() []string {
-	var ips []string
-	for _, ing := range info.loadBalancerStatus.Ingress {
-		ips = append(ips, ing.IP)
-	}
-	return ips
+// LoadBalancerIngress is part of ServicePort interface.
+func (info *BaseServiceInfo) LoadBalancerIngress() []v1.LoadBalancerIngress {
+	return info.loadBalancerStatus.Ingress
 }
 
 // OnlyNodeLocalEndpoints is part of ServicePort interface.
@@ -167,23 +163,16 @@ func (sct *ServiceChangeTracker) newBaseServiceInfo(port *v1.ServicePort, servic
 		klog.V(4).Infof("service change tracker(%v) ignored the following load balancer source ranges(%s) for service %v/%v as they don't match IPFamily", sct.ipFamily, strings.Join(incorrectIPs, ","), service.Namespace, service.Name)
 	}
 
-	// Obtain Load Balancer Ingress IPs
-	var ips []string
-	for _, ing := range service.Status.LoadBalancer.Ingress {
-		ips = append(ips, ing.IP)
+	correctIngresses, incorrectIngresses := utilproxy.FilterIncorrectLoadBalancerIngress(service.Status.LoadBalancer.Ingress, sct.ipFamily)
+
+	info.loadBalancerStatus.Ingress = correctIngresses
+
+	if len(incorrectIngresses) > 0 {
+		var incorrectIPs []string
+		for _, incorrectIng := range incorrectIngresses {
+			incorrectIPs = append(incorrectIPs, incorrectIng.IP)
 		}
-
-	if len(ips) > 0 {
-		correctIPs, incorrectIPs := utilproxy.FilterIncorrectIPVersion(ips, sct.ipFamily)
-
-		if len(incorrectIPs) > 0 {
 		klog.V(4).Infof("service change tracker(%v) ignored the following load balancer(%s) ingress ips for service %v/%v as they don't match IPFamily", sct.ipFamily, strings.Join(incorrectIPs, ","), service.Namespace, service.Name)
-
-		}
-		// Create the LoadBalancerStatus with the filtered IPs
-		for _, ip := range correctIPs {
-			info.loadBalancerStatus.Ingress = append(info.loadBalancerStatus.Ingress, v1.LoadBalancerIngress{IP: ip})
-		}
 	}
 
 	if apiservice.NeedsHealthCheck(service) {

pkg/proxy/types.go

@@ -73,8 +73,8 @@ type ServicePort interface {
 	StickyMaxAgeSeconds() int
 	// ExternalIPStrings returns service ExternalIPs as a string array.
 	ExternalIPStrings() []string
-	// LoadBalancerIPStrings returns service LoadBalancerIPs as a string array.
-	LoadBalancerIPStrings() []string
+	// LoadBalancerIngress returns service an array of LoadBalancerIngress.
+	LoadBalancerIngress() []v1.LoadBalancerIngress
 	// GetProtocol returns service protocol.
 	Protocol() v1.Protocol
 	// LoadBalancerSourceRanges returns service LoadBalancerSourceRanges if present empty array if not

pkg/proxy/util/utils.go

@@ -403,3 +403,26 @@ func GetClusterIPByFamily(ipFamily v1.IPFamily, service *v1.Service) string {
 
 	return ""
 }
+
+// FilterIncorrectLoadBalancerIngress filters out the ingresses with an IP version different from the given one
+func FilterIncorrectLoadBalancerIngress(ingresses []v1.LoadBalancerIngress, ipFamily v1.IPFamily) ([]v1.LoadBalancerIngress, []v1.LoadBalancerIngress) {
+	var validIngresses []v1.LoadBalancerIngress
+	var invalidIngresses []v1.LoadBalancerIngress
+
+	for _, ing := range ingresses {
+		// []string{ing.IP} have a len of 1, so len(correctIPs) + len(incorrectIPs) == 1
+		correctIPs, _ := FilterIncorrectIPVersion([]string{ing.IP}, ipFamily)
+
+		// len is either 1 or 0
+		if len(correctIPs) == 1 {
+			// Update the LoadBalancerStatus with the filtered IP
+			validIngresses = append(validIngresses, ing)
+			continue
+		}
+
+		// here len(incorrectIPs) == 1 since len(correctIPs) == 0
+		invalidIngresses = append(invalidIngresses, ing)
+	}
+
+	return validIngresses, invalidIngresses
+}

pkg/proxy/util/utils_test.go

@@ -822,3 +822,179 @@ func TestGetClusterIPByFamily(t *testing.T) {
 	}
 
 }
+
+func TestFilterIncorrectLoadBalancerIngress(t *testing.T) {
+	ipModeVIP := v1.LoadBalancerIPModeVIP
+	testCases := []struct {
+		name              string
+		ingresses         []v1.LoadBalancerIngress
+		ipFamily          v1.IPFamily
+		expectedCorrect   []v1.LoadBalancerIngress
+		expectedIncorrect []v1.LoadBalancerIngress
+	}{
+		{
+			name:     "IPv4 only valid ingresses",
+			ipFamily: v1.IPv4Protocol,
+			ingresses: []v1.LoadBalancerIngress{
+				{
+					IP:     "1.2.3.4",
+					IPMode: &ipModeVIP,
+				},
+				{
+					IP: "1.2.3.5",
+				},
+			},
+			expectedCorrect: []v1.LoadBalancerIngress{
+				{
+					IP:     "1.2.3.4",
+					IPMode: &ipModeVIP,
+				},
+				{
+					IP: "1.2.3.5",
+				},
+			},
+			expectedIncorrect: nil,
+		},
+		{
+			name:     "IPv4 some invalid ingresses",
+			ipFamily: v1.IPv4Protocol,
+			ingresses: []v1.LoadBalancerIngress{
+				{
+					IP:     "1.2.3.4",
+					IPMode: &ipModeVIP,
+				},
+				{
+					IP: "2000::1",
+				},
+				{
+					Hostname: "dummy",
+				},
+			},
+			expectedCorrect: []v1.LoadBalancerIngress{
+				{
+					IP:     "1.2.3.4",
+					IPMode: &ipModeVIP,
+				},
+				{
+					Hostname: "dummy", // weirdly no IP is a valid IPv4 but invalid IPv6
+				},
+			},
+			expectedIncorrect: []v1.LoadBalancerIngress{
+				{
+					IP: "2000::1",
+				},
+			},
+		},
+		{
+			name:     "IPv4 only invalid ingresses",
+			ipFamily: v1.IPv4Protocol,
+			ingresses: []v1.LoadBalancerIngress{
+				{
+					IP: "2000::1",
+				},
+				{
+					IP:     "2000::1",
+					IPMode: &ipModeVIP,
+				},
+			},
+			expectedCorrect: nil,
+			expectedIncorrect: []v1.LoadBalancerIngress{
+				{
+					IP: "2000::1",
+				},
+				{
+					IP:     "2000::1",
+					IPMode: &ipModeVIP,
+				},
+			},
+		},
+		{
+			name:     "IPv6 only valid ingresses",
+			ipFamily: v1.IPv6Protocol,
+			ingresses: []v1.LoadBalancerIngress{
+				{
+					IP:     "2000::1",
+					IPMode: &ipModeVIP,
+				},
+				{
+					IP: "2000::2",
+				},
+			},
+			expectedCorrect: []v1.LoadBalancerIngress{
+				{
+					IP:     "2000::1",
+					IPMode: &ipModeVIP,
+				},
+				{
+					IP: "2000::2",
+				},
+			},
+			expectedIncorrect: nil,
+		},
+		{
+			name:     "IPv6 some invalid ingresses",
+			ipFamily: v1.IPv6Protocol,
+			ingresses: []v1.LoadBalancerIngress{
+				{
+					IP:     "2000::1",
+					IPMode: &ipModeVIP,
+				},
+				{
+					IP: "1.2.3.4",
+				},
+				{
+					Hostname: "dummy",
+				},
+			},
+			expectedCorrect: []v1.LoadBalancerIngress{
+				{
+					IP:     "2000::1",
+					IPMode: &ipModeVIP,
+				},
+			},
+			expectedIncorrect: []v1.LoadBalancerIngress{
+				{
+					IP: "1.2.3.4",
+				},
+				{
+					Hostname: "dummy", // weirdly no IP is a valid IPv4 but invalid IPv6
+				},
+			},
+		},
+		{
+			name:     "IPv6 only invalid ingresses",
+			ipFamily: v1.IPv6Protocol,
+			ingresses: []v1.LoadBalancerIngress{
+				{
+					IP: "1.2.3.4",
+				},
+				{
+					IP:     "1.2.3.5",
+					IPMode: &ipModeVIP,
+				},
+			},
+			expectedCorrect: nil,
+			expectedIncorrect: []v1.LoadBalancerIngress{
+				{
+					IP: "1.2.3.4",
+				},
+				{
+					IP:     "1.2.3.5",
+					IPMode: &ipModeVIP,
+				},
+			},
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			correctIngresses, incorrectIngresses := FilterIncorrectLoadBalancerIngress(testCase.ingresses, testCase.ipFamily)
+			if !reflect.DeepEqual(correctIngresses, testCase.expectedCorrect) {
+				t.Errorf("Test %v failed: expected %v, got %v", testCase.name, testCase.expectedCorrect, correctIngresses)
+			}
+			if !reflect.DeepEqual(incorrectIngresses, testCase.expectedIncorrect) {
+				t.Errorf("Test %v failed: expected %v, got %v", testCase.name, testCase.expectedIncorrect, incorrectIngresses)
+			}
+		})
+	}
+}

pkg/registry/core/service/strategy.go

@@ -179,6 +179,12 @@ func dropServiceDisabledFields(newSvc *api.Service, oldSvc *api.Service) {
 	if !utilfeature.DefaultFeatureGate.Enabled(features.ServiceTopology) && !topologyKeysInUse(oldSvc) {
 		newSvc.Spec.TopologyKeys = nil
 	}
+
+	if !utilfeature.DefaultFeatureGate.Enabled(features.LoadBalancerIPMode) && !loadbalancerIPModeInUse(oldSvc) {
+		for _, ing := range newSvc.Status.LoadBalancer.Ingress {
+			ing.IPMode = nil
+		}
+	}
 }
 
 // returns true if svc.Spec.ServiceIPFamily field is in use
@@ -202,6 +208,19 @@ func topologyKeysInUse(svc *api.Service) bool {
 	return len(svc.Spec.TopologyKeys) > 0
 }
 
+// returns true when the LoadBalancer Ingress IPMode fields are in use.
+func loadbalancerIPModeInUse(svc *api.Service) bool {
+	if svc == nil {
+		return false
+	}
+	for _, ing := range svc.Status.LoadBalancer.Ingress {
+		if ing.IPMode != nil {
+			return true
+		}
+	}
+	return false
+}
+
 type serviceStatusStrategy struct {
 	Strategy
 }

staging/src/k8s.io/api/core/v1/generated.proto

@@ -2036,6 +2036,15 @@ message LoadBalancerIngress {
   // (typically AWS load-balancers)
   // +optional
   optional string hostname = 2;
+
+  // IPMode specifies how the load-balancer's IP behaves.
+  // Setting this to "VIP" indicates that the traffic passing through
+  // this load-balancer is delivered with the destination IP and port set to the load-balancer's IP and port.
+  // Setting this to "Proxy" indicates that the load-balancer acts like a proxy,
+  // delivering traffic with the destination IP and port set to the node's IP and nodePort or to the pod's IP and targetPort.
+  // This field can only be set when the ip field is also set, and defaults to "VIP" if not specified.
+  // +optional
+  optional string ipMode = 3;
 }
 
 // LoadBalancerStatus represents the status of a load-balancer.

staging/src/k8s.io/api/core/v1/types.go

@@ -3971,6 +3971,15 @@ type LoadBalancerIngress struct {
 	// (typically AWS load-balancers)
 	// +optional
 	Hostname string `json:"hostname,omitempty" protobuf:"bytes,2,opt,name=hostname"`
+
+	// IPMode specifies how the load-balancer's IP behaves.
+	// Setting this to "VIP" indicates that the traffic passing through
+	// this load-balancer is delivered with the destination IP and port set to the load-balancer's IP and port.
+	// Setting this to "Proxy" indicates that the load-balancer acts like a proxy,
+	// delivering traffic with the destination IP and port set to the node's IP and nodePort or to the pod's IP and targetPort.
+	// This field can only be set when the ip field is also set, and defaults to "VIP" if not specified.
+	// +optional
+	IPMode *LoadBalancerIPMode `json:"ipMode,omitempty" protobuf:"bytes,3,opt,name=ipMode"`
 }
 
 const (
@@ -3978,6 +3987,18 @@ const (
 	MaxServiceTopologyKeys = 16
 )
 
+// LoadBalancerIPMode represents the mode of the LoadBalancer ingress IP
+type LoadBalancerIPMode string
+
+const (
+	// LoadBalancerIPModeVIP indicates that the traffic passing through this LoadBalancer
+	// is delivered with the destination IP set to the specified LoadBalancer IP
+	LoadBalancerIPModeVIP LoadBalancerIPMode = "VIP"
+	// LoadBalancerIPModeProxy indicates that the specified LoadBalancer acts like a proxy,
+	// changing the destination IP to the node IP and the source IP to the LoadBalancer (mostly private) IP
+	LoadBalancerIPModeProxy LoadBalancerIPMode = "Proxy"
+)
+
 // IPFamily represents the IP Family (IPv4 or IPv6). This type is used
 // to express the family of an IP expressed by a type (e.g. service.spec.ipFamilies).
 type IPFamily string

staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go

@@ -953,6 +953,7 @@ var map_LoadBalancerIngress = map[string]string{
 	"":         "LoadBalancerIngress represents the status of a load-balancer ingress point: traffic intended for the service should be sent to an ingress point.",
 	"ip":       "IP is set for load-balancer ingress points that are IP based (typically GCE or OpenStack load-balancers)",
 	"hostname": "Hostname is set for load-balancer ingress points that are DNS based (typically AWS load-balancers)",
+	"ipMode":   "IPMode specifies how the load-balancer's IP behaves. Setting this to \"VIP\" indicates that the traffic passing through this load-balancer is delivered with the destination IP and port set to the load-balancer's IP and port. Setting this to \"Proxy\" indicates that the load-balancer acts like a proxy, delivering traffic with the destination IP and port set to the node's IP and nodePort or to the pod's IP and targetPort. This field can only be set when the ip field is also set, and defaults to \"VIP\" if not specified.",
 }
 
 func (LoadBalancerIngress) SwaggerDoc() map[string]string {

staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go

@@ -2144,6 +2144,11 @@ func (in *List) DeepCopyObject() runtime.Object {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *LoadBalancerIngress) DeepCopyInto(out *LoadBalancerIngress) {
 	*out = *in
+	if in.IPMode != nil {
+		in, out := &in.IPMode, &out.IPMode
+		*out = new(LoadBalancerIPMode)
+		**out = **in
+	}
 	return
 }
 
@@ -2163,7 +2168,9 @@ func (in *LoadBalancerStatus) DeepCopyInto(out *LoadBalancerStatus) {
 	if in.Ingress != nil {
 		in, out := &in.Ingress, &out.Ingress
 		*out = make([]LoadBalancerIngress, len(*in))
-		copy(*out, *in)
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
 	}
 	return
 }

staging/src/k8s.io/api/testdata/HEAD/core.v1.Service.json

@@ -89,7 +89,8 @@
       "ingress": [
         {
           "ip": "31",
-          "hostname": "32"
+          "hostname": "32",
+          "ipMode": "ƕP喂ƈ"
         }
       ]
     }

staging/src/k8s.io/api/testdata/HEAD/core.v1.Service.yaml

@@ -66,3 +66,4 @@ status:
     ingress:
     - hostname: "32"
       ip: "31"
+      ipMode: ƕP喂ƈ

staging/src/k8s.io/api/testdata/HEAD/extensions.v1beta1.Ingress.json

@@ -87,7 +87,8 @@
       "ingress": [
         {
           "ip": "33",
-          "hostname": "34"
+          "hostname": "34",
+          "ipMode": "ěĂ凗蓏Ŋ蛊ĉy緅縕"
         }
       ]
     }

staging/src/k8s.io/api/testdata/HEAD/extensions.v1beta1.Ingress.yaml

@@ -60,3 +60,4 @@ status:
     ingress:
     - hostname: "34"
       ip: "33"
+      ipMode: ěĂ凗蓏Ŋ蛊ĉy緅縕

staging/src/k8s.io/api/testdata/HEAD/networking.k8s.io.v1.Ingress.json

@@ -97,7 +97,8 @@
       "ingress": [
         {
           "ip": "34",
-          "hostname": "35"
+          "hostname": "35",
+          "ipMode": "ɑ"
         }
       ]
     }

staging/src/k8s.io/api/testdata/HEAD/networking.k8s.io.v1.Ingress.yaml

@@ -66,3 +66,4 @@ status:
     ingress:
     - hostname: "35"
       ip: "34"
+      ipMode: ɑ

staging/src/k8s.io/api/testdata/HEAD/networking.k8s.io.v1beta1.Ingress.json

@@ -87,7 +87,8 @@
       "ingress": [
         {
           "ip": "33",
-          "hostname": "34"
+          "hostname": "34",
+          "ipMode": "ěĂ凗蓏Ŋ蛊ĉy緅縕"
         }
       ]
     }

staging/src/k8s.io/api/testdata/HEAD/networking.k8s.io.v1beta1.Ingress.yaml

@@ -60,3 +60,4 @@ status:
     ingress:
     - hostname: "34"
       ip: "33"
+      ipMode: ěĂ凗蓏Ŋ蛊ĉy緅縕

staging/src/k8s.io/cloud-provider/service/helpers/helper.go

@@ -180,5 +180,8 @@ func ingressEqual(lhs, rhs *v1.LoadBalancerIngress) bool {
 	if lhs.Hostname != rhs.Hostname {
 		return false
 	}
+	if lhs.IPMode != rhs.IPMode {
+		return false
+	}
 	return true
 }