diff --git a/pkg/apis/core/v1/defaults.go b/pkg/apis/core/v1/defaults.go
index 5e8ef589b28..d23e31557d2 100644
--- a/pkg/apis/core/v1/defaults.go
+++ b/pkg/apis/core/v1/defaults.go
@@ -171,9 +171,9 @@ func SetDefaults_Service(obj *v1.Service) {
 		obj.Spec.Type == v1.ServiceTypeLoadBalancer {
 		ipMode := v1.LoadBalancerIPModeVIP
 
-		for _, ing := range obj.Status.LoadBalancer.Ingress {
-			if ing.IPMode == nil {
-				ing.IPMode = &ipMode
+		for i, ing := range obj.Status.LoadBalancer.Ingress {
+			if ing.IP != "" && ing.IPMode == nil {
+				obj.Status.LoadBalancer.Ingress[i].IPMode = &ipMode
 			}
 		}
 	}
diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go
index 6b3bd48432b..e4ba681d2a9 100644
--- a/pkg/proxy/iptables/proxier.go
+++ b/pkg/proxy/iptables/proxier.go
@@ -1175,7 +1175,7 @@ func (proxier *Proxier) syncProxyRules() {
 					// This currently works for loadbalancers that preserves source ips.
 					// For loadbalancers which direct traffic to service NodePort, the firewall rules will not apply.
 
-					if !utilfeature.DefaultFeatureGate.Enabled(features.LoadBalancerIPMode) || ingress.IPMode == nil || *ingress.IPMode == v1.LoadBalancerIPModeVIP {
+					if !utilfeature.DefaultFeatureGate.Enabled(features.LoadBalancerIPMode) || *ingress.IPMode == v1.LoadBalancerIPModeVIP {
 						args = append(args[:0],
 							"-A", string(kubeServicesChain),
 							"-m", "comment", "--comment", fmt.Sprintf(`"%s loadbalancer IP"`, svcNameString),
diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go
index 4a481a5b330..513a66ead51 100644
--- a/pkg/proxy/ipvs/proxier.go
+++ b/pkg/proxy/ipvs/proxier.go
@@ -1333,7 +1333,7 @@ func (proxier *Proxier) syncProxyRules() {
 
 		// Capture load-balancer ingress.
 		for _, ingress := range svcInfo.LoadBalancerIngress() {
-			if ingress.IP != "" && (!utilfeature.DefaultFeatureGate.Enabled(features.LoadBalancerIPMode) || ingress.IPMode == nil || *ingress.IPMode == v1.LoadBalancerIPModeVIP) {
+			if ingress.IP != "" && (!utilfeature.DefaultFeatureGate.Enabled(features.LoadBalancerIPMode) || *ingress.IPMode == v1.LoadBalancerIPModeVIP) {
 				// ipset call
 				entry = &utilipset.Entry{
 					IP:       ingress.IP,