diff --git a/pkg/cloudprovider/providers/azure/azure_loadbalancer.go b/pkg/cloudprovider/providers/azure/azure_loadbalancer.go
index c88860991bd..62c827960b4 100644
--- a/pkg/cloudprovider/providers/azure/azure_loadbalancer.go
+++ b/pkg/cloudprovider/providers/azure/azure_loadbalancer.go
@@ -604,7 +604,7 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
 	lbBackendPoolID := az.getBackendPoolID(lbName, lbBackendPoolName)
 
 	lbIdleTimeout, err := getIdleTimeout(service)
-	if err != nil {
+	if wantLb && err != nil {
 		return nil, err
 	}
 
@@ -789,7 +789,7 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
 		if az.serviceOwnsRule(service, *existingRule.Name) {
 			keepRule := false
 			klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb rule(%s) - considering evicting", serviceName, wantLb, *existingRule.Name)
-			if findRule(expectedRules, existingRule) {
+			if findRule(expectedRules, existingRule, wantLb) {
 				klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb rule(%s) - keeping", serviceName, wantLb, *existingRule.Name)
 				keepRule = true
 			}
@@ -803,7 +803,7 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
 	// update rules: add needed
 	for _, expectedRule := range expectedRules {
 		foundRule := false
-		if findRule(updatedRules, expectedRule) {
+		if findRule(updatedRules, expectedRule, wantLb) {
 			klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb rule(%s) - already exists", serviceName, wantLb, *expectedRule.Name)
 			foundRule = true
 		}
@@ -1474,10 +1474,10 @@ func findProbe(probes []network.Probe, probe network.Probe) bool {
 	return false
 }
 
-func findRule(rules []network.LoadBalancingRule, rule network.LoadBalancingRule) bool {
+func findRule(rules []network.LoadBalancingRule, rule network.LoadBalancingRule, wantLB bool) bool {
 	for _, existingRule := range rules {
 		if strings.EqualFold(to.String(existingRule.Name), to.String(rule.Name)) &&
-			equalLoadBalancingRulePropertiesFormat(existingRule.LoadBalancingRulePropertiesFormat, rule.LoadBalancingRulePropertiesFormat) {
+			equalLoadBalancingRulePropertiesFormat(existingRule.LoadBalancingRulePropertiesFormat, rule.LoadBalancingRulePropertiesFormat, wantLB) {
 			return true
 		}
 	}
@@ -1486,19 +1486,23 @@ func findRule(rules []network.LoadBalancingRule, rule network.LoadBalancingRule)
 
 // equalLoadBalancingRulePropertiesFormat checks whether the provided LoadBalancingRulePropertiesFormat are equal.
 // Note: only fields used in reconcileLoadBalancer are considered.
-func equalLoadBalancingRulePropertiesFormat(s, t *network.LoadBalancingRulePropertiesFormat) bool {
+func equalLoadBalancingRulePropertiesFormat(s *network.LoadBalancingRulePropertiesFormat, t *network.LoadBalancingRulePropertiesFormat, wantLB bool) bool {
 	if s == nil || t == nil {
 		return false
 	}
 
-	return reflect.DeepEqual(s.Protocol, t.Protocol) &&
+	properties := reflect.DeepEqual(s.Protocol, t.Protocol) &&
 		reflect.DeepEqual(s.FrontendIPConfiguration, t.FrontendIPConfiguration) &&
 		reflect.DeepEqual(s.BackendAddressPool, t.BackendAddressPool) &&
 		reflect.DeepEqual(s.LoadDistribution, t.LoadDistribution) &&
 		reflect.DeepEqual(s.FrontendPort, t.FrontendPort) &&
 		reflect.DeepEqual(s.BackendPort, t.BackendPort) &&
-		reflect.DeepEqual(s.EnableFloatingIP, t.EnableFloatingIP) &&
-		reflect.DeepEqual(s.IdleTimeoutInMinutes, t.IdleTimeoutInMinutes)
+		reflect.DeepEqual(s.EnableFloatingIP, t.EnableFloatingIP)
+
+	if wantLB {
+		return properties && reflect.DeepEqual(s.IdleTimeoutInMinutes, t.IdleTimeoutInMinutes)
+	}
+	return properties
 }
 
 // This compares rule's Name, Protocol, SourcePortRange, DestinationPortRange, SourceAddressPrefix, Access, and Direction.
diff --git a/pkg/cloudprovider/providers/azure/azure_loadbalancer_test.go b/pkg/cloudprovider/providers/azure/azure_loadbalancer_test.go
index f1368575b6d..06c26291c22 100644
--- a/pkg/cloudprovider/providers/azure/azure_loadbalancer_test.go
+++ b/pkg/cloudprovider/providers/azure/azure_loadbalancer_test.go
@@ -210,7 +210,7 @@ func TestFindRule(t *testing.T) {
 	}
 
 	for i, test := range tests {
-		findResult := findRule(test.existingRule, test.curRule)
+		findResult := findRule(test.existingRule, test.curRule, true)
 		assert.Equal(t, test.expected, findResult, fmt.Sprintf("TestCase[%d]: %s", i, test.msg))
 	}
 }