diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go
index a7cf38fdf08..e315086d24b 100644
--- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go
+++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go
@@ -1641,7 +1641,7 @@ func equalLoadBalancingRulePropertiesFormat(s *network.LoadBalancingRuleProperti
 		reflect.DeepEqual(s.EnableTCPReset, t.EnableTCPReset) &&
 		reflect.DeepEqual(s.DisableOutboundSnat, t.DisableOutboundSnat)
 
-	if wantLB {
+	if wantLB && s.IdleTimeoutInMinutes != nil && t.IdleTimeoutInMinutes != nil {
 		return properties && reflect.DeepEqual(s.IdleTimeoutInMinutes, t.IdleTimeoutInMinutes)
 	}
 	return properties
diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer_test.go
index 7560410bf58..fc65aefb712 100644
--- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer_test.go
+++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer_test.go
@@ -173,6 +173,40 @@ func TestFindRule(t *testing.T) {
 			},
 			expected: false,
 		},
+		{
+			msg: "rule names match while idletimeout unmatch should return false",
+			existingRule: []network.LoadBalancingRule{
+				{
+					Name: to.StringPtr("httpRule"),
+					LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
+						IdleTimeoutInMinutes: to.Int32Ptr(1),
+					},
+				},
+			},
+			curRule: network.LoadBalancingRule{
+				Name: to.StringPtr("httpRule"),
+				LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
+					IdleTimeoutInMinutes: to.Int32Ptr(2),
+				},
+			},
+			expected: false,
+		},
+		{
+			msg: "rule names match while idletimeout nil should return true",
+			existingRule: []network.LoadBalancingRule{
+				{
+					Name:                              to.StringPtr("httpRule"),
+					LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{},
+				},
+			},
+			curRule: network.LoadBalancingRule{
+				Name: to.StringPtr("httpRule"),
+				LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
+					IdleTimeoutInMinutes: to.Int32Ptr(2),
+				},
+			},
+			expected: true,
+		},
 		{
 			msg: "rule names match while LoadDistribution unmatch should return false",
 			existingRule: []network.LoadBalancingRule{