diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_backoff.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_backoff.go
index ec4e49c7810..097795b3ed4 100644
--- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_backoff.go
+++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_backoff.go
@@ -36,6 +36,9 @@ import (
 const (
 	// not active means the instance is under deleting from Azure VMSS.
 	vmssVMNotActiveErrorMessage = "not an active Virtual Machine Scale Set VM instanceId"
+
+	// operationCancledErrorMessage means the operation is canceled by another new operation.
+	operationCancledErrorMessage = "canceledandsupersededduetoanotheroperation"
 )
 
 // RequestBackoff if backoff is disabled in cloud provider it
@@ -197,6 +200,12 @@ func (az *Cloud) CreateOrUpdateSecurityGroup(service *v1.Service, sg network.Sec
 		if resp != nil && resp.StatusCode == http.StatusPreconditionFailed {
 			az.nsgCache.Delete(*sg.Name)
 		}
+
+		// Invalidate the cache because another new operation has canceled the current request.
+		if strings.Contains(strings.ToLower(err.Error()), operationCancledErrorMessage) {
+			az.nsgCache.Delete(*sg.Name)
+		}
+
 		return err
 	}
 
@@ -222,6 +231,13 @@ func (az *Cloud) CreateOrUpdateSGWithRetry(service *v1.Service, sg network.Secur
 			az.nsgCache.Delete(*sg.Name)
 			return true, err
 		}
+
+		// Invalidate the cache and abort backoff because another new operation has canceled the current request.
+		if err != nil && strings.Contains(strings.ToLower(err.Error()), operationCancledErrorMessage) {
+			az.nsgCache.Delete(*sg.Name)
+			return true, err
+		}
+
 		return done, retryError
 	})
 }
@@ -248,6 +264,10 @@ func (az *Cloud) CreateOrUpdateLB(service *v1.Service, lb network.LoadBalancer)
 		if resp != nil && resp.StatusCode == http.StatusPreconditionFailed {
 			az.lbCache.Delete(*lb.Name)
 		}
+		// Invalidate the cache because another new operation has canceled the current request.
+		if strings.Contains(strings.ToLower(err.Error()), operationCancledErrorMessage) {
+			az.lbCache.Delete(*lb.Name)
+		}
 		return err
 	}
 
@@ -271,7 +291,12 @@ func (az *Cloud) createOrUpdateLBWithRetry(service *v1.Service, lb network.LoadB
 
 		// Invalidate the cache and abort backoff because ETAG precondition mismatch.
 		if resp != nil && resp.StatusCode == http.StatusPreconditionFailed {
-			az.nsgCache.Delete(*lb.Name)
+			az.lbCache.Delete(*lb.Name)
+			return true, err
+		}
+		// Invalidate the cache and abort backoff because another new operation has canceled the current request.
+		if err != nil && strings.Contains(strings.ToLower(err.Error()), operationCancledErrorMessage) {
+			az.lbCache.Delete(*lb.Name)
 			return true, err
 		}
 		return done, retryError
@@ -497,6 +522,10 @@ func (az *Cloud) CreateOrUpdateRouteTable(routeTable network.RouteTable) error {
 		if resp != nil && resp.StatusCode == http.StatusPreconditionFailed {
 			az.rtCache.Delete(*routeTable.Name)
 		}
+		// Invalidate the cache because another new operation has canceled the current request.
+		if err != nil && strings.Contains(strings.ToLower(err.Error()), operationCancledErrorMessage) {
+			az.rtCache.Delete(*routeTable.Name)
+		}
 		return az.processHTTPResponse(nil, "", resp, err)
 	}
 
@@ -521,6 +550,11 @@ func (az *Cloud) createOrUpdateRouteTableWithRetry(routeTable network.RouteTable
 			az.rtCache.Delete(*routeTable.Name)
 			return true, err
 		}
+		// Invalidate the cache and abort backoff because another new operation has canceled the current request.
+		if err != nil && strings.Contains(strings.ToLower(err.Error()), operationCancledErrorMessage) {
+			az.rtCache.Delete(*routeTable.Name)
+			return true, err
+		}
 		return done, retryError
 	})
 }
@@ -536,6 +570,10 @@ func (az *Cloud) CreateOrUpdateRoute(route network.Route) error {
 		if resp != nil && resp.StatusCode == http.StatusPreconditionFailed {
 			az.rtCache.Delete(az.RouteTableName)
 		}
+		// Invalidate the cache because another new operation has canceled the current request.
+		if err != nil && strings.Contains(strings.ToLower(err.Error()), operationCancledErrorMessage) {
+			az.rtCache.Delete(az.RouteTableName)
+		}
 		return az.processHTTPResponse(nil, "", resp, err)
 	}
 
@@ -561,6 +599,13 @@ func (az *Cloud) createOrUpdateRouteWithRetry(route network.Route) error {
 			az.rtCache.Delete(az.RouteTableName)
 			return true, err
 		}
+
+		// Invalidate the cache and abort backoff because another new operation has canceled the current request.
+		if err != nil && strings.Contains(strings.ToLower(err.Error()), operationCancledErrorMessage) {
+			az.rtCache.Delete(az.RouteTableName)
+			return true, err
+		}
+
 		return done, retryError
 	})
 }
diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go
index c5481326fb3..d9a3418be87 100644
--- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go
+++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go
@@ -888,6 +888,8 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
 	if wantLb && nodes != nil {
 		// Add the machines to the backend pool if they're not already
 		vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName)
+		// Etag would be changed when updating backend pools, so invalidate lbCache after it.
+		defer az.lbCache.Delete(lbName)
 		err := az.vmSet.EnsureHostsInPool(service, nodes, lbBackendPoolID, vmSetName, isInternal)
 		if err != nil {
 			return nil, err