diff --git a/pkg/cloudprovider/providers/azure/azure_loadbalancer.go b/pkg/cloudprovider/providers/azure/azure_loadbalancer.go
index 44b59062314..cad0c2ceae1 100644
--- a/pkg/cloudprovider/providers/azure/azure_loadbalancer.go
+++ b/pkg/cloudprovider/providers/azure/azure_loadbalancer.go
@@ -781,7 +781,7 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
 			// Remove backend pools from vmSets. This is required for virtual machine scale sets before removing the LB.
 			vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName)
 			glog.V(10).Infof("EnsureBackendPoolDeleted(%s, %s): start", lbBackendPoolID, vmSetName)
-			err := az.vmSet.EnsureBackendPoolDeleted(lbBackendPoolID, vmSetName)
+			err := az.vmSet.EnsureBackendPoolDeleted(lbBackendPoolID, vmSetName, lb.BackendAddressPools)
 			if err != nil {
 				glog.Errorf("EnsureBackendPoolDeleted(%s, %s) failed: %v", lbBackendPoolID, vmSetName, err)
 				return nil, err
diff --git a/pkg/cloudprovider/providers/azure/azure_standard.go b/pkg/cloudprovider/providers/azure/azure_standard.go
index 6b43a58deb3..02c7c103a34 100644
--- a/pkg/cloudprovider/providers/azure/azure_standard.go
+++ b/pkg/cloudprovider/providers/azure/azure_standard.go
@@ -110,12 +110,6 @@ func (az *Cloud) getLoadBalancerProbeID(lbName, lbRuleName string) string {
 }
 
 func (az *Cloud) mapLoadBalancerNameToVMSet(lbName string, clusterName string) (vmSetName string) {
-	// Backends of Standard load balancer could belong to multiple VMAS or VMSS.
-	// Return "" to indicate so that following logic won't check this.
-	if az.useStandardLoadBalancer() {
-		return ""
-	}
-
 	vmSetName = strings.TrimSuffix(lbName, InternalLoadBalancerNameSuffix)
 	if strings.EqualFold(clusterName, vmSetName) {
 		vmSetName = az.vmSet.GetPrimaryVMSetName()
@@ -579,8 +573,9 @@ func (as *availabilitySet) GetPrimaryInterface(nodeName, vmSetName string) (netw
 		return network.Interface{}, err
 	}
 
-	// Check availability set
-	if vmSetName != "" {
+	// Check availability set.
+	// Backends of Standard load balancer could belong to multiple VMAS, so we don't check vmSet for it.
+	if vmSetName != "" && !as.useStandardLoadBalancer() {
 		expectedAvailabilitySetName := as.getAvailabilitySetID(vmSetName)
 		if machine.AvailabilitySet == nil || !strings.EqualFold(*machine.AvailabilitySet.ID, expectedAvailabilitySetName) {
 			glog.V(3).Infof(
@@ -704,7 +699,7 @@ func (as *availabilitySet) EnsureHostsInPool(serviceName string, nodes []*v1.Nod
 }
 
 // EnsureBackendPoolDeleted ensures the loadBalancer backendAddressPools deleted from the specified vmSet.
-func (as *availabilitySet) EnsureBackendPoolDeleted(poolID, vmSetName string) error {
+func (as *availabilitySet) EnsureBackendPoolDeleted(poolID, vmSetName string, backendAddressPools *[]network.BackendAddressPool) error {
 	// Do nothing for availability set.
 	return nil
 }
diff --git a/pkg/cloudprovider/providers/azure/azure_standard_test.go b/pkg/cloudprovider/providers/azure/azure_standard_test.go
index 4d6e7c24572..d5071192fbe 100644
--- a/pkg/cloudprovider/providers/azure/azure_standard_test.go
+++ b/pkg/cloudprovider/providers/azure/azure_standard_test.go
@@ -158,34 +158,6 @@ func TestMapLoadBalancerNameToVMSet(t *testing.T) {
 			clusterName:   "azure",
 			expectedVMSet: "azuretest",
 		},
-		{
-			description:   "default standard external LB should map to empty string",
-			lbName:        "azure",
-			useStandardLB: true,
-			clusterName:   "azure",
-			expectedVMSet: "",
-		},
-		{
-			description:   "default standard internal LB should map to empty string",
-			lbName:        "azure-internal",
-			useStandardLB: true,
-			clusterName:   "azure",
-			expectedVMSet: "",
-		},
-		{
-			description:   "non-default standard external LB should map to empty string",
-			lbName:        "azuretest",
-			useStandardLB: true,
-			clusterName:   "azure",
-			expectedVMSet: "",
-		},
-		{
-			description:   "non-default standard internal LB should map to empty string",
-			lbName:        "azuretest-internal",
-			useStandardLB: true,
-			clusterName:   "azure",
-			expectedVMSet: "",
-		},
 	}
 
 	for _, c := range cases {