Merge pull request #95542 from nilo19/bug/support-exclude-node-from-lb-label

Support the node label `node.kubernetes.io/exclude-from-external-load-balancers`

staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go

@@ -981,6 +981,15 @@ func (az *Cloud) findFrontendIPConfigOfService(
 	return nil, false, nil
 }
 
+func nodeNameInNodes(nodeName string, nodes []*v1.Node) bool {
+	for _, node := range nodes {
+		if strings.EqualFold(nodeName, node.Name) {
+			return true
+		}
+	}
+	return false
+}
+
 // This ensures load balancer exists and the frontend ip config is setup.
 // This also reconciles the Service's Ports  with the LoadBalancer config.
 // This entails adding rules/probes for expected Ports and removing stale rules/ports.
@@ -1022,6 +1031,42 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
 			if strings.EqualFold(*bp.Name, lbBackendPoolName) {
 				klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb backendpool - found wanted backendpool. not adding anything", serviceName, wantLb)
 				foundBackendPool = true
+
+				var backendIPConfigurationsToBeDeleted []network.InterfaceIPConfiguration
+				if bp.BackendAddressPoolPropertiesFormat != nil && bp.BackendIPConfigurations != nil {
+					for _, ipConf := range *bp.BackendIPConfigurations {
+						ipConfID := to.String(ipConf.ID)
+						nodeName, err := az.VMSet.GetNodeNameByIPConfigurationID(ipConfID)
+						if err != nil {
+							return nil, err
+						}
+						// If a node is not supposed to be included in the LB, it
+						// would not be in the `nodes` slice. We need to check the nodes that
+						// have been added to the LB's backendpool, find the unwanted ones and
+						// delete them from the pool.
+						if !nodeNameInNodes(nodeName, nodes) {
+							klog.V(2).Infof("reconcileLoadBalancer for service (%s)(%t): lb backendpool - found unwanted node %s, decouple it from the LB", serviceName, wantLb, nodeName)
+							// construct a backendPool that only contains the IP config of the node to be deleted
+							backendIPConfigurationsToBeDeleted = append(backendIPConfigurationsToBeDeleted, network.InterfaceIPConfiguration{ID: to.StringPtr(ipConfID)})
+						}
+					}
+					if len(backendIPConfigurationsToBeDeleted) > 0 {
+						backendpoolToBeDeleted := &[]network.BackendAddressPool{
+							{
+								ID: to.StringPtr(lbBackendPoolID),
+								BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{
+									BackendIPConfigurations: &backendIPConfigurationsToBeDeleted,
+								},
+							},
+						}
+						vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName)
+						// decouple the backendPool from the node
+						err = az.VMSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, backendpoolToBeDeleted)
+						if err != nil {
+							return nil, err
+						}
+					}
+				}
 				break
 			} else {
 				klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb backendpool - found other backendpool %s", serviceName, wantLb, *bp.Name)
@@ -1294,6 +1339,10 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
 				// Remove backend pools from vmSets. This is required for virtual machine scale sets before removing the LB.
 				vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName)
 				klog.V(10).Infof("EnsureBackendPoolDeleted(%s,%s) for service %s: start", lbBackendPoolID, vmSetName, serviceName)
+				if _, ok := az.VMSet.(*availabilitySet); ok {
+					// do nothing for availability set
+					lb.BackendAddressPools = nil
+				}
 				err := az.VMSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, lb.BackendAddressPools)
 				if err != nil {
 					klog.Errorf("EnsureBackendPoolDeleted(%s) for service %s failed: %v", lbBackendPoolID, serviceName, err)
@@ -2170,8 +2219,8 @@ func equalLoadBalancingRulePropertiesFormat(s *network.LoadBalancingRuleProperti
 		reflect.DeepEqual(s.FrontendPort, t.FrontendPort) &&
 		reflect.DeepEqual(s.BackendPort, t.BackendPort) &&
 		reflect.DeepEqual(s.EnableFloatingIP, t.EnableFloatingIP) &&
-		reflect.DeepEqual(s.EnableTCPReset, t.EnableTCPReset) &&
+		reflect.DeepEqual(to.Bool(s.EnableTCPReset), to.Bool(t.EnableTCPReset)) &&
-		reflect.DeepEqual(s.DisableOutboundSnat, t.DisableOutboundSnat)
+		reflect.DeepEqual(to.Bool(s.DisableOutboundSnat), to.Bool(t.DisableOutboundSnat))
 
 	if wantLB && s.IdleTimeoutInMinutes != nil && t.IdleTimeoutInMinutes != nil {
 		return properties && reflect.DeepEqual(s.IdleTimeoutInMinutes, t.IdleTimeoutInMinutes)

staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer_test.go

@@ -1762,11 +1762,12 @@ func getTestLoadBalancer(name, rgName, clusterName, identifier *string, service
 							ID: to.StringPtr("/subscriptions/subscription/resourceGroups/" + *rgName + "/providers/" +
 								"Microsoft.Network/loadBalancers/" + *name + "/backendAddressPools/" + *clusterName),
 						},
-						LoadDistribution: network.LoadDistribution("Default"),
+						LoadDistribution:    network.LoadDistribution("Default"),
-						FrontendPort:     to.Int32Ptr(service.Spec.Ports[0].Port),
+						FrontendPort:        to.Int32Ptr(service.Spec.Ports[0].Port),
-						BackendPort:      to.Int32Ptr(service.Spec.Ports[0].Port),
+						BackendPort:         to.Int32Ptr(service.Spec.Ports[0].Port),
-						EnableFloatingIP: to.BoolPtr(true),
+						EnableFloatingIP:    to.BoolPtr(true),
-						EnableTCPReset:   to.BoolPtr(strings.EqualFold(lbSku, "standard")),
+						EnableTCPReset:      to.BoolPtr(strings.EqualFold(lbSku, "standard")),
+						DisableOutboundSnat: to.BoolPtr(false),
 						Probe: &network.SubResource{
 							ID: to.StringPtr("/subscriptions/subscription/resourceGroups/" + *rgName + "/providers/Microsoft.Network/loadBalancers/testCluster/probes/aservice1-TCP-80"),
 						},
@@ -1833,8 +1834,6 @@ func TestReconcileLoadBalancer(t *testing.T) {
 		},
 	}
 	expectedLb1 := getTestLoadBalancer(to.StringPtr("testCluster"), to.StringPtr("rg"), to.StringPtr("testCluster"), to.StringPtr("aservice1"), service3, "Basic")
-	(*expectedLb1.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].DisableOutboundSnat = to.BoolPtr(false)
-	(*expectedLb1.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].EnableTCPReset = nil
 	expectedLb1.FrontendIPConfigurations = &[]network.FrontendIPConfiguration{
 		{
 			Name: to.StringPtr("aservice1"),
@@ -1970,9 +1969,7 @@ func TestReconcileLoadBalancer(t *testing.T) {
 	lb6.Probes = &[]network.Probe{}
 	expectedLB6 := getTestLoadBalancer(to.StringPtr("testCluster"), to.StringPtr("rg"), to.StringPtr("testCluster"), to.StringPtr("aservice1"), service6, "basic")
 	expectedLB6.Probes = &[]network.Probe{}
-	(*expectedLB6.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].Probe = nil
+	(*expectedLB6.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].Probe = &network.SubResource{ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/testCluster/probes/aservice1-TCP-80")}
-	(*expectedLB6.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].EnableTCPReset = nil
-	(*expectedLB6.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].DisableOutboundSnat = to.BoolPtr(false)
 	expectedLB6.FrontendIPConfigurations = &[]network.FrontendIPConfiguration{
 		{
 			Name: to.StringPtr("aservice1"),
@@ -1994,7 +1991,7 @@ func TestReconcileLoadBalancer(t *testing.T) {
 		ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/testCluster/probes/aservice1-UDP-80"),
 	}
 	(*expectedLB7.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].EnableTCPReset = nil
-	(*expectedLB7.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].DisableOutboundSnat = to.BoolPtr(false)
+	(*lb7.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].DisableOutboundSnat = to.BoolPtr(true)
 	expectedLB7.FrontendIPConfigurations = &[]network.FrontendIPConfiguration{
 		{
 			Name: to.StringPtr("aservice1"),

staging/src/k8s.io/legacy-cloud-providers/azure/azure_standard.go

@@ -71,10 +71,14 @@ const (
 	loadBalancerRuleNameMaxLength = 80
 )
 
-var errNotInVMSet = errors.New("vm is not in the vmset")
+var (
-var providerIDRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/Microsoft.Compute/virtualMachines/(.+)$`)
+	errNotInVMSet      = errors.New("vm is not in the vmset")
-var backendPoolIDRE = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Network/loadBalancers/(.+)/backendAddressPools/(?:.*)`)
+	providerIDRE       = regexp.MustCompile(`.*/subscriptions/(?:.*)/Microsoft.Compute/virtualMachines/(.+)$`)
-var nicResourceGroupRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Network/networkInterfaces/(?:.*)`)
+	backendPoolIDRE    = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Network/loadBalancers/(.+)/backendAddressPools/(?:.*)`)
+	nicResourceGroupRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Network/networkInterfaces/(?:.*)`)
+	nicIDRE            = regexp.MustCompile(`/subscriptions/(?:.*)/resourceGroups/(?:.+)/providers/Microsoft.Network/networkInterfaces/(.+)-nic-(.+)/ipConfigurations/(?:.*)`)
+	vmasIDRE           = regexp.MustCompile(`/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Compute/availabilitySets/(.+)`)
+)
 
 // getStandardMachineID returns the full identifier of a virtual machine.
 func (az *Cloud) getStandardMachineID(subscriptionID, resourceGroup, machineName string) string {
@@ -700,7 +704,8 @@ func (as *availabilitySet) GetVMSetNames(service *v1.Service, nodes []*v1.Node)
 
 // GetPrimaryInterface gets machine primary network interface by node name.
 func (as *availabilitySet) GetPrimaryInterface(nodeName string) (network.Interface, error) {
-	return as.getPrimaryInterfaceWithVMSet(nodeName, "")
+	nic, _, err := as.getPrimaryInterfaceWithVMSet(nodeName, "")
+	return nic, err
 }
 
 // extractResourceGroupByNicID extracts the resource group name by nicID.
@@ -714,26 +719,26 @@ func extractResourceGroupByNicID(nicID string) (string, error) {
 }
 
 // getPrimaryInterfaceWithVMSet gets machine primary network interface by node name and vmSet.
-func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName string) (network.Interface, error) {
+func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName string) (network.Interface, string, error) {
 	var machine compute.VirtualMachine
 
 	machine, err := as.GetVirtualMachineWithRetry(types.NodeName(nodeName), azcache.CacheReadTypeDefault)
 	if err != nil {
 		klog.V(2).Infof("GetPrimaryInterface(%s, %s) abort backoff", nodeName, vmSetName)
-		return network.Interface{}, err
+		return network.Interface{}, "", err
 	}
 
 	primaryNicID, err := getPrimaryInterfaceID(machine)
 	if err != nil {
-		return network.Interface{}, err
+		return network.Interface{}, "", err
 	}
 	nicName, err := getLastSegment(primaryNicID, "/")
 	if err != nil {
-		return network.Interface{}, err
+		return network.Interface{}, "", err
 	}
 	nodeResourceGroup, err := as.GetNodeResourceGroup(nodeName)
 	if err != nil {
-		return network.Interface{}, err
+		return network.Interface{}, "", err
 	}
 
 	// Check availability set name. Note that vmSetName is empty string when getting
@@ -748,23 +753,27 @@ func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName stri
 		if machine.AvailabilitySet == nil || !strings.EqualFold(*machine.AvailabilitySet.ID, expectedAvailabilitySetName) {
 			klog.V(3).Infof(
 				"GetPrimaryInterface: nic (%s) is not in the availabilitySet(%s)", nicName, vmSetName)
-			return network.Interface{}, errNotInVMSet
+			return network.Interface{}, "", errNotInVMSet
 		}
 	}
 
 	nicResourceGroup, err := extractResourceGroupByNicID(primaryNicID)
 	if err != nil {
-		return network.Interface{}, err
+		return network.Interface{}, "", err
 	}
 
 	ctx, cancel := getContextWithCancel()
 	defer cancel()
 	nic, rerr := as.InterfacesClient.Get(ctx, nicResourceGroup, nicName, "")
 	if rerr != nil {
-		return network.Interface{}, rerr.Error()
+		return network.Interface{}, "", rerr.Error()
 	}
 
-	return nic, nil
+	var availabilitySetID string
+	if machine.VirtualMachineProperties != nil && machine.AvailabilitySet != nil {
+		availabilitySetID = to.String(machine.AvailabilitySet.ID)
+	}
+	return nic, availabilitySetID, nil
 }
 
 // EnsureHostInPool ensures the given VM's Primary NIC's Primary IP Configuration is
@@ -772,7 +781,7 @@ func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName stri
 func (as *availabilitySet) EnsureHostInPool(service *v1.Service, nodeName types.NodeName, backendPoolID string, vmSetName string, isInternal bool) (string, string, string, *compute.VirtualMachineScaleSetVM, error) {
 	vmName := mapNodeNameToVMName(nodeName)
 	serviceName := getServiceName(service)
-	nic, err := as.getPrimaryInterfaceWithVMSet(vmName, vmSetName)
+	nic, _, err := as.getPrimaryInterfaceWithVMSet(vmName, vmSetName)
 	if err != nil {
 		if err == errNotInVMSet {
 			klog.V(3).Infof("EnsureHostInPool skips node %s because it is not in the vmSet %s", nodeName, vmSetName)
@@ -895,7 +904,111 @@ func (as *availabilitySet) EnsureHostsInPool(service *v1.Service, nodes []*v1.No
 
 // EnsureBackendPoolDeleted ensures the loadBalancer backendAddressPools deleted from the specified nodes.
 func (as *availabilitySet) EnsureBackendPoolDeleted(service *v1.Service, backendPoolID, vmSetName string, backendAddressPools *[]network.BackendAddressPool) error {
-	// Do nothing for availability set.
+	// Returns nil if backend address pools already deleted.
+	if backendAddressPools == nil {
+		return nil
+	}
+
+	mc := metrics.NewMetricContext("services", "vmas_ensure_backend_pool_deleted", as.ResourceGroup, as.SubscriptionID, service.Name)
+	isOperationSucceeded := false
+	defer func() {
+		mc.ObserveOperationWithResult(isOperationSucceeded)
+	}()
+
+	ipConfigurationIDs := []string{}
+	for _, backendPool := range *backendAddressPools {
+		if strings.EqualFold(to.String(backendPool.ID), backendPoolID) &&
+			backendPool.BackendAddressPoolPropertiesFormat != nil &&
+			backendPool.BackendIPConfigurations != nil {
+			for _, ipConf := range *backendPool.BackendIPConfigurations {
+				if ipConf.ID == nil {
+					continue
+				}
+
+				ipConfigurationIDs = append(ipConfigurationIDs, *ipConf.ID)
+			}
+		}
+	}
+	nicUpdaters := make([]func() error, 0)
+	errors := make([]error, 0)
+	for i := range ipConfigurationIDs {
+		ipConfigurationID := ipConfigurationIDs[i]
+		nodeName, err := as.GetNodeNameByIPConfigurationID(ipConfigurationID)
+		if err != nil {
+			klog.Errorf("Failed to GetNodeNameByIPConfigurationID(%s): %v", ipConfigurationID, err)
+			errors = append(errors, err)
+			continue
+		}
+
+		vmName := mapNodeNameToVMName(types.NodeName(nodeName))
+		nic, vmasID, err := as.getPrimaryInterfaceWithVMSet(vmName, vmSetName)
+		if err != nil {
+			if err == errNotInVMSet {
+				klog.V(3).Infof("EnsureBackendPoolDeleted skips node %s because it is not in the vmSet %s", nodeName, vmSetName)
+				return nil
+			}
+
+			klog.Errorf("error: az.EnsureBackendPoolDeleted(%s), az.VMSet.GetPrimaryInterface.Get(%s, %s), err=%v", nodeName, vmName, vmSetName, err)
+			return err
+		}
+		matches := vmasIDRE.FindStringSubmatch(vmasID)
+		if len(matches) != 2 {
+			return fmt.Errorf("EnsureBackendPoolDeleted: failed to parse the VMAS ID %s: %v", vmasID, err)
+		}
+		vmasName := matches[1]
+		// Only remove nodes belonging to specified vmSet to basic LB backends.
+		if !strings.EqualFold(vmasName, vmSetName) {
+			klog.V(2).Infof("EnsureBackendPoolDeleted: skipping the node %s belonging to another vm set %s", nodeName, vmasName)
+			continue
+		}
+
+		if nic.ProvisioningState != nil && *nic.ProvisioningState == nicFailedState {
+			klog.Warningf("EnsureBackendPoolDeleted skips node %s because its primary nic %s is in Failed state", nodeName, *nic.Name)
+			return nil
+		}
+
+		if nic.InterfacePropertiesFormat != nil && nic.InterfacePropertiesFormat.IPConfigurations != nil {
+			newIPConfigs := *nic.IPConfigurations
+			for j, ipConf := range newIPConfigs {
+				if !to.Bool(ipConf.Primary) {
+					continue
+				}
+				// found primary ip configuration
+				if ipConf.LoadBalancerBackendAddressPools != nil {
+					newLBAddressPools := *ipConf.LoadBalancerBackendAddressPools
+					for k, pool := range newLBAddressPools {
+						if strings.EqualFold(to.String(pool.ID), backendPoolID) {
+							newLBAddressPools = append(newLBAddressPools[:k], newLBAddressPools[k+1:]...)
+							break
+						}
+					}
+					newIPConfigs[j].LoadBalancerBackendAddressPools = &newLBAddressPools
+				}
+			}
+			nic.IPConfigurations = &newIPConfigs
+			nicUpdaters = append(nicUpdaters, func() error {
+				ctx, cancel := getContextWithCancel()
+				defer cancel()
+				klog.V(2).Infof("EnsureBackendPoolDeleted begins to CreateOrUpdate for NIC(%s, %s) with backendPoolID %s", as.resourceGroup, to.String(nic.Name), backendPoolID)
+				rerr := as.InterfacesClient.CreateOrUpdate(ctx, as.ResourceGroup, to.String(nic.Name), nic)
+				if rerr != nil {
+					klog.Errorf("EnsureBackendPoolDeleted CreateOrUpdate for NIC(%s, %s) failed with error %v", as.resourceGroup, to.String(nic.Name), rerr.Error())
+					return rerr.Error()
+				}
+				return nil
+			})
+		}
+	}
+	errs := utilerrors.AggregateGoroutines(nicUpdaters...)
+	if errs != nil {
+		return utilerrors.Flatten(errs)
+	}
+	// Fail if there are other errors.
+	if len(errors) > 0 {
+		return utilerrors.Flatten(utilerrors.NewAggregate(errors))
+	}
+
+	isOperationSucceeded = true
 	return nil
 }
 
@@ -908,3 +1021,16 @@ func generateStorageAccountName(accountNamePrefix string) string {
 	}
 	return accountName
 }
+
+// GetNodeNameByIPConfigurationID gets the node name by IP configuration ID.
+func (as *availabilitySet) GetNodeNameByIPConfigurationID(ipConfigurationID string) (string, error) {
+	matches := nicIDRE.FindStringSubmatch(ipConfigurationID)
+	if len(matches) != 3 {
+		klog.V(4).Infof("Can not extract VM name from ipConfigurationID (%s)", ipConfigurationID)
+		return "", fmt.Errorf("invalid ip config ID %s", ipConfigurationID)
+	}
+
+	prefix := matches[1]
+	suffix := matches[2]
+	return fmt.Sprintf("%s-%s", prefix, suffix), nil
+}

staging/src/k8s.io/legacy-cloud-providers/azure/azure_standard_test.go

@@ -1665,3 +1665,81 @@ func TestServiceOwnsFrontendIP(t *testing.T) {
 		assert.Equal(t, test.isPrimary, isPrimary, test.desc)
 	}
 }
+
+func TestStandardEnsureBackendPoolDeleted(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+	cloud := GetTestCloud(ctrl)
+	service := getTestService("test", v1.ProtocolTCP, nil, false, 80)
+	backendPoolID := "backendPoolID"
+	vmSetName := "AS"
+
+	tests := []struct {
+		desc                string
+		backendAddressPools *[]network.BackendAddressPool
+		loadBalancerSKU     string
+		existingVM          compute.VirtualMachine
+		existingNIC         network.Interface
+	}{
+		{
+			desc: "",
+			backendAddressPools: &[]network.BackendAddressPool{
+				{
+					ID: to.StringPtr(backendPoolID),
+					BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{
+						BackendIPConfigurations: &[]network.InterfaceIPConfiguration{
+							{
+								ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool1-00000000-nic-1/ipConfigurations/ipconfig1"),
+							},
+						},
+					},
+				},
+			},
+			existingVM: compute.VirtualMachine{
+				VirtualMachineProperties: &compute.VirtualMachineProperties{
+					AvailabilitySet: &compute.SubResource{
+						ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/availabilitySets/as"),
+					},
+					NetworkProfile: &compute.NetworkProfile{
+						NetworkInterfaces: &[]compute.NetworkInterfaceReference{
+							{
+								ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool1-00000000-nic-1"),
+							},
+						},
+					},
+				},
+			},
+			existingNIC: network.Interface{
+				InterfacePropertiesFormat: &network.InterfacePropertiesFormat{
+					ProvisioningState: to.StringPtr("Succeeded"),
+					IPConfigurations: &[]network.InterfaceIPConfiguration{
+						{
+							InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{
+								Primary: to.BoolPtr(true),
+								LoadBalancerBackendAddressPools: &[]network.BackendAddressPool{
+									{
+										ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool1-00000000-nic-1/ipConfigurations/ipconfig1"),
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		cloud.LoadBalancerSku = test.loadBalancerSKU
+		mockVMClient := mockvmclient.NewMockInterface(ctrl)
+		mockVMClient.EXPECT().Get(gomock.Any(), cloud.ResourceGroup, "k8s-agentpool1-00000000-1", gomock.Any()).Return(test.existingVM, nil)
+		cloud.VirtualMachinesClient = mockVMClient
+		mockNICClient := mockinterfaceclient.NewMockInterface(ctrl)
+		mockNICClient.EXPECT().Get(gomock.Any(), "rg", "k8s-agentpool1-00000000-nic-1", gomock.Any()).Return(test.existingNIC, nil)
+		mockNICClient.EXPECT().CreateOrUpdate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
+		cloud.InterfacesClient = mockNICClient
+
+		err := cloud.VMSet.EnsureBackendPoolDeleted(&service, backendPoolID, vmSetName, test.backendAddressPools)
+		assert.NoError(t, err, test.desc)
+	}
+}

staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmsets.go

@@ -78,4 +78,7 @@ type VMSet interface {
 
 	// GetPrivateIPsByNodeName returns a slice of all private ips assigned to node (ipv6 and ipv4)
 	GetPrivateIPsByNodeName(name string) ([]string, error)
+
+	// GetNodeNameByIPConfigurationID gets the node name by IP configuration ID.
+	GetNodeNameByIPConfigurationID(ipConfigurationID string) (string, error)
 }

staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go

@@ -1297,7 +1297,7 @@ func (ss *scaleSet) EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, bac
 
 // ensureBackendPoolDeletedFromNode ensures the loadBalancer backendAddressPools deleted
 // from the specified node, which returns (resourceGroup, vmssName, instanceID, vmssVM, error).
-func (ss *scaleSet) ensureBackendPoolDeletedFromNode(service *v1.Service, nodeName, backendPoolID string) (string, string, string, *compute.VirtualMachineScaleSetVM, error) {
+func (ss *scaleSet) ensureBackendPoolDeletedFromNode(nodeName, backendPoolID string) (string, string, string, *compute.VirtualMachineScaleSetVM, error) {
 	ssName, instanceID, vm, err := ss.getVmssVM(nodeName, azcache.CacheReadTypeDefault)
 	if err != nil {
 		return "", "", "", nil, err
@@ -1363,8 +1363,8 @@ func (ss *scaleSet) ensureBackendPoolDeletedFromNode(service *v1.Service, nodeNa
 	return nodeResourceGroup, ssName, instanceID, newVM, nil
 }
 
-// getNodeNameByIPConfigurationID gets the node name by IP configuration ID.
+// GetNodeNameByIPConfigurationID gets the node name by IP configuration ID.
-func (ss *scaleSet) getNodeNameByIPConfigurationID(ipConfigurationID string) (string, error) {
+func (ss *scaleSet) GetNodeNameByIPConfigurationID(ipConfigurationID string) (string, error) {
 	matches := vmssIPConfigurationRE.FindStringSubmatch(ipConfigurationID)
 	if len(matches) != 4 {
 		klog.V(4).Infof("Can not extract scale set name from ipConfigurationID (%s), assuming it is mananaged by availability set", ipConfigurationID)
@@ -1529,18 +1529,18 @@ func (ss *scaleSet) EnsureBackendPoolDeleted(service *v1.Service, backendPoolID,
 			}
 		}
 
-		nodeName, err := ss.getNodeNameByIPConfigurationID(ipConfigurationID)
+		nodeName, err := ss.GetNodeNameByIPConfigurationID(ipConfigurationID)
 		if err != nil {
 			if err == ErrorNotVmssInstance { // Do nothing for the VMAS nodes.
 				continue
 			}
 
-			klog.Errorf("Failed to getNodeNameByIPConfigurationID(%s): %v", ipConfigurationID, err)
+			klog.Errorf("Failed to GetNodeNameByIPConfigurationID(%s): %v", ipConfigurationID, err)
 			errors = append(errors, err)
 			continue
 		}
 
-		nodeResourceGroup, nodeVMSS, nodeInstanceID, nodeVMSSVM, err := ss.ensureBackendPoolDeletedFromNode(service, nodeName, backendPoolID)
+		nodeResourceGroup, nodeVMSS, nodeInstanceID, nodeVMSSVM, err := ss.ensureBackendPoolDeletedFromNode(nodeName, backendPoolID)
 		if err != nil {
 			klog.Errorf("EnsureBackendPoolDeleted(%s): backendPoolID(%s) - failed with error %v", getServiceName(service), backendPoolID, err)
 			errors = append(errors, err)

staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go

@@ -581,21 +581,21 @@ func TestGetNodeNameByIPConfigurationID(t *testing.T) {
 		expectError       bool
 	}{
 		{
-			description:       "getNodeNameByIPConfigurationID should get node's Name when the node is existing",
+			description:       "GetNodeNameByIPConfigurationID should get node's Name when the node is existing",
 			scaleSet:          "scaleset1",
 			ipConfigurationID: fmt.Sprintf(ipConfigurationIDTemplate, "scaleset1", "0", "scaleset1"),
 			vmList:            []string{"vmssee6c2000000", "vmssee6c2000001"},
 			expected:          "vmssee6c2000000",
 		},
 		{
-			description:       "getNodeNameByIPConfigurationID should return error for non-exist nodes",
+			description:       "GetNodeNameByIPConfigurationID should return error for non-exist nodes",
 			scaleSet:          "scaleset2",
 			ipConfigurationID: fmt.Sprintf(ipConfigurationIDTemplate, "scaleset2", "3", "scaleset1"),
 			vmList:            []string{"vmssee6c2000002", "vmssee6c2000003"},
 			expectError:       true,
 		},
 		{
-			description:       "getNodeNameByIPConfigurationID should return error for wrong ipConfigurationID",
+			description:       "GetNodeNameByIPConfigurationID should return error for wrong ipConfigurationID",
 			scaleSet:          "scaleset3",
 			ipConfigurationID: "invalid-configuration-id",
 			vmList:            []string{"vmssee6c2000004", "vmssee6c2000005"},
@@ -618,7 +618,7 @@ func TestGetNodeNameByIPConfigurationID(t *testing.T) {
 		expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "", false)
 		mockVMSSVMClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedVMs, nil).AnyTimes()
 
-		nodeName, err := ss.getNodeNameByIPConfigurationID(test.ipConfigurationID)
+		nodeName, err := ss.GetNodeNameByIPConfigurationID(test.ipConfigurationID)
 		if test.expectError {
 			assert.Error(t, err, test.description)
 			continue
@@ -2113,7 +2113,7 @@ func TestEnsureBackendPoolDeletedFromNode(t *testing.T) {
 		mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface)
 		mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes()
 
-		nodeResourceGroup, ssName, instanceID, vm, err := ss.ensureBackendPoolDeletedFromNode(&v1.Service{}, test.nodeName, test.backendpoolID)
+		nodeResourceGroup, ssName, instanceID, vm, err := ss.ensureBackendPoolDeletedFromNode(test.nodeName, test.backendpoolID)
 		assert.Equal(t, test.expectedErr, err, test.description+", but an error occurs")
 		assert.Equal(t, test.expectedNodeResourceGroup, nodeResourceGroup, test.description)
 		assert.Equal(t, test.expectedVMSSName, ssName, test.description)

staging/src/k8s.io/legacy-cloud-providers/azure/mockvmsets/azure_mock_vmsets.go

@@ -291,3 +291,18 @@ func (mr *MockVMSetMockRecorder) GetPrivateIPsByNodeName(name interface{}) *gomo
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPrivateIPsByNodeName", reflect.TypeOf((*MockVMSet)(nil).GetPrivateIPsByNodeName), name)
 }
+
+// GetNodeNameByIPConfigurationID mocks base method
+func (m *MockVMSet) GetNodeNameByIPConfigurationID(ipConfigurationID string) (string, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetNodeNameByIPConfigurationID", ipConfigurationID)
+	ret0, _ := ret[0].(string)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetNodeNameByIPConfigurationID indicates an expected call of GetNodeNameByIPConfigurationID
+func (mr *MockVMSetMockRecorder) GetNodeNameByIPConfigurationID(ipConfigurationID interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNodeNameByIPConfigurationID", reflect.TypeOf((*MockVMSet)(nil).GetNodeNameByIPConfigurationID), ipConfigurationID)
+}