diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go index 28b1c2dc62b..cd0af314f4b 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go @@ -981,6 +981,15 @@ func (az *Cloud) findFrontendIPConfigOfService( return nil, false, nil } +func nodeNameInNodes(nodeName string, nodes []*v1.Node) bool { + for _, node := range nodes { + if strings.EqualFold(nodeName, node.Name) { + return true + } + } + return false +} + // This ensures load balancer exists and the frontend ip config is setup. // This also reconciles the Service's Ports with the LoadBalancer config. // This entails adding rules/probes for expected Ports and removing stale rules/ports. @@ -1022,6 +1031,42 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, if strings.EqualFold(*bp.Name, lbBackendPoolName) { klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb backendpool - found wanted backendpool. not adding anything", serviceName, wantLb) foundBackendPool = true + + var backendIPConfigurationsToBeDeleted []network.InterfaceIPConfiguration + if bp.BackendAddressPoolPropertiesFormat != nil && bp.BackendIPConfigurations != nil { + for _, ipConf := range *bp.BackendIPConfigurations { + ipConfID := to.String(ipConf.ID) + nodeName, err := az.VMSet.GetNodeNameByIPConfigurationID(ipConfID) + if err != nil { + return nil, err + } + // If a node is not supposed to be included in the LB, it + // would not be in the `nodes` slice. We need to check the nodes that + // have been added to the LB's backendpool, find the unwanted ones and + // delete them from the pool. + if !nodeNameInNodes(nodeName, nodes) { + klog.V(2).Infof("reconcileLoadBalancer for service (%s)(%t): lb backendpool - found unwanted node %s, decouple it from the LB", serviceName, wantLb, nodeName) + // construct a backendPool that only contains the IP config of the node to be deleted + backendIPConfigurationsToBeDeleted = append(backendIPConfigurationsToBeDeleted, network.InterfaceIPConfiguration{ID: to.StringPtr(ipConfID)}) + } + } + if len(backendIPConfigurationsToBeDeleted) > 0 { + backendpoolToBeDeleted := &[]network.BackendAddressPool{ + { + ID: to.StringPtr(lbBackendPoolID), + BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{ + BackendIPConfigurations: &backendIPConfigurationsToBeDeleted, + }, + }, + } + vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName) + // decouple the backendPool from the node + err = az.VMSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, backendpoolToBeDeleted) + if err != nil { + return nil, err + } + } + } break } else { klog.V(10).Infof("reconcileLoadBalancer for service (%s)(%t): lb backendpool - found other backendpool %s", serviceName, wantLb, *bp.Name) @@ -1294,6 +1339,10 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, // Remove backend pools from vmSets. This is required for virtual machine scale sets before removing the LB. vmSetName := az.mapLoadBalancerNameToVMSet(lbName, clusterName) klog.V(10).Infof("EnsureBackendPoolDeleted(%s,%s) for service %s: start", lbBackendPoolID, vmSetName, serviceName) + if _, ok := az.VMSet.(*availabilitySet); ok { + // do nothing for availability set + lb.BackendAddressPools = nil + } err := az.VMSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, lb.BackendAddressPools) if err != nil { klog.Errorf("EnsureBackendPoolDeleted(%s) for service %s failed: %v", lbBackendPoolID, serviceName, err) @@ -2170,8 +2219,8 @@ func equalLoadBalancingRulePropertiesFormat(s *network.LoadBalancingRuleProperti reflect.DeepEqual(s.FrontendPort, t.FrontendPort) && reflect.DeepEqual(s.BackendPort, t.BackendPort) && reflect.DeepEqual(s.EnableFloatingIP, t.EnableFloatingIP) && - reflect.DeepEqual(s.EnableTCPReset, t.EnableTCPReset) && - reflect.DeepEqual(s.DisableOutboundSnat, t.DisableOutboundSnat) + reflect.DeepEqual(to.Bool(s.EnableTCPReset), to.Bool(t.EnableTCPReset)) && + reflect.DeepEqual(to.Bool(s.DisableOutboundSnat), to.Bool(t.DisableOutboundSnat)) if wantLB && s.IdleTimeoutInMinutes != nil && t.IdleTimeoutInMinutes != nil { return properties && reflect.DeepEqual(s.IdleTimeoutInMinutes, t.IdleTimeoutInMinutes) diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer_test.go index 931be2fa347..452a6b76fe6 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer_test.go @@ -1762,11 +1762,12 @@ func getTestLoadBalancer(name, rgName, clusterName, identifier *string, service ID: to.StringPtr("/subscriptions/subscription/resourceGroups/" + *rgName + "/providers/" + "Microsoft.Network/loadBalancers/" + *name + "/backendAddressPools/" + *clusterName), }, - LoadDistribution: network.LoadDistribution("Default"), - FrontendPort: to.Int32Ptr(service.Spec.Ports[0].Port), - BackendPort: to.Int32Ptr(service.Spec.Ports[0].Port), - EnableFloatingIP: to.BoolPtr(true), - EnableTCPReset: to.BoolPtr(strings.EqualFold(lbSku, "standard")), + LoadDistribution: network.LoadDistribution("Default"), + FrontendPort: to.Int32Ptr(service.Spec.Ports[0].Port), + BackendPort: to.Int32Ptr(service.Spec.Ports[0].Port), + EnableFloatingIP: to.BoolPtr(true), + EnableTCPReset: to.BoolPtr(strings.EqualFold(lbSku, "standard")), + DisableOutboundSnat: to.BoolPtr(false), Probe: &network.SubResource{ ID: to.StringPtr("/subscriptions/subscription/resourceGroups/" + *rgName + "/providers/Microsoft.Network/loadBalancers/testCluster/probes/aservice1-TCP-80"), }, @@ -1833,8 +1834,6 @@ func TestReconcileLoadBalancer(t *testing.T) { }, } expectedLb1 := getTestLoadBalancer(to.StringPtr("testCluster"), to.StringPtr("rg"), to.StringPtr("testCluster"), to.StringPtr("aservice1"), service3, "Basic") - (*expectedLb1.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].DisableOutboundSnat = to.BoolPtr(false) - (*expectedLb1.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].EnableTCPReset = nil expectedLb1.FrontendIPConfigurations = &[]network.FrontendIPConfiguration{ { Name: to.StringPtr("aservice1"), @@ -1970,9 +1969,7 @@ func TestReconcileLoadBalancer(t *testing.T) { lb6.Probes = &[]network.Probe{} expectedLB6 := getTestLoadBalancer(to.StringPtr("testCluster"), to.StringPtr("rg"), to.StringPtr("testCluster"), to.StringPtr("aservice1"), service6, "basic") expectedLB6.Probes = &[]network.Probe{} - (*expectedLB6.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].Probe = nil - (*expectedLB6.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].EnableTCPReset = nil - (*expectedLB6.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].DisableOutboundSnat = to.BoolPtr(false) + (*expectedLB6.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].Probe = &network.SubResource{ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/testCluster/probes/aservice1-TCP-80")} expectedLB6.FrontendIPConfigurations = &[]network.FrontendIPConfiguration{ { Name: to.StringPtr("aservice1"), @@ -1994,7 +1991,7 @@ func TestReconcileLoadBalancer(t *testing.T) { ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/testCluster/probes/aservice1-UDP-80"), } (*expectedLB7.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].EnableTCPReset = nil - (*expectedLB7.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].DisableOutboundSnat = to.BoolPtr(false) + (*lb7.LoadBalancerPropertiesFormat.LoadBalancingRules)[0].DisableOutboundSnat = to.BoolPtr(true) expectedLB7.FrontendIPConfigurations = &[]network.FrontendIPConfiguration{ { Name: to.StringPtr("aservice1"), diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_standard.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_standard.go index 762210f9887..21fae97ec05 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_standard.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_standard.go @@ -71,10 +71,14 @@ const ( loadBalancerRuleNameMaxLength = 80 ) -var errNotInVMSet = errors.New("vm is not in the vmset") -var providerIDRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/Microsoft.Compute/virtualMachines/(.+)$`) -var backendPoolIDRE = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Network/loadBalancers/(.+)/backendAddressPools/(?:.*)`) -var nicResourceGroupRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Network/networkInterfaces/(?:.*)`) +var ( + errNotInVMSet = errors.New("vm is not in the vmset") + providerIDRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/Microsoft.Compute/virtualMachines/(.+)$`) + backendPoolIDRE = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Network/loadBalancers/(.+)/backendAddressPools/(?:.*)`) + nicResourceGroupRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Network/networkInterfaces/(?:.*)`) + nicIDRE = regexp.MustCompile(`/subscriptions/(?:.*)/resourceGroups/(?:.+)/providers/Microsoft.Network/networkInterfaces/(.+)-nic-(.+)/ipConfigurations/(?:.*)`) + vmasIDRE = regexp.MustCompile(`/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Compute/availabilitySets/(.+)`) +) // getStandardMachineID returns the full identifier of a virtual machine. func (az *Cloud) getStandardMachineID(subscriptionID, resourceGroup, machineName string) string { @@ -700,7 +704,8 @@ func (as *availabilitySet) GetVMSetNames(service *v1.Service, nodes []*v1.Node) // GetPrimaryInterface gets machine primary network interface by node name. func (as *availabilitySet) GetPrimaryInterface(nodeName string) (network.Interface, error) { - return as.getPrimaryInterfaceWithVMSet(nodeName, "") + nic, _, err := as.getPrimaryInterfaceWithVMSet(nodeName, "") + return nic, err } // extractResourceGroupByNicID extracts the resource group name by nicID. @@ -714,26 +719,26 @@ func extractResourceGroupByNicID(nicID string) (string, error) { } // getPrimaryInterfaceWithVMSet gets machine primary network interface by node name and vmSet. -func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName string) (network.Interface, error) { +func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName string) (network.Interface, string, error) { var machine compute.VirtualMachine machine, err := as.GetVirtualMachineWithRetry(types.NodeName(nodeName), azcache.CacheReadTypeDefault) if err != nil { klog.V(2).Infof("GetPrimaryInterface(%s, %s) abort backoff", nodeName, vmSetName) - return network.Interface{}, err + return network.Interface{}, "", err } primaryNicID, err := getPrimaryInterfaceID(machine) if err != nil { - return network.Interface{}, err + return network.Interface{}, "", err } nicName, err := getLastSegment(primaryNicID, "/") if err != nil { - return network.Interface{}, err + return network.Interface{}, "", err } nodeResourceGroup, err := as.GetNodeResourceGroup(nodeName) if err != nil { - return network.Interface{}, err + return network.Interface{}, "", err } // Check availability set name. Note that vmSetName is empty string when getting @@ -748,23 +753,27 @@ func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName stri if machine.AvailabilitySet == nil || !strings.EqualFold(*machine.AvailabilitySet.ID, expectedAvailabilitySetName) { klog.V(3).Infof( "GetPrimaryInterface: nic (%s) is not in the availabilitySet(%s)", nicName, vmSetName) - return network.Interface{}, errNotInVMSet + return network.Interface{}, "", errNotInVMSet } } nicResourceGroup, err := extractResourceGroupByNicID(primaryNicID) if err != nil { - return network.Interface{}, err + return network.Interface{}, "", err } ctx, cancel := getContextWithCancel() defer cancel() nic, rerr := as.InterfacesClient.Get(ctx, nicResourceGroup, nicName, "") if rerr != nil { - return network.Interface{}, rerr.Error() + return network.Interface{}, "", rerr.Error() } - return nic, nil + var availabilitySetID string + if machine.VirtualMachineProperties != nil && machine.AvailabilitySet != nil { + availabilitySetID = to.String(machine.AvailabilitySet.ID) + } + return nic, availabilitySetID, nil } // EnsureHostInPool ensures the given VM's Primary NIC's Primary IP Configuration is @@ -772,7 +781,7 @@ func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName stri func (as *availabilitySet) EnsureHostInPool(service *v1.Service, nodeName types.NodeName, backendPoolID string, vmSetName string, isInternal bool) (string, string, string, *compute.VirtualMachineScaleSetVM, error) { vmName := mapNodeNameToVMName(nodeName) serviceName := getServiceName(service) - nic, err := as.getPrimaryInterfaceWithVMSet(vmName, vmSetName) + nic, _, err := as.getPrimaryInterfaceWithVMSet(vmName, vmSetName) if err != nil { if err == errNotInVMSet { klog.V(3).Infof("EnsureHostInPool skips node %s because it is not in the vmSet %s", nodeName, vmSetName) @@ -895,7 +904,111 @@ func (as *availabilitySet) EnsureHostsInPool(service *v1.Service, nodes []*v1.No // EnsureBackendPoolDeleted ensures the loadBalancer backendAddressPools deleted from the specified nodes. func (as *availabilitySet) EnsureBackendPoolDeleted(service *v1.Service, backendPoolID, vmSetName string, backendAddressPools *[]network.BackendAddressPool) error { - // Do nothing for availability set. + // Returns nil if backend address pools already deleted. + if backendAddressPools == nil { + return nil + } + + mc := metrics.NewMetricContext("services", "vmas_ensure_backend_pool_deleted", as.ResourceGroup, as.SubscriptionID, service.Name) + isOperationSucceeded := false + defer func() { + mc.ObserveOperationWithResult(isOperationSucceeded) + }() + + ipConfigurationIDs := []string{} + for _, backendPool := range *backendAddressPools { + if strings.EqualFold(to.String(backendPool.ID), backendPoolID) && + backendPool.BackendAddressPoolPropertiesFormat != nil && + backendPool.BackendIPConfigurations != nil { + for _, ipConf := range *backendPool.BackendIPConfigurations { + if ipConf.ID == nil { + continue + } + + ipConfigurationIDs = append(ipConfigurationIDs, *ipConf.ID) + } + } + } + nicUpdaters := make([]func() error, 0) + errors := make([]error, 0) + for i := range ipConfigurationIDs { + ipConfigurationID := ipConfigurationIDs[i] + nodeName, err := as.GetNodeNameByIPConfigurationID(ipConfigurationID) + if err != nil { + klog.Errorf("Failed to GetNodeNameByIPConfigurationID(%s): %v", ipConfigurationID, err) + errors = append(errors, err) + continue + } + + vmName := mapNodeNameToVMName(types.NodeName(nodeName)) + nic, vmasID, err := as.getPrimaryInterfaceWithVMSet(vmName, vmSetName) + if err != nil { + if err == errNotInVMSet { + klog.V(3).Infof("EnsureBackendPoolDeleted skips node %s because it is not in the vmSet %s", nodeName, vmSetName) + return nil + } + + klog.Errorf("error: az.EnsureBackendPoolDeleted(%s), az.VMSet.GetPrimaryInterface.Get(%s, %s), err=%v", nodeName, vmName, vmSetName, err) + return err + } + matches := vmasIDRE.FindStringSubmatch(vmasID) + if len(matches) != 2 { + return fmt.Errorf("EnsureBackendPoolDeleted: failed to parse the VMAS ID %s: %v", vmasID, err) + } + vmasName := matches[1] + // Only remove nodes belonging to specified vmSet to basic LB backends. + if !strings.EqualFold(vmasName, vmSetName) { + klog.V(2).Infof("EnsureBackendPoolDeleted: skipping the node %s belonging to another vm set %s", nodeName, vmasName) + continue + } + + if nic.ProvisioningState != nil && *nic.ProvisioningState == nicFailedState { + klog.Warningf("EnsureBackendPoolDeleted skips node %s because its primary nic %s is in Failed state", nodeName, *nic.Name) + return nil + } + + if nic.InterfacePropertiesFormat != nil && nic.InterfacePropertiesFormat.IPConfigurations != nil { + newIPConfigs := *nic.IPConfigurations + for j, ipConf := range newIPConfigs { + if !to.Bool(ipConf.Primary) { + continue + } + // found primary ip configuration + if ipConf.LoadBalancerBackendAddressPools != nil { + newLBAddressPools := *ipConf.LoadBalancerBackendAddressPools + for k, pool := range newLBAddressPools { + if strings.EqualFold(to.String(pool.ID), backendPoolID) { + newLBAddressPools = append(newLBAddressPools[:k], newLBAddressPools[k+1:]...) + break + } + } + newIPConfigs[j].LoadBalancerBackendAddressPools = &newLBAddressPools + } + } + nic.IPConfigurations = &newIPConfigs + nicUpdaters = append(nicUpdaters, func() error { + ctx, cancel := getContextWithCancel() + defer cancel() + klog.V(2).Infof("EnsureBackendPoolDeleted begins to CreateOrUpdate for NIC(%s, %s) with backendPoolID %s", as.resourceGroup, to.String(nic.Name), backendPoolID) + rerr := as.InterfacesClient.CreateOrUpdate(ctx, as.ResourceGroup, to.String(nic.Name), nic) + if rerr != nil { + klog.Errorf("EnsureBackendPoolDeleted CreateOrUpdate for NIC(%s, %s) failed with error %v", as.resourceGroup, to.String(nic.Name), rerr.Error()) + return rerr.Error() + } + return nil + }) + } + } + errs := utilerrors.AggregateGoroutines(nicUpdaters...) + if errs != nil { + return utilerrors.Flatten(errs) + } + // Fail if there are other errors. + if len(errors) > 0 { + return utilerrors.Flatten(utilerrors.NewAggregate(errors)) + } + + isOperationSucceeded = true return nil } @@ -908,3 +1021,16 @@ func generateStorageAccountName(accountNamePrefix string) string { } return accountName } + +// GetNodeNameByIPConfigurationID gets the node name by IP configuration ID. +func (as *availabilitySet) GetNodeNameByIPConfigurationID(ipConfigurationID string) (string, error) { + matches := nicIDRE.FindStringSubmatch(ipConfigurationID) + if len(matches) != 3 { + klog.V(4).Infof("Can not extract VM name from ipConfigurationID (%s)", ipConfigurationID) + return "", fmt.Errorf("invalid ip config ID %s", ipConfigurationID) + } + + prefix := matches[1] + suffix := matches[2] + return fmt.Sprintf("%s-%s", prefix, suffix), nil +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_standard_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_standard_test.go index 061df6e776c..e5f5dfa326e 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_standard_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_standard_test.go @@ -1665,3 +1665,81 @@ func TestServiceOwnsFrontendIP(t *testing.T) { assert.Equal(t, test.isPrimary, isPrimary, test.desc) } } + +func TestStandardEnsureBackendPoolDeleted(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + cloud := GetTestCloud(ctrl) + service := getTestService("test", v1.ProtocolTCP, nil, false, 80) + backendPoolID := "backendPoolID" + vmSetName := "AS" + + tests := []struct { + desc string + backendAddressPools *[]network.BackendAddressPool + loadBalancerSKU string + existingVM compute.VirtualMachine + existingNIC network.Interface + }{ + { + desc: "", + backendAddressPools: &[]network.BackendAddressPool{ + { + ID: to.StringPtr(backendPoolID), + BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{ + BackendIPConfigurations: &[]network.InterfaceIPConfiguration{ + { + ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool1-00000000-nic-1/ipConfigurations/ipconfig1"), + }, + }, + }, + }, + }, + existingVM: compute.VirtualMachine{ + VirtualMachineProperties: &compute.VirtualMachineProperties{ + AvailabilitySet: &compute.SubResource{ + ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/availabilitySets/as"), + }, + NetworkProfile: &compute.NetworkProfile{ + NetworkInterfaces: &[]compute.NetworkInterfaceReference{ + { + ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool1-00000000-nic-1"), + }, + }, + }, + }, + }, + existingNIC: network.Interface{ + InterfacePropertiesFormat: &network.InterfacePropertiesFormat{ + ProvisioningState: to.StringPtr("Succeeded"), + IPConfigurations: &[]network.InterfaceIPConfiguration{ + { + InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ + Primary: to.BoolPtr(true), + LoadBalancerBackendAddressPools: &[]network.BackendAddressPool{ + { + ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool1-00000000-nic-1/ipConfigurations/ipconfig1"), + }, + }, + }, + }, + }, + }, + }, + }, + } + + for _, test := range tests { + cloud.LoadBalancerSku = test.loadBalancerSKU + mockVMClient := mockvmclient.NewMockInterface(ctrl) + mockVMClient.EXPECT().Get(gomock.Any(), cloud.ResourceGroup, "k8s-agentpool1-00000000-1", gomock.Any()).Return(test.existingVM, nil) + cloud.VirtualMachinesClient = mockVMClient + mockNICClient := mockinterfaceclient.NewMockInterface(ctrl) + mockNICClient.EXPECT().Get(gomock.Any(), "rg", "k8s-agentpool1-00000000-nic-1", gomock.Any()).Return(test.existingNIC, nil) + mockNICClient.EXPECT().CreateOrUpdate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + cloud.InterfacesClient = mockNICClient + + err := cloud.VMSet.EnsureBackendPoolDeleted(&service, backendPoolID, vmSetName, test.backendAddressPools) + assert.NoError(t, err, test.desc) + } +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmsets.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmsets.go index 89ac79cb6d4..2cc3c040240 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmsets.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmsets.go @@ -78,4 +78,7 @@ type VMSet interface { // GetPrivateIPsByNodeName returns a slice of all private ips assigned to node (ipv6 and ipv4) GetPrivateIPsByNodeName(name string) ([]string, error) + + // GetNodeNameByIPConfigurationID gets the node name by IP configuration ID. + GetNodeNameByIPConfigurationID(ipConfigurationID string) (string, error) } diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go index c0512ff01eb..70d201c6d1b 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go @@ -1297,7 +1297,7 @@ func (ss *scaleSet) EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, bac // ensureBackendPoolDeletedFromNode ensures the loadBalancer backendAddressPools deleted // from the specified node, which returns (resourceGroup, vmssName, instanceID, vmssVM, error). -func (ss *scaleSet) ensureBackendPoolDeletedFromNode(service *v1.Service, nodeName, backendPoolID string) (string, string, string, *compute.VirtualMachineScaleSetVM, error) { +func (ss *scaleSet) ensureBackendPoolDeletedFromNode(nodeName, backendPoolID string) (string, string, string, *compute.VirtualMachineScaleSetVM, error) { ssName, instanceID, vm, err := ss.getVmssVM(nodeName, azcache.CacheReadTypeDefault) if err != nil { return "", "", "", nil, err @@ -1363,8 +1363,8 @@ func (ss *scaleSet) ensureBackendPoolDeletedFromNode(service *v1.Service, nodeNa return nodeResourceGroup, ssName, instanceID, newVM, nil } -// getNodeNameByIPConfigurationID gets the node name by IP configuration ID. -func (ss *scaleSet) getNodeNameByIPConfigurationID(ipConfigurationID string) (string, error) { +// GetNodeNameByIPConfigurationID gets the node name by IP configuration ID. +func (ss *scaleSet) GetNodeNameByIPConfigurationID(ipConfigurationID string) (string, error) { matches := vmssIPConfigurationRE.FindStringSubmatch(ipConfigurationID) if len(matches) != 4 { klog.V(4).Infof("Can not extract scale set name from ipConfigurationID (%s), assuming it is mananaged by availability set", ipConfigurationID) @@ -1529,18 +1529,18 @@ func (ss *scaleSet) EnsureBackendPoolDeleted(service *v1.Service, backendPoolID, } } - nodeName, err := ss.getNodeNameByIPConfigurationID(ipConfigurationID) + nodeName, err := ss.GetNodeNameByIPConfigurationID(ipConfigurationID) if err != nil { if err == ErrorNotVmssInstance { // Do nothing for the VMAS nodes. continue } - klog.Errorf("Failed to getNodeNameByIPConfigurationID(%s): %v", ipConfigurationID, err) + klog.Errorf("Failed to GetNodeNameByIPConfigurationID(%s): %v", ipConfigurationID, err) errors = append(errors, err) continue } - nodeResourceGroup, nodeVMSS, nodeInstanceID, nodeVMSSVM, err := ss.ensureBackendPoolDeletedFromNode(service, nodeName, backendPoolID) + nodeResourceGroup, nodeVMSS, nodeInstanceID, nodeVMSSVM, err := ss.ensureBackendPoolDeletedFromNode(nodeName, backendPoolID) if err != nil { klog.Errorf("EnsureBackendPoolDeleted(%s): backendPoolID(%s) - failed with error %v", getServiceName(service), backendPoolID, err) errors = append(errors, err) diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go index 5cd33dd3276..a519748cf57 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go @@ -581,21 +581,21 @@ func TestGetNodeNameByIPConfigurationID(t *testing.T) { expectError bool }{ { - description: "getNodeNameByIPConfigurationID should get node's Name when the node is existing", + description: "GetNodeNameByIPConfigurationID should get node's Name when the node is existing", scaleSet: "scaleset1", ipConfigurationID: fmt.Sprintf(ipConfigurationIDTemplate, "scaleset1", "0", "scaleset1"), vmList: []string{"vmssee6c2000000", "vmssee6c2000001"}, expected: "vmssee6c2000000", }, { - description: "getNodeNameByIPConfigurationID should return error for non-exist nodes", + description: "GetNodeNameByIPConfigurationID should return error for non-exist nodes", scaleSet: "scaleset2", ipConfigurationID: fmt.Sprintf(ipConfigurationIDTemplate, "scaleset2", "3", "scaleset1"), vmList: []string{"vmssee6c2000002", "vmssee6c2000003"}, expectError: true, }, { - description: "getNodeNameByIPConfigurationID should return error for wrong ipConfigurationID", + description: "GetNodeNameByIPConfigurationID should return error for wrong ipConfigurationID", scaleSet: "scaleset3", ipConfigurationID: "invalid-configuration-id", vmList: []string{"vmssee6c2000004", "vmssee6c2000005"}, @@ -618,7 +618,7 @@ func TestGetNodeNameByIPConfigurationID(t *testing.T) { expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "", false) mockVMSSVMClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedVMs, nil).AnyTimes() - nodeName, err := ss.getNodeNameByIPConfigurationID(test.ipConfigurationID) + nodeName, err := ss.GetNodeNameByIPConfigurationID(test.ipConfigurationID) if test.expectError { assert.Error(t, err, test.description) continue @@ -2113,7 +2113,7 @@ func TestEnsureBackendPoolDeletedFromNode(t *testing.T) { mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() - nodeResourceGroup, ssName, instanceID, vm, err := ss.ensureBackendPoolDeletedFromNode(&v1.Service{}, test.nodeName, test.backendpoolID) + nodeResourceGroup, ssName, instanceID, vm, err := ss.ensureBackendPoolDeletedFromNode(test.nodeName, test.backendpoolID) assert.Equal(t, test.expectedErr, err, test.description+", but an error occurs") assert.Equal(t, test.expectedNodeResourceGroup, nodeResourceGroup, test.description) assert.Equal(t, test.expectedVMSSName, ssName, test.description) diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/mockvmsets/azure_mock_vmsets.go b/staging/src/k8s.io/legacy-cloud-providers/azure/mockvmsets/azure_mock_vmsets.go index 1746b9bbb9f..8ebddca90b3 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/mockvmsets/azure_mock_vmsets.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/mockvmsets/azure_mock_vmsets.go @@ -291,3 +291,18 @@ func (mr *MockVMSetMockRecorder) GetPrivateIPsByNodeName(name interface{}) *gomo mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPrivateIPsByNodeName", reflect.TypeOf((*MockVMSet)(nil).GetPrivateIPsByNodeName), name) } + +// GetNodeNameByIPConfigurationID mocks base method +func (m *MockVMSet) GetNodeNameByIPConfigurationID(ipConfigurationID string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetNodeNameByIPConfigurationID", ipConfigurationID) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetNodeNameByIPConfigurationID indicates an expected call of GetNodeNameByIPConfigurationID +func (mr *MockVMSetMockRecorder) GetNodeNameByIPConfigurationID(ipConfigurationID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNodeNameByIPConfigurationID", reflect.TypeOf((*MockVMSet)(nil).GetNodeNameByIPConfigurationID), ipConfigurationID) +}