diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_standard.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_standard.go index f9aeb7c496f..73965ac1e55 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_standard.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_standard.go @@ -722,23 +722,23 @@ func (as *availabilitySet) getPrimaryInterfaceWithVMSet(nodeName, vmSetName stri // EnsureHostInPool ensures the given VM's Primary NIC's Primary IP Configuration is // participating in the specified LoadBalancer Backend Pool. -func (as *availabilitySet) EnsureHostInPool(service *v1.Service, nodeName types.NodeName, backendPoolID string, vmSetName string, isInternal bool) error { +func (as *availabilitySet) EnsureHostInPool(service *v1.Service, nodeName types.NodeName, backendPoolID string, vmSetName string, isInternal bool) (string, string, string, *compute.VirtualMachineScaleSetVM, error) { vmName := mapNodeNameToVMName(nodeName) serviceName := getServiceName(service) nic, err := as.getPrimaryInterfaceWithVMSet(vmName, vmSetName) if err != nil { if err == errNotInVMSet { klog.V(3).Infof("EnsureHostInPool skips node %s because it is not in the vmSet %s", nodeName, vmSetName) - return nil + return "", "", "", nil, nil } klog.Errorf("error: az.EnsureHostInPool(%s), az.vmSet.GetPrimaryInterface.Get(%s, %s), err=%v", nodeName, vmName, vmSetName, err) - return err + return "", "", "", nil, err } if nic.ProvisioningState != nil && *nic.ProvisioningState == nicFailedState { klog.Warningf("EnsureHostInPool skips node %s because its primary nic %s is in Failed state", nodeName, *nic.Name) - return nil + return "", "", "", nil, nil } var primaryIPConfig *network.InterfaceIPConfiguration @@ -746,12 +746,12 @@ func (as *availabilitySet) EnsureHostInPool(service *v1.Service, nodeName types. if !as.Cloud.ipv6DualStackEnabled && !ipv6 { primaryIPConfig, err = getPrimaryIPConfig(nic) if err != nil { - return err + return "", "", "", nil, err } } else { primaryIPConfig, err = getIPConfigByIPFamily(nic, ipv6) if err != nil { - return err + return "", "", "", nil, err } } @@ -780,11 +780,11 @@ func (as *availabilitySet) EnsureHostInPool(service *v1.Service, nodeName types. } isSameLB, oldLBName, err := isBackendPoolOnSameLB(backendPoolID, newBackendPoolsIDs) if err != nil { - return err + return "", "", "", nil, err } if !isSameLB { klog.V(4).Infof("Node %q has already been added to LB %q, omit adding it to a new one", nodeName, oldLBName) - return nil + return "", "", "", nil, nil } } @@ -799,10 +799,10 @@ func (as *availabilitySet) EnsureHostInPool(service *v1.Service, nodeName types. klog.V(3).Infof("nicupdate(%s): nic(%s) - updating", serviceName, nicName) err := as.CreateOrUpdateInterface(service, nic) if err != nil { - return err + return "", "", "", nil, err } } - return nil + return "", "", "", nil, nil } // EnsureHostsInPool ensures the given Node's primary IP configurations are @@ -822,7 +822,7 @@ func (as *availabilitySet) EnsureHostsInPool(service *v1.Service, nodes []*v1.No } f := func() error { - err := as.EnsureHostInPool(service, types.NodeName(localNodeName), backendPoolID, vmSetName, isInternal) + _, _, _, _, err := as.EnsureHostInPool(service, types.NodeName(localNodeName), backendPoolID, vmSetName, isInternal) if err != nil { return fmt.Errorf("ensure(%s): backendPoolID(%s) - failed to ensure host in pool: %q", getServiceName(service), backendPoolID, err) } diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmsets.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmsets.go index 43fe595cd98..948c281a1eb 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmsets.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmsets.go @@ -62,7 +62,7 @@ type VMSet interface { EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, backendPoolID string, vmSetName string, isInternal bool) error // EnsureHostInPool ensures the given VM's Primary NIC's Primary IP Configuration is // participating in the specified LoadBalancer Backend Pool. - EnsureHostInPool(service *v1.Service, nodeName types.NodeName, backendPoolID string, vmSetName string, isInternal bool) error + EnsureHostInPool(service *v1.Service, nodeName types.NodeName, backendPoolID string, vmSetName string, isInternal bool) (string, string, string, *compute.VirtualMachineScaleSetVM, error) // EnsureBackendPoolDeleted ensures the loadBalancer backendAddressPools deleted from the specified nodes. EnsureBackendPoolDeleted(service *v1.Service, backendPoolID, vmSetName string, backendAddressPools *[]network.BackendAddressPool) error diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go index b1c422f93a1..b61a3fc9379 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go @@ -63,6 +63,12 @@ const ( vmssVMInstanceUpdateDelay = 3 * time.Second ) +// vmssMetaInfo contains the metadata for a VMSS. +type vmssMetaInfo struct { + vmssName string + resourceGroup string +} + // scaleSet implements VMSet interface for Azure scale set. type scaleSet struct { *Cloud @@ -799,13 +805,13 @@ func (ss *scaleSet) getConfigForScaleSetByIPFamily(config *compute.VirtualMachin } // EnsureHostInPool ensures the given VM's Primary NIC's Primary IP Configuration is -// participating in the specified LoadBalancer Backend Pool. -func (ss *scaleSet) EnsureHostInPool(service *v1.Service, nodeName types.NodeName, backendPoolID string, vmSetName string, isInternal bool) error { +// participating in the specified LoadBalancer Backend Pool, which returns (resourceGroup, vmssName, instanceID, vmssVM, error). +func (ss *scaleSet) EnsureHostInPool(service *v1.Service, nodeName types.NodeName, backendPoolID string, vmSetName string, isInternal bool) (string, string, string, *compute.VirtualMachineScaleSetVM, error) { klog.V(3).Infof("ensuring node %q of scaleset %q in LB backendpool %q", nodeName, vmSetName, backendPoolID) vmName := mapNodeNameToVMName(nodeName) ssName, instanceID, vm, err := ss.getVmssVM(vmName, azcache.CacheReadTypeDefault) if err != nil { - return err + return "", "", "", nil, err } // Check scale set name: @@ -814,19 +820,19 @@ func (ss *scaleSet) EnsureHostInPool(service *v1.Service, nodeName types.NodeNam // don't check vmSet for it. if vmSetName != "" && !ss.useStandardLoadBalancer() && !strings.EqualFold(vmSetName, ssName) { klog.V(3).Infof("EnsureHostInPool skips node %s because it is not in the scaleSet %s", vmName, vmSetName) - return nil + return "", "", "", nil, nil } // Find primary network interface configuration. if vm.NetworkProfileConfiguration.NetworkInterfaceConfigurations == nil { klog.V(4).Infof("EnsureHostInPool: cannot obtain the primary network interface configuration, of vm %s, probably because the vm's being deleted", vmName) - return nil + return "", "", "", nil, nil } networkInterfaceConfigurations := *vm.NetworkProfileConfiguration.NetworkInterfaceConfigurations primaryNetworkInterfaceConfiguration, err := ss.getPrimaryNetworkInterfaceConfiguration(networkInterfaceConfigurations, vmName) if err != nil { - return err + return "", "", "", nil, err } var primaryIPConfiguration *compute.VirtualMachineScaleSetIPConfiguration @@ -836,12 +842,12 @@ func (ss *scaleSet) EnsureHostInPool(service *v1.Service, nodeName types.NodeNam // Find primary IP configuration. primaryIPConfiguration, err = getPrimaryIPConfigFromVMSSNetworkConfig(primaryNetworkInterfaceConfiguration) if err != nil { - return err + return "", "", "", nil, err } } else { primaryIPConfiguration, err = ss.getConfigForScaleSetByIPFamily(primaryNetworkInterfaceConfiguration, vmName, ipv6) if err != nil { - return err + return "", "", "", nil, err } } @@ -860,7 +866,7 @@ func (ss *scaleSet) EnsureHostInPool(service *v1.Service, nodeName types.NodeNam // The backendPoolID has already been found from existing LoadBalancerBackendAddressPools. if foundPool { - return nil + return "", "", "", nil, nil } if ss.useStandardLoadBalancer() && len(newBackendPools) > 0 { @@ -876,11 +882,11 @@ func (ss *scaleSet) EnsureHostInPool(service *v1.Service, nodeName types.NodeNam } isSameLB, oldLBName, err := isBackendPoolOnSameLB(backendPoolID, newBackendPoolsIDs) if err != nil { - return err + return "", "", "", nil, err } if !isSameLB { klog.V(4).Infof("Node %q has already been added to LB %q, omit adding it to a new one", nodeName, oldLBName) - return nil + return "", "", "", nil, nil } } @@ -890,7 +896,7 @@ func (ss *scaleSet) EnsureHostInPool(service *v1.Service, nodeName types.NodeNam ID: to.StringPtr(backendPoolID), }) primaryIPConfiguration.LoadBalancerBackendAddressPools = &newBackendPools - newVM := compute.VirtualMachineScaleSetVM{ + newVM := &compute.VirtualMachineScaleSetVM{ Sku: vm.Sku, Location: vm.Location, VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ @@ -904,23 +910,10 @@ func (ss *scaleSet) EnsureHostInPool(service *v1.Service, nodeName types.NodeNam // Get the node resource group. nodeResourceGroup, err := ss.GetNodeResourceGroup(vmName) if err != nil { - return err + return "", "", "", nil, err } - // Invalidate the cache since right after update - defer ss.deleteCacheForNode(vmName) - - // Update vmssVM with backoff. - ctx, cancel := getContextWithCancel() - defer cancel() - klog.V(2).Infof("EnsureHostInPool begins to update vmssVM(%s) with new backendPoolID %s", vmName, backendPoolID) - rerr := ss.VirtualMachineScaleSetVMsClient.Update(ctx, nodeResourceGroup, ssName, instanceID, newVM, "network_update") - if rerr != nil { - klog.Errorf("EnsureHostInPool VirtualMachineScaleSetVMsClient.Update(%s) with new backendPoolID %s, err: %v", vmName, backendPoolID, err) - return rerr.Error() - } - - return nil + return nodeResourceGroup, ssName, instanceID, newVM, nil } func getVmssAndResourceGroupNameByVMProviderID(providerID string) (string, string, error) { @@ -1050,6 +1043,8 @@ func (ss *scaleSet) ensureVMSSInPool(service *v1.Service, nodes []*v1.Node, back // participating in the specified LoadBalancer Backend Pool. func (ss *scaleSet) EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, backendPoolID string, vmSetName string, isInternal bool) error { hostUpdates := make([]func() error, 0, len(nodes)) + nodeUpdates := make(map[vmssMetaInfo]map[string]compute.VirtualMachineScaleSetVM) + errors := make([]error, 0) for _, node := range nodes { localNodeName := node.Name @@ -1063,39 +1058,80 @@ func (ss *scaleSet) EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, bac continue } - f := func() error { - // Check whether the node is VMAS virtual machine. - managedByAS, err := ss.isNodeManagedByAvailabilitySet(localNodeName, azcache.CacheReadTypeDefault) - if err != nil { - klog.Errorf("Failed to check isNodeManagedByAvailabilitySet(%s): %v", localNodeName, err) - return err - } - - if managedByAS { - // VMAS nodes should also be added to the SLB backends. - if ss.useStandardLoadBalancer() { - return ss.availabilitySet.EnsureHostInPool(service, types.NodeName(localNodeName), backendPoolID, vmSetName, isInternal) - } - - klog.V(3).Infof("EnsureHostsInPool skips node %s because VMAS nodes couldn't be added to basic LB with VMSS backends", localNodeName) - return nil - } - - err = ss.EnsureHostInPool(service, types.NodeName(localNodeName), backendPoolID, vmSetName, isInternal) - if err != nil { - return fmt.Errorf("EnsureHostInPool(%s): backendPoolID(%s) - failed to ensure host in pool: %q", getServiceName(service), backendPoolID, err) - } - return nil + // Check whether the node is VMAS virtual machine. + managedByAS, err := ss.isNodeManagedByAvailabilitySet(localNodeName, azcache.CacheReadTypeDefault) + if err != nil { + klog.Errorf("Failed to check isNodeManagedByAvailabilitySet(%s): %v", localNodeName, err) + errors = append(errors, err) + continue } - hostUpdates = append(hostUpdates, f) + + if managedByAS { + // VMAS nodes should also be added to the SLB backends. + if ss.useStandardLoadBalancer() { + hostUpdates = append(hostUpdates, func() error { + _, _, _, _, err := ss.availabilitySet.EnsureHostInPool(service, types.NodeName(localNodeName), backendPoolID, vmSetName, isInternal) + return err + }) + continue + } + + klog.V(3).Infof("EnsureHostsInPool skips node %s because VMAS nodes couldn't be added to basic LB with VMSS backends", localNodeName) + continue + } + + nodeResourceGroup, nodeVMSS, nodeInstanceID, nodeVMSSVM, err := ss.EnsureHostInPool(service, types.NodeName(localNodeName), backendPoolID, vmSetName, isInternal) + if err != nil { + klog.Errorf("EnsureHostInPool(%s): backendPoolID(%s) - failed to ensure host in pool: %q", getServiceName(service), backendPoolID, err) + errors = append(errors, err) + continue + } + + // No need to update if nodeVMSSVM is nil. + if nodeVMSSVM == nil { + continue + } + + nodeVMSSMetaInfo := vmssMetaInfo{vmssName: nodeVMSS, resourceGroup: nodeResourceGroup} + if v, ok := nodeUpdates[nodeVMSSMetaInfo]; ok { + v[nodeInstanceID] = *nodeVMSSVM + } else { + nodeUpdates[nodeVMSSMetaInfo] = map[string]compute.VirtualMachineScaleSetVM{ + nodeInstanceID: *nodeVMSSVM, + } + } + + // Invalidate the cache since the VMSS VM would be updated. + defer ss.deleteCacheForNode(localNodeName) } - errs := aggregateGoroutinesWithDelay(vmssVMInstanceUpdateDelay, hostUpdates...) + // Update VMs with best effort that have already been added to nodeUpdates. + for meta, update := range nodeUpdates { + hostUpdates = append(hostUpdates, func() error { + ctx, cancel := getContextWithCancel() + defer cancel() + klog.V(2).Infof("EnsureHostInPool begins to UpdateVMs for VMSS(%s, %s) with new backendPoolID %s", meta.resourceGroup, meta.vmssName, backendPoolID) + rerr := ss.VirtualMachineScaleSetVMsClient.UpdateVMs(ctx, meta.resourceGroup, meta.vmssName, update, "network_update") + if rerr != nil { + klog.Errorf("EnsureHostInPool UpdateVMs for VMSS(%s, %s) failed with error %v", meta.resourceGroup, meta.vmssName, rerr.Error()) + return rerr.Error() + } + + return nil + }) + } + errs := utilerrors.AggregateGoroutines(hostUpdates...) if errs != nil { return utilerrors.Flatten(errs) } - // we need to add the LB backend updates back to VMSS model, see issue kubernetes/kubernetes#80365 for detailed information + // Fail if there're other errors. + if len(errors) > 0 { + return utilerrors.Flatten(utilerrors.NewAggregate(errors)) + } + + // Ensure the backendPoolID is also added on VMSS itself. + // Refer to issue kubernetes/kubernetes#80365 for detailed information err := ss.ensureVMSSInPool(service, nodes, backendPoolID, vmSetName) if err != nil { return err @@ -1104,31 +1140,32 @@ func (ss *scaleSet) EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, bac return nil } -// ensureBackendPoolDeletedFromNode ensures the loadBalancer backendAddressPools deleted from the specified node. -func (ss *scaleSet) ensureBackendPoolDeletedFromNode(service *v1.Service, nodeName, backendPoolID string) error { +// ensureBackendPoolDeletedFromNode ensures the loadBalancer backendAddressPools deleted +// from the specified node, which returns (resourceGroup, vmssName, instanceID, vmssVM, error). +func (ss *scaleSet) ensureBackendPoolDeletedFromNode(service *v1.Service, nodeName, backendPoolID string) (string, string, string, *compute.VirtualMachineScaleSetVM, error) { ssName, instanceID, vm, err := ss.getVmssVM(nodeName, azcache.CacheReadTypeDefault) if err != nil { - return err + return "", "", "", nil, err } // Find primary network interface configuration. if vm.NetworkProfileConfiguration.NetworkInterfaceConfigurations == nil { klog.V(4).Infof("EnsureHostInPool: cannot obtain the primary network interface configuration, of vm %s, probably because the vm's being deleted", nodeName) - return nil + return "", "", "", nil, nil } networkInterfaceConfigurations := *vm.NetworkProfileConfiguration.NetworkInterfaceConfigurations primaryNetworkInterfaceConfiguration, err := ss.getPrimaryNetworkInterfaceConfiguration(networkInterfaceConfigurations, nodeName) if err != nil { - return err + return "", "", "", nil, err } // Find primary IP configuration. primaryIPConfiguration, err := getPrimaryIPConfigFromVMSSNetworkConfig(primaryNetworkInterfaceConfiguration) if err != nil { - return err + return "", "", "", nil, err } if primaryIPConfiguration.LoadBalancerBackendAddressPools == nil || len(*primaryIPConfiguration.LoadBalancerBackendAddressPools) == 0 { - return nil + return "", "", "", nil, nil } // Construct new loadBalancerBackendAddressPools and remove backendAddressPools from primary IP configuration. @@ -1146,12 +1183,12 @@ func (ss *scaleSet) ensureBackendPoolDeletedFromNode(service *v1.Service, nodeNa // Pool not found, assume it has been already removed. if !foundPool { - return nil + return "", "", "", nil, nil } // Compose a new vmssVM with added backendPoolID. primaryIPConfiguration.LoadBalancerBackendAddressPools = &newBackendPools - newVM := compute.VirtualMachineScaleSetVM{ + newVM := &compute.VirtualMachineScaleSetVM{ Sku: vm.Sku, Location: vm.Location, VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ @@ -1165,23 +1202,10 @@ func (ss *scaleSet) ensureBackendPoolDeletedFromNode(service *v1.Service, nodeNa // Get the node resource group. nodeResourceGroup, err := ss.GetNodeResourceGroup(nodeName) if err != nil { - return err + return "", "", "", nil, err } - // Invalidate the cache since right after update - defer ss.deleteCacheForNode(nodeName) - - // Update vmssVM with backoff. - ctx, cancel := getContextWithCancel() - defer cancel() - klog.V(2).Infof("ensureBackendPoolDeletedFromNode begins to update vmssVM(%s) with backendPoolID %s", nodeName, backendPoolID) - rerr := ss.VirtualMachineScaleSetVMsClient.Update(ctx, nodeResourceGroup, ssName, instanceID, newVM, "network_update") - if rerr != nil { - klog.Errorf("ensureBackendPoolDeletedFromNode failed to update vmssVM(%s) with backendPoolID %s: %v", nodeName, backendPoolID, err) - } else { - klog.V(2).Infof("ensureBackendPoolDeletedFromNode update vmssVM(%s) with backendPoolID %s succeeded", nodeName, backendPoolID) - } - return rerr.Error() + return nodeResourceGroup, ssName, instanceID, newVM, nil } // getNodeNameByIPConfigurationID gets the node name by IP configuration ID. @@ -1331,43 +1355,82 @@ func (ss *scaleSet) EnsureBackendPoolDeleted(service *v1.Service, backendPoolID, } hostUpdates := make([]func() error, 0, len(ipConfigurationIDs)) + nodeUpdates := make(map[vmssMetaInfo]map[string]compute.VirtualMachineScaleSetVM) + errors := make([]error, 0) for i := range ipConfigurationIDs { ipConfigurationID := ipConfigurationIDs[i] - f := func() error { - var scaleSetName string - var err error - if scaleSetName, err = extractScaleSetNameByProviderID(ipConfigurationID); err == nil { - // Only remove nodes belonging to specified vmSet to basic LB backends. - if !ss.useStandardLoadBalancer() && !strings.EqualFold(scaleSetName, vmSetName) { - return nil - } + var scaleSetName string + var err error + if scaleSetName, err = extractScaleSetNameByProviderID(ipConfigurationID); err == nil { + // Only remove nodes belonging to specified vmSet to basic LB backends. + if !ss.useStandardLoadBalancer() && !strings.EqualFold(scaleSetName, vmSetName) { + continue + } + } + + nodeName, err := ss.getNodeNameByIPConfigurationID(ipConfigurationID) + if err != nil { + if err == ErrorNotVmssInstance { // Do nothing for the VMAS nodes. + continue } - nodeName, err := ss.getNodeNameByIPConfigurationID(ipConfigurationID) - if err != nil { - if err == ErrorNotVmssInstance { // Do nothing for the VMAS nodes. - return nil - } - klog.Errorf("Failed to getNodeNameByIPConfigurationID(%s): %v", ipConfigurationID, err) - return err - } + klog.Errorf("Failed to getNodeNameByIPConfigurationID(%s): %v", ipConfigurationID, err) + errors = append(errors, err) + continue + } - err = ss.ensureBackendPoolDeletedFromNode(service, nodeName, backendPoolID) - if err != nil { - return fmt.Errorf("failed to ensure backend pool %s deleted from node %s: %v", backendPoolID, nodeName, err) + nodeResourceGroup, nodeVMSS, nodeInstanceID, nodeVMSSVM, err := ss.ensureBackendPoolDeletedFromNode(service, nodeName, backendPoolID) + if err != nil { + klog.Errorf("EnsureBackendPoolDeleted(%s): backendPoolID(%s) - failed with error %v", getServiceName(service), backendPoolID, err) + errors = append(errors, err) + continue + } + + // No need to update if nodeVMSSVM is nil. + if nodeVMSSVM == nil { + continue + } + + nodeVMSSMetaInfo := vmssMetaInfo{vmssName: nodeVMSS, resourceGroup: nodeResourceGroup} + if v, ok := nodeUpdates[nodeVMSSMetaInfo]; ok { + v[nodeInstanceID] = *nodeVMSSVM + } else { + nodeUpdates[nodeVMSSMetaInfo] = map[string]compute.VirtualMachineScaleSetVM{ + nodeInstanceID: *nodeVMSSVM, + } + } + + // Invalidate the cache since the VMSS VM would be updated. + defer ss.deleteCacheForNode(nodeName) + } + + // Update VMs with best effort that have already been added to nodeUpdates. + for meta, update := range nodeUpdates { + hostUpdates = append(hostUpdates, func() error { + ctx, cancel := getContextWithCancel() + defer cancel() + klog.V(2).Infof("EnsureBackendPoolDeleted begins to UpdateVMs for VMSS(%s, %s) with backendPoolID %s", meta.resourceGroup, meta.vmssName, backendPoolID) + rerr := ss.VirtualMachineScaleSetVMsClient.UpdateVMs(ctx, meta.resourceGroup, meta.vmssName, update, "network_update") + if rerr != nil { + klog.Errorf("EnsureBackendPoolDeleted UpdateVMs for VMSS(%s, %s) failed with error %v", meta.resourceGroup, meta.vmssName, rerr.Error()) + return rerr.Error() } return nil - } - hostUpdates = append(hostUpdates, f) + }) } - - errs := aggregateGoroutinesWithDelay(vmssVMInstanceUpdateDelay, hostUpdates...) + errs := utilerrors.AggregateGoroutines(hostUpdates...) if errs != nil { return utilerrors.Flatten(errs) } + // Fail if there're other errors. + if len(errors) > 0 { + return utilerrors.Flatten(utilerrors.NewAggregate(errors)) + } + + // Ensure the backendPoolID is also deleted on VMSS itself. err := ss.ensureBackendPoolDeletedFromVMSS(service, backendPoolID, vmSetName, ipConfigurationIDs) if err != nil { return err diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/mockvmsets/azure_mock_vmsets.go b/staging/src/k8s.io/legacy-cloud-providers/azure/mockvmsets/azure_mock_vmsets.go index 93849f77156..9e26b796539 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/mockvmsets/azure_mock_vmsets.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/mockvmsets/azure_mock_vmsets.go @@ -26,7 +26,7 @@ import ( gomock "github.com/golang/mock/gomock" v1 "k8s.io/api/core/v1" types "k8s.io/apimachinery/pkg/types" - cloudprovider "k8s.io/cloud-provider" + cloud_provider "k8s.io/cloud-provider" cache "k8s.io/legacy-cloud-providers/azure/cache" ) @@ -130,10 +130,10 @@ func (mr *MockVMSetMockRecorder) GetNodeNameByProviderID(providerID interface{}) } // GetZoneByNodeName mocks base method -func (m *MockVMSet) GetZoneByNodeName(name string) (cloudprovider.Zone, error) { +func (m *MockVMSet) GetZoneByNodeName(name string) (cloud_provider.Zone, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetZoneByNodeName", name) - ret0, _ := ret[0].(cloudprovider.Zone) + ret0, _ := ret[0].(cloud_provider.Zone) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -188,11 +188,15 @@ func (mr *MockVMSetMockRecorder) EnsureHostsInPool(service, nodes, backendPoolID } // EnsureHostInPool mocks base method -func (m *MockVMSet) EnsureHostInPool(service *v1.Service, nodeName types.NodeName, backendPoolID, vmSetName string, isInternal bool) error { +func (m *MockVMSet) EnsureHostInPool(service *v1.Service, nodeName types.NodeName, backendPoolID, vmSetName string, isInternal bool) (string, string, string, *compute.VirtualMachineScaleSetVM, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "EnsureHostInPool", service, nodeName, backendPoolID, vmSetName, isInternal) - ret0, _ := ret[0].(error) - return ret0 + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(string) + ret2, _ := ret[2].(string) + ret3, _ := ret[3].(*compute.VirtualMachineScaleSetVM) + ret4, _ := ret[4].(error) + return ret0, ret1, ret2, ret3, ret4 } // EnsureHostInPool indicates an expected call of EnsureHostInPool