From 85ecd0e17c2abfa85de37079f56f3707036f36a4 Mon Sep 17 00:00:00 2001 From: Benjamin Pineau Date: Sun, 12 Jul 2020 17:49:57 +0200 Subject: [PATCH 1/2] Azure: per VMSS, incremental VMSS VMs cache Azure's cloud provider VMSS VMs API accesses are mediated through a cache holding and refreshing all VMSS together. Due to that we hit VMSSVM.List API more often than we could: an instance's cache miss or expiration should only require a single VMSS re-list, while it's currently O(n) relative to the number of attached Scale Sets. Under hard pressure (clusters with many attached VMSS that can't all be listed in one sequence of successive API calls) the controller manager might be stuck trying to re-list everything from scratch, then aborting the whole operation; then re-trying and re-triggering API rate-limits, affecting the whole Subscription. This patch replaces the global VMSS VMs cache by per-VMSS VMs caches. Refreshes (VMSS VMs lists) are scoped to the single relevant VMSS; under severe throttling the various caches can be incrementally refreshed. Signed-off-by: Benjamin Pineau --- .../azure/azure_vmss.go | 114 +++++++++-- .../azure/azure_vmss_cache.go | 185 +++++++++++------- 2 files changed, 215 insertions(+), 84 deletions(-) diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go index 7670608ca14..87bec02dafb 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go @@ -61,6 +61,13 @@ type vmssMetaInfo struct { resourceGroup string } +// nodeIdentity identifies a node within a subscription. +type nodeIdentity struct { + resourceGroup string + vmssName string + nodeName string +} + // scaleSet implements VMSet interface for Azure scale set. type scaleSet struct { *Cloud @@ -70,7 +77,7 @@ type scaleSet struct { availabilitySet VMSet vmssCache *azcache.TimedCache - vmssVMCache *azcache.TimedCache + vmssVMCache *sync.Map // [resourcegroup/vmssname]*azcache.TimedCache availabilitySetNodesCache *azcache.TimedCache } @@ -80,6 +87,7 @@ func newScaleSet(az *Cloud) (VMSet, error) { ss := &scaleSet{ Cloud: az, availabilitySet: newAvailabilitySet(az), + vmssVMCache: &sync.Map{}, } if !ss.DisableAvailabilitySetNodes { @@ -94,11 +102,6 @@ func newScaleSet(az *Cloud) (VMSet, error) { return nil, err } - ss.vmssVMCache, err = ss.newVMSSVirtualMachinesCache() - if err != nil { - return nil, err - } - return ss, nil } @@ -139,12 +142,17 @@ func (ss *scaleSet) getVMSS(vmssName string, crt azcache.AzureCacheReadType) (*c return vmss, nil } -// getVmssVM gets virtualMachineScaleSetVM by nodeName from cache. -// It returns cloudprovider.InstanceNotFound if node does not belong to any scale sets. -func (ss *scaleSet) getVmssVM(nodeName string, crt azcache.AzureCacheReadType) (string, string, *compute.VirtualMachineScaleSetVM, error) { +// getVmssVMByNodeIdentity find virtualMachineScaleSetVM by nodeIdentity, using node's parent VMSS cache. +// Returns cloudprovider.InstanceNotFound if the node does not belong to the scale set named in nodeIdentity. +func (ss *scaleSet) getVmssVMByNodeIdentity(node *nodeIdentity, crt azcache.AzureCacheReadType) (string, string, *compute.VirtualMachineScaleSetVM, error) { + cacheKey, cache, err := ss.getVMSSVMCache(node.resourceGroup, node.vmssName) + if err != nil { + return "", "", nil, err + } + getter := func(nodeName string, crt azcache.AzureCacheReadType) (string, string, *compute.VirtualMachineScaleSetVM, bool, error) { var found bool - cached, err := ss.vmssVMCache.Get(vmssVirtualMachinesKey, crt) + cached, err := cache.Get(cacheKey, crt) if err != nil { return "", "", nil, found, err } @@ -159,19 +167,19 @@ func (ss *scaleSet) getVmssVM(nodeName string, crt azcache.AzureCacheReadType) ( return "", "", nil, found, nil } - _, err := getScaleSetVMInstanceID(nodeName) + _, err = getScaleSetVMInstanceID(node.nodeName) if err != nil { return "", "", nil, err } - vmssName, instanceID, vm, found, err := getter(nodeName, crt) + vmssName, instanceID, vm, found, err := getter(node.nodeName, crt) if err != nil { return "", "", nil, err } if !found { - klog.V(2).Infof("Couldn't find VMSS VM with nodeName %s, refreshing the cache", nodeName) - vmssName, instanceID, vm, found, err = getter(nodeName, azcache.CacheReadTypeForceRefresh) + klog.V(2).Infof("Couldn't find VMSS VM with nodeName %s, refreshing the cache", node.nodeName) + vmssName, instanceID, vm, found, err = getter(node.nodeName, azcache.CacheReadTypeForceRefresh) if err != nil { return "", "", nil, err } @@ -187,6 +195,17 @@ func (ss *scaleSet) getVmssVM(nodeName string, crt azcache.AzureCacheReadType) ( return vmssName, instanceID, vm, nil } +// getVmssVM gets virtualMachineScaleSetVM by nodeName from cache. +// Returns cloudprovider.InstanceNotFound if nodeName does not belong to any scale set. +func (ss *scaleSet) getVmssVM(nodeName string, crt azcache.AzureCacheReadType) (string, string, *compute.VirtualMachineScaleSetVM, error) { + node, err := ss.getNodeIdentityByNodeName(nodeName, crt) + if err != nil { + return "", "", nil, err + } + + return ss.getVmssVMByNodeIdentity(node, crt) +} + // GetPowerStatusByNodeName returns the power state of the specified node. func (ss *scaleSet) GetPowerStatusByNodeName(name string) (powerState string, err error) { managedByAS, err := ss.isNodeManagedByAvailabilitySet(name, azcache.CacheReadTypeUnsafe) @@ -222,8 +241,13 @@ func (ss *scaleSet) GetPowerStatusByNodeName(name string) (powerState string, er // getCachedVirtualMachineByInstanceID gets scaleSetVMInfo from cache. // The node must belong to one of scale sets. func (ss *scaleSet) getVmssVMByInstanceID(resourceGroup, scaleSetName, instanceID string, crt azcache.AzureCacheReadType) (*compute.VirtualMachineScaleSetVM, error) { + cacheKey, cache, err := ss.getVMSSVMCache(resourceGroup, scaleSetName) + if err != nil { + return nil, err + } + getter := func(crt azcache.AzureCacheReadType) (vm *compute.VirtualMachineScaleSetVM, found bool, err error) { - cached, err := ss.vmssVMCache.Get(vmssVirtualMachinesKey, crt) + cached, err := cache.Get(cacheKey, crt) if err != nil { return nil, false, err } @@ -590,6 +614,66 @@ func (ss *scaleSet) listScaleSets(resourceGroup string) ([]string, error) { return ssNames, nil } +// getNodeIdentityByNodeName use the VMSS cache to find a node's resourcegroup and vmss, returned in a nodeIdentity. +func (ss *scaleSet) getNodeIdentityByNodeName(nodeName string, crt azcache.AzureCacheReadType) (*nodeIdentity, error) { + getter := func(nodeName string, crt azcache.AzureCacheReadType) (*nodeIdentity, error) { + node := &nodeIdentity{ + nodeName: nodeName, + } + + cached, err := ss.vmssCache.Get(vmssKey, crt) + if err != nil { + return nil, err + } + + vmsses := cached.(*sync.Map) + vmsses.Range(func(key, value interface{}) bool { + v := value.(*vmssEntry) + if v.vmss.Name == nil { + return true + } + + vmssPrefix := *v.vmss.Name + if v.vmss.VirtualMachineProfile != nil && + v.vmss.VirtualMachineProfile.OsProfile != nil && + v.vmss.VirtualMachineProfile.OsProfile.ComputerNamePrefix != nil { + vmssPrefix = *v.vmss.VirtualMachineProfile.OsProfile.ComputerNamePrefix + } + + if strings.EqualFold(vmssPrefix, nodeName[:len(nodeName)-6]) { + node.vmssName = *v.vmss.Name + node.resourceGroup = v.resourceGroup + return false + } + + return true + }) + return node, nil + } + + if _, err := getScaleSetVMInstanceID(nodeName); err != nil { + return nil, err + } + + node, err := getter(nodeName, crt) + if err != nil { + return nil, err + } + if node.vmssName != "" { + return node, nil + } + + klog.V(2).Infof("Couldn't find VMSS for node %s, refreshing the cache", nodeName) + node, err = getter(nodeName, azcache.CacheReadTypeForceRefresh) + if err != nil { + return nil, err + } + if node.vmssName == "" { + return nil, cloudprovider.InstanceNotFound + } + return node, nil +} + // listScaleSetVMs lists VMs belonging to the specified scale set. func (ss *scaleSet) listScaleSetVMs(scaleSetName, resourceGroup string) ([]compute.VirtualMachineScaleSetVM, error) { ctx, cancel := getContextWithCancel() diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache.go index 94a85aade66..44dc282d417 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache.go @@ -20,6 +20,7 @@ package azure import ( "context" + "fmt" "strings" "sync" "time" @@ -36,7 +37,6 @@ var ( vmssNameSeparator = "_" vmssKey = "k8svmssKey" - vmssVirtualMachinesKey = "k8svmssVirtualMachinesKey" availabilitySetNodesKey = "k8sAvailabilitySetNodesKey" availabilitySetNodesCacheTTLDefaultInSeconds = 900 @@ -53,8 +53,9 @@ type vmssVirtualMachinesEntry struct { } type vmssEntry struct { - vmss *compute.VirtualMachineScaleSet - lastUpdate time.Time + vmss *compute.VirtualMachineScaleSet + resourceGroup string + lastUpdate time.Time } func (ss *scaleSet) newVMSSCache() (*azcache.TimedCache, error) { @@ -80,8 +81,9 @@ func (ss *scaleSet) newVMSSCache() (*azcache.TimedCache, error) { continue } localCache.Store(*scaleSet.Name, &vmssEntry{ - vmss: &scaleSet, - lastUpdate: time.Now().UTC(), + vmss: &scaleSet, + resourceGroup: resourceGroup, + lastUpdate: time.Now().UTC(), }) } } @@ -109,15 +111,58 @@ func extractVmssVMName(name string) (string, string, error) { return ssName, instanceID, nil } -func (ss *scaleSet) newVMSSVirtualMachinesCache() (*azcache.TimedCache, error) { +// getVMSSVMCache returns an *azcache.TimedCache and cache key for a VMSS (creating that cache if new). +func (ss *scaleSet) getVMSSVMCache(resourceGroup, vmssName string) (string, *azcache.TimedCache, error) { + cacheKey := strings.ToLower(fmt.Sprintf("%s/%s", resourceGroup, vmssName)) + if entry, ok := ss.vmssVMCache.Load(cacheKey); ok { + cache := entry.(*azcache.TimedCache) + return cacheKey, cache, nil + } + + cache, err := ss.newVMSSVirtualMachinesCache(resourceGroup, vmssName, cacheKey) + if err != nil { + return "", nil, err + } + ss.vmssVMCache.Store(cacheKey, cache) + return cacheKey, cache, nil +} + +// gcVMSSVMCache delete stale VMSS VMs caches from deleted VMSSes. +func (ss *scaleSet) gcVMSSVMCache() error { + cached, err := ss.vmssCache.Get(vmssKey, azcache.CacheReadTypeUnsafe) + if err != nil { + return err + } + + vmsses := cached.(*sync.Map) + removed := map[string]bool{} + ss.vmssVMCache.Range(func(key, value interface{}) bool { + cacheKey := key.(string) + vlistIdx := cacheKey[strings.LastIndex(cacheKey, "/")+1:] + if _, ok := vmsses.Load(vlistIdx); !ok { + removed[cacheKey] = true + } + return true + }) + + for key := range removed { + ss.vmssVMCache.Delete(key) + } + + return nil +} + +// newVMSSVirtualMachinesCache instanciates a new VMs cache for VMs belonging to the provided VMSS. +func (ss *scaleSet) newVMSSVirtualMachinesCache(resourceGroupName, vmssName, cacheKey string) (*azcache.TimedCache, error) { getter := func(key string) (interface{}, error) { localCache := &sync.Map{} // [nodeName]*vmssVirtualMachinesEntry oldCache := make(map[string]vmssVirtualMachinesEntry) - if ss.vmssVMCache != nil { + if vmssCache, ok := ss.vmssVMCache.Load(cacheKey); ok { // get old cache before refreshing the cache - entry, exists, err := ss.vmssVMCache.Store.GetByKey(vmssVirtualMachinesKey) + cache := vmssCache.(*azcache.TimedCache) + entry, exists, err := cache.Store.GetByKey(cacheKey) if err != nil { return nil, err } @@ -133,75 +178,61 @@ func (ss *scaleSet) newVMSSVirtualMachinesCache() (*azcache.TimedCache, error) { } } - allResourceGroups, err := ss.GetResourceGroups() + vms, err := ss.listScaleSetVMs(vmssName, resourceGroupName) if err != nil { return nil, err } - for _, resourceGroup := range allResourceGroups.List() { - scaleSetNames, err := ss.listScaleSets(resourceGroup) - if err != nil { - return nil, err + for i := range vms { + vm := vms[i] + if vm.OsProfile == nil || vm.OsProfile.ComputerName == nil { + klog.Warningf("failed to get computerName for vmssVM (%q)", vmssName) + continue } - for _, ssName := range scaleSetNames { - vms, err := ss.listScaleSetVMs(ssName, resourceGroup) - if err != nil { - return nil, err - } + computerName := strings.ToLower(*vm.OsProfile.ComputerName) + vmssVMCacheEntry := &vmssVirtualMachinesEntry{ + resourceGroup: resourceGroupName, + vmssName: vmssName, + instanceID: to.String(vm.InstanceID), + virtualMachine: &vm, + lastUpdate: time.Now().UTC(), + } + // set cache entry to nil when the VM is under deleting. + if vm.VirtualMachineScaleSetVMProperties != nil && + strings.EqualFold(to.String(vm.VirtualMachineScaleSetVMProperties.ProvisioningState), string(compute.ProvisioningStateDeleting)) { + klog.V(4).Infof("VMSS virtualMachine %q is under deleting, setting its cache to nil", computerName) + vmssVMCacheEntry.virtualMachine = nil + } + localCache.Store(computerName, vmssVMCacheEntry) - for i := range vms { - vm := vms[i] - if vm.OsProfile == nil || vm.OsProfile.ComputerName == nil { - klog.Warningf("failed to get computerName for vmssVM (%q)", ssName) - continue - } + delete(oldCache, computerName) + } - computerName := strings.ToLower(*vm.OsProfile.ComputerName) - vmssVMCacheEntry := &vmssVirtualMachinesEntry{ - resourceGroup: resourceGroup, - vmssName: ssName, - instanceID: to.String(vm.InstanceID), - virtualMachine: &vm, - lastUpdate: time.Now().UTC(), - } - // set cache entry to nil when the VM is under deleting. - if vm.VirtualMachineScaleSetVMProperties != nil && - strings.EqualFold(to.String(vm.VirtualMachineScaleSetVMProperties.ProvisioningState), string(compute.ProvisioningStateDeleting)) { - klog.V(4).Infof("VMSS virtualMachine %q is under deleting, setting its cache to nil", computerName) - vmssVMCacheEntry.virtualMachine = nil - } - localCache.Store(computerName, vmssVMCacheEntry) - - delete(oldCache, computerName) - } + // add old missing cache data with nil entries to prevent aggressive + // ARM calls during cache invalidation + for name, vmEntry := range oldCache { + // if the nil cache entry has existed for 15 minutes in the cache + // then it should not be added back to the cache + if vmEntry.virtualMachine == nil && time.Since(vmEntry.lastUpdate) > 15*time.Minute { + klog.V(5).Infof("ignoring expired entries from old cache for %s", name) + continue + } + lastUpdate := time.Now().UTC() + if vmEntry.virtualMachine == nil { + // if this is already a nil entry then keep the time the nil + // entry was first created, so we can cleanup unwanted entries + lastUpdate = vmEntry.lastUpdate } - // add old missing cache data with nil entries to prevent aggressive - // ARM calls during cache invalidation - for name, vmEntry := range oldCache { - // if the nil cache entry has existed for 15 minutes in the cache - // then it should not be added back to the cache - if vmEntry.virtualMachine == nil && time.Since(vmEntry.lastUpdate) > 15*time.Minute { - klog.V(5).Infof("ignoring expired entries from old cache for %s", name) - continue - } - lastUpdate := time.Now().UTC() - if vmEntry.virtualMachine == nil { - // if this is already a nil entry then keep the time the nil - // entry was first created, so we can cleanup unwanted entries - lastUpdate = vmEntry.lastUpdate - } - - klog.V(5).Infof("adding old entries to new cache for %s", name) - localCache.Store(name, &vmssVirtualMachinesEntry{ - resourceGroup: vmEntry.resourceGroup, - vmssName: vmEntry.vmssName, - instanceID: vmEntry.instanceID, - virtualMachine: nil, - lastUpdate: lastUpdate, - }) - } + klog.V(5).Infof("adding old entries to new cache for %s", name) + localCache.Store(name, &vmssVirtualMachinesEntry{ + resourceGroup: vmEntry.resourceGroup, + vmssName: vmEntry.vmssName, + instanceID: vmEntry.instanceID, + virtualMachine: nil, + lastUpdate: lastUpdate, + }) } return localCache, nil @@ -214,14 +245,30 @@ func (ss *scaleSet) newVMSSVirtualMachinesCache() (*azcache.TimedCache, error) { } func (ss *scaleSet) deleteCacheForNode(nodeName string) error { - cached, err := ss.vmssVMCache.Get(vmssVirtualMachinesKey, azcache.CacheReadTypeUnsafe) + node, err := ss.getNodeIdentityByNodeName(nodeName, azcache.CacheReadTypeUnsafe) if err != nil { klog.Errorf("deleteCacheForNode(%s) failed with error: %v", nodeName, err) return err } - virtualMachines := cached.(*sync.Map) + cacheKey, timedcache, err := ss.getVMSSVMCache(node.resourceGroup, node.vmssName) + if err != nil { + klog.Errorf("deleteCacheForNode(%s) failed with error: %v", nodeName, err) + return err + } + + vmcache, err := timedcache.Get(cacheKey, azcache.CacheReadTypeUnsafe) + if err != nil { + klog.Errorf("deleteCacheForNode(%s) failed with error: %v", nodeName, err) + return err + } + virtualMachines := vmcache.(*sync.Map) virtualMachines.Delete(nodeName) + + if err := ss.gcVMSSVMCache(); err != nil { + klog.Errorf("deleteCacheForNode(%s) failed to gc stale vmss caches: %v", nodeName, err) + } + return nil } From fcb3f1f64c175871252365b9892d1f26ee89d219 Mon Sep 17 00:00:00 2001 From: Benjamin Pineau Date: Tue, 7 Jul 2020 09:12:03 +0200 Subject: [PATCH 2/2] Tests fixes for Azure per-VMSS VMs caches Signed-off-by: Benjamin Pineau --- .../azure/azure_controller_vmss_test.go | 6 +- .../azure/azure_vmss_cache_test.go | 11 +- .../azure/azure_vmss_test.go | 203 ++++++++++-------- 3 files changed, 124 insertions(+), 96 deletions(-) diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_vmss_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_vmss_test.go index 3324929d460..bdf47d0670f 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_vmss_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_vmss_test.go @@ -98,7 +98,7 @@ func TestAttachDiskWithVMSS(t *testing.T) { assert.NoError(t, err, test.desc) testCloud := ss.cloud testCloud.PrimaryScaleSetName = scaleSetName - expectedVMSS := buildTestVMSS(scaleSetName, []string{testLBBackendpoolID0}, false) + expectedVMSS := buildTestVMSSWithLB(scaleSetName, "vmss00-vm-", []string{testLBBackendpoolID0}, false) mockVMSSClient := testCloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), testCloud.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() mockVMSSClient.EXPECT().Get(gomock.Any(), testCloud.ResourceGroup, scaleSetName).Return(expectedVMSS, nil).MaxTimes(1) @@ -194,7 +194,7 @@ func TestDetachDiskWithVMSS(t *testing.T) { assert.NoError(t, err, test.desc) testCloud := ss.cloud testCloud.PrimaryScaleSetName = scaleSetName - expectedVMSS := buildTestVMSS(scaleSetName, []string{testLBBackendpoolID0}, false) + expectedVMSS := buildTestVMSSWithLB(scaleSetName, "vmss00-vm-", []string{testLBBackendpoolID0}, false) mockVMSSClient := testCloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), testCloud.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() mockVMSSClient.EXPECT().Get(gomock.Any(), testCloud.ResourceGroup, scaleSetName).Return(expectedVMSS, nil).MaxTimes(1) @@ -297,7 +297,7 @@ func TestGetDataDisksWithVMSS(t *testing.T) { assert.NoError(t, err, test.desc) testCloud := ss.cloud testCloud.PrimaryScaleSetName = scaleSetName - expectedVMSS := buildTestVMSS(scaleSetName, []string{testLBBackendpoolID0}, false) + expectedVMSS := buildTestVMSSWithLB(scaleSetName, "vmss00-vm-", []string{testLBBackendpoolID0}, false) mockVMSSClient := testCloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), testCloud.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() mockVMSSClient.EXPECT().Get(gomock.Any(), testCloud.ResourceGroup, scaleSetName).Return(expectedVMSS, nil).MaxTimes(1) diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache_test.go index 07fb2073921..8d3f8371f36 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache_test.go @@ -91,12 +91,7 @@ func TestVMSSVMCache(t *testing.T) { ss.cloud.VirtualMachineScaleSetsClient = mockVMSSClient ss.cloud.VirtualMachineScaleSetVMsClient = mockVMSSVMClient - expectedScaleSet := compute.VirtualMachineScaleSet{ - Name: &vmssName, - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedScaleSet := buildTestVMSS(vmssName, "vmssee6c2") mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).AnyTimes() expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, vmssName, "", 0, vmList, "", false) @@ -120,7 +115,9 @@ func TestVMSSVMCache(t *testing.T) { assert.NoError(t, err) // the VM should be removed from cache after deleteCacheForNode(). - cached, err := ss.vmssVMCache.Get(vmssVirtualMachinesKey, azcache.CacheReadTypeDefault) + cacheKey, cache, err := ss.getVMSSVMCache("rg", vmssName) + assert.NoError(t, err) + cached, err := cache.Get(cacheKey, azcache.CacheReadTypeDefault) assert.NoError(t, err) cachedVirtualMachines := cached.(*sync.Map) _, ok := cachedVirtualMachines.Load(vmName) diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go index c0eb15fe87f..8ef023e0342 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go @@ -67,7 +67,7 @@ func newTestScaleSetWithState(ctrl *gomock.Controller) (*scaleSet, error) { return ss.(*scaleSet), nil } -func buildTestVMSS(name string, lbBackendpoolIDs []string, ipv6 bool) compute.VirtualMachineScaleSet { +func buildTestVMSSWithLB(name, namePrefix string, lbBackendpoolIDs []string, ipv6 bool) compute.VirtualMachineScaleSet { lbBackendpools := make([]compute.SubResource, 0) for _, id := range lbBackendpoolIDs { lbBackendpools = append(lbBackendpools, compute.SubResource{ID: to.StringPtr(id)}) @@ -93,6 +93,9 @@ func buildTestVMSS(name string, lbBackendpoolIDs []string, ipv6 bool) compute.Vi VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ ProvisioningState: to.StringPtr("Running"), VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{ + OsProfile: &compute.VirtualMachineScaleSetOSProfile{ + ComputerNamePrefix: &namePrefix, + }, NetworkProfile: &compute.VirtualMachineScaleSetNetworkProfile{ NetworkInterfaceConfigurations: &[]compute.VirtualMachineScaleSetNetworkConfiguration{ { @@ -110,6 +113,19 @@ func buildTestVMSS(name string, lbBackendpoolIDs []string, ipv6 bool) compute.Vi return expectedVMSS } +func buildTestVMSS(name, computerNamePrefix string) compute.VirtualMachineScaleSet { + return compute.VirtualMachineScaleSet{ + Name: &name, + VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ + VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{ + OsProfile: &compute.VirtualMachineScaleSetOSProfile{ + ComputerNamePrefix: &computerNamePrefix, + }, + }, + }, + } +} + func buildTestVirtualMachineEnv(ss *Cloud, scaleSetName, zone string, faultDomain int32, vmList []string, state string, isIPv6 bool) ([]compute.VirtualMachineScaleSetVM, network.Interface, network.PublicIPAddress) { expectedVMSSVMs := make([]compute.VirtualMachineScaleSetVM, 0) expectedInterface := network.Interface{} @@ -261,6 +277,82 @@ func TestGetScaleSetVMInstanceID(t *testing.T) { } } +func TestGetNodeIdentityByNodeName(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + testCases := []struct { + description string + vmList []string + nodeName string + expected *nodeIdentity + scaleSet string + computerName string + expectError bool + }{ + { + description: "scaleSet should get node identity by node name", + vmList: []string{"vmssee6c2000000", "vmssee6c2000001"}, + nodeName: "vmssee6c2000001", + scaleSet: "vmssee6c2", + computerName: "vmssee6c2", + expected: &nodeIdentity{"rg", "vmssee6c2", "vmssee6c2000001"}, + }, + { + description: "scaleSet should get node identity when computerNamePrefix differs from vmss name", + vmList: []string{"vmssee6c2000000", "vmssee6c2000001"}, + nodeName: "vmssee6c2000001", + scaleSet: "ss", + computerName: "vmssee6c2", + expected: &nodeIdentity{"rg", "ss", "vmssee6c2000001"}, + }, + { + description: "scaleSet should get node identity by node name with upper cases hostname", + vmList: []string{"VMSSEE6C2000000", "VMSSEE6C2000001"}, + nodeName: "vmssee6c2000001", + scaleSet: "ss", + computerName: "vmssee6c2", + expected: &nodeIdentity{"rg", "ss", "vmssee6c2000001"}, + }, + { + description: "scaleSet should not get node identity for non-existing nodes", + vmList: []string{"vmssee6c2000000", "vmssee6c2000001"}, + nodeName: "agente6c2000005", + scaleSet: "ss", + computerName: "vmssee6c2", + expectError: true, + }, + } + + for _, test := range testCases { + ss, err := newTestScaleSet(ctrl) + assert.NoError(t, err, test.description) + + mockVMSSClient := mockvmssclient.NewMockInterface(ctrl) + mockVMSSVMClient := mockvmssvmclient.NewMockInterface(ctrl) + ss.cloud.VirtualMachineScaleSetsClient = mockVMSSClient + ss.cloud.VirtualMachineScaleSetVMsClient = mockVMSSVMClient + + expectedScaleSet := buildTestVMSS(test.scaleSet, test.computerName) + mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).AnyTimes() + + expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "", false) + mockVMSSVMClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedVMs, nil).AnyTimes() + + mockVMsClient := ss.cloud.VirtualMachinesClient.(*mockvmclient.MockInterface) + mockVMsClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachine{}, nil).AnyTimes() + + nodeID, err := ss.getNodeIdentityByNodeName(test.nodeName, azcache.CacheReadTypeDefault) + if test.expectError { + assert.Error(t, err, test.description) + continue + } + + assert.NoError(t, err, test.description) + assert.Equal(t, test.expected, nodeID, test.description) + } +} + func TestGetInstanceIDByNodeName(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -305,12 +397,7 @@ func TestGetInstanceIDByNodeName(t *testing.T) { ss.cloud.VirtualMachineScaleSetsClient = mockVMSSClient ss.cloud.VirtualMachineScaleSetVMsClient = mockVMSSVMClient - expectedScaleSet := compute.VirtualMachineScaleSet{ - Name: &test.scaleSet, - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedScaleSet := buildTestVMSS(test.scaleSet, "vmssee6c2") mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).AnyTimes() expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "", false) @@ -395,12 +482,7 @@ func TestGetZoneByNodeName(t *testing.T) { ss.cloud.VirtualMachineScaleSetsClient = mockVMSSClient ss.cloud.VirtualMachineScaleSetVMsClient = mockVMSSVMClient - expectedScaleSet := compute.VirtualMachineScaleSet{ - Name: &test.scaleSet, - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedScaleSet := buildTestVMSS(test.scaleSet, "vmssee6c2") mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).AnyTimes() expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, test.zone, test.faultDomain, test.vmList, "", false) @@ -462,12 +544,7 @@ func TestGetIPByNodeName(t *testing.T) { ss.cloud.InterfacesClient = mockInterfaceClient ss.cloud.PublicIPAddressesClient = mockPIPClient - expectedScaleSet := compute.VirtualMachineScaleSet{ - Name: &test.scaleSet, - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedScaleSet := buildTestVMSS(test.scaleSet, "vmssee6c2") mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).AnyTimes() expectedVMs, expectedInterface, expectedPIP := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "", false) @@ -535,12 +612,7 @@ func TestGetNodeNameByIPConfigurationID(t *testing.T) { ss.cloud.VirtualMachineScaleSetsClient = mockVMSSClient ss.cloud.VirtualMachineScaleSetVMsClient = mockVMSSVMClient - expectedScaleSet := compute.VirtualMachineScaleSet{ - Name: &test.scaleSet, - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedScaleSet := buildTestVMSS(test.scaleSet, "vmssee6c2") mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).AnyTimes() expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "", false) @@ -679,12 +751,7 @@ func TestGetVmssVM(t *testing.T) { ss, err := newTestScaleSet(ctrl) assert.NoError(t, err, test.description) - expectedVMSS := compute.VirtualMachineScaleSet{ - Name: to.StringPtr(test.existedVMSSName), - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedVMSS := buildTestVMSS(test.existedVMSSName, "vmss-vm-") mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() @@ -735,12 +802,7 @@ func TestGetPowerStatusByNodeName(t *testing.T) { ss, err := newTestScaleSet(ctrl) assert.NoError(t, err, "unexpected error when creating test VMSS") - expectedVMSS := compute.VirtualMachineScaleSet{ - Name: to.StringPtr(testVMSSName), - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-") mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() @@ -829,12 +891,7 @@ func TestGetInstanceTypeByNodeName(t *testing.T) { ss, err := newTestScaleSet(ctrl) assert.NoError(t, err, "unexpected error when creating test VMSS") - expectedVMSS := compute.VirtualMachineScaleSet{ - Name: to.StringPtr(testVMSSName), - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-") mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() @@ -987,11 +1044,7 @@ func TestGetPrimaryInterface(t *testing.T) { ss, err := newTestScaleSet(ctrl) assert.NoError(t, err, "unexpected error when creating test VMSS") - expectedVMSS := compute.VirtualMachineScaleSet{ - Name: to.StringPtr(testVMSSName), - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}}, - } + expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-") mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, test.vmssClientErr).AnyTimes() @@ -1114,12 +1167,7 @@ func TestGetPrivateIPsByNodeName(t *testing.T) { ss, err := newTestScaleSet(ctrl) assert.NoError(t, err, "unexpected error when creating test VMSS") - expectedVMSS := compute.VirtualMachineScaleSet{ - Name: to.StringPtr(testVMSSName), - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-") mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() @@ -1318,12 +1366,7 @@ func TestGetAgentPoolScaleSets(t *testing.T) { ss, err := newTestScaleSet(ctrl) assert.NoError(t, err, "unexpected error when creating test VMSS") - expectedVMSS := compute.VirtualMachineScaleSet{ - Name: to.StringPtr(testVMSSName), - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-") mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() @@ -1419,12 +1462,7 @@ func TestGetVMSetNames(t *testing.T) { ss, err := newTestScaleSet(ctrl) assert.NoError(t, err, "unexpected error when creating test VMSS") - expectedVMSS := compute.VirtualMachineScaleSet{ - Name: to.StringPtr(testVMSSName), - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-") mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() @@ -1703,12 +1741,7 @@ func TestEnsureHostInPool(t *testing.T) { ss.LoadBalancerSku = loadBalancerSkuStandard } - expectedVMSS := compute.VirtualMachineScaleSet{ - Name: to.StringPtr(testVMSSName), - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-") mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() @@ -1876,7 +1909,7 @@ func TestEnsureVMSSInPool(t *testing.T) { ss.LoadBalancerSku = loadBalancerSkuStandard } - expectedVMSS := buildTestVMSS(testVMSSName, []string{testLBBackendpoolID0}, test.setIPv6Config) + expectedVMSS := buildTestVMSSWithLB(testVMSSName, "vmss-vm-", []string{testLBBackendpoolID0}, test.setIPv6Config) if test.isVMSSDeallocating { expectedVMSS.ProvisioningState = &virtualMachineScaleSetsDeallocating } @@ -1973,7 +2006,7 @@ func TestEnsureHostsInPool(t *testing.T) { ss.LoadBalancerSku = loadBalancerSkuStandard ss.ExcludeMasterFromStandardLB = to.BoolPtr(true) - expectedVMSS := buildTestVMSS(testVMSSName, []string{testLBBackendpoolID0}, false) + expectedVMSS := buildTestVMSSWithLB(testVMSSName, "vmss-vm-", []string{testLBBackendpoolID0}, false) mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() mockVMSSClient.EXPECT().Get(gomock.Any(), ss.ResourceGroup, testVMSSName).Return(expectedVMSS, nil).MaxTimes(1) @@ -2061,12 +2094,7 @@ func TestEnsureBackendPoolDeletedFromNode(t *testing.T) { ss, err := newTestScaleSet(ctrl) assert.NoError(t, err, test.description) - expectedVMSS := compute.VirtualMachineScaleSet{ - Name: to.StringPtr(testVMSSName), - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-") mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() @@ -2155,7 +2183,7 @@ func TestEnsureBackendPoolDeletedFromVMSS(t *testing.T) { ss.LoadBalancerSku = loadBalancerSkuStandard - expectedVMSS := buildTestVMSS(testVMSSName, []string{testLBBackendpoolID0}, false) + expectedVMSS := buildTestVMSSWithLB(testVMSSName, "vmss-vm-", []string{testLBBackendpoolID0}, false) if test.isVMSSDeallocating { expectedVMSS.ProvisioningState = &virtualMachineScaleSetsDeallocating } @@ -2249,7 +2277,7 @@ func TestEnsureBackendPoolDeleted(t *testing.T) { ss, err := newTestScaleSet(ctrl) assert.NoError(t, err, test.description) - expectedVMSS := buildTestVMSS(testVMSSName, []string{testLBBackendpoolID0}, false) + expectedVMSS := buildTestVMSSWithLB(testVMSSName, "vmss-vm-", []string{testLBBackendpoolID0}, false) mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() mockVMSSClient.EXPECT().Get(gomock.Any(), ss.ResourceGroup, testVMSSName).Return(expectedVMSS, nil).MaxTimes(1) @@ -2308,11 +2336,12 @@ func TestEnsureBackendPoolDeletedConcurrently(t *testing.T) { }, }, } - vmss0 := buildTestVMSS("vmss-0", []string{testLBBackendpoolID0, testLBBackendpoolID1}, false) - vmss1 := buildTestVMSS("vmss-1", []string{testLBBackendpoolID0, testLBBackendpoolID1}, false) - expectedVMSSVMsOfVMSS0, _, _ := buildTestVirtualMachineEnv(ss.cloud, "vmss-0", "", 0, []string{"vmss-vm-000000"}, "succeeded", false) - expectedVMSSVMsOfVMSS1, _, _ := buildTestVirtualMachineEnv(ss.cloud, "vmss-1", "", 0, []string{"vmss-vm-000001"}, "succeeded", false) + vmss0 := buildTestVMSSWithLB("vmss-0", "vmss-0-vm-", []string{testLBBackendpoolID0, testLBBackendpoolID1}, false) + vmss1 := buildTestVMSSWithLB("vmss-1", "vmss-1-vm-", []string{testLBBackendpoolID0, testLBBackendpoolID1}, false) + + expectedVMSSVMsOfVMSS0, _, _ := buildTestVirtualMachineEnv(ss.cloud, "vmss-0", "", 0, []string{"vmss-0-vm-000000"}, "succeeded", false) + expectedVMSSVMsOfVMSS1, _, _ := buildTestVirtualMachineEnv(ss.cloud, "vmss-1", "", 0, []string{"vmss-1-vm-000001"}, "succeeded", false) for _, expectedVMSSVMs := range [][]compute.VirtualMachineScaleSetVM{expectedVMSSVMsOfVMSS0, expectedVMSSVMsOfVMSS1} { vmssVMNetworkConfigs := expectedVMSSVMs[0].NetworkProfileConfiguration vmssVMIPConfigs := (*vmssVMNetworkConfigs.NetworkInterfaceConfigurations)[0].VirtualMachineScaleSetNetworkConfigurationProperties.IPConfigurations @@ -2322,11 +2351,13 @@ func TestEnsureBackendPoolDeletedConcurrently(t *testing.T) { mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{vmss0, vmss1}, nil).AnyTimes() + mockVMSSClient.EXPECT().List(gomock.Any(), "rg1").Return(nil, nil).AnyTimes() mockVMSSClient.EXPECT().Get(gomock.Any(), ss.ResourceGroup, "vmss-0").Return(vmss0, nil).MaxTimes(2) mockVMSSClient.EXPECT().Get(gomock.Any(), ss.ResourceGroup, "vmss-1").Return(vmss1, nil).MaxTimes(2) mockVMSSClient.EXPECT().CreateOrUpdate(gomock.Any(), ss.ResourceGroup, gomock.Any(), gomock.Any()).Return(nil).Times(2) mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) + mockVMSSVMClient.EXPECT().List(gomock.Any(), "rg1", "vmss-0", gomock.Any()).Return(nil, nil).AnyTimes() mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, "vmss-0", gomock.Any()).Return(expectedVMSSVMsOfVMSS0, nil).AnyTimes() mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, "vmss-1", gomock.Any()).Return(expectedVMSSVMsOfVMSS1, nil).AnyTimes() mockVMSSVMClient.EXPECT().UpdateVMs(gomock.Any(), ss.ResourceGroup, gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(2)