diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_vmss_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_vmss_test.go index 3324929d460..bdf47d0670f 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_vmss_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_vmss_test.go @@ -98,7 +98,7 @@ func TestAttachDiskWithVMSS(t *testing.T) { assert.NoError(t, err, test.desc) testCloud := ss.cloud testCloud.PrimaryScaleSetName = scaleSetName - expectedVMSS := buildTestVMSS(scaleSetName, []string{testLBBackendpoolID0}, false) + expectedVMSS := buildTestVMSSWithLB(scaleSetName, "vmss00-vm-", []string{testLBBackendpoolID0}, false) mockVMSSClient := testCloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), testCloud.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() mockVMSSClient.EXPECT().Get(gomock.Any(), testCloud.ResourceGroup, scaleSetName).Return(expectedVMSS, nil).MaxTimes(1) @@ -194,7 +194,7 @@ func TestDetachDiskWithVMSS(t *testing.T) { assert.NoError(t, err, test.desc) testCloud := ss.cloud testCloud.PrimaryScaleSetName = scaleSetName - expectedVMSS := buildTestVMSS(scaleSetName, []string{testLBBackendpoolID0}, false) + expectedVMSS := buildTestVMSSWithLB(scaleSetName, "vmss00-vm-", []string{testLBBackendpoolID0}, false) mockVMSSClient := testCloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), testCloud.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() mockVMSSClient.EXPECT().Get(gomock.Any(), testCloud.ResourceGroup, scaleSetName).Return(expectedVMSS, nil).MaxTimes(1) @@ -297,7 +297,7 @@ func TestGetDataDisksWithVMSS(t *testing.T) { assert.NoError(t, err, test.desc) testCloud := ss.cloud testCloud.PrimaryScaleSetName = scaleSetName - expectedVMSS := buildTestVMSS(scaleSetName, []string{testLBBackendpoolID0}, false) + expectedVMSS := buildTestVMSSWithLB(scaleSetName, "vmss00-vm-", []string{testLBBackendpoolID0}, false) mockVMSSClient := testCloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), testCloud.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() mockVMSSClient.EXPECT().Get(gomock.Any(), testCloud.ResourceGroup, scaleSetName).Return(expectedVMSS, nil).MaxTimes(1) diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go index 7670608ca14..87bec02dafb 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go @@ -61,6 +61,13 @@ type vmssMetaInfo struct { resourceGroup string } +// nodeIdentity identifies a node within a subscription. +type nodeIdentity struct { + resourceGroup string + vmssName string + nodeName string +} + // scaleSet implements VMSet interface for Azure scale set. type scaleSet struct { *Cloud @@ -70,7 +77,7 @@ type scaleSet struct { availabilitySet VMSet vmssCache *azcache.TimedCache - vmssVMCache *azcache.TimedCache + vmssVMCache *sync.Map // [resourcegroup/vmssname]*azcache.TimedCache availabilitySetNodesCache *azcache.TimedCache } @@ -80,6 +87,7 @@ func newScaleSet(az *Cloud) (VMSet, error) { ss := &scaleSet{ Cloud: az, availabilitySet: newAvailabilitySet(az), + vmssVMCache: &sync.Map{}, } if !ss.DisableAvailabilitySetNodes { @@ -94,11 +102,6 @@ func newScaleSet(az *Cloud) (VMSet, error) { return nil, err } - ss.vmssVMCache, err = ss.newVMSSVirtualMachinesCache() - if err != nil { - return nil, err - } - return ss, nil } @@ -139,12 +142,17 @@ func (ss *scaleSet) getVMSS(vmssName string, crt azcache.AzureCacheReadType) (*c return vmss, nil } -// getVmssVM gets virtualMachineScaleSetVM by nodeName from cache. -// It returns cloudprovider.InstanceNotFound if node does not belong to any scale sets. -func (ss *scaleSet) getVmssVM(nodeName string, crt azcache.AzureCacheReadType) (string, string, *compute.VirtualMachineScaleSetVM, error) { +// getVmssVMByNodeIdentity find virtualMachineScaleSetVM by nodeIdentity, using node's parent VMSS cache. +// Returns cloudprovider.InstanceNotFound if the node does not belong to the scale set named in nodeIdentity. +func (ss *scaleSet) getVmssVMByNodeIdentity(node *nodeIdentity, crt azcache.AzureCacheReadType) (string, string, *compute.VirtualMachineScaleSetVM, error) { + cacheKey, cache, err := ss.getVMSSVMCache(node.resourceGroup, node.vmssName) + if err != nil { + return "", "", nil, err + } + getter := func(nodeName string, crt azcache.AzureCacheReadType) (string, string, *compute.VirtualMachineScaleSetVM, bool, error) { var found bool - cached, err := ss.vmssVMCache.Get(vmssVirtualMachinesKey, crt) + cached, err := cache.Get(cacheKey, crt) if err != nil { return "", "", nil, found, err } @@ -159,19 +167,19 @@ func (ss *scaleSet) getVmssVM(nodeName string, crt azcache.AzureCacheReadType) ( return "", "", nil, found, nil } - _, err := getScaleSetVMInstanceID(nodeName) + _, err = getScaleSetVMInstanceID(node.nodeName) if err != nil { return "", "", nil, err } - vmssName, instanceID, vm, found, err := getter(nodeName, crt) + vmssName, instanceID, vm, found, err := getter(node.nodeName, crt) if err != nil { return "", "", nil, err } if !found { - klog.V(2).Infof("Couldn't find VMSS VM with nodeName %s, refreshing the cache", nodeName) - vmssName, instanceID, vm, found, err = getter(nodeName, azcache.CacheReadTypeForceRefresh) + klog.V(2).Infof("Couldn't find VMSS VM with nodeName %s, refreshing the cache", node.nodeName) + vmssName, instanceID, vm, found, err = getter(node.nodeName, azcache.CacheReadTypeForceRefresh) if err != nil { return "", "", nil, err } @@ -187,6 +195,17 @@ func (ss *scaleSet) getVmssVM(nodeName string, crt azcache.AzureCacheReadType) ( return vmssName, instanceID, vm, nil } +// getVmssVM gets virtualMachineScaleSetVM by nodeName from cache. +// Returns cloudprovider.InstanceNotFound if nodeName does not belong to any scale set. +func (ss *scaleSet) getVmssVM(nodeName string, crt azcache.AzureCacheReadType) (string, string, *compute.VirtualMachineScaleSetVM, error) { + node, err := ss.getNodeIdentityByNodeName(nodeName, crt) + if err != nil { + return "", "", nil, err + } + + return ss.getVmssVMByNodeIdentity(node, crt) +} + // GetPowerStatusByNodeName returns the power state of the specified node. func (ss *scaleSet) GetPowerStatusByNodeName(name string) (powerState string, err error) { managedByAS, err := ss.isNodeManagedByAvailabilitySet(name, azcache.CacheReadTypeUnsafe) @@ -222,8 +241,13 @@ func (ss *scaleSet) GetPowerStatusByNodeName(name string) (powerState string, er // getCachedVirtualMachineByInstanceID gets scaleSetVMInfo from cache. // The node must belong to one of scale sets. func (ss *scaleSet) getVmssVMByInstanceID(resourceGroup, scaleSetName, instanceID string, crt azcache.AzureCacheReadType) (*compute.VirtualMachineScaleSetVM, error) { + cacheKey, cache, err := ss.getVMSSVMCache(resourceGroup, scaleSetName) + if err != nil { + return nil, err + } + getter := func(crt azcache.AzureCacheReadType) (vm *compute.VirtualMachineScaleSetVM, found bool, err error) { - cached, err := ss.vmssVMCache.Get(vmssVirtualMachinesKey, crt) + cached, err := cache.Get(cacheKey, crt) if err != nil { return nil, false, err } @@ -590,6 +614,66 @@ func (ss *scaleSet) listScaleSets(resourceGroup string) ([]string, error) { return ssNames, nil } +// getNodeIdentityByNodeName use the VMSS cache to find a node's resourcegroup and vmss, returned in a nodeIdentity. +func (ss *scaleSet) getNodeIdentityByNodeName(nodeName string, crt azcache.AzureCacheReadType) (*nodeIdentity, error) { + getter := func(nodeName string, crt azcache.AzureCacheReadType) (*nodeIdentity, error) { + node := &nodeIdentity{ + nodeName: nodeName, + } + + cached, err := ss.vmssCache.Get(vmssKey, crt) + if err != nil { + return nil, err + } + + vmsses := cached.(*sync.Map) + vmsses.Range(func(key, value interface{}) bool { + v := value.(*vmssEntry) + if v.vmss.Name == nil { + return true + } + + vmssPrefix := *v.vmss.Name + if v.vmss.VirtualMachineProfile != nil && + v.vmss.VirtualMachineProfile.OsProfile != nil && + v.vmss.VirtualMachineProfile.OsProfile.ComputerNamePrefix != nil { + vmssPrefix = *v.vmss.VirtualMachineProfile.OsProfile.ComputerNamePrefix + } + + if strings.EqualFold(vmssPrefix, nodeName[:len(nodeName)-6]) { + node.vmssName = *v.vmss.Name + node.resourceGroup = v.resourceGroup + return false + } + + return true + }) + return node, nil + } + + if _, err := getScaleSetVMInstanceID(nodeName); err != nil { + return nil, err + } + + node, err := getter(nodeName, crt) + if err != nil { + return nil, err + } + if node.vmssName != "" { + return node, nil + } + + klog.V(2).Infof("Couldn't find VMSS for node %s, refreshing the cache", nodeName) + node, err = getter(nodeName, azcache.CacheReadTypeForceRefresh) + if err != nil { + return nil, err + } + if node.vmssName == "" { + return nil, cloudprovider.InstanceNotFound + } + return node, nil +} + // listScaleSetVMs lists VMs belonging to the specified scale set. func (ss *scaleSet) listScaleSetVMs(scaleSetName, resourceGroup string) ([]compute.VirtualMachineScaleSetVM, error) { ctx, cancel := getContextWithCancel() diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache.go index 94a85aade66..44dc282d417 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache.go @@ -20,6 +20,7 @@ package azure import ( "context" + "fmt" "strings" "sync" "time" @@ -36,7 +37,6 @@ var ( vmssNameSeparator = "_" vmssKey = "k8svmssKey" - vmssVirtualMachinesKey = "k8svmssVirtualMachinesKey" availabilitySetNodesKey = "k8sAvailabilitySetNodesKey" availabilitySetNodesCacheTTLDefaultInSeconds = 900 @@ -53,8 +53,9 @@ type vmssVirtualMachinesEntry struct { } type vmssEntry struct { - vmss *compute.VirtualMachineScaleSet - lastUpdate time.Time + vmss *compute.VirtualMachineScaleSet + resourceGroup string + lastUpdate time.Time } func (ss *scaleSet) newVMSSCache() (*azcache.TimedCache, error) { @@ -80,8 +81,9 @@ func (ss *scaleSet) newVMSSCache() (*azcache.TimedCache, error) { continue } localCache.Store(*scaleSet.Name, &vmssEntry{ - vmss: &scaleSet, - lastUpdate: time.Now().UTC(), + vmss: &scaleSet, + resourceGroup: resourceGroup, + lastUpdate: time.Now().UTC(), }) } } @@ -109,15 +111,58 @@ func extractVmssVMName(name string) (string, string, error) { return ssName, instanceID, nil } -func (ss *scaleSet) newVMSSVirtualMachinesCache() (*azcache.TimedCache, error) { +// getVMSSVMCache returns an *azcache.TimedCache and cache key for a VMSS (creating that cache if new). +func (ss *scaleSet) getVMSSVMCache(resourceGroup, vmssName string) (string, *azcache.TimedCache, error) { + cacheKey := strings.ToLower(fmt.Sprintf("%s/%s", resourceGroup, vmssName)) + if entry, ok := ss.vmssVMCache.Load(cacheKey); ok { + cache := entry.(*azcache.TimedCache) + return cacheKey, cache, nil + } + + cache, err := ss.newVMSSVirtualMachinesCache(resourceGroup, vmssName, cacheKey) + if err != nil { + return "", nil, err + } + ss.vmssVMCache.Store(cacheKey, cache) + return cacheKey, cache, nil +} + +// gcVMSSVMCache delete stale VMSS VMs caches from deleted VMSSes. +func (ss *scaleSet) gcVMSSVMCache() error { + cached, err := ss.vmssCache.Get(vmssKey, azcache.CacheReadTypeUnsafe) + if err != nil { + return err + } + + vmsses := cached.(*sync.Map) + removed := map[string]bool{} + ss.vmssVMCache.Range(func(key, value interface{}) bool { + cacheKey := key.(string) + vlistIdx := cacheKey[strings.LastIndex(cacheKey, "/")+1:] + if _, ok := vmsses.Load(vlistIdx); !ok { + removed[cacheKey] = true + } + return true + }) + + for key := range removed { + ss.vmssVMCache.Delete(key) + } + + return nil +} + +// newVMSSVirtualMachinesCache instanciates a new VMs cache for VMs belonging to the provided VMSS. +func (ss *scaleSet) newVMSSVirtualMachinesCache(resourceGroupName, vmssName, cacheKey string) (*azcache.TimedCache, error) { getter := func(key string) (interface{}, error) { localCache := &sync.Map{} // [nodeName]*vmssVirtualMachinesEntry oldCache := make(map[string]vmssVirtualMachinesEntry) - if ss.vmssVMCache != nil { + if vmssCache, ok := ss.vmssVMCache.Load(cacheKey); ok { // get old cache before refreshing the cache - entry, exists, err := ss.vmssVMCache.Store.GetByKey(vmssVirtualMachinesKey) + cache := vmssCache.(*azcache.TimedCache) + entry, exists, err := cache.Store.GetByKey(cacheKey) if err != nil { return nil, err } @@ -133,75 +178,61 @@ func (ss *scaleSet) newVMSSVirtualMachinesCache() (*azcache.TimedCache, error) { } } - allResourceGroups, err := ss.GetResourceGroups() + vms, err := ss.listScaleSetVMs(vmssName, resourceGroupName) if err != nil { return nil, err } - for _, resourceGroup := range allResourceGroups.List() { - scaleSetNames, err := ss.listScaleSets(resourceGroup) - if err != nil { - return nil, err + for i := range vms { + vm := vms[i] + if vm.OsProfile == nil || vm.OsProfile.ComputerName == nil { + klog.Warningf("failed to get computerName for vmssVM (%q)", vmssName) + continue } - for _, ssName := range scaleSetNames { - vms, err := ss.listScaleSetVMs(ssName, resourceGroup) - if err != nil { - return nil, err - } + computerName := strings.ToLower(*vm.OsProfile.ComputerName) + vmssVMCacheEntry := &vmssVirtualMachinesEntry{ + resourceGroup: resourceGroupName, + vmssName: vmssName, + instanceID: to.String(vm.InstanceID), + virtualMachine: &vm, + lastUpdate: time.Now().UTC(), + } + // set cache entry to nil when the VM is under deleting. + if vm.VirtualMachineScaleSetVMProperties != nil && + strings.EqualFold(to.String(vm.VirtualMachineScaleSetVMProperties.ProvisioningState), string(compute.ProvisioningStateDeleting)) { + klog.V(4).Infof("VMSS virtualMachine %q is under deleting, setting its cache to nil", computerName) + vmssVMCacheEntry.virtualMachine = nil + } + localCache.Store(computerName, vmssVMCacheEntry) - for i := range vms { - vm := vms[i] - if vm.OsProfile == nil || vm.OsProfile.ComputerName == nil { - klog.Warningf("failed to get computerName for vmssVM (%q)", ssName) - continue - } + delete(oldCache, computerName) + } - computerName := strings.ToLower(*vm.OsProfile.ComputerName) - vmssVMCacheEntry := &vmssVirtualMachinesEntry{ - resourceGroup: resourceGroup, - vmssName: ssName, - instanceID: to.String(vm.InstanceID), - virtualMachine: &vm, - lastUpdate: time.Now().UTC(), - } - // set cache entry to nil when the VM is under deleting. - if vm.VirtualMachineScaleSetVMProperties != nil && - strings.EqualFold(to.String(vm.VirtualMachineScaleSetVMProperties.ProvisioningState), string(compute.ProvisioningStateDeleting)) { - klog.V(4).Infof("VMSS virtualMachine %q is under deleting, setting its cache to nil", computerName) - vmssVMCacheEntry.virtualMachine = nil - } - localCache.Store(computerName, vmssVMCacheEntry) - - delete(oldCache, computerName) - } + // add old missing cache data with nil entries to prevent aggressive + // ARM calls during cache invalidation + for name, vmEntry := range oldCache { + // if the nil cache entry has existed for 15 minutes in the cache + // then it should not be added back to the cache + if vmEntry.virtualMachine == nil && time.Since(vmEntry.lastUpdate) > 15*time.Minute { + klog.V(5).Infof("ignoring expired entries from old cache for %s", name) + continue + } + lastUpdate := time.Now().UTC() + if vmEntry.virtualMachine == nil { + // if this is already a nil entry then keep the time the nil + // entry was first created, so we can cleanup unwanted entries + lastUpdate = vmEntry.lastUpdate } - // add old missing cache data with nil entries to prevent aggressive - // ARM calls during cache invalidation - for name, vmEntry := range oldCache { - // if the nil cache entry has existed for 15 minutes in the cache - // then it should not be added back to the cache - if vmEntry.virtualMachine == nil && time.Since(vmEntry.lastUpdate) > 15*time.Minute { - klog.V(5).Infof("ignoring expired entries from old cache for %s", name) - continue - } - lastUpdate := time.Now().UTC() - if vmEntry.virtualMachine == nil { - // if this is already a nil entry then keep the time the nil - // entry was first created, so we can cleanup unwanted entries - lastUpdate = vmEntry.lastUpdate - } - - klog.V(5).Infof("adding old entries to new cache for %s", name) - localCache.Store(name, &vmssVirtualMachinesEntry{ - resourceGroup: vmEntry.resourceGroup, - vmssName: vmEntry.vmssName, - instanceID: vmEntry.instanceID, - virtualMachine: nil, - lastUpdate: lastUpdate, - }) - } + klog.V(5).Infof("adding old entries to new cache for %s", name) + localCache.Store(name, &vmssVirtualMachinesEntry{ + resourceGroup: vmEntry.resourceGroup, + vmssName: vmEntry.vmssName, + instanceID: vmEntry.instanceID, + virtualMachine: nil, + lastUpdate: lastUpdate, + }) } return localCache, nil @@ -214,14 +245,30 @@ func (ss *scaleSet) newVMSSVirtualMachinesCache() (*azcache.TimedCache, error) { } func (ss *scaleSet) deleteCacheForNode(nodeName string) error { - cached, err := ss.vmssVMCache.Get(vmssVirtualMachinesKey, azcache.CacheReadTypeUnsafe) + node, err := ss.getNodeIdentityByNodeName(nodeName, azcache.CacheReadTypeUnsafe) if err != nil { klog.Errorf("deleteCacheForNode(%s) failed with error: %v", nodeName, err) return err } - virtualMachines := cached.(*sync.Map) + cacheKey, timedcache, err := ss.getVMSSVMCache(node.resourceGroup, node.vmssName) + if err != nil { + klog.Errorf("deleteCacheForNode(%s) failed with error: %v", nodeName, err) + return err + } + + vmcache, err := timedcache.Get(cacheKey, azcache.CacheReadTypeUnsafe) + if err != nil { + klog.Errorf("deleteCacheForNode(%s) failed with error: %v", nodeName, err) + return err + } + virtualMachines := vmcache.(*sync.Map) virtualMachines.Delete(nodeName) + + if err := ss.gcVMSSVMCache(); err != nil { + klog.Errorf("deleteCacheForNode(%s) failed to gc stale vmss caches: %v", nodeName, err) + } + return nil } diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache_test.go index 07fb2073921..8d3f8371f36 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache_test.go @@ -91,12 +91,7 @@ func TestVMSSVMCache(t *testing.T) { ss.cloud.VirtualMachineScaleSetsClient = mockVMSSClient ss.cloud.VirtualMachineScaleSetVMsClient = mockVMSSVMClient - expectedScaleSet := compute.VirtualMachineScaleSet{ - Name: &vmssName, - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedScaleSet := buildTestVMSS(vmssName, "vmssee6c2") mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).AnyTimes() expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, vmssName, "", 0, vmList, "", false) @@ -120,7 +115,9 @@ func TestVMSSVMCache(t *testing.T) { assert.NoError(t, err) // the VM should be removed from cache after deleteCacheForNode(). - cached, err := ss.vmssVMCache.Get(vmssVirtualMachinesKey, azcache.CacheReadTypeDefault) + cacheKey, cache, err := ss.getVMSSVMCache("rg", vmssName) + assert.NoError(t, err) + cached, err := cache.Get(cacheKey, azcache.CacheReadTypeDefault) assert.NoError(t, err) cachedVirtualMachines := cached.(*sync.Map) _, ok := cachedVirtualMachines.Load(vmName) diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go index c0eb15fe87f..8ef023e0342 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go @@ -67,7 +67,7 @@ func newTestScaleSetWithState(ctrl *gomock.Controller) (*scaleSet, error) { return ss.(*scaleSet), nil } -func buildTestVMSS(name string, lbBackendpoolIDs []string, ipv6 bool) compute.VirtualMachineScaleSet { +func buildTestVMSSWithLB(name, namePrefix string, lbBackendpoolIDs []string, ipv6 bool) compute.VirtualMachineScaleSet { lbBackendpools := make([]compute.SubResource, 0) for _, id := range lbBackendpoolIDs { lbBackendpools = append(lbBackendpools, compute.SubResource{ID: to.StringPtr(id)}) @@ -93,6 +93,9 @@ func buildTestVMSS(name string, lbBackendpoolIDs []string, ipv6 bool) compute.Vi VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ ProvisioningState: to.StringPtr("Running"), VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{ + OsProfile: &compute.VirtualMachineScaleSetOSProfile{ + ComputerNamePrefix: &namePrefix, + }, NetworkProfile: &compute.VirtualMachineScaleSetNetworkProfile{ NetworkInterfaceConfigurations: &[]compute.VirtualMachineScaleSetNetworkConfiguration{ { @@ -110,6 +113,19 @@ func buildTestVMSS(name string, lbBackendpoolIDs []string, ipv6 bool) compute.Vi return expectedVMSS } +func buildTestVMSS(name, computerNamePrefix string) compute.VirtualMachineScaleSet { + return compute.VirtualMachineScaleSet{ + Name: &name, + VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ + VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{ + OsProfile: &compute.VirtualMachineScaleSetOSProfile{ + ComputerNamePrefix: &computerNamePrefix, + }, + }, + }, + } +} + func buildTestVirtualMachineEnv(ss *Cloud, scaleSetName, zone string, faultDomain int32, vmList []string, state string, isIPv6 bool) ([]compute.VirtualMachineScaleSetVM, network.Interface, network.PublicIPAddress) { expectedVMSSVMs := make([]compute.VirtualMachineScaleSetVM, 0) expectedInterface := network.Interface{} @@ -261,6 +277,82 @@ func TestGetScaleSetVMInstanceID(t *testing.T) { } } +func TestGetNodeIdentityByNodeName(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + testCases := []struct { + description string + vmList []string + nodeName string + expected *nodeIdentity + scaleSet string + computerName string + expectError bool + }{ + { + description: "scaleSet should get node identity by node name", + vmList: []string{"vmssee6c2000000", "vmssee6c2000001"}, + nodeName: "vmssee6c2000001", + scaleSet: "vmssee6c2", + computerName: "vmssee6c2", + expected: &nodeIdentity{"rg", "vmssee6c2", "vmssee6c2000001"}, + }, + { + description: "scaleSet should get node identity when computerNamePrefix differs from vmss name", + vmList: []string{"vmssee6c2000000", "vmssee6c2000001"}, + nodeName: "vmssee6c2000001", + scaleSet: "ss", + computerName: "vmssee6c2", + expected: &nodeIdentity{"rg", "ss", "vmssee6c2000001"}, + }, + { + description: "scaleSet should get node identity by node name with upper cases hostname", + vmList: []string{"VMSSEE6C2000000", "VMSSEE6C2000001"}, + nodeName: "vmssee6c2000001", + scaleSet: "ss", + computerName: "vmssee6c2", + expected: &nodeIdentity{"rg", "ss", "vmssee6c2000001"}, + }, + { + description: "scaleSet should not get node identity for non-existing nodes", + vmList: []string{"vmssee6c2000000", "vmssee6c2000001"}, + nodeName: "agente6c2000005", + scaleSet: "ss", + computerName: "vmssee6c2", + expectError: true, + }, + } + + for _, test := range testCases { + ss, err := newTestScaleSet(ctrl) + assert.NoError(t, err, test.description) + + mockVMSSClient := mockvmssclient.NewMockInterface(ctrl) + mockVMSSVMClient := mockvmssvmclient.NewMockInterface(ctrl) + ss.cloud.VirtualMachineScaleSetsClient = mockVMSSClient + ss.cloud.VirtualMachineScaleSetVMsClient = mockVMSSVMClient + + expectedScaleSet := buildTestVMSS(test.scaleSet, test.computerName) + mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).AnyTimes() + + expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "", false) + mockVMSSVMClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedVMs, nil).AnyTimes() + + mockVMsClient := ss.cloud.VirtualMachinesClient.(*mockvmclient.MockInterface) + mockVMsClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachine{}, nil).AnyTimes() + + nodeID, err := ss.getNodeIdentityByNodeName(test.nodeName, azcache.CacheReadTypeDefault) + if test.expectError { + assert.Error(t, err, test.description) + continue + } + + assert.NoError(t, err, test.description) + assert.Equal(t, test.expected, nodeID, test.description) + } +} + func TestGetInstanceIDByNodeName(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -305,12 +397,7 @@ func TestGetInstanceIDByNodeName(t *testing.T) { ss.cloud.VirtualMachineScaleSetsClient = mockVMSSClient ss.cloud.VirtualMachineScaleSetVMsClient = mockVMSSVMClient - expectedScaleSet := compute.VirtualMachineScaleSet{ - Name: &test.scaleSet, - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedScaleSet := buildTestVMSS(test.scaleSet, "vmssee6c2") mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).AnyTimes() expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "", false) @@ -395,12 +482,7 @@ func TestGetZoneByNodeName(t *testing.T) { ss.cloud.VirtualMachineScaleSetsClient = mockVMSSClient ss.cloud.VirtualMachineScaleSetVMsClient = mockVMSSVMClient - expectedScaleSet := compute.VirtualMachineScaleSet{ - Name: &test.scaleSet, - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedScaleSet := buildTestVMSS(test.scaleSet, "vmssee6c2") mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).AnyTimes() expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, test.zone, test.faultDomain, test.vmList, "", false) @@ -462,12 +544,7 @@ func TestGetIPByNodeName(t *testing.T) { ss.cloud.InterfacesClient = mockInterfaceClient ss.cloud.PublicIPAddressesClient = mockPIPClient - expectedScaleSet := compute.VirtualMachineScaleSet{ - Name: &test.scaleSet, - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedScaleSet := buildTestVMSS(test.scaleSet, "vmssee6c2") mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).AnyTimes() expectedVMs, expectedInterface, expectedPIP := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "", false) @@ -535,12 +612,7 @@ func TestGetNodeNameByIPConfigurationID(t *testing.T) { ss.cloud.VirtualMachineScaleSetsClient = mockVMSSClient ss.cloud.VirtualMachineScaleSetVMsClient = mockVMSSVMClient - expectedScaleSet := compute.VirtualMachineScaleSet{ - Name: &test.scaleSet, - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedScaleSet := buildTestVMSS(test.scaleSet, "vmssee6c2") mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).AnyTimes() expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "", false) @@ -679,12 +751,7 @@ func TestGetVmssVM(t *testing.T) { ss, err := newTestScaleSet(ctrl) assert.NoError(t, err, test.description) - expectedVMSS := compute.VirtualMachineScaleSet{ - Name: to.StringPtr(test.existedVMSSName), - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedVMSS := buildTestVMSS(test.existedVMSSName, "vmss-vm-") mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() @@ -735,12 +802,7 @@ func TestGetPowerStatusByNodeName(t *testing.T) { ss, err := newTestScaleSet(ctrl) assert.NoError(t, err, "unexpected error when creating test VMSS") - expectedVMSS := compute.VirtualMachineScaleSet{ - Name: to.StringPtr(testVMSSName), - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-") mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() @@ -829,12 +891,7 @@ func TestGetInstanceTypeByNodeName(t *testing.T) { ss, err := newTestScaleSet(ctrl) assert.NoError(t, err, "unexpected error when creating test VMSS") - expectedVMSS := compute.VirtualMachineScaleSet{ - Name: to.StringPtr(testVMSSName), - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-") mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() @@ -987,11 +1044,7 @@ func TestGetPrimaryInterface(t *testing.T) { ss, err := newTestScaleSet(ctrl) assert.NoError(t, err, "unexpected error when creating test VMSS") - expectedVMSS := compute.VirtualMachineScaleSet{ - Name: to.StringPtr(testVMSSName), - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}}, - } + expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-") mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, test.vmssClientErr).AnyTimes() @@ -1114,12 +1167,7 @@ func TestGetPrivateIPsByNodeName(t *testing.T) { ss, err := newTestScaleSet(ctrl) assert.NoError(t, err, "unexpected error when creating test VMSS") - expectedVMSS := compute.VirtualMachineScaleSet{ - Name: to.StringPtr(testVMSSName), - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-") mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() @@ -1318,12 +1366,7 @@ func TestGetAgentPoolScaleSets(t *testing.T) { ss, err := newTestScaleSet(ctrl) assert.NoError(t, err, "unexpected error when creating test VMSS") - expectedVMSS := compute.VirtualMachineScaleSet{ - Name: to.StringPtr(testVMSSName), - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-") mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() @@ -1419,12 +1462,7 @@ func TestGetVMSetNames(t *testing.T) { ss, err := newTestScaleSet(ctrl) assert.NoError(t, err, "unexpected error when creating test VMSS") - expectedVMSS := compute.VirtualMachineScaleSet{ - Name: to.StringPtr(testVMSSName), - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-") mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() @@ -1703,12 +1741,7 @@ func TestEnsureHostInPool(t *testing.T) { ss.LoadBalancerSku = loadBalancerSkuStandard } - expectedVMSS := compute.VirtualMachineScaleSet{ - Name: to.StringPtr(testVMSSName), - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-") mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() @@ -1876,7 +1909,7 @@ func TestEnsureVMSSInPool(t *testing.T) { ss.LoadBalancerSku = loadBalancerSkuStandard } - expectedVMSS := buildTestVMSS(testVMSSName, []string{testLBBackendpoolID0}, test.setIPv6Config) + expectedVMSS := buildTestVMSSWithLB(testVMSSName, "vmss-vm-", []string{testLBBackendpoolID0}, test.setIPv6Config) if test.isVMSSDeallocating { expectedVMSS.ProvisioningState = &virtualMachineScaleSetsDeallocating } @@ -1973,7 +2006,7 @@ func TestEnsureHostsInPool(t *testing.T) { ss.LoadBalancerSku = loadBalancerSkuStandard ss.ExcludeMasterFromStandardLB = to.BoolPtr(true) - expectedVMSS := buildTestVMSS(testVMSSName, []string{testLBBackendpoolID0}, false) + expectedVMSS := buildTestVMSSWithLB(testVMSSName, "vmss-vm-", []string{testLBBackendpoolID0}, false) mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() mockVMSSClient.EXPECT().Get(gomock.Any(), ss.ResourceGroup, testVMSSName).Return(expectedVMSS, nil).MaxTimes(1) @@ -2061,12 +2094,7 @@ func TestEnsureBackendPoolDeletedFromNode(t *testing.T) { ss, err := newTestScaleSet(ctrl) assert.NoError(t, err, test.description) - expectedVMSS := compute.VirtualMachineScaleSet{ - Name: to.StringPtr(testVMSSName), - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}, - }, - } + expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-") mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() @@ -2155,7 +2183,7 @@ func TestEnsureBackendPoolDeletedFromVMSS(t *testing.T) { ss.LoadBalancerSku = loadBalancerSkuStandard - expectedVMSS := buildTestVMSS(testVMSSName, []string{testLBBackendpoolID0}, false) + expectedVMSS := buildTestVMSSWithLB(testVMSSName, "vmss-vm-", []string{testLBBackendpoolID0}, false) if test.isVMSSDeallocating { expectedVMSS.ProvisioningState = &virtualMachineScaleSetsDeallocating } @@ -2249,7 +2277,7 @@ func TestEnsureBackendPoolDeleted(t *testing.T) { ss, err := newTestScaleSet(ctrl) assert.NoError(t, err, test.description) - expectedVMSS := buildTestVMSS(testVMSSName, []string{testLBBackendpoolID0}, false) + expectedVMSS := buildTestVMSSWithLB(testVMSSName, "vmss-vm-", []string{testLBBackendpoolID0}, false) mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() mockVMSSClient.EXPECT().Get(gomock.Any(), ss.ResourceGroup, testVMSSName).Return(expectedVMSS, nil).MaxTimes(1) @@ -2308,11 +2336,12 @@ func TestEnsureBackendPoolDeletedConcurrently(t *testing.T) { }, }, } - vmss0 := buildTestVMSS("vmss-0", []string{testLBBackendpoolID0, testLBBackendpoolID1}, false) - vmss1 := buildTestVMSS("vmss-1", []string{testLBBackendpoolID0, testLBBackendpoolID1}, false) - expectedVMSSVMsOfVMSS0, _, _ := buildTestVirtualMachineEnv(ss.cloud, "vmss-0", "", 0, []string{"vmss-vm-000000"}, "succeeded", false) - expectedVMSSVMsOfVMSS1, _, _ := buildTestVirtualMachineEnv(ss.cloud, "vmss-1", "", 0, []string{"vmss-vm-000001"}, "succeeded", false) + vmss0 := buildTestVMSSWithLB("vmss-0", "vmss-0-vm-", []string{testLBBackendpoolID0, testLBBackendpoolID1}, false) + vmss1 := buildTestVMSSWithLB("vmss-1", "vmss-1-vm-", []string{testLBBackendpoolID0, testLBBackendpoolID1}, false) + + expectedVMSSVMsOfVMSS0, _, _ := buildTestVirtualMachineEnv(ss.cloud, "vmss-0", "", 0, []string{"vmss-0-vm-000000"}, "succeeded", false) + expectedVMSSVMsOfVMSS1, _, _ := buildTestVirtualMachineEnv(ss.cloud, "vmss-1", "", 0, []string{"vmss-1-vm-000001"}, "succeeded", false) for _, expectedVMSSVMs := range [][]compute.VirtualMachineScaleSetVM{expectedVMSSVMsOfVMSS0, expectedVMSSVMsOfVMSS1} { vmssVMNetworkConfigs := expectedVMSSVMs[0].NetworkProfileConfiguration vmssVMIPConfigs := (*vmssVMNetworkConfigs.NetworkInterfaceConfigurations)[0].VirtualMachineScaleSetNetworkConfigurationProperties.IPConfigurations @@ -2322,11 +2351,13 @@ func TestEnsureBackendPoolDeletedConcurrently(t *testing.T) { mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{vmss0, vmss1}, nil).AnyTimes() + mockVMSSClient.EXPECT().List(gomock.Any(), "rg1").Return(nil, nil).AnyTimes() mockVMSSClient.EXPECT().Get(gomock.Any(), ss.ResourceGroup, "vmss-0").Return(vmss0, nil).MaxTimes(2) mockVMSSClient.EXPECT().Get(gomock.Any(), ss.ResourceGroup, "vmss-1").Return(vmss1, nil).MaxTimes(2) mockVMSSClient.EXPECT().CreateOrUpdate(gomock.Any(), ss.ResourceGroup, gomock.Any(), gomock.Any()).Return(nil).Times(2) mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) + mockVMSSVMClient.EXPECT().List(gomock.Any(), "rg1", "vmss-0", gomock.Any()).Return(nil, nil).AnyTimes() mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, "vmss-0", gomock.Any()).Return(expectedVMSSVMsOfVMSS0, nil).AnyTimes() mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, "vmss-1", gomock.Any()).Return(expectedVMSSVMsOfVMSS1, nil).AnyTimes() mockVMSSVMClient.EXPECT().UpdateVMs(gomock.Any(), ss.ResourceGroup, gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(2)