github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-25 04:33:26 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #93107 from DataDog/azure-per-vmss-vmssvm-incremental-cache

Browse Source 
Azure: use per-vmss vmssvm incremental cache
This commit is contained in:
Kubernetes Prow Robot 2020-07-21 15:00:06 -07:00 committed by GitHub
commit 5df1e53dac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 339 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_vmss_test.go
View File

@ -98,7 +98,7 @@ func TestAttachDiskWithVMSS(t *testing.T) {
assert.NoError(t, err, test.desc)
testCloud := ss.cloud
testCloud.PrimaryScaleSetName = scaleSetName
expectedVMSS := buildTestVMSS(scaleSetName, []string{testLBBackendpoolID0}, false)
expectedVMSS := buildTestVMSSWithLB(scaleSetName, "vmss00-vm-", []string{testLBBackendpoolID0}, false)
mockVMSSClient := testCloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface)
mockVMSSClient.EXPECT().List(gomock.Any(), testCloud.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes()
mockVMSSClient.EXPECT().Get(gomock.Any(), testCloud.ResourceGroup, scaleSetName).Return(expectedVMSS, nil).MaxTimes(1)
@ -194,7 +194,7 @@ func TestDetachDiskWithVMSS(t *testing.T) {
assert.NoError(t, err, test.desc)
testCloud := ss.cloud
testCloud.PrimaryScaleSetName = scaleSetName
expectedVMSS := buildTestVMSS(scaleSetName, []string{testLBBackendpoolID0}, false)
expectedVMSS := buildTestVMSSWithLB(scaleSetName, "vmss00-vm-", []string{testLBBackendpoolID0}, false)
mockVMSSClient := testCloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface)
mockVMSSClient.EXPECT().List(gomock.Any(), testCloud.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes()
mockVMSSClient.EXPECT().Get(gomock.Any(), testCloud.ResourceGroup, scaleSetName).Return(expectedVMSS, nil).MaxTimes(1)
@ -297,7 +297,7 @@ func TestGetDataDisksWithVMSS(t *testing.T) {
assert.NoError(t, err, test.desc)
testCloud := ss.cloud
testCloud.PrimaryScaleSetName = scaleSetName
expectedVMSS := buildTestVMSS(scaleSetName, []string{testLBBackendpoolID0}, false)
expectedVMSS := buildTestVMSSWithLB(scaleSetName, "vmss00-vm-", []string{testLBBackendpoolID0}, false)
mockVMSSClient := testCloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface)
mockVMSSClient.EXPECT().List(gomock.Any(), testCloud.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes()
mockVMSSClient.EXPECT().Get(gomock.Any(), testCloud.ResourceGroup, scaleSetName).Return(expectedVMSS, nil).MaxTimes(1)

114
staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go
View File

@ -61,6 +61,13 @@ type vmssMetaInfo struct {
resourceGroup string
}
// nodeIdentity identifies a node within a subscription.
type nodeIdentity struct {
resourceGroup string
vmssName string
nodeName string
}
// scaleSet implements VMSet interface for Azure scale set.
type scaleSet struct {
*Cloud
@ -70,7 +77,7 @@ type scaleSet struct {
availabilitySet VMSet
vmssCache *azcache.TimedCache
vmssVMCache *azcache.TimedCache
vmssVMCache *sync.Map // [resourcegroup/vmssname]*azcache.TimedCache
availabilitySetNodesCache *azcache.TimedCache
}
@ -80,6 +87,7 @@ func newScaleSet(az *Cloud) (VMSet, error) {
ss := &scaleSet{
Cloud: az,
availabilitySet: newAvailabilitySet(az),
vmssVMCache: &sync.Map{},
}
if !ss.DisableAvailabilitySetNodes {
@ -94,11 +102,6 @@ func newScaleSet(az *Cloud) (VMSet, error) {
return nil, err
}
ss.vmssVMCache, err = ss.newVMSSVirtualMachinesCache()
if err != nil {
return nil, err
}
return ss, nil
}
@ -139,12 +142,17 @@ func (ss *scaleSet) getVMSS(vmssName string, crt azcache.AzureCacheReadType) (*c
return vmss, nil
}
// getVmssVM gets virtualMachineScaleSetVM by nodeName from cache.
// It returns cloudprovider.InstanceNotFound if node does not belong to any scale sets.
func (ss *scaleSet) getVmssVM(nodeName string, crt azcache.AzureCacheReadType) (string, string, *compute.VirtualMachineScaleSetVM, error) {
// getVmssVMByNodeIdentity find virtualMachineScaleSetVM by nodeIdentity, using node's parent VMSS cache.
// Returns cloudprovider.InstanceNotFound if the node does not belong to the scale set named in nodeIdentity.
func (ss *scaleSet) getVmssVMByNodeIdentity(node *nodeIdentity, crt azcache.AzureCacheReadType) (string, string, *compute.VirtualMachineScaleSetVM, error) {
cacheKey, cache, err := ss.getVMSSVMCache(node.resourceGroup, node.vmssName)
if err != nil {
return "", "", nil, err
}
getter := func(nodeName string, crt azcache.AzureCacheReadType) (string, string, *compute.VirtualMachineScaleSetVM, bool, error) {
var found bool
cached, err := ss.vmssVMCache.Get(vmssVirtualMachinesKey, crt)
cached, err := cache.Get(cacheKey, crt)
if err != nil {
return "", "", nil, found, err
}
@ -159,19 +167,19 @@ func (ss *scaleSet) getVmssVM(nodeName string, crt azcache.AzureCacheReadType) (
return "", "", nil, found, nil
}
_, err := getScaleSetVMInstanceID(nodeName)
_, err = getScaleSetVMInstanceID(node.nodeName)
if err != nil {
return "", "", nil, err
}
vmssName, instanceID, vm, found, err := getter(nodeName, crt)
vmssName, instanceID, vm, found, err := getter(node.nodeName, crt)
if err != nil {
return "", "", nil, err
}
if !found {
klog.V(2).Infof("Couldn't find VMSS VM with nodeName %s, refreshing the cache", nodeName)
vmssName, instanceID, vm, found, err = getter(nodeName, azcache.CacheReadTypeForceRefresh)
klog.V(2).Infof("Couldn't find VMSS VM with nodeName %s, refreshing the cache", node.nodeName)
vmssName, instanceID, vm, found, err = getter(node.nodeName, azcache.CacheReadTypeForceRefresh)
if err != nil {
return "", "", nil, err
}
@ -187,6 +195,17 @@ func (ss *scaleSet) getVmssVM(nodeName string, crt azcache.AzureCacheReadType) (
return vmssName, instanceID, vm, nil
}
// getVmssVM gets virtualMachineScaleSetVM by nodeName from cache.
// Returns cloudprovider.InstanceNotFound if nodeName does not belong to any scale set.
func (ss *scaleSet) getVmssVM(nodeName string, crt azcache.AzureCacheReadType) (string, string, *compute.VirtualMachineScaleSetVM, error) {
node, err := ss.getNodeIdentityByNodeName(nodeName, crt)
if err != nil {
return "", "", nil, err
}
return ss.getVmssVMByNodeIdentity(node, crt)
}
// GetPowerStatusByNodeName returns the power state of the specified node.
func (ss *scaleSet) GetPowerStatusByNodeName(name string) (powerState string, err error) {
managedByAS, err := ss.isNodeManagedByAvailabilitySet(name, azcache.CacheReadTypeUnsafe)
@ -222,8 +241,13 @@ func (ss *scaleSet) GetPowerStatusByNodeName(name string) (powerState string, er
// getCachedVirtualMachineByInstanceID gets scaleSetVMInfo from cache.
// The node must belong to one of scale sets.
func (ss *scaleSet) getVmssVMByInstanceID(resourceGroup, scaleSetName, instanceID string, crt azcache.AzureCacheReadType) (*compute.VirtualMachineScaleSetVM, error) {
cacheKey, cache, err := ss.getVMSSVMCache(resourceGroup, scaleSetName)
if err != nil {
return nil, err
}
getter := func(crt azcache.AzureCacheReadType) (vm *compute.VirtualMachineScaleSetVM, found bool, err error) {
cached, err := ss.vmssVMCache.Get(vmssVirtualMachinesKey, crt)
cached, err := cache.Get(cacheKey, crt)
if err != nil {
return nil, false, err
}
@ -590,6 +614,66 @@ func (ss *scaleSet) listScaleSets(resourceGroup string) ([]string, error) {
return ssNames, nil
}
// getNodeIdentityByNodeName use the VMSS cache to find a node's resourcegroup and vmss, returned in a nodeIdentity.
func (ss *scaleSet) getNodeIdentityByNodeName(nodeName string, crt azcache.AzureCacheReadType) (*nodeIdentity, error) {
getter := func(nodeName string, crt azcache.AzureCacheReadType) (*nodeIdentity, error) {
node := &nodeIdentity{
nodeName: nodeName,
}
cached, err := ss.vmssCache.Get(vmssKey, crt)
if err != nil {
return nil, err
}
vmsses := cached.(*sync.Map)
vmsses.Range(func(key, value interface{}) bool {
v := value.(*vmssEntry)
if v.vmss.Name == nil {
return true
}
vmssPrefix := *v.vmss.Name
if v.vmss.VirtualMachineProfile != nil &&
v.vmss.VirtualMachineProfile.OsProfile != nil &&
v.vmss.VirtualMachineProfile.OsProfile.ComputerNamePrefix != nil {
vmssPrefix = *v.vmss.VirtualMachineProfile.OsProfile.ComputerNamePrefix
}
if strings.EqualFold(vmssPrefix, nodeName[:len(nodeName)-6]) {
node.vmssName = *v.vmss.Name
node.resourceGroup = v.resourceGroup
return false
}
return true
})
return node, nil
}
if _, err := getScaleSetVMInstanceID(nodeName); err != nil {
return nil, err
}
node, err := getter(nodeName, crt)
if err != nil {
return nil, err
}
if node.vmssName != "" {
return node, nil
}
klog.V(2).Infof("Couldn't find VMSS for node %s, refreshing the cache", nodeName)
node, err = getter(nodeName, azcache.CacheReadTypeForceRefresh)
if err != nil {
return nil, err
}
if node.vmssName == "" {
return nil, cloudprovider.InstanceNotFound
}
return node, nil
}
// listScaleSetVMs lists VMs belonging to the specified scale set.
func (ss *scaleSet) listScaleSetVMs(scaleSetName, resourceGroup string) ([]compute.VirtualMachineScaleSetVM, error) {
ctx, cancel := getContextWithCancel()

185
staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache.go
View File

@ -20,6 +20,7 @@ package azure
import (
"context"
"fmt"
"strings"
"sync"
"time"
@ -36,7 +37,6 @@ var (
vmssNameSeparator = "_"
vmssKey = "k8svmssKey"
vmssVirtualMachinesKey = "k8svmssVirtualMachinesKey"
availabilitySetNodesKey = "k8sAvailabilitySetNodesKey"
availabilitySetNodesCacheTTLDefaultInSeconds = 900
@ -53,8 +53,9 @@ type vmssVirtualMachinesEntry struct {
}
type vmssEntry struct {
vmss *compute.VirtualMachineScaleSet
lastUpdate time.Time
vmss *compute.VirtualMachineScaleSet
resourceGroup string
lastUpdate time.Time
}
func (ss *scaleSet) newVMSSCache() (*azcache.TimedCache, error) {
@ -80,8 +81,9 @@ func (ss *scaleSet) newVMSSCache() (*azcache.TimedCache, error) {
continue
}
localCache.Store(*scaleSet.Name, &vmssEntry{
vmss: &scaleSet,
lastUpdate: time.Now().UTC(),
vmss: &scaleSet,
resourceGroup: resourceGroup,
lastUpdate: time.Now().UTC(),
})
}
}
@ -109,15 +111,58 @@ func extractVmssVMName(name string) (string, string, error) {
return ssName, instanceID, nil
}
func (ss *scaleSet) newVMSSVirtualMachinesCache() (*azcache.TimedCache, error) {
// getVMSSVMCache returns an *azcache.TimedCache and cache key for a VMSS (creating that cache if new).
func (ss *scaleSet) getVMSSVMCache(resourceGroup, vmssName string) (string, *azcache.TimedCache, error) {
cacheKey := strings.ToLower(fmt.Sprintf("%s/%s", resourceGroup, vmssName))
if entry, ok := ss.vmssVMCache.Load(cacheKey); ok {
cache := entry.(*azcache.TimedCache)
return cacheKey, cache, nil
}
cache, err := ss.newVMSSVirtualMachinesCache(resourceGroup, vmssName, cacheKey)
if err != nil {
return "", nil, err
}
ss.vmssVMCache.Store(cacheKey, cache)
return cacheKey, cache, nil
}
// gcVMSSVMCache delete stale VMSS VMs caches from deleted VMSSes.
func (ss *scaleSet) gcVMSSVMCache() error {
cached, err := ss.vmssCache.Get(vmssKey, azcache.CacheReadTypeUnsafe)
if err != nil {
return err
}
vmsses := cached.(*sync.Map)
removed := map[string]bool{}
ss.vmssVMCache.Range(func(key, value interface{}) bool {
cacheKey := key.(string)
vlistIdx := cacheKey[strings.LastIndex(cacheKey, "/")+1:]
if _, ok := vmsses.Load(vlistIdx); !ok {
removed[cacheKey] = true
}
return true
})
for key := range removed {
ss.vmssVMCache.Delete(key)
}
return nil
}
// newVMSSVirtualMachinesCache instanciates a new VMs cache for VMs belonging to the provided VMSS.
func (ss *scaleSet) newVMSSVirtualMachinesCache(resourceGroupName, vmssName, cacheKey string) (*azcache.TimedCache, error) {
getter := func(key string) (interface{}, error) {
localCache := &sync.Map{} // [nodeName]*vmssVirtualMachinesEntry
oldCache := make(map[string]vmssVirtualMachinesEntry)
if ss.vmssVMCache != nil {
if vmssCache, ok := ss.vmssVMCache.Load(cacheKey); ok {
// get old cache before refreshing the cache
entry, exists, err := ss.vmssVMCache.Store.GetByKey(vmssVirtualMachinesKey)
cache := vmssCache.(*azcache.TimedCache)
entry, exists, err := cache.Store.GetByKey(cacheKey)
if err != nil {
return nil, err
}
@ -133,75 +178,61 @@ func (ss *scaleSet) newVMSSVirtualMachinesCache() (*azcache.TimedCache, error) {
}
}
allResourceGroups, err := ss.GetResourceGroups()
vms, err := ss.listScaleSetVMs(vmssName, resourceGroupName)
if err != nil {
return nil, err
}
for _, resourceGroup := range allResourceGroups.List() {
scaleSetNames, err := ss.listScaleSets(resourceGroup)
if err != nil {
return nil, err
for i := range vms {
vm := vms[i]
if vm.OsProfile == nil || vm.OsProfile.ComputerName == nil {
klog.Warningf("failed to get computerName for vmssVM (%q)", vmssName)
continue
}
for _, ssName := range scaleSetNames {
vms, err := ss.listScaleSetVMs(ssName, resourceGroup)
if err != nil {
return nil, err
}
computerName := strings.ToLower(*vm.OsProfile.ComputerName)
vmssVMCacheEntry := &vmssVirtualMachinesEntry{
resourceGroup: resourceGroupName,
vmssName: vmssName,
instanceID: to.String(vm.InstanceID),
virtualMachine: &vm,
lastUpdate: time.Now().UTC(),
}
// set cache entry to nil when the VM is under deleting.
if vm.VirtualMachineScaleSetVMProperties != nil &&
strings.EqualFold(to.String(vm.VirtualMachineScaleSetVMProperties.ProvisioningState), string(compute.ProvisioningStateDeleting)) {
klog.V(4).Infof("VMSS virtualMachine %q is under deleting, setting its cache to nil", computerName)
vmssVMCacheEntry.virtualMachine = nil
}
localCache.Store(computerName, vmssVMCacheEntry)
for i := range vms {
vm := vms[i]
if vm.OsProfile == nil || vm.OsProfile.ComputerName == nil {
klog.Warningf("failed to get computerName for vmssVM (%q)", ssName)
continue
}
delete(oldCache, computerName)
}
computerName := strings.ToLower(*vm.OsProfile.ComputerName)
vmssVMCacheEntry := &vmssVirtualMachinesEntry{
resourceGroup: resourceGroup,
vmssName: ssName,
instanceID: to.String(vm.InstanceID),
virtualMachine: &vm,
lastUpdate: time.Now().UTC(),
}
// set cache entry to nil when the VM is under deleting.
if vm.VirtualMachineScaleSetVMProperties != nil &&
strings.EqualFold(to.String(vm.VirtualMachineScaleSetVMProperties.ProvisioningState), string(compute.ProvisioningStateDeleting)) {
klog.V(4).Infof("VMSS virtualMachine %q is under deleting, setting its cache to nil", computerName)
vmssVMCacheEntry.virtualMachine = nil
}
localCache.Store(computerName, vmssVMCacheEntry)
delete(oldCache, computerName)
}
// add old missing cache data with nil entries to prevent aggressive
// ARM calls during cache invalidation
for name, vmEntry := range oldCache {
// if the nil cache entry has existed for 15 minutes in the cache
// then it should not be added back to the cache
if vmEntry.virtualMachine == nil && time.Since(vmEntry.lastUpdate) > 15*time.Minute {
klog.V(5).Infof("ignoring expired entries from old cache for %s", name)
continue
}
lastUpdate := time.Now().UTC()
if vmEntry.virtualMachine == nil {
// if this is already a nil entry then keep the time the nil
// entry was first created, so we can cleanup unwanted entries
lastUpdate = vmEntry.lastUpdate
}
// add old missing cache data with nil entries to prevent aggressive
// ARM calls during cache invalidation
for name, vmEntry := range oldCache {
// if the nil cache entry has existed for 15 minutes in the cache
// then it should not be added back to the cache
if vmEntry.virtualMachine == nil && time.Since(vmEntry.lastUpdate) > 15*time.Minute {
klog.V(5).Infof("ignoring expired entries from old cache for %s", name)
continue
}
lastUpdate := time.Now().UTC()
if vmEntry.virtualMachine == nil {
// if this is already a nil entry then keep the time the nil
// entry was first created, so we can cleanup unwanted entries
lastUpdate = vmEntry.lastUpdate
}
klog.V(5).Infof("adding old entries to new cache for %s", name)
localCache.Store(name, &vmssVirtualMachinesEntry{
resourceGroup: vmEntry.resourceGroup,
vmssName: vmEntry.vmssName,
instanceID: vmEntry.instanceID,
virtualMachine: nil,
lastUpdate: lastUpdate,
})
}
klog.V(5).Infof("adding old entries to new cache for %s", name)
localCache.Store(name, &vmssVirtualMachinesEntry{
resourceGroup: vmEntry.resourceGroup,
vmssName: vmEntry.vmssName,
instanceID: vmEntry.instanceID,
virtualMachine: nil,
lastUpdate: lastUpdate,
})
}
return localCache, nil
@ -214,14 +245,30 @@ func (ss *scaleSet) newVMSSVirtualMachinesCache() (*azcache.TimedCache, error) {
}
func (ss *scaleSet) deleteCacheForNode(nodeName string) error {
cached, err := ss.vmssVMCache.Get(vmssVirtualMachinesKey, azcache.CacheReadTypeUnsafe)
node, err := ss.getNodeIdentityByNodeName(nodeName, azcache.CacheReadTypeUnsafe)
if err != nil {
klog.Errorf("deleteCacheForNode(%s) failed with error: %v", nodeName, err)
return err
}
virtualMachines := cached.(*sync.Map)
cacheKey, timedcache, err := ss.getVMSSVMCache(node.resourceGroup, node.vmssName)
if err != nil {
klog.Errorf("deleteCacheForNode(%s) failed with error: %v", nodeName, err)
return err
}
vmcache, err := timedcache.Get(cacheKey, azcache.CacheReadTypeUnsafe)
if err != nil {
klog.Errorf("deleteCacheForNode(%s) failed with error: %v", nodeName, err)
return err
}
virtualMachines := vmcache.(*sync.Map)
virtualMachines.Delete(nodeName)
if err := ss.gcVMSSVMCache(); err != nil {
klog.Errorf("deleteCacheForNode(%s) failed to gc stale vmss caches: %v", nodeName, err)
}
return nil
}

11
staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache_test.go
View File

@ -91,12 +91,7 @@ func TestVMSSVMCache(t *testing.T) {
ss.cloud.VirtualMachineScaleSetsClient = mockVMSSClient
ss.cloud.VirtualMachineScaleSetVMsClient = mockVMSSVMClient
expectedScaleSet := compute.VirtualMachineScaleSet{
Name: &vmssName,
VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{},
},
}
expectedScaleSet := buildTestVMSS(vmssName, "vmssee6c2")
mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).AnyTimes()
expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, vmssName, "", 0, vmList, "", false)
@ -120,7 +115,9 @@ func TestVMSSVMCache(t *testing.T) {
assert.NoError(t, err)
// the VM should be removed from cache after deleteCacheForNode().
cached, err := ss.vmssVMCache.Get(vmssVirtualMachinesKey, azcache.CacheReadTypeDefault)
cacheKey, cache, err := ss.getVMSSVMCache("rg", vmssName)
assert.NoError(t, err)
cached, err := cache.Get(cacheKey, azcache.CacheReadTypeDefault)
assert.NoError(t, err)
cachedVirtualMachines := cached.(*sync.Map)
_, ok := cachedVirtualMachines.Load(vmName)

203
staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go
View File

@ -67,7 +67,7 @@ func newTestScaleSetWithState(ctrl *gomock.Controller) (*scaleSet, error) {
return ss.(*scaleSet), nil
}
func buildTestVMSS(name string, lbBackendpoolIDs []string, ipv6 bool) compute.VirtualMachineScaleSet {
func buildTestVMSSWithLB(name, namePrefix string, lbBackendpoolIDs []string, ipv6 bool) compute.VirtualMachineScaleSet {
lbBackendpools := make([]compute.SubResource, 0)
for _, id := range lbBackendpoolIDs {
lbBackendpools = append(lbBackendpools, compute.SubResource{ID: to.StringPtr(id)})
@ -93,6 +93,9 @@ func buildTestVMSS(name string, lbBackendpoolIDs []string, ipv6 bool) compute.Vi
VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
ProvisioningState: to.StringPtr("Running"),
VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{
OsProfile: &compute.VirtualMachineScaleSetOSProfile{
ComputerNamePrefix: &namePrefix,
},
NetworkProfile: &compute.VirtualMachineScaleSetNetworkProfile{
NetworkInterfaceConfigurations: &[]compute.VirtualMachineScaleSetNetworkConfiguration{
{
@ -110,6 +113,19 @@ func buildTestVMSS(name string, lbBackendpoolIDs []string, ipv6 bool) compute.Vi
return expectedVMSS
}
func buildTestVMSS(name, computerNamePrefix string) compute.VirtualMachineScaleSet {
return compute.VirtualMachineScaleSet{
Name: &name,
VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{
OsProfile: &compute.VirtualMachineScaleSetOSProfile{
ComputerNamePrefix: &computerNamePrefix,
},
},
},
}
}
func buildTestVirtualMachineEnv(ss *Cloud, scaleSetName, zone string, faultDomain int32, vmList []string, state string, isIPv6 bool) ([]compute.VirtualMachineScaleSetVM, network.Interface, network.PublicIPAddress) {
expectedVMSSVMs := make([]compute.VirtualMachineScaleSetVM, 0)
expectedInterface := network.Interface{}
@ -261,6 +277,82 @@ func TestGetScaleSetVMInstanceID(t *testing.T) {
}
}
func TestGetNodeIdentityByNodeName(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
testCases := []struct {
description string
vmList []string
nodeName string
expected *nodeIdentity
scaleSet string
computerName string
expectError bool
}{
{
description: "scaleSet should get node identity by node name",
vmList: []string{"vmssee6c2000000", "vmssee6c2000001"},
nodeName: "vmssee6c2000001",
scaleSet: "vmssee6c2",
computerName: "vmssee6c2",
expected: &nodeIdentity{"rg", "vmssee6c2", "vmssee6c2000001"},
},
{
description: "scaleSet should get node identity when computerNamePrefix differs from vmss name",
vmList: []string{"vmssee6c2000000", "vmssee6c2000001"},
nodeName: "vmssee6c2000001",
scaleSet: "ss",
computerName: "vmssee6c2",
expected: &nodeIdentity{"rg", "ss", "vmssee6c2000001"},
},
{
description: "scaleSet should get node identity by node name with upper cases hostname",
vmList: []string{"VMSSEE6C2000000", "VMSSEE6C2000001"},
nodeName: "vmssee6c2000001",
scaleSet: "ss",
computerName: "vmssee6c2",
expected: &nodeIdentity{"rg", "ss", "vmssee6c2000001"},
},
{
description: "scaleSet should not get node identity for non-existing nodes",
vmList: []string{"vmssee6c2000000", "vmssee6c2000001"},
nodeName: "agente6c2000005",
scaleSet: "ss",
computerName: "vmssee6c2",
expectError: true,
},
}
for _, test := range testCases {
ss, err := newTestScaleSet(ctrl)
assert.NoError(t, err, test.description)
mockVMSSClient := mockvmssclient.NewMockInterface(ctrl)
mockVMSSVMClient := mockvmssvmclient.NewMockInterface(ctrl)
ss.cloud.VirtualMachineScaleSetsClient = mockVMSSClient
ss.cloud.VirtualMachineScaleSetVMsClient = mockVMSSVMClient
expectedScaleSet := buildTestVMSS(test.scaleSet, test.computerName)
mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).AnyTimes()
expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "", false)
mockVMSSVMClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedVMs, nil).AnyTimes()
mockVMsClient := ss.cloud.VirtualMachinesClient.(*mockvmclient.MockInterface)
mockVMsClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachine{}, nil).AnyTimes()
nodeID, err := ss.getNodeIdentityByNodeName(test.nodeName, azcache.CacheReadTypeDefault)
if test.expectError {
assert.Error(t, err, test.description)
continue
}
assert.NoError(t, err, test.description)
assert.Equal(t, test.expected, nodeID, test.description)
}
}
func TestGetInstanceIDByNodeName(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
@ -305,12 +397,7 @@ func TestGetInstanceIDByNodeName(t *testing.T) {
ss.cloud.VirtualMachineScaleSetsClient = mockVMSSClient
ss.cloud.VirtualMachineScaleSetVMsClient = mockVMSSVMClient
expectedScaleSet := compute.VirtualMachineScaleSet{
Name: &test.scaleSet,
VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{},
},
}
expectedScaleSet := buildTestVMSS(test.scaleSet, "vmssee6c2")
mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).AnyTimes()
expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "", false)
@ -395,12 +482,7 @@ func TestGetZoneByNodeName(t *testing.T) {
ss.cloud.VirtualMachineScaleSetsClient = mockVMSSClient
ss.cloud.VirtualMachineScaleSetVMsClient = mockVMSSVMClient
expectedScaleSet := compute.VirtualMachineScaleSet{
Name: &test.scaleSet,
VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{},
},
}
expectedScaleSet := buildTestVMSS(test.scaleSet, "vmssee6c2")
mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).AnyTimes()
expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, test.zone, test.faultDomain, test.vmList, "", false)
@ -462,12 +544,7 @@ func TestGetIPByNodeName(t *testing.T) {
ss.cloud.InterfacesClient = mockInterfaceClient
ss.cloud.PublicIPAddressesClient = mockPIPClient
expectedScaleSet := compute.VirtualMachineScaleSet{
Name: &test.scaleSet,
VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{},
},
}
expectedScaleSet := buildTestVMSS(test.scaleSet, "vmssee6c2")
mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).AnyTimes()
expectedVMs, expectedInterface, expectedPIP := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "", false)
@ -535,12 +612,7 @@ func TestGetNodeNameByIPConfigurationID(t *testing.T) {
ss.cloud.VirtualMachineScaleSetsClient = mockVMSSClient
ss.cloud.VirtualMachineScaleSetVMsClient = mockVMSSVMClient
expectedScaleSet := compute.VirtualMachineScaleSet{
Name: &test.scaleSet,
VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{},
},
}
expectedScaleSet := buildTestVMSS(test.scaleSet, "vmssee6c2")
mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).AnyTimes()
expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "", false)
@ -679,12 +751,7 @@ func TestGetVmssVM(t *testing.T) {
ss, err := newTestScaleSet(ctrl)
assert.NoError(t, err, test.description)
expectedVMSS := compute.VirtualMachineScaleSet{
Name: to.StringPtr(test.existedVMSSName),
VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{},
},
}
expectedVMSS := buildTestVMSS(test.existedVMSSName, "vmss-vm-")
mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface)
mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes()
@ -735,12 +802,7 @@ func TestGetPowerStatusByNodeName(t *testing.T) {
ss, err := newTestScaleSet(ctrl)
assert.NoError(t, err, "unexpected error when creating test VMSS")
expectedVMSS := compute.VirtualMachineScaleSet{
Name: to.StringPtr(testVMSSName),
VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{},
},
}
expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-")
mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface)
mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes()
@ -829,12 +891,7 @@ func TestGetInstanceTypeByNodeName(t *testing.T) {
ss, err := newTestScaleSet(ctrl)
assert.NoError(t, err, "unexpected error when creating test VMSS")
expectedVMSS := compute.VirtualMachineScaleSet{
Name: to.StringPtr(testVMSSName),
VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{},
},
}
expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-")
mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface)
mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes()
@ -987,11 +1044,7 @@ func TestGetPrimaryInterface(t *testing.T) {
ss, err := newTestScaleSet(ctrl)
assert.NoError(t, err, "unexpected error when creating test VMSS")
expectedVMSS := compute.VirtualMachineScaleSet{
Name: to.StringPtr(testVMSSName),
VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{}},
}
expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-")
mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface)
mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, test.vmssClientErr).AnyTimes()
@ -1114,12 +1167,7 @@ func TestGetPrivateIPsByNodeName(t *testing.T) {
ss, err := newTestScaleSet(ctrl)
assert.NoError(t, err, "unexpected error when creating test VMSS")
expectedVMSS := compute.VirtualMachineScaleSet{
Name: to.StringPtr(testVMSSName),
VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{},
},
}
expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-")
mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface)
mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes()
@ -1318,12 +1366,7 @@ func TestGetAgentPoolScaleSets(t *testing.T) {
ss, err := newTestScaleSet(ctrl)
assert.NoError(t, err, "unexpected error when creating test VMSS")
expectedVMSS := compute.VirtualMachineScaleSet{
Name: to.StringPtr(testVMSSName),
VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{},
},
}
expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-")
mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface)
mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes()
@ -1419,12 +1462,7 @@ func TestGetVMSetNames(t *testing.T) {
ss, err := newTestScaleSet(ctrl)
assert.NoError(t, err, "unexpected error when creating test VMSS")
expectedVMSS := compute.VirtualMachineScaleSet{
Name: to.StringPtr(testVMSSName),
VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{},
},
}
expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-")
mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface)
mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes()
@ -1703,12 +1741,7 @@ func TestEnsureHostInPool(t *testing.T) {
ss.LoadBalancerSku = loadBalancerSkuStandard
}
expectedVMSS := compute.VirtualMachineScaleSet{
Name: to.StringPtr(testVMSSName),
VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{},
},
}
expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-")
mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface)
mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes()
@ -1876,7 +1909,7 @@ func TestEnsureVMSSInPool(t *testing.T) {
ss.LoadBalancerSku = loadBalancerSkuStandard
}
expectedVMSS := buildTestVMSS(testVMSSName, []string{testLBBackendpoolID0}, test.setIPv6Config)
expectedVMSS := buildTestVMSSWithLB(testVMSSName, "vmss-vm-", []string{testLBBackendpoolID0}, test.setIPv6Config)
if test.isVMSSDeallocating {
expectedVMSS.ProvisioningState = &virtualMachineScaleSetsDeallocating
}
@ -1973,7 +2006,7 @@ func TestEnsureHostsInPool(t *testing.T) {
ss.LoadBalancerSku = loadBalancerSkuStandard
ss.ExcludeMasterFromStandardLB = to.BoolPtr(true)
expectedVMSS := buildTestVMSS(testVMSSName, []string{testLBBackendpoolID0}, false)
expectedVMSS := buildTestVMSSWithLB(testVMSSName, "vmss-vm-", []string{testLBBackendpoolID0}, false)
mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface)
mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes()
mockVMSSClient.EXPECT().Get(gomock.Any(), ss.ResourceGroup, testVMSSName).Return(expectedVMSS, nil).MaxTimes(1)
@ -2061,12 +2094,7 @@ func TestEnsureBackendPoolDeletedFromNode(t *testing.T) {
ss, err := newTestScaleSet(ctrl)
assert.NoError(t, err, test.description)
expectedVMSS := compute.VirtualMachineScaleSet{
Name: to.StringPtr(testVMSSName),
VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{},
},
}
expectedVMSS := buildTestVMSS(testVMSSName, "vmss-vm-")
mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface)
mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes()
@ -2155,7 +2183,7 @@ func TestEnsureBackendPoolDeletedFromVMSS(t *testing.T) {
ss.LoadBalancerSku = loadBalancerSkuStandard
expectedVMSS := buildTestVMSS(testVMSSName, []string{testLBBackendpoolID0}, false)
expectedVMSS := buildTestVMSSWithLB(testVMSSName, "vmss-vm-", []string{testLBBackendpoolID0}, false)
if test.isVMSSDeallocating {
expectedVMSS.ProvisioningState = &virtualMachineScaleSetsDeallocating
}
@ -2249,7 +2277,7 @@ func TestEnsureBackendPoolDeleted(t *testing.T) {
ss, err := newTestScaleSet(ctrl)
assert.NoError(t, err, test.description)
expectedVMSS := buildTestVMSS(testVMSSName, []string{testLBBackendpoolID0}, false)
expectedVMSS := buildTestVMSSWithLB(testVMSSName, "vmss-vm-", []string{testLBBackendpoolID0}, false)
mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface)
mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes()
mockVMSSClient.EXPECT().Get(gomock.Any(), ss.ResourceGroup, testVMSSName).Return(expectedVMSS, nil).MaxTimes(1)
@ -2308,11 +2336,12 @@ func TestEnsureBackendPoolDeletedConcurrently(t *testing.T) {
},
},
}
vmss0 := buildTestVMSS("vmss-0", []string{testLBBackendpoolID0, testLBBackendpoolID1}, false)
vmss1 := buildTestVMSS("vmss-1", []string{testLBBackendpoolID0, testLBBackendpoolID1}, false)
expectedVMSSVMsOfVMSS0, _, _ := buildTestVirtualMachineEnv(ss.cloud, "vmss-0", "", 0, []string{"vmss-vm-000000"}, "succeeded", false)
expectedVMSSVMsOfVMSS1, _, _ := buildTestVirtualMachineEnv(ss.cloud, "vmss-1", "", 0, []string{"vmss-vm-000001"}, "succeeded", false)
vmss0 := buildTestVMSSWithLB("vmss-0", "vmss-0-vm-", []string{testLBBackendpoolID0, testLBBackendpoolID1}, false)
vmss1 := buildTestVMSSWithLB("vmss-1", "vmss-1-vm-", []string{testLBBackendpoolID0, testLBBackendpoolID1}, false)
expectedVMSSVMsOfVMSS0, _, _ := buildTestVirtualMachineEnv(ss.cloud, "vmss-0", "", 0, []string{"vmss-0-vm-000000"}, "succeeded", false)
expectedVMSSVMsOfVMSS1, _, _ := buildTestVirtualMachineEnv(ss.cloud, "vmss-1", "", 0, []string{"vmss-1-vm-000001"}, "succeeded", false)
for _, expectedVMSSVMs := range [][]compute.VirtualMachineScaleSetVM{expectedVMSSVMsOfVMSS0, expectedVMSSVMsOfVMSS1} {
vmssVMNetworkConfigs := expectedVMSSVMs[0].NetworkProfileConfiguration
vmssVMIPConfigs := (*vmssVMNetworkConfigs.NetworkInterfaceConfigurations)[0].VirtualMachineScaleSetNetworkConfigurationProperties.IPConfigurations
@ -2322,11 +2351,13 @@ func TestEnsureBackendPoolDeletedConcurrently(t *testing.T) {
mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface)
mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{vmss0, vmss1}, nil).AnyTimes()
mockVMSSClient.EXPECT().List(gomock.Any(), "rg1").Return(nil, nil).AnyTimes()
mockVMSSClient.EXPECT().Get(gomock.Any(), ss.ResourceGroup, "vmss-0").Return(vmss0, nil).MaxTimes(2)
mockVMSSClient.EXPECT().Get(gomock.Any(), ss.ResourceGroup, "vmss-1").Return(vmss1, nil).MaxTimes(2)
mockVMSSClient.EXPECT().CreateOrUpdate(gomock.Any(), ss.ResourceGroup, gomock.Any(), gomock.Any()).Return(nil).Times(2)
mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface)
mockVMSSVMClient.EXPECT().List(gomock.Any(), "rg1", "vmss-0", gomock.Any()).Return(nil, nil).AnyTimes()
mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, "vmss-0", gomock.Any()).Return(expectedVMSSVMsOfVMSS0, nil).AnyTimes()
mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, "vmss-1", gomock.Any()).Return(expectedVMSSVMsOfVMSS1, nil).AnyTimes()
mockVMSSVMClient.EXPECT().UpdateVMs(gomock.Any(), ss.ResourceGroup, gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(2)