From 3fdceba86f4c464926545047e57e01dffb120ac4 Mon Sep 17 00:00:00 2001 From: t-qini Date: Sun, 19 Apr 2020 16:55:10 +0800 Subject: [PATCH] Enrich the unit tests for azure_vmss.go. --- .../azure/azure_fakes.go | 1 + .../azure/azure_loadbalancer.go | 2 +- .../azure/azure_vmss.go | 14 +- .../azure/azure_vmss_test.go | 1604 ++++++++++++++++- 4 files changed, 1607 insertions(+), 14 deletions(-) diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_fakes.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_fakes.go index fafc771e607..8b4f64cb7a3 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_fakes.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_fakes.go @@ -61,6 +61,7 @@ func GetTestCloud(ctrl *gomock.Controller) (az *Cloud) { SecurityGroupName: "nsg", RouteTableName: "rt", PrimaryAvailabilitySetName: "as", + PrimaryScaleSetName: "vmss", MaximumLoadBalancerRuleCount: 250, VMType: vmTypeStandard, }, diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go index 597e3e70cee..ec276c1ca02 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go @@ -1733,7 +1733,7 @@ func subnet(service *v1.Service) *string { // getServiceLoadBalancerMode parses the mode value. // if the value is __auto__ it returns isAuto = TRUE. -// if anything else it returns the unique VM set names after triming spaces. +// if anything else it returns the unique VM set names after trimming spaces. func getServiceLoadBalancerMode(service *v1.Service) (hasMode bool, isAuto bool, vmSetNames []string) { mode, hasMode := service.Annotations[ServiceAnnotationLoadBalancerMode] mode = strings.TrimSpace(mode) diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go index f70ad7e535f..0a10027a461 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go @@ -44,7 +44,7 @@ var ( // virtualMachineScaleSetsDeallocating indicates VMSS instances are in Deallocating state. virtualMachineScaleSetsDeallocating = "Deallocating" - // ErrorNotVmssInstance indicates an instance is not belongint to any vmss. + // ErrorNotVmssInstance indicates an instance is not belonging to any vmss. ErrorNotVmssInstance = errors.New("not a vmss instance") scaleSetNameRE = regexp.MustCompile(`.*/subscriptions/(?:.*)/Microsoft.Compute/virtualMachineScaleSets/(.+)/virtualMachines(?:.*)`) @@ -289,7 +289,7 @@ func (ss *scaleSet) GetNodeNameByProviderID(providerID string) (types.NodeName, // NodeName is not part of providerID for vmss instances. scaleSetName, err := extractScaleSetNameByProviderID(providerID) if err != nil { - klog.V(4).Infof("Can not extract scale set name from providerID (%s), assuming it is mananaged by availability set: %v", providerID, err) + klog.V(4).Infof("Can not extract scale set name from providerID (%s), assuming it is managed by availability set: %v", providerID, err) return ss.availabilitySet.GetNodeNameByProviderID(providerID) } @@ -300,7 +300,7 @@ func (ss *scaleSet) GetNodeNameByProviderID(providerID string) (types.NodeName, instanceID, err := getLastSegment(providerID) if err != nil { - klog.V(4).Infof("Can not extract instanceID from providerID (%s), assuming it is mananaged by availability set: %v", providerID, err) + klog.V(4).Infof("Can not extract instanceID from providerID (%s), assuming it is managed by availability set: %v", providerID, err) return ss.availabilitySet.GetNodeNameByProviderID(providerID) } @@ -433,11 +433,11 @@ func (ss *scaleSet) GetIPByNodeName(nodeName string) (string, string, error) { return internalIP, publicIP, nil } -func (ss *scaleSet) getVMSSPublicIPAddress(resourceGroupName string, virtualMachineScaleSetName string, virtualmachineIndex string, networkInterfaceName string, IPConfigurationName string, publicIPAddressName string) (network.PublicIPAddress, bool, error) { +func (ss *scaleSet) getVMSSPublicIPAddress(resourceGroupName string, virtualMachineScaleSetName string, virtualMachineIndex string, networkInterfaceName string, IPConfigurationName string, publicIPAddressName string) (network.PublicIPAddress, bool, error) { ctx, cancel := getContextWithCancel() defer cancel() - pip, err := ss.PublicIPAddressesClient.GetVirtualMachineScaleSetPublicIPAddress(ctx, resourceGroupName, virtualMachineScaleSetName, virtualmachineIndex, networkInterfaceName, IPConfigurationName, publicIPAddressName, "") + pip, err := ss.PublicIPAddressesClient.GetVirtualMachineScaleSetPublicIPAddress(ctx, resourceGroupName, virtualMachineScaleSetName, virtualMachineIndex, networkInterfaceName, IPConfigurationName, publicIPAddressName, "") exists, rerr := checkResourceExistsFromError(err) if rerr != nil { return pip, false, rerr.Error() @@ -712,7 +712,7 @@ func (ss *scaleSet) GetPrimaryInterface(nodeName string) (network.Interface, err exists, realErr := checkResourceExistsFromError(rerr) if realErr != nil { klog.Errorf("error: ss.GetPrimaryInterface(%s), ss.GetVirtualMachineScaleSetNetworkInterface.Get(%s, %s, %s), err=%v", nodeName, resourceGroup, ssName, nicName, realErr) - return network.Interface{}, err + return network.Interface{}, realErr.Error() } if !exists { @@ -956,7 +956,7 @@ func (ss *scaleSet) ensureVMSSInPool(service *v1.Service, nodes []*v1.Node, back } if vmss.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations == nil { - klog.V(4).Infof("EnsureHostInPool: cannot obtain the primary network interface configuration, of vmss %s", vmssName) + klog.V(4).Infof("EnsureHostInPool: cannot obtain the primary network interface configuration of vmss %s", vmssName) continue } vmssNIC := *vmss.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go index fa4d473bff3..b7509521a23 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go @@ -23,6 +23,9 @@ import ( "strings" "testing" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" cloudprovider "k8s.io/cloud-provider" azcache "k8s.io/legacy-cloud-providers/azure/cache" "k8s.io/legacy-cloud-providers/azure/clients/interfaceclient/mockinterfaceclient" @@ -40,8 +43,11 @@ import ( ) const ( - fakePrivateIP = "10.240.0.10" - fakePublicIP = "10.10.10.10" + fakePrivateIP = "10.240.0.10" + fakePublicIP = "10.10.10.10" + testVMSSName = "vmss" + testVMPowerState = "PowerState/Running" + testLBBackendpoolID = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-0" ) func newTestScaleSet(ctrl *gomock.Controller) (*scaleSet, error) { @@ -75,12 +81,18 @@ func buildTestVirtualMachineEnv(ss *Cloud, scaleSetName, zone string, faultDomai networkInterfaces := []compute.NetworkInterfaceReference{ { ID: &interfaceID, + NetworkInterfaceReferenceProperties: &compute.NetworkInterfaceReferenceProperties{ + Primary: to.BoolPtr(true), + }, }, } ipConfigurations := []compute.VirtualMachineScaleSetIPConfiguration{ { Name: to.StringPtr("ipconfig1"), - VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{}, + VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ + Primary: to.BoolPtr(true), + LoadBalancerBackendAddressPools: &[]compute.SubResource{{ID: to.StringPtr(testLBBackendpoolID)}}, + }, }, } networkConfigurations := []compute.VirtualMachineScaleSetNetworkConfiguration{ @@ -106,12 +118,16 @@ func buildTestVirtualMachineEnv(ss *Cloud, scaleSetName, zone string, faultDomai }, InstanceView: &compute.VirtualMachineScaleSetVMInstanceView{ PlatformFaultDomain: &faultDomain, + Statuses: &[]compute.InstanceViewStatus{ + {Code: to.StringPtr(testVMPowerState)}, + }, }, }, ID: &ID, InstanceID: &instanceID, Name: &vmName, Location: &ss.Location, + Sku: &compute.Sku{Name: to.StringPtr("sku")}, } if zone != "" { zones := []string{zone} @@ -120,7 +136,8 @@ func buildTestVirtualMachineEnv(ss *Cloud, scaleSetName, zone string, faultDomai // set interfaces. expectedInterface = network.Interface{ - ID: &interfaceID, + Name: to.StringPtr("nic"), + ID: &interfaceID, InterfacePropertiesFormat: &network.InterfacePropertiesFormat{ IPConfigurations: &[]network.InterfaceIPConfiguration{ { @@ -557,13 +574,13 @@ func TestGetVmssVM(t *testing.T) { description: "getVmssVM should return the correct name of vmss, the instance id of the node, and the corresponding vmss instance", nodeName: "vmss-vm-000000", existedNodeNames: []string{"vmss-vm-000000"}, - existedVMSSName: "vmss", + existedVMSSName: testVMSSName, }, { description: "getVmssVM should report an error of instance not found if there's no matches", nodeName: "vmss-vm-000001", existedNodeNames: []string{"vmss-vm-000000"}, - existedVMSSName: "vmss", + existedVMSSName: testVMSSName, expectedError: cloudprovider.InstanceNotFound, }, } @@ -594,3 +611,1578 @@ func TestGetVmssVM(t *testing.T) { assert.Equal(t, test.expectedError, err, test.description) } } + +func TestGetPowerStatusByNodeName(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + testCases := []struct { + description string + vmList []string + nilStatus bool + expectedPowerState string + expectedErr error + }{ + { + description: "GetPowerStatusByNodeName should return the correct power state", + vmList: []string{"vmss-vm-000001"}, + expectedPowerState: "Running", + }, + { + description: "GetPowerStatusByNodeName should return vmPowerStateStopped when the vm.InstanceView.Statuses is nil", + vmList: []string{"vmss-vm-000001"}, + nilStatus: true, + expectedPowerState: vmPowerStateStopped, + }, + } + + for _, test := range testCases { + ss, err := newTestScaleSet(ctrl) + assert.NoError(t, err, "unexpected error when creating test VMSS") + + expectedVMSS := compute.VirtualMachineScaleSet{Name: to.StringPtr(testVMSSName)} + mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) + mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() + + expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, test.vmList, "") + mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) + if test.nilStatus { + expectedVMSSVMs[0].InstanceView.Statuses = nil + } + mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() + + powerState, err := ss.GetPowerStatusByNodeName("vmss-vm-000001") + assert.Equal(t, test.expectedErr, err, test.description+", but an error occurs") + assert.Equal(t, test.expectedPowerState, powerState, test.description) + } +} + +func TestGetVmssVMByInstanceID(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + testCases := []struct { + description string + instanceID string + vmList []string + expectedErr error + }{ + { + description: "GetVmssVMByInstanceID should return the correct VMSS VM", + instanceID: "0", + vmList: []string{"vmss-vm-000000"}, + }, + } + + for _, test := range testCases { + ss, err := newTestScaleSet(ctrl) + assert.NoError(t, err, "unexpected error when creating test VMSS") + + expectedVMSS := compute.VirtualMachineScaleSet{Name: to.StringPtr(testVMSSName)} + mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) + mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() + + expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, test.vmList, "") + mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) + mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() + + vm, err := ss.getVmssVMByInstanceID(ss.ResourceGroup, testVMSSName, test.instanceID, azcache.CacheReadTypeDefault) + assert.Equal(t, test.expectedErr, err, test.description+", but an error occurs") + assert.Equal(t, expectedVMSSVMs[0], *vm, test.description) + } +} + +func TestGetInstanceTypeByNodeName(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + testCases := []struct { + description string + vmList []string + vmClientErr *retry.Error + expectedType string + expectedErr error + }{ + { + description: "GetInstanceTypeByNodeName should return the correct instance type", + vmList: []string{"vmss-vm-000000"}, + expectedType: "sku", + }, + { + description: "GetInstanceTypeByNodeName should report the error that occurs", + vmList: []string{"vmss-vm-000000"}, + vmClientErr: &retry.Error{RawError: fmt.Errorf("error")}, + expectedType: "", + expectedErr: fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 0, RawError: error"), + }, + } + + for _, test := range testCases { + ss, err := newTestScaleSet(ctrl) + assert.NoError(t, err, "unexpected error when creating test VMSS") + + expectedVMSS := compute.VirtualMachineScaleSet{Name: to.StringPtr(testVMSSName)} + mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) + mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() + + expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, test.vmList, "") + mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) + mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() + + mockVMClient := ss.cloud.VirtualMachinesClient.(*mockvmclient.MockInterface) + mockVMClient.EXPECT().List(gomock.Any(), gomock.Any()).Return(nil, test.vmClientErr).AnyTimes() + + sku, err := ss.GetInstanceTypeByNodeName("vmss-vm-000000") + assert.Equal(t, test.expectedErr, err, test.description+", but an error occurs") + assert.Equal(t, test.expectedType, sku, test.description) + } +} + +func TestGetPrimaryInterfaceID(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + testCases := []struct { + description string + existedInterfaces []compute.NetworkInterfaceReference + expectedID string + expectedErr error + }{ + { + description: "GetPrimaryInterfaceID should return the ID of the primary NIC on the VMSS VM", + existedInterfaces: []compute.NetworkInterfaceReference{ + { + ID: to.StringPtr("1"), + NetworkInterfaceReferenceProperties: &compute.NetworkInterfaceReferenceProperties{ + Primary: to.BoolPtr(true), + }, + }, + {ID: to.StringPtr("2")}, + }, + expectedID: "1", + }, + { + description: "GetPrimaryInterfaceID should report an error if there's no primary NIC on the VMSS VM", + existedInterfaces: []compute.NetworkInterfaceReference{ + { + ID: to.StringPtr("1"), + NetworkInterfaceReferenceProperties: &compute.NetworkInterfaceReferenceProperties{ + Primary: to.BoolPtr(false), + }, + }, + { + ID: to.StringPtr("2"), + NetworkInterfaceReferenceProperties: &compute.NetworkInterfaceReferenceProperties{ + Primary: to.BoolPtr(false), + }, + }, + }, + expectedErr: fmt.Errorf("failed to find a primary nic for the vm. vmname=\"vm\""), + }, + } + + for _, test := range testCases { + ss, err := newTestScaleSet(ctrl) + assert.NoError(t, err, "unexpected error when creating test VMSS") + + vm := compute.VirtualMachineScaleSetVM{ + Name: to.StringPtr("vm"), + VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ + NetworkProfile: &compute.NetworkProfile{ + NetworkInterfaces: &test.existedInterfaces, + }, + }, + } + + id, err := ss.getPrimaryInterfaceID(vm) + assert.Equal(t, test.expectedErr, err, test.description+", but an error occurs") + assert.Equal(t, test.expectedID, id, test.description) + } +} + +func TestGetPrimaryInterface(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + testCases := []struct { + description string + nodeName string + vmList []string + vmClientErr *retry.Error + vmssClientErr *retry.Error + nicClientErr *retry.Error + hasPrimaryInterface bool + isInvalidNICID bool + expectedErr error + }{ + { + description: "GetPrimaryInterface should return the correct network interface", + nodeName: "vmss-vm-000000", + vmList: []string{"vmss-vm-000000"}, + hasPrimaryInterface: true, + }, + { + description: "GetPrimaryInterface should report the error if vm client returns retry error", + nodeName: "vmss-vm-000000", + vmList: []string{"vmss-vm-000000"}, + hasPrimaryInterface: true, + vmClientErr: &retry.Error{RawError: fmt.Errorf("error")}, + expectedErr: fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 0, RawError: error"), + }, + { + description: "GetPrimaryInterface should report the error if vmss client returns retry error", + nodeName: "vmss-vm-000000", + vmList: []string{"vmss-vm-000000"}, + hasPrimaryInterface: true, + vmssClientErr: &retry.Error{RawError: fmt.Errorf("error")}, + expectedErr: fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 0, RawError: error"), + }, + { + description: "GetPrimaryInterface should report the error if there is no primary interface", + nodeName: "vmss-vm-000000", + vmList: []string{"vmss-vm-000000"}, + hasPrimaryInterface: false, + expectedErr: fmt.Errorf("failed to find a primary nic for the vm. vmname=\"vmss_0\""), + }, + { + description: "GetPrimaryInterface should report the error if the id of the primary nic is not valid", + nodeName: "vmss-vm-000000", + vmList: []string{"vmss-vm-000000"}, + isInvalidNICID: true, + hasPrimaryInterface: true, + expectedErr: fmt.Errorf("resource name was missing from identifier"), + }, + { + description: "GetPrimaryInterface should report the error if nic client returns retry error", + nodeName: "vmss-vm-000000", + vmList: []string{"vmss-vm-000000"}, + hasPrimaryInterface: true, + nicClientErr: &retry.Error{RawError: fmt.Errorf("error")}, + expectedErr: fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 0, RawError: error"), + }, + { + description: "GetPrimaryInterface should report the error if the NIC instance is not found", + nodeName: "vmss-vm-000000", + vmList: []string{"vmss-vm-000000"}, + hasPrimaryInterface: true, + nicClientErr: &retry.Error{HTTPStatusCode: 404, RawError: fmt.Errorf("not found")}, + expectedErr: cloudprovider.InstanceNotFound, + }, + } + + for _, test := range testCases { + ss, err := newTestScaleSet(ctrl) + assert.NoError(t, err, "unexpected error when creating test VMSS") + + expectedVMSS := compute.VirtualMachineScaleSet{Name: to.StringPtr(testVMSSName)} + mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) + mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, test.vmssClientErr).AnyTimes() + + expectedVMSSVMs, expectedInterface, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, test.vmList, "") + if !test.hasPrimaryInterface { + networkInterfaces := *expectedVMSSVMs[0].NetworkProfile.NetworkInterfaces + networkInterfaces[0].Primary = to.BoolPtr(false) + networkInterfaces = append(networkInterfaces, compute.NetworkInterfaceReference{ + NetworkInterfaceReferenceProperties: &compute.NetworkInterfaceReferenceProperties{Primary: to.BoolPtr(false)}, + }) + expectedVMSSVMs[0].NetworkProfile.NetworkInterfaces = &networkInterfaces + } + if test.isInvalidNICID { + networkInterfaces := *expectedVMSSVMs[0].NetworkProfile.NetworkInterfaces + networkInterfaces[0].ID = to.StringPtr("invalid/id/") + expectedVMSSVMs[0].NetworkProfile.NetworkInterfaces = &networkInterfaces + } + mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) + mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() + + mockVMClient := ss.cloud.VirtualMachinesClient.(*mockvmclient.MockInterface) + mockVMClient.EXPECT().List(gomock.Any(), gomock.Any()).Return(nil, test.vmClientErr).AnyTimes() + + mockInterfaceClient := ss.cloud.InterfacesClient.(*mockinterfaceclient.MockInterface) + mockInterfaceClient.EXPECT().GetVirtualMachineScaleSetNetworkInterface(gomock.Any(), ss.ResourceGroup, testVMSSName, "0", test.nodeName, gomock.Any()).Return(expectedInterface, test.nicClientErr).AnyTimes() + expectedInterface.Location = &ss.Location + + if test.vmClientErr != nil || test.vmssClientErr != nil || test.nicClientErr != nil || !test.hasPrimaryInterface || test.isInvalidNICID { + expectedInterface = network.Interface{} + } + + nic, err := ss.GetPrimaryInterface(test.nodeName) + assert.Equal(t, test.expectedErr, err, test.description+", but an error occurs") + assert.Equal(t, expectedInterface, nic, test.description) + } +} + +func TestGetVMSSPublicIPAddress(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + testCases := []struct { + description string + pipClientErr *retry.Error + pipName string + found bool + expectedErr error + }{ + { + description: "GetVMSSPublicIPAddress should return the correct public IP address", + pipName: "pip", + found: true, + }, + { + description: "GetVMSSPublicIPAddress should report the error if the pip client returns retry.Error", + pipName: "pip", + found: false, + pipClientErr: &retry.Error{RawError: fmt.Errorf("error")}, + expectedErr: fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 0, RawError: error"), + }, + { + description: "GetVMSSPublicIPAddress should not report errors if the pip cannot be found", + pipName: "pip-1", + found: false, + }, + } + + for _, test := range testCases { + ss, err := newTestScaleSet(ctrl) + assert.NoError(t, err, "unexpected error when creating test VMSS") + + mockPIPClient := ss.cloud.PublicIPAddressesClient.(*mockpublicipclient.MockInterface) + mockPIPClient.EXPECT().GetVirtualMachineScaleSetPublicIPAddress(gomock.Any(), ss.ResourceGroup, testVMSSName, "0", "nic", "ip", "pip", "").Return(network.PublicIPAddress{}, test.pipClientErr).AnyTimes() + mockPIPClient.EXPECT().GetVirtualMachineScaleSetPublicIPAddress(gomock.Any(), ss.ResourceGroup, testVMSSName, "0", "nic", "ip", gomock.Not("pip"), "").Return(network.PublicIPAddress{}, &retry.Error{HTTPStatusCode: 404, RawError: fmt.Errorf("not found")}).AnyTimes() + + _, found, err := ss.getVMSSPublicIPAddress(ss.ResourceGroup, testVMSSName, "0", "nic", "ip", test.pipName) + assert.Equal(t, test.expectedErr, err, test.description+", but an error occurs") + assert.Equal(t, test.found, found, test.description) + } +} + +func TestGetPrivateIPsByNodeName(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + testCases := []struct { + description string + nodeName string + vmList []string + isNilIPConfigs bool + vmClientErr *retry.Error + expectedPrivateIPs []string + expectedErr error + }{ + { + description: "GetPrivateIPsByNodeName should return the correct private IPs", + nodeName: "vmss-vm-000000", + vmList: []string{"vmss-vm-000000"}, + expectedPrivateIPs: []string{fakePrivateIP}, + }, + { + description: "GetPrivateIPsByNodeName should report the error if the ipconfig of the nic is nil", + nodeName: "vmss-vm-000000", + vmList: []string{"vmss-vm-000000"}, + isNilIPConfigs: true, + expectedPrivateIPs: []string{}, + expectedErr: fmt.Errorf("nic.IPConfigurations for nic (nicname=\"nic\") is nil"), + }, + { + description: "GetPrivateIPsByNodeName should report the error if error happens during GetPrimaryInterface", + nodeName: "vmss-vm-000000", + vmList: []string{"vmss-vm-000000"}, + vmClientErr: &retry.Error{RawError: fmt.Errorf("error")}, + expectedPrivateIPs: []string{}, + expectedErr: fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 0, RawError: error"), + }, + } + + for _, test := range testCases { + ss, err := newTestScaleSet(ctrl) + assert.NoError(t, err, "unexpected error when creating test VMSS") + + expectedVMSS := compute.VirtualMachineScaleSet{Name: to.StringPtr(testVMSSName)} + mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) + mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() + + expectedVMSSVMs, expectedInterface, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, test.vmList, "") + + mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) + mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() + + mockVMClient := ss.cloud.VirtualMachinesClient.(*mockvmclient.MockInterface) + mockVMClient.EXPECT().List(gomock.Any(), gomock.Any()).Return(nil, test.vmClientErr).AnyTimes() + + if test.isNilIPConfigs { + expectedInterface.IPConfigurations = nil + } + mockInterfaceClient := ss.cloud.InterfacesClient.(*mockinterfaceclient.MockInterface) + mockInterfaceClient.EXPECT().GetVirtualMachineScaleSetNetworkInterface(gomock.Any(), ss.ResourceGroup, testVMSSName, "0", test.nodeName, gomock.Any()).Return(expectedInterface, nil).AnyTimes() + + privateIPs, err := ss.GetPrivateIPsByNodeName(test.nodeName) + assert.Equal(t, test.expectedErr, err, test.description+", but an error occurs") + assert.Equal(t, test.expectedPrivateIPs, privateIPs, test.description) + } +} + +func TestGetVmssMachineID(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + ss, err := newTestScaleSet(ctrl) + assert.NoError(t, err, "unexpected error when creating test VMSS") + + subscriptionID, resourceGroup, scaleSetName, instanceID := "sub", "RG", "vmss", "id" + VMSSMachineID := ss.cloud.getVmssMachineID(subscriptionID, resourceGroup, scaleSetName, instanceID) + expectedVMSSMachineID := fmt.Sprintf(vmssMachineIDTemplate, subscriptionID, strings.ToLower(resourceGroup), scaleSetName, instanceID) + assert.Equal(t, expectedVMSSMachineID, VMSSMachineID, "GetVmssMachineID should return the correct VMSS machine ID") +} + +func TestExtractScaleSetNameByProviderID(t *testing.T) { + providerID := "/subscriptions/script/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/vmss-vm-000000" + vmssName, err := extractScaleSetNameByProviderID(providerID) + assert.Nil(t, err, fmt.Errorf("unexpected error %v happened", err)) + assert.Equal(t, "vmss", vmssName, "extractScaleSetNameByProviderID should return the correct vmss name") + + providerID = "/invalid/id" + vmssName, err = extractScaleSetNameByProviderID(providerID) + assert.Equal(t, ErrorNotVmssInstance, err, "extractScaleSetNameByProviderID should return the error of ErrorNotVmssInstance if the providerID is not a valid vmss ID") + assert.Equal(t, "", vmssName, "extractScaleSetNameByProviderID should return an empty string") +} + +func TestExtractResourceGroupByProviderID(t *testing.T) { + providerID := "/subscriptions/script/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/vmss-vm-000000" + vmssName, err := extractResourceGroupByProviderID(providerID) + assert.Nil(t, err, fmt.Errorf("unexpected error %v happened", err)) + assert.Equal(t, "rg", vmssName, "extractScaleSetNameByProviderID should return the correct vmss name") + + providerID = "/invalid/id" + vmssName, err = extractResourceGroupByProviderID(providerID) + assert.Equal(t, ErrorNotVmssInstance, err, "extractScaleSetNameByProviderID should return the error of ErrorNotVmssInstance if the providerID is not a valid vmss ID") + assert.Equal(t, "", vmssName, "extractScaleSetNameByProviderID should return an empty string") +} + +func TestListScaleSets(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + testCases := []struct { + description string + existedScaleSets []compute.VirtualMachineScaleSet + vmssClientErr *retry.Error + expectedVMSSNames []string + expectedErr error + }{ + { + description: "listScaleSets should return the correct scale sets", + existedScaleSets: []compute.VirtualMachineScaleSet{ + { + Name: to.StringPtr("vmss-0"), + Sku: &compute.Sku{Capacity: to.Int64Ptr(1)}, + }, + { + Name: to.StringPtr("vmss-1"), + }, + { + Name: to.StringPtr("vmss-2"), + Sku: &compute.Sku{Capacity: to.Int64Ptr(0)}, + }, + }, + expectedVMSSNames: []string{"vmss-0", "vmss-1"}, + }, + { + description: "listScaleSets should report the error if vmss client returns an retry.Error", + vmssClientErr: &retry.Error{RawError: fmt.Errorf("error")}, + expectedErr: fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 0, RawError: error"), + }, + } + + for _, test := range testCases { + ss, err := newTestScaleSet(ctrl) + assert.NoError(t, err, "unexpected error when creating test VMSS") + + mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) + mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return(test.existedScaleSets, test.vmssClientErr).AnyTimes() + + vmssNames, err := ss.listScaleSets(ss.ResourceGroup) + assert.Equal(t, test.expectedErr, err, test.description+", but an error occurs") + assert.Equal(t, test.expectedVMSSNames, vmssNames, test.description) + } +} + +func TestListScaleSetVMs(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + testCases := []struct { + description string + existedVMSSVMs []compute.VirtualMachineScaleSetVM + vmssVMClientErr *retry.Error + expectedErr error + }{ + { + description: "listScaleSetVMs should return the correct vmss vms", + existedVMSSVMs: []compute.VirtualMachineScaleSetVM{ + {Name: to.StringPtr("vmss-vm-000000")}, + {Name: to.StringPtr("vmss-vm-000001")}, + }, + }, + { + description: "listScaleSetVMs should report the error that the vmss vm client hits", + vmssVMClientErr: &retry.Error{RawError: fmt.Errorf("error")}, + expectedErr: fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 0, RawError: error"), + }, + } + + for _, test := range testCases { + ss, err := newTestScaleSet(ctrl) + assert.NoError(t, err, "unexpected error when creating test VMSS") + + mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) + mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(test.existedVMSSVMs, test.vmssVMClientErr).AnyTimes() + + expectedVMSSVMs := test.existedVMSSVMs + + vmssVMs, err := ss.listScaleSetVMs(testVMSSName, ss.ResourceGroup) + assert.Equal(t, test.expectedErr, err, test.description+", but an error occurs") + assert.Equal(t, expectedVMSSVMs, vmssVMs, test.description) + } +} + +func TestGetAgentPoolScaleSets(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + testCases := []struct { + description string + nodes []*v1.Node + expectedVMSSNames *[]string + expectedErr error + }{ + { + description: "getAgentPoolScaleSets should return the correct vmss names", + nodes: []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "vmss-vm-000000", + Labels: map[string]string{nodeLabelRole: "master"}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "vmss-vm-000001", + Labels: map[string]string{managedByAzureLabel: "false"}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "vmss-vm-000002", + }, + }, + }, + expectedVMSSNames: &[]string{"vmss"}, + }, + } + + for _, test := range testCases { + ss, err := newTestScaleSet(ctrl) + assert.NoError(t, err, "unexpected error when creating test VMSS") + + expectedVMSS := compute.VirtualMachineScaleSet{Name: to.StringPtr(testVMSSName)} + mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) + mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() + + expectedVMSSVMs := []compute.VirtualMachineScaleSetVM{ + { + VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ + OsProfile: &compute.OSProfile{ComputerName: to.StringPtr("vmss-vm-000000")}, + }, + }, + { + VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ + OsProfile: &compute.OSProfile{ComputerName: to.StringPtr("vmss-vm-000001")}, + }, + }, + { + VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ + OsProfile: &compute.OSProfile{ComputerName: to.StringPtr("vmss-vm-000002")}, + }, + }, + } + mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) + mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() + + mockVMClient := ss.cloud.VirtualMachinesClient.(*mockvmclient.MockInterface) + mockVMClient.EXPECT().List(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() + + vmssNames, err := ss.getAgentPoolScaleSets(test.nodes) + assert.Equal(t, test.expectedErr, err, test.description+", but an error occurs") + assert.Equal(t, test.expectedVMSSNames, vmssNames) + } +} + +func TestGetVMSetNames(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + testCases := []struct { + description string + service *v1.Service + nodes []*v1.Node + expectedVMSetNames *[]string + expectedErr error + }{ + { + description: "GetVMSetNames should return the primary vm set name if the service has no mode annotation", + service: &v1.Service{}, + expectedVMSetNames: &[]string{"vmss"}, + }, + { + description: "GetVMSetNames should return nil if the service has auto mode annotation", + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ServiceAnnotationLoadBalancerMode: ServiceAnnotationLoadBalancerAutoModeValue}}, + }, + nodes: []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "vmss-vm-000002", + }, + }, + }, + }, + { + description: "GetVMSetNames should report the error if there's no such vmss", + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ServiceAnnotationLoadBalancerMode: "vmss-1"}}, + }, + nodes: []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "vmss-vm-000002", + }, + }, + }, + expectedErr: fmt.Errorf("scale set (vmss-1) - not found"), + }, + { + description: "GetVMSetNames should return the correct vmss names", + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ServiceAnnotationLoadBalancerMode: "vmss"}}, + }, + nodes: []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "vmss-vm-000002", + }, + }, + }, + expectedVMSetNames: &[]string{"vmss"}, + }, + } + + for _, test := range testCases { + ss, err := newTestScaleSet(ctrl) + assert.NoError(t, err, "unexpected error when creating test VMSS") + + expectedVMSS := compute.VirtualMachineScaleSet{Name: to.StringPtr(testVMSSName)} + mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) + mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() + + expectedVMSSVMs := []compute.VirtualMachineScaleSetVM{ + { + VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ + OsProfile: &compute.OSProfile{ComputerName: to.StringPtr("vmss-vm-000000")}, + }, + }, + { + VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ + OsProfile: &compute.OSProfile{ComputerName: to.StringPtr("vmss-vm-000001")}, + }, + }, + { + VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ + OsProfile: &compute.OSProfile{ComputerName: to.StringPtr("vmss-vm-000002")}, + }, + }, + } + mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) + mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() + + mockVMClient := ss.cloud.VirtualMachinesClient.(*mockvmclient.MockInterface) + mockVMClient.EXPECT().List(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() + + vmSetNames, err := ss.GetVMSetNames(test.service, test.nodes) + assert.Equal(t, test.expectedErr, err, test.description+", but an error occurs") + assert.Equal(t, test.expectedVMSetNames, vmSetNames) + } +} + +func TestGetPrimaryNetworkInterfaceConfigurationForScaleSet(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + ss, err := newTestScaleSet(ctrl) + assert.NoError(t, err, "unexpected error when creating test VMSS") + + networkConfigs := []compute.VirtualMachineScaleSetNetworkConfiguration{ + {Name: to.StringPtr("config-0")}, + } + config, err := ss.getPrimaryNetworkInterfaceConfigurationForScaleSet(networkConfigs, testVMSSName) + assert.Nil(t, err, "getPrimaryNetworkInterfaceConfigurationForScaleSet should return the correct network config") + assert.Equal(t, &networkConfigs[0], config, "getPrimaryNetworkInterfaceConfigurationForScaleSet should return the correct network config") + + networkConfigs = []compute.VirtualMachineScaleSetNetworkConfiguration{ + { + Name: to.StringPtr("config-0"), + VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ + Primary: to.BoolPtr(false), + }, + }, + { + Name: to.StringPtr("config-1"), + VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ + Primary: to.BoolPtr(true), + }, + }, + } + config, err = ss.getPrimaryNetworkInterfaceConfigurationForScaleSet(networkConfigs, testVMSSName) + assert.Nil(t, err, "getPrimaryNetworkInterfaceConfigurationForScaleSet should return the correct network config") + assert.Equal(t, &networkConfigs[1], config, "getPrimaryNetworkInterfaceConfigurationForScaleSet should return the correct network config") + + networkConfigs = []compute.VirtualMachineScaleSetNetworkConfiguration{ + { + Name: to.StringPtr("config-0"), + VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ + Primary: to.BoolPtr(false), + }, + }, + { + Name: to.StringPtr("config-1"), + VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ + Primary: to.BoolPtr(false), + }, + }, + } + config, err = ss.getPrimaryNetworkInterfaceConfigurationForScaleSet(networkConfigs, testVMSSName) + assert.Equal(t, fmt.Errorf("failed to find a primary network configuration for the scale set \"vmss\""), err, "getPrimaryNetworkInterfaceConfigurationForScaleSet should report an error if there is no primary nic") + assert.Nil(t, config, "getPrimaryNetworkInterfaceConfigurationForScaleSet should report an error if there is no primary nic") +} + +func TestGetPrimaryIPConfigFromVMSSNetworkConfig(t *testing.T) { + config := &compute.VirtualMachineScaleSetNetworkConfiguration{ + VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ + IPConfigurations: &[]compute.VirtualMachineScaleSetIPConfiguration{ + { + Name: to.StringPtr("config-0"), + }, + }, + }, + } + + ipConfig, err := getPrimaryIPConfigFromVMSSNetworkConfig(config) + assert.Nil(t, err, "getPrimaryIPConfigFromVMSSNetworkConfig should return the correct IP config") + assert.Equal(t, (*config.IPConfigurations)[0], *ipConfig, "getPrimaryIPConfigFromVMSSNetworkConfig should return the correct IP config") + + config = &compute.VirtualMachineScaleSetNetworkConfiguration{ + VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ + IPConfigurations: &[]compute.VirtualMachineScaleSetIPConfiguration{ + { + Name: to.StringPtr("config-0"), + VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ + Primary: to.BoolPtr(false), + }, + }, + { + Name: to.StringPtr("config-1"), + VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ + Primary: to.BoolPtr(true), + }, + }, + }, + }, + } + + ipConfig, err = getPrimaryIPConfigFromVMSSNetworkConfig(config) + assert.Nil(t, err, "getPrimaryIPConfigFromVMSSNetworkConfig should return the correct IP config") + assert.Equal(t, (*config.IPConfigurations)[1], *ipConfig, "getPrimaryIPConfigFromVMSSNetworkConfig should return the correct IP config") + + config = &compute.VirtualMachineScaleSetNetworkConfiguration{ + VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ + IPConfigurations: &[]compute.VirtualMachineScaleSetIPConfiguration{ + { + Name: to.StringPtr("config-0"), + VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ + Primary: to.BoolPtr(false), + }, + }, + { + Name: to.StringPtr("config-1"), + VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ + Primary: to.BoolPtr(false), + }, + }, + }, + }, + } + + ipConfig, err = getPrimaryIPConfigFromVMSSNetworkConfig(config) + assert.Equal(t, err, fmt.Errorf("failed to find a primary IP configuration"), "getPrimaryIPConfigFromVMSSNetworkConfig should report an error if there is no primary IP config") + assert.Nil(t, ipConfig, "getPrimaryIPConfigFromVMSSNetworkConfig should report an error if there is no primary IP config") +} + +func TestGetConfigForScaleSetByIPFamily(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + ss, err := newTestScaleSet(ctrl) + assert.NoError(t, err, "unexpected error when creating test VMSS") + + config := &compute.VirtualMachineScaleSetNetworkConfiguration{ + VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ + IPConfigurations: &[]compute.VirtualMachineScaleSetIPConfiguration{ + { + Name: to.StringPtr("config-0"), + VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ + PrivateIPAddressVersion: compute.IPv4, + }, + }, + { + Name: to.StringPtr("config-0"), + VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ + PrivateIPAddressVersion: compute.IPv6, + }, + }, + }, + }, + } + + ipConfig, err := ss.getConfigForScaleSetByIPFamily(config, "vmss-vm-000000", true) + assert.Nil(t, err, "getConfigForScaleSetByIPFamily should find the IPV6 config") + assert.Equal(t, (*config.IPConfigurations)[1], *ipConfig, "getConfigForScaleSetByIPFamily should find the IPV6 config") + + ipConfig, err = ss.getConfigForScaleSetByIPFamily(config, "vmss-vm-000000", false) + assert.Nil(t, err, "getConfigForScaleSetByIPFamily should find the IPV4 config") + assert.Equal(t, (*config.IPConfigurations)[0], *ipConfig, "getConfigForScaleSetByIPFamily should find the IPV4 config") +} + +func TestEnsureHostInPool(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + testCases := []struct { + description string + service *v1.Service + nodeName types.NodeName + backendPoolID string + vmSetName string + isBasicLB bool + isNilVMNetworkConfigs bool + expectedNodeResourceGroup string + expectedVMSSName string + expectedInstanceID string + expectedVMSSVM *compute.VirtualMachineScaleSetVM + expectedErr error + }{ + { + description: "EnsureHostInPool should skip the current node if the vmSetName is not equal to the node's vmss name and the basic LB is used", + nodeName: "vmss-vm-000000", + vmSetName: "vmss-1", + isBasicLB: true, + }, + { + description: "EnsureHostInPool should skip the current node if the network configs of the VMSS VM is nil", + nodeName: "vmss-vm-000000", + vmSetName: "vmss", + isNilVMNetworkConfigs: true, + }, + { + description: "EnsureHostInPool should skip the current node if the backend pool has existed", + service: &v1.Service{Spec: v1.ServiceSpec{ClusterIP: "clusterIP"}}, + nodeName: "vmss-vm-000000", + vmSetName: "vmss", + backendPoolID: testLBBackendpoolID, + }, + { + description: "EnsureHostInPool should skip the current node if it has already been added to another LB", + service: &v1.Service{Spec: v1.ServiceSpec{ClusterIP: "clusterIP"}}, + nodeName: "vmss-vm-000000", + vmSetName: "vmss", + backendPoolID: "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb1-internal/backendAddressPools/backendpool-1", + isBasicLB: false, + }, + { + description: "EnsureHostInPool should add a new backend pool to the vm", + service: &v1.Service{Spec: v1.ServiceSpec{ClusterIP: "clusterIP"}}, + nodeName: "vmss-vm-000000", + vmSetName: "vmss", + backendPoolID: "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb-internal/backendAddressPools/backendpool-1", + isBasicLB: false, + expectedNodeResourceGroup: "rg", + expectedVMSSName: testVMSSName, + expectedInstanceID: "0", + expectedVMSSVM: &compute.VirtualMachineScaleSetVM{ + Sku: &compute.Sku{Name: to.StringPtr("sku")}, + Location: to.StringPtr("westus"), + VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ + NetworkProfileConfiguration: &compute.VirtualMachineScaleSetVMNetworkProfileConfiguration{ + NetworkInterfaceConfigurations: &[]compute.VirtualMachineScaleSetNetworkConfiguration{ + { + Name: to.StringPtr("ipconfig1"), + ID: to.StringPtr("fakeNetworkConfiguration"), + VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ + IPConfigurations: &[]compute.VirtualMachineScaleSetIPConfiguration{ + { + Name: to.StringPtr("ipconfig1"), + VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ + Primary: to.BoolPtr(true), + LoadBalancerBackendAddressPools: &[]compute.SubResource{ + { + ID: to.StringPtr(testLBBackendpoolID), + }, + { + ID: to.StringPtr("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb-internal/backendAddressPools/backendpool-1"), + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + for _, test := range testCases { + ss, err := newTestScaleSet(ctrl) + assert.NoError(t, err, test.description) + + if !test.isBasicLB { + ss.LoadBalancerSku = loadBalancerSkuStandard + } + + expectedVMSS := compute.VirtualMachineScaleSet{Name: to.StringPtr(testVMSSName)} + mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) + mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() + + expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, []string{string(test.nodeName)}, "") + if test.isNilVMNetworkConfigs { + expectedVMSSVMs[0].NetworkProfileConfiguration.NetworkInterfaceConfigurations = nil + } + mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) + mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() + + nodeResourceGroup, ssName, instanceID, vm, err := ss.EnsureHostInPool(test.service, test.nodeName, test.backendPoolID, test.vmSetName, false) + assert.Equal(t, test.expectedErr, err, test.description+", but an error occurs") + assert.Equal(t, test.expectedNodeResourceGroup, nodeResourceGroup, test.description) + assert.Equal(t, test.expectedVMSSName, ssName, test.description) + assert.Equal(t, test.expectedInstanceID, instanceID, test.description) + assert.Equal(t, test.expectedVMSSVM, vm, test.description) + } +} + +func TestGetVmssAndResourceGroupNameByVMProviderID(t *testing.T) { + providerID := "azure:///subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0" + rgName, vmssName, err := getVmssAndResourceGroupNameByVMProviderID(providerID) + assert.Nil(t, err) + assert.Equal(t, "rg", rgName) + assert.Equal(t, "vmss", vmssName) + + providerID = "invalid/id" + rgName, vmssName, err = getVmssAndResourceGroupNameByVMProviderID(providerID) + assert.Equal(t, err, ErrorNotVmssInstance) + assert.Equal(t, "", rgName) + assert.Equal(t, "", vmssName) +} + +func TestEnsureVMSSInPool(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + testCases := []struct { + description string + nodes []*v1.Node + backendPoolID string + vmSetName string + isBasicLB bool + isVMSSDeallocating bool + isVMSSNilNICConfig bool + expectedPutVMSS bool + expectedErr error + }{ + { + description: "ensureVMSSInPool should skip the node if it isn't managed by VMSS", + nodes: []*v1.Node{ + { + Spec: v1.NodeSpec{ + ProviderID: "invalid/id", + }, + }, + }, + isBasicLB: false, + expectedPutVMSS: false, + }, + { + description: "ensureVMSSInPool should skip the node if the corresponding VMSS is deallocating", + nodes: []*v1.Node{ + { + Spec: v1.NodeSpec{ + ProviderID: "azure:///subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0", + }, + }, + }, + isBasicLB: false, + isVMSSDeallocating: true, + expectedPutVMSS: false, + }, + { + description: "ensureVMSSInPool should skip the node if the NIC config of the corresponding VMSS is nil", + nodes: []*v1.Node{ + { + Spec: v1.NodeSpec{ + ProviderID: "azure:///subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0", + }, + }, + }, + isBasicLB: false, + isVMSSNilNICConfig: true, + expectedPutVMSS: false, + }, + { + description: "ensureVMSSInPool should skip the node if the backendpool ID has been added already", + nodes: []*v1.Node{ + { + Spec: v1.NodeSpec{ + ProviderID: "azure:///subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0", + }, + }, + }, + isBasicLB: false, + backendPoolID: testLBBackendpoolID, + expectedPutVMSS: false, + }, + { + description: "ensureVMSSInPool should skip the node if the VMSS has been added to another LB's backendpool already", + nodes: []*v1.Node{ + { + Spec: v1.NodeSpec{ + ProviderID: "azure:///subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0", + }, + }, + }, + isBasicLB: false, + backendPoolID: "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb1/backendAddressPools/backendpool-0", + expectedPutVMSS: false, + }, + { + description: "ensureVMSSInPool should update the VMSS correctly", + nodes: []*v1.Node{ + { + Spec: v1.NodeSpec{ + ProviderID: "azure:///subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0", + }, + }, + }, + isBasicLB: false, + backendPoolID: "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-1", + expectedPutVMSS: true, + }, + } + + for _, test := range testCases { + ss, err := newTestScaleSet(ctrl) + assert.NoError(t, err, test.description) + + if !test.isBasicLB { + ss.LoadBalancerSku = loadBalancerSkuStandard + } + + expectedVMSS := compute.VirtualMachineScaleSet{ + Name: to.StringPtr(testVMSSName), + VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ + ProvisioningState: to.StringPtr("Running"), + VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{ + NetworkProfile: &compute.VirtualMachineScaleSetNetworkProfile{ + NetworkInterfaceConfigurations: &[]compute.VirtualMachineScaleSetNetworkConfiguration{ + { + VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ + Primary: to.BoolPtr(true), + IPConfigurations: &[]compute.VirtualMachineScaleSetIPConfiguration{ + { + VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ + LoadBalancerBackendAddressPools: &[]compute.SubResource{ + { + ID: to.StringPtr(testLBBackendpoolID), + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + if test.isVMSSDeallocating { + expectedVMSS.ProvisioningState = &virtualMachineScaleSetsDeallocating + } + if test.isVMSSNilNICConfig { + expectedVMSS.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations = nil + } + mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) + mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() + vmssPutTimes := 0 + if test.expectedPutVMSS { + vmssPutTimes = 1 + mockVMSSClient.EXPECT().Get(gomock.Any(), ss.ResourceGroup, testVMSSName).Return(expectedVMSS, nil) + } + mockVMSSClient.EXPECT().CreateOrUpdate(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(nil).Times(vmssPutTimes) + + expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, []string{"vmss-vm-000000"}, "") + mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) + mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() + + err = ss.ensureVMSSInPool(&v1.Service{}, test.nodes, test.backendPoolID, test.vmSetName) + assert.Equal(t, test.expectedErr, err, test.description+", but an error occurs") + } +} + +func TestEnsureHostsInPool(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + testCases := []struct { + description string + nodes []*v1.Node + backendpoolID string + vmSetName string + expectedVMSSVMPutTimes int + expectedErr bool + }{ + { + description: "EnsureHostsInPool should skip the invalid node and update the VMSS VM correctly", + nodes: []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "vmss-vm-000000", + Labels: map[string]string{nodeLabelRole: "master"}, + }, + Spec: v1.NodeSpec{ + ProviderID: "azure:///subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "vmss-vm-000001", + Labels: map[string]string{managedByAzureLabel: "false"}, + }, + Spec: v1.NodeSpec{ + ProviderID: "azure:///subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/1", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "vmss-vm-000002", + }, + Spec: v1.NodeSpec{ + ProviderID: "azure:///subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/2", + }, + }, + }, + backendpoolID: "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-1", + vmSetName: testVMSSName, + expectedVMSSVMPutTimes: 1, + }, + { + description: "EnsureHostsInPool should gather report the error if something goes wrong in EnsureHostInPool", + nodes: []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "vmss-vm-000003", + }, + Spec: v1.NodeSpec{ + ProviderID: "azure:///subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/3", + }, + }, + }, + backendpoolID: "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-1", + vmSetName: testVMSSName, + expectedVMSSVMPutTimes: 0, + expectedErr: true, + }, + } + + for _, test := range testCases { + ss, err := newTestScaleSet(ctrl) + assert.NoError(t, err, test.description) + + ss.LoadBalancerSku = loadBalancerSkuStandard + ss.ExcludeMasterFromStandardLB = to.BoolPtr(true) + + expectedVMSS := compute.VirtualMachineScaleSet{ + Name: to.StringPtr(testVMSSName), + VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ + ProvisioningState: to.StringPtr("Running"), + VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{ + NetworkProfile: &compute.VirtualMachineScaleSetNetworkProfile{ + NetworkInterfaceConfigurations: &[]compute.VirtualMachineScaleSetNetworkConfiguration{ + { + VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ + Primary: to.BoolPtr(true), + IPConfigurations: &[]compute.VirtualMachineScaleSetIPConfiguration{ + { + VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ + LoadBalancerBackendAddressPools: &[]compute.SubResource{ + { + ID: to.StringPtr(testLBBackendpoolID), + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) + mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() + mockVMSSClient.EXPECT().Get(gomock.Any(), ss.ResourceGroup, testVMSSName).Return(expectedVMSS, nil).MaxTimes(1) + mockVMSSClient.EXPECT().CreateOrUpdate(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(nil).MaxTimes(1) + + expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, []string{"vmss-vm-000000", "vmss-vm-000001", "vmss-vm-000002"}, "") + mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) + mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() + mockVMSSVMClient.EXPECT().UpdateVMs(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any(), gomock.Any()).Return(nil).Times(test.expectedVMSSVMPutTimes) + + mockVMClient := ss.cloud.VirtualMachinesClient.(*mockvmclient.MockInterface) + mockVMClient.EXPECT().List(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() + + err = ss.EnsureHostsInPool(&v1.Service{}, test.nodes, test.backendpoolID, test.vmSetName, false) + assert.Equal(t, test.expectedErr, err != nil, test.description+", but an error occurs") + } +} + +func TestEnsureBackendPoolDeletedFromNode(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + testCases := []struct { + description string + nodeName string + backendpoolID string + isNilVMNetworkConfigs bool + expectedNodeResourceGroup string + expectedVMSSName string + expectedInstanceID string + expectedVMSSVM *compute.VirtualMachineScaleSetVM + expectedErr error + }{ + { + description: "ensureBackendPoolDeletedFromNode should report the error that occurs during getVmssVM", + nodeName: "vmss-vm-000001", + expectedErr: cloudprovider.InstanceNotFound, + }, + { + description: "ensureBackendPoolDeletedFromNode skip the node if the VM's NIC config is nil", + nodeName: "vmss-vm-000000", + isNilVMNetworkConfigs: true, + }, + { + description: "ensureBackendPoolDeletedFromNode should skip the node if there's no wanted lb backendpool ID on that VM", + nodeName: "vmss-vm-000000", + backendpoolID: "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-1", + }, + { + description: "ensureBackendPoolDeletedFromNode should delete the given backendpool ID", + nodeName: "vmss-vm-000000", + backendpoolID: testLBBackendpoolID, + expectedNodeResourceGroup: "rg", + expectedVMSSName: testVMSSName, + expectedInstanceID: "0", + expectedVMSSVM: &compute.VirtualMachineScaleSetVM{ + Sku: &compute.Sku{Name: to.StringPtr("sku")}, + Location: to.StringPtr("westus"), + VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ + NetworkProfileConfiguration: &compute.VirtualMachineScaleSetVMNetworkProfileConfiguration{ + NetworkInterfaceConfigurations: &[]compute.VirtualMachineScaleSetNetworkConfiguration{ + { + Name: to.StringPtr("ipconfig1"), + ID: to.StringPtr("fakeNetworkConfiguration"), + VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ + IPConfigurations: &[]compute.VirtualMachineScaleSetIPConfiguration{ + { + Name: to.StringPtr("ipconfig1"), + VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ + Primary: to.BoolPtr(true), + LoadBalancerBackendAddressPools: &[]compute.SubResource{}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + for _, test := range testCases { + ss, err := newTestScaleSet(ctrl) + assert.NoError(t, err, test.description) + + expectedVMSS := compute.VirtualMachineScaleSet{Name: to.StringPtr(testVMSSName)} + mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) + mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() + + expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, []string{"vmss-vm-000000"}, "") + if test.isNilVMNetworkConfigs { + expectedVMSSVMs[0].NetworkProfileConfiguration.NetworkInterfaceConfigurations = nil + } + mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) + mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() + + nodeResourceGroup, ssName, instanceID, vm, err := ss.ensureBackendPoolDeletedFromNode(&v1.Service{}, test.nodeName, test.backendpoolID) + assert.Equal(t, test.expectedErr, err, test.description+", but an error occurs") + assert.Equal(t, test.expectedNodeResourceGroup, nodeResourceGroup, test.description) + assert.Equal(t, test.expectedVMSSName, ssName, test.description) + assert.Equal(t, test.expectedInstanceID, instanceID, test.description) + assert.Equal(t, test.expectedVMSSVM, vm, test.description) + } +} + +func TestGetScaleSetAndResourceGroupNameByIPConfigurationID(t *testing.T) { + ipConfigID := "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/vmss-vm-000000/networkInterfaces/nic" + scaleSetName, resourceGroup, err := getScaleSetAndResourceGroupNameByIPConfigurationID(ipConfigID) + assert.Equal(t, "vmss", scaleSetName) + assert.Equal(t, "rg", resourceGroup) + assert.NoError(t, err) + + ipConfigID = "invalid/id" + scaleSetName, resourceGroup, err = getScaleSetAndResourceGroupNameByIPConfigurationID(ipConfigID) + assert.Equal(t, ErrorNotVmssInstance, err) + assert.Equal(t, "", scaleSetName) + assert.Equal(t, "", resourceGroup) +} + +func TestEnsureBackendPoolDeletedFromVMSS(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + testCases := []struct { + description string + backendPoolID string + ipConfigurationIDs []string + isVMSSDeallocating bool + isVMSSNilNICConfig bool + expectedPutVMSS bool + vmssClientErr *retry.Error + expectedErr error + }{ + { + description: "ensureBackendPoolDeletedFromVMSS should skip the IP config if it's ID is invalid", + ipConfigurationIDs: []string{"invalid/id"}, + }, + { + description: "ensureBackendPoolDeletedFromVMSS should skip the VMSS if it's being deleting", + ipConfigurationIDs: []string{"/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/vmss-vm-000000/networkInterfaces/nic"}, + isVMSSDeallocating: true, + }, + { + description: "ensureBackendPoolDeletedFromVMSS should skip the VMSS if it's NIC config is nil", + ipConfigurationIDs: []string{"/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/vmss-vm-000000/networkInterfaces/nic"}, + isVMSSNilNICConfig: true, + }, + { + description: "ensureBackendPoolDeletedFromVMSS should delete the corresponding LB backendpool ID", + ipConfigurationIDs: []string{"/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/vmss-vm-000000/networkInterfaces/nic"}, + backendPoolID: testLBBackendpoolID, + expectedPutVMSS: true, + }, + { + description: "ensureBackendPoolDeletedFromVMSS should skip the VMSS if there's no wanted LB backendpool ID", + ipConfigurationIDs: []string{"/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/vmss-vm-000000/networkInterfaces/nic"}, + backendPoolID: "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-1", + }, + { + description: "ensureBackendPoolDeletedFromVMSS should report the error that occurs during VMSS client's call", + ipConfigurationIDs: []string{"/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/vmss-vm-000000/networkInterfaces/nic"}, + backendPoolID: testLBBackendpoolID, + expectedPutVMSS: true, + vmssClientErr: &retry.Error{RawError: fmt.Errorf("error")}, + expectedErr: fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 0, RawError: error"), + }, + } + + for _, test := range testCases { + ss, err := newTestScaleSet(ctrl) + assert.NoError(t, err, test.description) + + ss.LoadBalancerSku = loadBalancerSkuStandard + + expectedVMSS := compute.VirtualMachineScaleSet{ + Name: to.StringPtr(testVMSSName), + VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ + ProvisioningState: to.StringPtr("Running"), + VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{ + NetworkProfile: &compute.VirtualMachineScaleSetNetworkProfile{ + NetworkInterfaceConfigurations: &[]compute.VirtualMachineScaleSetNetworkConfiguration{ + { + VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ + Primary: to.BoolPtr(true), + IPConfigurations: &[]compute.VirtualMachineScaleSetIPConfiguration{ + { + VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ + LoadBalancerBackendAddressPools: &[]compute.SubResource{ + { + ID: to.StringPtr(testLBBackendpoolID), + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + if test.isVMSSDeallocating { + expectedVMSS.ProvisioningState = &virtualMachineScaleSetsDeallocating + } + if test.isVMSSNilNICConfig { + expectedVMSS.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations = nil + } + mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) + mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() + vmssPutTimes := 0 + if test.expectedPutVMSS { + vmssPutTimes = 1 + mockVMSSClient.EXPECT().Get(gomock.Any(), ss.ResourceGroup, testVMSSName).Return(expectedVMSS, nil) + } + mockVMSSClient.EXPECT().CreateOrUpdate(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(test.vmssClientErr).Times(vmssPutTimes) + + expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, []string{"vmss-vm-000000"}, "") + mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) + mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() + + err = ss.ensureBackendPoolDeletedFromVMSS(&v1.Service{}, test.backendPoolID, testVMSSName, test.ipConfigurationIDs) + assert.Equal(t, test.expectedErr, err, test.description+", but an error occurs") + } +} + +func TestEnsureBackendPoolDeleted(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + testCases := []struct { + description string + backendpoolID string + backendAddressPools *[]network.BackendAddressPool + expectedVMSSVMPutTimes int + vmClientErr *retry.Error + expectedErr bool + }{ + { + description: "EnsureBackendPoolDeleted should skip the unwanted backend address pools and update the VMSS VM correctly", + backendpoolID: testLBBackendpoolID, + backendAddressPools: &[]network.BackendAddressPool{ + { + ID: to.StringPtr(testLBBackendpoolID), + BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{ + BackendIPConfigurations: &[]network.InterfaceIPConfiguration{ + { + Name: to.StringPtr("ip-1"), + ID: to.StringPtr("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0/networkInterfaces/nic"), + }, + { + Name: to.StringPtr("ip-2"), + }, + { + Name: to.StringPtr("ip-3"), + ID: to.StringPtr("/invalid/id"), + }, + }, + }, + }, + { + ID: to.StringPtr("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-1"), + }, + }, + expectedVMSSVMPutTimes: 1, + }, + { + description: "EnsureBackendPoolDeleted should report the error that occurs during the call of VMSS VM client", + backendpoolID: testLBBackendpoolID, + backendAddressPools: &[]network.BackendAddressPool{ + { + ID: to.StringPtr(testLBBackendpoolID), + BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{ + BackendIPConfigurations: &[]network.InterfaceIPConfiguration{ + { + Name: to.StringPtr("ip-1"), + ID: to.StringPtr("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0/networkInterfaces/nic"), + }, + }, + }, + }, + { + ID: to.StringPtr("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-1"), + }, + }, + expectedVMSSVMPutTimes: 1, + expectedErr: true, + vmClientErr: &retry.Error{RawError: fmt.Errorf("error")}, + }, + } + + for _, test := range testCases { + ss, err := newTestScaleSet(ctrl) + assert.NoError(t, err, test.description) + + expectedVMSS := compute.VirtualMachineScaleSet{ + Name: to.StringPtr(testVMSSName), + VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ + ProvisioningState: to.StringPtr("Running"), + VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{ + NetworkProfile: &compute.VirtualMachineScaleSetNetworkProfile{ + NetworkInterfaceConfigurations: &[]compute.VirtualMachineScaleSetNetworkConfiguration{ + { + VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ + Primary: to.BoolPtr(true), + IPConfigurations: &[]compute.VirtualMachineScaleSetIPConfiguration{ + { + VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ + LoadBalancerBackendAddressPools: &[]compute.SubResource{ + { + ID: to.StringPtr(testLBBackendpoolID), + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) + mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() + mockVMSSClient.EXPECT().Get(gomock.Any(), ss.ResourceGroup, testVMSSName).Return(expectedVMSS, nil).MaxTimes(1) + mockVMSSClient.EXPECT().CreateOrUpdate(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(nil).MaxTimes(1) + + expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, []string{"vmss-vm-000000", "vmss-vm-000001", "vmss-vm-000002"}, "") + mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) + mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() + mockVMSSVMClient.EXPECT().UpdateVMs(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any(), gomock.Any()).Return(test.vmClientErr).Times(test.expectedVMSSVMPutTimes) + + err = ss.EnsureBackendPoolDeleted(&v1.Service{}, test.backendpoolID, testVMSSName, test.backendAddressPools) + assert.Equal(t, test.expectedErr, err != nil, test.description+", but an error occurs") + } +}