From e48f69de10a910bd410b9c854791d75900936847 Mon Sep 17 00:00:00 2001 From: t-qini Date: Sat, 25 Apr 2020 17:33:03 +0800 Subject: [PATCH] Add unit tests for azure race conditions. --- .../k8s.io/legacy-cloud-providers/azure/BUILD | 4 + .../azure/azure_backoff.go | 14 +- .../azure/azure_backoff_test.go | 516 ++++++++++++++++++ .../azure/azure_test.go | 3 +- .../azure/azure_vmss_test.go | 272 ++++----- 5 files changed, 665 insertions(+), 144 deletions(-) create mode 100644 staging/src/k8s.io/legacy-cloud-providers/azure/azure_backoff_test.go diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/BUILD b/staging/src/k8s.io/legacy-cloud-providers/azure/BUILD index cbce39f4bbe..72331c8025e 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/BUILD +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/BUILD @@ -109,6 +109,7 @@ go_library( go_test( name = "go_default_test", srcs = [ + "azure_backoff_test.go", "azure_config_test.go", "azure_controller_common_test.go", "azure_controller_standard_test.go", @@ -132,7 +133,9 @@ go_test( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", "//staging/src/k8s.io/cloud-provider/service/helpers:go_default_library", @@ -144,6 +147,7 @@ go_test( "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/interfaceclient/mockinterfaceclient:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/loadbalancerclient/mockloadbalancerclient:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/publicipclient/mockpublicipclient:go_default_library", + "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/routeclient/mockrouteclient:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/routetableclient/mockroutetableclient:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/securitygroupclient/mocksecuritygroupclient:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/storageaccountclient/mockstorageaccountclient:go_default_library", diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_backoff.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_backoff.go index 89bbb01a365..1046ef51cc3 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_backoff.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_backoff.go @@ -40,8 +40,8 @@ const ( // not active means the instance is under deleting from Azure VMSS. vmssVMNotActiveErrorMessage = "not an active Virtual Machine Scale Set VM instanceId" - // operationCancledErrorMessage means the operation is canceled by another new operation. - operationCancledErrorMessage = "canceledandsupersededduetoanotheroperation" + // operationCanceledErrorMessage means the operation is canceled by another new operation. + operationCanceledErrorMessage = "canceledandsupersededduetoanotheroperation" ) // RequestBackoff if backoff is disabled in cloud provider it @@ -166,8 +166,8 @@ func (az *Cloud) CreateOrUpdateSecurityGroup(service *v1.Service, sg network.Sec } // Invalidate the cache because another new operation has canceled the current request. - if strings.Contains(strings.ToLower(rerr.Error().Error()), operationCancledErrorMessage) { - klog.V(3).Infof("SecurityGroup cache for %s is cleanup because CreateOrUpdateSecurityGroup is canceld by another operation", *sg.Name) + if strings.Contains(strings.ToLower(rerr.Error().Error()), operationCanceledErrorMessage) { + klog.V(3).Infof("SecurityGroup cache for %s is cleanup because CreateOrUpdateSecurityGroup is canceled by another operation", *sg.Name) az.nsgCache.Delete(*sg.Name) } @@ -194,7 +194,7 @@ func (az *Cloud) CreateOrUpdateLB(service *v1.Service, lb network.LoadBalancer) az.lbCache.Delete(*lb.Name) } // Invalidate the cache because another new operation has canceled the current request. - if strings.Contains(strings.ToLower(rerr.Error().Error()), operationCancledErrorMessage) { + if strings.Contains(strings.ToLower(rerr.Error().Error()), operationCanceledErrorMessage) { klog.V(3).Infof("LoadBalancer cache for %s is cleanup because CreateOrUpdate is canceled by another operation", *lb.Name) az.lbCache.Delete(*lb.Name) } @@ -317,7 +317,7 @@ func (az *Cloud) CreateOrUpdateRouteTable(routeTable network.RouteTable) error { az.rtCache.Delete(*routeTable.Name) } // Invalidate the cache because another new operation has canceled the current request. - if strings.Contains(strings.ToLower(rerr.Error().Error()), operationCancledErrorMessage) { + if strings.Contains(strings.ToLower(rerr.Error().Error()), operationCanceledErrorMessage) { klog.V(3).Infof("Route table cache for %s is cleanup because CreateOrUpdateRouteTable is canceld by another operation", *routeTable.Name) az.rtCache.Delete(*routeTable.Name) } @@ -342,7 +342,7 @@ func (az *Cloud) CreateOrUpdateRoute(route network.Route) error { az.rtCache.Delete(az.RouteTableName) } // Invalidate the cache because another new operation has canceled the current request. - if strings.Contains(strings.ToLower(rerr.Error().Error()), operationCancledErrorMessage) { + if strings.Contains(strings.ToLower(rerr.Error().Error()), operationCanceledErrorMessage) { klog.V(3).Infof("Route cache for %s is cleanup because CreateOrUpdateRouteTable is canceld by another operation", *route.Name) az.rtCache.Delete(az.RouteTableName) } diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_backoff_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_backoff_test.go new file mode 100644 index 00000000000..85388857aab --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_backoff_test.go @@ -0,0 +1,516 @@ +// +build !providerless + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "fmt" + "net/http" + "testing" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/wait" + cloudprovider "k8s.io/cloud-provider" + "k8s.io/legacy-cloud-providers/azure/cache" + "k8s.io/legacy-cloud-providers/azure/clients/interfaceclient/mockinterfaceclient" + "k8s.io/legacy-cloud-providers/azure/clients/loadbalancerclient/mockloadbalancerclient" + "k8s.io/legacy-cloud-providers/azure/clients/publicipclient/mockpublicipclient" + "k8s.io/legacy-cloud-providers/azure/clients/routeclient/mockrouteclient" + "k8s.io/legacy-cloud-providers/azure/clients/routetableclient/mockroutetableclient" + "k8s.io/legacy-cloud-providers/azure/clients/securitygroupclient/mocksecuritygroupclient" + "k8s.io/legacy-cloud-providers/azure/clients/vmclient/mockvmclient" + "k8s.io/legacy-cloud-providers/azure/clients/vmssclient/mockvmssclient" + "k8s.io/legacy-cloud-providers/azure/retry" + + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" + "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + "github.com/Azure/go-autorest/autorest/to" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" +) + +func TestGetVirtualMachineWithRetry(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tests := []struct { + vmClientErr *retry.Error + expectedErr error + }{ + { + vmClientErr: &retry.Error{HTTPStatusCode: http.StatusNotFound}, + expectedErr: cloudprovider.InstanceNotFound, + }, + { + vmClientErr: &retry.Error{HTTPStatusCode: http.StatusInternalServerError}, + expectedErr: fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 500, RawError: "), + }, + } + + for _, test := range tests { + az := GetTestCloud(ctrl) + mockVMClient := az.VirtualMachinesClient.(*mockvmclient.MockInterface) + mockVMClient.EXPECT().Get(gomock.Any(), az.ResourceGroup, "vm", gomock.Any()).Return(compute.VirtualMachine{}, test.vmClientErr) + + vm, err := az.GetVirtualMachineWithRetry("vm", cache.CacheReadTypeDefault) + assert.Empty(t, vm) + assert.Equal(t, test.expectedErr, err) + } +} + +func TestGetPrivateIPsForMachine(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tests := []struct { + vmClientErr *retry.Error + expectedPrivateIPs []string + expectedErr error + }{ + { + expectedPrivateIPs: []string{"1.2.3.4"}, + }, + { + vmClientErr: &retry.Error{HTTPStatusCode: http.StatusNotFound}, + expectedErr: cloudprovider.InstanceNotFound, + expectedPrivateIPs: []string{}, + }, + { + vmClientErr: &retry.Error{HTTPStatusCode: http.StatusInternalServerError}, + expectedErr: wait.ErrWaitTimeout, + expectedPrivateIPs: []string{}, + }, + } + + expectedVM := compute.VirtualMachine{ + VirtualMachineProperties: &compute.VirtualMachineProperties{ + AvailabilitySet: &compute.SubResource{ID: to.StringPtr("availability-set")}, + NetworkProfile: &compute.NetworkProfile{ + NetworkInterfaces: &[]compute.NetworkInterfaceReference{ + { + NetworkInterfaceReferenceProperties: &compute.NetworkInterfaceReferenceProperties{ + Primary: to.BoolPtr(true), + }, + ID: to.StringPtr("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/nic"), + }, + }, + }, + }, + } + + expectedInterface := network.Interface{ + InterfacePropertiesFormat: &network.InterfacePropertiesFormat{ + IPConfigurations: &[]network.InterfaceIPConfiguration{ + { + InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ + PrivateIPAddress: to.StringPtr("1.2.3.4"), + }, + }, + }, + }, + } + + for _, test := range tests { + az := GetTestCloud(ctrl) + mockVMClient := az.VirtualMachinesClient.(*mockvmclient.MockInterface) + mockVMClient.EXPECT().Get(gomock.Any(), az.ResourceGroup, "vm", gomock.Any()).Return(expectedVM, test.vmClientErr) + + mockInterfaceClient := az.InterfacesClient.(*mockinterfaceclient.MockInterface) + mockInterfaceClient.EXPECT().Get(gomock.Any(), az.ResourceGroup, "nic", gomock.Any()).Return(expectedInterface, nil).MaxTimes(1) + + privateIPs, err := az.getPrivateIPsForMachine("vm") + assert.Equal(t, test.expectedErr, err) + assert.Equal(t, test.expectedPrivateIPs, privateIPs) + } +} + +func TestGetIPForMachineWithRetry(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tests := []struct { + clientErr *retry.Error + expectedPrivateIP string + expectedPublicIP string + expectedErr error + }{ + { + expectedPrivateIP: "1.2.3.4", + expectedPublicIP: "5.6.7.8", + }, + { + clientErr: &retry.Error{HTTPStatusCode: http.StatusNotFound}, + expectedErr: wait.ErrWaitTimeout, + }, + } + + expectedVM := compute.VirtualMachine{ + VirtualMachineProperties: &compute.VirtualMachineProperties{ + AvailabilitySet: &compute.SubResource{ID: to.StringPtr("availability-set")}, + NetworkProfile: &compute.NetworkProfile{ + NetworkInterfaces: &[]compute.NetworkInterfaceReference{ + { + NetworkInterfaceReferenceProperties: &compute.NetworkInterfaceReferenceProperties{ + Primary: to.BoolPtr(true), + }, + ID: to.StringPtr("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/nic"), + }, + }, + }, + }, + } + + expectedInterface := network.Interface{ + InterfacePropertiesFormat: &network.InterfacePropertiesFormat{ + IPConfigurations: &[]network.InterfaceIPConfiguration{ + { + InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ + PrivateIPAddress: to.StringPtr("1.2.3.4"), + PublicIPAddress: &network.PublicIPAddress{ + ID: to.StringPtr("test/pip"), + }, + }, + }, + }, + }, + } + + expectedPIP := network.PublicIPAddress{ + PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ + IPAddress: to.StringPtr("5.6.7.8"), + }, + } + + for _, test := range tests { + az := GetTestCloud(ctrl) + mockVMClient := az.VirtualMachinesClient.(*mockvmclient.MockInterface) + mockVMClient.EXPECT().Get(gomock.Any(), az.ResourceGroup, "vm", gomock.Any()).Return(expectedVM, test.clientErr) + + mockInterfaceClient := az.InterfacesClient.(*mockinterfaceclient.MockInterface) + mockInterfaceClient.EXPECT().Get(gomock.Any(), az.ResourceGroup, "nic", gomock.Any()).Return(expectedInterface, nil).MaxTimes(1) + + mockPIPClient := az.PublicIPAddressesClient.(*mockpublicipclient.MockInterface) + mockPIPClient.EXPECT().Get(gomock.Any(), az.ResourceGroup, "pip", gomock.Any()).Return(expectedPIP, nil).MaxTimes(1) + + privateIP, publicIP, err := az.GetIPForMachineWithRetry("vm") + assert.Equal(t, test.expectedErr, err) + assert.Equal(t, test.expectedPrivateIP, privateIP) + assert.Equal(t, test.expectedPublicIP, publicIP) + } +} + +func TestCreateOrUpdateSecurityGroupCanceled(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + az := GetTestCloud(ctrl) + az.nsgCache.Set("sg", "test") + + mockSGClient := az.SecurityGroupsClient.(*mocksecuritygroupclient.MockInterface) + mockSGClient.EXPECT().CreateOrUpdate(gomock.Any(), az.ResourceGroup, gomock.Any(), gomock.Any(), gomock.Any()).Return(&retry.Error{ + RawError: fmt.Errorf(operationCanceledErrorMessage), + }) + mockSGClient.EXPECT().Get(gomock.Any(), az.ResourceGroup, "sg", gomock.Any()).Return(network.SecurityGroup{}, nil) + + err := az.CreateOrUpdateSecurityGroup(&v1.Service{}, network.SecurityGroup{Name: to.StringPtr("sg")}) + assert.Equal(t, fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 0, RawError: canceledandsupersededduetoanotheroperation"), err) + + // security group should be removed from cache if the operation is canceled + shouldBeEmpty, err := az.nsgCache.Get("sg", cache.CacheReadTypeDefault) + assert.NoError(t, err) + assert.Empty(t, shouldBeEmpty) +} + +func TestCreateOrUpdateLB(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tests := []struct { + clientErr *retry.Error + expectedErr error + }{ + { + clientErr: &retry.Error{HTTPStatusCode: http.StatusPreconditionFailed}, + expectedErr: fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 412, RawError: "), + }, + { + clientErr: &retry.Error{RawError: fmt.Errorf(operationCanceledErrorMessage)}, + expectedErr: fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 0, RawError: canceledandsupersededduetoanotheroperation"), + }, + } + + for _, test := range tests { + az := GetTestCloud(ctrl) + az.lbCache.Set("lb", "test") + + mockLBClient := az.LoadBalancerClient.(*mockloadbalancerclient.MockInterface) + mockLBClient.EXPECT().CreateOrUpdate(gomock.Any(), az.ResourceGroup, gomock.Any(), gomock.Any(), gomock.Any()).Return(test.clientErr) + mockLBClient.EXPECT().Get(gomock.Any(), az.ResourceGroup, "lb", gomock.Any()).Return(network.LoadBalancer{}, nil) + + err := az.CreateOrUpdateLB(&v1.Service{}, network.LoadBalancer{ + Name: to.StringPtr("lb"), + Etag: to.StringPtr("etag"), + }) + assert.Equal(t, test.expectedErr, err) + + // loadbalancer should be removed from cache if the etag is mismatch or the operation is canceled + shouldBeEmpty, err := az.lbCache.Get("lb", cache.CacheReadTypeDefault) + assert.NoError(t, err) + assert.Empty(t, shouldBeEmpty) + } +} + +func TestListLB(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + az := GetTestCloud(ctrl) + mockLBClient := az.LoadBalancerClient.(*mockloadbalancerclient.MockInterface) + mockLBClient.EXPECT().List(gomock.Any(), az.ResourceGroup).Return(nil, &retry.Error{HTTPStatusCode: http.StatusInternalServerError}) + + pips, err := az.ListLB(&v1.Service{}) + assert.Equal(t, fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 500, RawError: "), err) + assert.Empty(t, pips) +} + +func TestListPIP(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + az := GetTestCloud(ctrl) + mockPIPClient := az.PublicIPAddressesClient.(*mockpublicipclient.MockInterface) + mockPIPClient.EXPECT().List(gomock.Any(), az.ResourceGroup).Return(nil, &retry.Error{HTTPStatusCode: http.StatusInternalServerError}) + + pips, err := az.ListPIP(&v1.Service{}, az.ResourceGroup) + assert.Equal(t, fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 500, RawError: "), err) + assert.Empty(t, pips) +} + +func TestCreateOrUpdatePIP(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + az := GetTestCloud(ctrl) + mockPIPClient := az.PublicIPAddressesClient.(*mockpublicipclient.MockInterface) + mockPIPClient.EXPECT().CreateOrUpdate(gomock.Any(), az.ResourceGroup, "nic", gomock.Any()).Return(&retry.Error{HTTPStatusCode: http.StatusInternalServerError}) + + err := az.CreateOrUpdatePIP(&v1.Service{}, az.ResourceGroup, network.PublicIPAddress{Name: to.StringPtr("nic")}) + assert.Equal(t, fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 500, RawError: "), err) +} + +func TestCreateOrUpdateInterface(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + az := GetTestCloud(ctrl) + mockInterfaceClient := az.InterfacesClient.(*mockinterfaceclient.MockInterface) + mockInterfaceClient.EXPECT().CreateOrUpdate(gomock.Any(), az.ResourceGroup, "nic", gomock.Any()).Return(&retry.Error{HTTPStatusCode: http.StatusInternalServerError}) + + err := az.CreateOrUpdateInterface(&v1.Service{}, network.Interface{Name: to.StringPtr("nic")}) + assert.Equal(t, fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 500, RawError: "), err) +} + +func TestDeletePublicIP(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + az := GetTestCloud(ctrl) + mockPIPClient := az.PublicIPAddressesClient.(*mockpublicipclient.MockInterface) + mockPIPClient.EXPECT().Delete(gomock.Any(), az.ResourceGroup, "pip").Return(&retry.Error{HTTPStatusCode: http.StatusInternalServerError}) + + err := az.DeletePublicIP(&v1.Service{}, az.ResourceGroup, "pip") + assert.Equal(t, fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 500, RawError: "), err) +} + +func TestDeleteLB(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + az := GetTestCloud(ctrl) + mockLBClient := az.LoadBalancerClient.(*mockloadbalancerclient.MockInterface) + mockLBClient.EXPECT().Delete(gomock.Any(), az.ResourceGroup, "lb").Return(&retry.Error{HTTPStatusCode: http.StatusInternalServerError}) + + err := az.DeleteLB(&v1.Service{}, "lb") + assert.Equal(t, fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 500, RawError: "), err) +} + +func TestCreateOrUpdateRouteTable(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tests := []struct { + clientErr *retry.Error + expectedErr error + }{ + { + clientErr: &retry.Error{HTTPStatusCode: http.StatusPreconditionFailed}, + expectedErr: fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 412, RawError: "), + }, + { + clientErr: &retry.Error{RawError: fmt.Errorf(operationCanceledErrorMessage)}, + expectedErr: fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 0, RawError: canceledandsupersededduetoanotheroperation"), + }, + } + + for _, test := range tests { + az := GetTestCloud(ctrl) + az.rtCache.Set("rt", "test") + + mockRTClient := az.RouteTablesClient.(*mockroutetableclient.MockInterface) + mockRTClient.EXPECT().CreateOrUpdate(gomock.Any(), az.ResourceGroup, gomock.Any(), gomock.Any(), gomock.Any()).Return(test.clientErr) + mockRTClient.EXPECT().Get(gomock.Any(), az.ResourceGroup, "rt", gomock.Any()).Return(network.RouteTable{}, nil) + + err := az.CreateOrUpdateRouteTable(network.RouteTable{ + Name: to.StringPtr("rt"), + Etag: to.StringPtr("etag"), + }) + assert.Equal(t, test.expectedErr, err) + + // route table should be removed from cache if the etag is mismatch or the operation is canceled + shouldBeEmpty, err := az.rtCache.Get("rt", cache.CacheReadTypeDefault) + assert.NoError(t, err) + assert.Empty(t, shouldBeEmpty) + } +} + +func TestCreateOrUpdateRoute(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tests := []struct { + clientErr *retry.Error + expectedErr error + }{ + { + clientErr: &retry.Error{HTTPStatusCode: http.StatusPreconditionFailed}, + expectedErr: fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 412, RawError: "), + }, + { + clientErr: &retry.Error{RawError: fmt.Errorf(operationCanceledErrorMessage)}, + expectedErr: fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 0, RawError: canceledandsupersededduetoanotheroperation"), + }, + { + clientErr: nil, + expectedErr: nil, + }, + } + + for _, test := range tests { + az := GetTestCloud(ctrl) + az.rtCache.Set("rt", "test") + + mockRTClient := az.RoutesClient.(*mockrouteclient.MockInterface) + mockRTClient.EXPECT().CreateOrUpdate(gomock.Any(), az.ResourceGroup, "rt", gomock.Any(), gomock.Any(), gomock.Any()).Return(test.clientErr) + + mockRTableClient := az.RouteTablesClient.(*mockroutetableclient.MockInterface) + mockRTableClient.EXPECT().Get(gomock.Any(), az.ResourceGroup, "rt", gomock.Any()).Return(network.RouteTable{}, nil) + + err := az.CreateOrUpdateRoute(network.Route{ + Name: to.StringPtr("rt"), + Etag: to.StringPtr("etag"), + }) + assert.Equal(t, test.expectedErr, err) + + shouldBeEmpty, err := az.rtCache.Get("rt", cache.CacheReadTypeDefault) + assert.NoError(t, err) + assert.Empty(t, shouldBeEmpty) + } +} + +func TestDeleteRouteWithName(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tests := []struct { + clientErr *retry.Error + expectedErr error + }{ + { + clientErr: &retry.Error{HTTPStatusCode: http.StatusInternalServerError}, + expectedErr: fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 500, RawError: "), + }, + { + clientErr: nil, + expectedErr: nil, + }, + } + + for _, test := range tests { + az := GetTestCloud(ctrl) + + mockRTClient := az.RoutesClient.(*mockrouteclient.MockInterface) + mockRTClient.EXPECT().Delete(gomock.Any(), az.ResourceGroup, "rt", "rt").Return(test.clientErr) + + err := az.DeleteRouteWithName("rt") + assert.Equal(t, test.expectedErr, err) + } +} + +func TestCreateOrUpdateVMSS(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tests := []struct { + vmss compute.VirtualMachineScaleSet + clientErr *retry.Error + expectedErr *retry.Error + }{ + { + clientErr: &retry.Error{HTTPStatusCode: http.StatusInternalServerError}, + expectedErr: &retry.Error{HTTPStatusCode: http.StatusInternalServerError}, + }, + { + clientErr: &retry.Error{HTTPStatusCode: http.StatusTooManyRequests}, + expectedErr: &retry.Error{HTTPStatusCode: http.StatusTooManyRequests}, + }, + { + clientErr: &retry.Error{RawError: fmt.Errorf("azure cloud provider rate limited(write) for operation CreateOrUpdate")}, + expectedErr: &retry.Error{RawError: fmt.Errorf("azure cloud provider rate limited(write) for operation CreateOrUpdate")}, + }, + { + vmss: compute.VirtualMachineScaleSet{ + VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ + ProvisioningState: &virtualMachineScaleSetsDeallocating, + }, + }, + }, + } + + for _, test := range tests { + az := GetTestCloud(ctrl) + + mockVMSSClient := az.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) + mockVMSSClient.EXPECT().Get(gomock.Any(), az.ResourceGroup, testVMSSName).Return(test.vmss, test.clientErr) + + err := az.CreateOrUpdateVMSS(az.ResourceGroup, testVMSSName, compute.VirtualMachineScaleSet{}) + assert.Equal(t, test.expectedErr, err) + } +} + +func TestRequestBackoff(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + az := GetTestCloud(ctrl) + az.CloudProviderBackoff = true + az.ResourceRequestBackoff = wait.Backoff{Steps: 3} + + backoff := az.RequestBackoff() + assert.Equal(t, wait.Backoff{Steps: 3}, backoff) + +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_test.go index 3b9eeff627c..4d9f21dd091 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_test.go @@ -26,8 +26,6 @@ import ( "strings" "testing" - cloudprovider "k8s.io/cloud-provider" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" "github.com/Azure/go-autorest/autorest/to" @@ -38,6 +36,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + cloudprovider "k8s.io/cloud-provider" servicehelpers "k8s.io/cloud-provider/service/helpers" "k8s.io/legacy-cloud-providers/azure/clients/interfaceclient/mockinterfaceclient" "k8s.io/legacy-cloud-providers/azure/clients/loadbalancerclient/mockloadbalancerclient" diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go index b7509521a23..8769c32b0bd 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go @@ -26,6 +26,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + utilerrors "k8s.io/apimachinery/pkg/util/errors" cloudprovider "k8s.io/cloud-provider" azcache "k8s.io/legacy-cloud-providers/azure/cache" "k8s.io/legacy-cloud-providers/azure/clients/interfaceclient/mockinterfaceclient" @@ -43,11 +44,13 @@ import ( ) const ( - fakePrivateIP = "10.240.0.10" - fakePublicIP = "10.10.10.10" - testVMSSName = "vmss" - testVMPowerState = "PowerState/Running" - testLBBackendpoolID = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-0" + fakePrivateIP = "10.240.0.10" + fakePublicIP = "10.10.10.10" + testVMSSName = "vmss" + testVMPowerState = "PowerState/Running" + testLBBackendpoolID0 = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-0" + testLBBackendpoolID1 = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-1" + testLBBackendpoolID2 = "/subscriptions/sub/resourceGroups/rg1/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-2" ) func newTestScaleSet(ctrl *gomock.Controller) (*scaleSet, error) { @@ -64,6 +67,40 @@ func newTestScaleSetWithState(ctrl *gomock.Controller) (*scaleSet, error) { return ss.(*scaleSet), nil } +func buildTestVMSS(lbBackendpoolIDs []string) compute.VirtualMachineScaleSet { + lbBackendpools := make([]compute.SubResource, 0) + for _, id := range lbBackendpoolIDs { + lbBackendpools = append(lbBackendpools, compute.SubResource{ID: to.StringPtr(id)}) + } + + expectedVMSS := compute.VirtualMachineScaleSet{ + Name: to.StringPtr(testVMSSName), + VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ + ProvisioningState: to.StringPtr("Running"), + VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{ + NetworkProfile: &compute.VirtualMachineScaleSetNetworkProfile{ + NetworkInterfaceConfigurations: &[]compute.VirtualMachineScaleSetNetworkConfiguration{ + { + VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ + Primary: to.BoolPtr(true), + IPConfigurations: &[]compute.VirtualMachineScaleSetIPConfiguration{ + { + VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ + LoadBalancerBackendAddressPools: &lbBackendpools, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + return expectedVMSS +} + func buildTestVirtualMachineEnv(ss *Cloud, scaleSetName, zone string, faultDomain int32, vmList []string, state string) ([]compute.VirtualMachineScaleSetVM, network.Interface, network.PublicIPAddress) { expectedVMSSVMs := make([]compute.VirtualMachineScaleSetVM, 0) expectedInterface := network.Interface{} @@ -91,7 +128,7 @@ func buildTestVirtualMachineEnv(ss *Cloud, scaleSetName, zone string, faultDomai Name: to.StringPtr("ipconfig1"), VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ Primary: to.BoolPtr(true), - LoadBalancerBackendAddressPools: &[]compute.SubResource{{ID: to.StringPtr(testLBBackendpoolID)}}, + LoadBalancerBackendAddressPools: &[]compute.SubResource{{ID: to.StringPtr(testLBBackendpoolID0)}}, }, }, } @@ -1496,7 +1533,7 @@ func TestEnsureHostInPool(t *testing.T) { service: &v1.Service{Spec: v1.ServiceSpec{ClusterIP: "clusterIP"}}, nodeName: "vmss-vm-000000", vmSetName: "vmss", - backendPoolID: testLBBackendpoolID, + backendPoolID: testLBBackendpoolID0, }, { description: "EnsureHostInPool should skip the current node if it has already been added to another LB", @@ -1533,7 +1570,7 @@ func TestEnsureHostInPool(t *testing.T) { Primary: to.BoolPtr(true), LoadBalancerBackendAddressPools: &[]compute.SubResource{ { - ID: to.StringPtr(testLBBackendpoolID), + ID: to.StringPtr(testLBBackendpoolID0), }, { ID: to.StringPtr("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb-internal/backendAddressPools/backendpool-1"), @@ -1656,7 +1693,7 @@ func TestEnsureVMSSInPool(t *testing.T) { }, }, isBasicLB: false, - backendPoolID: testLBBackendpoolID, + backendPoolID: testLBBackendpoolID0, expectedPutVMSS: false, }, { @@ -1682,7 +1719,7 @@ func TestEnsureVMSSInPool(t *testing.T) { }, }, isBasicLB: false, - backendPoolID: "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-1", + backendPoolID: testLBBackendpoolID1, expectedPutVMSS: true, }, } @@ -1695,34 +1732,7 @@ func TestEnsureVMSSInPool(t *testing.T) { ss.LoadBalancerSku = loadBalancerSkuStandard } - expectedVMSS := compute.VirtualMachineScaleSet{ - Name: to.StringPtr(testVMSSName), - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - ProvisioningState: to.StringPtr("Running"), - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{ - NetworkProfile: &compute.VirtualMachineScaleSetNetworkProfile{ - NetworkInterfaceConfigurations: &[]compute.VirtualMachineScaleSetNetworkConfiguration{ - { - VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ - Primary: to.BoolPtr(true), - IPConfigurations: &[]compute.VirtualMachineScaleSetIPConfiguration{ - { - VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ - LoadBalancerBackendAddressPools: &[]compute.SubResource{ - { - ID: to.StringPtr(testLBBackendpoolID), - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } + expectedVMSS := buildTestVMSS([]string{testLBBackendpoolID0}) if test.isVMSSDeallocating { expectedVMSS.ProvisioningState = &virtualMachineScaleSetsDeallocating } @@ -1789,7 +1799,7 @@ func TestEnsureHostsInPool(t *testing.T) { }, }, }, - backendpoolID: "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-1", + backendpoolID: testLBBackendpoolID1, vmSetName: testVMSSName, expectedVMSSVMPutTimes: 1, }, @@ -1805,7 +1815,7 @@ func TestEnsureHostsInPool(t *testing.T) { }, }, }, - backendpoolID: "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-1", + backendpoolID: testLBBackendpoolID1, vmSetName: testVMSSName, expectedVMSSVMPutTimes: 0, expectedErr: true, @@ -1819,34 +1829,7 @@ func TestEnsureHostsInPool(t *testing.T) { ss.LoadBalancerSku = loadBalancerSkuStandard ss.ExcludeMasterFromStandardLB = to.BoolPtr(true) - expectedVMSS := compute.VirtualMachineScaleSet{ - Name: to.StringPtr(testVMSSName), - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - ProvisioningState: to.StringPtr("Running"), - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{ - NetworkProfile: &compute.VirtualMachineScaleSetNetworkProfile{ - NetworkInterfaceConfigurations: &[]compute.VirtualMachineScaleSetNetworkConfiguration{ - { - VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ - Primary: to.BoolPtr(true), - IPConfigurations: &[]compute.VirtualMachineScaleSetIPConfiguration{ - { - VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ - LoadBalancerBackendAddressPools: &[]compute.SubResource{ - { - ID: to.StringPtr(testLBBackendpoolID), - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } + expectedVMSS := buildTestVMSS([]string{testLBBackendpoolID0}) mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() mockVMSSClient.EXPECT().Get(gomock.Any(), ss.ResourceGroup, testVMSSName).Return(expectedVMSS, nil).MaxTimes(1) @@ -1893,12 +1876,12 @@ func TestEnsureBackendPoolDeletedFromNode(t *testing.T) { { description: "ensureBackendPoolDeletedFromNode should skip the node if there's no wanted lb backendpool ID on that VM", nodeName: "vmss-vm-000000", - backendpoolID: "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-1", + backendpoolID: testLBBackendpoolID1, }, { description: "ensureBackendPoolDeletedFromNode should delete the given backendpool ID", nodeName: "vmss-vm-000000", - backendpoolID: testLBBackendpoolID, + backendpoolID: testLBBackendpoolID0, expectedNodeResourceGroup: "rg", expectedVMSSName: testVMSSName, expectedInstanceID: "0", @@ -1999,18 +1982,18 @@ func TestEnsureBackendPoolDeletedFromVMSS(t *testing.T) { { description: "ensureBackendPoolDeletedFromVMSS should delete the corresponding LB backendpool ID", ipConfigurationIDs: []string{"/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/vmss-vm-000000/networkInterfaces/nic"}, - backendPoolID: testLBBackendpoolID, + backendPoolID: testLBBackendpoolID0, expectedPutVMSS: true, }, { description: "ensureBackendPoolDeletedFromVMSS should skip the VMSS if there's no wanted LB backendpool ID", ipConfigurationIDs: []string{"/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/vmss-vm-000000/networkInterfaces/nic"}, - backendPoolID: "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-1", + backendPoolID: testLBBackendpoolID1, }, { description: "ensureBackendPoolDeletedFromVMSS should report the error that occurs during VMSS client's call", ipConfigurationIDs: []string{"/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/vmss-vm-000000/networkInterfaces/nic"}, - backendPoolID: testLBBackendpoolID, + backendPoolID: testLBBackendpoolID0, expectedPutVMSS: true, vmssClientErr: &retry.Error{RawError: fmt.Errorf("error")}, expectedErr: fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 0, RawError: error"), @@ -2023,34 +2006,7 @@ func TestEnsureBackendPoolDeletedFromVMSS(t *testing.T) { ss.LoadBalancerSku = loadBalancerSkuStandard - expectedVMSS := compute.VirtualMachineScaleSet{ - Name: to.StringPtr(testVMSSName), - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - ProvisioningState: to.StringPtr("Running"), - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{ - NetworkProfile: &compute.VirtualMachineScaleSetNetworkProfile{ - NetworkInterfaceConfigurations: &[]compute.VirtualMachineScaleSetNetworkConfiguration{ - { - VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ - Primary: to.BoolPtr(true), - IPConfigurations: &[]compute.VirtualMachineScaleSetIPConfiguration{ - { - VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ - LoadBalancerBackendAddressPools: &[]compute.SubResource{ - { - ID: to.StringPtr(testLBBackendpoolID), - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } + expectedVMSS := buildTestVMSS([]string{testLBBackendpoolID0}) if test.isVMSSDeallocating { expectedVMSS.ProvisioningState = &virtualMachineScaleSetsDeallocating } @@ -2089,10 +2045,10 @@ func TestEnsureBackendPoolDeleted(t *testing.T) { }{ { description: "EnsureBackendPoolDeleted should skip the unwanted backend address pools and update the VMSS VM correctly", - backendpoolID: testLBBackendpoolID, + backendpoolID: testLBBackendpoolID0, backendAddressPools: &[]network.BackendAddressPool{ { - ID: to.StringPtr(testLBBackendpoolID), + ID: to.StringPtr(testLBBackendpoolID0), BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{ BackendIPConfigurations: &[]network.InterfaceIPConfiguration{ { @@ -2110,17 +2066,17 @@ func TestEnsureBackendPoolDeleted(t *testing.T) { }, }, { - ID: to.StringPtr("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-1"), + ID: to.StringPtr(testLBBackendpoolID1), }, }, expectedVMSSVMPutTimes: 1, }, { description: "EnsureBackendPoolDeleted should report the error that occurs during the call of VMSS VM client", - backendpoolID: testLBBackendpoolID, + backendpoolID: testLBBackendpoolID0, backendAddressPools: &[]network.BackendAddressPool{ { - ID: to.StringPtr(testLBBackendpoolID), + ID: to.StringPtr(testLBBackendpoolID0), BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{ BackendIPConfigurations: &[]network.InterfaceIPConfiguration{ { @@ -2131,7 +2087,7 @@ func TestEnsureBackendPoolDeleted(t *testing.T) { }, }, { - ID: to.StringPtr("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-1"), + ID: to.StringPtr(testLBBackendpoolID1), }, }, expectedVMSSVMPutTimes: 1, @@ -2144,34 +2100,7 @@ func TestEnsureBackendPoolDeleted(t *testing.T) { ss, err := newTestScaleSet(ctrl) assert.NoError(t, err, test.description) - expectedVMSS := compute.VirtualMachineScaleSet{ - Name: to.StringPtr(testVMSSName), - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - ProvisioningState: to.StringPtr("Running"), - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{ - NetworkProfile: &compute.VirtualMachineScaleSetNetworkProfile{ - NetworkInterfaceConfigurations: &[]compute.VirtualMachineScaleSetNetworkConfiguration{ - { - VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ - Primary: to.BoolPtr(true), - IPConfigurations: &[]compute.VirtualMachineScaleSetIPConfiguration{ - { - VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ - LoadBalancerBackendAddressPools: &[]compute.SubResource{ - { - ID: to.StringPtr(testLBBackendpoolID), - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } + expectedVMSS := buildTestVMSS([]string{testLBBackendpoolID0}) mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() mockVMSSClient.EXPECT().Get(gomock.Any(), ss.ResourceGroup, testVMSSName).Return(expectedVMSS, nil).MaxTimes(1) @@ -2186,3 +2115,76 @@ func TestEnsureBackendPoolDeleted(t *testing.T) { assert.Equal(t, test.expectedErr, err != nil, test.description+", but an error occurs") } } + +func TestEnsureBackendPoolDeletedConcurrently(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + ss, err := newTestScaleSet(ctrl) + assert.NoError(t, err) + + backendAddressPools := &[]network.BackendAddressPool{ + { + ID: to.StringPtr(testLBBackendpoolID0), + BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{ + BackendIPConfigurations: &[]network.InterfaceIPConfiguration{ + { + Name: to.StringPtr("ip-1"), + ID: to.StringPtr("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0/networkInterfaces/nic"), + }, + }, + }, + }, + { + ID: to.StringPtr(testLBBackendpoolID1), + BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{ + BackendIPConfigurations: &[]network.InterfaceIPConfiguration{ + { + Name: to.StringPtr("ip-1"), + ID: to.StringPtr("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0/networkInterfaces/nic"), + }, + }, + }, + }, + { + // this would fail + ID: to.StringPtr(testLBBackendpoolID2), + BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{ + BackendIPConfigurations: &[]network.InterfaceIPConfiguration{ + { + Name: to.StringPtr("ip-1"), + ID: to.StringPtr("/subscriptions/sub/resourceGroups/rg1/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0/networkInterfaces/nic"), + }, + }, + }, + }, + } + expectedVMSS := buildTestVMSS([]string{testLBBackendpoolID0, testLBBackendpoolID1}) + + expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, []string{"vmss-vm-000000"}, "succeeded") + vmssVMNetworkConfigs := expectedVMSSVMs[0].NetworkProfileConfiguration + vmssVMIPConfigs := (*vmssVMNetworkConfigs.NetworkInterfaceConfigurations)[0].VirtualMachineScaleSetNetworkConfigurationProperties.IPConfigurations + lbBackendpools := (*vmssVMIPConfigs)[0].LoadBalancerBackendAddressPools + *lbBackendpools = append(*lbBackendpools, compute.SubResource{ID: to.StringPtr(testLBBackendpoolID1)}) + + mockVMSSClient := ss.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface) + mockVMSSClient.EXPECT().List(gomock.Any(), ss.ResourceGroup).Return([]compute.VirtualMachineScaleSet{expectedVMSS}, nil).AnyTimes() + mockVMSSClient.EXPECT().Get(gomock.Any(), ss.ResourceGroup, testVMSSName).Return(expectedVMSS, nil).MaxTimes(2) + mockVMSSClient.EXPECT().CreateOrUpdate(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(nil).Times(2) + + mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface) + mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes() + mockVMSSVMClient.EXPECT().UpdateVMs(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any(), gomock.Any()).Return(nil).Times(2) + + backendpoolAddressIDs := []string{testLBBackendpoolID0, testLBBackendpoolID1, testLBBackendpoolID2} + testFunc := make([]func() error, 0) + for _, id := range backendpoolAddressIDs { + id := id + testFunc = append(testFunc, func() error { + return ss.EnsureBackendPoolDeleted(&v1.Service{}, id, testVMSSName, backendAddressPools) + }) + } + errs := utilerrors.AggregateGoroutines(testFunc...) + assert.Equal(t, 1, len(errs.Errors())) + assert.Equal(t, "instance not found", errs.Error()) +}