diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/BUILD b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/BUILD index 0665336d5e8..72c070260c0 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/BUILD +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/BUILD @@ -29,9 +29,11 @@ go_test( srcs = ["azure_vmclient_test.go"], embed = [":go_default_library"], deps = [ + "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure/clients:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient:go_default_library", + "//staging/src/k8s.io/legacy-cloud-providers/azure/retry:go_default_library", "//vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute:go_default_library", "//vendor/github.com/Azure/go-autorest/autorest:go_default_library", "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library", diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/azure_vmclient_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/azure_vmclient_test.go index 4ad0c722b0a..8fb4c0831fe 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/azure_vmclient_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmclient/azure_vmclient_test.go @@ -26,6 +26,7 @@ import ( "io/ioutil" "net/http" "testing" + "time" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" "github.com/Azure/go-autorest/autorest" @@ -33,11 +34,90 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + "k8s.io/client-go/util/flowcontrol" azclients "k8s.io/legacy-cloud-providers/azure/clients" "k8s.io/legacy-cloud-providers/azure/clients/armclient" "k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient" + "k8s.io/legacy-cloud-providers/azure/retry" ) +func TestNew(t *testing.T) { + config := &azclients.ClientConfig{ + SubscriptionID: "sub", + ResourceManagerEndpoint: "endpoint", + Location: "eastus", + RateLimitConfig: &azclients.RateLimitConfig{ + CloudProviderRateLimit: true, + CloudProviderRateLimitQPS: 0.5, + CloudProviderRateLimitBucket: 1, + CloudProviderRateLimitQPSWrite: 0.5, + CloudProviderRateLimitBucketWrite: 1, + }, + Backoff: &retry.Backoff{Steps: 1}, + } + + vmClient := New(config) + assert.Equal(t, "sub", vmClient.subscriptionID) + assert.NotEmpty(t, vmClient.rateLimiterReader) + assert.NotEmpty(t, vmClient.rateLimiterWriter) +} + +func TestGet(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm1" + response := &http.Response{ + StatusCode: http.StatusOK, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "InstanceView").Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + expected := compute.VirtualMachine{Response: autorest.Response{Response: response}} + vmClient := getTestVMClient(armClient) + result, rerr := vmClient.Get(context.TODO(), "rg", "vm1", "InstanceView") + assert.Equal(t, expected, result) + assert.Nil(t, rerr) +} + +func TestGetNeverRateLimiter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmGetErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "read", "VMGet"), + Retriable: true, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + vmClient := getTestVMClientWithNeverRateLimiter(armClient) + expected := compute.VirtualMachine{} + result, rerr := vmClient.Get(context.TODO(), "rg", "vm1", "InstanceView") + assert.Equal(t, expected, result) + assert.Equal(t, vmGetErr, rerr) +} + +func TestGetRetryAfterReader(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmGetErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "VMGet", "client throttled"), + Retriable: true, + RetryAfter: getFutureTime(), + } + + armClient := mockarmclient.NewMockInterface(ctrl) + vmClient := getTestVMClientWithRetryAfterReader(armClient) + expected := compute.VirtualMachine{} + result, rerr := vmClient.Get(context.TODO(), "rg", "vm1", "InstanceView") + assert.Equal(t, expected, result) + assert.Equal(t, vmGetErr, rerr) +} + func TestGetNotFound(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -80,6 +160,31 @@ func TestGetInternalError(t *testing.T) { assert.Equal(t, http.StatusInternalServerError, rerr.HTTPStatusCode) } +func TestGetThrottle(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm1" + response := &http.Response{ + StatusCode: http.StatusTooManyRequests, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + throttleErr := &retry.Error{ + HTTPStatusCode: http.StatusTooManyRequests, + RawError: fmt.Errorf("error"), + Retriable: true, + RetryAfter: time.Unix(100, 0), + } + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "InstanceView").Return(response, throttleErr).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + vmClient := getTestVMClient(armClient) + result, rerr := vmClient.Get(context.TODO(), "rg", "vm1", "InstanceView") + assert.Empty(t, result) + assert.Equal(t, throttleErr, rerr) +} + func TestList(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -102,6 +207,245 @@ func TestList(t *testing.T) { assert.Equal(t, 3, len(result)) } +func TestListNotFound(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines" + response := &http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + vmClient := getTestVMClient(armClient) + expected := []compute.VirtualMachine{} + result, rerr := vmClient.List(context.TODO(), "rg") + assert.Equal(t, expected, result) + assert.NotNil(t, rerr) + assert.Equal(t, http.StatusNotFound, rerr.HTTPStatusCode) +} + +func TestListInternalError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines" + response := &http.Response{ + StatusCode: http.StatusInternalServerError, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + vmClient := getTestVMClient(armClient) + expected := []compute.VirtualMachine{} + result, rerr := vmClient.List(context.TODO(), "rg") + assert.Equal(t, expected, result) + assert.NotNil(t, rerr) + assert.Equal(t, http.StatusInternalServerError, rerr.HTTPStatusCode) +} + +func TestListThrottle(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines" + armClient := mockarmclient.NewMockInterface(ctrl) + response := &http.Response{ + StatusCode: http.StatusTooManyRequests, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + throttleErr := &retry.Error{ + HTTPStatusCode: http.StatusTooManyRequests, + RawError: fmt.Errorf("error"), + Retriable: true, + RetryAfter: time.Unix(100, 0), + } + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, throttleErr).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + vmClient := getTestVMClient(armClient) + result, rerr := vmClient.List(context.TODO(), "rg") + assert.Empty(t, result) + assert.NotNil(t, rerr) + assert.Equal(t, throttleErr, rerr) +} + +func TestListWithListResponderError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines" + armClient := mockarmclient.NewMockInterface(ctrl) + vmList := []compute.VirtualMachine{getTestVM("vm1"), getTestVM("vm2"), getTestVM("vm3")} + responseBody, err := json.Marshal(compute.VirtualMachineListResult{Value: &vmList}) + assert.Nil(t, err) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return( + &http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewReader(responseBody)), + }, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + vmClient := getTestVMClient(armClient) + result, rerr := vmClient.List(context.TODO(), "rg") + assert.NotNil(t, rerr) + assert.Equal(t, 0, len(result)) +} + +func TestListNeverRateLimiter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmListErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "read", "VMList"), + Retriable: true, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + vmClient := getTestVMClientWithNeverRateLimiter(armClient) + result, rerr := vmClient.List(context.TODO(), "rg") + assert.Equal(t, 0, len(result)) + assert.NotNil(t, rerr) + assert.Equal(t, vmListErr, rerr) +} + +func TestListRetryAfterReader(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmListErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "VMList", "client throttled"), + Retriable: true, + RetryAfter: getFutureTime(), + } + + armClient := mockarmclient.NewMockInterface(ctrl) + vmClient := getTestVMClientWithRetryAfterReader(armClient) + result, rerr := vmClient.List(context.TODO(), "rg") + assert.Equal(t, 0, len(result)) + assert.NotNil(t, rerr) + assert.Equal(t, vmListErr, rerr) +} + +func TestListNextResultsMultiPages(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tests := []struct { + name string + prepareErr error + sendErr *retry.Error + expectedErrMsg string + }{ + { + name: "testlistNextResultsSuccessful", + prepareErr: nil, + sendErr: nil, + }, + { + name: "testPrepareGetRequestError", + prepareErr: fmt.Errorf("error"), + expectedErrMsg: "Failure preparing next results request", + }, + { + name: "testSendError", + sendErr: &retry.Error{RawError: fmt.Errorf("error")}, + expectedErrMsg: "Failure sending next results request", + }, + } + + lastResult := compute.VirtualMachineListResult{ + NextLink: to.StringPtr("next"), + } + + for _, test := range tests { + armClient := mockarmclient.NewMockInterface(ctrl) + req := &http.Request{ + Method: "GET", + } + armClient.EXPECT().PrepareGetRequest(gomock.Any(), gomock.Any()).Return(req, test.prepareErr) + if test.prepareErr == nil { + armClient.EXPECT().Send(gomock.Any(), req).Return(&http.Response{ + StatusCode: http.StatusOK, + Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"foo":"bar"}`))), + }, test.sendErr) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()) + } + + vmssClient := getTestVMClient(armClient) + result, err := vmssClient.listNextResults(context.TODO(), lastResult) + if err != nil { + assert.Equal(t, err.(autorest.DetailedError).Message, test.expectedErrMsg) + } else { + assert.Nil(t, err) + } + + if test.prepareErr != nil { + assert.Empty(t, result) + } else { + assert.NotEmpty(t, result) + } + } +} + +func TestListNextResultsMultiPagesWithListResponderError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tests := []struct { + name string + prepareErr error + sendErr *retry.Error + }{ + { + name: "testListResponderError", + prepareErr: nil, + sendErr: nil, + }, + { + name: "testSendError", + sendErr: &retry.Error{RawError: fmt.Errorf("error")}, + }, + } + + lastResult := compute.VirtualMachineListResult{ + NextLink: to.StringPtr("next"), + } + + for _, test := range tests { + armClient := mockarmclient.NewMockInterface(ctrl) + req := &http.Request{ + Method: "GET", + } + armClient.EXPECT().PrepareGetRequest(gomock.Any(), gomock.Any()).Return(req, test.prepareErr) + if test.prepareErr == nil { + armClient.EXPECT().Send(gomock.Any(), req).Return(&http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"foo":"bar"}`))), + }, test.sendErr) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()) + } + + response := &http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewBuffer([]byte(`{"foo":"bar"}`))), + } + expected := compute.VirtualMachineListResult{} + expected.Response = autorest.Response{Response: response} + vmssClient := getTestVMClient(armClient) + result, err := vmssClient.listNextResults(context.TODO(), lastResult) + assert.NotNil(t, err) + if test.sendErr != nil { + assert.NotEqual(t, expected, result) + } else { + assert.Equal(t, expected, result) + } + } +} + func TestUpdate(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -121,6 +465,87 @@ func TestUpdate(t *testing.T) { assert.Nil(t, rerr) } +func TestUpdateWithUpdateResponderError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm1" + testVM := compute.VirtualMachineUpdate{} + armClient := mockarmclient.NewMockInterface(ctrl) + response := &http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewReader([]byte(""))), + } + armClient.EXPECT().PatchResource(gomock.Any(), resourceID, testVM).Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + vmClient := getTestVMClient(armClient) + rerr := vmClient.Update(context.TODO(), "rg", "vm1", testVM, "test") + assert.NotNil(t, rerr) +} + +func TestUpdateNeverRateLimiter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmUpdateErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "write", "VMUpdate"), + Retriable: true, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + vmClient := getTestVMClientWithNeverRateLimiter(armClient) + testVM := compute.VirtualMachineUpdate{} + rerr := vmClient.Update(context.TODO(), "rg", "vm1", testVM, "test") + assert.NotNil(t, rerr) + assert.Equal(t, vmUpdateErr, rerr) +} + +func TestUpdateRetryAfterReader(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmUpdateErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "VMUpdate", "client throttled"), + Retriable: true, + RetryAfter: getFutureTime(), + } + + testVM := compute.VirtualMachineUpdate{} + armClient := mockarmclient.NewMockInterface(ctrl) + vmClient := getTestVMClientWithRetryAfterReader(armClient) + rerr := vmClient.Update(context.TODO(), "rg", "vm1", testVM, "test") + assert.NotNil(t, rerr) + assert.Equal(t, vmUpdateErr, rerr) +} + +func TestUpdateThrottle(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm1" + response := &http.Response{ + StatusCode: http.StatusTooManyRequests, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + throttleErr := &retry.Error{ + HTTPStatusCode: http.StatusTooManyRequests, + RawError: fmt.Errorf("error"), + Retriable: true, + RetryAfter: time.Unix(100, 0), + } + + testVM := compute.VirtualMachineUpdate{} + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().PatchResource(gomock.Any(), resourceID, testVM).Return(response, throttleErr).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + vmClient := getTestVMClient(armClient) + rerr := vmClient.Update(context.TODO(), "rg", "vm1", testVM, "test") + assert.NotNil(t, rerr) + assert.Equal(t, throttleErr, rerr) +} + func TestCreateOrUpdate(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -139,6 +564,84 @@ func TestCreateOrUpdate(t *testing.T) { assert.Nil(t, rerr) } +func TestCreateOrUpdateWithCreateOrUpdateResponderError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + testVM := getTestVM("vm1") + armClient := mockarmclient.NewMockInterface(ctrl) + response := &http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewReader([]byte(""))), + } + armClient.EXPECT().PutResource(gomock.Any(), to.String(testVM.ID), testVM).Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + vmClient := getTestVMClient(armClient) + rerr := vmClient.CreateOrUpdate(context.TODO(), "rg", "vm1", testVM, "test") + assert.NotNil(t, rerr) +} + +func TestCreateOrUpdateNeverRateLimiter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmCreateOrUpdateErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "write", "VMCreateOrUpdate"), + Retriable: true, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + vmClient := getTestVMClientWithNeverRateLimiter(armClient) + testVM := getTestVM("vm1") + rerr := vmClient.CreateOrUpdate(context.TODO(), "rg", "vm1", testVM, "test") + assert.NotNil(t, rerr) + assert.Equal(t, vmCreateOrUpdateErr, rerr) +} + +func TestCreateOrUpdateRetryAfterReader(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmCreateOrUpdateErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "VMCreateOrUpdate", "client throttled"), + Retriable: true, + RetryAfter: getFutureTime(), + } + + testVM := getTestVM("vm1") + armClient := mockarmclient.NewMockInterface(ctrl) + vmClient := getTestVMClientWithRetryAfterReader(armClient) + rerr := vmClient.CreateOrUpdate(context.TODO(), "rg", "vm1", testVM, "test") + assert.NotNil(t, rerr) + assert.Equal(t, vmCreateOrUpdateErr, rerr) +} + +func TestCreateOrUpdateThrottle(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + response := &http.Response{ + StatusCode: http.StatusTooManyRequests, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + throttleErr := &retry.Error{ + HTTPStatusCode: http.StatusTooManyRequests, + RawError: fmt.Errorf("error"), + Retriable: true, + RetryAfter: time.Unix(100, 0), + } + + testVM := getTestVM("vm1") + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().PutResource(gomock.Any(), to.String(testVM.ID), testVM).Return(response, throttleErr).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + vmClient := getTestVMClient(armClient) + rerr := vmClient.CreateOrUpdate(context.TODO(), "rg", "vm1", testVM, "test") + assert.NotNil(t, rerr) + assert.Equal(t, throttleErr, rerr) +} + func TestDelete(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -152,6 +655,60 @@ func TestDelete(t *testing.T) { assert.Nil(t, rerr) } +func TestDeleteNeverRateLimiter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmDeleteErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "write", "VMDelete"), + Retriable: true, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + vmClient := getTestVMClientWithNeverRateLimiter(armClient) + rerr := vmClient.Delete(context.TODO(), "rg", "vm1") + assert.NotNil(t, rerr) + assert.Equal(t, vmDeleteErr, rerr) +} + +func TestDeleteRetryAfterReader(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmDeleteErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "VMDelete", "client throttled"), + Retriable: true, + RetryAfter: getFutureTime(), + } + + armClient := mockarmclient.NewMockInterface(ctrl) + vmClient := getTestVMClientWithRetryAfterReader(armClient) + rerr := vmClient.Delete(context.TODO(), "rg", "vm1") + assert.NotNil(t, rerr) + assert.Equal(t, vmDeleteErr, rerr) +} + +func TestDeleteThrottle(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + throttleErr := &retry.Error{ + HTTPStatusCode: http.StatusTooManyRequests, + RawError: fmt.Errorf("error"), + Retriable: true, + RetryAfter: time.Unix(100, 0), + } + + testVM := getTestVM("vm1") + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().DeleteResource(gomock.Any(), to.String(testVM.ID), "").Return(throttleErr).Times(1) + + vmClient := getTestVMClient(armClient) + rerr := vmClient.Delete(context.TODO(), "rg", "vm1") + assert.NotNil(t, rerr) + assert.Equal(t, throttleErr, rerr) +} + func getTestVM(vmName string) compute.VirtualMachine { resourceID := fmt.Sprintf("/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/%s", vmName) return compute.VirtualMachine{ @@ -170,3 +727,31 @@ func getTestVMClient(armClient armclient.Interface) *Client { rateLimiterWriter: rateLimiterWriter, } } + +func getTestVMClientWithNeverRateLimiter(armClient armclient.Interface) *Client { + rateLimiterReader := flowcontrol.NewFakeNeverRateLimiter() + rateLimiterWriter := flowcontrol.NewFakeNeverRateLimiter() + return &Client{ + armClient: armClient, + subscriptionID: "subscriptionID", + rateLimiterReader: rateLimiterReader, + rateLimiterWriter: rateLimiterWriter, + } +} + +func getTestVMClientWithRetryAfterReader(armClient armclient.Interface) *Client { + rateLimiterReader := flowcontrol.NewFakeAlwaysRateLimiter() + rateLimiterWriter := flowcontrol.NewFakeAlwaysRateLimiter() + return &Client{ + armClient: armClient, + subscriptionID: "subscriptionID", + rateLimiterReader: rateLimiterReader, + rateLimiterWriter: rateLimiterWriter, + RetryAfterReader: getFutureTime(), + RetryAfterWriter: getFutureTime(), + } +} + +func getFutureTime() time.Time { + return time.Unix(3000000000, 0) +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient/BUILD b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient/BUILD index 9b9fb370bd9..e90d2d3bbec 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient/BUILD +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient/BUILD @@ -29,6 +29,7 @@ go_test( srcs = ["azure_vmssclient_test.go"], embed = [":go_default_library"], deps = [ + "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure/clients:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient:go_default_library", diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient/azure_vmssclient_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient/azure_vmssclient_test.go index ea6a20f8854..cc6c7aeb4b3 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient/azure_vmssclient_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssclient/azure_vmssclient_test.go @@ -26,6 +26,7 @@ import ( "io/ioutil" "net/http" "testing" + "time" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" "github.com/Azure/go-autorest/autorest" @@ -34,12 +35,90 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + "k8s.io/client-go/util/flowcontrol" azclients "k8s.io/legacy-cloud-providers/azure/clients" "k8s.io/legacy-cloud-providers/azure/clients/armclient" "k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient" "k8s.io/legacy-cloud-providers/azure/retry" ) +func TestNew(t *testing.T) { + config := &azclients.ClientConfig{ + SubscriptionID: "sub", + ResourceManagerEndpoint: "endpoint", + Location: "eastus", + RateLimitConfig: &azclients.RateLimitConfig{ + CloudProviderRateLimit: true, + CloudProviderRateLimitQPS: 0.5, + CloudProviderRateLimitBucket: 1, + CloudProviderRateLimitQPSWrite: 0.5, + CloudProviderRateLimitBucketWrite: 1, + }, + Backoff: &retry.Backoff{Steps: 1}, + } + + vmssClient := New(config) + assert.Equal(t, "sub", vmssClient.subscriptionID) + assert.NotEmpty(t, vmssClient.rateLimiterReader) + assert.NotEmpty(t, vmssClient.rateLimiterWriter) +} + +func TestGet(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss1" + response := &http.Response{ + StatusCode: http.StatusOK, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + expected := compute.VirtualMachineScaleSet{Response: autorest.Response{Response: response}} + vmssClient := getTestVMSSClient(armClient) + result, rerr := vmssClient.Get(context.TODO(), "rg", "vmss1") + assert.Equal(t, expected, result) + assert.Nil(t, rerr) +} + +func TestGetNeverRateLimiter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmssGetErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "read", "VMSSGet"), + Retriable: true, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + vmssClient := getTestVMSSClientWithNeverRateLimiter(armClient) + expected := compute.VirtualMachineScaleSet{} + result, rerr := vmssClient.Get(context.TODO(), "rg", "vmss1") + assert.Equal(t, expected, result) + assert.Equal(t, vmssGetErr, rerr) +} + +func TestGetRetryAfterReader(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmssGetErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "VMSSGet", "client throttled"), + Retriable: true, + RetryAfter: getFutureTime(), + } + + armClient := mockarmclient.NewMockInterface(ctrl) + vmssClient := getTestVMSSClientWithRetryAfterReader(armClient) + expected := compute.VirtualMachineScaleSet{} + result, rerr := vmssClient.Get(context.TODO(), "rg", "vmss1") + assert.Equal(t, expected, result) + assert.Equal(t, vmssGetErr, rerr) +} + func TestGetNotFound(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -82,6 +161,31 @@ func TestGetInternalError(t *testing.T) { assert.Equal(t, http.StatusInternalServerError, rerr.HTTPStatusCode) } +func TestGetThrottle(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss1" + response := &http.Response{ + StatusCode: http.StatusTooManyRequests, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + throttleErr := &retry.Error{ + HTTPStatusCode: http.StatusTooManyRequests, + RawError: fmt.Errorf("error"), + Retriable: true, + RetryAfter: time.Unix(100, 0), + } + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, throttleErr).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + vmssClient := getTestVMSSClient(armClient) + result, rerr := vmssClient.Get(context.TODO(), "rg", "vmss1") + assert.Empty(t, result) + assert.Equal(t, throttleErr, rerr) +} + func TestList(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -104,6 +208,245 @@ func TestList(t *testing.T) { assert.Equal(t, 3, len(result)) } +func TestListNotFound(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets" + response := &http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + vmssClient := getTestVMSSClient(armClient) + expected := []compute.VirtualMachineScaleSet{} + result, rerr := vmssClient.List(context.TODO(), "rg") + assert.Equal(t, expected, result) + assert.NotNil(t, rerr) + assert.Equal(t, http.StatusNotFound, rerr.HTTPStatusCode) +} + +func TestListInternalError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets" + response := &http.Response{ + StatusCode: http.StatusInternalServerError, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + vmssClient := getTestVMSSClient(armClient) + expected := []compute.VirtualMachineScaleSet{} + result, rerr := vmssClient.List(context.TODO(), "rg") + assert.Equal(t, expected, result) + assert.NotNil(t, rerr) + assert.Equal(t, http.StatusInternalServerError, rerr.HTTPStatusCode) +} + +func TestListThrottle(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets" + armClient := mockarmclient.NewMockInterface(ctrl) + response := &http.Response{ + StatusCode: http.StatusTooManyRequests, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + throttleErr := &retry.Error{ + HTTPStatusCode: http.StatusTooManyRequests, + RawError: fmt.Errorf("error"), + Retriable: true, + RetryAfter: time.Unix(100, 0), + } + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, throttleErr).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + vmssClient := getTestVMSSClient(armClient) + result, rerr := vmssClient.List(context.TODO(), "rg") + assert.Empty(t, result) + assert.NotNil(t, rerr) + assert.Equal(t, throttleErr, rerr) +} + +func TestListWithListResponderError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets" + armClient := mockarmclient.NewMockInterface(ctrl) + vmssList := []compute.VirtualMachineScaleSet{getTestVMSS("vmss1"), getTestVMSS("vmss2"), getTestVMSS("vmss3")} + responseBody, err := json.Marshal(compute.VirtualMachineScaleSetListResult{Value: &vmssList}) + assert.Nil(t, err) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return( + &http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewReader(responseBody)), + }, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + vmssClient := getTestVMSSClient(armClient) + result, rerr := vmssClient.List(context.TODO(), "rg") + assert.NotNil(t, rerr) + assert.Equal(t, 0, len(result)) +} + +func TestListNeverRateLimiter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmssListErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "read", "VMSSList"), + Retriable: true, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + vmssClient := getTestVMSSClientWithNeverRateLimiter(armClient) + result, rerr := vmssClient.List(context.TODO(), "rg") + assert.Equal(t, 0, len(result)) + assert.NotNil(t, rerr) + assert.Equal(t, vmssListErr, rerr) +} + +func TestListRetryAfterReader(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmssListErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "VMSSList", "client throttled"), + Retriable: true, + RetryAfter: getFutureTime(), + } + + armClient := mockarmclient.NewMockInterface(ctrl) + vmssClient := getTestVMSSClientWithRetryAfterReader(armClient) + result, rerr := vmssClient.List(context.TODO(), "rg") + assert.Equal(t, 0, len(result)) + assert.NotNil(t, rerr) + assert.Equal(t, vmssListErr, rerr) +} + +func TestListNextResultsMultiPages(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tests := []struct { + name string + prepareErr error + sendErr *retry.Error + expectedErrMsg string + }{ + { + name: "testlistNextResultsSuccessful", + prepareErr: nil, + sendErr: nil, + }, + { + name: "testPrepareGetRequestError", + prepareErr: fmt.Errorf("error"), + expectedErrMsg: "Failure preparing next results request", + }, + { + name: "testSendError", + sendErr: &retry.Error{RawError: fmt.Errorf("error")}, + expectedErrMsg: "Failure sending next results request", + }, + } + + lastResult := compute.VirtualMachineScaleSetListResult{ + NextLink: to.StringPtr("next"), + } + + for _, test := range tests { + armClient := mockarmclient.NewMockInterface(ctrl) + req := &http.Request{ + Method: "GET", + } + armClient.EXPECT().PrepareGetRequest(gomock.Any(), gomock.Any()).Return(req, test.prepareErr) + if test.prepareErr == nil { + armClient.EXPECT().Send(gomock.Any(), req).Return(&http.Response{ + StatusCode: http.StatusOK, + Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"foo":"bar"}`))), + }, test.sendErr) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()) + } + + vmssClient := getTestVMSSClient(armClient) + result, err := vmssClient.listNextResults(context.TODO(), lastResult) + if err != nil { + assert.Equal(t, err.(autorest.DetailedError).Message, test.expectedErrMsg) + } else { + assert.Nil(t, err) + } + + if test.prepareErr != nil { + assert.Empty(t, result) + } else { + assert.NotEmpty(t, result) + } + } +} + +func TestListNextResultsMultiPagesWithListResponderError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tests := []struct { + name string + prepareErr error + sendErr *retry.Error + }{ + { + name: "testListResponderError", + prepareErr: nil, + sendErr: nil, + }, + { + name: "testSendError", + sendErr: &retry.Error{RawError: fmt.Errorf("error")}, + }, + } + + lastResult := compute.VirtualMachineScaleSetListResult{ + NextLink: to.StringPtr("next"), + } + + for _, test := range tests { + armClient := mockarmclient.NewMockInterface(ctrl) + req := &http.Request{ + Method: "GET", + } + armClient.EXPECT().PrepareGetRequest(gomock.Any(), gomock.Any()).Return(req, test.prepareErr) + if test.prepareErr == nil { + armClient.EXPECT().Send(gomock.Any(), req).Return(&http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"foo":"bar"}`))), + }, test.sendErr) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()) + } + + response := &http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewBuffer([]byte(`{"foo":"bar"}`))), + } + expected := compute.VirtualMachineScaleSetListResult{} + expected.Response = autorest.Response{Response: response} + vmssClient := getTestVMSSClient(armClient) + result, err := vmssClient.listNextResults(context.TODO(), lastResult) + assert.NotNil(t, err) + if test.sendErr != nil { + assert.NotEqual(t, expected, result) + } else { + assert.Equal(t, expected, result) + } + } +} + func TestCreateOrUpdate(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -122,6 +465,84 @@ func TestCreateOrUpdate(t *testing.T) { assert.Nil(t, rerr) } +func TestCreateOrUpdateWithCreateOrUpdateResponderError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + vmss := getTestVMSS("vmss1") + armClient := mockarmclient.NewMockInterface(ctrl) + response := &http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewReader([]byte(""))), + } + armClient.EXPECT().PutResource(gomock.Any(), to.String(vmss.ID), vmss).Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + vmssClient := getTestVMSSClient(armClient) + rerr := vmssClient.CreateOrUpdate(context.TODO(), "rg", "vmss1", vmss) + assert.NotNil(t, rerr) +} + +func TestCreateOrUpdateNeverRateLimiter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmssCreateOrUpdateErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "write", "VMSSCreateOrUpdate"), + Retriable: true, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + vmssClient := getTestVMSSClientWithNeverRateLimiter(armClient) + vmss := getTestVMSS("vmss1") + rerr := vmssClient.CreateOrUpdate(context.TODO(), "rg", "vmss1", vmss) + assert.NotNil(t, rerr) + assert.Equal(t, vmssCreateOrUpdateErr, rerr) +} + +func TestCreateOrUpdateRetryAfterReader(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmssCreateOrUpdateErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "VMSSCreateOrUpdate", "client throttled"), + Retriable: true, + RetryAfter: getFutureTime(), + } + + vmss := getTestVMSS("vmss1") + armClient := mockarmclient.NewMockInterface(ctrl) + vmssClient := getTestVMSSClientWithRetryAfterReader(armClient) + rerr := vmssClient.CreateOrUpdate(context.TODO(), "rg", "vmss1", vmss) + assert.NotNil(t, rerr) + assert.Equal(t, vmssCreateOrUpdateErr, rerr) +} + +func TestCreateOrUpdateThrottle(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + response := &http.Response{ + StatusCode: http.StatusTooManyRequests, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + throttleErr := &retry.Error{ + HTTPStatusCode: http.StatusTooManyRequests, + RawError: fmt.Errorf("error"), + Retriable: true, + RetryAfter: time.Unix(100, 0), + } + + vmss := getTestVMSS("vmss1") + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().PutResource(gomock.Any(), to.String(vmss.ID), vmss).Return(response, throttleErr).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + vmssClient := getTestVMSSClient(armClient) + rerr := vmssClient.CreateOrUpdate(context.TODO(), "rg", "vmss1", vmss) + assert.NotNil(t, rerr) + assert.Equal(t, throttleErr, rerr) +} + func TestCreateOrUpdateAsync(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -141,6 +562,63 @@ func TestCreateOrUpdateAsync(t *testing.T) { assert.Equal(t, retryErr, rerr) } +func TestCreateOrUpdateAsyncNeverRateLimiter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmssCreateOrUpdateAsyncErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "write", "VMSSCreateOrUpdateAsync"), + Retriable: true, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + vmssClient := getTestVMSSClientWithNeverRateLimiter(armClient) + vmss := getTestVMSS("vmss1") + _, rerr := vmssClient.CreateOrUpdateAsync(context.TODO(), "rg", "vmss1", vmss) + assert.NotNil(t, rerr) + assert.Equal(t, vmssCreateOrUpdateAsyncErr, rerr) +} + +func TestCreateOrUpdateAsyncRetryAfterReader(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmssCreateOrUpdateAsyncErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "VMSSCreateOrUpdateAsync", "client throttled"), + Retriable: true, + RetryAfter: getFutureTime(), + } + + vmss := getTestVMSS("vmss1") + armClient := mockarmclient.NewMockInterface(ctrl) + vmssClient := getTestVMSSClientWithRetryAfterReader(armClient) + _, rerr := vmssClient.CreateOrUpdateAsync(context.TODO(), "rg", "vmss1", vmss) + assert.NotNil(t, rerr) + assert.Equal(t, vmssCreateOrUpdateAsyncErr, rerr) +} + +func TestCreateOrUpdateAsyncThrottle(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + throttleErr := &retry.Error{ + HTTPStatusCode: http.StatusTooManyRequests, + RawError: fmt.Errorf("error"), + Retriable: true, + RetryAfter: time.Unix(100, 0), + } + + vmss := getTestVMSS("vmss1") + future := &azure.Future{} + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().PutResourceAsync(gomock.Any(), to.String(vmss.ID), vmss).Return(future, throttleErr).Times(1) + + vmssClient := getTestVMSSClient(armClient) + _, rerr := vmssClient.CreateOrUpdateAsync(context.TODO(), "rg", "vmss1", vmss) + assert.NotNil(t, rerr) + assert.Equal(t, throttleErr, rerr) +} + func TestWaitForAsyncOperationResult(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -180,6 +658,104 @@ func TestDeleteInstances(t *testing.T) { assert.Nil(t, rerr) } +func TestDeleteInstancesNeverRateLimiter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmInstanceIDs := compute.VirtualMachineScaleSetVMInstanceRequiredIDs{ + InstanceIds: &[]string{"0", "1", "2"}, + } + vmssDeleteInstancesErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "write", "VMSSDeleteInstances"), + Retriable: true, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + vmssClient := getTestVMSSClientWithNeverRateLimiter(armClient) + rerr := vmssClient.DeleteInstances(context.TODO(), "rg", "vmss1", vmInstanceIDs) + assert.NotNil(t, rerr) + assert.Equal(t, vmssDeleteInstancesErr, rerr) +} + +func TestDeleteInstancesRetryAfterReader(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmInstanceIDs := compute.VirtualMachineScaleSetVMInstanceRequiredIDs{ + InstanceIds: &[]string{"0", "1", "2"}, + } + vmssDeleteInstancesErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "VMSSDeleteInstances", "client throttled"), + Retriable: true, + RetryAfter: getFutureTime(), + } + + armClient := mockarmclient.NewMockInterface(ctrl) + vmssClient := getTestVMSSClientWithRetryAfterReader(armClient) + rerr := vmssClient.DeleteInstances(context.TODO(), "rg", "vmss1", vmInstanceIDs) + assert.NotNil(t, rerr) + assert.Equal(t, vmssDeleteInstancesErr, rerr) +} + +func TestDeleteInstancesThrottle(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmss := getTestVMSS("vmss1") + vmInstanceIDs := compute.VirtualMachineScaleSetVMInstanceRequiredIDs{ + InstanceIds: &[]string{"0", "1", "2"}, + } + response := &http.Response{ + StatusCode: http.StatusTooManyRequests, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + throttleErr := &retry.Error{ + HTTPStatusCode: http.StatusTooManyRequests, + RawError: fmt.Errorf("error"), + Retriable: true, + RetryAfter: time.Unix(100, 0), + } + + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().PostResource(gomock.Any(), to.String(vmss.ID), "delete", vmInstanceIDs).Return(response, throttleErr).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + vmssClient := getTestVMSSClient(armClient) + rerr := vmssClient.DeleteInstances(context.TODO(), "rg", "vmss1", vmInstanceIDs) + assert.NotNil(t, rerr) + assert.Equal(t, throttleErr, rerr) +} + +func TestDeleteInstancesWaitError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmss := getTestVMSS("vmss1") + vmInstanceIDs := compute.VirtualMachineScaleSetVMInstanceRequiredIDs{ + InstanceIds: &[]string{"0", "1", "2"}, + } + response := &http.Response{ + StatusCode: http.StatusOK, + Request: &http.Request{Method: "POST"}, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + err := fmt.Errorf("%s", string("Wait error")) + vmssDeleteInstancesErr := &retry.Error{ + RawError: err, + Retriable: false, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().PostResource(gomock.Any(), to.String(vmss.ID), "delete", vmInstanceIDs).Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + armClient.EXPECT().WaitForAsyncOperationCompletion(gomock.Any(), gomock.Any(), "vmssclient.DeleteInstances").Return(err).Times(1) + + vmssClient := getTestVMSSClient(armClient) + rerr := vmssClient.DeleteInstances(context.TODO(), "rg", "vmss1", vmInstanceIDs) + assert.NotNil(t, rerr) + assert.Equal(t, vmssDeleteInstancesErr, rerr) +} + func getTestVMSS(name string) compute.VirtualMachineScaleSet { return compute.VirtualMachineScaleSet{ ID: to.StringPtr("/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss1"), @@ -201,3 +777,31 @@ func getTestVMSSClient(armClient armclient.Interface) *Client { rateLimiterWriter: rateLimiterWriter, } } + +func getTestVMSSClientWithNeverRateLimiter(armClient armclient.Interface) *Client { + rateLimiterReader := flowcontrol.NewFakeNeverRateLimiter() + rateLimiterWriter := flowcontrol.NewFakeNeverRateLimiter() + return &Client{ + armClient: armClient, + subscriptionID: "subscriptionID", + rateLimiterReader: rateLimiterReader, + rateLimiterWriter: rateLimiterWriter, + } +} + +func getTestVMSSClientWithRetryAfterReader(armClient armclient.Interface) *Client { + rateLimiterReader := flowcontrol.NewFakeAlwaysRateLimiter() + rateLimiterWriter := flowcontrol.NewFakeAlwaysRateLimiter() + return &Client{ + armClient: armClient, + subscriptionID: "subscriptionID", + rateLimiterReader: rateLimiterReader, + rateLimiterWriter: rateLimiterWriter, + RetryAfterReader: getFutureTime(), + RetryAfterWriter: getFutureTime(), + } +} + +func getFutureTime() time.Time { + return time.Unix(3000000000, 0) +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssvmclient/BUILD b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssvmclient/BUILD index b3c9bd146b0..b1b538132d6 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssvmclient/BUILD +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssvmclient/BUILD @@ -30,9 +30,11 @@ go_test( srcs = ["azure_vmssvmclient_test.go"], embed = [":go_default_library"], deps = [ + "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure/clients:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient:go_default_library", + "//staging/src/k8s.io/legacy-cloud-providers/azure/retry:go_default_library", "//vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute:go_default_library", "//vendor/github.com/Azure/go-autorest/autorest:go_default_library", "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library", diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssvmclient/azure_vmssvmclient_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssvmclient/azure_vmssvmclient_test.go index 803f1870511..55411d1c27e 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssvmclient/azure_vmssvmclient_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmssvmclient/azure_vmssvmclient_test.go @@ -26,6 +26,7 @@ import ( "io/ioutil" "net/http" "testing" + "time" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" "github.com/Azure/go-autorest/autorest" @@ -33,11 +34,90 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + "k8s.io/client-go/util/flowcontrol" azclients "k8s.io/legacy-cloud-providers/azure/clients" "k8s.io/legacy-cloud-providers/azure/clients/armclient" "k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient" + "k8s.io/legacy-cloud-providers/azure/retry" ) +func TestNew(t *testing.T) { + config := &azclients.ClientConfig{ + SubscriptionID: "sub", + ResourceManagerEndpoint: "endpoint", + Location: "eastus", + RateLimitConfig: &azclients.RateLimitConfig{ + CloudProviderRateLimit: true, + CloudProviderRateLimitQPS: 0.5, + CloudProviderRateLimitBucket: 1, + CloudProviderRateLimitQPSWrite: 0.5, + CloudProviderRateLimitBucketWrite: 1, + }, + Backoff: &retry.Backoff{Steps: 1}, + } + + vmssvmClient := New(config) + assert.Equal(t, "sub", vmssvmClient.subscriptionID) + assert.NotEmpty(t, vmssvmClient.rateLimiterReader) + assert.NotEmpty(t, vmssvmClient.rateLimiterWriter) +} + +func TestGet(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss1/virtualMachines/0" + response := &http.Response{ + StatusCode: http.StatusOK, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "InstanceView").Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + expected := compute.VirtualMachineScaleSetVM{Response: autorest.Response{Response: response}} + vmssvmClient := getTestVMSSVMClient(armClient) + result, rerr := vmssvmClient.Get(context.TODO(), "rg", "vmss1", "0", "InstanceView") + assert.Equal(t, expected, result) + assert.Nil(t, rerr) +} + +func TestGetNeverRateLimiter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmssvmGetErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "read", "VMSSVMGet"), + Retriable: true, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + vmssvmClient := getTestVMSSVMClientWithNeverRateLimiter(armClient) + expected := compute.VirtualMachineScaleSetVM{} + result, rerr := vmssvmClient.Get(context.TODO(), "rg", "vmss1", "0", "InstanceView") + assert.Equal(t, expected, result) + assert.Equal(t, vmssvmGetErr, rerr) +} + +func TestGetRetryAfterReader(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmssvmGetErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "VMSSVMGet", "client throttled"), + Retriable: true, + RetryAfter: getFutureTime(), + } + + armClient := mockarmclient.NewMockInterface(ctrl) + vmssvmClient := getTestVMSSVMClientWithRetryAfterReader(armClient) + expected := compute.VirtualMachineScaleSetVM{} + result, rerr := vmssvmClient.Get(context.TODO(), "rg", "vmss1", "0", "InstanceView") + assert.Equal(t, expected, result) + assert.Equal(t, vmssvmGetErr, rerr) +} + func TestGetNotFound(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -80,6 +160,31 @@ func TestGetInternalError(t *testing.T) { assert.Equal(t, http.StatusInternalServerError, rerr.HTTPStatusCode) } +func TestGetThrottle(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss1/virtualMachines/0" + response := &http.Response{ + StatusCode: http.StatusTooManyRequests, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + throttleErr := &retry.Error{ + HTTPStatusCode: http.StatusTooManyRequests, + RawError: fmt.Errorf("error"), + Retriable: true, + RetryAfter: time.Unix(100, 0), + } + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "InstanceView").Return(response, throttleErr).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + vmssvmClient := getTestVMSSVMClient(armClient) + result, rerr := vmssvmClient.Get(context.TODO(), "rg", "vmss1", "0", "InstanceView") + assert.Empty(t, result) + assert.Equal(t, throttleErr, rerr) +} + func TestList(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -102,6 +207,245 @@ func TestList(t *testing.T) { assert.Equal(t, 3, len(result)) } +func TestListNotFound(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss1/virtualMachines" + response := &http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "InstanceView").Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + vmssvmClient := getTestVMSSVMClient(armClient) + expected := []compute.VirtualMachineScaleSetVM{} + result, rerr := vmssvmClient.List(context.TODO(), "rg", "vmss1", "InstanceView") + assert.Equal(t, expected, result) + assert.NotNil(t, rerr) + assert.Equal(t, http.StatusNotFound, rerr.HTTPStatusCode) +} + +func TestListInternalError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss1/virtualMachines" + response := &http.Response{ + StatusCode: http.StatusInternalServerError, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "InstanceView").Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + vmssvmClient := getTestVMSSVMClient(armClient) + expected := []compute.VirtualMachineScaleSetVM{} + result, rerr := vmssvmClient.List(context.TODO(), "rg", "vmss1", "InstanceView") + assert.Equal(t, expected, result) + assert.NotNil(t, rerr) + assert.Equal(t, http.StatusInternalServerError, rerr.HTTPStatusCode) +} + +func TestListThrottle(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss1/virtualMachines" + armClient := mockarmclient.NewMockInterface(ctrl) + response := &http.Response{ + StatusCode: http.StatusTooManyRequests, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + throttleErr := &retry.Error{ + HTTPStatusCode: http.StatusTooManyRequests, + RawError: fmt.Errorf("error"), + Retriable: true, + RetryAfter: time.Unix(100, 0), + } + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "InstanceView").Return(response, throttleErr).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + vmssvmClient := getTestVMSSVMClient(armClient) + result, rerr := vmssvmClient.List(context.TODO(), "rg", "vmss1", "InstanceView") + assert.Empty(t, result) + assert.NotNil(t, rerr) + assert.Equal(t, throttleErr, rerr) +} + +func TestListWithListResponderError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss1/virtualMachines" + armClient := mockarmclient.NewMockInterface(ctrl) + vmssvmList := []compute.VirtualMachineScaleSetVM{getTestVMSSVM("vmss1", "1"), getTestVMSSVM("vmss1", "2"), getTestVMSSVM("vmss1", "3")} + responseBody, err := json.Marshal(compute.VirtualMachineScaleSetVMListResult{Value: &vmssvmList}) + assert.Nil(t, err) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "InstanceView").Return( + &http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewReader(responseBody)), + }, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + vmssvmClient := getTestVMSSVMClient(armClient) + result, rerr := vmssvmClient.List(context.TODO(), "rg", "vmss1", "InstanceView") + assert.NotNil(t, rerr) + assert.Equal(t, 0, len(result)) +} + +func TestListNeverRateLimiter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmssvmListErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "read", "VMSSVMList"), + Retriable: true, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + vmssvmClient := getTestVMSSVMClientWithNeverRateLimiter(armClient) + result, rerr := vmssvmClient.List(context.TODO(), "rg", "vmss1", "InstanceView") + assert.Equal(t, 0, len(result)) + assert.NotNil(t, rerr) + assert.Equal(t, vmssvmListErr, rerr) +} + +func TestListRetryAfterReader(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmssvmListErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "VMSSVMList", "client throttled"), + Retriable: true, + RetryAfter: getFutureTime(), + } + + armClient := mockarmclient.NewMockInterface(ctrl) + vmssvmClient := getTestVMSSVMClientWithRetryAfterReader(armClient) + result, rerr := vmssvmClient.List(context.TODO(), "rg", "vmss1", "InstanceView") + assert.Equal(t, 0, len(result)) + assert.NotNil(t, rerr) + assert.Equal(t, vmssvmListErr, rerr) +} + +func TestListNextResultsMultiPages(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tests := []struct { + name string + prepareErr error + sendErr *retry.Error + expectedErrMsg string + }{ + { + name: "testlistNextResultsSuccessful", + prepareErr: nil, + sendErr: nil, + }, + { + name: "testPrepareGetRequestError", + prepareErr: fmt.Errorf("error"), + expectedErrMsg: "Failure preparing next results request", + }, + { + name: "testSendError", + sendErr: &retry.Error{RawError: fmt.Errorf("error")}, + expectedErrMsg: "Failure sending next results request", + }, + } + + lastResult := compute.VirtualMachineScaleSetVMListResult{ + NextLink: to.StringPtr("next"), + } + + for _, test := range tests { + armClient := mockarmclient.NewMockInterface(ctrl) + req := &http.Request{ + Method: "GET", + } + armClient.EXPECT().PrepareGetRequest(gomock.Any(), gomock.Any()).Return(req, test.prepareErr) + if test.prepareErr == nil { + armClient.EXPECT().Send(gomock.Any(), req).Return(&http.Response{ + StatusCode: http.StatusOK, + Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"foo":"bar"}`))), + }, test.sendErr) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()) + } + + vmssClient := getTestVMSSVMClient(armClient) + result, err := vmssClient.listNextResults(context.TODO(), lastResult) + if err != nil { + assert.Equal(t, err.(autorest.DetailedError).Message, test.expectedErrMsg) + } else { + assert.Nil(t, err) + } + + if test.prepareErr != nil { + assert.Empty(t, result) + } else { + assert.NotEmpty(t, result) + } + } +} + +func TestListNextResultsMultiPagesWithListResponderError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tests := []struct { + name string + prepareErr error + sendErr *retry.Error + }{ + { + name: "testListResponderError", + prepareErr: nil, + sendErr: nil, + }, + { + name: "testSendError", + sendErr: &retry.Error{RawError: fmt.Errorf("error")}, + }, + } + + lastResult := compute.VirtualMachineScaleSetVMListResult{ + NextLink: to.StringPtr("next"), + } + + for _, test := range tests { + armClient := mockarmclient.NewMockInterface(ctrl) + req := &http.Request{ + Method: "GET", + } + armClient.EXPECT().PrepareGetRequest(gomock.Any(), gomock.Any()).Return(req, test.prepareErr) + if test.prepareErr == nil { + armClient.EXPECT().Send(gomock.Any(), req).Return(&http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"foo":"bar"}`))), + }, test.sendErr) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()) + } + + response := &http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewBuffer([]byte(`{"foo":"bar"}`))), + } + expected := compute.VirtualMachineScaleSetVMListResult{} + expected.Response = autorest.Response{Response: response} + vmssClient := getTestVMSSVMClient(armClient) + result, err := vmssClient.listNextResults(context.TODO(), lastResult) + assert.NotNil(t, err) + if test.sendErr != nil { + assert.NotEqual(t, expected, result) + } else { + assert.Equal(t, expected, result) + } + } +} + func TestUpdate(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -120,6 +464,222 @@ func TestUpdate(t *testing.T) { assert.Nil(t, rerr) } +func TestUpdateWithUpdateResponderError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmssVM := getTestVMSSVM("vmss1", "0") + armClient := mockarmclient.NewMockInterface(ctrl) + response := &http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewReader([]byte(""))), + } + armClient.EXPECT().PutResource(gomock.Any(), to.String(vmssVM.ID), vmssVM).Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + vmssvmClient := getTestVMSSVMClient(armClient) + rerr := vmssvmClient.Update(context.TODO(), "rg", "vmss1", "0", vmssVM, "test") + assert.NotNil(t, rerr) +} + +func TestUpdateNeverRateLimiter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmssvmUpdateErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "write", "VMSSVMUpdate"), + Retriable: true, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + vmssvmClient := getTestVMSSVMClientWithNeverRateLimiter(armClient) + vmssVM := getTestVMSSVM("vmss1", "0") + rerr := vmssvmClient.Update(context.TODO(), "rg", "vmss1", "0", vmssVM, "test") + assert.NotNil(t, rerr) + assert.Equal(t, vmssvmUpdateErr, rerr) +} + +func TestUpdateRetryAfterReader(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmssvmUpdateErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "VMSSVMUpdate", "client throttled"), + Retriable: true, + RetryAfter: getFutureTime(), + } + + vmssVM := getTestVMSSVM("vmss1", "0") + armClient := mockarmclient.NewMockInterface(ctrl) + vmClient := getTestVMSSVMClientWithRetryAfterReader(armClient) + rerr := vmClient.Update(context.TODO(), "rg", "vmss1", "0", vmssVM, "test") + assert.NotNil(t, rerr) + assert.Equal(t, vmssvmUpdateErr, rerr) +} + +func TestUpdateThrottle(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + response := &http.Response{ + StatusCode: http.StatusTooManyRequests, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + throttleErr := &retry.Error{ + HTTPStatusCode: http.StatusTooManyRequests, + RawError: fmt.Errorf("error"), + Retriable: true, + RetryAfter: time.Unix(100, 0), + } + + vmssVM := getTestVMSSVM("vmss1", "0") + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().PutResource(gomock.Any(), to.String(vmssVM.ID), vmssVM).Return(response, throttleErr).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + vmssvmClient := getTestVMSSVMClient(armClient) + rerr := vmssvmClient.Update(context.TODO(), "rg", "vmss1", "0", vmssVM, "test") + assert.NotNil(t, rerr) + assert.Equal(t, throttleErr, rerr) +} + +func TestUpdateVMs(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmssVM1 := getTestVMSSVM("vmss1", "1") + vmssVM2 := getTestVMSSVM("vmss1", "2") + instances := map[string]compute.VirtualMachineScaleSetVM{ + "1": vmssVM1, + "2": vmssVM2, + } + testvmssVMs := map[string]interface{}{ + to.String(vmssVM1.ID): vmssVM1, + to.String(vmssVM2.ID): vmssVM2, + } + response := &http.Response{ + StatusCode: http.StatusOK, + Body: ioutil.NopCloser(bytes.NewReader([]byte(""))), + } + responses := map[string]*armclient.PutResourcesResponse{ + to.String(vmssVM1.ID): { + Response: response, + }, + to.String(vmssVM2.ID): { + Response: response, + }, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().PutResources(gomock.Any(), testvmssVMs).Return(responses).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(2) + + vmssvmClient := getTestVMSSVMClient(armClient) + rerr := vmssvmClient.UpdateVMs(context.TODO(), "rg", "vmss1", instances, "test") + assert.Nil(t, rerr) +} + +func TestUpdateVMsWithUpdateVMsResponderError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmssVM := getTestVMSSVM("vmss1", "1") + instances := map[string]compute.VirtualMachineScaleSetVM{ + "1": vmssVM, + } + testvmssVMs := map[string]interface{}{ + to.String(vmssVM.ID): vmssVM, + } + response := &http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewReader([]byte(""))), + } + responses := map[string]*armclient.PutResourcesResponse{ + to.String(vmssVM.ID): { + Response: response, + }, + } + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().PutResources(gomock.Any(), testvmssVMs).Return(responses).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + vmssvmClient := getTestVMSSVMClient(armClient) + rerr := vmssvmClient.UpdateVMs(context.TODO(), "rg", "vmss1", instances, "test") + assert.NotNil(t, rerr) +} + +func TestUpdateVMsNeverRateLimiter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + instances := map[string]compute.VirtualMachineScaleSetVM{} + vmssvmUpdateVMsErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "write", "VMSSVMUpdateVMs"), + Retriable: true, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + vmssvmClient := getTestVMSSVMClientWithNeverRateLimiter(armClient) + rerr := vmssvmClient.UpdateVMs(context.TODO(), "rg", "vmss1", instances, "test") + assert.NotNil(t, rerr) + assert.Equal(t, vmssvmUpdateVMsErr, rerr) +} + +func TestUpdateVMsRetryAfterReader(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + instances := map[string]compute.VirtualMachineScaleSetVM{} + vmssvmUpdateVMsErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "VMSSVMUpdateVMs", "client throttled"), + Retriable: true, + RetryAfter: getFutureTime(), + } + + armClient := mockarmclient.NewMockInterface(ctrl) + vmClient := getTestVMSSVMClientWithRetryAfterReader(armClient) + rerr := vmClient.UpdateVMs(context.TODO(), "rg", "vmss1", instances, "test") + assert.NotNil(t, rerr) + assert.Equal(t, vmssvmUpdateVMsErr, rerr) +} + +func TestUpdateVMsThrottle(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmssVM := getTestVMSSVM("vmss1", "1") + instances := map[string]compute.VirtualMachineScaleSetVM{ + "1": vmssVM, + } + testvmssVMs := map[string]interface{}{ + to.String(vmssVM.ID): vmssVM, + } + throttleErr := retry.Error{ + HTTPStatusCode: http.StatusTooManyRequests, + RawError: fmt.Errorf("error"), + Retriable: true, + RetryAfter: time.Unix(100, 0), + } + responses := map[string]*armclient.PutResourcesResponse{ + to.String(vmssVM.ID): { + Response: &http.Response{ + StatusCode: http.StatusTooManyRequests, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + }, + Error: &throttleErr, + }, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().PutResources(gomock.Any(), testvmssVMs).Return(responses).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + vmssvmClient := getTestVMSSVMClient(armClient) + rerr := vmssvmClient.UpdateVMs(context.TODO(), "rg", "vmss1", instances, "test") + assert.NotNil(t, rerr) + assert.Equal(t, throttleErr.Error(), fmt.Errorf(rerr.RawError.Error())) +} + func getTestVMSSVM(vmssName, instanceID string) compute.VirtualMachineScaleSetVM { resourceID := fmt.Sprintf("/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/%s/virtualMachines/%s", vmssName, instanceID) return compute.VirtualMachineScaleSetVM{ @@ -138,3 +698,31 @@ func getTestVMSSVMClient(armClient armclient.Interface) *Client { rateLimiterWriter: rateLimiterWriter, } } + +func getTestVMSSVMClientWithNeverRateLimiter(armClient armclient.Interface) *Client { + rateLimiterReader := flowcontrol.NewFakeNeverRateLimiter() + rateLimiterWriter := flowcontrol.NewFakeNeverRateLimiter() + return &Client{ + armClient: armClient, + subscriptionID: "subscriptionID", + rateLimiterReader: rateLimiterReader, + rateLimiterWriter: rateLimiterWriter, + } +} + +func getTestVMSSVMClientWithRetryAfterReader(armClient armclient.Interface) *Client { + rateLimiterReader := flowcontrol.NewFakeAlwaysRateLimiter() + rateLimiterWriter := flowcontrol.NewFakeAlwaysRateLimiter() + return &Client{ + armClient: armClient, + subscriptionID: "subscriptionID", + rateLimiterReader: rateLimiterReader, + rateLimiterWriter: rateLimiterWriter, + RetryAfterReader: getFutureTime(), + RetryAfterWriter: getFutureTime(), + } +} + +func getFutureTime() time.Time { + return time.Unix(3000000000, 0) +}