diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/snapshotclient/BUILD b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/snapshotclient/BUILD index 34ab1d31e16..2fcb0313315 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/snapshotclient/BUILD +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/snapshotclient/BUILD @@ -29,9 +29,11 @@ go_test( srcs = ["azure_snapshotclient_test.go"], embed = [":go_default_library"], deps = [ + "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure/clients:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient:go_default_library", + "//staging/src/k8s.io/legacy-cloud-providers/azure/retry:go_default_library", "//vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute:go_default_library", "//vendor/github.com/Azure/go-autorest/autorest:go_default_library", "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library", diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/snapshotclient/azure_snapshotclient_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/snapshotclient/azure_snapshotclient_test.go index ad85bccb41a..7ac5883c034 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/snapshotclient/azure_snapshotclient_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/snapshotclient/azure_snapshotclient_test.go @@ -26,17 +26,98 @@ import ( "io/ioutil" "net/http" "testing" + "time" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/to" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + + "k8s.io/client-go/util/flowcontrol" azclients "k8s.io/legacy-cloud-providers/azure/clients" "k8s.io/legacy-cloud-providers/azure/clients/armclient" "k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient" + "k8s.io/legacy-cloud-providers/azure/retry" ) +func TestNew(t *testing.T) { + config := &azclients.ClientConfig{ + SubscriptionID: "sub", + ResourceManagerEndpoint: "endpoint", + Location: "eastus", + RateLimitConfig: &azclients.RateLimitConfig{ + CloudProviderRateLimit: true, + CloudProviderRateLimitQPS: 0.5, + CloudProviderRateLimitBucket: 1, + CloudProviderRateLimitQPSWrite: 0.5, + CloudProviderRateLimitBucketWrite: 1, + }, + Backoff: &retry.Backoff{Steps: 1}, + } + + snClient := New(config) + assert.Equal(t, "sub", snClient.subscriptionID) + assert.NotEmpty(t, snClient.rateLimiterReader) + assert.NotEmpty(t, snClient.rateLimiterWriter) +} + +func TestGet(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/snapshots/sn1" + response := &http.Response{ + StatusCode: http.StatusOK, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + expected := compute.Snapshot{Response: autorest.Response{Response: response}} + snClient := getTestSnapshotClient(armClient) + result, rerr := snClient.Get(context.TODO(), "rg", "sn1") + assert.Equal(t, expected, result) + assert.Nil(t, rerr) +} + +func TestGetNeverRateLimiter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + snGetErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "read", "SnapshotGet"), + Retriable: true, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + snClient := getTestSnapshotClientWithNeverRateLimiter(armClient) + expected := compute.Snapshot{} + result, rerr := snClient.Get(context.TODO(), "rg", "sn1") + assert.Equal(t, expected, result) + assert.Equal(t, snGetErr, rerr) +} + +func TestGetRetryAfterReader(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + snGetErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "SnapshotGet", "client throttled"), + Retriable: true, + RetryAfter: getFutureTime(), + } + + armClient := mockarmclient.NewMockInterface(ctrl) + snClient := getTestSnapshotClientWithRetryAfterReader(armClient) + expected := compute.Snapshot{} + result, rerr := snClient.Get(context.TODO(), "rg", "sn1") + assert.Equal(t, expected, result) + assert.Equal(t, snGetErr, rerr) +} + func TestGetNotFound(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -58,6 +139,31 @@ func TestGetNotFound(t *testing.T) { assert.Equal(t, http.StatusNotFound, rerr.HTTPStatusCode) } +func TestGetThrottle(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/snapshots/sn1" + response := &http.Response{ + StatusCode: http.StatusTooManyRequests, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + throttleErr := &retry.Error{ + HTTPStatusCode: http.StatusTooManyRequests, + RawError: fmt.Errorf("error"), + Retriable: true, + RetryAfter: time.Unix(100, 0), + } + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, throttleErr).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + snClient := getTestSnapshotClient(armClient) + result, rerr := snClient.Get(context.TODO(), "rg", "sn1") + assert.Empty(t, result) + assert.Equal(t, throttleErr, rerr) +} + func TestGetInternalError(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -85,7 +191,7 @@ func TestListByResourceGroup(t *testing.T) { resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/snapshots" armClient := mockarmclient.NewMockInterface(ctrl) - snList := []compute.Snapshot{getTestSnapshot("sn1"), getTestSnapshot("pip2"), getTestSnapshot("pip3")} + snList := []compute.Snapshot{getTestSnapshot("sn1"), getTestSnapshot("sn2"), getTestSnapshot("sn3")} responseBody, err := json.Marshal(compute.SnapshotList{Value: &snList}) assert.Nil(t, err) armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return( @@ -101,6 +207,223 @@ func TestListByResourceGroup(t *testing.T) { assert.Equal(t, 3, len(result)) } +func TestListByResourceGroupNotFound(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/snapshots" + response := &http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + snClient := getTestSnapshotClient(armClient) + expected := []compute.Snapshot{} + result, rerr := snClient.ListByResourceGroup(context.TODO(), "rg") + assert.Equal(t, expected, result) + assert.NotNil(t, rerr) + assert.Equal(t, http.StatusNotFound, rerr.HTTPStatusCode) +} + +func TestListByResourceGroupInternalError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/snapshots" + response := &http.Response{ + StatusCode: http.StatusInternalServerError, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + snClient := getTestSnapshotClient(armClient) + expected := []compute.Snapshot{} + result, rerr := snClient.ListByResourceGroup(context.TODO(), "rg") + assert.Equal(t, expected, result) + assert.NotNil(t, rerr) + assert.Equal(t, http.StatusInternalServerError, rerr.HTTPStatusCode) +} + +func TestListByResourceGroupThrottle(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/snapshots" + armClient := mockarmclient.NewMockInterface(ctrl) + response := &http.Response{ + StatusCode: http.StatusTooManyRequests, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + throttleErr := &retry.Error{ + HTTPStatusCode: http.StatusTooManyRequests, + RawError: fmt.Errorf("error"), + Retriable: true, + RetryAfter: time.Unix(100, 0), + } + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, throttleErr).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + snClient := getTestSnapshotClient(armClient) + result, rerr := snClient.ListByResourceGroup(context.TODO(), "rg") + assert.Empty(t, result) + assert.NotNil(t, rerr) + assert.Equal(t, throttleErr, rerr) +} + +func TestListByResourceGroupWithListResponderError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/snapshots" + armClient := mockarmclient.NewMockInterface(ctrl) + snList := []compute.Snapshot{getTestSnapshot("sn1"), getTestSnapshot("sn2"), getTestSnapshot("sn3")} + responseBody, err := json.Marshal(compute.SnapshotList{Value: &snList}) + assert.Nil(t, err) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return( + &http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewReader(responseBody)), + }, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + snClient := getTestSnapshotClient(armClient) + result, rerr := snClient.ListByResourceGroup(context.TODO(), "rg") + assert.NotNil(t, rerr) + assert.Equal(t, 0, len(result)) +} + +func TestListByResourceGroupNeverRateLimiter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + snListErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "read", "SnapshotListByResourceGroup"), + Retriable: true, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + snClient := getTestSnapshotClientWithNeverRateLimiter(armClient) + result, rerr := snClient.ListByResourceGroup(context.TODO(), "rg") + assert.Equal(t, 0, len(result)) + assert.NotNil(t, rerr) + assert.Equal(t, snListErr, rerr) +} + +func TestListByResourceGroupRetryAfterReader(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + snListErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "SnapshotListByResourceGroup", "client throttled"), + Retriable: true, + RetryAfter: getFutureTime(), + } + + armClient := mockarmclient.NewMockInterface(ctrl) + snClient := getTestSnapshotClientWithRetryAfterReader(armClient) + result, rerr := snClient.ListByResourceGroup(context.TODO(), "rg") + assert.Equal(t, 0, len(result)) + assert.NotNil(t, rerr) + assert.Equal(t, snListErr, rerr) +} + +func TestListNextResultsMultiPages(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tests := []struct { + prepareErr error + sendErr *retry.Error + }{ + { + prepareErr: nil, + sendErr: nil, + }, + { + prepareErr: fmt.Errorf("error"), + }, + { + sendErr: &retry.Error{RawError: fmt.Errorf("error")}, + }, + } + + lastResult := compute.SnapshotList{ + NextLink: to.StringPtr("next"), + } + + for _, test := range tests { + armClient := mockarmclient.NewMockInterface(ctrl) + req := &http.Request{ + Method: "GET", + } + armClient.EXPECT().PrepareGetRequest(gomock.Any(), gomock.Any()).Return(req, test.prepareErr) + if test.prepareErr == nil { + armClient.EXPECT().Send(gomock.Any(), req).Return(&http.Response{ + StatusCode: http.StatusOK, + Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"foo":"bar"}`))), + }, test.sendErr) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()) + } + + snClient := getTestSnapshotClient(armClient) + result, err := snClient.listNextResults(context.TODO(), lastResult) + if test.prepareErr != nil || test.sendErr != nil { + assert.NotNil(t, err) + } else { + assert.NoError(t, err) + } + if test.prepareErr != nil { + assert.Empty(t, result) + } else { + assert.NotEmpty(t, result) + } + } +} + +func TestListNextResultsMultiPagesWithListResponderError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + test := struct { + prepareErr error + sendErr *retry.Error + }{ + prepareErr: nil, + sendErr: nil, + } + + lastResult := compute.SnapshotList{ + NextLink: to.StringPtr("next"), + } + + armClient := mockarmclient.NewMockInterface(ctrl) + req := &http.Request{ + Method: "GET", + } + armClient.EXPECT().PrepareGetRequest(gomock.Any(), gomock.Any()).Return(req, test.prepareErr) + if test.prepareErr == nil { + armClient.EXPECT().Send(gomock.Any(), req).Return(&http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"foo":"bar"}`))), + }, test.sendErr) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()) + } + + response := &http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewBuffer([]byte(`{"foo":"bar"}`))), + } + expected := compute.SnapshotList{} + expected.Response = autorest.Response{Response: response} + snClient := getTestSnapshotClient(armClient) + result, err := snClient.listNextResults(context.TODO(), lastResult) + assert.NotNil(t, err) + assert.Equal(t, expected, result) +} + func TestCreateOrUpdate(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -119,6 +442,84 @@ func TestCreateOrUpdate(t *testing.T) { assert.Nil(t, rerr) } +func TestCreateOrUpdateWithCreateOrUpdateResponderError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + sn := getTestSnapshot("sn1") + armClient := mockarmclient.NewMockInterface(ctrl) + response := &http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewReader([]byte(""))), + } + armClient.EXPECT().PutResource(gomock.Any(), to.String(sn.ID), sn).Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + snClient := getTestSnapshotClient(armClient) + rerr := snClient.CreateOrUpdate(context.TODO(), "rg", "sn1", sn) + assert.NotNil(t, rerr) +} + +func TestCreateOrUpdateNeverRateLimiter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + snCreateOrUpdateErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "write", "SnapshotCreateOrUpdate"), + Retriable: true, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + snClient := getTestSnapshotClientWithNeverRateLimiter(armClient) + sn := getTestSnapshot("sn1") + rerr := snClient.CreateOrUpdate(context.TODO(), "rg", "sn1", sn) + assert.NotNil(t, rerr) + assert.Equal(t, snCreateOrUpdateErr, rerr) +} + +func TestCreateOrUpdateRetryAfterReader(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + snCreateOrUpdateErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "SnapshotCreateOrUpdate", "client throttled"), + Retriable: true, + RetryAfter: getFutureTime(), + } + + sn := getTestSnapshot("sn1") + armClient := mockarmclient.NewMockInterface(ctrl) + snClient := getTestSnapshotClientWithRetryAfterReader(armClient) + rerr := snClient.CreateOrUpdate(context.TODO(), "rg", "sn1", sn) + assert.NotNil(t, rerr) + assert.Equal(t, snCreateOrUpdateErr, rerr) +} + +func TestCreateOrUpdateThrottle(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + response := &http.Response{ + StatusCode: http.StatusTooManyRequests, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + throttleErr := &retry.Error{ + HTTPStatusCode: http.StatusTooManyRequests, + RawError: fmt.Errorf("error"), + Retriable: true, + RetryAfter: time.Unix(100, 0), + } + + sn := getTestSnapshot("sn1") + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().PutResource(gomock.Any(), to.String(sn.ID), sn).Return(response, throttleErr).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + snClient := getTestSnapshotClient(armClient) + rerr := snClient.CreateOrUpdate(context.TODO(), "rg", "sn1", sn) + assert.NotNil(t, rerr) + assert.Equal(t, throttleErr, rerr) +} + func TestDelete(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -132,6 +533,60 @@ func TestDelete(t *testing.T) { assert.Nil(t, rerr) } +func TestDeleteNeverRateLimiter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + snDeleteErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "write", "SnapshotDelete"), + Retriable: true, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + snClient := getTestSnapshotClientWithNeverRateLimiter(armClient) + rerr := snClient.Delete(context.TODO(), "rg", "sn1") + assert.NotNil(t, rerr) + assert.Equal(t, snDeleteErr, rerr) +} + +func TestDeleteRetryAfterReader(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + snDeleteErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "SnapshotDelete", "client throttled"), + Retriable: true, + RetryAfter: getFutureTime(), + } + + armClient := mockarmclient.NewMockInterface(ctrl) + snClient := getTestSnapshotClientWithRetryAfterReader(armClient) + rerr := snClient.Delete(context.TODO(), "rg", "sn1") + assert.NotNil(t, rerr) + assert.Equal(t, snDeleteErr, rerr) +} + +func TestDeleteThrottle(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + throttleErr := &retry.Error{ + HTTPStatusCode: http.StatusTooManyRequests, + RawError: fmt.Errorf("error"), + Retriable: true, + RetryAfter: time.Unix(100, 0), + } + + sn := getTestSnapshot("sn1") + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().DeleteResource(gomock.Any(), to.String(sn.ID), "").Return(throttleErr).Times(1) + + snClient := getTestSnapshotClient(armClient) + rerr := snClient.Delete(context.TODO(), "rg", "sn1") + assert.NotNil(t, rerr) + assert.Equal(t, throttleErr, rerr) +} + func getTestSnapshot(name string) compute.Snapshot { return compute.Snapshot{ ID: to.StringPtr(fmt.Sprintf("/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/snapshots/%s", name)), @@ -149,3 +604,31 @@ func getTestSnapshotClient(armClient armclient.Interface) *Client { rateLimiterWriter: rateLimiterWriter, } } + +func getTestSnapshotClientWithNeverRateLimiter(armClient armclient.Interface) *Client { + rateLimiterReader := flowcontrol.NewFakeNeverRateLimiter() + rateLimiterWriter := flowcontrol.NewFakeNeverRateLimiter() + return &Client{ + armClient: armClient, + subscriptionID: "subscriptionID", + rateLimiterReader: rateLimiterReader, + rateLimiterWriter: rateLimiterWriter, + } +} + +func getTestSnapshotClientWithRetryAfterReader(armClient armclient.Interface) *Client { + rateLimiterReader := flowcontrol.NewFakeAlwaysRateLimiter() + rateLimiterWriter := flowcontrol.NewFakeAlwaysRateLimiter() + return &Client{ + armClient: armClient, + subscriptionID: "subscriptionID", + rateLimiterReader: rateLimiterReader, + rateLimiterWriter: rateLimiterWriter, + RetryAfterReader: getFutureTime(), + RetryAfterWriter: getFutureTime(), + } +} + +func getFutureTime() time.Time { + return time.Unix(3000000000, 0) +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/subnetclient/BUILD b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/subnetclient/BUILD index 08c612d10da..34e04209a3f 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/subnetclient/BUILD +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/subnetclient/BUILD @@ -29,9 +29,11 @@ go_test( srcs = ["azure_subnetclient_test.go"], embed = [":go_default_library"], deps = [ + "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure/clients:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient:go_default_library", + "//staging/src/k8s.io/legacy-cloud-providers/azure/retry:go_default_library", "//vendor/github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network:go_default_library", "//vendor/github.com/Azure/go-autorest/autorest:go_default_library", "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library", diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/subnetclient/azure_subnetclient_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/subnetclient/azure_subnetclient_test.go index a437d5a62d6..b83c6cff9a2 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/subnetclient/azure_subnetclient_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/subnetclient/azure_subnetclient_test.go @@ -26,6 +26,7 @@ import ( "io/ioutil" "net/http" "testing" + "time" "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" "github.com/Azure/go-autorest/autorest" @@ -33,11 +34,63 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + "k8s.io/client-go/util/flowcontrol" azclients "k8s.io/legacy-cloud-providers/azure/clients" "k8s.io/legacy-cloud-providers/azure/clients/armclient" "k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient" + "k8s.io/legacy-cloud-providers/azure/retry" ) +func TestNew(t *testing.T) { + config := &azclients.ClientConfig{ + SubscriptionID: "sub", + ResourceManagerEndpoint: "endpoint", + Location: "eastus", + RateLimitConfig: &azclients.RateLimitConfig{ + CloudProviderRateLimit: true, + CloudProviderRateLimitQPS: 0.5, + CloudProviderRateLimitBucket: 1, + CloudProviderRateLimitQPSWrite: 0.5, + CloudProviderRateLimitBucketWrite: 1, + }, + Backoff: &retry.Backoff{Steps: 1}, + } + + subnetClient := New(config) + assert.Equal(t, "sub", subnetClient.subscriptionID) + assert.NotEmpty(t, subnetClient.rateLimiterReader) + assert.NotEmpty(t, subnetClient.rateLimiterWriter) +} + +func TestGet(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnet/subnets/subnet1" + testSubnet := network.Subnet{ + Name: to.StringPtr("subnet1"), + } + subnet, err := testSubnet.MarshalJSON() + assert.Nil(t, err) + response := &http.Response{ + StatusCode: http.StatusOK, + Body: ioutil.NopCloser(bytes.NewReader(subnet)), + } + + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + expected := network.Subnet{ + Response: autorest.Response{Response: response}, + Name: to.StringPtr("subnet1"), + } + subnetClient := getTestSubnetClient(armClient) + result, rerr := subnetClient.Get(context.TODO(), "rg", "vnet", "subnet1", "") + assert.Equal(t, expected, result) + assert.Nil(t, rerr) +} + func TestGetNotFound(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -80,6 +133,67 @@ func TestGetInternalError(t *testing.T) { assert.Equal(t, http.StatusInternalServerError, rerr.HTTPStatusCode) } +func TestGetNeverRateLimiter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + subnetGetErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "read", "SubnetGet"), + Retriable: true, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + subnetClient := getTestSubnetClientWithNeverRateLimiter(armClient) + expected := network.Subnet{} + result, rerr := subnetClient.Get(context.TODO(), "rg", "vnet", "subnet1", "") + assert.Equal(t, expected, result) + assert.Equal(t, subnetGetErr, rerr) +} + +func TestGetRetryAfterReader(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + subnetGetErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "SubnetGet", "client throttled"), + Retriable: true, + RetryAfter: getFutureTime(), + } + + armClient := mockarmclient.NewMockInterface(ctrl) + subnetClient := getTestSubnetClientWithRetryAfterReader(armClient) + expected := network.Subnet{} + result, rerr := subnetClient.Get(context.TODO(), "rg", "vnet", "subnet1", "") + assert.Equal(t, expected, result) + assert.Equal(t, subnetGetErr, rerr) +} + +func TestGetThrottle(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnet/subnets/subnet1" + response := &http.Response{ + StatusCode: http.StatusTooManyRequests, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + throttleErr := &retry.Error{ + HTTPStatusCode: http.StatusTooManyRequests, + RawError: fmt.Errorf("error"), + Retriable: true, + RetryAfter: time.Unix(100, 0), + } + + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, throttleErr).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + subnetClient := getTestSubnetClient(armClient) + result, rerr := subnetClient.Get(context.TODO(), "rg", "vnet", "subnet1", "") + assert.Empty(t, result) + assert.Equal(t, throttleErr, rerr) +} + func TestList(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -102,6 +216,224 @@ func TestList(t *testing.T) { assert.Equal(t, 3, len(result)) } +func TestListNotFound(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnet/subnets" + response := &http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + subnetClient := getTestSubnetClient(armClient) + expected := []network.Subnet{} + result, rerr := subnetClient.List(context.TODO(), "rg", "vnet") + assert.Equal(t, expected, result) + assert.NotNil(t, rerr) + assert.Equal(t, http.StatusNotFound, rerr.HTTPStatusCode) +} + +func TestListInternalError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnet/subnets" + response := &http.Response{ + StatusCode: http.StatusInternalServerError, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + subnetClient := getTestSubnetClient(armClient) + expected := []network.Subnet{} + result, rerr := subnetClient.List(context.TODO(), "rg", "vnet") + assert.Equal(t, expected, result) + assert.NotNil(t, rerr) + assert.Equal(t, http.StatusInternalServerError, rerr.HTTPStatusCode) +} + +func TestListThrottle(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnet/subnets" + response := &http.Response{ + StatusCode: http.StatusTooManyRequests, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + throttleErr := &retry.Error{ + HTTPStatusCode: http.StatusTooManyRequests, + RawError: fmt.Errorf("error"), + Retriable: true, + RetryAfter: time.Unix(100, 0), + } + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, throttleErr).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + subnetClient := getTestSubnetClient(armClient) + result, rerr := subnetClient.List(context.TODO(), "rg", "vnet") + assert.Empty(t, result) + assert.NotNil(t, rerr) + assert.Equal(t, throttleErr, rerr) +} + +func TestListWithListResponderError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnet/subnets" + armClient := mockarmclient.NewMockInterface(ctrl) + subnetList := []network.Subnet{getTestSubnet("subnet1"), getTestSubnet("subnet2"), getTestSubnet("subnet3")} + responseBody, err := json.Marshal(network.SubnetListResult{Value: &subnetList}) + assert.Nil(t, err) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return( + &http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewReader(responseBody)), + }, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + subnetClient := getTestSubnetClient(armClient) + result, rerr := subnetClient.List(context.TODO(), "rg", "vnet") + assert.NotNil(t, rerr) + assert.Equal(t, 0, len(result)) +} + +func TestListNeverRateLimiter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + subnetListErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "read", "SubnetList"), + Retriable: true, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + subnetClient := getTestSubnetClientWithNeverRateLimiter(armClient) + result, rerr := subnetClient.List(context.TODO(), "rg", "vnet") + assert.Equal(t, 0, len(result)) + assert.NotNil(t, rerr) + assert.Equal(t, subnetListErr, rerr) +} + +func TestListRetryAfterReader(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + subnetListErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "SubnetList", "client throttled"), + Retriable: true, + RetryAfter: getFutureTime(), + } + + armClient := mockarmclient.NewMockInterface(ctrl) + subnetClient := getTestSubnetClientWithRetryAfterReader(armClient) + result, rerr := subnetClient.List(context.TODO(), "rg", "vnet") + assert.Equal(t, 0, len(result)) + assert.NotNil(t, rerr) + assert.Equal(t, subnetListErr, rerr) +} + +func TestListNextResultsMultiPages(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + tests := []struct { + prepareErr error + sendErr *retry.Error + }{ + { + prepareErr: nil, + sendErr: nil, + }, + { + prepareErr: fmt.Errorf("error"), + }, + { + sendErr: &retry.Error{RawError: fmt.Errorf("error")}, + }, + } + + lastResult := network.SubnetListResult{ + NextLink: to.StringPtr("next"), + } + + for _, test := range tests { + armClient := mockarmclient.NewMockInterface(ctrl) + req := &http.Request{ + Method: "GET", + } + armClient.EXPECT().PrepareGetRequest(gomock.Any(), gomock.Any()).Return(req, test.prepareErr) + if test.prepareErr == nil { + armClient.EXPECT().Send(gomock.Any(), req).Return(&http.Response{ + StatusCode: http.StatusOK, + Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"foo":"bar"}`))), + }, test.sendErr) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()) + } + + subnetClient := getTestSubnetClient(armClient) + result, err := subnetClient.listNextResults(context.TODO(), lastResult) + if test.prepareErr != nil || test.sendErr != nil { + assert.NotNil(t, err) + } else { + assert.NoError(t, err) + } + if test.prepareErr != nil { + assert.Empty(t, result) + } else { + assert.NotEmpty(t, result) + } + } +} + +func TestListNextResultsMultiPagesWithListResponderError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + test := struct { + prepareErr error + sendErr *retry.Error + }{ + prepareErr: nil, + sendErr: nil, + } + + lastResult := network.SubnetListResult{ + NextLink: to.StringPtr("next"), + } + + armClient := mockarmclient.NewMockInterface(ctrl) + req := &http.Request{ + Method: "GET", + } + armClient.EXPECT().PrepareGetRequest(gomock.Any(), gomock.Any()).Return(req, test.prepareErr) + if test.prepareErr == nil { + armClient.EXPECT().Send(gomock.Any(), req).Return(&http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewReader([]byte(`{"foo":"bar"}`))), + }, test.sendErr) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()) + } + + response := &http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewBuffer([]byte(`{"foo":"bar"}`))), + } + expected := network.SubnetListResult{} + expected.Response = autorest.Response{Response: response} + subnetClient := getTestSubnetClient(armClient) + result, err := subnetClient.listNextResults(context.TODO(), lastResult) + assert.NotNil(t, err) + assert.Equal(t, expected, result) +} + func TestCreateOrUpdate(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -120,6 +452,84 @@ func TestCreateOrUpdate(t *testing.T) { assert.Nil(t, rerr) } +func TestCreateOrUpdateWithCreateOrUpdateResponderError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + subnet := getTestSubnet("subnet1") + armClient := mockarmclient.NewMockInterface(ctrl) + response := &http.Response{ + StatusCode: http.StatusNotFound, + Body: ioutil.NopCloser(bytes.NewReader([]byte(""))), + } + armClient.EXPECT().PutResource(gomock.Any(), to.String(subnet.ID), subnet).Return(response, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + subnetClient := getTestSubnetClient(armClient) + rerr := subnetClient.CreateOrUpdate(context.TODO(), "rg", "vnet", "subnet1", subnet) + assert.NotNil(t, rerr) +} + +func TestCreateOrUpdateNeverRateLimiter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + subnetCreateOrUpdateErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "write", "SubnetCreateOrUpdate"), + Retriable: true, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + subnetClient := getTestSubnetClientWithNeverRateLimiter(armClient) + subnet := getTestSubnet("subnet1") + rerr := subnetClient.CreateOrUpdate(context.TODO(), "rg", "vnet", "subnet1", subnet) + assert.NotNil(t, rerr) + assert.Equal(t, subnetCreateOrUpdateErr, rerr) +} + +func TestCreateOrUpdateRetryAfterReader(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + subnetCreateOrUpdateErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "SubnetCreateOrUpdate", "client throttled"), + Retriable: true, + RetryAfter: getFutureTime(), + } + + subnet := getTestSubnet("subnet1") + armClient := mockarmclient.NewMockInterface(ctrl) + subnetClient := getTestSubnetClientWithRetryAfterReader(armClient) + rerr := subnetClient.CreateOrUpdate(context.TODO(), "rg", "vnet", "subnet1", subnet) + assert.NotNil(t, rerr) + assert.Equal(t, subnetCreateOrUpdateErr, rerr) +} + +func TestCreateOrUpdateThrottle(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + response := &http.Response{ + StatusCode: http.StatusTooManyRequests, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + throttleErr := &retry.Error{ + HTTPStatusCode: http.StatusTooManyRequests, + RawError: fmt.Errorf("error"), + Retriable: true, + RetryAfter: time.Unix(100, 0), + } + + subnet := getTestSubnet("subnet1") + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().PutResource(gomock.Any(), to.String(subnet.ID), subnet).Return(response, throttleErr).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + + subnetClient := getTestSubnetClient(armClient) + rerr := subnetClient.CreateOrUpdate(context.TODO(), "rg", "vnet", "subnet1", subnet) + assert.NotNil(t, rerr) + assert.Equal(t, throttleErr, rerr) +} + func TestDelete(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -133,6 +543,60 @@ func TestDelete(t *testing.T) { assert.Nil(t, rerr) } +func TestDeleteNeverRateLimiter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + subnetDeleteErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "write", "SubnetDelete"), + Retriable: true, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + subnetClient := getTestSubnetClientWithNeverRateLimiter(armClient) + rerr := subnetClient.Delete(context.TODO(), "rg", "vnet", "subnet1") + assert.NotNil(t, rerr) + assert.Equal(t, subnetDeleteErr, rerr) +} + +func TestDeleteRetryAfterReader(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + subnetDeleteErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "SubnetDelete", "client throttled"), + Retriable: true, + RetryAfter: getFutureTime(), + } + + armClient := mockarmclient.NewMockInterface(ctrl) + subnetClient := getTestSubnetClientWithRetryAfterReader(armClient) + rerr := subnetClient.Delete(context.TODO(), "rg", "vnet", "subnet1") + assert.NotNil(t, rerr) + assert.Equal(t, subnetDeleteErr, rerr) +} + +func TestDeleteThrottle(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + throttleErr := &retry.Error{ + HTTPStatusCode: http.StatusTooManyRequests, + RawError: fmt.Errorf("error"), + Retriable: true, + RetryAfter: time.Unix(100, 0), + } + + subnet := getTestSubnet("subnet1") + armClient := mockarmclient.NewMockInterface(ctrl) + armClient.EXPECT().DeleteResource(gomock.Any(), to.String(subnet.ID), "").Return(throttleErr).Times(1) + + subnetClient := getTestSubnetClient(armClient) + rerr := subnetClient.Delete(context.TODO(), "rg", "vnet", "subnet1") + assert.NotNil(t, rerr) + assert.Equal(t, throttleErr, rerr) +} + func getTestSubnet(name string) network.Subnet { return network.Subnet{ ID: to.StringPtr(fmt.Sprintf("/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnet/subnets/%s", name)), @@ -149,3 +613,31 @@ func getTestSubnetClient(armClient armclient.Interface) *Client { rateLimiterWriter: rateLimiterWriter, } } + +func getTestSubnetClientWithNeverRateLimiter(armClient armclient.Interface) *Client { + rateLimiterReader := flowcontrol.NewFakeNeverRateLimiter() + rateLimiterWriter := flowcontrol.NewFakeNeverRateLimiter() + return &Client{ + armClient: armClient, + subscriptionID: "subscriptionID", + rateLimiterReader: rateLimiterReader, + rateLimiterWriter: rateLimiterWriter, + } +} + +func getTestSubnetClientWithRetryAfterReader(armClient armclient.Interface) *Client { + rateLimiterReader := flowcontrol.NewFakeAlwaysRateLimiter() + rateLimiterWriter := flowcontrol.NewFakeAlwaysRateLimiter() + return &Client{ + armClient: armClient, + subscriptionID: "subscriptionID", + rateLimiterReader: rateLimiterReader, + rateLimiterWriter: rateLimiterWriter, + RetryAfterReader: getFutureTime(), + RetryAfterWriter: getFutureTime(), + } +} + +func getFutureTime() time.Time { + return time.Unix(3000000000, 0) +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmsizeclient/BUILD b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmsizeclient/BUILD index 44da0b704b0..00bd45b76e3 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmsizeclient/BUILD +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmsizeclient/BUILD @@ -28,11 +28,14 @@ go_test( srcs = ["azure_vmsizeclient_test.go"], embed = [":go_default_library"], deps = [ + "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure/clients:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient:go_default_library", + "//staging/src/k8s.io/legacy-cloud-providers/azure/retry:go_default_library", "//vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute:go_default_library", "//vendor/github.com/Azure/go-autorest/autorest:go_default_library", + "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library", "//vendor/github.com/golang/mock/gomock:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", ], diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmsizeclient/azure_vmsizeclient_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmsizeclient/azure_vmsizeclient_test.go index b10be264531..539376639e4 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmsizeclient/azure_vmsizeclient_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmsizeclient/azure_vmsizeclient_test.go @@ -21,20 +21,68 @@ package vmsizeclient import ( "bytes" "context" + "encoding/json" + "fmt" "io/ioutil" "net/http" "testing" + "time" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/to" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + "k8s.io/client-go/util/flowcontrol" azclients "k8s.io/legacy-cloud-providers/azure/clients" "k8s.io/legacy-cloud-providers/azure/clients/armclient" "k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient" + "k8s.io/legacy-cloud-providers/azure/retry" ) +func TestNew(t *testing.T) { + config := &azclients.ClientConfig{ + SubscriptionID: "sub", + ResourceManagerEndpoint: "endpoint", + Location: "eastus", + RateLimitConfig: &azclients.RateLimitConfig{ + CloudProviderRateLimit: true, + CloudProviderRateLimitQPS: 0.5, + CloudProviderRateLimitBucket: 1, + CloudProviderRateLimitQPSWrite: 0.5, + CloudProviderRateLimitBucketWrite: 1, + }, + Backoff: &retry.Backoff{Steps: 1}, + } + + vmsizeClient := New(config) + assert.Equal(t, "sub", vmsizeClient.subscriptionID) + assert.NotEmpty(t, vmsizeClient.rateLimiterReader) + assert.NotEmpty(t, vmsizeClient.rateLimiterWriter) +} + +func TestList(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/providers/Microsoft.Compute/locations/eastus/vmSizes" + armClient := mockarmclient.NewMockInterface(ctrl) + vmsizeList := []compute.VirtualMachineSize{getTestVMSize("Standard_D2s_v3"), getTestVMSize("Standard_D4s_v3"), getTestVMSize("Standard_D8s_v3")} + responseBody, err := json.Marshal(compute.VirtualMachineSizeListResult{Value: &vmsizeList}) + assert.Nil(t, err) + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return( + &http.Response{ + StatusCode: http.StatusOK, + Body: ioutil.NopCloser(bytes.NewReader(responseBody)), + }, nil).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + vmsizeClient := getTestVMSizeClient(armClient) + result, rerr := vmsizeClient.List(context.TODO(), "eastus") + assert.Nil(t, rerr) + assert.Equal(t, 3, len(*result.Value)) +} + func TestListNotFound(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -77,6 +125,69 @@ func TestListInternalError(t *testing.T) { assert.Equal(t, http.StatusInternalServerError, rerr.HTTPStatusCode) } +func TestListThrottle(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + resourceID := "/subscriptions/subscriptionID/providers/Microsoft.Compute/locations/eastus/vmSizes" + armClient := mockarmclient.NewMockInterface(ctrl) + response := &http.Response{ + StatusCode: http.StatusTooManyRequests, + Body: ioutil.NopCloser(bytes.NewReader([]byte("{}"))), + } + throttleErr := &retry.Error{ + HTTPStatusCode: http.StatusTooManyRequests, + RawError: fmt.Errorf("error"), + Retriable: true, + RetryAfter: time.Unix(100, 0), + } + armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, throttleErr).Times(1) + armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1) + vmsizeClient := getTestVMSizeClient(armClient) + expected := compute.VirtualMachineSizeListResult{Response: autorest.Response{}} + result, rerr := vmsizeClient.List(context.TODO(), "eastus") + assert.Equal(t, expected, result) + assert.NotNil(t, rerr) + assert.Equal(t, throttleErr, rerr) +} + +func TestListNeverRateLimiter(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmsizeListErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "read", "VMSizesList"), + Retriable: true, + } + + armClient := mockarmclient.NewMockInterface(ctrl) + expected := compute.VirtualMachineSizeListResult{Response: autorest.Response{}} + vmsizeClient := getTestVMSizeClientWithNeverRateLimiter(armClient) + result, rerr := vmsizeClient.List(context.TODO(), "eastus") + assert.Equal(t, expected, result) + assert.NotNil(t, rerr) + assert.Equal(t, vmsizeListErr, rerr) +} + +func TestListRetryAfterReader(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + vmsizeListErr := &retry.Error{ + RawError: fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "VMSizesList", "client throttled"), + Retriable: true, + RetryAfter: getFutureTime(), + } + + armClient := mockarmclient.NewMockInterface(ctrl) + expected := compute.VirtualMachineSizeListResult{Response: autorest.Response{}} + vmsizeClient := getTestVMSizeClientWithRetryAfterReader(armClient) + result, rerr := vmsizeClient.List(context.TODO(), "eastus") + assert.Equal(t, expected, result) + assert.NotNil(t, rerr) + assert.Equal(t, vmsizeListErr, rerr) +} + func getTestVMSizeClient(armClient armclient.Interface) *Client { rateLimiterReader, rateLimiterWriter := azclients.NewRateLimiter(&azclients.RateLimitConfig{}) return &Client{ @@ -86,3 +197,37 @@ func getTestVMSizeClient(armClient armclient.Interface) *Client { rateLimiterWriter: rateLimiterWriter, } } + +func getTestVMSize(name string) compute.VirtualMachineSize { + return compute.VirtualMachineSize{ + Name: to.StringPtr(name), + } +} + +func getTestVMSizeClientWithNeverRateLimiter(armClient armclient.Interface) *Client { + rateLimiterReader := flowcontrol.NewFakeNeverRateLimiter() + rateLimiterWriter := flowcontrol.NewFakeNeverRateLimiter() + return &Client{ + armClient: armClient, + subscriptionID: "subscriptionID", + rateLimiterReader: rateLimiterReader, + rateLimiterWriter: rateLimiterWriter, + } +} + +func getTestVMSizeClientWithRetryAfterReader(armClient armclient.Interface) *Client { + rateLimiterReader := flowcontrol.NewFakeAlwaysRateLimiter() + rateLimiterWriter := flowcontrol.NewFakeAlwaysRateLimiter() + return &Client{ + armClient: armClient, + subscriptionID: "subscriptionID", + rateLimiterReader: rateLimiterReader, + rateLimiterWriter: rateLimiterWriter, + RetryAfterReader: getFutureTime(), + RetryAfterWriter: getFutureTime(), + } +} + +func getFutureTime() time.Time { + return time.Unix(3000000000, 0) +}