Merge pull request #91086 from v-xuxin/enrich-unit-test

Azure: Enrich the unit tests for vmsizeclient and snapshotclient and s…

staging/src/k8s.io/legacy-cloud-providers/azure/clients/snapshotclient/BUILD

@@ -29,9 +29,11 @@ go_test(
     srcs = ["azure_snapshotclient_test.go"],
     embed = [":go_default_library"],
     deps = [
+        "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library",
         "//staging/src/k8s.io/legacy-cloud-providers/azure/clients:go_default_library",
         "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient:go_default_library",
         "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient:go_default_library",
+        "//staging/src/k8s.io/legacy-cloud-providers/azure/retry:go_default_library",
         "//vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute:go_default_library",
         "//vendor/github.com/Azure/go-autorest/autorest:go_default_library",
         "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library",

staging/src/k8s.io/legacy-cloud-providers/azure/clients/snapshotclient/azure_snapshotclient_test.go

@@ -26,17 +26,98 @@ import (
 	"io/ioutil"
 	"net/http"
 	"testing"
+	"time"
 
 	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute"
 	"github.com/Azure/go-autorest/autorest"
 	"github.com/Azure/go-autorest/autorest/to"
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/assert"
+
+	"k8s.io/client-go/util/flowcontrol"
 	azclients "k8s.io/legacy-cloud-providers/azure/clients"
 	"k8s.io/legacy-cloud-providers/azure/clients/armclient"
 	"k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient"
+	"k8s.io/legacy-cloud-providers/azure/retry"
 )
 
+func TestNew(t *testing.T) {
+	config := &azclients.ClientConfig{
+		SubscriptionID:          "sub",
+		ResourceManagerEndpoint: "endpoint",
+		Location:                "eastus",
+		RateLimitConfig: &azclients.RateLimitConfig{
+			CloudProviderRateLimit:            true,
+			CloudProviderRateLimitQPS:         0.5,
+			CloudProviderRateLimitBucket:      1,
+			CloudProviderRateLimitQPSWrite:    0.5,
+			CloudProviderRateLimitBucketWrite: 1,
+		},
+		Backoff: &retry.Backoff{Steps: 1},
+	}
+
+	snClient := New(config)
+	assert.Equal(t, "sub", snClient.subscriptionID)
+	assert.NotEmpty(t, snClient.rateLimiterReader)
+	assert.NotEmpty(t, snClient.rateLimiterWriter)
+}
+
+func TestGet(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/snapshots/sn1"
+	response := &http.Response{
+		StatusCode: http.StatusOK,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
+	}
+
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, nil).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+
+	expected := compute.Snapshot{Response: autorest.Response{Response: response}}
+	snClient := getTestSnapshotClient(armClient)
+	result, rerr := snClient.Get(context.TODO(), "rg", "sn1")
+	assert.Equal(t, expected, result)
+	assert.Nil(t, rerr)
+}
+
+func TestGetNeverRateLimiter(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	snGetErr := &retry.Error{
+		RawError:  fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "read", "SnapshotGet"),
+		Retriable: true,
+	}
+
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	snClient := getTestSnapshotClientWithNeverRateLimiter(armClient)
+	expected := compute.Snapshot{}
+	result, rerr := snClient.Get(context.TODO(), "rg", "sn1")
+	assert.Equal(t, expected, result)
+	assert.Equal(t, snGetErr, rerr)
+}
+
+func TestGetRetryAfterReader(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	snGetErr := &retry.Error{
+		RawError:   fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "SnapshotGet", "client throttled"),
+		Retriable:  true,
+		RetryAfter: getFutureTime(),
+	}
+
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	snClient := getTestSnapshotClientWithRetryAfterReader(armClient)
+	expected := compute.Snapshot{}
+	result, rerr := snClient.Get(context.TODO(), "rg", "sn1")
+	assert.Equal(t, expected, result)
+	assert.Equal(t, snGetErr, rerr)
+}
+
 func TestGetNotFound(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
@@ -58,6 +139,31 @@ func TestGetNotFound(t *testing.T) {
 	assert.Equal(t, http.StatusNotFound, rerr.HTTPStatusCode)
 }
 
+func TestGetThrottle(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/snapshots/sn1"
+	response := &http.Response{
+		StatusCode: http.StatusTooManyRequests,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
+	}
+	throttleErr := &retry.Error{
+		HTTPStatusCode: http.StatusTooManyRequests,
+		RawError:       fmt.Errorf("error"),
+		Retriable:      true,
+		RetryAfter:     time.Unix(100, 0),
+	}
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, throttleErr).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+
+	snClient := getTestSnapshotClient(armClient)
+	result, rerr := snClient.Get(context.TODO(), "rg", "sn1")
+	assert.Empty(t, result)
+	assert.Equal(t, throttleErr, rerr)
+}
+
 func TestGetInternalError(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
@@ -85,7 +191,7 @@ func TestListByResourceGroup(t *testing.T) {
 
 	resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/snapshots"
 	armClient := mockarmclient.NewMockInterface(ctrl)
-	snList := []compute.Snapshot{getTestSnapshot("sn1"), getTestSnapshot("pip2"), getTestSnapshot("pip3")}
+	snList := []compute.Snapshot{getTestSnapshot("sn1"), getTestSnapshot("sn2"), getTestSnapshot("sn3")}
 	responseBody, err := json.Marshal(compute.SnapshotList{Value: &snList})
 	assert.Nil(t, err)
 	armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(
@@ -101,6 +207,223 @@ func TestListByResourceGroup(t *testing.T) {
 	assert.Equal(t, 3, len(result))
 }
 
+func TestListByResourceGroupNotFound(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/snapshots"
+	response := &http.Response{
+		StatusCode: http.StatusNotFound,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
+	}
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, nil).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+
+	snClient := getTestSnapshotClient(armClient)
+	expected := []compute.Snapshot{}
+	result, rerr := snClient.ListByResourceGroup(context.TODO(), "rg")
+	assert.Equal(t, expected, result)
+	assert.NotNil(t, rerr)
+	assert.Equal(t, http.StatusNotFound, rerr.HTTPStatusCode)
+}
+
+func TestListByResourceGroupInternalError(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/snapshots"
+	response := &http.Response{
+		StatusCode: http.StatusInternalServerError,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
+	}
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, nil).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+
+	snClient := getTestSnapshotClient(armClient)
+	expected := []compute.Snapshot{}
+	result, rerr := snClient.ListByResourceGroup(context.TODO(), "rg")
+	assert.Equal(t, expected, result)
+	assert.NotNil(t, rerr)
+	assert.Equal(t, http.StatusInternalServerError, rerr.HTTPStatusCode)
+}
+
+func TestListByResourceGroupThrottle(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/snapshots"
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	response := &http.Response{
+		StatusCode: http.StatusTooManyRequests,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
+	}
+	throttleErr := &retry.Error{
+		HTTPStatusCode: http.StatusTooManyRequests,
+		RawError:       fmt.Errorf("error"),
+		Retriable:      true,
+		RetryAfter:     time.Unix(100, 0),
+	}
+	armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, throttleErr).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+	snClient := getTestSnapshotClient(armClient)
+	result, rerr := snClient.ListByResourceGroup(context.TODO(), "rg")
+	assert.Empty(t, result)
+	assert.NotNil(t, rerr)
+	assert.Equal(t, throttleErr, rerr)
+}
+
+func TestListByResourceGroupWithListResponderError(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/snapshots"
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	snList := []compute.Snapshot{getTestSnapshot("sn1"), getTestSnapshot("sn2"), getTestSnapshot("sn3")}
+	responseBody, err := json.Marshal(compute.SnapshotList{Value: &snList})
+	assert.Nil(t, err)
+	armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(
+		&http.Response{
+			StatusCode: http.StatusNotFound,
+			Body:       ioutil.NopCloser(bytes.NewReader(responseBody)),
+		}, nil).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+	snClient := getTestSnapshotClient(armClient)
+	result, rerr := snClient.ListByResourceGroup(context.TODO(), "rg")
+	assert.NotNil(t, rerr)
+	assert.Equal(t, 0, len(result))
+}
+
+func TestListByResourceGroupNeverRateLimiter(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	snListErr := &retry.Error{
+		RawError:  fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "read", "SnapshotListByResourceGroup"),
+		Retriable: true,
+	}
+
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	snClient := getTestSnapshotClientWithNeverRateLimiter(armClient)
+	result, rerr := snClient.ListByResourceGroup(context.TODO(), "rg")
+	assert.Equal(t, 0, len(result))
+	assert.NotNil(t, rerr)
+	assert.Equal(t, snListErr, rerr)
+}
+
+func TestListByResourceGroupRetryAfterReader(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	snListErr := &retry.Error{
+		RawError:   fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "SnapshotListByResourceGroup", "client throttled"),
+		Retriable:  true,
+		RetryAfter: getFutureTime(),
+	}
+
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	snClient := getTestSnapshotClientWithRetryAfterReader(armClient)
+	result, rerr := snClient.ListByResourceGroup(context.TODO(), "rg")
+	assert.Equal(t, 0, len(result))
+	assert.NotNil(t, rerr)
+	assert.Equal(t, snListErr, rerr)
+}
+
+func TestListNextResultsMultiPages(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	tests := []struct {
+		prepareErr error
+		sendErr    *retry.Error
+	}{
+		{
+			prepareErr: nil,
+			sendErr:    nil,
+		},
+		{
+			prepareErr: fmt.Errorf("error"),
+		},
+		{
+			sendErr: &retry.Error{RawError: fmt.Errorf("error")},
+		},
+	}
+
+	lastResult := compute.SnapshotList{
+		NextLink: to.StringPtr("next"),
+	}
+
+	for _, test := range tests {
+		armClient := mockarmclient.NewMockInterface(ctrl)
+		req := &http.Request{
+			Method: "GET",
+		}
+		armClient.EXPECT().PrepareGetRequest(gomock.Any(), gomock.Any()).Return(req, test.prepareErr)
+		if test.prepareErr == nil {
+			armClient.EXPECT().Send(gomock.Any(), req).Return(&http.Response{
+				StatusCode: http.StatusOK,
+				Body:       ioutil.NopCloser(bytes.NewReader([]byte(`{"foo":"bar"}`))),
+			}, test.sendErr)
+			armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any())
+		}
+
+		snClient := getTestSnapshotClient(armClient)
+		result, err := snClient.listNextResults(context.TODO(), lastResult)
+		if test.prepareErr != nil || test.sendErr != nil {
+			assert.NotNil(t, err)
+		} else {
+			assert.NoError(t, err)
+		}
+		if test.prepareErr != nil {
+			assert.Empty(t, result)
+		} else {
+			assert.NotEmpty(t, result)
+		}
+	}
+}
+
+func TestListNextResultsMultiPagesWithListResponderError(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	test := struct {
+		prepareErr error
+		sendErr    *retry.Error
+	}{
+		prepareErr: nil,
+		sendErr:    nil,
+	}
+
+	lastResult := compute.SnapshotList{
+		NextLink: to.StringPtr("next"),
+	}
+
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	req := &http.Request{
+		Method: "GET",
+	}
+	armClient.EXPECT().PrepareGetRequest(gomock.Any(), gomock.Any()).Return(req, test.prepareErr)
+	if test.prepareErr == nil {
+		armClient.EXPECT().Send(gomock.Any(), req).Return(&http.Response{
+			StatusCode: http.StatusNotFound,
+			Body:       ioutil.NopCloser(bytes.NewReader([]byte(`{"foo":"bar"}`))),
+		}, test.sendErr)
+		armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any())
+	}
+
+	response := &http.Response{
+		StatusCode: http.StatusNotFound,
+		Body:       ioutil.NopCloser(bytes.NewBuffer([]byte(`{"foo":"bar"}`))),
+	}
+	expected := compute.SnapshotList{}
+	expected.Response = autorest.Response{Response: response}
+	snClient := getTestSnapshotClient(armClient)
+	result, err := snClient.listNextResults(context.TODO(), lastResult)
+	assert.NotNil(t, err)
+	assert.Equal(t, expected, result)
+}
+
 func TestCreateOrUpdate(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
@@ -119,6 +442,84 @@ func TestCreateOrUpdate(t *testing.T) {
 	assert.Nil(t, rerr)
 }
 
+func TestCreateOrUpdateWithCreateOrUpdateResponderError(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+	sn := getTestSnapshot("sn1")
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	response := &http.Response{
+		StatusCode: http.StatusNotFound,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte(""))),
+	}
+	armClient.EXPECT().PutResource(gomock.Any(), to.String(sn.ID), sn).Return(response, nil).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+
+	snClient := getTestSnapshotClient(armClient)
+	rerr := snClient.CreateOrUpdate(context.TODO(), "rg", "sn1", sn)
+	assert.NotNil(t, rerr)
+}
+
+func TestCreateOrUpdateNeverRateLimiter(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	snCreateOrUpdateErr := &retry.Error{
+		RawError:  fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "write", "SnapshotCreateOrUpdate"),
+		Retriable: true,
+	}
+
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	snClient := getTestSnapshotClientWithNeverRateLimiter(armClient)
+	sn := getTestSnapshot("sn1")
+	rerr := snClient.CreateOrUpdate(context.TODO(), "rg", "sn1", sn)
+	assert.NotNil(t, rerr)
+	assert.Equal(t, snCreateOrUpdateErr, rerr)
+}
+
+func TestCreateOrUpdateRetryAfterReader(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	snCreateOrUpdateErr := &retry.Error{
+		RawError:   fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "SnapshotCreateOrUpdate", "client throttled"),
+		Retriable:  true,
+		RetryAfter: getFutureTime(),
+	}
+
+	sn := getTestSnapshot("sn1")
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	snClient := getTestSnapshotClientWithRetryAfterReader(armClient)
+	rerr := snClient.CreateOrUpdate(context.TODO(), "rg", "sn1", sn)
+	assert.NotNil(t, rerr)
+	assert.Equal(t, snCreateOrUpdateErr, rerr)
+}
+
+func TestCreateOrUpdateThrottle(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	response := &http.Response{
+		StatusCode: http.StatusTooManyRequests,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
+	}
+	throttleErr := &retry.Error{
+		HTTPStatusCode: http.StatusTooManyRequests,
+		RawError:       fmt.Errorf("error"),
+		Retriable:      true,
+		RetryAfter:     time.Unix(100, 0),
+	}
+
+	sn := getTestSnapshot("sn1")
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	armClient.EXPECT().PutResource(gomock.Any(), to.String(sn.ID), sn).Return(response, throttleErr).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+
+	snClient := getTestSnapshotClient(armClient)
+	rerr := snClient.CreateOrUpdate(context.TODO(), "rg", "sn1", sn)
+	assert.NotNil(t, rerr)
+	assert.Equal(t, throttleErr, rerr)
+}
+
 func TestDelete(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
@@ -132,6 +533,60 @@ func TestDelete(t *testing.T) {
 	assert.Nil(t, rerr)
 }
 
+func TestDeleteNeverRateLimiter(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	snDeleteErr := &retry.Error{
+		RawError:  fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "write", "SnapshotDelete"),
+		Retriable: true,
+	}
+
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	snClient := getTestSnapshotClientWithNeverRateLimiter(armClient)
+	rerr := snClient.Delete(context.TODO(), "rg", "sn1")
+	assert.NotNil(t, rerr)
+	assert.Equal(t, snDeleteErr, rerr)
+}
+
+func TestDeleteRetryAfterReader(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	snDeleteErr := &retry.Error{
+		RawError:   fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "SnapshotDelete", "client throttled"),
+		Retriable:  true,
+		RetryAfter: getFutureTime(),
+	}
+
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	snClient := getTestSnapshotClientWithRetryAfterReader(armClient)
+	rerr := snClient.Delete(context.TODO(), "rg", "sn1")
+	assert.NotNil(t, rerr)
+	assert.Equal(t, snDeleteErr, rerr)
+}
+
+func TestDeleteThrottle(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	throttleErr := &retry.Error{
+		HTTPStatusCode: http.StatusTooManyRequests,
+		RawError:       fmt.Errorf("error"),
+		Retriable:      true,
+		RetryAfter:     time.Unix(100, 0),
+	}
+
+	sn := getTestSnapshot("sn1")
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	armClient.EXPECT().DeleteResource(gomock.Any(), to.String(sn.ID), "").Return(throttleErr).Times(1)
+
+	snClient := getTestSnapshotClient(armClient)
+	rerr := snClient.Delete(context.TODO(), "rg", "sn1")
+	assert.NotNil(t, rerr)
+	assert.Equal(t, throttleErr, rerr)
+}
+
 func getTestSnapshot(name string) compute.Snapshot {
 	return compute.Snapshot{
 		ID:       to.StringPtr(fmt.Sprintf("/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Compute/snapshots/%s", name)),
@@ -149,3 +604,31 @@ func getTestSnapshotClient(armClient armclient.Interface) *Client {
 		rateLimiterWriter: rateLimiterWriter,
 	}
 }
+
+func getTestSnapshotClientWithNeverRateLimiter(armClient armclient.Interface) *Client {
+	rateLimiterReader := flowcontrol.NewFakeNeverRateLimiter()
+	rateLimiterWriter := flowcontrol.NewFakeNeverRateLimiter()
+	return &Client{
+		armClient:         armClient,
+		subscriptionID:    "subscriptionID",
+		rateLimiterReader: rateLimiterReader,
+		rateLimiterWriter: rateLimiterWriter,
+	}
+}
+
+func getTestSnapshotClientWithRetryAfterReader(armClient armclient.Interface) *Client {
+	rateLimiterReader := flowcontrol.NewFakeAlwaysRateLimiter()
+	rateLimiterWriter := flowcontrol.NewFakeAlwaysRateLimiter()
+	return &Client{
+		armClient:         armClient,
+		subscriptionID:    "subscriptionID",
+		rateLimiterReader: rateLimiterReader,
+		rateLimiterWriter: rateLimiterWriter,
+		RetryAfterReader:  getFutureTime(),
+		RetryAfterWriter:  getFutureTime(),
+	}
+}
+
+func getFutureTime() time.Time {
+	return time.Unix(3000000000, 0)
+}

staging/src/k8s.io/legacy-cloud-providers/azure/clients/subnetclient/BUILD

@@ -29,9 +29,11 @@ go_test(
     srcs = ["azure_subnetclient_test.go"],
     embed = [":go_default_library"],
     deps = [
+        "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library",
         "//staging/src/k8s.io/legacy-cloud-providers/azure/clients:go_default_library",
         "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient:go_default_library",
         "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient:go_default_library",
+        "//staging/src/k8s.io/legacy-cloud-providers/azure/retry:go_default_library",
         "//vendor/github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network:go_default_library",
         "//vendor/github.com/Azure/go-autorest/autorest:go_default_library",
         "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library",

staging/src/k8s.io/legacy-cloud-providers/azure/clients/subnetclient/azure_subnetclient_test.go

@@ -26,6 +26,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"testing"
+	"time"
 
 	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network"
 	"github.com/Azure/go-autorest/autorest"
@@ -33,11 +34,63 @@ import (
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/assert"
 
+	"k8s.io/client-go/util/flowcontrol"
 	azclients "k8s.io/legacy-cloud-providers/azure/clients"
 	"k8s.io/legacy-cloud-providers/azure/clients/armclient"
 	"k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient"
+	"k8s.io/legacy-cloud-providers/azure/retry"
 )
 
+func TestNew(t *testing.T) {
+	config := &azclients.ClientConfig{
+		SubscriptionID:          "sub",
+		ResourceManagerEndpoint: "endpoint",
+		Location:                "eastus",
+		RateLimitConfig: &azclients.RateLimitConfig{
+			CloudProviderRateLimit:            true,
+			CloudProviderRateLimitQPS:         0.5,
+			CloudProviderRateLimitBucket:      1,
+			CloudProviderRateLimitQPSWrite:    0.5,
+			CloudProviderRateLimitBucketWrite: 1,
+		},
+		Backoff: &retry.Backoff{Steps: 1},
+	}
+
+	subnetClient := New(config)
+	assert.Equal(t, "sub", subnetClient.subscriptionID)
+	assert.NotEmpty(t, subnetClient.rateLimiterReader)
+	assert.NotEmpty(t, subnetClient.rateLimiterWriter)
+}
+
+func TestGet(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnet/subnets/subnet1"
+	testSubnet := network.Subnet{
+		Name: to.StringPtr("subnet1"),
+	}
+	subnet, err := testSubnet.MarshalJSON()
+	assert.Nil(t, err)
+	response := &http.Response{
+		StatusCode: http.StatusOK,
+		Body:       ioutil.NopCloser(bytes.NewReader(subnet)),
+	}
+
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, nil).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+
+	expected := network.Subnet{
+		Response: autorest.Response{Response: response},
+		Name:     to.StringPtr("subnet1"),
+	}
+	subnetClient := getTestSubnetClient(armClient)
+	result, rerr := subnetClient.Get(context.TODO(), "rg", "vnet", "subnet1", "")
+	assert.Equal(t, expected, result)
+	assert.Nil(t, rerr)
+}
+
 func TestGetNotFound(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
@@ -80,6 +133,67 @@ func TestGetInternalError(t *testing.T) {
 	assert.Equal(t, http.StatusInternalServerError, rerr.HTTPStatusCode)
 }
 
+func TestGetNeverRateLimiter(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	subnetGetErr := &retry.Error{
+		RawError:  fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "read", "SubnetGet"),
+		Retriable: true,
+	}
+
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	subnetClient := getTestSubnetClientWithNeverRateLimiter(armClient)
+	expected := network.Subnet{}
+	result, rerr := subnetClient.Get(context.TODO(), "rg", "vnet", "subnet1", "")
+	assert.Equal(t, expected, result)
+	assert.Equal(t, subnetGetErr, rerr)
+}
+
+func TestGetRetryAfterReader(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	subnetGetErr := &retry.Error{
+		RawError:   fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "SubnetGet", "client throttled"),
+		Retriable:  true,
+		RetryAfter: getFutureTime(),
+	}
+
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	subnetClient := getTestSubnetClientWithRetryAfterReader(armClient)
+	expected := network.Subnet{}
+	result, rerr := subnetClient.Get(context.TODO(), "rg", "vnet", "subnet1", "")
+	assert.Equal(t, expected, result)
+	assert.Equal(t, subnetGetErr, rerr)
+}
+
+func TestGetThrottle(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnet/subnets/subnet1"
+	response := &http.Response{
+		StatusCode: http.StatusTooManyRequests,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
+	}
+	throttleErr := &retry.Error{
+		HTTPStatusCode: http.StatusTooManyRequests,
+		RawError:       fmt.Errorf("error"),
+		Retriable:      true,
+		RetryAfter:     time.Unix(100, 0),
+	}
+
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, throttleErr).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+
+	subnetClient := getTestSubnetClient(armClient)
+	result, rerr := subnetClient.Get(context.TODO(), "rg", "vnet", "subnet1", "")
+	assert.Empty(t, result)
+	assert.Equal(t, throttleErr, rerr)
+}
+
 func TestList(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
@@ -102,6 +216,224 @@ func TestList(t *testing.T) {
 	assert.Equal(t, 3, len(result))
 }
 
+func TestListNotFound(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnet/subnets"
+	response := &http.Response{
+		StatusCode: http.StatusNotFound,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
+	}
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, nil).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+
+	subnetClient := getTestSubnetClient(armClient)
+	expected := []network.Subnet{}
+	result, rerr := subnetClient.List(context.TODO(), "rg", "vnet")
+	assert.Equal(t, expected, result)
+	assert.NotNil(t, rerr)
+	assert.Equal(t, http.StatusNotFound, rerr.HTTPStatusCode)
+}
+
+func TestListInternalError(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnet/subnets"
+	response := &http.Response{
+		StatusCode: http.StatusInternalServerError,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
+	}
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, nil).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+
+	subnetClient := getTestSubnetClient(armClient)
+	expected := []network.Subnet{}
+	result, rerr := subnetClient.List(context.TODO(), "rg", "vnet")
+	assert.Equal(t, expected, result)
+	assert.NotNil(t, rerr)
+	assert.Equal(t, http.StatusInternalServerError, rerr.HTTPStatusCode)
+}
+
+func TestListThrottle(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnet/subnets"
+	response := &http.Response{
+		StatusCode: http.StatusTooManyRequests,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
+	}
+	throttleErr := &retry.Error{
+		HTTPStatusCode: http.StatusTooManyRequests,
+		RawError:       fmt.Errorf("error"),
+		Retriable:      true,
+		RetryAfter:     time.Unix(100, 0),
+	}
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, throttleErr).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+
+	subnetClient := getTestSubnetClient(armClient)
+	result, rerr := subnetClient.List(context.TODO(), "rg", "vnet")
+	assert.Empty(t, result)
+	assert.NotNil(t, rerr)
+	assert.Equal(t, throttleErr, rerr)
+}
+
+func TestListWithListResponderError(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	resourceID := "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnet/subnets"
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	subnetList := []network.Subnet{getTestSubnet("subnet1"), getTestSubnet("subnet2"), getTestSubnet("subnet3")}
+	responseBody, err := json.Marshal(network.SubnetListResult{Value: &subnetList})
+	assert.Nil(t, err)
+	armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(
+		&http.Response{
+			StatusCode: http.StatusNotFound,
+			Body:       ioutil.NopCloser(bytes.NewReader(responseBody)),
+		}, nil).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+	subnetClient := getTestSubnetClient(armClient)
+	result, rerr := subnetClient.List(context.TODO(), "rg", "vnet")
+	assert.NotNil(t, rerr)
+	assert.Equal(t, 0, len(result))
+}
+
+func TestListNeverRateLimiter(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	subnetListErr := &retry.Error{
+		RawError:  fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "read", "SubnetList"),
+		Retriable: true,
+	}
+
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	subnetClient := getTestSubnetClientWithNeverRateLimiter(armClient)
+	result, rerr := subnetClient.List(context.TODO(), "rg", "vnet")
+	assert.Equal(t, 0, len(result))
+	assert.NotNil(t, rerr)
+	assert.Equal(t, subnetListErr, rerr)
+}
+
+func TestListRetryAfterReader(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	subnetListErr := &retry.Error{
+		RawError:   fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "SubnetList", "client throttled"),
+		Retriable:  true,
+		RetryAfter: getFutureTime(),
+	}
+
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	subnetClient := getTestSubnetClientWithRetryAfterReader(armClient)
+	result, rerr := subnetClient.List(context.TODO(), "rg", "vnet")
+	assert.Equal(t, 0, len(result))
+	assert.NotNil(t, rerr)
+	assert.Equal(t, subnetListErr, rerr)
+}
+
+func TestListNextResultsMultiPages(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	tests := []struct {
+		prepareErr error
+		sendErr    *retry.Error
+	}{
+		{
+			prepareErr: nil,
+			sendErr:    nil,
+		},
+		{
+			prepareErr: fmt.Errorf("error"),
+		},
+		{
+			sendErr: &retry.Error{RawError: fmt.Errorf("error")},
+		},
+	}
+
+	lastResult := network.SubnetListResult{
+		NextLink: to.StringPtr("next"),
+	}
+
+	for _, test := range tests {
+		armClient := mockarmclient.NewMockInterface(ctrl)
+		req := &http.Request{
+			Method: "GET",
+		}
+		armClient.EXPECT().PrepareGetRequest(gomock.Any(), gomock.Any()).Return(req, test.prepareErr)
+		if test.prepareErr == nil {
+			armClient.EXPECT().Send(gomock.Any(), req).Return(&http.Response{
+				StatusCode: http.StatusOK,
+				Body:       ioutil.NopCloser(bytes.NewReader([]byte(`{"foo":"bar"}`))),
+			}, test.sendErr)
+			armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any())
+		}
+
+		subnetClient := getTestSubnetClient(armClient)
+		result, err := subnetClient.listNextResults(context.TODO(), lastResult)
+		if test.prepareErr != nil || test.sendErr != nil {
+			assert.NotNil(t, err)
+		} else {
+			assert.NoError(t, err)
+		}
+		if test.prepareErr != nil {
+			assert.Empty(t, result)
+		} else {
+			assert.NotEmpty(t, result)
+		}
+	}
+}
+
+func TestListNextResultsMultiPagesWithListResponderError(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	test := struct {
+		prepareErr error
+		sendErr    *retry.Error
+	}{
+		prepareErr: nil,
+		sendErr:    nil,
+	}
+
+	lastResult := network.SubnetListResult{
+		NextLink: to.StringPtr("next"),
+	}
+
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	req := &http.Request{
+		Method: "GET",
+	}
+	armClient.EXPECT().PrepareGetRequest(gomock.Any(), gomock.Any()).Return(req, test.prepareErr)
+	if test.prepareErr == nil {
+		armClient.EXPECT().Send(gomock.Any(), req).Return(&http.Response{
+			StatusCode: http.StatusNotFound,
+			Body:       ioutil.NopCloser(bytes.NewReader([]byte(`{"foo":"bar"}`))),
+		}, test.sendErr)
+		armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any())
+	}
+
+	response := &http.Response{
+		StatusCode: http.StatusNotFound,
+		Body:       ioutil.NopCloser(bytes.NewBuffer([]byte(`{"foo":"bar"}`))),
+	}
+	expected := network.SubnetListResult{}
+	expected.Response = autorest.Response{Response: response}
+	subnetClient := getTestSubnetClient(armClient)
+	result, err := subnetClient.listNextResults(context.TODO(), lastResult)
+	assert.NotNil(t, err)
+	assert.Equal(t, expected, result)
+}
+
 func TestCreateOrUpdate(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
@@ -120,6 +452,84 @@ func TestCreateOrUpdate(t *testing.T) {
 	assert.Nil(t, rerr)
 }
 
+func TestCreateOrUpdateWithCreateOrUpdateResponderError(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+	subnet := getTestSubnet("subnet1")
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	response := &http.Response{
+		StatusCode: http.StatusNotFound,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte(""))),
+	}
+	armClient.EXPECT().PutResource(gomock.Any(), to.String(subnet.ID), subnet).Return(response, nil).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+
+	subnetClient := getTestSubnetClient(armClient)
+	rerr := subnetClient.CreateOrUpdate(context.TODO(), "rg", "vnet", "subnet1", subnet)
+	assert.NotNil(t, rerr)
+}
+
+func TestCreateOrUpdateNeverRateLimiter(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	subnetCreateOrUpdateErr := &retry.Error{
+		RawError:  fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "write", "SubnetCreateOrUpdate"),
+		Retriable: true,
+	}
+
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	subnetClient := getTestSubnetClientWithNeverRateLimiter(armClient)
+	subnet := getTestSubnet("subnet1")
+	rerr := subnetClient.CreateOrUpdate(context.TODO(), "rg", "vnet", "subnet1", subnet)
+	assert.NotNil(t, rerr)
+	assert.Equal(t, subnetCreateOrUpdateErr, rerr)
+}
+
+func TestCreateOrUpdateRetryAfterReader(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	subnetCreateOrUpdateErr := &retry.Error{
+		RawError:   fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "SubnetCreateOrUpdate", "client throttled"),
+		Retriable:  true,
+		RetryAfter: getFutureTime(),
+	}
+
+	subnet := getTestSubnet("subnet1")
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	subnetClient := getTestSubnetClientWithRetryAfterReader(armClient)
+	rerr := subnetClient.CreateOrUpdate(context.TODO(), "rg", "vnet", "subnet1", subnet)
+	assert.NotNil(t, rerr)
+	assert.Equal(t, subnetCreateOrUpdateErr, rerr)
+}
+
+func TestCreateOrUpdateThrottle(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	response := &http.Response{
+		StatusCode: http.StatusTooManyRequests,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
+	}
+	throttleErr := &retry.Error{
+		HTTPStatusCode: http.StatusTooManyRequests,
+		RawError:       fmt.Errorf("error"),
+		Retriable:      true,
+		RetryAfter:     time.Unix(100, 0),
+	}
+
+	subnet := getTestSubnet("subnet1")
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	armClient.EXPECT().PutResource(gomock.Any(), to.String(subnet.ID), subnet).Return(response, throttleErr).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+
+	subnetClient := getTestSubnetClient(armClient)
+	rerr := subnetClient.CreateOrUpdate(context.TODO(), "rg", "vnet", "subnet1", subnet)
+	assert.NotNil(t, rerr)
+	assert.Equal(t, throttleErr, rerr)
+}
+
 func TestDelete(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
@@ -133,6 +543,60 @@ func TestDelete(t *testing.T) {
 	assert.Nil(t, rerr)
 }
 
+func TestDeleteNeverRateLimiter(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	subnetDeleteErr := &retry.Error{
+		RawError:  fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "write", "SubnetDelete"),
+		Retriable: true,
+	}
+
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	subnetClient := getTestSubnetClientWithNeverRateLimiter(armClient)
+	rerr := subnetClient.Delete(context.TODO(), "rg", "vnet", "subnet1")
+	assert.NotNil(t, rerr)
+	assert.Equal(t, subnetDeleteErr, rerr)
+}
+
+func TestDeleteRetryAfterReader(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	subnetDeleteErr := &retry.Error{
+		RawError:   fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "SubnetDelete", "client throttled"),
+		Retriable:  true,
+		RetryAfter: getFutureTime(),
+	}
+
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	subnetClient := getTestSubnetClientWithRetryAfterReader(armClient)
+	rerr := subnetClient.Delete(context.TODO(), "rg", "vnet", "subnet1")
+	assert.NotNil(t, rerr)
+	assert.Equal(t, subnetDeleteErr, rerr)
+}
+
+func TestDeleteThrottle(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	throttleErr := &retry.Error{
+		HTTPStatusCode: http.StatusTooManyRequests,
+		RawError:       fmt.Errorf("error"),
+		Retriable:      true,
+		RetryAfter:     time.Unix(100, 0),
+	}
+
+	subnet := getTestSubnet("subnet1")
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	armClient.EXPECT().DeleteResource(gomock.Any(), to.String(subnet.ID), "").Return(throttleErr).Times(1)
+
+	subnetClient := getTestSubnetClient(armClient)
+	rerr := subnetClient.Delete(context.TODO(), "rg", "vnet", "subnet1")
+	assert.NotNil(t, rerr)
+	assert.Equal(t, throttleErr, rerr)
+}
+
 func getTestSubnet(name string) network.Subnet {
 	return network.Subnet{
 		ID:   to.StringPtr(fmt.Sprintf("/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnet/subnets/%s", name)),
@@ -149,3 +613,31 @@ func getTestSubnetClient(armClient armclient.Interface) *Client {
 		rateLimiterWriter: rateLimiterWriter,
 	}
 }
+
+func getTestSubnetClientWithNeverRateLimiter(armClient armclient.Interface) *Client {
+	rateLimiterReader := flowcontrol.NewFakeNeverRateLimiter()
+	rateLimiterWriter := flowcontrol.NewFakeNeverRateLimiter()
+	return &Client{
+		armClient:         armClient,
+		subscriptionID:    "subscriptionID",
+		rateLimiterReader: rateLimiterReader,
+		rateLimiterWriter: rateLimiterWriter,
+	}
+}
+
+func getTestSubnetClientWithRetryAfterReader(armClient armclient.Interface) *Client {
+	rateLimiterReader := flowcontrol.NewFakeAlwaysRateLimiter()
+	rateLimiterWriter := flowcontrol.NewFakeAlwaysRateLimiter()
+	return &Client{
+		armClient:         armClient,
+		subscriptionID:    "subscriptionID",
+		rateLimiterReader: rateLimiterReader,
+		rateLimiterWriter: rateLimiterWriter,
+		RetryAfterReader:  getFutureTime(),
+		RetryAfterWriter:  getFutureTime(),
+	}
+}
+
+func getFutureTime() time.Time {
+	return time.Unix(3000000000, 0)
+}

staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmsizeclient/BUILD

@@ -28,11 +28,14 @@ go_test(
     srcs = ["azure_vmsizeclient_test.go"],
     embed = [":go_default_library"],
     deps = [
+        "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library",
         "//staging/src/k8s.io/legacy-cloud-providers/azure/clients:go_default_library",
         "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient:go_default_library",
         "//staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient:go_default_library",
+        "//staging/src/k8s.io/legacy-cloud-providers/azure/retry:go_default_library",
         "//vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute:go_default_library",
         "//vendor/github.com/Azure/go-autorest/autorest:go_default_library",
+        "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library",
         "//vendor/github.com/golang/mock/gomock:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
     ],

staging/src/k8s.io/legacy-cloud-providers/azure/clients/vmsizeclient/azure_vmsizeclient_test.go

@@ -21,20 +21,68 @@ package vmsizeclient
 import (
 	"bytes"
 	"context"
+	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"testing"
+	"time"
 
 	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute"
 	"github.com/Azure/go-autorest/autorest"
+	"github.com/Azure/go-autorest/autorest/to"
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/assert"
 
+	"k8s.io/client-go/util/flowcontrol"
 	azclients "k8s.io/legacy-cloud-providers/azure/clients"
 	"k8s.io/legacy-cloud-providers/azure/clients/armclient"
 	"k8s.io/legacy-cloud-providers/azure/clients/armclient/mockarmclient"
+	"k8s.io/legacy-cloud-providers/azure/retry"
 )
 
+func TestNew(t *testing.T) {
+	config := &azclients.ClientConfig{
+		SubscriptionID:          "sub",
+		ResourceManagerEndpoint: "endpoint",
+		Location:                "eastus",
+		RateLimitConfig: &azclients.RateLimitConfig{
+			CloudProviderRateLimit:            true,
+			CloudProviderRateLimitQPS:         0.5,
+			CloudProviderRateLimitBucket:      1,
+			CloudProviderRateLimitQPSWrite:    0.5,
+			CloudProviderRateLimitBucketWrite: 1,
+		},
+		Backoff: &retry.Backoff{Steps: 1},
+	}
+
+	vmsizeClient := New(config)
+	assert.Equal(t, "sub", vmsizeClient.subscriptionID)
+	assert.NotEmpty(t, vmsizeClient.rateLimiterReader)
+	assert.NotEmpty(t, vmsizeClient.rateLimiterWriter)
+}
+
+func TestList(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	resourceID := "/subscriptions/subscriptionID/providers/Microsoft.Compute/locations/eastus/vmSizes"
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	vmsizeList := []compute.VirtualMachineSize{getTestVMSize("Standard_D2s_v3"), getTestVMSize("Standard_D4s_v3"), getTestVMSize("Standard_D8s_v3")}
+	responseBody, err := json.Marshal(compute.VirtualMachineSizeListResult{Value: &vmsizeList})
+	assert.Nil(t, err)
+	armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(
+		&http.Response{
+			StatusCode: http.StatusOK,
+			Body:       ioutil.NopCloser(bytes.NewReader(responseBody)),
+		}, nil).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+	vmsizeClient := getTestVMSizeClient(armClient)
+	result, rerr := vmsizeClient.List(context.TODO(), "eastus")
+	assert.Nil(t, rerr)
+	assert.Equal(t, 3, len(*result.Value))
+}
+
 func TestListNotFound(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
@@ -77,6 +125,69 @@ func TestListInternalError(t *testing.T) {
 	assert.Equal(t, http.StatusInternalServerError, rerr.HTTPStatusCode)
 }
 
+func TestListThrottle(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	resourceID := "/subscriptions/subscriptionID/providers/Microsoft.Compute/locations/eastus/vmSizes"
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	response := &http.Response{
+		StatusCode: http.StatusTooManyRequests,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
+	}
+	throttleErr := &retry.Error{
+		HTTPStatusCode: http.StatusTooManyRequests,
+		RawError:       fmt.Errorf("error"),
+		Retriable:      true,
+		RetryAfter:     time.Unix(100, 0),
+	}
+	armClient.EXPECT().GetResource(gomock.Any(), resourceID, "").Return(response, throttleErr).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+	vmsizeClient := getTestVMSizeClient(armClient)
+	expected := compute.VirtualMachineSizeListResult{Response: autorest.Response{}}
+	result, rerr := vmsizeClient.List(context.TODO(), "eastus")
+	assert.Equal(t, expected, result)
+	assert.NotNil(t, rerr)
+	assert.Equal(t, throttleErr, rerr)
+}
+
+func TestListNeverRateLimiter(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	vmsizeListErr := &retry.Error{
+		RawError:  fmt.Errorf("azure cloud provider rate limited(%s) for operation %q", "read", "VMSizesList"),
+		Retriable: true,
+	}
+
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	expected := compute.VirtualMachineSizeListResult{Response: autorest.Response{}}
+	vmsizeClient := getTestVMSizeClientWithNeverRateLimiter(armClient)
+	result, rerr := vmsizeClient.List(context.TODO(), "eastus")
+	assert.Equal(t, expected, result)
+	assert.NotNil(t, rerr)
+	assert.Equal(t, vmsizeListErr, rerr)
+}
+
+func TestListRetryAfterReader(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	vmsizeListErr := &retry.Error{
+		RawError:   fmt.Errorf("azure cloud provider throttled for operation %s with reason %q", "VMSizesList", "client throttled"),
+		Retriable:  true,
+		RetryAfter: getFutureTime(),
+	}
+
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	expected := compute.VirtualMachineSizeListResult{Response: autorest.Response{}}
+	vmsizeClient := getTestVMSizeClientWithRetryAfterReader(armClient)
+	result, rerr := vmsizeClient.List(context.TODO(), "eastus")
+	assert.Equal(t, expected, result)
+	assert.NotNil(t, rerr)
+	assert.Equal(t, vmsizeListErr, rerr)
+}
+
 func getTestVMSizeClient(armClient armclient.Interface) *Client {
 	rateLimiterReader, rateLimiterWriter := azclients.NewRateLimiter(&azclients.RateLimitConfig{})
 	return &Client{
@@ -86,3 +197,37 @@ func getTestVMSizeClient(armClient armclient.Interface) *Client {
 		rateLimiterWriter: rateLimiterWriter,
 	}
 }
+
+func getTestVMSize(name string) compute.VirtualMachineSize {
+	return compute.VirtualMachineSize{
+		Name: to.StringPtr(name),
+	}
+}
+
+func getTestVMSizeClientWithNeverRateLimiter(armClient armclient.Interface) *Client {
+	rateLimiterReader := flowcontrol.NewFakeNeverRateLimiter()
+	rateLimiterWriter := flowcontrol.NewFakeNeverRateLimiter()
+	return &Client{
+		armClient:         armClient,
+		subscriptionID:    "subscriptionID",
+		rateLimiterReader: rateLimiterReader,
+		rateLimiterWriter: rateLimiterWriter,
+	}
+}
+
+func getTestVMSizeClientWithRetryAfterReader(armClient armclient.Interface) *Client {
+	rateLimiterReader := flowcontrol.NewFakeAlwaysRateLimiter()
+	rateLimiterWriter := flowcontrol.NewFakeAlwaysRateLimiter()
+	return &Client{
+		armClient:         armClient,
+		subscriptionID:    "subscriptionID",
+		rateLimiterReader: rateLimiterReader,
+		rateLimiterWriter: rateLimiterWriter,
+		RetryAfterReader:  getFutureTime(),
+		RetryAfterWriter:  getFutureTime(),
+	}
+}
+
+func getFutureTime() time.Time {
+	return time.Unix(3000000000, 0)
+}