Add retry Error definition

staging/src/k8s.io/legacy-cloud-providers/azure/retry/BUILD

@@ -0,0 +1,31 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["azure_error.go"],
+    importmap = "k8s.io/kubernetes/vendor/k8s.io/legacy-cloud-providers/azure/retry",
+    importpath = "k8s.io/legacy-cloud-providers/azure/retry",
+    visibility = ["//visibility:public"],
+    deps = ["//vendor/k8s.io/klog:go_default_library"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["azure_error_test.go"],
+    embed = [":go_default_library"],
+    deps = ["//vendor/github.com/stretchr/testify/assert:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

staging/src/k8s.io/legacy-cloud-providers/azure/retry/azure_error.go

@@ -0,0 +1,198 @@
+// +build !providerless
+
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package retry
+
+import (
+	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"k8s.io/klog"
+)
+
+var (
+	// The function to get current time.
+	now = time.Now
+)
+
+// Error indicates an error returned by Azure APIs.
+type Error struct {
+	// Retriable indicates whether the request is retriable.
+	Retriable bool
+	// HTTPStatusCode indicates the response HTTP status code.
+	HTTPStatusCode int
+	// RetryAfter indicates the time when the request should retry after throttling.
+	// A throttled request is retriable.
+	RetryAfter time.Time
+	// RetryAfter indicates the raw error from API.
+	RawError error
+}
+
+// Error returns the error.
+// Note that Error doesn't implement error interface because (nil *Error) != (nil error).
+func (err *Error) Error() error {
+	if err == nil {
+		return nil
+	}
+
+	return fmt.Errorf("Retriable: %v, RetryAfter: %s, HTTPStatusCode: %d, RawError: %v",
+		err.Retriable, err.RetryAfter.String(), err.HTTPStatusCode, err.RawError)
+}
+
+// NewError creates a new Error.
+func NewError(retriable bool, err error) *Error {
+	return &Error{
+		Retriable: retriable,
+		RawError:  err,
+	}
+}
+
+// GetRetriableError gets new retriable Error.
+func GetRetriableError(err error) *Error {
+	return &Error{
+		Retriable: true,
+		RawError:  err,
+	}
+}
+
+// GetError gets a new Error based on resp and error.
+func GetError(resp *http.Response, err error) *Error {
+	if err == nil && resp == nil {
+		return nil
+	}
+
+	if err == nil && resp != nil && isSuccessHTTPResponse(resp) {
+		// HTTP 2xx suggests a successful response
+		return nil
+	}
+
+	retryAfter := time.Time{}
+	if retryAfterDuration := getRetryAfter(resp); retryAfterDuration != 0 {
+		retryAfter = now().Add(retryAfterDuration)
+	}
+	rawError := err
+	if err == nil && resp != nil {
+		rawError = fmt.Errorf("HTTP response: %v", resp.StatusCode)
+	}
+	return &Error{
+		RawError:       rawError,
+		RetryAfter:     retryAfter,
+		Retriable:      shouldRetryHTTPRequest(resp, err),
+		HTTPStatusCode: getHTTPStatusCode(resp),
+	}
+}
+
+// isSuccessHTTPResponse determines if the response from an HTTP request suggests success
+func isSuccessHTTPResponse(resp *http.Response) bool {
+	if resp == nil {
+		return false
+	}
+
+	// HTTP 2xx suggests a successful response
+	if 199 < resp.StatusCode && resp.StatusCode < 300 {
+		return true
+	}
+
+	return false
+}
+
+func getHTTPStatusCode(resp *http.Response) int {
+	if resp == nil {
+		return -1
+	}
+
+	return resp.StatusCode
+}
+
+// shouldRetryHTTPRequest determines if the request is retriable.
+func shouldRetryHTTPRequest(resp *http.Response, err error) bool {
+	if resp != nil {
+		// HTTP 412 (StatusPreconditionFailed) means etag mismatch, hence we shouldn't retry.
+		if resp.StatusCode == http.StatusPreconditionFailed {
+			return false
+		}
+
+		// HTTP 4xx (except 412) or 5xx suggests we should retry.
+		if 399 < resp.StatusCode && resp.StatusCode < 600 {
+			return true
+		}
+	}
+
+	if err != nil {
+		return true
+	}
+
+	return false
+}
+
+// getRetryAfter gets the retryAfter from http response.
+// The value of Retry-After can be either the number of seconds or a date in RFC1123 format.
+func getRetryAfter(resp *http.Response) time.Duration {
+	if resp == nil {
+		return 0
+	}
+
+	ra := resp.Header.Get("Retry-After")
+	if ra == "" {
+		return 0
+	}
+
+	var dur time.Duration
+	if retryAfter, _ := strconv.Atoi(ra); retryAfter > 0 {
+		dur = time.Duration(retryAfter) * time.Second
+	} else if t, err := time.Parse(time.RFC1123, ra); err == nil {
+		dur = t.Sub(now())
+	}
+	return dur
+}
+
+// GetStatusNotFoundAndForbiddenIgnoredError gets an error with StatusNotFound and StatusForbidden ignored.
+// It is only used in DELETE operations.
+func GetStatusNotFoundAndForbiddenIgnoredError(resp *http.Response, err error) *Error {
+	rerr := GetError(resp, err)
+	if rerr == nil {
+		return nil
+	}
+
+	// Returns nil when it is StatusNotFound error.
+	if rerr.HTTPStatusCode == http.StatusNotFound {
+		klog.V(3).Infof("Ignoring StatusNotFound error: %v", rerr)
+		return nil
+	}
+
+	// Returns nil if the status code is StatusForbidden.
+	// This happens when AuthorizationFailed is reported from Azure API.
+	if rerr.HTTPStatusCode == http.StatusForbidden {
+		klog.V(3).Infof("Ignoring StatusForbidden error: %v", rerr)
+		return nil
+	}
+
+	return rerr
+}
+
+// IsErrorRetriable returns true if the error is retriable.
+func IsErrorRetriable(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	return strings.Contains(err.Error(), "Retriable: true")
+}

staging/src/k8s.io/legacy-cloud-providers/azure/retry/azure_error_test.go

@@ -0,0 +1,253 @@
+// +build !providerless
+
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package retry
+
+import (
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetError(t *testing.T) {
+	now = func() time.Time {
+		return time.Time{}
+	}
+
+	tests := []struct {
+		code       int
+		retryAfter int
+		err        error
+		expected   *Error
+	}{
+		{
+			code:     http.StatusOK,
+			expected: nil,
+		},
+		{
+			code: http.StatusOK,
+			err:  fmt.Errorf("some error"),
+			expected: &Error{
+				Retriable:      true,
+				HTTPStatusCode: http.StatusOK,
+				RawError:       fmt.Errorf("some error"),
+			},
+		},
+		{
+			code: http.StatusBadRequest,
+			expected: &Error{
+				Retriable:      true,
+				HTTPStatusCode: http.StatusBadRequest,
+				RawError:       fmt.Errorf("HTTP response: 400"),
+			},
+		},
+		{
+			code: http.StatusInternalServerError,
+			expected: &Error{
+				Retriable:      true,
+				HTTPStatusCode: http.StatusInternalServerError,
+				RawError:       fmt.Errorf("HTTP response: 500"),
+			},
+		},
+		{
+			code: http.StatusSeeOther,
+			err:  fmt.Errorf("some error"),
+			expected: &Error{
+				Retriable:      true,
+				HTTPStatusCode: http.StatusSeeOther,
+				RawError:       fmt.Errorf("some error"),
+			},
+		},
+		{
+			code:       http.StatusTooManyRequests,
+			retryAfter: 100,
+			expected: &Error{
+				Retriable:      true,
+				HTTPStatusCode: http.StatusTooManyRequests,
+				RetryAfter:     now().Add(100 * time.Second),
+				RawError:       fmt.Errorf("HTTP response: 429"),
+			},
+		},
+	}
+
+	for _, test := range tests {
+		resp := &http.Response{
+			StatusCode: test.code,
+			Header:     http.Header{},
+		}
+		if test.retryAfter != 0 {
+			resp.Header.Add("Retry-After", fmt.Sprintf("%d", test.retryAfter))
+		}
+		rerr := GetError(resp, test.err)
+		assert.Equal(t, test.expected, rerr)
+	}
+}
+
+func TestGetStatusNotFoundAndForbiddenIgnoredError(t *testing.T) {
+	now = func() time.Time {
+		return time.Time{}
+	}
+
+	tests := []struct {
+		code       int
+		retryAfter int
+		err        error
+		expected   *Error
+	}{
+		{
+			code:     http.StatusOK,
+			expected: nil,
+		},
+		{
+			code:     http.StatusNotFound,
+			expected: nil,
+		},
+		{
+			code:     http.StatusForbidden,
+			expected: nil,
+		},
+		{
+			code: http.StatusOK,
+			err:  fmt.Errorf("some error"),
+			expected: &Error{
+				Retriable:      true,
+				HTTPStatusCode: http.StatusOK,
+				RawError:       fmt.Errorf("some error"),
+			},
+		},
+		{
+			code: http.StatusBadRequest,
+			expected: &Error{
+				Retriable:      true,
+				HTTPStatusCode: http.StatusBadRequest,
+				RawError:       fmt.Errorf("HTTP response: 400"),
+			},
+		},
+		{
+			code: http.StatusInternalServerError,
+			expected: &Error{
+				Retriable:      true,
+				HTTPStatusCode: http.StatusInternalServerError,
+				RawError:       fmt.Errorf("HTTP response: 500"),
+			},
+		},
+		{
+			code: http.StatusSeeOther,
+			err:  fmt.Errorf("some error"),
+			expected: &Error{
+				Retriable:      true,
+				HTTPStatusCode: http.StatusSeeOther,
+				RawError:       fmt.Errorf("some error"),
+			},
+		},
+		{
+			code:       http.StatusTooManyRequests,
+			retryAfter: 100,
+			expected: &Error{
+				Retriable:      true,
+				HTTPStatusCode: http.StatusTooManyRequests,
+				RetryAfter:     now().Add(100 * time.Second),
+				RawError:       fmt.Errorf("HTTP response: 429"),
+			},
+		},
+	}
+
+	for _, test := range tests {
+		resp := &http.Response{
+			StatusCode: test.code,
+			Header:     http.Header{},
+		}
+		if test.retryAfter != 0 {
+			resp.Header.Add("Retry-After", fmt.Sprintf("%d", test.retryAfter))
+		}
+		rerr := GetStatusNotFoundAndForbiddenIgnoredError(resp, test.err)
+		assert.Equal(t, test.expected, rerr)
+	}
+}
+
+func TestShouldRetryHTTPRequest(t *testing.T) {
+	tests := []struct {
+		code     int
+		err      error
+		expected bool
+	}{
+		{
+			code:     http.StatusBadRequest,
+			expected: true,
+		},
+		{
+			code:     http.StatusInternalServerError,
+			expected: true,
+		},
+		{
+			code:     http.StatusOK,
+			err:      fmt.Errorf("some error"),
+			expected: true,
+		},
+		{
+			code:     http.StatusOK,
+			expected: false,
+		},
+		{
+			code:     399,
+			expected: false,
+		},
+	}
+	for _, test := range tests {
+		resp := &http.Response{
+			StatusCode: test.code,
+		}
+		res := shouldRetryHTTPRequest(resp, test.err)
+		if res != test.expected {
+			t.Errorf("expected: %v, saw: %v", test.expected, res)
+		}
+	}
+}
+
+func TestIsSuccessResponse(t *testing.T) {
+	tests := []struct {
+		code     int
+		expected bool
+	}{
+		{
+			code:     http.StatusNotFound,
+			expected: false,
+		},
+		{
+			code:     http.StatusInternalServerError,
+			expected: false,
+		},
+		{
+			code:     http.StatusOK,
+			expected: true,
+		},
+	}
+
+	for _, test := range tests {
+		resp := http.Response{
+			StatusCode: test.code,
+		}
+		res := isSuccessHTTPResponse(&resp)
+		if res != test.expected {
+			t.Errorf("expected: %v, saw: %v", test.expected, res)
+		}
+	}
+}