diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/retry/BUILD b/staging/src/k8s.io/legacy-cloud-providers/azure/retry/BUILD new file mode 100644 index 00000000000..e3ecf81dfb1 --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/retry/BUILD @@ -0,0 +1,31 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["azure_error.go"], + importmap = "k8s.io/kubernetes/vendor/k8s.io/legacy-cloud-providers/azure/retry", + importpath = "k8s.io/legacy-cloud-providers/azure/retry", + visibility = ["//visibility:public"], + deps = ["//vendor/k8s.io/klog:go_default_library"], +) + +go_test( + name = "go_default_test", + srcs = ["azure_error_test.go"], + embed = [":go_default_library"], + deps = ["//vendor/github.com/stretchr/testify/assert:go_default_library"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/retry/azure_error.go b/staging/src/k8s.io/legacy-cloud-providers/azure/retry/azure_error.go new file mode 100644 index 00000000000..03d043cb9d9 --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/retry/azure_error.go @@ -0,0 +1,198 @@ +// +build !providerless + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package retry + +import ( + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "k8s.io/klog" +) + +var ( + // The function to get current time. + now = time.Now +) + +// Error indicates an error returned by Azure APIs. +type Error struct { + // Retriable indicates whether the request is retriable. + Retriable bool + // HTTPStatusCode indicates the response HTTP status code. + HTTPStatusCode int + // RetryAfter indicates the time when the request should retry after throttling. + // A throttled request is retriable. + RetryAfter time.Time + // RetryAfter indicates the raw error from API. + RawError error +} + +// Error returns the error. +// Note that Error doesn't implement error interface because (nil *Error) != (nil error). +func (err *Error) Error() error { + if err == nil { + return nil + } + + return fmt.Errorf("Retriable: %v, RetryAfter: %s, HTTPStatusCode: %d, RawError: %v", + err.Retriable, err.RetryAfter.String(), err.HTTPStatusCode, err.RawError) +} + +// NewError creates a new Error. +func NewError(retriable bool, err error) *Error { + return &Error{ + Retriable: retriable, + RawError: err, + } +} + +// GetRetriableError gets new retriable Error. +func GetRetriableError(err error) *Error { + return &Error{ + Retriable: true, + RawError: err, + } +} + +// GetError gets a new Error based on resp and error. +func GetError(resp *http.Response, err error) *Error { + if err == nil && resp == nil { + return nil + } + + if err == nil && resp != nil && isSuccessHTTPResponse(resp) { + // HTTP 2xx suggests a successful response + return nil + } + + retryAfter := time.Time{} + if retryAfterDuration := getRetryAfter(resp); retryAfterDuration != 0 { + retryAfter = now().Add(retryAfterDuration) + } + rawError := err + if err == nil && resp != nil { + rawError = fmt.Errorf("HTTP response: %v", resp.StatusCode) + } + return &Error{ + RawError: rawError, + RetryAfter: retryAfter, + Retriable: shouldRetryHTTPRequest(resp, err), + HTTPStatusCode: getHTTPStatusCode(resp), + } +} + +// isSuccessHTTPResponse determines if the response from an HTTP request suggests success +func isSuccessHTTPResponse(resp *http.Response) bool { + if resp == nil { + return false + } + + // HTTP 2xx suggests a successful response + if 199 < resp.StatusCode && resp.StatusCode < 300 { + return true + } + + return false +} + +func getHTTPStatusCode(resp *http.Response) int { + if resp == nil { + return -1 + } + + return resp.StatusCode +} + +// shouldRetryHTTPRequest determines if the request is retriable. +func shouldRetryHTTPRequest(resp *http.Response, err error) bool { + if resp != nil { + // HTTP 412 (StatusPreconditionFailed) means etag mismatch, hence we shouldn't retry. + if resp.StatusCode == http.StatusPreconditionFailed { + return false + } + + // HTTP 4xx (except 412) or 5xx suggests we should retry. + if 399 < resp.StatusCode && resp.StatusCode < 600 { + return true + } + } + + if err != nil { + return true + } + + return false +} + +// getRetryAfter gets the retryAfter from http response. +// The value of Retry-After can be either the number of seconds or a date in RFC1123 format. +func getRetryAfter(resp *http.Response) time.Duration { + if resp == nil { + return 0 + } + + ra := resp.Header.Get("Retry-After") + if ra == "" { + return 0 + } + + var dur time.Duration + if retryAfter, _ := strconv.Atoi(ra); retryAfter > 0 { + dur = time.Duration(retryAfter) * time.Second + } else if t, err := time.Parse(time.RFC1123, ra); err == nil { + dur = t.Sub(now()) + } + return dur +} + +// GetStatusNotFoundAndForbiddenIgnoredError gets an error with StatusNotFound and StatusForbidden ignored. +// It is only used in DELETE operations. +func GetStatusNotFoundAndForbiddenIgnoredError(resp *http.Response, err error) *Error { + rerr := GetError(resp, err) + if rerr == nil { + return nil + } + + // Returns nil when it is StatusNotFound error. + if rerr.HTTPStatusCode == http.StatusNotFound { + klog.V(3).Infof("Ignoring StatusNotFound error: %v", rerr) + return nil + } + + // Returns nil if the status code is StatusForbidden. + // This happens when AuthorizationFailed is reported from Azure API. + if rerr.HTTPStatusCode == http.StatusForbidden { + klog.V(3).Infof("Ignoring StatusForbidden error: %v", rerr) + return nil + } + + return rerr +} + +// IsErrorRetriable returns true if the error is retriable. +func IsErrorRetriable(err error) bool { + if err == nil { + return false + } + + return strings.Contains(err.Error(), "Retriable: true") +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/retry/azure_error_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/retry/azure_error_test.go new file mode 100644 index 00000000000..01ab273a510 --- /dev/null +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/retry/azure_error_test.go @@ -0,0 +1,253 @@ +// +build !providerless + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package retry + +import ( + "fmt" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestGetError(t *testing.T) { + now = func() time.Time { + return time.Time{} + } + + tests := []struct { + code int + retryAfter int + err error + expected *Error + }{ + { + code: http.StatusOK, + expected: nil, + }, + { + code: http.StatusOK, + err: fmt.Errorf("some error"), + expected: &Error{ + Retriable: true, + HTTPStatusCode: http.StatusOK, + RawError: fmt.Errorf("some error"), + }, + }, + { + code: http.StatusBadRequest, + expected: &Error{ + Retriable: true, + HTTPStatusCode: http.StatusBadRequest, + RawError: fmt.Errorf("HTTP response: 400"), + }, + }, + { + code: http.StatusInternalServerError, + expected: &Error{ + Retriable: true, + HTTPStatusCode: http.StatusInternalServerError, + RawError: fmt.Errorf("HTTP response: 500"), + }, + }, + { + code: http.StatusSeeOther, + err: fmt.Errorf("some error"), + expected: &Error{ + Retriable: true, + HTTPStatusCode: http.StatusSeeOther, + RawError: fmt.Errorf("some error"), + }, + }, + { + code: http.StatusTooManyRequests, + retryAfter: 100, + expected: &Error{ + Retriable: true, + HTTPStatusCode: http.StatusTooManyRequests, + RetryAfter: now().Add(100 * time.Second), + RawError: fmt.Errorf("HTTP response: 429"), + }, + }, + } + + for _, test := range tests { + resp := &http.Response{ + StatusCode: test.code, + Header: http.Header{}, + } + if test.retryAfter != 0 { + resp.Header.Add("Retry-After", fmt.Sprintf("%d", test.retryAfter)) + } + rerr := GetError(resp, test.err) + assert.Equal(t, test.expected, rerr) + } +} + +func TestGetStatusNotFoundAndForbiddenIgnoredError(t *testing.T) { + now = func() time.Time { + return time.Time{} + } + + tests := []struct { + code int + retryAfter int + err error + expected *Error + }{ + { + code: http.StatusOK, + expected: nil, + }, + { + code: http.StatusNotFound, + expected: nil, + }, + { + code: http.StatusForbidden, + expected: nil, + }, + { + code: http.StatusOK, + err: fmt.Errorf("some error"), + expected: &Error{ + Retriable: true, + HTTPStatusCode: http.StatusOK, + RawError: fmt.Errorf("some error"), + }, + }, + { + code: http.StatusBadRequest, + expected: &Error{ + Retriable: true, + HTTPStatusCode: http.StatusBadRequest, + RawError: fmt.Errorf("HTTP response: 400"), + }, + }, + { + code: http.StatusInternalServerError, + expected: &Error{ + Retriable: true, + HTTPStatusCode: http.StatusInternalServerError, + RawError: fmt.Errorf("HTTP response: 500"), + }, + }, + { + code: http.StatusSeeOther, + err: fmt.Errorf("some error"), + expected: &Error{ + Retriable: true, + HTTPStatusCode: http.StatusSeeOther, + RawError: fmt.Errorf("some error"), + }, + }, + { + code: http.StatusTooManyRequests, + retryAfter: 100, + expected: &Error{ + Retriable: true, + HTTPStatusCode: http.StatusTooManyRequests, + RetryAfter: now().Add(100 * time.Second), + RawError: fmt.Errorf("HTTP response: 429"), + }, + }, + } + + for _, test := range tests { + resp := &http.Response{ + StatusCode: test.code, + Header: http.Header{}, + } + if test.retryAfter != 0 { + resp.Header.Add("Retry-After", fmt.Sprintf("%d", test.retryAfter)) + } + rerr := GetStatusNotFoundAndForbiddenIgnoredError(resp, test.err) + assert.Equal(t, test.expected, rerr) + } +} + +func TestShouldRetryHTTPRequest(t *testing.T) { + tests := []struct { + code int + err error + expected bool + }{ + { + code: http.StatusBadRequest, + expected: true, + }, + { + code: http.StatusInternalServerError, + expected: true, + }, + { + code: http.StatusOK, + err: fmt.Errorf("some error"), + expected: true, + }, + { + code: http.StatusOK, + expected: false, + }, + { + code: 399, + expected: false, + }, + } + for _, test := range tests { + resp := &http.Response{ + StatusCode: test.code, + } + res := shouldRetryHTTPRequest(resp, test.err) + if res != test.expected { + t.Errorf("expected: %v, saw: %v", test.expected, res) + } + } +} + +func TestIsSuccessResponse(t *testing.T) { + tests := []struct { + code int + expected bool + }{ + { + code: http.StatusNotFound, + expected: false, + }, + { + code: http.StatusInternalServerError, + expected: false, + }, + { + code: http.StatusOK, + expected: true, + }, + } + + for _, test := range tests { + resp := http.Response{ + StatusCode: test.code, + } + res := isSuccessHTTPResponse(&resp) + if res != test.expected { + t.Errorf("expected: %v, saw: %v", test.expected, res) + } + } +}