package retry import ( "context" "io" "math" "net" "net/url" "syscall" "time" "github.com/docker/distribution/registry/api/errcode" errcodev2 "github.com/docker/distribution/registry/api/v2" "github.com/hashicorp/go-multierror" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) // RetryOptions defines the option to retry type RetryOptions struct { MaxRetry int // The number of times to possibly retry Delay time.Duration // The delay to use between retries, if set } // RetryIfNecessary retries the operation in exponential backoff with the retryOptions func RetryIfNecessary(ctx context.Context, operation func() error, retryOptions *RetryOptions) error { err := operation() for attempt := 0; err != nil && isRetryable(err) && attempt < retryOptions.MaxRetry; attempt++ { delay := time.Duration(int(math.Pow(2, float64(attempt)))) * time.Second if retryOptions.Delay != 0 { delay = retryOptions.Delay } logrus.Warnf("Failed, retrying in %s ... (%d/%d). Error: %v", delay, attempt+1, retryOptions.MaxRetry, err) select { case <-time.After(delay): break case <-ctx.Done(): return err } err = operation() } return err } func isRetryable(err error) bool { err = errors.Cause(err) switch err { case nil: return false case context.Canceled, context.DeadlineExceeded: return false default: // continue } type unwrapper interface { Unwrap() error } switch e := err.(type) { case errcode.Error: switch e.Code { case errcode.ErrorCodeUnauthorized, errcode.ErrorCodeDenied, errcodev2.ErrorCodeNameUnknown, errcodev2.ErrorCodeManifestUnknown: return false } return true case *net.OpError: return isRetryable(e.Err) case *url.Error: // This includes errors returned by the net/http client. if e.Err == io.EOF { // Happens when a server accepts a HTTP connection and sends EOF return true } return isRetryable(e.Err) case syscall.Errno: return isErrnoRetryable(e) case errcode.Errors: // if this error is a group of errors, process them all in turn for i := range e { if !isRetryable(e[i]) { return false } } return true case *multierror.Error: // if this error is a group of errors, process them all in turn for i := range e.Errors { if !isRetryable(e.Errors[i]) { return false } } return true case unwrapper: // Test this last, because various error types might implement .Unwrap() err = e.Unwrap() return isRetryable(err) } return false } func isErrnoRetryable(e error) bool { switch e { case syscall.ECONNREFUSED, syscall.EINTR, syscall.EAGAIN, syscall.EBUSY, syscall.ENETDOWN, syscall.ENETUNREACH, syscall.ENETRESET, syscall.ECONNABORTED, syscall.ECONNRESET, syscall.ETIMEDOUT, syscall.EHOSTDOWN, syscall.EHOSTUNREACH: return true } return isErrnoERESTART(e) }