github/skopeo
1
0
Fork 0
mirror of https://github.com/containers/skopeo.git synced 2025-08-01 23:07:51 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #1002 from QiWang19/common-retry

Browse Source 
Use c/common retry package
This commit is contained in:
Miloslav Trmač 2020-08-08 08:04:35 +02:00 committed by GitHub
commit baeaad61d9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 118 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cmd/skopeo/copy.go
View File

@ -6,6 +6,7 @@ import (
"io"
"strings"
"github.com/containers/common/pkg/retry"
"github.com/containers/image/v5/copy"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/manifest"
@ -22,7 +23,7 @@ type copyOptions struct {
global *globalOptions
srcImage *imageOptions
destImage *imageDestOptions
retryOpts *retryOptions
retryOpts *retry.RetryOptions
additionalTags []string // For docker-archive: destinations, in addition to the name:tag specified as destination, also add these
removeSignatures bool // Do not copy signatures from the source image
signByFingerprint string // Sign the image using a GPG key with the specified fingerprint
@ -182,7 +183,7 @@ func (opts *copyOptions) run(args []string, stdout io.Writer) error {
decConfig = cc.DecryptConfig
}
return retryIfNecessary(ctx, func() error {
return retry.RetryIfNecessary(ctx, func() error {
_, err = copy.Image(ctx, policyContext, destRef, srcRef, &copy.Options{
RemoveSignatures: opts.removeSignatures,
SignBy: opts.signByFingerprint,

5
cmd/skopeo/delete.go
View File

@ -6,6 +6,7 @@ import (
"io"
"strings"
"github.com/containers/common/pkg/retry"
"github.com/containers/image/v5/transports"
"github.com/containers/image/v5/transports/alltransports"
"github.com/spf13/cobra"
@ -14,7 +15,7 @@ import (
type deleteOptions struct {
global *globalOptions
image *imageOptions
retryOpts *retryOptions
retryOpts *retry.RetryOptions
}
func deleteCmd(global *globalOptions) *cobra.Command {
@ -68,7 +69,7 @@ func (opts *deleteOptions) run(args []string, stdout io.Writer) error {
ctx, cancel := opts.global.commandTimeoutContext()
defer cancel()
return retryIfNecessary(ctx, func() error {
return retry.RetryIfNecessary(ctx, func() error {
return ref.DeleteImage(ctx, sys)
}, opts.retryOpts)
}

13
cmd/skopeo/inspect.go
View File

@ -6,6 +6,7 @@ import (
"io"
"strings"
"github.com/containers/common/pkg/retry"
"github.com/containers/image/v5/docker"
"github.com/containers/image/v5/image"
"github.com/containers/image/v5/manifest"
@ -21,7 +22,7 @@ import (
type inspectOptions struct {
global *globalOptions
image *imageOptions
retryOpts *retryOptions
retryOpts *retry.RetryOptions
raw bool // Output the raw manifest instead of parsing information about the image
config bool // Output the raw config blob instead of parsing information about the image
}
@ -80,7 +81,7 @@ func (opts *inspectOptions) run(args []string, stdout io.Writer) (retErr error)
return err
}
if err := retryIfNecessary(ctx, func() error {
if err := retry.RetryIfNecessary(ctx, func() error {
src, err = parseImageSource(ctx, opts.image, imageName)
return err
}, opts.retryOpts); err != nil {
@ -93,7 +94,7 @@ func (opts *inspectOptions) run(args []string, stdout io.Writer) (retErr error)
}
}()
if err := retryIfNecessary(ctx, func() error {
if err := retry.RetryIfNecessary(ctx, func() error {
rawManifest, _, err = src.GetManifest(ctx, nil)
return err
}, opts.retryOpts); err != nil {
@ -115,7 +116,7 @@ func (opts *inspectOptions) run(args []string, stdout io.Writer) (retErr error)
if opts.config && opts.raw {
var configBlob []byte
if err := retryIfNecessary(ctx, func() error {
if err := retry.RetryIfNecessary(ctx, func() error {
configBlob, err = img.ConfigBlob(ctx)
return err
}, opts.retryOpts); err != nil {
@ -128,7 +129,7 @@ func (opts *inspectOptions) run(args []string, stdout io.Writer) (retErr error)
return nil
} else if opts.config {
var config *v1.Image
if err := retryIfNecessary(ctx, func() error {
if err := retry.RetryIfNecessary(ctx, func() error {
config, err = img.OCIConfig(ctx)
return err
}, opts.retryOpts); err != nil {
@ -141,7 +142,7 @@ func (opts *inspectOptions) run(args []string, stdout io.Writer) (retErr error)
return nil
}
if err := retryIfNecessary(ctx, func() error {
if err := retry.RetryIfNecessary(ctx, func() error {
imgInspect, err = img.Inspect(ctx)
return err
}, opts.retryOpts); err != nil {

11
cmd/skopeo/layers.go
View File

@ -7,6 +7,7 @@ import (
"os"
"strings"
"github.com/containers/common/pkg/retry"
"github.com/containers/image/v5/directory"
"github.com/containers/image/v5/image"
"github.com/containers/image/v5/pkg/blobinfocache"
@ -19,7 +20,7 @@ import (
type layersOptions struct {
global *globalOptions
image *imageOptions
retryOpts *retryOptions
retryOpts *retry.RetryOptions
}
func layersCmd(global *globalOptions) *cobra.Command {
@ -68,13 +69,13 @@ func (opts *layersOptions) run(args []string, stdout io.Writer) (retErr error) {
rawSource types.ImageSource
src types.ImageCloser
)
if err = retryIfNecessary(ctx, func() error {
if err = retry.RetryIfNecessary(ctx, func() error {
rawSource, err = parseImageSource(ctx, opts.image, imageName)
return err
}, opts.retryOpts); err != nil {
return err
}
if err = retryIfNecessary(ctx, func() error {
if err = retry.RetryIfNecessary(ctx, func() error {
src, err = image.FromSource(ctx, sys, rawSource)
return err
}, opts.retryOpts); err != nil {
@ -145,7 +146,7 @@ func (opts *layersOptions) run(args []string, stdout io.Writer) (retErr error) {
r io.ReadCloser
blobSize int64
)
if err = retryIfNecessary(ctx, func() error {
if err = retry.RetryIfNecessary(ctx, func() error {
r, blobSize, err = rawSource.GetBlob(ctx, types.BlobInfo{Digest: bd.digest, Size: -1}, cache)
return err
}, opts.retryOpts); err != nil {
@ -160,7 +161,7 @@ func (opts *layersOptions) run(args []string, stdout io.Writer) (retErr error) {
}
var manifest []byte
if err = retryIfNecessary(ctx, func() error {
if err = retry.RetryIfNecessary(ctx, func() error {
manifest, _, err = src.Manifest(ctx)
return err
}, opts.retryOpts); err != nil {

5
cmd/skopeo/list_tags.go
View File

@ -7,6 +7,7 @@ import (
"io"
"strings"
"github.com/containers/common/pkg/retry"
"github.com/containers/image/v5/docker"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/transports/alltransports"
@ -24,7 +25,7 @@ type tagListOutput struct {
type tagsOptions struct {
global *globalOptions
image *imageOptions
retryOpts *retryOptions
retryOpts *retry.RetryOptions
}
func tagsCmd(global *globalOptions) *cobra.Command {
@ -124,7 +125,7 @@ func (opts *tagsOptions) run(args []string, stdout io.Writer) (retErr error) {
var repositoryName string
var tagListing []string
if err = retryIfNecessary(ctx, func() error {
if err = retry.RetryIfNecessary(ctx, func() error {
repositoryName, tagListing, err = listDockerTags(ctx, sys, imgRef)
return err
}, opts.retryOpts); err != nil {

7
cmd/skopeo/sync.go
View File

@ -11,6 +11,7 @@ import (
"regexp"
"strings"
"github.com/containers/common/pkg/retry"
"github.com/containers/image/v5/copy"
"github.com/containers/image/v5/directory"
"github.com/containers/image/v5/docker"
@ -28,7 +29,7 @@ type syncOptions struct {
global *globalOptions // Global (not command dependant) skopeo options
srcImage *imageOptions // Source image options
destImage *imageDestOptions // Destination image options
retryOpts *retryOptions
retryOpts *retry.RetryOptions
removeSignatures bool // Do not copy signatures from the source image
signByFingerprint string // Sign the image using a GPG key with the specified fingerprint
source string // Source repository name
@ -518,7 +519,7 @@ func (opts *syncOptions) run(args []string, stdout io.Writer) error {
sourceArg := args[0]
var srcRepoList []repoDescriptor
if err = retryIfNecessary(ctx, func() error {
if err = retry.RetryIfNecessary(ctx, func() error {
srcRepoList, err = imagesToCopy(sourceArg, opts.source, sourceCtx)
return err
}, opts.retryOpts); err != nil {
@ -570,7 +571,7 @@ func (opts *syncOptions) run(args []string, stdout io.Writer) error {
"to": transports.ImageName(destRef),
}).Infof("Copying image tag %d/%d", counter+1, len(srcRepo.TaggedImages))
if err = retryIfNecessary(ctx, func() error {
if err = retry.RetryIfNecessary(ctx, func() error {
_, err = copy.Image(ctx, policyContext, destRef, ref, &options)
return err
}, opts.retryOpts); err != nil {

81
cmd/skopeo/utils.go
View File

@ -3,22 +3,14 @@ package main
import (
"context"
"io"
"math"
"net"
"net/url"
"os"
"strings"
"syscall"
"time"
"github.com/containers/common/pkg/retry"
"github.com/containers/image/v5/pkg/compression"
"github.com/containers/image/v5/transports/alltransports"
"github.com/containers/image/v5/types"
"github.com/docker/distribution/registry/api/errcode"
errcodev2 "github.com/docker/distribution/registry/api/v2"
multierror "github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
@ -121,10 +113,10 @@ type retryOptions struct {
maxRetry int // The number of times to possibly retry
}
func retryFlags() (pflag.FlagSet, *retryOptions) {
opts := retryOptions{}
func retryFlags() (pflag.FlagSet, *retry.RetryOptions) {
opts := retry.RetryOptions{}
fs := pflag.FlagSet{}
fs.IntVar(&opts.maxRetry, "retry-times", 0, "the number of times to possibly retry")
fs.IntVar(&opts.MaxRetry, "retry-times", 0, "the number of times to possibly retry")
return fs, &opts
}
@ -277,68 +269,3 @@ func adjustUsage(c *cobra.Command) {
c.SetUsageTemplate(usageTemplate)
c.DisableFlagsInUseLine = true
}
func isRetryable(err error) bool {
err = errors.Cause(err)
if err == context.Canceled || err == context.DeadlineExceeded {
return false
}
type unwrapper interface {
Unwrap() error
}
switch e := err.(type) {
case unwrapper:
err = e.Unwrap()
return isRetryable(err)
case errcode.Error:
switch e.Code {
case errcode.ErrorCodeUnauthorized, errcodev2.ErrorCodeNameUnknown, errcodev2.ErrorCodeManifestUnknown:
return false
}
return true
case *net.OpError:
return isRetryable(e.Err)
case *url.Error:
return isRetryable(e.Err)
case syscall.Errno:
return e != syscall.ECONNREFUSED
case errcode.Errors:
// if this error is a group of errors, process them all in turn
for i := range e {
if !isRetryable(e[i]) {
return false
}
}
return true
case *multierror.Error:
// if this error is a group of errors, process them all in turn
for i := range e.Errors {
if !isRetryable(e.Errors[i]) {
return false
}
}
return true
}
return false
}
func retryIfNecessary(ctx context.Context, operation func() error, retryOptions *retryOptions) error {
err := operation()
for attempt := 0; err != nil && isRetryable(err) && attempt < retryOptions.maxRetry; attempt++ {
delay := time.Duration(int(math.Pow(2, float64(attempt)))) * time.Second
logrus.Infof("Warning: failed, retrying in %s ... (%d/%d)", delay, attempt+1, retryOptions.maxRetry)
select {
case <-time.After(delay):
break
case <-ctx.Done():
return err
}
err = operation()
}
return err
}

2
go.mod
View File

@ -7,11 +7,9 @@ require (
github.com/containers/image/v5 v5.5.1
github.com/containers/ocicrypt v1.0.3
github.com/containers/storage v1.22.0
github.com/docker/distribution v2.7.1+incompatible
github.com/docker/docker v1.4.2-0.20191219165747-a9416c67da9f
github.com/dsnet/compress v0.0.1 // indirect
github.com/go-check/check v0.0.0-20180628173108-788fd7840127
github.com/hashicorp/go-multierror v1.1.0
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6
github.com/opencontainers/image-tools v0.0.0-20170926011501-6d941547fa1d

87
vendor/github.com/containers/common/pkg/retry/retry.go generated vendored Normal file
View File

@ -0,0 +1,87 @@
package retry
import (
"context"
"math"
"net"
"net/url"
"syscall"
"time"
"github.com/docker/distribution/registry/api/errcode"
errcodev2 "github.com/docker/distribution/registry/api/v2"
"github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
// RetryOptions defines the option to retry
type RetryOptions struct {
MaxRetry int // The number of times to possibly retry
}
// RetryIfNecessary retries the operation in exponential backoff with the retryOptions
func RetryIfNecessary(ctx context.Context, operation func() error, retryOptions *RetryOptions) error {
err := operation()
for attempt := 0; err != nil && isRetryable(err) && attempt < retryOptions.MaxRetry; attempt++ {
delay := time.Duration(int(math.Pow(2, float64(attempt)))) * time.Second
logrus.Infof("Warning: failed, retrying in %s ... (%d/%d)", delay, attempt+1, retryOptions.MaxRetry)
select {
case <-time.After(delay):
break
case <-ctx.Done():
return err
}
err = operation()
}
return err
}
func isRetryable(err error) bool {
err = errors.Cause(err)
if err == context.Canceled || err == context.DeadlineExceeded {
return false
}
type unwrapper interface {
Unwrap() error
}
switch e := err.(type) {
case errcode.Error:
switch e.Code {
case errcode.ErrorCodeUnauthorized, errcodev2.ErrorCodeNameUnknown, errcodev2.ErrorCodeManifestUnknown:
return false
}
return true
case *net.OpError:
return isRetryable(e.Err)
case *url.Error:
return isRetryable(e.Err)
case syscall.Errno:
return e != syscall.ECONNREFUSED
case errcode.Errors:
// if this error is a group of errors, process them all in turn
for i := range e {
if !isRetryable(e[i]) {
return false
}
}
return true
case *multierror.Error:
// if this error is a group of errors, process them all in turn
for i := range e.Errors {
if !isRetryable(e.Errors[i]) {
return false
}
}
return true
case unwrapper:
err = e.Unwrap()
return isRetryable(err)
}
return false
}

1
vendor/modules.txt vendored
View File

@ -37,6 +37,7 @@ github.com/containerd/cgroups/stats/v1
github.com/containerd/containerd/errdefs
# github.com/containers/common v0.18.0
github.com/containers/common/pkg/auth
github.com/containers/common/pkg/retry
# github.com/containers/image/v5 v5.5.1
github.com/containers/image/v5/copy
github.com/containers/image/v5/directory