Merge pull request #933 from QiWang19/retry

Retry skopeo inspect command

cmd/skopeo/inspect.go

@@ -11,7 +11,9 @@ import (
 	"github.com/containers/image/v5/image"
 	"github.com/containers/image/v5/manifest"
 	"github.com/containers/image/v5/transports"
+	"github.com/containers/image/v5/types"
 	digest "github.com/opencontainers/go-digest"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -33,18 +35,21 @@ type inspectOutput struct {
 }
 
 type inspectOptions struct {
-	global *globalOptions
+	global    *globalOptions
-	image  *imageOptions
+	image     *imageOptions
-	raw    bool // Output the raw manifest instead of parsing information about the image
+	retryOpts *retryOptions
-	config bool // Output the raw config blob instead of parsing information about the image
+	raw       bool // Output the raw manifest instead of parsing information about the image
+	config    bool // Output the raw config blob instead of parsing information about the image
 }
 
 func inspectCmd(global *globalOptions) *cobra.Command {
 	sharedFlags, sharedOpts := sharedImageFlags()
 	imageFlags, imageOpts := imageFlags(global, sharedOpts, "", "")
+	retryFlags, retryOpts := retryFlags()
 	opts := inspectOptions{
-		global: global,
+		global:    global,
-		image:  imageOpts,
+		image:     imageOpts,
+		retryOpts: retryOpts,
 	}
 	cmd := &cobra.Command{
 		Use:   "inspect [command options] IMAGE-NAME",
@@ -64,10 +69,16 @@ See skopeo(1) section "IMAGE NAMES" for the expected format
 	flags.BoolVar(&opts.config, "config", false, "output configuration")
 	flags.AddFlagSet(&sharedFlags)
 	flags.AddFlagSet(&imageFlags)
+	flags.AddFlagSet(&retryFlags)
 	return cmd
 }
 
 func (opts *inspectOptions) run(args []string, stdout io.Writer) (retErr error) {
+	var (
+		rawManifest []byte
+		src         types.ImageSource
+		imgInspect  *types.ImageInspectInfo
+	)
 	ctx, cancel := opts.global.commandTimeoutContext()
 	defer cancel()
 
@@ -85,9 +96,11 @@ func (opts *inspectOptions) run(args []string, stdout io.Writer) (retErr error)
 		return err
 	}
 
-	src, err := parseImageSource(ctx, opts.image, imageName)
+	if err := retryIfNecessary(ctx, func() error {
-	if err != nil {
+		src, err = parseImageSource(ctx, opts.image, imageName)
-		return fmt.Errorf("Error parsing image name %q: %v", imageName, err)
+		return err
+	}, opts.retryOpts); err != nil {
+		return errors.Wrapf(err, "Error parsing image name %q", imageName)
 	}
 
 	defer func() {
@@ -96,9 +109,11 @@ func (opts *inspectOptions) run(args []string, stdout io.Writer) (retErr error)
 		}
 	}()
 
-	rawManifest, _, err := src.GetManifest(ctx, nil)
+	if err := retryIfNecessary(ctx, func() error {
-	if err != nil {
+		rawManifest, _, err = src.GetManifest(ctx, nil)
-		return fmt.Errorf("Error retrieving manifest for image: %v", err)
+		return err
+	}, opts.retryOpts); err != nil {
+		return errors.Wrapf(err, "Error retrieving manifest for image")
 	}
 
 	if opts.raw && !opts.config {
@@ -115,9 +130,12 @@ func (opts *inspectOptions) run(args []string, stdout io.Writer) (retErr error)
 	}
 
 	if opts.config && opts.raw {
-		configBlob, err := img.ConfigBlob(ctx)
+		var configBlob []byte
-		if err != nil {
+		if err := retryIfNecessary(ctx, func() error {
-			return fmt.Errorf("Error reading configuration blob: %v", err)
+			configBlob, err = img.ConfigBlob(ctx)
+			return err
+		}, opts.retryOpts); err != nil {
+			return errors.Wrapf(err, "Error reading configuration blob")
 		}
 		_, err = stdout.Write(configBlob)
 		if err != nil {
@@ -125,9 +143,12 @@ func (opts *inspectOptions) run(args []string, stdout io.Writer) (retErr error)
 		}
 		return nil
 	} else if opts.config {
-		config, err := img.OCIConfig(ctx)
+		var config *v1.Image
-		if err != nil {
+		if err := retryIfNecessary(ctx, func() error {
-			return fmt.Errorf("Error reading OCI-formatted configuration data: %v", err)
+			config, err = img.OCIConfig(ctx)
+			return err
+		}, opts.retryOpts); err != nil {
+			return errors.Wrapf(err, "Error reading OCI-formatted configuration data")
 		}
 		err = json.NewEncoder(stdout).Encode(config)
 		if err != nil {
@@ -136,10 +157,13 @@ func (opts *inspectOptions) run(args []string, stdout io.Writer) (retErr error)
 		return nil
 	}
 
-	imgInspect, err := img.Inspect(ctx)
+	if err := retryIfNecessary(ctx, func() error {
-	if err != nil {
+		imgInspect, err = img.Inspect(ctx)
+		return err
+	}, opts.retryOpts); err != nil {
 		return err
 	}
+
 	outputData := inspectOutput{
 		Name: "", // Set below if DockerReference() is known
 		Tag:  imgInspect.Tag,

cmd/skopeo/utils.go

@@ -3,13 +3,22 @@ package main
 import (
 	"context"
 	"io"
+	"math"
+	"net"
+	"net/url"
 	"os"
 	"strings"
+	"syscall"
+	"time"
 
 	"github.com/containers/image/v5/pkg/compression"
 	"github.com/containers/image/v5/transports/alltransports"
 	"github.com/containers/image/v5/types"
+	"github.com/docker/distribution/registry/api/errcode"
+	errcodev2 "github.com/docker/distribution/registry/api/v2"
+	multierror "github.com/hashicorp/go-multierror"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 )
@@ -108,6 +117,17 @@ func imageFlags(global *globalOptions, shared *sharedImageOptions, flagPrefix, c
 	return fs, opts
 }
 
+type retryOptions struct {
+	maxRetry int // The number of times to possibly retry
+}
+
+func retryFlags() (pflag.FlagSet, *retryOptions) {
+	opts := retryOptions{}
+	fs := pflag.FlagSet{}
+	fs.IntVar(&opts.maxRetry, "retry-times", 0, "the number of times to possibly retry")
+	return fs, &opts
+}
+
 // newSystemContext returns a *types.SystemContext corresponding to opts.
 // It is guaranteed to return a fresh instance, so it is safe to make additional updates to it.
 func (opts *imageOptions) newSystemContext() (*types.SystemContext, error) {
@@ -257,3 +277,68 @@ func adjustUsage(c *cobra.Command) {
 	c.SetUsageTemplate(usageTemplate)
 	c.DisableFlagsInUseLine = true
 }
+
+func isRetryable(err error) bool {
+	err = errors.Cause(err)
+
+	if err == context.Canceled || err == context.DeadlineExceeded {
+		return false
+	}
+
+	type unwrapper interface {
+		Unwrap() error
+	}
+
+	switch e := err.(type) {
+
+	case unwrapper:
+		err = e.Unwrap()
+		return isRetryable(err)
+	case errcode.Error:
+		switch e.Code {
+		case errcode.ErrorCodeUnauthorized, errcodev2.ErrorCodeNameUnknown, errcodev2.ErrorCodeManifestUnknown:
+			return false
+		}
+		return true
+	case *net.OpError:
+		return isRetryable(e.Err)
+	case *url.Error:
+		return isRetryable(e.Err)
+	case syscall.Errno:
+		return e != syscall.ECONNREFUSED
+	case errcode.Errors:
+		// if this error is a group of errors, process them all in turn
+		for i := range e {
+			if !isRetryable(e[i]) {
+				return false
+			}
+		}
+		return true
+	case *multierror.Error:
+		// if this error is a group of errors, process them all in turn
+		for i := range e.Errors {
+			if !isRetryable(e.Errors[i]) {
+				return false
+			}
+		}
+		return true
+	}
+
+	return false
+}
+
+func retryIfNecessary(ctx context.Context, operation func() error, retryOptions *retryOptions) error {
+	err := operation()
+	for attempt := 0; err != nil && isRetryable(err) && attempt < retryOptions.maxRetry; attempt++ {
+		delay := time.Duration(int(math.Pow(2, float64(attempt)))) * time.Second
+		logrus.Infof("Warning: failed, retrying in %s ... (%d/%d)", delay, attempt+1, retryOptions.maxRetry)
+		select {
+		case <-time.After(delay):
+			break
+		case <-ctx.Done():
+			return err
+		}
+		err = operation()
+	}
+	return err
+}

completions/bash/skopeo

@@ -73,6 +73,7 @@ _skopeo_inspect() {
      --authfile
      --creds
      --cert-dir
+     --retry-times
      "
      local boolean_options="
      --config

docs/skopeo-inspect.1.md

@@ -29,6 +29,8 @@ Return low-level information about _image-name_ in a registry
 
   **--cert-dir** _path_ Use certificates at _path_ (\*.crt, \*.cert, \*.key) to connect to the registry
 
+  **--retry-times**  the number of times to retry, retry wait time will be exponentially increased based on the number of failed attempts
+
   **--tls-verify** _bool-value_ Require HTTPS and verify certificates when talking to container registries (defaults to true)
 
   **--no-creds** _bool-value_ Access the registry anonymously.

go.mod

@@ -7,9 +7,11 @@ require (
 	github.com/containers/image/v5 v5.5.1
 	github.com/containers/ocicrypt v1.0.2
 	github.com/containers/storage v1.20.2
+	github.com/docker/distribution v2.7.1+incompatible
 	github.com/docker/docker v1.4.2-0.20191219165747-a9416c67da9f
 	github.com/dsnet/compress v0.0.1 // indirect
 	github.com/go-check/check v0.0.0-20180628173108-788fd7840127
+	github.com/hashicorp/go-multierror v1.0.0
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6
 	github.com/opencontainers/image-tools v0.0.0-20170926011501-6d941547fa1d