Merge branch 'master' into dependabot/go_modules/github.com/containers/storage-1.30.1

.cirrus.yml

@@ -23,13 +23,13 @@ env:
     ####
     #### Cache-image names to test with (double-quotes around names are critical)
     ####
-    FEDORA_NAME: "fedora-34beta"
+    FEDORA_NAME: "fedora-34"
     PRIOR_FEDORA_NAME: "fedora-33"
-    UBUNTU_NAME: "ubuntu-2010"
+    UBUNTU_NAME: "ubuntu-2104"
-    PRIOR_UBUNTU_NAME: "ubuntu-2004"
+    PRIOR_UBUNTU_NAME: "ubuntu-2010"
 
     # Google-cloud VM Images
-    IMAGE_SUFFIX: "c5032481331085312"
+    IMAGE_SUFFIX: "c6032583541653504"
     FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}"
     PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${IMAGE_SUFFIX}"
     UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${IMAGE_SUFFIX}"

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+- package-ecosystem: gomod
+  directory: "/"
+  schedule:
+    interval: daily
+    time: "10:00"
+    timezone: Europe/Berlin
+  open-pull-requests-limit: 10
+

cmd/skopeo/copy.go

@@ -9,14 +9,11 @@ import (
 	"github.com/containers/common/pkg/retry"
 	"github.com/containers/image/v5/copy"
 	"github.com/containers/image/v5/docker/reference"
-	"github.com/containers/image/v5/manifest"
 	"github.com/containers/image/v5/transports"
 	"github.com/containers/image/v5/transports/alltransports"
-	"github.com/spf13/cobra"
-
 	encconfig "github.com/containers/ocicrypt/config"
 	enchelpers "github.com/containers/ocicrypt/helpers"
-	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/spf13/cobra"
 )
 
 type copyOptions struct {
@@ -112,15 +109,9 @@ func (opts *copyOptions) run(args []string, stdout io.Writer) error {
 
 	var manifestType string
 	if opts.format.present {
-		switch opts.format.value {
+		manifestType, err = parseManifestFormat(opts.format.value)
-		case "oci":
+		if err != nil {
-			manifestType = imgspecv1.MediaTypeImageManifest
+			return err
-		case "v2s1":
-			manifestType = manifest.DockerV2Schema1SignedMediaType
-		case "v2s2":
-			manifestType = manifest.DockerV2Schema2MediaType
-		default:
-			return fmt.Errorf("unknown format %q. Choose one of the supported formats: 'oci', 'v2s1', or 'v2s2'", opts.format.value)
 		}
 	}
 

cmd/skopeo/sync.go

@@ -33,6 +33,7 @@ type syncOptions struct {
 	retryOpts         *retry.RetryOptions
 	removeSignatures  bool           // Do not copy signatures from the source image
 	signByFingerprint string         // Sign the image using a GPG key with the specified fingerprint
+	format            optionalString // Force conversion of the image to a specified format
 	source            string         // Source repository name
 	destination       string         // Destination registry name
 	scoped            bool           // When true, namespace copied images at destination using the source repository name
@@ -95,6 +96,7 @@ See skopeo-sync(1) for details.
 	flags := cmd.Flags()
 	flags.BoolVar(&opts.removeSignatures, "remove-signatures", false, "Do not copy signatures from SOURCE images")
 	flags.StringVar(&opts.signByFingerprint, "sign-by", "", "Sign the image using a GPG key with the specified `FINGERPRINT`")
+	flags.VarP(newOptionalStringValue(&opts.format), "format", "f", `MANIFEST TYPE (oci, v2s1, or v2s2) to use when syncing image(s) to a destination (default is manifest type of source)`)
 	flags.StringVarP(&opts.source, "src", "s", "", "SOURCE transport type")
 	flags.StringVarP(&opts.destination, "dest", "d", "", "DESTINATION transport type")
 	flags.BoolVar(&opts.scoped, "scoped", false, "Images at DESTINATION are prefix using the full source image path as scope")
@@ -536,6 +538,14 @@ func (opts *syncOptions) run(args []string, stdout io.Writer) error {
 		return err
 	}
 
+	var manifestType string
+	if opts.format.present {
+		manifestType, err = parseManifestFormat(opts.format.value)
+		if err != nil {
+			return err
+		}
+	}
+
 	ctx, cancel := opts.global.commandTimeoutContext()
 	defer cancel()
 
@@ -562,6 +572,7 @@ func (opts *syncOptions) run(args []string, stdout io.Writer) error {
 		DestinationCtx:                        destinationCtx,
 		ImageListSelection:                    imageListSelection,
 		OptimizeDestinationImageAlreadyExists: true,
+		ForceManifestMIMEType:                 manifestType,
 	}
 
 	for _, srcRepo := range srcRepoList {

cmd/skopeo/utils.go

@@ -2,14 +2,17 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"os"
 	"strings"
 
 	"github.com/containers/common/pkg/retry"
+	"github.com/containers/image/v5/manifest"
 	"github.com/containers/image/v5/pkg/compression"
 	"github.com/containers/image/v5/transports/alltransports"
 	"github.com/containers/image/v5/types"
+	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
@@ -246,6 +249,21 @@ func parseImageSource(ctx context.Context, opts *imageOptions, name string) (typ
 	return ref.NewImageSource(ctx, sys)
 }
 
+// parseManifestFormat parses format parameter for copy and sync command.
+// It returns string value to use as manifest MIME type
+func parseManifestFormat(manifestFormat string) (string, error) {
+	switch manifestFormat {
+	case "oci":
+		return imgspecv1.MediaTypeImageManifest, nil
+	case "v2s1":
+		return manifest.DockerV2Schema1SignedMediaType, nil
+	case "v2s2":
+		return manifest.DockerV2Schema2MediaType, nil
+	default:
+		return "", fmt.Errorf("unknown format %q. Choose one of the supported formats: 'oci', 'v2s1', or 'v2s2'", manifestFormat)
+	}
+}
+
 // usageTemplate returns the usage template for skopeo commands
 // This blocks the displaying of the global options. The main skopeo
 // command should not use this.

cmd/skopeo/utils_test.go

@@ -4,7 +4,9 @@ import (
 	"os"
 	"testing"
 
+	"github.com/containers/image/v5/manifest"
 	"github.com/containers/image/v5/types"
+	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -203,6 +205,38 @@ func TestImageDestOptionsNewSystemContext(t *testing.T) {
 	assert.Error(t, err)
 }
 
+func TestParseManifestFormat(t *testing.T) {
+	for _, testCase := range []struct {
+		formatParam          string
+		expectedManifestType string
+		expectErr            bool
+	}{
+		{"oci",
+			imgspecv1.MediaTypeImageManifest,
+			false},
+		{"v2s1",
+			manifest.DockerV2Schema1SignedMediaType,
+			false},
+		{"v2s2",
+			manifest.DockerV2Schema2MediaType,
+			false},
+		{"",
+			"",
+			true},
+		{"badValue",
+			"",
+			true},
+	} {
+		manifestType, err := parseManifestFormat(testCase.formatParam)
+		if testCase.expectErr {
+			require.Error(t, err)
+		} else {
+			require.NoError(t, err)
+		}
+		assert.Equal(t, manifestType, testCase.expectedManifestType)
+	}
+}
+
 // since there is a shared authfile image option and a non-shared (prefixed) one, make sure the override logic
 // works correctly.
 func TestImageOptionsAuthfileOverride(t *testing.T) {

completions/bash/skopeo

@@ -70,6 +70,42 @@ _skopeo_copy() {
     _complete_ "$options_with_args" "$boolean_options" "$transports"
 }
 
+_skopeo_sync() {
+    local options_with_args="
+    --authfile
+    --dest
+    --dest-authfile
+    --dest-cert-
+    --dest-creds
+    --dest-registry-token string
+    --format
+    --retry-times
+    --sign-by
+    --src
+    --src-authfile
+    --src-cert-dir
+    --src-creds
+    --src-registry-token
+    "
+
+    local boolean_options="
+    --all
+    --dest-no-creds
+    --dest-tls-verify
+    --remove-signatures
+    --scoped
+    --src-no-creds
+    --src-tls-verify
+    "
+
+    local transports
+    transports="
+    $(_skopeo_supported_transports "${FUNCNAME//"_skopeo_"/}")
+    "
+
+    _complete_ "$options_with_args" "$boolean_options" "$transports"
+}
+
 _skopeo_inspect() {
      local options_with_args="
      --authfile
@@ -260,7 +296,7 @@ _cli_bash_autocomplete() {
      local counter=1
      while [ $counter -lt "$cword" ]; do
          case "${words[$counter]}" in
-             skopeo|copy|inspect|delete|manifest-digest|standalone-sign|standalone-verify|help|h|list-repository-tags)
+             skopeo|copy|sync|inspect|delete|manifest-digest|standalone-sign|standalone-verify|help|h|list-repository-tags)
                  command="${words[$counter]//-/_}"
                  cpos=$counter
                  (( cpos++ ))

docs/skopeo-sync.1.md

@@ -54,6 +54,8 @@ Path of the authentication file for the destination registry. Uses path given by
 
 **--dest** _transport_ Destination transport.
 
+**--format, -f** _manifest-type_ Manifest Type (oci, v2s1, or v2s2) to use when syncing image(s) to a destination (default is manifest type of source).
+
 **--scoped** Prefix images with the source image path, so that multiple images with the same name can be stored at _destination_.
 
 **--remove-signatures** Do not copy signatures, if any, from _source-image_. This is necessary when copying a signed image to a destination which does not support signatures.

go.mod

@@ -3,7 +3,7 @@ module github.com/containers/skopeo
 go 1.12
 
 require (
-	github.com/containers/common v0.37.0
+	github.com/containers/common v0.37.1
 	github.com/containers/image/v5 v5.11.1
 	github.com/containers/ocicrypt v1.1.1
 	github.com/containers/storage v1.30.1

go.sum

@@ -171,8 +171,8 @@ github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1Dv
 github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
 github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
 github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM=
-github.com/containers/common v0.37.0 h1:RRyR8FITTJXfrF7J9KXKSplywY4zsXoA2kuQXMaUaNo=
+github.com/containers/common v0.37.1 h1:V71FK6k2KsNgcNtspGlrdCaKrSml/SO6bKmJdWjSnaY=
-github.com/containers/common v0.37.0/go.mod h1:dgbJcccCPTmncqxhma56+XW+6d5VzqGF6jtkMHyu3v0=
+github.com/containers/common v0.37.1/go.mod h1:ONPdpc69oQG9e75v/eBzzAReuv0we5NcGdEzK4meDv4=
 github.com/containers/image/v5 v5.11.1 h1:mNybUvU6zXUwcMsQaa3n+Idsru5pV+GE7k4oRuPzYi0=
 github.com/containers/image/v5 v5.11.1/go.mod h1:HC9lhJ/Nz5v3w/5Co7H431kLlgzlVlOC+auD/er3OqE=
 github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b h1:Q8ePgVfHDplZ7U33NwHZkrVELsZP5fYj9pM5WBZB2GE=
@@ -181,10 +181,7 @@ github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/
 github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4=
 github.com/containers/ocicrypt v1.1.1 h1:prL8l9w3ntVqXvNH1CiNn5ENjcCnr38JqpSyvKKB4GI=
 github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY=
-github.com/containers/storage v1.29.0 h1:l3Vh6+IiMKLGfQZ3rDkF84m+KF1Qb0XEcilWC+pYo2o=
 github.com/containers/storage v1.29.0/go.mod h1:u84RU4CCufGeJBNTRNwMB+FoE+AiFeFw4SsMoqAOeCM=
-github.com/containers/storage v1.30.0 h1:KS6zmoPyy0Qcx1HCCiseQ0ysSckRvtiuoVpIGh9iwQA=
-github.com/containers/storage v1.30.0/go.mod h1:M/xn0pg6ReYFrLtWl5YELI/a4Xjq+Z3e5GJxQrJCcDI=
 github.com/containers/storage v1.30.1 h1:+87sZDoUp0uNsP45dWypHTWTEoy0eNDgFYjTU1XIRVQ=
 github.com/containers/storage v1.30.1/go.mod h1:NDJkiwxnSHD1Is+4DGcyR3SIEYSDOa0xnAW+uGQFx9E=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
@@ -423,10 +420,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4=
 github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.12.1 h1:/+xsCsk06wE38cyiqOR/o7U2fSftcH72xD+BQXmja/g=
-github.com/klauspost/compress v1.12.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
 github.com/klauspost/compress v1.12.2 h1:2KCfW3I9M7nSc5wOqXAlW2v2U6v+w6cbjvbfp+OykW8=
 github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=

integration/copy_test.go

@@ -1251,14 +1251,6 @@ func (s *CopySuite) testCopySchemaConversionRegistries(c *check.C, schema1Regist
 	verifyManifestMIMEType(c, destDir, manifest.DockerV2Schema1SignedMediaType)
 }
 
-// Verify manifest in a dir: image at dir is expectedMIMEType.
-func verifyManifestMIMEType(c *check.C, dir string, expectedMIMEType string) {
-	manifestBlob, err := ioutil.ReadFile(filepath.Join(dir, "manifest.json"))
-	c.Assert(err, check.IsNil)
-	mimeType := manifest.GuessMIMEType(manifestBlob)
-	c.Assert(mimeType, check.Equals, expectedMIMEType)
-}
-
 const regConfFixture = "./fixtures/registries.conf"
 
 func (s *SkopeoSuite) TestSuccessCopySrcWithMirror(c *check.C) {

integration/sync_test.go

@@ -12,8 +12,10 @@ import (
 
 	"github.com/containers/image/v5/docker"
 	"github.com/containers/image/v5/docker/reference"
+	"github.com/containers/image/v5/manifest"
 	"github.com/containers/image/v5/types"
 	"github.com/go-check/check"
+	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 const (
@@ -472,6 +474,26 @@ func (s *SyncSuite) TestYamlTLSVerify(c *check.C) {
 
 }
 
+func (s *SyncSuite) TestSyncManifestOutput(c *check.C) {
+	tmpDir, err := ioutil.TempDir("", "sync-manifest-output")
+	c.Assert(err, check.IsNil)
+	defer os.RemoveAll(tmpDir)
+
+	destDir1 := filepath.Join(tmpDir, "dest1")
+	destDir2 := filepath.Join(tmpDir, "dest2")
+	destDir3 := filepath.Join(tmpDir, "dest3")
+
+	//Split image:tag path from image URI for manifest comparison
+	imageDir := pullableTaggedImage[strings.LastIndex(pullableTaggedImage, "/")+1:]
+
+	assertSkopeoSucceeds(c, "", "sync", "--format=oci", "--all", "--src", "docker", "--dest", "dir", pullableTaggedImage, destDir1)
+	verifyManifestMIMEType(c, filepath.Join(destDir1, imageDir), imgspecv1.MediaTypeImageManifest)
+	assertSkopeoSucceeds(c, "", "sync", "--format=v2s2", "--all", "--src", "docker", "--dest", "dir", pullableTaggedImage, destDir2)
+	verifyManifestMIMEType(c, filepath.Join(destDir2, imageDir), manifest.DockerV2Schema2MediaType)
+	assertSkopeoSucceeds(c, "", "sync", "--format=v2s1", "--all", "--src", "docker", "--dest", "dir", pullableTaggedImage, destDir3)
+	verifyManifestMIMEType(c, filepath.Join(destDir3, imageDir), manifest.DockerV2Schema1SignedMediaType)
+}
+
 func (s *SyncSuite) TestDocker2DockerTagged(c *check.C) {
 	const localRegURL = "docker://" + v2DockerRegistryURL + "/"
 

integration/utils.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/containers/image/v5/manifest"
 	"github.com/go-check/check"
 )
 
@@ -200,3 +201,11 @@ func runDecompressDirs(c *check.C, regexp string, args ...string) {
 		c.Assert(string(out), check.Matches, "(?s)"+regexp) // (?s) : '.' will also match newlines
 	}
 }
+
+// Verify manifest in a dir: image at dir is expectedMIMEType.
+func verifyManifestMIMEType(c *check.C, dir string, expectedMIMEType string) {
+	manifestBlob, err := ioutil.ReadFile(filepath.Join(dir, "manifest.json"))
+	c.Assert(err, check.IsNil)
+	mimeType := manifest.GuessMIMEType(manifestBlob)
+	c.Assert(mimeType, check.Equals, expectedMIMEType)
+}

vendor/modules.txt

@@ -41,7 +41,7 @@ github.com/containerd/cgroups/stats/v1
 github.com/containerd/containerd/errdefs
 github.com/containerd/containerd/log
 github.com/containerd/containerd/platforms
-# github.com/containers/common v0.37.0
+# github.com/containers/common v0.37.1
 github.com/containers/common/pkg/auth
 github.com/containers/common/pkg/capabilities
 github.com/containers/common/pkg/completion