diff --git a/.cirrus.yml b/.cirrus.yml index fa9f0462..41cde27f 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -23,13 +23,13 @@ env: #### #### Cache-image names to test with (double-quotes around names are critical) #### - FEDORA_NAME: "fedora-34beta" + FEDORA_NAME: "fedora-34" PRIOR_FEDORA_NAME: "fedora-33" - UBUNTU_NAME: "ubuntu-2010" - PRIOR_UBUNTU_NAME: "ubuntu-2004" + UBUNTU_NAME: "ubuntu-2104" + PRIOR_UBUNTU_NAME: "ubuntu-2010" # Google-cloud VM Images - IMAGE_SUFFIX: "c5032481331085312" + IMAGE_SUFFIX: "c6032583541653504" FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}" PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${IMAGE_SUFFIX}" UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${IMAGE_SUFFIX}" diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..55b5a9c8 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: +- package-ecosystem: gomod + directory: "/" + schedule: + interval: daily + time: "10:00" + timezone: Europe/Berlin + open-pull-requests-limit: 10 + diff --git a/cmd/skopeo/copy.go b/cmd/skopeo/copy.go index 4a261bac..0027d781 100644 --- a/cmd/skopeo/copy.go +++ b/cmd/skopeo/copy.go @@ -9,14 +9,11 @@ import ( "github.com/containers/common/pkg/retry" "github.com/containers/image/v5/copy" "github.com/containers/image/v5/docker/reference" - "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/transports" "github.com/containers/image/v5/transports/alltransports" - "github.com/spf13/cobra" - encconfig "github.com/containers/ocicrypt/config" enchelpers "github.com/containers/ocicrypt/helpers" - imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/spf13/cobra" ) type copyOptions struct { @@ -112,15 +109,9 @@ func (opts *copyOptions) run(args []string, stdout io.Writer) error { var manifestType string if opts.format.present { - switch opts.format.value { - case "oci": - manifestType = imgspecv1.MediaTypeImageManifest - case "v2s1": - manifestType = manifest.DockerV2Schema1SignedMediaType - case "v2s2": - manifestType = manifest.DockerV2Schema2MediaType - default: - return fmt.Errorf("unknown format %q. Choose one of the supported formats: 'oci', 'v2s1', or 'v2s2'", opts.format.value) + manifestType, err = parseManifestFormat(opts.format.value) + if err != nil { + return err } } diff --git a/cmd/skopeo/sync.go b/cmd/skopeo/sync.go index 4063932f..263087ed 100644 --- a/cmd/skopeo/sync.go +++ b/cmd/skopeo/sync.go @@ -31,12 +31,13 @@ type syncOptions struct { srcImage *imageOptions // Source image options destImage *imageDestOptions // Destination image options retryOpts *retry.RetryOptions - removeSignatures bool // Do not copy signatures from the source image - signByFingerprint string // Sign the image using a GPG key with the specified fingerprint - source string // Source repository name - destination string // Destination registry name - scoped bool // When true, namespace copied images at destination using the source repository name - all bool // Copy all of the images if an image in the source is a list + removeSignatures bool // Do not copy signatures from the source image + signByFingerprint string // Sign the image using a GPG key with the specified fingerprint + format optionalString // Force conversion of the image to a specified format + source string // Source repository name + destination string // Destination registry name + scoped bool // When true, namespace copied images at destination using the source repository name + all bool // Copy all of the images if an image in the source is a list } // repoDescriptor contains information of a single repository used as a sync source. @@ -95,6 +96,7 @@ See skopeo-sync(1) for details. flags := cmd.Flags() flags.BoolVar(&opts.removeSignatures, "remove-signatures", false, "Do not copy signatures from SOURCE images") flags.StringVar(&opts.signByFingerprint, "sign-by", "", "Sign the image using a GPG key with the specified `FINGERPRINT`") + flags.VarP(newOptionalStringValue(&opts.format), "format", "f", `MANIFEST TYPE (oci, v2s1, or v2s2) to use when syncing image(s) to a destination (default is manifest type of source)`) flags.StringVarP(&opts.source, "src", "s", "", "SOURCE transport type") flags.StringVarP(&opts.destination, "dest", "d", "", "DESTINATION transport type") flags.BoolVar(&opts.scoped, "scoped", false, "Images at DESTINATION are prefix using the full source image path as scope") @@ -536,6 +538,14 @@ func (opts *syncOptions) run(args []string, stdout io.Writer) error { return err } + var manifestType string + if opts.format.present { + manifestType, err = parseManifestFormat(opts.format.value) + if err != nil { + return err + } + } + ctx, cancel := opts.global.commandTimeoutContext() defer cancel() @@ -562,6 +572,7 @@ func (opts *syncOptions) run(args []string, stdout io.Writer) error { DestinationCtx: destinationCtx, ImageListSelection: imageListSelection, OptimizeDestinationImageAlreadyExists: true, + ForceManifestMIMEType: manifestType, } for _, srcRepo := range srcRepoList { diff --git a/cmd/skopeo/utils.go b/cmd/skopeo/utils.go index 219c2c02..61018294 100644 --- a/cmd/skopeo/utils.go +++ b/cmd/skopeo/utils.go @@ -2,14 +2,17 @@ package main import ( "context" + "fmt" "io" "os" "strings" "github.com/containers/common/pkg/retry" + "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/pkg/compression" "github.com/containers/image/v5/transports/alltransports" "github.com/containers/image/v5/types" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -246,6 +249,21 @@ func parseImageSource(ctx context.Context, opts *imageOptions, name string) (typ return ref.NewImageSource(ctx, sys) } +// parseManifestFormat parses format parameter for copy and sync command. +// It returns string value to use as manifest MIME type +func parseManifestFormat(manifestFormat string) (string, error) { + switch manifestFormat { + case "oci": + return imgspecv1.MediaTypeImageManifest, nil + case "v2s1": + return manifest.DockerV2Schema1SignedMediaType, nil + case "v2s2": + return manifest.DockerV2Schema2MediaType, nil + default: + return "", fmt.Errorf("unknown format %q. Choose one of the supported formats: 'oci', 'v2s1', or 'v2s2'", manifestFormat) + } +} + // usageTemplate returns the usage template for skopeo commands // This blocks the displaying of the global options. The main skopeo // command should not use this. diff --git a/cmd/skopeo/utils_test.go b/cmd/skopeo/utils_test.go index 0344613e..6c0992b0 100644 --- a/cmd/skopeo/utils_test.go +++ b/cmd/skopeo/utils_test.go @@ -4,7 +4,9 @@ import ( "os" "testing" + "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/types" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -203,6 +205,38 @@ func TestImageDestOptionsNewSystemContext(t *testing.T) { assert.Error(t, err) } +func TestParseManifestFormat(t *testing.T) { + for _, testCase := range []struct { + formatParam string + expectedManifestType string + expectErr bool + }{ + {"oci", + imgspecv1.MediaTypeImageManifest, + false}, + {"v2s1", + manifest.DockerV2Schema1SignedMediaType, + false}, + {"v2s2", + manifest.DockerV2Schema2MediaType, + false}, + {"", + "", + true}, + {"badValue", + "", + true}, + } { + manifestType, err := parseManifestFormat(testCase.formatParam) + if testCase.expectErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + assert.Equal(t, manifestType, testCase.expectedManifestType) + } +} + // since there is a shared authfile image option and a non-shared (prefixed) one, make sure the override logic // works correctly. func TestImageOptionsAuthfileOverride(t *testing.T) { diff --git a/completions/bash/skopeo b/completions/bash/skopeo index 9cdefce7..6ebc6e4b 100644 --- a/completions/bash/skopeo +++ b/completions/bash/skopeo @@ -70,6 +70,42 @@ _skopeo_copy() { _complete_ "$options_with_args" "$boolean_options" "$transports" } +_skopeo_sync() { + local options_with_args=" + --authfile + --dest + --dest-authfile + --dest-cert- + --dest-creds + --dest-registry-token string + --format + --retry-times + --sign-by + --src + --src-authfile + --src-cert-dir + --src-creds + --src-registry-token + " + + local boolean_options=" + --all + --dest-no-creds + --dest-tls-verify + --remove-signatures + --scoped + --src-no-creds + --src-tls-verify + " + + local transports + transports=" + $(_skopeo_supported_transports "${FUNCNAME//"_skopeo_"/}") + " + + _complete_ "$options_with_args" "$boolean_options" "$transports" +} + _skopeo_inspect() { local options_with_args=" --authfile @@ -260,7 +296,7 @@ _cli_bash_autocomplete() { local counter=1 while [ $counter -lt "$cword" ]; do case "${words[$counter]}" in - skopeo|copy|inspect|delete|manifest-digest|standalone-sign|standalone-verify|help|h|list-repository-tags) + skopeo|copy|sync|inspect|delete|manifest-digest|standalone-sign|standalone-verify|help|h|list-repository-tags) command="${words[$counter]//-/_}" cpos=$counter (( cpos++ )) diff --git a/docs/skopeo-sync.1.md b/docs/skopeo-sync.1.md index d03d80a8..c8304bdc 100644 --- a/docs/skopeo-sync.1.md +++ b/docs/skopeo-sync.1.md @@ -54,6 +54,8 @@ Path of the authentication file for the destination registry. Uses path given by **--dest** _transport_ Destination transport. +**--format, -f** _manifest-type_ Manifest Type (oci, v2s1, or v2s2) to use when syncing image(s) to a destination (default is manifest type of source). + **--scoped** Prefix images with the source image path, so that multiple images with the same name can be stored at _destination_. **--remove-signatures** Do not copy signatures, if any, from _source-image_. This is necessary when copying a signed image to a destination which does not support signatures. diff --git a/go.mod b/go.mod index 1ee3187a..09131a82 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/containers/skopeo go 1.12 require ( - github.com/containers/common v0.37.0 + github.com/containers/common v0.37.1 github.com/containers/image/v5 v5.11.1 github.com/containers/ocicrypt v1.1.1 github.com/containers/storage v1.30.1 diff --git a/go.sum b/go.sum index 3304600b..e46408bc 100644 --- a/go.sum +++ b/go.sum @@ -171,8 +171,8 @@ github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1Dv github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= -github.com/containers/common v0.37.0 h1:RRyR8FITTJXfrF7J9KXKSplywY4zsXoA2kuQXMaUaNo= -github.com/containers/common v0.37.0/go.mod h1:dgbJcccCPTmncqxhma56+XW+6d5VzqGF6jtkMHyu3v0= +github.com/containers/common v0.37.1 h1:V71FK6k2KsNgcNtspGlrdCaKrSml/SO6bKmJdWjSnaY= +github.com/containers/common v0.37.1/go.mod h1:ONPdpc69oQG9e75v/eBzzAReuv0we5NcGdEzK4meDv4= github.com/containers/image/v5 v5.11.1 h1:mNybUvU6zXUwcMsQaa3n+Idsru5pV+GE7k4oRuPzYi0= github.com/containers/image/v5 v5.11.1/go.mod h1:HC9lhJ/Nz5v3w/5Co7H431kLlgzlVlOC+auD/er3OqE= github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b h1:Q8ePgVfHDplZ7U33NwHZkrVELsZP5fYj9pM5WBZB2GE= @@ -181,10 +181,7 @@ github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/ github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= github.com/containers/ocicrypt v1.1.1 h1:prL8l9w3ntVqXvNH1CiNn5ENjcCnr38JqpSyvKKB4GI= github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= -github.com/containers/storage v1.29.0 h1:l3Vh6+IiMKLGfQZ3rDkF84m+KF1Qb0XEcilWC+pYo2o= github.com/containers/storage v1.29.0/go.mod h1:u84RU4CCufGeJBNTRNwMB+FoE+AiFeFw4SsMoqAOeCM= -github.com/containers/storage v1.30.0 h1:KS6zmoPyy0Qcx1HCCiseQ0ysSckRvtiuoVpIGh9iwQA= -github.com/containers/storage v1.30.0/go.mod h1:M/xn0pg6ReYFrLtWl5YELI/a4Xjq+Z3e5GJxQrJCcDI= github.com/containers/storage v1.30.1 h1:+87sZDoUp0uNsP45dWypHTWTEoy0eNDgFYjTU1XIRVQ= github.com/containers/storage v1.30.1/go.mod h1:NDJkiwxnSHD1Is+4DGcyR3SIEYSDOa0xnAW+uGQFx9E= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -423,10 +420,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.12.1 h1:/+xsCsk06wE38cyiqOR/o7U2fSftcH72xD+BQXmja/g= -github.com/klauspost/compress v1.12.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.12.2 h1:2KCfW3I9M7nSc5wOqXAlW2v2U6v+w6cbjvbfp+OykW8= github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= diff --git a/integration/copy_test.go b/integration/copy_test.go index b86b3289..1d460f96 100644 --- a/integration/copy_test.go +++ b/integration/copy_test.go @@ -1251,14 +1251,6 @@ func (s *CopySuite) testCopySchemaConversionRegistries(c *check.C, schema1Regist verifyManifestMIMEType(c, destDir, manifest.DockerV2Schema1SignedMediaType) } -// Verify manifest in a dir: image at dir is expectedMIMEType. -func verifyManifestMIMEType(c *check.C, dir string, expectedMIMEType string) { - manifestBlob, err := ioutil.ReadFile(filepath.Join(dir, "manifest.json")) - c.Assert(err, check.IsNil) - mimeType := manifest.GuessMIMEType(manifestBlob) - c.Assert(mimeType, check.Equals, expectedMIMEType) -} - const regConfFixture = "./fixtures/registries.conf" func (s *SkopeoSuite) TestSuccessCopySrcWithMirror(c *check.C) { diff --git a/integration/sync_test.go b/integration/sync_test.go index 8d1bd05e..3255cf8c 100644 --- a/integration/sync_test.go +++ b/integration/sync_test.go @@ -12,8 +12,10 @@ import ( "github.com/containers/image/v5/docker" "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/types" "github.com/go-check/check" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" ) const ( @@ -472,6 +474,26 @@ func (s *SyncSuite) TestYamlTLSVerify(c *check.C) { } +func (s *SyncSuite) TestSyncManifestOutput(c *check.C) { + tmpDir, err := ioutil.TempDir("", "sync-manifest-output") + c.Assert(err, check.IsNil) + defer os.RemoveAll(tmpDir) + + destDir1 := filepath.Join(tmpDir, "dest1") + destDir2 := filepath.Join(tmpDir, "dest2") + destDir3 := filepath.Join(tmpDir, "dest3") + + //Split image:tag path from image URI for manifest comparison + imageDir := pullableTaggedImage[strings.LastIndex(pullableTaggedImage, "/")+1:] + + assertSkopeoSucceeds(c, "", "sync", "--format=oci", "--all", "--src", "docker", "--dest", "dir", pullableTaggedImage, destDir1) + verifyManifestMIMEType(c, filepath.Join(destDir1, imageDir), imgspecv1.MediaTypeImageManifest) + assertSkopeoSucceeds(c, "", "sync", "--format=v2s2", "--all", "--src", "docker", "--dest", "dir", pullableTaggedImage, destDir2) + verifyManifestMIMEType(c, filepath.Join(destDir2, imageDir), manifest.DockerV2Schema2MediaType) + assertSkopeoSucceeds(c, "", "sync", "--format=v2s1", "--all", "--src", "docker", "--dest", "dir", pullableTaggedImage, destDir3) + verifyManifestMIMEType(c, filepath.Join(destDir3, imageDir), manifest.DockerV2Schema1SignedMediaType) +} + func (s *SyncSuite) TestDocker2DockerTagged(c *check.C) { const localRegURL = "docker://" + v2DockerRegistryURL + "/" diff --git a/integration/utils.go b/integration/utils.go index a19e41bf..d54a7a89 100644 --- a/integration/utils.go +++ b/integration/utils.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/containers/image/v5/manifest" "github.com/go-check/check" ) @@ -200,3 +201,11 @@ func runDecompressDirs(c *check.C, regexp string, args ...string) { c.Assert(string(out), check.Matches, "(?s)"+regexp) // (?s) : '.' will also match newlines } } + +// Verify manifest in a dir: image at dir is expectedMIMEType. +func verifyManifestMIMEType(c *check.C, dir string, expectedMIMEType string) { + manifestBlob, err := ioutil.ReadFile(filepath.Join(dir, "manifest.json")) + c.Assert(err, check.IsNil) + mimeType := manifest.GuessMIMEType(manifestBlob) + c.Assert(mimeType, check.Equals, expectedMIMEType) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 984b7b0a..195f540d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -41,7 +41,7 @@ github.com/containerd/cgroups/stats/v1 github.com/containerd/containerd/errdefs github.com/containerd/containerd/log github.com/containerd/containerd/platforms -# github.com/containers/common v0.37.0 +# github.com/containers/common v0.37.1 github.com/containers/common/pkg/auth github.com/containers/common/pkg/capabilities github.com/containers/common/pkg/completion