diff --git a/cmd/skopeo/copy.go b/cmd/skopeo/copy.go index fc522d9f..9979685d 100644 --- a/cmd/skopeo/copy.go +++ b/cmd/skopeo/copy.go @@ -7,9 +7,11 @@ import ( "strings" "github.com/containers/image/copy" + "github.com/containers/image/manifest" "github.com/containers/image/transports" "github.com/containers/image/transports/alltransports" "github.com/containers/image/types" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/urfave/cli" ) @@ -55,12 +57,27 @@ func copyHandler(context *cli.Context) error { return err } + var manifestType string + if context.IsSet("format") { + switch context.String("format") { + case "oci": + manifestType = imgspecv1.MediaTypeImageManifest + case "v2s1": + manifestType = manifest.DockerV2Schema1SignedMediaType + case "v2s2": + manifestType = manifest.DockerV2Schema2MediaType + default: + return fmt.Errorf("unknown format %q. Choose on of the supported formats: 'oci', 'v2s1', or 'v2s2'", context.String("format")) + } + } + return copy.Image(policyContext, destRef, srcRef, ©.Options{ - RemoveSignatures: removeSignatures, - SignBy: signBy, - ReportWriter: os.Stdout, - SourceCtx: sourceCtx, - DestinationCtx: destinationCtx, + RemoveSignatures: removeSignatures, + SignBy: signBy, + ReportWriter: os.Stdout, + SourceCtx: sourceCtx, + DestinationCtx: destinationCtx, + ForceManifestMIMEType: manifestType, }) } @@ -131,5 +148,13 @@ var copyCmd = cli.Command{ Value: "", Usage: "`DIRECTORY` to use to store retrieved blobs (OCI layout destinations only)", }, + cli.StringFlag{ + Name: "format, f", + Usage: "`MANIFEST TYPE` (oci, v2s1, or v2s2) to use when saving image to directory using the 'dir:' transport (default is manifest type of source)", + }, + cli.BoolFlag{ + Name: "dest-compress", + Usage: "Compress tarball image layers when saving to directory using the 'dir' transport. (default is same compression type as source)", + }, }, } diff --git a/cmd/skopeo/utils.go b/cmd/skopeo/utils.go index e78fce6d..7bd4d953 100644 --- a/cmd/skopeo/utils.go +++ b/cmd/skopeo/utils.go @@ -18,6 +18,7 @@ func contextFromGlobalOptions(c *cli.Context, flagPrefix string) (*types.SystemC DockerInsecureSkipTLSVerify: !c.GlobalBoolT("tls-verify"), OSTreeTmpDirPath: c.String(flagPrefix + "ostree-tmp-dir"), OCISharedBlobDirPath: c.String(flagPrefix + "shared-blob-dir"), + DirForceCompress: c.Bool(flagPrefix + "compress"), } if c.IsSet(flagPrefix + "tls-verify") { ctx.DockerInsecureSkipTLSVerify = !c.BoolT(flagPrefix + "tls-verify") diff --git a/completions/bash/skopeo b/completions/bash/skopeo index b21644c0..10f621f0 100644 --- a/completions/bash/skopeo +++ b/completions/bash/skopeo @@ -20,20 +20,24 @@ _complete_() { } _skopeo_copy() { - local options_with_args=" - --sign-by - --src-creds --screds - --src-cert-dir - --src-tls-verify - --dest-creds --dcreds - --dest-cert-dir - --dest-ostree-tmp-dir - --dest-tls-verify - " - local boolean_options=" - --remove-signatures - " - _complete_ "$options_with_args" "$boolean_options" + local options_with_args=" + --format -f + --sign-by + --src-creds --screds + --src-cert-dir + --src-tls-verify + --dest-creds --dcreds + --dest-cert-dir + --dest-ostree-tmp-dir + --dest-tls-verify + " + + local boolean_options=" + --dest-compress + --remove-signatures + " + + _complete_ "$options_with_args" "$boolean_options" } _skopeo_inspect() { diff --git a/docs/skopeo.1.md b/docs/skopeo.1.md index a955b828..bf050a4e 100644 --- a/docs/skopeo.1.md +++ b/docs/skopeo.1.md @@ -60,12 +60,16 @@ Uses the system's trust policy to validate images, rejects images not trusted by _destination-image_ use the "image name" format described above + **--format, -f** _manifest-type_ Manifest type (oci, v2s1, or v2s2) to use when saving image to directory using the 'dir:' transport (default is manifest type of source) + **--remove-signatures** do not copy signatures, if any, from _source-image_. Necessary when copying a signed image to a destination which does not support signatures. **--sign-by=**_key-id_ add a signature using that key ID for an image name corresponding to _destination-image_ **--src-creds** _username[:password]_ for accessing the source registry + **--dest-compress** _bool-value_ Compress tarball image layers when saving to directory using the 'dir' transport. (default is same compression type as source) + **--dest-creds** _username[:password]_ for accessing the destination registry **--src-cert-dir** _path_ Use certificates at _path_ (*.crt, *.cert, *.key) to connect to the source registry diff --git a/integration/copy_test.go b/integration/copy_test.go index 95e939bb..506a1b3f 100644 --- a/integration/copy_test.go +++ b/integration/copy_test.go @@ -15,6 +15,7 @@ import ( "github.com/containers/image/signature" "github.com/go-check/check" "github.com/opencontainers/go-digest" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/opencontainers/image-tools/image" ) @@ -591,6 +592,32 @@ func (s *CopySuite) TestCopySchemaConversion(c *check.C) { s.testCopySchemaConversionRegistries(c, "docker://"+v2s1DockerRegistryURL+"/schema1", "docker://"+v2DockerRegistryURL+"/schema2") } +func (s *CopySuite) TestCopyManifestConversion(c *check.C) { + topDir, err := ioutil.TempDir("", "manifest-conversion") + c.Assert(err, check.IsNil) + defer os.RemoveAll(topDir) + srcDir := filepath.Join(topDir, "source") + destDir1 := filepath.Join(topDir, "dest1") + destDir2 := filepath.Join(topDir, "dest2") + + // oci to v2s1 and vice-versa not supported yet + // get v2s2 manifest type + assertSkopeoSucceeds(c, "", "copy", "docker://busybox", "dir:"+srcDir) + verifyManifestMIMEType(c, srcDir, manifest.DockerV2Schema2MediaType) + // convert from v2s2 to oci + assertSkopeoSucceeds(c, "", "copy", "--format=oci", "dir:"+srcDir, "dir:"+destDir1) + verifyManifestMIMEType(c, destDir1, imgspecv1.MediaTypeImageManifest) + // convert from oci to v2s2 + assertSkopeoSucceeds(c, "", "copy", "--format=v2s2", "dir:"+destDir1, "dir:"+destDir2) + verifyManifestMIMEType(c, destDir2, manifest.DockerV2Schema2MediaType) + // convert from v2s2 to v2s1 + assertSkopeoSucceeds(c, "", "copy", "--format=v2s1", "dir:"+srcDir, "dir:"+destDir1) + verifyManifestMIMEType(c, destDir1, manifest.DockerV2Schema1SignedMediaType) + // convert from v2s1 to v2s2 + assertSkopeoSucceeds(c, "", "copy", "--format=v2s2", "dir:"+destDir1, "dir:"+destDir2) + verifyManifestMIMEType(c, destDir2, manifest.DockerV2Schema2MediaType) +} + func (s *CopySuite) testCopySchemaConversionRegistries(c *check.C, schema1Registry, schema2Registry string) { topDir, err := ioutil.TempDir("", "schema-conversion") c.Assert(err, check.IsNil) diff --git a/vendor.conf b/vendor.conf index 6aa535b9..f8ff8289 100644 --- a/vendor.conf +++ b/vendor.conf @@ -1,5 +1,5 @@ github.com/urfave/cli v1.17.0 -github.com/containers/image master +github.com/containers/image f950aa3529148eb0dea90888c24b6682da641b13 github.com/opencontainers/go-digest master gopkg.in/cheggaaa/pb.v1 ad4efe000aa550bb54918c06ebbadc0ff17687b9 https://github.com/cheggaaa/pb github.com/containers/storage master diff --git a/vendor/github.com/containers/image/copy/copy.go b/vendor/github.com/containers/image/copy/copy.go index 590b3787..be96b520 100644 --- a/vendor/github.com/containers/image/copy/copy.go +++ b/vendor/github.com/containers/image/copy/copy.go @@ -12,8 +12,6 @@ import ( "strings" "time" - pb "gopkg.in/cheggaaa/pb.v1" - "github.com/containers/image/image" "github.com/containers/image/pkg/compression" "github.com/containers/image/signature" @@ -22,6 +20,7 @@ import ( "github.com/opencontainers/go-digest" "github.com/pkg/errors" "github.com/sirupsen/logrus" + pb "gopkg.in/cheggaaa/pb.v1" ) type digestingReader struct { @@ -95,6 +94,8 @@ type Options struct { DestinationCtx *types.SystemContext ProgressInterval time.Duration // time to wait between reports to signal the progress channel Progress chan types.ProgressProperties // Reported to when ProgressInterval has arrived for a single artifact+offset. + // manifest MIME type of image set by user. "" is default and means use the autodetection to the the manifest MIME type + ForceManifestMIMEType string } // Image copies image from srcRef to destRef, using policyContext to validate @@ -193,7 +194,7 @@ func Image(policyContext *signature.PolicyContext, destRef, srcRef types.ImageRe // We compute preferredManifestMIMEType only to show it in error messages. // Without having to add this context in an error message, we would be happy enough to know only that no conversion is needed. - preferredManifestMIMEType, otherManifestMIMETypeCandidates, err := determineManifestConversion(&manifestUpdates, src, dest.SupportedManifestMIMETypes(), canModifyManifest) + preferredManifestMIMEType, otherManifestMIMETypeCandidates, err := determineManifestConversion(&manifestUpdates, src, dest.SupportedManifestMIMETypes(), canModifyManifest, options.ForceManifestMIMEType) if err != nil { return err } diff --git a/vendor/github.com/containers/image/copy/manifest.go b/vendor/github.com/containers/image/copy/manifest.go index e3b294dd..c4f582cb 100644 --- a/vendor/github.com/containers/image/copy/manifest.go +++ b/vendor/github.com/containers/image/copy/manifest.go @@ -41,12 +41,16 @@ func (os *orderedSet) append(s string) { // Note that the conversion will only happen later, through src.UpdatedImage // Returns the preferred manifest MIME type (whether we are converting to it or using it unmodified), // and a list of other possible alternatives, in order. -func determineManifestConversion(manifestUpdates *types.ManifestUpdateOptions, src types.Image, destSupportedManifestMIMETypes []string, canModifyManifest bool) (string, []string, error) { +func determineManifestConversion(manifestUpdates *types.ManifestUpdateOptions, src types.Image, destSupportedManifestMIMETypes []string, canModifyManifest bool, forceManifestMIMEType string) (string, []string, error) { _, srcType, err := src.Manifest() if err != nil { // This should have been cached?! return "", nil, errors.Wrap(err, "Error reading manifest") } + if forceManifestMIMEType != "" { + destSupportedManifestMIMETypes = []string{forceManifestMIMEType} + } + if len(destSupportedManifestMIMETypes) == 0 { return srcType, []string{}, nil // Anything goes; just use the original as is, do not try any conversions. } diff --git a/vendor/github.com/containers/image/directory/directory_dest.go b/vendor/github.com/containers/image/directory/directory_dest.go index ea46a27e..47d59d9f 100644 --- a/vendor/github.com/containers/image/directory/directory_dest.go +++ b/vendor/github.com/containers/image/directory/directory_dest.go @@ -4,19 +4,77 @@ import ( "io" "io/ioutil" "os" + "path/filepath" "github.com/containers/image/types" "github.com/opencontainers/go-digest" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) +const version = "Directory Transport Version: 1.0\n" + +// ErrNotContainerImageDir indicates that the directory doesn't match the expected contents of a directory created +// using the 'dir' transport +var ErrNotContainerImageDir = errors.New("not a containers image directory, don't want to overwrite important data") + type dirImageDestination struct { - ref dirReference + ref dirReference + compress bool } -// newImageDestination returns an ImageDestination for writing to an existing directory. -func newImageDestination(ref dirReference) types.ImageDestination { - return &dirImageDestination{ref} +// newImageDestination returns an ImageDestination for writing to a directory. +func newImageDestination(ref dirReference, compress bool) (types.ImageDestination, error) { + d := &dirImageDestination{ref: ref, compress: compress} + + // If directory exists check if it is empty + // if not empty, check whether the contents match that of a container image directory and overwrite the contents + // if the contents don't match throw an error + dirExists, err := pathExists(d.ref.resolvedPath) + if err != nil { + return nil, errors.Wrapf(err, "error checking for path %q", d.ref.resolvedPath) + } + if dirExists { + isEmpty, err := isDirEmpty(d.ref.resolvedPath) + if err != nil { + return nil, err + } + + if !isEmpty { + versionExists, err := pathExists(d.ref.versionPath()) + if err != nil { + return nil, errors.Wrapf(err, "error checking if path exists %q", d.ref.versionPath()) + } + if versionExists { + contents, err := ioutil.ReadFile(d.ref.versionPath()) + if err != nil { + return nil, err + } + // check if contents of version file is what we expect it to be + if string(contents) != version { + return nil, ErrNotContainerImageDir + } + } else { + return nil, ErrNotContainerImageDir + } + // delete directory contents so that only one image is in the directory at a time + if err = removeDirContents(d.ref.resolvedPath); err != nil { + return nil, errors.Wrapf(err, "error erasing contents in %q", d.ref.resolvedPath) + } + logrus.Debugf("overwriting existing container image directory %q", d.ref.resolvedPath) + } + } else { + // create directory if it doesn't exist + if err := os.MkdirAll(d.ref.resolvedPath, 0755); err != nil { + return nil, errors.Wrapf(err, "unable to create directory %q", d.ref.resolvedPath) + } + } + // create version file + err = ioutil.WriteFile(d.ref.versionPath(), []byte(version), 0755) + if err != nil { + return nil, errors.Wrapf(err, "error creating version file %q", d.ref.versionPath()) + } + return d, nil } // Reference returns the reference used to set up this destination. Note that this should directly correspond to user's intent, @@ -42,7 +100,7 @@ func (d *dirImageDestination) SupportsSignatures() error { // ShouldCompressLayers returns true iff it is desirable to compress layer blobs written to this destination. func (d *dirImageDestination) ShouldCompressLayers() bool { - return false + return d.compress } // AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually @@ -147,3 +205,39 @@ func (d *dirImageDestination) PutSignatures(signatures [][]byte) error { func (d *dirImageDestination) Commit() error { return nil } + +// returns true if path exists +func pathExists(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } + if err != nil && os.IsNotExist(err) { + return false, nil + } + return false, err +} + +// returns true if directory is empty +func isDirEmpty(path string) (bool, error) { + files, err := ioutil.ReadDir(path) + if err != nil { + return false, err + } + return len(files) == 0, nil +} + +// deletes the contents of a directory +func removeDirContents(path string) error { + files, err := ioutil.ReadDir(path) + if err != nil { + return err + } + + for _, file := range files { + if err := os.RemoveAll(filepath.Join(path, file.Name())); err != nil { + return err + } + } + return nil +} diff --git a/vendor/github.com/containers/image/directory/directory_transport.go b/vendor/github.com/containers/image/directory/directory_transport.go index b9ce01a2..48f0eb3c 100644 --- a/vendor/github.com/containers/image/directory/directory_transport.go +++ b/vendor/github.com/containers/image/directory/directory_transport.go @@ -152,7 +152,11 @@ func (ref dirReference) NewImageSource(ctx *types.SystemContext) (types.ImageSou // NewImageDestination returns a types.ImageDestination for this reference. // The caller must call .Close() on the returned ImageDestination. func (ref dirReference) NewImageDestination(ctx *types.SystemContext) (types.ImageDestination, error) { - return newImageDestination(ref), nil + compress := false + if ctx != nil { + compress = ctx.DirForceCompress + } + return newImageDestination(ref, compress) } // DeleteImage deletes the named image from the registry, if supported. @@ -175,3 +179,8 @@ func (ref dirReference) layerPath(digest digest.Digest) string { func (ref dirReference) signaturePath(index int) string { return filepath.Join(ref.path, fmt.Sprintf("signature-%d", index+1)) } + +// versionPath returns a path for the version file within a directory using our conventions. +func (ref dirReference) versionPath() string { + return filepath.Join(ref.path, "version") +} diff --git a/vendor/github.com/containers/image/docker/docker_image_dest.go b/vendor/github.com/containers/image/docker/docker_image_dest.go index 32d5a18b..79c38622 100644 --- a/vendor/github.com/containers/image/docker/docker_image_dest.go +++ b/vendor/github.com/containers/image/docker/docker_image_dest.go @@ -236,7 +236,7 @@ func (d *dockerImageDestination) PutManifest(m []byte) error { return err } defer res.Body.Close() - if res.StatusCode != http.StatusCreated { + if !successStatus(res.StatusCode) { err = errors.Wrapf(client.HandleErrorResponse(res), "Error uploading manifest to %s", path) if isManifestInvalidError(errors.Cause(err)) { err = types.ManifestTypeRejectedError{Err: err} @@ -246,6 +246,12 @@ func (d *dockerImageDestination) PutManifest(m []byte) error { return nil } +// successStatus returns true if the argument is a successful HTTP response +// code (in the range 200 - 399 inclusive). +func successStatus(status int) bool { + return status >= 200 && status <= 399 +} + // isManifestInvalidError returns true iff err from client.HandleErrorReponse is a “manifest invalid” error. func isManifestInvalidError(err error) bool { errors, ok := err.(errcode.Errors) diff --git a/vendor/github.com/containers/image/types/types.go b/vendor/github.com/containers/image/types/types.go index 685e67fa..17434c4e 100644 --- a/vendor/github.com/containers/image/types/types.go +++ b/vendor/github.com/containers/image/types/types.go @@ -349,6 +349,10 @@ type SystemContext struct { DockerDaemonHost string // Used to skip TLS verification, off by default. To take effect DockerDaemonCertPath needs to be specified as well. DockerDaemonInsecureSkipTLSVerify bool + + // === dir.Transport overrides === + // DirForceCompress compresses the image layers if set to true + DirForceCompress bool } // ProgressProperties is used to pass information from the copy code to a monitor which