add possibility to download to OCI image-layout

- vendor containers/image c703326038d30c3422168dd9a1a5afaf51740331
- fix copy tests relying on v2s1 manifests

Signed-off-by: Antonio Murdaca <runcom@redhat.com>

cmd/skopeo/copy.go

@@ -14,16 +14,17 @@ func copyHandler(context *cli.Context) error {
 		return errors.New("Usage: copy source destination")
 	}
 
-	rawSource, err := parseImageSource(context, context.Args()[0])
-	if err != nil {
-		return fmt.Errorf("Error initializing %s: %v", context.Args()[0], err)
-	}
-	src := image.FromSource(rawSource)
-
 	dest, err := parseImageDestination(context, context.Args()[1])
 	if err != nil {
 		return fmt.Errorf("Error initializing %s: %v", context.Args()[1], err)
 	}
+
+	rawSource, err := parseImageSource(context, context.Args()[0])
+	if err != nil {
+		return fmt.Errorf("Error initializing %s: %v", context.Args()[0], err)
+	}
+	src := image.FromSource(rawSource, dest.SupportedManifestMIMETypes())
+
 	signBy := context.String("sign-by")
 
 	manifest, _, err := src.Manifest()

cmd/skopeo/inspect.go

@@ -13,7 +13,7 @@ import (
 // inspectOutput is the output format of (skopeo inspect), primarily so that we can format it with a simple json.MarshalIndent.
 type inspectOutput struct {
 	Name          string `json:",omitempty"`
-	Tag           string
+	Tag           string `json:",omitempty"`
 	Digest        string
 	RepoTags      []string
 	Created       time.Time

cmd/skopeo/layers.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/containers/image/directory"
 	"github.com/containers/image/image"
+	"github.com/containers/image/manifest"
 	"github.com/urfave/cli"
 )
 
@@ -18,7 +19,12 @@ var layersCmd = cli.Command{
 		if err != nil {
 			return err
 		}
-		src := image.FromSource(rawSource)
+		src := image.FromSource(rawSource, []string{
+			// TODO: skopeo layers only support these now
+			// eventually we'll remove this command altogether...
+			manifest.DockerV2Schema1SignedMIMEType,
+			manifest.DockerV2Schema1MIMEType,
+		})
 		blobDigests := c.Args().Tail()
 		if len(blobDigests) == 0 {
 			b, err := src.BlobDigests()

cmd/skopeo/utils.go

@@ -8,6 +8,7 @@ import (
 	"github.com/containers/image/directory"
 	"github.com/containers/image/docker"
 	"github.com/containers/image/image"
+	"github.com/containers/image/oci"
 	"github.com/containers/image/openshift"
 	"github.com/containers/image/types"
 	"github.com/urfave/cli"
@@ -20,6 +21,8 @@ const (
 	dockerPrefix = "docker://"
 	// directoryPrefix is the URL-like schema prefix used for local directories (for debugging)
 	directoryPrefix = "dir:"
+	// ociPrefix is the URL-like schema prefix used for OCI images.
+	ociPrefix = "oci:"
 )
 
 // ParseImage converts image URL-like string to an initialized handler for that image.
@@ -36,7 +39,7 @@ func parseImage(c *cli.Context) (types.Image, error) {
 		//
 	case strings.HasPrefix(imgName, directoryPrefix):
 		src := directory.NewDirImageSource(strings.TrimPrefix(imgName, directoryPrefix))
-		return image.FromSource(src), nil
+		return image.FromSource(src, nil), nil
 	}
 	return nil, errors.New("no valid prefix provided")
 }
@@ -71,6 +74,8 @@ func parseImageDestination(c *cli.Context, name string) (types.ImageDestination,
 		return openshift.NewOpenshiftImageDestination(strings.TrimPrefix(name, atomicPrefix), certPath, tlsVerify)
 	case strings.HasPrefix(name, directoryPrefix):
 		return directory.NewDirImageDestination(strings.TrimPrefix(name, directoryPrefix)), nil
+	case strings.HasPrefix(name, ociPrefix):
+		return oci.NewOCIImageDestination(strings.TrimPrefix(name, ociPrefix))
 	}
 	return nil, fmt.Errorf("Unrecognized image reference %s", name)
 }

integration/copy_test.go

@@ -55,7 +55,7 @@ func (s *CopySuite) TestCopySimple(c *check.C) {
 
 	// FIXME: It would be nice to use one of the local Docker registries instead of neeeding an Internet connection.
 	// "pull": docker: → dir:
-	assertSkopeoSucceeds(c, "", "copy", "docker://busybox:latest", "dir:"+dir1)
+	assertSkopeoSucceeds(c, "", "copy", "docker://estesp/busybox:latest", "dir:"+dir1)
 	// "push": dir: → atomic:
 	assertSkopeoSucceeds(c, "", "--debug", "copy", "dir:"+dir1, "atomic:myns/unsigned:unsigned")
 	// The result of pushing and pulling is an unmodified image.
@@ -63,6 +63,15 @@ func (s *CopySuite) TestCopySimple(c *check.C) {
 	out := combinedOutputOfCommand(c, "diff", "-urN", dir1, dir2)
 	c.Assert(out, check.Equals, "")
 
+	// docker v2s2 -> OCI image layout
+	// ociDest will be created by oci: if it doesn't exist
+	// so don't create it here to exercise auto-creation
+	ociDest := "busybox-latest"
+	defer os.RemoveAll(ociDest)
+	assertSkopeoSucceeds(c, "", "copy", "docker://busybox:latest", "oci:"+ociDest)
+	_, err = os.Stat(ociDest)
+	c.Assert(err, check.IsNil)
+
 	// FIXME: Also check pushing to docker://
 }
 
@@ -77,9 +86,9 @@ func (s *CopySuite) TestCopyStreaming(c *check.C) {
 
 	// FIXME: It would be nice to use one of the local Docker registries instead of neeeding an Internet connection.
 	// streaming: docker: → atomic:
-	assertSkopeoSucceeds(c, "", "--debug", "copy", "docker://busybox:1-glibc", "atomic:myns/unsigned:streaming")
+	assertSkopeoSucceeds(c, "", "--debug", "copy", "docker://estesp/busybox:amd64", "atomic:myns/unsigned:streaming")
 	// Compare (copies of) the original and the copy:
-	assertSkopeoSucceeds(c, "", "copy", "docker://busybox:1-glibc", "dir:"+dir1)
+	assertSkopeoSucceeds(c, "", "copy", "docker://estesp/busybox:amd64", "dir:"+dir1)
 	assertSkopeoSucceeds(c, "", "copy", "atomic:myns/unsigned:streaming", "dir:"+dir2)
 	// The manifests will have different JWS signatures; so, compare the manifests by digests, which
 	// strips the signatures, and remove them, comparing the rest file by file.

vendor/github.com/containers/image/directory/directory_dest.go

@@ -22,6 +22,10 @@ func (d *dirImageDestination) CanonicalDockerReference() (string, error) {
 	return "", fmt.Errorf("Can not determine canonical Docker reference for a local directory")
 }
 
+func (d *dirImageDestination) SupportedManifestMIMETypes() []string {
+	return nil
+}
+
 func (d *dirImageDestination) PutManifest(manifest []byte) error {
 	return ioutil.WriteFile(manifestPath(d.dir), manifest, 0644)
 }

vendor/github.com/containers/image/docker/docker_image.go

@@ -23,7 +23,7 @@ func NewDockerImage(img, certPath string, tlsVerify bool) (types.Image, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &Image{Image: image.FromSource(s), src: s}, nil
+	return &Image{Image: image.FromSource(s, nil), src: s}, nil
 }
 
 // SourceRefFullName returns a fully expanded name for the repository this image is in.

vendor/github.com/containers/image/docker/docker_image_dest.go

@@ -8,8 +8,8 @@ import (
 	"net/http"
 
 	"github.com/Sirupsen/logrus"
-	"github.com/containers/image/manifest"
 	"github.com/containers/image/docker/reference"
+	"github.com/containers/image/manifest"
 	"github.com/containers/image/types"
 )
 
@@ -36,6 +36,15 @@ func NewDockerImageDestination(img, certPath string, tlsVerify bool) (types.Imag
 	}, nil
 }
 
+func (d *dockerImageDestination) SupportedManifestMIMETypes() []string {
+	return []string{
+		// TODO(runcom): we'll add OCI as part of another PR here
+		manifest.DockerV2Schema2MIMEType,
+		manifest.DockerV2Schema1SignedMIMEType,
+		manifest.DockerV2Schema1MIMEType,
+	}
+}
+
 func (d *dockerImageDestination) CanonicalDockerReference() (string, error) {
 	return fmt.Sprintf("%s:%s", d.ref.Name(), d.tag), nil
 }

vendor/github.com/containers/image/docker/docker_image_src.go

@@ -95,7 +95,7 @@ func (s *dockerImageSource) GetManifest(mimetypes []string) ([]byte, string, err
 
 func (s *dockerImageSource) GetBlob(digest string) (io.ReadCloser, int64, error) {
 	url := fmt.Sprintf(blobsURL, s.ref.RemoteName(), digest)
-	logrus.Infof("Downloading %s", url)
+	logrus.Debugf("Downloading %s", url)
 	res, err := s.c.makeRequest("GET", url, nil, nil)
 	if err != nil {
 		return nil, 0, err

vendor/github.com/containers/image/image/image.go

@@ -33,12 +33,21 @@ type genericImage struct {
 	// this field is valid only if cachedManifest is not nil
 	cachedManifestMIMEType string
 	// private cache for Signatures(); nil if not yet known.
-	cachedSignatures [][]byte
+	cachedSignatures           [][]byte
+	requestedManifestMIMETypes []string
 }
 
 // FromSource returns a types.Image implementation for source.
-func FromSource(src types.ImageSource) types.Image {
+func FromSource(src types.ImageSource, requestedManifestMIMETypes []string) types.Image {
-	return &genericImage{src: src}
+	if len(requestedManifestMIMETypes) == 0 {
+		requestedManifestMIMETypes = []string{
+			manifest.OCIV1ImageManifestMIMEType,
+			manifest.DockerV2Schema2MIMEType,
+			manifest.DockerV2Schema1SignedMIMEType,
+			manifest.DockerV2Schema1MIMEType,
+		}
+	}
+	return &genericImage{src: src, requestedManifestMIMETypes: requestedManifestMIMETypes}
 }
 
 // IntendedDockerReference returns the full, unambiguous, Docker reference for this image, _as specified by the user_
@@ -52,7 +61,7 @@ func (i *genericImage) IntendedDockerReference() string {
 // NOTE: It is essential for signature verification that Manifest returns the manifest from which BlobDigests is computed.
 func (i *genericImage) Manifest() ([]byte, string, error) {
 	if i.cachedManifest == nil {
-		m, mt, err := i.src.GetManifest([]string{manifest.DockerV2Schema1SignedMIMEType, manifest.DockerV2Schema1MIMEType})
+		m, mt, err := i.src.GetManifest(i.requestedManifestMIMETypes)
 		if err != nil {
 			return nil, "", err
 		}

vendor/github.com/containers/image/manifest/manifest.go

@@ -20,18 +20,17 @@ const (
 	DockerV2Schema2MIMEType = "application/vnd.docker.distribution.manifest.v2+json"
 	// DockerV2ListMIMEType MIME type represents Docker manifest schema 2 list
 	DockerV2ListMIMEType = "application/vnd.docker.distribution.manifest.list.v2+json"
-	// OCIV1DescriptorMIMEType TODO
+
+	// OCIV1DescriptorMIMEType specifies the mediaType for a content descriptor.
 	OCIV1DescriptorMIMEType = "application/vnd.oci.descriptor.v1+json"
-	// OCIV1ImageManifestMIMEType TODO
+	// OCIV1ImageManifestMIMEType specifies the mediaType for an image manifest.
 	OCIV1ImageManifestMIMEType = "application/vnd.oci.image.manifest.v1+json"
-	// OCIV1ImageManifestListMIMEType TODO
+	// OCIV1ImageManifestListMIMEType specifies the mediaType for an image manifest list.
 	OCIV1ImageManifestListMIMEType = "application/vnd.oci.image.manifest.list.v1+json"
-	// OCIV1ImageSerializationRootfsTarGzipMIMEType TODO)
+	// OCIV1ImageSerializationMIMEType is the mediaType used for layers referenced by the manifest.
-	OCIV1ImageSerializationRootfsTarGzipMIMEType = "application/vnd.oci.image.serialization.rootfs.tar.gzip"
+	OCIV1ImageSerializationMIMEType = "application/vnd.oci.image.serialization.rootfs.tar.gzip"
-	// OCIV1ImageSerializationConfigMIMEType TODO
+	// OCIV1ImageSerializationConfigMIMEType specifies the mediaType for the image configuration.
 	OCIV1ImageSerializationConfigMIMEType = "application/vnd.oci.image.serialization.config.v1+json"
-	// OCIV1ImageSerializationCombinedMIMEType TODO
-	OCIV1ImageSerializationCombinedMIMEType = "application/vnd.oci.image.serialization.combined.v1+json"
 )
 
 // GuessMIMEType guesses MIME type of a manifest and returns it _if it is recognized_, or "" if unknown or unrecognized.
@@ -50,7 +49,7 @@ func GuessMIMEType(manifest []byte) string {
 	}
 
 	switch meta.MediaType {
-	case DockerV2Schema2MIMEType, DockerV2ListMIMEType, OCIV1DescriptorMIMEType, OCIV1ImageManifestMIMEType, OCIV1ImageManifestListMIMEType, OCIV1ImageSerializationRootfsTarGzipMIMEType, OCIV1ImageSerializationConfigMIMEType, OCIV1ImageSerializationCombinedMIMEType: // A recognized type.
+	case DockerV2Schema2MIMEType, DockerV2ListMIMEType, OCIV1DescriptorMIMEType, OCIV1ImageManifestMIMEType, OCIV1ImageManifestListMIMEType: // A recognized type.
 		return meta.MediaType
 	}
 	// this is the only way the function can return DockerV2Schema1MIMEType, and recognizing that is essential for stripping the JWS signatures = computing the correct manifest digest.

vendor/github.com/containers/image/oci/oci_dest.go

@@ -0,0 +1,183 @@
+package oci
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/containers/image/manifest"
+	"github.com/containers/image/types"
+)
+
+type ociManifest struct {
+	SchemaVersion int               `json:"schemaVersion"`
+	MediaType     string            `json:"mediaType"`
+	Config        descriptor        `json:"config"`
+	Layers        []descriptor      `json:"layers"`
+	Annotations   map[string]string `json:"annotations"`
+}
+
+type descriptor struct {
+	Digest    string `json:"digest"`
+	MediaType string `json:"mediaType"`
+	Size      int64  `json:"size"`
+}
+
+type ociImageDestination struct {
+	dir string
+	tag string
+}
+
+var refRegexp = regexp.MustCompile(`^([A-Za-z0-9._-]+)+$`)
+
+// NewOCIImageDestination returns an ImageDestination for writing to an existing directory.
+func NewOCIImageDestination(dest string) (types.ImageDestination, error) {
+	dir := dest
+	sep := strings.LastIndex(dest, ":")
+	tag := "latest"
+	if sep != -1 {
+		dir = dest[:sep]
+		tag = dest[sep+1:]
+		if !refRegexp.MatchString(tag) {
+			return nil, fmt.Errorf("Invalid reference %s", tag)
+		}
+	}
+	return &ociImageDestination{
+		dir: dir,
+		tag: tag,
+	}, nil
+}
+
+func (d *ociImageDestination) CanonicalDockerReference() (string, error) {
+	return "", fmt.Errorf("Can not determine canonical Docker reference for an OCI image")
+}
+
+func createManifest(m []byte) ([]byte, string, error) {
+	om := ociManifest{}
+	mt := manifest.GuessMIMEType(m)
+	switch mt {
+	case manifest.DockerV2Schema1MIMEType:
+		// There a simple reason about not yet implementing this.
+		// OCI image-spec assure about backward compatibility with docker v2s2 but not v2s1
+		// generating a v2s2 is a migration docker does when upgrading to 1.10.3
+		// and I don't think we should bother about this now (I don't want to have migration code here in skopeo)
+		return nil, "", fmt.Errorf("can't create OCI manifest from Docker V2 schema 1 manifest")
+	case manifest.DockerV2Schema2MIMEType:
+		if err := json.Unmarshal(m, &om); err != nil {
+			return nil, "", err
+		}
+		om.MediaType = manifest.OCIV1ImageManifestMIMEType
+		for i := range om.Layers {
+			om.Layers[i].MediaType = manifest.OCIV1ImageSerializationMIMEType
+		}
+		om.Config.MediaType = manifest.OCIV1ImageSerializationConfigMIMEType
+		b, err := json.Marshal(om)
+		if err != nil {
+			return nil, "", err
+		}
+		return b, om.MediaType, nil
+	case manifest.DockerV2ListMIMEType:
+		return nil, "", fmt.Errorf("can't create OCI manifest from Docker V2 schema 2 manifest list")
+	case manifest.OCIV1ImageManifestListMIMEType:
+		return nil, "", fmt.Errorf("can't create OCI manifest from OCI manifest list")
+	case manifest.OCIV1ImageManifestMIMEType:
+		return m, om.MediaType, nil
+	}
+	return nil, "", fmt.Errorf("Unrecognized manifest media type")
+}
+
+func (d *ociImageDestination) PutManifest(m []byte) error {
+	if err := d.ensureParentDirectoryExists("refs"); err != nil {
+		return err
+	}
+	// TODO(mitr, runcom): this breaks signatures entirely since at this point we're creating a new manifest
+	// and signatures don't apply anymore. Will fix.
+	ociMan, mt, err := createManifest(m)
+	if err != nil {
+		return err
+	}
+	digest, err := manifest.Digest(ociMan)
+	if err != nil {
+		return err
+	}
+	desc := descriptor{}
+	desc.Digest = digest
+	// TODO(runcom): beaware and add support for OCI manifest list
+	desc.MediaType = mt
+	desc.Size = int64(len(ociMan))
+	data, err := json.Marshal(desc)
+	if err != nil {
+		return err
+	}
+
+	if err := ioutil.WriteFile(blobPath(d.dir, digest), ociMan, 0644); err != nil {
+		return err
+	}
+	// TODO(runcom): ugly here?
+	if err := ioutil.WriteFile(ociLayoutPath(d.dir), []byte(`{"imageLayoutVersion": "1.0.0"}`), 0644); err != nil {
+		return err
+	}
+	return ioutil.WriteFile(descriptorPath(d.dir, d.tag), data, 0644)
+}
+
+func (d *ociImageDestination) PutBlob(digest string, stream io.Reader) error {
+	if err := d.ensureParentDirectoryExists("blobs"); err != nil {
+		return err
+	}
+	blob, err := os.Create(blobPath(d.dir, digest))
+	if err != nil {
+		return err
+	}
+	defer blob.Close()
+	if _, err := io.Copy(blob, stream); err != nil {
+		return err
+	}
+	if err := blob.Sync(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (d *ociImageDestination) ensureParentDirectoryExists(parent string) error {
+	path := filepath.Join(d.dir, parent)
+	if _, err := os.Stat(path); err != nil && os.IsNotExist(err) {
+		if err := os.MkdirAll(path, 0755); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (d *ociImageDestination) SupportedManifestMIMETypes() []string {
+	return []string{
+		manifest.OCIV1ImageManifestMIMEType,
+		manifest.DockerV2Schema2MIMEType,
+	}
+}
+
+func (d *ociImageDestination) PutSignatures(signatures [][]byte) error {
+	if len(signatures) != 0 {
+		return fmt.Errorf("Pushing signatures for OCI images is not supported")
+	}
+	return nil
+}
+
+// ociLayoutPathPath returns a path for the oci-layout within a directory using OCI conventions.
+func ociLayoutPath(dir string) string {
+	return filepath.Join(dir, "oci-layout")
+}
+
+// blobPath returns a path for a blob within a directory using OCI image-layout conventions.
+func blobPath(dir string, digest string) string {
+	return filepath.Join(dir, "blobs", strings.Replace(digest, ":", "-", -1))
+}
+
+// descriptorPath returns a path for the manifest within a directory using OCI conventions.
+func descriptorPath(dir string, digest string) string {
+	return filepath.Join(dir, "refs", digest)
+}

vendor/github.com/containers/image/openshift/openshift.go

@@ -283,6 +283,13 @@ func NewOpenshiftImageDestination(imageName, certPath string, tlsVerify bool) (t
 	}, nil
 }
 
+func (d *openshiftImageDestination) SupportedManifestMIMETypes() []string {
+	return []string{
+		manifest.DockerV2Schema1SignedMIMEType,
+		manifest.DockerV2Schema1MIMEType,
+	}
+}
+
 func (d *openshiftImageDestination) CanonicalDockerReference() (string, error) {
 	return d.client.canonicalDockerReference(), nil
 }

vendor/github.com/containers/image/types/types.go

@@ -34,6 +34,9 @@ type ImageDestination interface {
 	// Note: Calling PutBlob() and other methods may have ordering dependencies WRT other methods of this type. FIXME: Figure out and document.
 	PutBlob(digest string, stream io.Reader) error
 	PutSignatures(signatures [][]byte) error
+	// SupportedManifestMIMETypes tells which manifest mime types the destination supports
+	// If an empty slice or nil it's returned, then any mime type can be tried to upload
+	SupportedManifestMIMETypes() []string
 }
 
 // Image is the primary API for inspecting properties of images.