Fix skopeo tests with changes to dir transport

The dir transport has been changed to save the blobs without the .tar extension
Fixes the skopeo tests failing due to this change

Signed-off-by: umohnani8 <umohnani@redhat.com>

integration/copy_test.go

@@ -379,7 +379,7 @@ func (s *CopySuite) TestCopyDirSignatures(c *check.C) {
 
 // Compression during copy
 func (s *CopySuite) TestCopyCompression(c *check.C) {
-	const uncompresssedLayerFile = "160d823fdc48e62f97ba62df31e55424f8f5eb6b679c865eec6e59adfe304710.tar"
+	const uncompresssedLayerFile = "160d823fdc48e62f97ba62df31e55424f8f5eb6b679c865eec6e59adfe304710"
 
 	topDir, err := ioutil.TempDir("", "compression-top")
 	c.Assert(err, check.IsNil)
@@ -411,9 +411,7 @@ func (s *CopySuite) TestCopyCompression(c *check.C) {
 		fis, err := dirf.Readdir(-1)
 		c.Assert(err, check.IsNil)
 		for _, fi := range fis {
-			if strings.HasSuffix(fi.Name(), ".tar") {
-				c.Assert(fi.Size() < 2048, check.Equals, true)
-			}
+			c.Assert(fi.Size() < 2048, check.Equals, true)
 		}
 	}
 }

vendor/github.com/containers/image/copy/copy.go

@@ -368,7 +368,10 @@ func (ic *imageCopier) copyLayers() error {
 	srcInfos := ic.src.LayerInfos()
 	destInfos := []types.BlobInfo{}
 	diffIDs := []digest.Digest{}
-	updatedSrcInfos := ic.src.LayerInfosForCopy()
+	updatedSrcInfos, err := ic.src.LayerInfosForCopy()
+	if err != nil {
+		return err
+	}
 	srcInfosUpdated := false
 	if updatedSrcInfos != nil && !reflect.DeepEqual(srcInfos, updatedSrcInfos) {
 		if !ic.canModifyManifest {

vendor/github.com/containers/image/copy/manifest.go

@@ -46,6 +46,11 @@ func (ic *imageCopier) determineManifestConversion(destSupportedManifestMIMEType
 	if err != nil { // This should have been cached?!
 		return "", nil, errors.Wrap(err, "Error reading manifest")
 	}
+	normalizedSrcType := manifest.NormalizedMIMEType(srcType)
+	if srcType != normalizedSrcType {
+		logrus.Debugf("Source manifest MIME type %s, treating it as %s", srcType, normalizedSrcType)
+		srcType = normalizedSrcType
+	}
 
 	if forceManifestMIMEType != "" {
 		destSupportedManifestMIMETypes = []string{forceManifestMIMEType}

vendor/github.com/containers/image/directory/directory_dest.go

@@ -12,7 +12,7 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-const version = "Directory Transport Version: 1.0\n"
+const version = "Directory Transport Version: 1.1\n"
 
 // ErrNotContainerImageDir indicates that the directory doesn't match the expected contents of a directory created
 // using the 'dir' transport

vendor/github.com/containers/image/directory/directory_src.go

@@ -52,11 +52,11 @@ func (s *dirImageSource) GetManifest(instanceDigest *digest.Digest) ([]byte, str
 func (s *dirImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int64, error) {
 	r, err := os.Open(s.ref.layerPath(info.Digest))
 	if err != nil {
-		return nil, 0, nil
+		return nil, -1, err
 	}
 	fi, err := r.Stat()
 	if err != nil {
-		return nil, 0, nil
+		return nil, -1, err
 	}
 	return r, fi.Size(), nil
 }
@@ -84,6 +84,6 @@ func (s *dirImageSource) GetSignatures(ctx context.Context, instanceDigest *dige
 }
 
 // LayerInfosForCopy() returns updated layer info that should be used when copying, in preference to values in the manifest, if specified.
-func (s *dirImageSource) LayerInfosForCopy() []types.BlobInfo {
-	return nil
+func (s *dirImageSource) LayerInfosForCopy() ([]types.BlobInfo, error) {
+	return nil, nil
 }

vendor/github.com/containers/image/directory/directory_transport.go

@@ -5,14 +5,13 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/pkg/errors"
-
 	"github.com/containers/image/directory/explicitfilepath"
 	"github.com/containers/image/docker/reference"
 	"github.com/containers/image/image"
 	"github.com/containers/image/transports"
 	"github.com/containers/image/types"
 	"github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
 )
 
 func init() {
@@ -173,7 +172,7 @@ func (ref dirReference) manifestPath() string {
 // layerPath returns a path for a layer tarball within a directory using our conventions.
 func (ref dirReference) layerPath(digest digest.Digest) string {
 	// FIXME: Should we keep the digest identification?
-	return filepath.Join(ref.path, digest.Hex()+".tar")
+	return filepath.Join(ref.path, digest.Hex())
 }
 
 // signaturePath returns a path for a signature within a directory using our conventions.

vendor/github.com/containers/image/docker/archive/src.go

@@ -36,6 +36,6 @@ func (s *archiveImageSource) Close() error {
 }
 
 // LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified.
-func (s *archiveImageSource) LayerInfosForCopy() []types.BlobInfo {
-	return nil
+func (s *archiveImageSource) LayerInfosForCopy() ([]types.BlobInfo, error) {
+	return nil, nil
 }

vendor/github.com/containers/image/docker/daemon/daemon_src.go

@@ -83,6 +83,6 @@ func (s *daemonImageSource) Close() error {
 }
 
 // LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified.
-func (s *daemonImageSource) LayerInfosForCopy() []types.BlobInfo {
-	return nil
+func (s *daemonImageSource) LayerInfosForCopy() ([]types.BlobInfo, error) {
+	return nil, nil
 }

vendor/github.com/containers/image/docker/docker_image_src.go

@@ -53,8 +53,8 @@ func (s *dockerImageSource) Close() error {
 }
 
 // LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified.
-func (s *dockerImageSource) LayerInfosForCopy() []types.BlobInfo {
-	return nil
+func (s *dockerImageSource) LayerInfosForCopy() ([]types.BlobInfo, error) {
+	return nil, nil
 }
 
 // simplifyContentType drops parameters from a HTTP media type (see https://tools.ietf.org/html/rfc7231#section-3.1.1.1)

vendor/github.com/containers/image/image/docker_schema1.go

@@ -95,7 +95,7 @@ func (m *manifestSchema1) imageInspectInfo() (*types.ImageInspectInfo, error) {
 // This is a horribly specific interface, but computing InformationOnly.LayerDiffIDs can be very expensive to compute
 // (most importantly it forces us to download the full layers even if they are already present at the destination).
 func (m *manifestSchema1) UpdatedImageNeedsLayerDiffIDs(options types.ManifestUpdateOptions) bool {
-	return options.ManifestMIMEType == manifest.DockerV2Schema2MediaType
+	return (options.ManifestMIMEType == manifest.DockerV2Schema2MediaType || options.ManifestMIMEType == imgspecv1.MediaTypeImageManifest)
 }
 
 // UpdatedImage returns a types.Image modified according to options.

vendor/github.com/containers/image/image/memory.go

@@ -65,6 +65,6 @@ func (i *memoryImage) Inspect() (*types.ImageInspectInfo, error) {
 // LayerInfosForCopy returns an updated set of layer blob information which may not match the manifest.
 // The Digest field is guaranteed to be provided; Size may be -1.
 // WARNING: The list may contain duplicates, and they are semantically relevant.
-func (i *memoryImage) LayerInfosForCopy() []types.BlobInfo {
-	return nil
+func (i *memoryImage) LayerInfosForCopy() ([]types.BlobInfo, error) {
+	return nil, nil
 }

vendor/github.com/containers/image/image/sourced.go

@@ -101,6 +101,6 @@ func (i *sourcedImage) Inspect() (*types.ImageInspectInfo, error) {
 	return inspectManifest(i.genericManifest)
 }
 
-func (i *sourcedImage) LayerInfosForCopy() []types.BlobInfo {
+func (i *sourcedImage) LayerInfosForCopy() ([]types.BlobInfo, error) {
 	return i.UnparsedImage.LayerInfosForCopy()
 }

vendor/github.com/containers/image/image/unparsed.go

@@ -97,6 +97,6 @@ func (i *UnparsedImage) Signatures(ctx context.Context) ([][]byte, error) {
 // LayerInfosForCopy returns an updated set of layer blob information which may not match the manifest.
 // The Digest field is guaranteed to be provided; Size may be -1.
 // WARNING: The list may contain duplicates, and they are semantically relevant.
-func (i *UnparsedImage) LayerInfosForCopy() []types.BlobInfo {
+func (i *UnparsedImage) LayerInfosForCopy() ([]types.BlobInfo, error) {
 	return i.src.LayerInfosForCopy()
 }

vendor/github.com/containers/image/oci/archive/oci_src.go

@@ -90,6 +90,6 @@ func (s *ociArchiveImageSource) GetSignatures(ctx context.Context, instanceDiges
 }
 
 // LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified.
-func (s *ociArchiveImageSource) LayerInfosForCopy() []types.BlobInfo {
-	return nil
+func (s *ociArchiveImageSource) LayerInfosForCopy() ([]types.BlobInfo, error) {
+	return nil, nil
 }

vendor/github.com/containers/image/oci/layout/oci_src.go

@@ -144,8 +144,8 @@ func (s *ociImageSource) getExternalBlob(urls []string) (io.ReadCloser, int64, e
 }
 
 // LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified.
-func (s *ociImageSource) LayerInfosForCopy() []types.BlobInfo {
-	return nil
+func (s *ociImageSource) LayerInfosForCopy() ([]types.BlobInfo, error) {
+	return nil, nil
 }
 
 func getBlobSize(resp *http.Response) int64 {

vendor/github.com/containers/image/openshift/openshift.go

@@ -247,8 +247,8 @@ func (s *openshiftImageSource) GetSignatures(ctx context.Context, instanceDigest
 }
 
 // LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified.
-func (s *openshiftImageSource) LayerInfosForCopy() []types.BlobInfo {
-	return nil
+func (s *openshiftImageSource) LayerInfosForCopy() ([]types.BlobInfo, error) {
+	return nil, nil
 }
 
 // ensureImageIsResolved sets up s.docker and s.imageStreamImageName

vendor/github.com/containers/image/ostree/ostree_dest.go

@@ -230,33 +230,36 @@ func (d *ostreeImageDestination) ostreeCommit(repo *otbuiltin.Repo, branch strin
 	return err
 }
 
-func generateTarSplitMetadata(output *bytes.Buffer, file string) error {
+func generateTarSplitMetadata(output *bytes.Buffer, file string) (digest.Digest, int64, error) {
 	mfz := gzip.NewWriter(output)
 	defer mfz.Close()
 	metaPacker := storage.NewJSONPacker(mfz)
 
 	stream, err := os.OpenFile(file, os.O_RDONLY, 0)
 	if err != nil {
-		return err
+		return "", -1, err
 	}
 	defer stream.Close()
 
 	gzReader, err := archive.DecompressStream(stream)
 	if err != nil {
-		return err
+		return "", -1, err
 	}
 	defer gzReader.Close()
 
 	its, err := asm.NewInputTarStream(gzReader, metaPacker, nil)
 	if err != nil {
-		return err
+		return "", -1, err
 	}
 
-	_, err = io.Copy(ioutil.Discard, its)
+	digester := digest.Canonical.Digester()
+
+	written, err := io.Copy(digester.Hash(), its)
 	if err != nil {
-		return err
+		return "", -1, err
 	}
-	return nil
+
+	return digester.Digest(), written, nil
 }
 
 func (d *ostreeImageDestination) importBlob(selinuxHnd *C.struct_selabel_handle, repo *otbuiltin.Repo, blob *blobToImport) error {
@@ -271,7 +274,8 @@ func (d *ostreeImageDestination) importBlob(selinuxHnd *C.struct_selabel_handle,
 	}()
 
 	var tarSplitOutput bytes.Buffer
-	if err := generateTarSplitMetadata(&tarSplitOutput, blob.BlobPath); err != nil {
+	uncompressedDigest, uncompressedSize, err := generateTarSplitMetadata(&tarSplitOutput, blob.BlobPath)
+	if err != nil {
 		return err
 	}
 
@@ -293,6 +297,8 @@ func (d *ostreeImageDestination) importBlob(selinuxHnd *C.struct_selabel_handle,
 		}
 	}
 	return d.ostreeCommit(repo, ostreeBranch, destinationPath, []string{fmt.Sprintf("docker.size=%d", blob.Size),
+		fmt.Sprintf("docker.uncompressed_size=%d", uncompressedSize),
+		fmt.Sprintf("docker.uncompressed_digest=%s", uncompressedDigest.String()),
 		fmt.Sprintf("tarsplit.output=%s", base64.StdEncoding.EncodeToString(tarSplitOutput.Bytes()))})
 
 }
@@ -315,7 +321,17 @@ func (d *ostreeImageDestination) HasBlob(info types.BlobInfo) (bool, int64, erro
 	}
 	branch := fmt.Sprintf("ociimage/%s", info.Digest.Hex())
 
-	found, data, err := readMetadata(d.repo, branch, "docker.size")
+	found, data, err := readMetadata(d.repo, branch, "docker.uncompressed_digest")
+	if err != nil || !found {
+		return found, -1, err
+	}
+
+	found, data, err = readMetadata(d.repo, branch, "docker.uncompressed_size")
+	if err != nil || !found {
+		return found, -1, err
+	}
+
+	found, data, err = readMetadata(d.repo, branch, "docker.size")
 	if err != nil || !found {
 		return found, -1, err
 	}

vendor/github.com/containers/image/ostree/ostree_src.go

@@ -37,11 +37,13 @@ type ostreeImageSource struct {
 	ref    ostreeReference
 	tmpDir string
 	repo   *C.struct_OstreeRepo
+	// get the compressed layer by its uncompressed checksum
+	compressed map[digest.Digest]digest.Digest
 }
 
 // newImageSource returns an ImageSource for reading from an existing directory.
 func newImageSource(ctx *types.SystemContext, tmpDir string, ref ostreeReference) (types.ImageSource, error) {
-	return &ostreeImageSource{ref: ref, tmpDir: tmpDir}, nil
+	return &ostreeImageSource{ref: ref, tmpDir: tmpDir, compressed: nil}, nil
 }
 
 // Reference returns the reference used to set up this source.
@@ -255,7 +257,21 @@ func (s *ostreeImageSource) readSingleFile(commit, path string) (io.ReadCloser,
 
 // GetBlob returns a stream for the specified blob, and the blob's size.
 func (s *ostreeImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int64, error) {
+
 	blob := info.Digest.Hex()
+
+	// Ensure s.compressed is initialized.  It is build by LayerInfosForCopy.
+	if s.compressed == nil {
+		_, err := s.LayerInfosForCopy()
+		if err != nil {
+			return nil, -1, err
+		}
+
+	}
+	compressedBlob, found := s.compressed[info.Digest]
+	if found {
+		blob = compressedBlob.Hex()
+	}
 	branch := fmt.Sprintf("ociimage/%s", blob)
 
 	if s.repo == nil {
@@ -348,7 +364,45 @@ func (s *ostreeImageSource) GetSignatures(ctx context.Context, instanceDigest *d
 	return signatures, nil
 }
 
-// LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified.
-func (s *ostreeImageSource) LayerInfosForCopy() []types.BlobInfo {
-	return nil
+// LayerInfosForCopy() returns the list of layer blobs that make up the root filesystem of
+// the image, after they've been decompressed.
+func (s *ostreeImageSource) LayerInfosForCopy() ([]types.BlobInfo, error) {
+	updatedBlobInfos := []types.BlobInfo{}
+	manifestBlob, manifestType, err := s.GetManifest(nil)
+	if err != nil {
+		return nil, err
+	}
+
+	man, err := manifest.FromBlob(manifestBlob, manifestType)
+
+	s.compressed = make(map[digest.Digest]digest.Digest)
+
+	layerBlobs := man.LayerInfos()
+
+	for _, layerBlob := range layerBlobs {
+		branch := fmt.Sprintf("ociimage/%s", layerBlob.Digest.Hex())
+		found, uncompressedDigestStr, err := readMetadata(s.repo, branch, "docker.uncompressed_digest")
+		if err != nil || !found {
+			return nil, err
+		}
+
+		found, uncompressedSizeStr, err := readMetadata(s.repo, branch, "docker.uncompressed_size")
+		if err != nil || !found {
+			return nil, err
+		}
+
+		uncompressedSize, err := strconv.ParseInt(uncompressedSizeStr, 10, 64)
+		if err != nil {
+			return nil, err
+		}
+		uncompressedDigest := digest.Digest(uncompressedDigestStr)
+		blobInfo := types.BlobInfo{
+			Digest:    uncompressedDigest,
+			Size:      uncompressedSize,
+			MediaType: layerBlob.MediaType,
+		}
+		s.compressed[uncompressedDigest] = layerBlob.Digest
+		updatedBlobInfos = append(updatedBlobInfos, blobInfo)
+	}
+	return updatedBlobInfos, nil
 }

vendor/github.com/containers/image/storage/storage_image.go

@@ -177,18 +177,16 @@ func (s *storageImageSource) GetManifest(instanceDigest *digest.Digest) (manifes
 
 // LayerInfosForCopy() returns the list of layer blobs that make up the root filesystem of
 // the image, after they've been decompressed.
-func (s *storageImageSource) LayerInfosForCopy() []types.BlobInfo {
+func (s *storageImageSource) LayerInfosForCopy() ([]types.BlobInfo, error) {
 	simg, err := s.imageRef.transport.store.Image(s.ID)
 	if err != nil {
-		logrus.Errorf("error reading image %q: %v", s.ID, err)
-		return nil
+		return nil, errors.Wrapf(err, "error reading image %q", s.ID)
 	}
 	updatedBlobInfos := []types.BlobInfo{}
 	layerID := simg.TopLayer
 	_, manifestType, err := s.GetManifest(nil)
 	if err != nil {
-		logrus.Errorf("error reading image manifest for %q: %v", s.ID, err)
-		return nil
+		return nil, errors.Wrapf(err, "error reading image manifest for %q", s.ID)
 	}
 	uncompressedLayerType := ""
 	switch manifestType {
@@ -201,16 +199,13 @@ func (s *storageImageSource) LayerInfosForCopy() []types.BlobInfo {
 	for layerID != "" {
 		layer, err := s.imageRef.transport.store.Layer(layerID)
 		if err != nil {
-			logrus.Errorf("error reading layer %q in image %q: %v", layerID, s.ID, err)
-			return nil
+			return nil, errors.Wrapf(err, "error reading layer %q in image %q", layerID, s.ID)
 		}
 		if layer.UncompressedDigest == "" {
-			logrus.Errorf("uncompressed digest for layer %q is unknown", layerID)
-			return nil
+			return nil, errors.Errorf("uncompressed digest for layer %q is unknown", layerID)
 		}
 		if layer.UncompressedSize < 0 {
-			logrus.Errorf("uncompressed size for layer %q is unknown", layerID)
-			return nil
+			return nil, errors.Errorf("uncompressed size for layer %q is unknown", layerID)
 		}
 		blobInfo := types.BlobInfo{
 			Digest:    layer.UncompressedDigest,
@@ -220,7 +215,7 @@ func (s *storageImageSource) LayerInfosForCopy() []types.BlobInfo {
 		updatedBlobInfos = append([]types.BlobInfo{blobInfo}, updatedBlobInfos...)
 		layerID = layer.Parent
 	}
-	return updatedBlobInfos
+	return updatedBlobInfos, nil
 }
 
 // GetSignatures() parses the image's signatures blob into a slice of byte slices.

vendor/github.com/containers/image/tarball/tarball_src.go

@@ -255,6 +255,6 @@ func (is *tarballImageSource) Reference() types.ImageReference {
 }
 
 // LayerInfosForCopy() returns updated layer info that should be used when reading, in preference to values in the manifest, if specified.
-func (*tarballImageSource) LayerInfosForCopy() []types.BlobInfo {
-	return nil
+func (*tarballImageSource) LayerInfosForCopy() ([]types.BlobInfo, error) {
+	return nil, nil
 }

vendor/github.com/containers/image/types/types.go

@@ -129,7 +129,7 @@ type ImageSource interface {
 	// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer blobsums that are listed in the image's manifest.
 	// The Digest field is guaranteed to be provided; Size may be -1.
 	// WARNING: The list may contain duplicates, and they are semantically relevant.
-	LayerInfosForCopy() []BlobInfo
+	LayerInfosForCopy() ([]BlobInfo, error)
 }
 
 // ImageDestination is a service, possibly remote (= slow), to store components of a single image.
@@ -218,7 +218,7 @@ type UnparsedImage interface {
 	// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer blobsums that are listed in the image's manifest.
 	// The Digest field is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided.
 	// WARNING: The list may contain duplicates, and they are semantically relevant.
-	LayerInfosForCopy() []BlobInfo
+	LayerInfosForCopy() ([]BlobInfo, error)
 }
 
 // Image is the primary API for inspecting properties of images.