Merge pull request #190 from runcom/fix-putblob

vendor containers/image for PutBlob returns

cmd/skopeo/layers.go

@@ -58,7 +58,7 @@ var layersCmd = cli.Command{
 			if err != nil {
 				return err
 			}
-			if err := dest.PutBlob(digest, blobSize, r); err != nil {
+			if _, _, err := dest.PutBlob(r, digest, blobSize); err != nil {
 				r.Close()
 				return err
 			}

vendor/github.com/containers/image/copy/copy.go

@@ -137,7 +137,7 @@ func Image(ctx *types.SystemContext, policyContext *signature.PolicyContext, des
 		if err != nil {
 			return fmt.Errorf("Error preparing to verify blob %s: %v", digest, err)
 		}
-		if err := dest.PutBlob(digest, blobSize, digestingReader); err != nil {
+		if _, _, err := dest.PutBlob(digestingReader, digest, blobSize); err != nil {
 			return fmt.Errorf("Error writing blob: %v", err)
 		}
 		if digestingReader.validationFailed { // Coverage: This should never happen.

vendor/github.com/containers/image/directory/directory_dest.go

@@ -1,11 +1,12 @@
 package directory
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
-	"path/filepath"
 
 	"github.com/containers/image/types"
 )
@@ -33,16 +34,16 @@ func (d *dirImageDestination) SupportedManifestMIMETypes() []string {
 	return nil
 }
 
-// PutBlob writes contents of stream as a blob identified by digest.
+// PutBlob writes contents of stream and returns its computed digest and size.
+// A digest can be optionally provided if known, the specific image destination can decide to play with it or not.
 // The length of stream is expected to be expectedSize; if expectedSize == -1, it is not known.
 // WARNING: The contents of stream are being verified on the fly.  Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available
 // to any other readers for download using the supplied digest.
 // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far.
-func (d *dirImageDestination) PutBlob(digest string, expectedSize int64, stream io.Reader) error {
-	blobPath := d.ref.layerPath(digest)
-	blobFile, err := ioutil.TempFile(filepath.Dir(blobPath), filepath.Base(blobPath))
+func (d *dirImageDestination) PutBlob(stream io.Reader, digest string, expectedSize int64) (string, int64, error) {
+	blobFile, err := ioutil.TempFile(d.ref.path, "dir-put-blob")
 	if err != nil {
-		return err
+		return "", -1, err
 	}
 	succeeded := false
 	defer func() {
@@ -52,24 +53,29 @@ func (d *dirImageDestination) PutBlob(digest string, expectedSize int64, stream
 		}
 	}()
 
-	size, err := io.Copy(blobFile, stream)
+	h := sha256.New()
+	tee := io.TeeReader(stream, h)
+
+	size, err := io.Copy(blobFile, tee)
 	if err != nil {
-		return err
+		return "", -1, err
 	}
+	computedDigest := hex.EncodeToString(h.Sum(nil))
 	if expectedSize != -1 && size != expectedSize {
-		return fmt.Errorf("Size mismatch when copying %s, expected %d, got %d", digest, expectedSize, size)
+		return "", -1, fmt.Errorf("Size mismatch when copying %s, expected %d, got %d", computedDigest, expectedSize, size)
 	}
 	if err := blobFile.Sync(); err != nil {
-		return err
+		return "", -1, err
 	}
 	if err := blobFile.Chmod(0644); err != nil {
-		return err
+		return "", -1, err
 	}
+	blobPath := d.ref.layerPath(computedDigest)
 	if err := os.Rename(blobFile.Name(), blobPath); err != nil {
-		return nil
+		return "", -1, err
 	}
 	succeeded = true
-	return nil
+	return "sha256:" + computedDigest, size, nil
 }
 
 func (d *dirImageDestination) PutManifest(manifest []byte) error {

vendor/github.com/containers/image/docker/docker_image_dest.go

@@ -2,10 +2,13 @@ package docker
 
 import (
 	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
+	"strconv"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/containers/image/manifest"
@@ -48,60 +51,84 @@ func (d *dockerImageDestination) SupportedManifestMIMETypes() []string {
 	}
 }
 
-// PutBlob writes contents of stream as a blob identified by digest.
+// PutBlob writes contents of stream and returns its computed digest and size.
+// A digest can be optionally provided if known, the specific image destination can decide to play with it or not.
 // The length of stream is expected to be expectedSize; if expectedSize == -1, it is not known.
 // WARNING: The contents of stream are being verified on the fly.  Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available
 // to any other readers for download using the supplied digest.
 // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far.
-func (d *dockerImageDestination) PutBlob(digest string, expectedSize int64, stream io.Reader) error {
-	checkURL := fmt.Sprintf(blobsURL, d.ref.ref.RemoteName(), digest)
+func (d *dockerImageDestination) PutBlob(stream io.Reader, digest string, expectedSize int64) (string, int64, error) {
+	if digest != "" {
+		checkURL := fmt.Sprintf(blobsURL, d.ref.ref.RemoteName(), digest)
 
-	logrus.Debugf("Checking %s", checkURL)
-	res, err := d.c.makeRequest("HEAD", checkURL, nil, nil)
-	if err != nil {
-		return err
+		logrus.Debugf("Checking %s", checkURL)
+		res, err := d.c.makeRequest("HEAD", checkURL, nil, nil)
+		if err != nil {
+			return "", -1, err
+		}
+		defer res.Body.Close()
+		if res.StatusCode == http.StatusOK && res.Header.Get("Docker-Content-Digest") == digest {
+			logrus.Debugf("... already exists, not uploading")
+			blobLength, err := strconv.ParseInt(res.Header.Get("Content-Length"), 10, 64)
+			if err != nil {
+				return "", -1, err
+			}
+			return digest, blobLength, nil
+		}
+		logrus.Debugf("... failed, status %d", res.StatusCode)
 	}
-	defer res.Body.Close()
-	if res.StatusCode == http.StatusOK && res.Header.Get("Docker-Content-Digest") == digest {
-		logrus.Debugf("... already exists, not uploading")
-		return nil
-	}
-	logrus.Debugf("... failed, status %d", res.StatusCode)
 
 	// FIXME? Chunked upload, progress reporting, etc.
 	uploadURL := fmt.Sprintf(blobUploadURL, d.ref.ref.RemoteName())
 	logrus.Debugf("Uploading %s", uploadURL)
-	res, err = d.c.makeRequest("POST", uploadURL, nil, nil)
+	res, err := d.c.makeRequest("POST", uploadURL, nil, nil)
 	if err != nil {
-		return err
+		return "", -1, err
 	}
 	defer res.Body.Close()
 	if res.StatusCode != http.StatusAccepted {
 		logrus.Debugf("Error initiating layer upload, response %#v", *res)
-		return fmt.Errorf("Error initiating layer upload to %s, status %d", uploadURL, res.StatusCode)
+		return "", -1, fmt.Errorf("Error initiating layer upload to %s, status %d", uploadURL, res.StatusCode)
 	}
 	uploadLocation, err := res.Location()
 	if err != nil {
-		return fmt.Errorf("Error determining upload URL: %s", err.Error())
+		return "", -1, fmt.Errorf("Error determining upload URL: %s", err.Error())
+	}
+
+	h := sha256.New()
+	tee := io.TeeReader(stream, h)
+	res, err = d.c.makeRequestToResolvedURL("PATCH", uploadLocation.String(), map[string][]string{"Content-Type": {"application/octet-stream"}}, tee, expectedSize)
+	if err != nil {
+		logrus.Debugf("Error uploading layer chunked, response %#v", *res)
+		return "", -1, err
+	}
+	defer res.Body.Close()
+	hash := h.Sum(nil)
+	computedDigest := "sha256:" + hex.EncodeToString(hash[:])
+
+	uploadLocation, err = res.Location()
+	if err != nil {
+		return "", -1, fmt.Errorf("Error determining upload URL: %s", err.Error())
 	}
 
 	// FIXME: DELETE uploadLocation on failure
 
 	locationQuery := uploadLocation.Query()
-	locationQuery.Set("digest", digest)
+	// TODO: check digest == computedDigest https://github.com/containers/image/pull/70#discussion_r77646717
+	locationQuery.Set("digest", computedDigest)
 	uploadLocation.RawQuery = locationQuery.Encode()
-	res, err = d.c.makeRequestToResolvedURL("PUT", uploadLocation.String(), map[string][]string{"Content-Type": {"application/octet-stream"}}, stream, expectedSize)
+	res, err = d.c.makeRequestToResolvedURL("PUT", uploadLocation.String(), map[string][]string{"Content-Type": {"application/octet-stream"}}, nil, -1)
 	if err != nil {
-		return err
+		return "", -1, err
 	}
 	defer res.Body.Close()
 	if res.StatusCode != http.StatusCreated {
 		logrus.Debugf("Error uploading layer, response %#v", *res)
-		return fmt.Errorf("Error uploading layer to %s, status %d", uploadLocation, res.StatusCode)
+		return "", -1, fmt.Errorf("Error uploading layer to %s, status %d", uploadLocation, res.StatusCode)
 	}
 
 	logrus.Debugf("Upload of layer %s complete", digest)
-	return nil
+	return computedDigest, res.Request.ContentLength, nil
 }
 
 func (d *dockerImageDestination) PutManifest(m []byte) error {

vendor/github.com/containers/image/oci/oci_dest.go

@@ -1,6 +1,8 @@
 package oci
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -33,22 +35,19 @@ func (d *ociImageDestination) Reference() types.ImageReference {
 func (d *ociImageDestination) Close() {
 }
 
-// PutBlob writes contents of stream as a blob identified by digest.
+// PutBlob writes contents of stream and returns its computed digest and size.
+// A digest can be optionally provided if known, the specific image destination can decide to play with it or not.
 // The length of stream is expected to be expectedSize; if expectedSize == -1, it is not known.
 // WARNING: The contents of stream are being verified on the fly.  Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available
 // to any other readers for download using the supplied digest.
 // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far.
-func (d *ociImageDestination) PutBlob(digest string, expectedSize int64, stream io.Reader) error {
-	blobPath, err := d.ref.blobPath(digest)
-	if err != nil {
-		return err
+func (d *ociImageDestination) PutBlob(stream io.Reader, _ string, expectedSize int64) (string, int64, error) {
+	if err := ensureDirectoryExists(d.ref.dir); err != nil {
+		return "", -1, err
 	}
-	if err := ensureParentDirectoryExists(blobPath); err != nil {
-		return err
-	}
-	blobFile, err := ioutil.TempFile(filepath.Dir(blobPath), filepath.Base(blobPath))
+	blobFile, err := ioutil.TempFile(d.ref.dir, "oci-put-blob")
 	if err != nil {
-		return err
+		return "", -1, err
 	}
 	succeeded := false
 	defer func() {
@@ -58,24 +57,36 @@ func (d *ociImageDestination) PutBlob(digest string, expectedSize int64, stream
 		}
 	}()
 
-	size, err := io.Copy(blobFile, stream)
+	h := sha256.New()
+	tee := io.TeeReader(stream, h)
+
+	size, err := io.Copy(blobFile, tee)
 	if err != nil {
-		return err
+		return "", -1, err
 	}
+	computedDigest := "sha256:" + hex.EncodeToString(h.Sum(nil))
 	if expectedSize != -1 && size != expectedSize {
-		return fmt.Errorf("Size mismatch when copying %s, expected %d, got %d", digest, expectedSize, size)
+		return "", -1, fmt.Errorf("Size mismatch when copying %s, expected %d, got %d", computedDigest, expectedSize, size)
 	}
 	if err := blobFile.Sync(); err != nil {
-		return err
+		return "", -1, err
 	}
 	if err := blobFile.Chmod(0644); err != nil {
-		return err
+		return "", -1, err
+	}
+
+	blobPath, err := d.ref.blobPath(computedDigest)
+	if err != nil {
+		return "", -1, err
+	}
+	if err := ensureParentDirectoryExists(blobPath); err != nil {
+		return "", -1, err
 	}
 	if err := os.Rename(blobFile.Name(), blobPath); err != nil {
-		return nil
+		return "", -1, err
 	}
 	succeeded = true
-	return nil
+	return computedDigest, size, nil
 }
 
 func createManifest(m []byte) ([]byte, string, error) {
@@ -151,17 +162,20 @@ func (d *ociImageDestination) PutManifest(m []byte) error {
 	return ioutil.WriteFile(descriptorPath, data, 0644)
 }
 
-// ensureParentDirectoryExists ensures the parent of the supplied path exists.
-func ensureParentDirectoryExists(path string) error {
-	parent := filepath.Dir(path)
-	if _, err := os.Stat(parent); err != nil && os.IsNotExist(err) {
-		if err := os.MkdirAll(parent, 0755); err != nil {
+func ensureDirectoryExists(path string) error {
+	if _, err := os.Stat(path); err != nil && os.IsNotExist(err) {
+		if err := os.MkdirAll(path, 0755); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
+// ensureParentDirectoryExists ensures the parent of the supplied path exists.
+func ensureParentDirectoryExists(path string) error {
+	return ensureDirectoryExists(filepath.Dir(path))
+}
+
 func (d *ociImageDestination) SupportedManifestMIMETypes() []string {
 	return []string{
 		imgspecv1.MediaTypeImageManifest,

vendor/github.com/containers/image/openshift/openshift.go

@@ -342,13 +342,14 @@ func (d *openshiftImageDestination) SupportedManifestMIMETypes() []string {
 	}
 }
 
-// PutBlob writes contents of stream as a blob identified by digest.
+// PutBlob writes contents of stream and returns its computed digest and size.
+// A digest can be optionally provided if known, the specific image destination can decide to play with it or not.
 // The length of stream is expected to be expectedSize; if expectedSize == -1, it is not known.
 // WARNING: The contents of stream are being verified on the fly.  Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available
 // to any other readers for download using the supplied digest.
 // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far.
-func (d *openshiftImageDestination) PutBlob(digest string, expectedSize int64, stream io.Reader) error {
-	return d.docker.PutBlob(digest, expectedSize, stream)
+func (d *openshiftImageDestination) PutBlob(stream io.Reader, digest string, expectedSize int64) (string, int64, error) {
+	return d.docker.PutBlob(stream, digest, expectedSize)
 }
 
 func (d *openshiftImageDestination) PutManifest(m []byte) error {

vendor/github.com/containers/image/types/types.go

@@ -119,12 +119,13 @@ type ImageDestination interface {
 	Reference() ImageReference
 	// Close removes resources associated with an initialized ImageDestination, if any.
 	Close()
-	// PutBlob writes contents of stream as a blob identified by digest.
+	// PutBlob writes contents of stream and returns its computed digest and size.
+	// A digest can be optionally provided if known, the specific image destination can decide to play with it or not.
 	// The length of stream is expected to be expectedSize; if expectedSize == -1, it is not known.
 	// WARNING: The contents of stream are being verified on the fly.  Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available
 	// to any other readers for download using the supplied digest.
 	// If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far.
-	PutBlob(digest string, expectedSize int64, stream io.Reader) error
+	PutBlob(stream io.Reader, digest string, expectedSize int64) (string, int64, error)
 	// FIXME? This should also receive a MIME type if known, to differentiate between schema versions.
 	PutManifest([]byte) error
 	PutSignatures(signatures [][]byte) error