From 4e40830eaef708a126045920eb1a89b5fe627256 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Tue, 6 Sep 2016 19:18:05 +0200 Subject: [PATCH] vendor containers/image for PutBlob returns Signed-off-by: Antonio Murdaca --- cmd/skopeo/layers.go | 2 +- .../github.com/containers/image/copy/copy.go | 2 +- .../image/directory/directory_dest.go | 32 +++++---- .../image/docker/docker_image_dest.go | 71 +++++++++++++------ .../containers/image/oci/oci_dest.go | 58 +++++++++------ .../containers/image/openshift/openshift.go | 7 +- .../containers/image/types/types.go | 5 +- 7 files changed, 113 insertions(+), 64 deletions(-) diff --git a/cmd/skopeo/layers.go b/cmd/skopeo/layers.go index 6c1f31d7..c90ff1fe 100644 --- a/cmd/skopeo/layers.go +++ b/cmd/skopeo/layers.go @@ -58,7 +58,7 @@ var layersCmd = cli.Command{ if err != nil { return err } - if err := dest.PutBlob(digest, blobSize, r); err != nil { + if _, _, err := dest.PutBlob(r, digest, blobSize); err != nil { r.Close() return err } diff --git a/vendor/github.com/containers/image/copy/copy.go b/vendor/github.com/containers/image/copy/copy.go index 5e769717..c463d3ca 100644 --- a/vendor/github.com/containers/image/copy/copy.go +++ b/vendor/github.com/containers/image/copy/copy.go @@ -137,7 +137,7 @@ func Image(ctx *types.SystemContext, policyContext *signature.PolicyContext, des if err != nil { return fmt.Errorf("Error preparing to verify blob %s: %v", digest, err) } - if err := dest.PutBlob(digest, blobSize, digestingReader); err != nil { + if _, _, err := dest.PutBlob(digestingReader, digest, blobSize); err != nil { return fmt.Errorf("Error writing blob: %v", err) } if digestingReader.validationFailed { // Coverage: This should never happen. diff --git a/vendor/github.com/containers/image/directory/directory_dest.go b/vendor/github.com/containers/image/directory/directory_dest.go index 7f2f0732..8702ce85 100644 --- a/vendor/github.com/containers/image/directory/directory_dest.go +++ b/vendor/github.com/containers/image/directory/directory_dest.go @@ -1,11 +1,12 @@ package directory import ( + "crypto/sha256" + "encoding/hex" "fmt" "io" "io/ioutil" "os" - "path/filepath" "github.com/containers/image/types" ) @@ -33,16 +34,16 @@ func (d *dirImageDestination) SupportedManifestMIMETypes() []string { return nil } -// PutBlob writes contents of stream as a blob identified by digest. +// PutBlob writes contents of stream and returns its computed digest and size. +// A digest can be optionally provided if known, the specific image destination can decide to play with it or not. // The length of stream is expected to be expectedSize; if expectedSize == -1, it is not known. // WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available // to any other readers for download using the supplied digest. // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. -func (d *dirImageDestination) PutBlob(digest string, expectedSize int64, stream io.Reader) error { - blobPath := d.ref.layerPath(digest) - blobFile, err := ioutil.TempFile(filepath.Dir(blobPath), filepath.Base(blobPath)) +func (d *dirImageDestination) PutBlob(stream io.Reader, digest string, expectedSize int64) (string, int64, error) { + blobFile, err := ioutil.TempFile(d.ref.path, "dir-put-blob") if err != nil { - return err + return "", -1, err } succeeded := false defer func() { @@ -52,24 +53,29 @@ func (d *dirImageDestination) PutBlob(digest string, expectedSize int64, stream } }() - size, err := io.Copy(blobFile, stream) + h := sha256.New() + tee := io.TeeReader(stream, h) + + size, err := io.Copy(blobFile, tee) if err != nil { - return err + return "", -1, err } + computedDigest := hex.EncodeToString(h.Sum(nil)) if expectedSize != -1 && size != expectedSize { - return fmt.Errorf("Size mismatch when copying %s, expected %d, got %d", digest, expectedSize, size) + return "", -1, fmt.Errorf("Size mismatch when copying %s, expected %d, got %d", computedDigest, expectedSize, size) } if err := blobFile.Sync(); err != nil { - return err + return "", -1, err } if err := blobFile.Chmod(0644); err != nil { - return err + return "", -1, err } + blobPath := d.ref.layerPath(computedDigest) if err := os.Rename(blobFile.Name(), blobPath); err != nil { - return nil + return "", -1, err } succeeded = true - return nil + return "sha256:" + computedDigest, size, nil } func (d *dirImageDestination) PutManifest(manifest []byte) error { diff --git a/vendor/github.com/containers/image/docker/docker_image_dest.go b/vendor/github.com/containers/image/docker/docker_image_dest.go index 68deb60e..163fab00 100644 --- a/vendor/github.com/containers/image/docker/docker_image_dest.go +++ b/vendor/github.com/containers/image/docker/docker_image_dest.go @@ -2,10 +2,13 @@ package docker import ( "bytes" + "crypto/sha256" + "encoding/hex" "fmt" "io" "io/ioutil" "net/http" + "strconv" "github.com/Sirupsen/logrus" "github.com/containers/image/manifest" @@ -48,60 +51,84 @@ func (d *dockerImageDestination) SupportedManifestMIMETypes() []string { } } -// PutBlob writes contents of stream as a blob identified by digest. +// PutBlob writes contents of stream and returns its computed digest and size. +// A digest can be optionally provided if known, the specific image destination can decide to play with it or not. // The length of stream is expected to be expectedSize; if expectedSize == -1, it is not known. // WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available // to any other readers for download using the supplied digest. // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. -func (d *dockerImageDestination) PutBlob(digest string, expectedSize int64, stream io.Reader) error { - checkURL := fmt.Sprintf(blobsURL, d.ref.ref.RemoteName(), digest) +func (d *dockerImageDestination) PutBlob(stream io.Reader, digest string, expectedSize int64) (string, int64, error) { + if digest != "" { + checkURL := fmt.Sprintf(blobsURL, d.ref.ref.RemoteName(), digest) - logrus.Debugf("Checking %s", checkURL) - res, err := d.c.makeRequest("HEAD", checkURL, nil, nil) - if err != nil { - return err + logrus.Debugf("Checking %s", checkURL) + res, err := d.c.makeRequest("HEAD", checkURL, nil, nil) + if err != nil { + return "", -1, err + } + defer res.Body.Close() + if res.StatusCode == http.StatusOK && res.Header.Get("Docker-Content-Digest") == digest { + logrus.Debugf("... already exists, not uploading") + blobLength, err := strconv.ParseInt(res.Header.Get("Content-Length"), 10, 64) + if err != nil { + return "", -1, err + } + return digest, blobLength, nil + } + logrus.Debugf("... failed, status %d", res.StatusCode) } - defer res.Body.Close() - if res.StatusCode == http.StatusOK && res.Header.Get("Docker-Content-Digest") == digest { - logrus.Debugf("... already exists, not uploading") - return nil - } - logrus.Debugf("... failed, status %d", res.StatusCode) // FIXME? Chunked upload, progress reporting, etc. uploadURL := fmt.Sprintf(blobUploadURL, d.ref.ref.RemoteName()) logrus.Debugf("Uploading %s", uploadURL) - res, err = d.c.makeRequest("POST", uploadURL, nil, nil) + res, err := d.c.makeRequest("POST", uploadURL, nil, nil) if err != nil { - return err + return "", -1, err } defer res.Body.Close() if res.StatusCode != http.StatusAccepted { logrus.Debugf("Error initiating layer upload, response %#v", *res) - return fmt.Errorf("Error initiating layer upload to %s, status %d", uploadURL, res.StatusCode) + return "", -1, fmt.Errorf("Error initiating layer upload to %s, status %d", uploadURL, res.StatusCode) } uploadLocation, err := res.Location() if err != nil { - return fmt.Errorf("Error determining upload URL: %s", err.Error()) + return "", -1, fmt.Errorf("Error determining upload URL: %s", err.Error()) + } + + h := sha256.New() + tee := io.TeeReader(stream, h) + res, err = d.c.makeRequestToResolvedURL("PATCH", uploadLocation.String(), map[string][]string{"Content-Type": {"application/octet-stream"}}, tee, expectedSize) + if err != nil { + logrus.Debugf("Error uploading layer chunked, response %#v", *res) + return "", -1, err + } + defer res.Body.Close() + hash := h.Sum(nil) + computedDigest := "sha256:" + hex.EncodeToString(hash[:]) + + uploadLocation, err = res.Location() + if err != nil { + return "", -1, fmt.Errorf("Error determining upload URL: %s", err.Error()) } // FIXME: DELETE uploadLocation on failure locationQuery := uploadLocation.Query() - locationQuery.Set("digest", digest) + // TODO: check digest == computedDigest https://github.com/containers/image/pull/70#discussion_r77646717 + locationQuery.Set("digest", computedDigest) uploadLocation.RawQuery = locationQuery.Encode() - res, err = d.c.makeRequestToResolvedURL("PUT", uploadLocation.String(), map[string][]string{"Content-Type": {"application/octet-stream"}}, stream, expectedSize) + res, err = d.c.makeRequestToResolvedURL("PUT", uploadLocation.String(), map[string][]string{"Content-Type": {"application/octet-stream"}}, nil, -1) if err != nil { - return err + return "", -1, err } defer res.Body.Close() if res.StatusCode != http.StatusCreated { logrus.Debugf("Error uploading layer, response %#v", *res) - return fmt.Errorf("Error uploading layer to %s, status %d", uploadLocation, res.StatusCode) + return "", -1, fmt.Errorf("Error uploading layer to %s, status %d", uploadLocation, res.StatusCode) } logrus.Debugf("Upload of layer %s complete", digest) - return nil + return computedDigest, res.Request.ContentLength, nil } func (d *dockerImageDestination) PutManifest(m []byte) error { diff --git a/vendor/github.com/containers/image/oci/oci_dest.go b/vendor/github.com/containers/image/oci/oci_dest.go index 40e2ec25..1663b0f5 100644 --- a/vendor/github.com/containers/image/oci/oci_dest.go +++ b/vendor/github.com/containers/image/oci/oci_dest.go @@ -1,6 +1,8 @@ package oci import ( + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "io" @@ -33,22 +35,19 @@ func (d *ociImageDestination) Reference() types.ImageReference { func (d *ociImageDestination) Close() { } -// PutBlob writes contents of stream as a blob identified by digest. +// PutBlob writes contents of stream and returns its computed digest and size. +// A digest can be optionally provided if known, the specific image destination can decide to play with it or not. // The length of stream is expected to be expectedSize; if expectedSize == -1, it is not known. // WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available // to any other readers for download using the supplied digest. // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. -func (d *ociImageDestination) PutBlob(digest string, expectedSize int64, stream io.Reader) error { - blobPath, err := d.ref.blobPath(digest) - if err != nil { - return err +func (d *ociImageDestination) PutBlob(stream io.Reader, _ string, expectedSize int64) (string, int64, error) { + if err := ensureDirectoryExists(d.ref.dir); err != nil { + return "", -1, err } - if err := ensureParentDirectoryExists(blobPath); err != nil { - return err - } - blobFile, err := ioutil.TempFile(filepath.Dir(blobPath), filepath.Base(blobPath)) + blobFile, err := ioutil.TempFile(d.ref.dir, "oci-put-blob") if err != nil { - return err + return "", -1, err } succeeded := false defer func() { @@ -58,24 +57,36 @@ func (d *ociImageDestination) PutBlob(digest string, expectedSize int64, stream } }() - size, err := io.Copy(blobFile, stream) + h := sha256.New() + tee := io.TeeReader(stream, h) + + size, err := io.Copy(blobFile, tee) if err != nil { - return err + return "", -1, err } + computedDigest := "sha256:" + hex.EncodeToString(h.Sum(nil)) if expectedSize != -1 && size != expectedSize { - return fmt.Errorf("Size mismatch when copying %s, expected %d, got %d", digest, expectedSize, size) + return "", -1, fmt.Errorf("Size mismatch when copying %s, expected %d, got %d", computedDigest, expectedSize, size) } if err := blobFile.Sync(); err != nil { - return err + return "", -1, err } if err := blobFile.Chmod(0644); err != nil { - return err + return "", -1, err + } + + blobPath, err := d.ref.blobPath(computedDigest) + if err != nil { + return "", -1, err + } + if err := ensureParentDirectoryExists(blobPath); err != nil { + return "", -1, err } if err := os.Rename(blobFile.Name(), blobPath); err != nil { - return nil + return "", -1, err } succeeded = true - return nil + return computedDigest, size, nil } func createManifest(m []byte) ([]byte, string, error) { @@ -151,17 +162,20 @@ func (d *ociImageDestination) PutManifest(m []byte) error { return ioutil.WriteFile(descriptorPath, data, 0644) } -// ensureParentDirectoryExists ensures the parent of the supplied path exists. -func ensureParentDirectoryExists(path string) error { - parent := filepath.Dir(path) - if _, err := os.Stat(parent); err != nil && os.IsNotExist(err) { - if err := os.MkdirAll(parent, 0755); err != nil { +func ensureDirectoryExists(path string) error { + if _, err := os.Stat(path); err != nil && os.IsNotExist(err) { + if err := os.MkdirAll(path, 0755); err != nil { return err } } return nil } +// ensureParentDirectoryExists ensures the parent of the supplied path exists. +func ensureParentDirectoryExists(path string) error { + return ensureDirectoryExists(filepath.Dir(path)) +} + func (d *ociImageDestination) SupportedManifestMIMETypes() []string { return []string{ imgspecv1.MediaTypeImageManifest, diff --git a/vendor/github.com/containers/image/openshift/openshift.go b/vendor/github.com/containers/image/openshift/openshift.go index 8179205f..e32977d5 100644 --- a/vendor/github.com/containers/image/openshift/openshift.go +++ b/vendor/github.com/containers/image/openshift/openshift.go @@ -342,13 +342,14 @@ func (d *openshiftImageDestination) SupportedManifestMIMETypes() []string { } } -// PutBlob writes contents of stream as a blob identified by digest. +// PutBlob writes contents of stream and returns its computed digest and size. +// A digest can be optionally provided if known, the specific image destination can decide to play with it or not. // The length of stream is expected to be expectedSize; if expectedSize == -1, it is not known. // WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available // to any other readers for download using the supplied digest. // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. -func (d *openshiftImageDestination) PutBlob(digest string, expectedSize int64, stream io.Reader) error { - return d.docker.PutBlob(digest, expectedSize, stream) +func (d *openshiftImageDestination) PutBlob(stream io.Reader, digest string, expectedSize int64) (string, int64, error) { + return d.docker.PutBlob(stream, digest, expectedSize) } func (d *openshiftImageDestination) PutManifest(m []byte) error { diff --git a/vendor/github.com/containers/image/types/types.go b/vendor/github.com/containers/image/types/types.go index 9f0542dd..e45ca97c 100644 --- a/vendor/github.com/containers/image/types/types.go +++ b/vendor/github.com/containers/image/types/types.go @@ -119,12 +119,13 @@ type ImageDestination interface { Reference() ImageReference // Close removes resources associated with an initialized ImageDestination, if any. Close() - // PutBlob writes contents of stream as a blob identified by digest. + // PutBlob writes contents of stream and returns its computed digest and size. + // A digest can be optionally provided if known, the specific image destination can decide to play with it or not. // The length of stream is expected to be expectedSize; if expectedSize == -1, it is not known. // WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available // to any other readers for download using the supplied digest. // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. - PutBlob(digest string, expectedSize int64, stream io.Reader) error + PutBlob(stream io.Reader, digest string, expectedSize int64) (string, int64, error) // FIXME? This should also receive a MIME type if known, to differentiate between schema versions. PutManifest([]byte) error PutSignatures(signatures [][]byte) error