From e8794bd9ff44161d007bf101c69869755e12c524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miloslav=20Trma=C4=8D?= Date: Fri, 26 Aug 2016 23:39:17 +0200 Subject: [PATCH] Vendor after merging in mtrmac/image:api-changes ... and update for the API changes. --- cmd/skopeo/inspect.go | 2 + cmd/skopeo/layers.go | 36 ++++--- cmd/skopeo/utils.go | 2 + .../github.com/containers/image/copy/copy.go | 14 ++- .../image/directory/directory_dest.go | 31 ++++-- .../image/directory/directory_src.go | 6 ++ .../image/directory/directory_transport.go | 5 +- .../containers/image/docker/docker_client.go | 8 +- .../containers/image/docker/docker_image.go | 1 + .../image/docker/docker_image_dest.go | 78 ++++++++------- .../image/docker/docker_image_src.go | 10 +- .../image/docker/docker_transport.go | 5 +- .../containers/image/image/image.go | 11 +++ .../containers/image/manifest/manifest.go | 4 +- .../containers/image/oci/oci_dest.go | 98 +++++++++++-------- .../containers/image/oci/oci_transport.go | 5 +- .../containers/image/openshift/openshift.go | 46 ++++++--- .../image/openshift/openshift_transport.go | 5 +- .../containers/image/types/types.go | 36 +++++-- 19 files changed, 276 insertions(+), 127 deletions(-) diff --git a/cmd/skopeo/inspect.go b/cmd/skopeo/inspect.go index 6c5f4b7c..6b8f897b 100644 --- a/cmd/skopeo/inspect.go +++ b/cmd/skopeo/inspect.go @@ -39,6 +39,8 @@ var inspectCmd = cli.Command{ if err != nil { return err } + defer img.Close() + rawManifest, _, err := img.Manifest() if err != nil { return err diff --git a/cmd/skopeo/layers.go b/cmd/skopeo/layers.go index 8c70d6d8..6c1f31d7 100644 --- a/cmd/skopeo/layers.go +++ b/cmd/skopeo/layers.go @@ -26,6 +26,8 @@ var layersCmd = cli.Command{ return err } src := image.FromSource(rawSource) + defer src.Close() + blobDigests := c.Args().Tail() if len(blobDigests) == 0 { b, err := src.BlobDigests() @@ -46,6 +48,23 @@ var layersCmd = cli.Command{ if err != nil { return err } + defer dest.Close() + + for _, digest := range blobDigests { + if !strings.HasPrefix(digest, "sha256:") { + digest = "sha256:" + digest + } + r, blobSize, err := rawSource.GetBlob(digest) + if err != nil { + return err + } + if err := dest.PutBlob(digest, blobSize, r); err != nil { + r.Close() + return err + } + r.Close() + } + manifest, _, err := src.Manifest() if err != nil { return err @@ -53,20 +72,11 @@ var layersCmd = cli.Command{ if err := dest.PutManifest(manifest); err != nil { return err } - for _, digest := range blobDigests { - if !strings.HasPrefix(digest, "sha256:") { - digest = "sha256:" + digest - } - r, _, err := rawSource.GetBlob(digest) - if err != nil { - return err - } - if err := dest.PutBlob(digest, r); err != nil { - r.Close() - return err - } - r.Close() + + if err := dest.Commit(); err != nil { + return err } + return nil }, } diff --git a/cmd/skopeo/utils.go b/cmd/skopeo/utils.go index a646e9a1..93f9e331 100644 --- a/cmd/skopeo/utils.go +++ b/cmd/skopeo/utils.go @@ -17,6 +17,7 @@ func contextFromGlobalOptions(c *cli.Context) *types.SystemContext { } // ParseImage converts image URL-like string to an initialized handler for that image. +// The caller must call .Close() on the returned Image. func parseImage(c *cli.Context) (types.Image, error) { imgName := c.Args().First() ref, err := transports.ParseImageName(imgName) @@ -28,6 +29,7 @@ func parseImage(c *cli.Context) (types.Image, error) { // parseImageSource converts image URL-like string to an ImageSource. // requestedManifestMIMETypes is as in types.ImageReference.NewImageSource. +// The caller must call .Close() on the returned ImageSource. func parseImageSource(c *cli.Context, name string, requestedManifestMIMETypes []string) (types.ImageSource, error) { ref, err := transports.ParseImageName(name) if err != nil { diff --git a/vendor/github.com/containers/image/copy/copy.go b/vendor/github.com/containers/image/copy/copy.go index 712eaae7..5e769717 100644 --- a/vendor/github.com/containers/image/copy/copy.go +++ b/vendor/github.com/containers/image/copy/copy.go @@ -86,11 +86,14 @@ func Image(ctx *types.SystemContext, policyContext *signature.PolicyContext, des if err != nil { return fmt.Errorf("Error initializing destination %s: %v", transports.ImageName(destRef), err) } + defer dest.Close() + rawSource, err := srcRef.NewImageSource(ctx, dest.SupportedManifestMIMETypes()) if err != nil { return fmt.Errorf("Error initializing source %s: %v", transports.ImageName(srcRef), err) } src := image.FromSource(rawSource) + defer src.Close() // Please keep this policy check BEFORE reading any other information about the image. if allowed, err := policyContext.IsRunningImageAllowed(src); !allowed || err != nil { // Be paranoid and fail if either return value indicates so. @@ -119,8 +122,7 @@ func Image(ctx *types.SystemContext, policyContext *signature.PolicyContext, des return fmt.Errorf("Error parsing manifest: %v", err) } for _, digest := range blobDigests { - // TODO(mitr): do not ignore the size param returned here - stream, _, err := rawSource.GetBlob(digest) + stream, blobSize, err := rawSource.GetBlob(digest) if err != nil { return fmt.Errorf("Error reading blob %s: %v", digest, err) } @@ -135,7 +137,7 @@ func Image(ctx *types.SystemContext, policyContext *signature.PolicyContext, des if err != nil { return fmt.Errorf("Error preparing to verify blob %s: %v", digest, err) } - if err := dest.PutBlob(digest, digestingReader); err != nil { + if err := dest.PutBlob(digest, blobSize, digestingReader); err != nil { return fmt.Errorf("Error writing blob: %v", err) } if digestingReader.validationFailed { // Coverage: This should never happen. @@ -160,7 +162,6 @@ func Image(ctx *types.SystemContext, policyContext *signature.PolicyContext, des sigs = append(sigs, newSig) } - // FIXME: We need to call PutManifest after PutBlob and before PutSignatures. This seems ugly; move to a "set properties" + "commit" model? if err := dest.PutManifest(manifest); err != nil { return fmt.Errorf("Error writing manifest: %v", err) } @@ -168,5 +169,10 @@ func Image(ctx *types.SystemContext, policyContext *signature.PolicyContext, des if err := dest.PutSignatures(sigs); err != nil { return fmt.Errorf("Error writing signatures: %v", err) } + + if err := dest.Commit(); err != nil { + return fmt.Errorf("Error committing the finished image: %v", err) + } + return nil } diff --git a/vendor/github.com/containers/image/directory/directory_dest.go b/vendor/github.com/containers/image/directory/directory_dest.go index fb5ff757..7f2f0732 100644 --- a/vendor/github.com/containers/image/directory/directory_dest.go +++ b/vendor/github.com/containers/image/directory/directory_dest.go @@ -1,6 +1,7 @@ package directory import ( + "fmt" "io" "io/ioutil" "os" @@ -24,20 +25,20 @@ func (d *dirImageDestination) Reference() types.ImageReference { return d.ref } +// Close removes resources associated with an initialized ImageDestination, if any. +func (d *dirImageDestination) Close() { +} + func (d *dirImageDestination) SupportedManifestMIMETypes() []string { return nil } -func (d *dirImageDestination) PutManifest(manifest []byte) error { - return ioutil.WriteFile(d.ref.manifestPath(), manifest, 0644) -} - // PutBlob writes contents of stream as a blob identified by digest. +// The length of stream is expected to be expectedSize; if expectedSize == -1, it is not known. // WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available // to any other readers for download using the supplied digest. // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. -// Note: Calling PutBlob() and other methods may have ordering dependencies WRT other methods of this type. FIXME: Figure out and document. -func (d *dirImageDestination) PutBlob(digest string, stream io.Reader) error { +func (d *dirImageDestination) PutBlob(digest string, expectedSize int64, stream io.Reader) error { blobPath := d.ref.layerPath(digest) blobFile, err := ioutil.TempFile(filepath.Dir(blobPath), filepath.Base(blobPath)) if err != nil { @@ -51,9 +52,13 @@ func (d *dirImageDestination) PutBlob(digest string, stream io.Reader) error { } }() - if _, err := io.Copy(blobFile, stream); err != nil { + size, err := io.Copy(blobFile, stream) + if err != nil { return err } + if expectedSize != -1 && size != expectedSize { + return fmt.Errorf("Size mismatch when copying %s, expected %d, got %d", digest, expectedSize, size) + } if err := blobFile.Sync(); err != nil { return err } @@ -67,6 +72,10 @@ func (d *dirImageDestination) PutBlob(digest string, stream io.Reader) error { return nil } +func (d *dirImageDestination) PutManifest(manifest []byte) error { + return ioutil.WriteFile(d.ref.manifestPath(), manifest, 0644) +} + func (d *dirImageDestination) PutSignatures(signatures [][]byte) error { for i, sig := range signatures { if err := ioutil.WriteFile(d.ref.signaturePath(i), sig, 0644); err != nil { @@ -75,3 +84,11 @@ func (d *dirImageDestination) PutSignatures(signatures [][]byte) error { } return nil } + +// Commit marks the process of storing the image as successful and asks for the image to be persisted. +// WARNING: This does not have any transactional semantics: +// - Uploaded data MAY be visible to others before Commit() is called +// - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed) +func (d *dirImageDestination) Commit() error { + return nil +} diff --git a/vendor/github.com/containers/image/directory/directory_src.go b/vendor/github.com/containers/image/directory/directory_src.go index 1f343597..c87b0a3b 100644 --- a/vendor/github.com/containers/image/directory/directory_src.go +++ b/vendor/github.com/containers/image/directory/directory_src.go @@ -13,6 +13,7 @@ type dirImageSource struct { } // newImageSource returns an ImageSource reading from an existing directory. +// The caller must call .Close() on the returned ImageSource. func newImageSource(ref dirReference) types.ImageSource { return &dirImageSource{ref} } @@ -23,6 +24,10 @@ func (s *dirImageSource) Reference() types.ImageReference { return s.ref } +// Close removes resources associated with an initialized ImageSource, if any. +func (s *dirImageSource) Close() { +} + // it's up to the caller to determine the MIME type of the returned manifest's bytes func (s *dirImageSource) GetManifest() ([]byte, string, error) { m, err := ioutil.ReadFile(s.ref.manifestPath()) @@ -32,6 +37,7 @@ func (s *dirImageSource) GetManifest() ([]byte, string, error) { return m, "", err } +// GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). func (s *dirImageSource) GetBlob(digest string) (io.ReadCloser, int64, error) { r, err := os.Open(s.ref.layerPath(digest)) if err != nil { diff --git a/vendor/github.com/containers/image/directory/directory_transport.go b/vendor/github.com/containers/image/directory/directory_transport.go index 1fd30cfd..4979572d 100644 --- a/vendor/github.com/containers/image/directory/directory_transport.go +++ b/vendor/github.com/containers/image/directory/directory_transport.go @@ -128,19 +128,22 @@ func (ref dirReference) PolicyConfigurationNamespaces() []string { } // NewImage returns a types.Image for this reference. +// The caller must call .Close() on the returned Image. func (ref dirReference) NewImage(ctx *types.SystemContext) (types.Image, error) { src := newImageSource(ref) return image.FromSource(src), nil } // NewImageSource returns a types.ImageSource for this reference, -// asking the backend to use a manifest from requestedManifestMIMETypes if possible +// asking the backend to use a manifest from requestedManifestMIMETypes if possible. // nil requestedManifestMIMETypes means manifest.DefaultRequestedManifestMIMETypes. +// The caller must call .Close() on the returned ImageSource. func (ref dirReference) NewImageSource(ctx *types.SystemContext, requestedManifestMIMETypes []string) (types.ImageSource, error) { return newImageSource(ref), nil } // NewImageDestination returns a types.ImageDestination for this reference. +// The caller must call .Close() on the returned ImageDestination. func (ref dirReference) NewImageDestination(ctx *types.SystemContext) (types.ImageDestination, error) { return newImageDestination(ref), nil } diff --git a/vendor/github.com/containers/image/docker/docker_client.go b/vendor/github.com/containers/image/docker/docker_client.go index fd016565..f5b31841 100644 --- a/vendor/github.com/containers/image/docker/docker_client.go +++ b/vendor/github.com/containers/image/docker/docker_client.go @@ -99,16 +99,20 @@ func (c *dockerClient) makeRequest(method, url string, headers map[string][]stri } url = fmt.Sprintf(baseURL, c.scheme, c.registry) + url - return c.makeRequestToResolvedURL(method, url, headers, stream) + return c.makeRequestToResolvedURL(method, url, headers, stream, -1) } // makeRequestToResolvedURL creates and executes a http.Request with the specified parameters, adding authentication and TLS options for the Docker client. +// streamLen, if not -1, specifies the length of the data expected on stream. // makeRequest should generally be preferred. -func (c *dockerClient) makeRequestToResolvedURL(method, url string, headers map[string][]string, stream io.Reader) (*http.Response, error) { +func (c *dockerClient) makeRequestToResolvedURL(method, url string, headers map[string][]string, stream io.Reader, streamLen int64) (*http.Response, error) { req, err := http.NewRequest(method, url, stream) if err != nil { return nil, err } + if streamLen != -1 { // Do not blindly overwrite if streamLen == -1, http.NewRequest above can figure out the length of bytes.Reader and similar objects without us having to compute it. + req.ContentLength = streamLen + } req.Header.Set("Docker-Distribution-API-Version", "registry/2.0") for n, h := range headers { for _, hh := range h { diff --git a/vendor/github.com/containers/image/docker/docker_image.go b/vendor/github.com/containers/image/docker/docker_image.go index 57216e8f..b02a8cd9 100644 --- a/vendor/github.com/containers/image/docker/docker_image.go +++ b/vendor/github.com/containers/image/docker/docker_image.go @@ -18,6 +18,7 @@ type Image struct { // newImage returns a new Image interface type after setting up // a client to the registry hosting the given image. +// The caller must call .Close() on the returned Image. func newImage(ctx *types.SystemContext, ref dockerReference) (types.Image, error) { s, err := newImageSource(ctx, ref, nil) if err != nil { diff --git a/vendor/github.com/containers/image/docker/docker_image_dest.go b/vendor/github.com/containers/image/docker/docker_image_dest.go index 1e5ab498..68deb60e 100644 --- a/vendor/github.com/containers/image/docker/docker_image_dest.go +++ b/vendor/github.com/containers/image/docker/docker_image_dest.go @@ -35,6 +35,10 @@ func (d *dockerImageDestination) Reference() types.ImageReference { return d.ref } +// Close removes resources associated with an initialized ImageDestination, if any. +func (d *dockerImageDestination) Close() { +} + func (d *dockerImageDestination) SupportedManifestMIMETypes() []string { return []string{ // TODO(runcom): we'll add OCI as part of another PR here @@ -44,42 +48,12 @@ func (d *dockerImageDestination) SupportedManifestMIMETypes() []string { } } -func (d *dockerImageDestination) PutManifest(m []byte) error { - // FIXME: This only allows upload by digest, not creating a tag. See the - // corresponding comment in openshift.NewImageDestination. - digest, err := manifest.Digest(m) - if err != nil { - return err - } - url := fmt.Sprintf(manifestURL, d.ref.ref.RemoteName(), digest) - - headers := map[string][]string{} - mimeType := manifest.GuessMIMEType(m) - if mimeType != "" { - headers["Content-Type"] = []string{mimeType} - } - res, err := d.c.makeRequest("PUT", url, headers, bytes.NewReader(m)) - if err != nil { - return err - } - defer res.Body.Close() - if res.StatusCode != http.StatusCreated { - body, err := ioutil.ReadAll(res.Body) - if err == nil { - logrus.Debugf("Error body %s", string(body)) - } - logrus.Debugf("Error uploading manifest, status %d, %#v", res.StatusCode, res) - return fmt.Errorf("Error uploading manifest to %s, status %d", url, res.StatusCode) - } - return nil -} - // PutBlob writes contents of stream as a blob identified by digest. +// The length of stream is expected to be expectedSize; if expectedSize == -1, it is not known. // WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available // to any other readers for download using the supplied digest. // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. -// Note: Calling PutBlob() and other methods may have ordering dependencies WRT other methods of this type. FIXME: Figure out and document. -func (d *dockerImageDestination) PutBlob(digest string, stream io.Reader) error { +func (d *dockerImageDestination) PutBlob(digest string, expectedSize int64, stream io.Reader) error { checkURL := fmt.Sprintf(blobsURL, d.ref.ref.RemoteName(), digest) logrus.Debugf("Checking %s", checkURL) @@ -116,7 +90,7 @@ func (d *dockerImageDestination) PutBlob(digest string, stream io.Reader) error locationQuery := uploadLocation.Query() locationQuery.Set("digest", digest) uploadLocation.RawQuery = locationQuery.Encode() - res, err = d.c.makeRequestToResolvedURL("PUT", uploadLocation.String(), map[string][]string{"Content-Type": {"application/octet-stream"}}, stream) + res, err = d.c.makeRequestToResolvedURL("PUT", uploadLocation.String(), map[string][]string{"Content-Type": {"application/octet-stream"}}, stream, expectedSize) if err != nil { return err } @@ -130,9 +104,47 @@ func (d *dockerImageDestination) PutBlob(digest string, stream io.Reader) error return nil } +func (d *dockerImageDestination) PutManifest(m []byte) error { + // FIXME: This only allows upload by digest, not creating a tag. See the + // corresponding comment in openshift.NewImageDestination. + digest, err := manifest.Digest(m) + if err != nil { + return err + } + url := fmt.Sprintf(manifestURL, d.ref.ref.RemoteName(), digest) + + headers := map[string][]string{} + mimeType := manifest.GuessMIMEType(m) + if mimeType != "" { + headers["Content-Type"] = []string{mimeType} + } + res, err := d.c.makeRequest("PUT", url, headers, bytes.NewReader(m)) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != http.StatusCreated { + body, err := ioutil.ReadAll(res.Body) + if err == nil { + logrus.Debugf("Error body %s", string(body)) + } + logrus.Debugf("Error uploading manifest, status %d, %#v", res.StatusCode, res) + return fmt.Errorf("Error uploading manifest to %s, status %d", url, res.StatusCode) + } + return nil +} + func (d *dockerImageDestination) PutSignatures(signatures [][]byte) error { if len(signatures) != 0 { return fmt.Errorf("Pushing signatures to a Docker Registry is not supported") } return nil } + +// Commit marks the process of storing the image as successful and asks for the image to be persisted. +// WARNING: This does not have any transactional semantics: +// - Uploaded data MAY be visible to others before Commit() is called +// - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed) +func (d *dockerImageDestination) Commit() error { + return nil +} diff --git a/vendor/github.com/containers/image/docker/docker_image_src.go b/vendor/github.com/containers/image/docker/docker_image_src.go index 943918f9..5267c5d4 100644 --- a/vendor/github.com/containers/image/docker/docker_image_src.go +++ b/vendor/github.com/containers/image/docker/docker_image_src.go @@ -29,8 +29,9 @@ type dockerImageSource struct { } // newImageSource creates a new ImageSource for the specified image reference, -// asking the backend to use a manifest from requestedManifestMIMETypes if possible +// asking the backend to use a manifest from requestedManifestMIMETypes if possible. // nil requestedManifestMIMETypes means manifest.DefaultRequestedManifestMIMETypes. +// The caller must call .Close() on the returned ImageSource. func newImageSource(ctx *types.SystemContext, ref dockerReference, requestedManifestMIMETypes []string) (*dockerImageSource, error) { c, err := newDockerClient(ctx, ref.ref.Hostname()) if err != nil { @@ -52,6 +53,10 @@ func (s *dockerImageSource) Reference() types.ImageReference { return s.ref } +// Close removes resources associated with an initialized ImageSource, if any. +func (s *dockerImageSource) Close() { +} + // simplifyContentType drops parameters from a HTTP media type (see https://tools.ietf.org/html/rfc7231#section-3.1.1.1) // Alternatively, an empty string is returned unchanged, and invalid values are "simplified" to an empty string. func simplifyContentType(contentType string) string { @@ -91,6 +96,7 @@ func (s *dockerImageSource) GetManifest() ([]byte, string, error) { return manblob, simplifyContentType(res.Header.Get("Content-Type")), nil } +// GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). func (s *dockerImageSource) GetBlob(digest string) (io.ReadCloser, int64, error) { url := fmt.Sprintf(blobsURL, s.ref.ref.RemoteName(), digest) logrus.Debugf("Downloading %s", url) @@ -104,7 +110,7 @@ func (s *dockerImageSource) GetBlob(digest string) (io.ReadCloser, int64, error) } size, err := strconv.ParseInt(res.Header.Get("Content-Length"), 10, 64) if err != nil { - size = 0 + size = -1 } return res.Body, size, nil } diff --git a/vendor/github.com/containers/image/docker/docker_transport.go b/vendor/github.com/containers/image/docker/docker_transport.go index 20f77dff..5ab5efe3 100644 --- a/vendor/github.com/containers/image/docker/docker_transport.go +++ b/vendor/github.com/containers/image/docker/docker_transport.go @@ -116,18 +116,21 @@ func (ref dockerReference) PolicyConfigurationNamespaces() []string { } // NewImage returns a types.Image for this reference. +// The caller must call .Close() on the returned Image. func (ref dockerReference) NewImage(ctx *types.SystemContext) (types.Image, error) { return newImage(ctx, ref) } // NewImageSource returns a types.ImageSource for this reference, -// asking the backend to use a manifest from requestedManifestMIMETypes if possible +// asking the backend to use a manifest from requestedManifestMIMETypes if possible. // nil requestedManifestMIMETypes means manifest.DefaultRequestedManifestMIMETypes. +// The caller must call .Close() on the returned ImageSource. func (ref dockerReference) NewImageSource(ctx *types.SystemContext, requestedManifestMIMETypes []string) (types.ImageSource, error) { return newImageSource(ctx, ref, requestedManifestMIMETypes) } // NewImageDestination returns a types.ImageDestination for this reference. +// The caller must call .Close() on the returned ImageDestination. func (ref dockerReference) NewImageDestination(ctx *types.SystemContext) (types.ImageDestination, error) { return newImageDestination(ctx, ref) } diff --git a/vendor/github.com/containers/image/image/image.go b/vendor/github.com/containers/image/image/image.go index ee64ce28..3241d85b 100644 --- a/vendor/github.com/containers/image/image/image.go +++ b/vendor/github.com/containers/image/image/image.go @@ -37,6 +37,12 @@ type genericImage struct { } // FromSource returns a types.Image implementation for source. +// The caller must call .Close() on the returned Image. +// +// FromSource “takes ownership” of the input ImageSource and will call src.Close() +// when the image is closed. (This does not prevent callers from using both the +// Image and ImageSource objects simultaneously, but it means that they only need to +// the Image.) func FromSource(src types.ImageSource) types.Image { return &genericImage{src: src} } @@ -47,6 +53,11 @@ func (i *genericImage) Reference() types.ImageReference { return i.src.Reference() } +// Close removes resources associated with an initialized Image, if any. +func (i *genericImage) Close() { + i.src.Close() +} + // Manifest is like ImageSource.GetManifest, but the result is cached; it is OK to call this however often you need. // NOTE: It is essential for signature verification that Manifest returns the manifest from which BlobDigests is computed. func (i *genericImage) Manifest() ([]byte, string, error) { diff --git a/vendor/github.com/containers/image/manifest/manifest.go b/vendor/github.com/containers/image/manifest/manifest.go index 1d5d63a7..f56290a4 100644 --- a/vendor/github.com/containers/image/manifest/manifest.go +++ b/vendor/github.com/containers/image/manifest/manifest.go @@ -28,9 +28,9 @@ const ( // OCIV1ImageManifestListMIMEType specifies the mediaType for an image manifest list. OCIV1ImageManifestListMIMEType = "application/vnd.oci.image.manifest.list.v1+json" // OCIV1ImageSerializationMIMEType is the mediaType used for layers referenced by the manifest. - OCIV1ImageSerializationMIMEType = "application/vnd.oci.image.serialization.rootfs.tar.gzip" + OCIV1ImageSerializationMIMEType = "application/vnd.oci.image.layer.tar+gzip" // OCIV1ImageSerializationConfigMIMEType specifies the mediaType for the image configuration. - OCIV1ImageSerializationConfigMIMEType = "application/vnd.oci.image.serialization.config.v1+json" + OCIV1ImageSerializationConfigMIMEType = "application/vnd.oci.image.config.v1+json" ) // DefaultRequestedManifestMIMETypes is a list of MIME types a types.ImageSource diff --git a/vendor/github.com/containers/image/oci/oci_dest.go b/vendor/github.com/containers/image/oci/oci_dest.go index 6dd7f46e..6222061e 100644 --- a/vendor/github.com/containers/image/oci/oci_dest.go +++ b/vendor/github.com/containers/image/oci/oci_dest.go @@ -41,6 +41,55 @@ func (d *ociImageDestination) Reference() types.ImageReference { return d.ref } +// Close removes resources associated with an initialized ImageDestination, if any. +func (d *ociImageDestination) Close() { +} + +// PutBlob writes contents of stream as a blob identified by digest. +// The length of stream is expected to be expectedSize; if expectedSize == -1, it is not known. +// WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available +// to any other readers for download using the supplied digest. +// If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. +func (d *ociImageDestination) PutBlob(digest string, expectedSize int64, stream io.Reader) error { + blobPath, err := d.ref.blobPath(digest) + if err != nil { + return err + } + if err := ensureParentDirectoryExists(blobPath); err != nil { + return err + } + blobFile, err := ioutil.TempFile(filepath.Dir(blobPath), filepath.Base(blobPath)) + if err != nil { + return err + } + succeeded := false + defer func() { + blobFile.Close() + if !succeeded { + os.Remove(blobFile.Name()) + } + }() + + size, err := io.Copy(blobFile, stream) + if err != nil { + return err + } + if expectedSize != -1 && size != expectedSize { + return fmt.Errorf("Size mismatch when copying %s, expected %d, got %d", digest, expectedSize, size) + } + if err := blobFile.Sync(); err != nil { + return err + } + if err := blobFile.Chmod(0644); err != nil { + return err + } + if err := os.Rename(blobFile.Name(), blobPath); err != nil { + return nil + } + succeeded = true + return nil +} + func createManifest(m []byte) ([]byte, string, error) { om := ociManifest{} mt := manifest.GuessMIMEType(m) @@ -114,47 +163,6 @@ func (d *ociImageDestination) PutManifest(m []byte) error { return ioutil.WriteFile(descriptorPath, data, 0644) } -// PutBlob writes contents of stream as a blob identified by digest. -// WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available -// to any other readers for download using the supplied digest. -// If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. -// Note: Calling PutBlob() and other methods may have ordering dependencies WRT other methods of this type. FIXME: Figure out and document. -func (d *ociImageDestination) PutBlob(digest string, stream io.Reader) error { - blobPath, err := d.ref.blobPath(digest) - if err != nil { - return err - } - if err := ensureParentDirectoryExists(blobPath); err != nil { - return err - } - blobFile, err := ioutil.TempFile(filepath.Dir(blobPath), filepath.Base(blobPath)) - if err != nil { - return err - } - succeeded := false - defer func() { - blobFile.Close() - if !succeeded { - os.Remove(blobFile.Name()) - } - }() - - if _, err := io.Copy(blobFile, stream); err != nil { - return err - } - if err := blobFile.Sync(); err != nil { - return err - } - if err := blobFile.Chmod(0644); err != nil { - return err - } - if err := os.Rename(blobFile.Name(), blobPath); err != nil { - return nil - } - succeeded = true - return nil -} - // ensureParentDirectoryExists ensures the parent of the supplied path exists. func ensureParentDirectoryExists(path string) error { parent := filepath.Dir(path) @@ -179,3 +187,11 @@ func (d *ociImageDestination) PutSignatures(signatures [][]byte) error { } return nil } + +// Commit marks the process of storing the image as successful and asks for the image to be persisted. +// WARNING: This does not have any transactional semantics: +// - Uploaded data MAY be visible to others before Commit() is called +// - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed) +func (d *ociImageDestination) Commit() error { + return nil +} diff --git a/vendor/github.com/containers/image/oci/oci_transport.go b/vendor/github.com/containers/image/oci/oci_transport.go index 26ba4cbd..ee5211b6 100644 --- a/vendor/github.com/containers/image/oci/oci_transport.go +++ b/vendor/github.com/containers/image/oci/oci_transport.go @@ -165,18 +165,21 @@ func (ref ociReference) PolicyConfigurationNamespaces() []string { } // NewImage returns a types.Image for this reference. +// The caller must call .Close() on the returned Image. func (ref ociReference) NewImage(ctx *types.SystemContext) (types.Image, error) { return nil, errors.New("Full Image support not implemented for oci: image names") } // NewImageSource returns a types.ImageSource for this reference, -// asking the backend to use a manifest from requestedManifestMIMETypes if possible +// asking the backend to use a manifest from requestedManifestMIMETypes if possible. // nil requestedManifestMIMETypes means manifest.DefaultRequestedManifestMIMETypes. +// The caller must call .Close() on the returned ImageSource. func (ref ociReference) NewImageSource(ctx *types.SystemContext, requestedManifestMIMETypes []string) (types.ImageSource, error) { return nil, errors.New("Reading images not implemented for oci: image names") } // NewImageDestination returns a types.ImageDestination for this reference. +// The caller must call .Close() on the returned ImageDestination. func (ref ociReference) NewImageDestination(ctx *types.SystemContext) (types.ImageDestination, error) { return newImageDestination(ref), nil } diff --git a/vendor/github.com/containers/image/openshift/openshift.go b/vendor/github.com/containers/image/openshift/openshift.go index 3e951440..8179205f 100644 --- a/vendor/github.com/containers/image/openshift/openshift.go +++ b/vendor/github.com/containers/image/openshift/openshift.go @@ -179,8 +179,9 @@ type openshiftImageSource struct { } // newImageSource creates a new ImageSource for the specified reference, -// asking the backend to use a manifest from requestedManifestMIMETypes if possible +// asking the backend to use a manifest from requestedManifestMIMETypes if possible. // nil requestedManifestMIMETypes means manifest.DefaultRequestedManifestMIMETypes. +// The caller must call .Close() on the returned ImageSource. func newImageSource(ctx *types.SystemContext, ref openshiftReference, requestedManifestMIMETypes []string) (types.ImageSource, error) { client, err := newOpenshiftClient(ref) if err != nil { @@ -200,6 +201,14 @@ func (s *openshiftImageSource) Reference() types.ImageReference { return s.client.ref } +// Close removes resources associated with an initialized ImageSource, if any. +func (s *openshiftImageSource) Close() { + if s.docker != nil { + s.docker.Close() + s.docker = nil + } +} + func (s *openshiftImageSource) GetManifest() ([]byte, string, error) { if err := s.ensureImageIsResolved(); err != nil { return nil, "", err @@ -207,6 +216,7 @@ func (s *openshiftImageSource) GetManifest() ([]byte, string, error) { return s.docker.GetManifest() } +// GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). func (s *openshiftImageSource) GetBlob(digest string) (io.ReadCloser, int64, error) { if err := s.ensureImageIsResolved(); err != nil { return nil, 0, err @@ -320,6 +330,11 @@ func (d *openshiftImageDestination) Reference() types.ImageReference { return d.client.ref } +// Close removes resources associated with an initialized ImageDestination, if any. +func (d *openshiftImageDestination) Close() { + d.docker.Close() +} + func (d *openshiftImageDestination) SupportedManifestMIMETypes() []string { return []string{ manifest.DockerV2Schema1SignedMIMEType, @@ -327,6 +342,15 @@ func (d *openshiftImageDestination) SupportedManifestMIMETypes() []string { } } +// PutBlob writes contents of stream as a blob identified by digest. +// The length of stream is expected to be expectedSize; if expectedSize == -1, it is not known. +// WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available +// to any other readers for download using the supplied digest. +// If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. +func (d *openshiftImageDestination) PutBlob(digest string, expectedSize int64, stream io.Reader) error { + return d.docker.PutBlob(digest, expectedSize, stream) +} + func (d *openshiftImageDestination) PutManifest(m []byte) error { // FIXME? Can this eventually just call d.docker.PutManifest()? // Right now we need this as a skeleton to attach signatures to, and @@ -373,19 +397,9 @@ func (d *openshiftImageDestination) PutManifest(m []byte) error { return nil } -// PutBlob writes contents of stream as a blob identified by digest. -// WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available -// to any other readers for download using the supplied digest. -// If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. -// Note: Calling PutBlob() and other methods may have ordering dependencies WRT other methods of this type. FIXME: Figure out and document. -func (d *openshiftImageDestination) PutBlob(digest string, stream io.Reader) error { - return d.docker.PutBlob(digest, stream) -} - func (d *openshiftImageDestination) PutSignatures(signatures [][]byte) error { - // FIXME: This assumption that signatures are stored after the manifest rather breaks the model. if d.imageStreamImageName == "" { - return fmt.Errorf("Unknown manifest digest, can't add signatures") + return fmt.Errorf("Internal error: Unknown manifest digest, can't add signatures") } // Because image signatures are a shared resource in Atomic Registry, the default upload // always adds signatures. Eventually we should also allow removing signatures. @@ -444,6 +458,14 @@ sigExists: return nil } +// Commit marks the process of storing the image as successful and asks for the image to be persisted. +// WARNING: This does not have any transactional semantics: +// - Uploaded data MAY be visible to others before Commit() is called +// - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed) +func (d *openshiftImageDestination) Commit() error { + return d.docker.Commit() +} + // These structs are subsets of github.com/openshift/origin/pkg/image/api/v1 and its dependencies. type imageStream struct { Status imageStreamStatus `json:"status,omitempty"` diff --git a/vendor/github.com/containers/image/openshift/openshift_transport.go b/vendor/github.com/containers/image/openshift/openshift_transport.go index de8dd9d7..240d05ac 100644 --- a/vendor/github.com/containers/image/openshift/openshift_transport.go +++ b/vendor/github.com/containers/image/openshift/openshift_transport.go @@ -154,18 +154,21 @@ func (ref openshiftReference) PolicyConfigurationNamespaces() []string { } // NewImage returns a types.Image for this reference. +// The caller must call .Close() on the returned Image. func (ref openshiftReference) NewImage(ctx *types.SystemContext) (types.Image, error) { return nil, errors.New("Full Image support not implemented for atomic: image names") } // NewImageSource returns a types.ImageSource for this reference, -// asking the backend to use a manifest from requestedManifestMIMETypes if possible +// asking the backend to use a manifest from requestedManifestMIMETypes if possible. // nil requestedManifestMIMETypes means manifest.DefaultRequestedManifestMIMETypes. +// The caller must call .Close() on the returned ImageSource. func (ref openshiftReference) NewImageSource(ctx *types.SystemContext, requestedManifestMIMETypes []string) (types.ImageSource, error) { return newImageSource(ctx, ref, requestedManifestMIMETypes) } // NewImageDestination returns a types.ImageDestination for this reference. +// The caller must call .Close() on the returned ImageDestination. func (ref openshiftReference) NewImageDestination(ctx *types.SystemContext) (types.ImageDestination, error) { return newImageDestination(ctx, ref) } diff --git a/vendor/github.com/containers/image/types/types.go b/vendor/github.com/containers/image/types/types.go index eee1cd49..9f0542dd 100644 --- a/vendor/github.com/containers/image/types/types.go +++ b/vendor/github.com/containers/image/types/types.go @@ -71,12 +71,15 @@ type ImageReference interface { PolicyConfigurationNamespaces() []string // NewImage returns a types.Image for this reference. + // The caller must call .Close() on the returned Image. NewImage(ctx *SystemContext) (Image, error) // NewImageSource returns a types.ImageSource for this reference, - // asking the backend to use a manifest from requestedManifestMIMETypes if possible + // asking the backend to use a manifest from requestedManifestMIMETypes if possible. // nil requestedManifestMIMETypes means manifest.DefaultRequestedManifestMIMETypes. + // The caller must call .Close() on the returned ImageSource. NewImageSource(ctx *SystemContext, requestedManifestMIMETypes []string) (ImageSource, error) // NewImageDestination returns a types.ImageDestination for this reference. + // The caller must call .Close() on the returned ImageDestination. NewImageDestination(ctx *SystemContext) (ImageDestination, error) // DeleteImage deletes the named image from the registry, if supported. @@ -86,44 +89,63 @@ type ImageReference interface { // ImageSource is a service, possibly remote (= slow), to download components of a single image. // This is primarily useful for copying images around; for examining their properties, Image (below) // is usually more useful. +// Each ImageSource should eventually be closed by calling Close(). type ImageSource interface { // Reference returns the reference used to set up this source, _as specified by the user_ // (not as the image itself, or its underlying storage, claims). This can be used e.g. to determine which public keys are trusted for this image. Reference() ImageReference + // Close removes resources associated with an initialized ImageSource, if any. + Close() // GetManifest returns the image's manifest along with its MIME type. The empty string is returned if the MIME type is unknown. // It may use a remote (= slow) service. GetManifest() ([]byte, string, error) - // Note: Calling GetBlob() may have ordering dependencies WRT other methods of this type. FIXME: How does this work with (docker save) on stdin? - // the second return value is the size of the blob. If not known 0 is returned + // GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). GetBlob(digest string) (io.ReadCloser, int64, error) // GetSignatures returns the image's signatures. It may use a remote (= slow) service. GetSignatures() ([][]byte, error) } // ImageDestination is a service, possibly remote (= slow), to store components of a single image. +// +// There is a specific required order for some of the calls: +// PutBlob on the various blobs, if any, MUST be called before PutManifest (manifest references blobs, which may be created or compressed only at push time) +// PutSignatures, if called, MUST be called after PutManifest (signatures reference manifest contents) +// Finally, Commit MUST be called if the caller wants the image, as formed by the components saved above, to persist. +// +// Each ImageDestination should eventually be closed by calling Close(). type ImageDestination interface { // Reference returns the reference used to set up this destination. Note that this should directly correspond to user's intent, // e.g. it should use the public hostname instead of the result of resolving CNAMEs or following redirects. Reference() ImageReference - // FIXME? This should also receive a MIME type if known, to differentiate between schema versions. - PutManifest([]byte) error + // Close removes resources associated with an initialized ImageDestination, if any. + Close() // PutBlob writes contents of stream as a blob identified by digest. + // The length of stream is expected to be expectedSize; if expectedSize == -1, it is not known. // WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available // to any other readers for download using the supplied digest. // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. - // Note: Calling PutBlob() and other methods may have ordering dependencies WRT other methods of this type. FIXME: Figure out and document. - PutBlob(digest string, stream io.Reader) error + PutBlob(digest string, expectedSize int64, stream io.Reader) error + // FIXME? This should also receive a MIME type if known, to differentiate between schema versions. + PutManifest([]byte) error PutSignatures(signatures [][]byte) error + // Commit marks the process of storing the image as successful and asks for the image to be persisted. + // WARNING: This does not have any transactional semantics: + // - Uploaded data MAY be visible to others before Commit() is called + // - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed) + Commit() error // SupportedManifestMIMETypes tells which manifest mime types the destination supports // If an empty slice or nil it's returned, then any mime type can be tried to upload SupportedManifestMIMETypes() []string } // Image is the primary API for inspecting properties of images. +// Each Image should eventually be closed by calling Close(). type Image interface { // Reference returns the reference used to set up this source, _as specified by the user_ // (not as the image itself, or its underlying storage, claims). This can be used e.g. to determine which public keys are trusted for this image. Reference() ImageReference + // Close removes resources associated with an initialized Image, if any. + Close() // ref to repository? // Manifest is like ImageSource.GetManifest, but the result is cached; it is OK to call this however often you need. // NOTE: It is essential for signature verification that Manifest returns the manifest from which BlobDigests is computed.