Merge pull request #443 from nstack/shared-blobs

copy: add shared blob directory support for OCI sources/destinations

cmd/skopeo/copy.go

@@ -121,5 +121,15 @@ var copyCmd = cli.Command{
 			Value: "",
 			Usage: "`DIRECTORY` to use for OSTree temporary files",
 		},
+		cli.StringFlag{
+			Name:  "src-shared-blob-dir",
+			Value: "",
+			Usage: "`DIRECTORY` to use to fetch retrieved blobs (OCI layout sources only)",
+		},
+		cli.StringFlag{
+			Name:  "dest-shared-blob-dir",
+			Value: "",
+			Usage: "`DIRECTORY` to use to store retrieved blobs (OCI layout destinations only)",
+		},
 	},
 }

cmd/skopeo/utils.go

@@ -17,6 +17,7 @@ func contextFromGlobalOptions(c *cli.Context, flagPrefix string) (*types.SystemC
 		// them if per subcommand flags are provided (see below).
 		DockerInsecureSkipTLSVerify: !c.GlobalBoolT("tls-verify"),
 		OSTreeTmpDirPath:            c.String(flagPrefix + "ostree-tmp-dir"),
+		OCISharedBlobDirPath:        c.String(flagPrefix + "shared-blob-dir"),
 	}
 	if c.IsSet(flagPrefix + "tls-verify") {
 		ctx.DockerInsecureSkipTLSVerify = !c.BoolT(flagPrefix + "tls-verify")

vendor/github.com/containers/image/oci/layout/oci_dest.go

@@ -18,12 +18,13 @@ import (
 )
 
 type ociImageDestination struct {
-	ref   ociReference
-	index imgspecv1.Index
+	ref           ociReference
+	index         imgspecv1.Index
+	sharedBlobDir string
 }
 
 // newImageDestination returns an ImageDestination for writing to an existing directory.
-func newImageDestination(ref ociReference) (types.ImageDestination, error) {
+func newImageDestination(ctx *types.SystemContext, ref ociReference) (types.ImageDestination, error) {
 	if ref.image == "" {
 		return nil, errors.Errorf("cannot save image with empty image.ref.name")
 	}
@@ -43,7 +44,21 @@ func newImageDestination(ref ociReference) (types.ImageDestination, error) {
 		}
 	}
 
-	return &ociImageDestination{ref: ref, index: *index}, nil
+	d := &ociImageDestination{ref: ref, index: *index}
+	if ctx != nil {
+		d.sharedBlobDir = ctx.OCISharedBlobDirPath
+	}
+
+	if err := ensureDirectoryExists(d.ref.dir); err != nil {
+		return nil, err
+	}
+	// Per the OCI image specification, layouts MUST have a "blobs" subdirectory,
+	// but it MAY be empty (e.g. if we never end up calling PutBlob)
+	// https://github.com/opencontainers/image-spec/blame/7c889fafd04a893f5c5f50b7ab9963d5d64e5242/image-layout.md#L19
+	if err := ensureDirectoryExists(filepath.Join(d.ref.dir, "blobs")); err != nil {
+		return nil, err
+	}
+	return d, nil
 }
 
 // Reference returns the reference used to set up this destination.  Note that this should directly correspond to user's intent,
@@ -92,9 +107,6 @@ func (d *ociImageDestination) MustMatchRuntimeOS() bool {
 // to any other readers for download using the supplied digest.
 // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far.
 func (d *ociImageDestination) PutBlob(stream io.Reader, inputInfo types.BlobInfo) (types.BlobInfo, error) {
-	if err := ensureDirectoryExists(d.ref.dir); err != nil {
-		return types.BlobInfo{}, err
-	}
 	blobFile, err := ioutil.TempFile(d.ref.dir, "oci-put-blob")
 	if err != nil {
 		return types.BlobInfo{}, err
@@ -125,7 +137,7 @@ func (d *ociImageDestination) PutBlob(stream io.Reader, inputInfo types.BlobInfo
 		return types.BlobInfo{}, err
 	}
 
-	blobPath, err := d.ref.blobPath(computedDigest)
+	blobPath, err := d.ref.blobPath(computedDigest, d.sharedBlobDir)
 	if err != nil {
 		return types.BlobInfo{}, err
 	}
@@ -147,7 +159,7 @@ func (d *ociImageDestination) HasBlob(info types.BlobInfo) (bool, int64, error)
 	if info.Digest == "" {
 		return false, -1, errors.Errorf(`"Can not check for a blob with unknown digest`)
 	}
-	blobPath, err := d.ref.blobPath(info.Digest)
+	blobPath, err := d.ref.blobPath(info.Digest, d.sharedBlobDir)
 	if err != nil {
 		return false, -1, err
 	}
@@ -180,7 +192,7 @@ func (d *ociImageDestination) PutManifest(m []byte) error {
 	desc.MediaType = imgspecv1.MediaTypeImageManifest
 	desc.Size = int64(len(m))
 
-	blobPath, err := d.ref.blobPath(digest)
+	blobPath, err := d.ref.blobPath(digest, d.sharedBlobDir)
 	if err != nil {
 		return err
 	}

vendor/github.com/containers/image/oci/layout/oci_src.go

@@ -17,9 +17,10 @@ import (
 )
 
 type ociImageSource struct {
-	ref        ociReference
-	descriptor imgspecv1.Descriptor
-	client     *http.Client
+	ref           ociReference
+	descriptor    imgspecv1.Descriptor
+	client        *http.Client
+	sharedBlobDir string
 }
 
 // newImageSource returns an ImageSource for reading from an existing directory.
@@ -40,7 +41,12 @@ func newImageSource(ctx *types.SystemContext, ref ociReference) (types.ImageSour
 	if err != nil {
 		return nil, err
 	}
-	return &ociImageSource{ref: ref, descriptor: descriptor, client: client}, nil
+	d := &ociImageSource{ref: ref, descriptor: descriptor, client: client}
+	if ctx != nil {
+		// TODO(jonboulle): check dir existence?
+		d.sharedBlobDir = ctx.OCISharedBlobDirPath
+	}
+	return d, nil
 }
 
 // Reference returns the reference used to set up this source.
@@ -56,7 +62,7 @@ func (s *ociImageSource) Close() error {
 // GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available).
 // It may use a remote (= slow) service.
 func (s *ociImageSource) GetManifest() ([]byte, string, error) {
-	manifestPath, err := s.ref.blobPath(digest.Digest(s.descriptor.Digest))
+	manifestPath, err := s.ref.blobPath(digest.Digest(s.descriptor.Digest), s.sharedBlobDir)
 	if err != nil {
 		return nil, "", err
 	}
@@ -69,7 +75,7 @@ func (s *ociImageSource) GetManifest() ([]byte, string, error) {
 }
 
 func (s *ociImageSource) GetTargetManifest(digest digest.Digest) ([]byte, string, error) {
-	manifestPath, err := s.ref.blobPath(digest)
+	manifestPath, err := s.ref.blobPath(digest, s.sharedBlobDir)
 	if err != nil {
 		return nil, "", err
 	}
@@ -92,7 +98,7 @@ func (s *ociImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int64, err
 		return s.getExternalBlob(info.URLs)
 	}
 
-	path, err := s.ref.blobPath(info.Digest)
+	path, err := s.ref.blobPath(info.Digest, s.sharedBlobDir)
 	if err != nil {
 		return nil, 0, err
 	}

vendor/github.com/containers/image/oci/layout/oci_transport.go

@@ -261,7 +261,7 @@ func (ref ociReference) NewImageSource(ctx *types.SystemContext) (types.ImageSou
 // NewImageDestination returns a types.ImageDestination for this reference.
 // The caller must call .Close() on the returned ImageDestination.
 func (ref ociReference) NewImageDestination(ctx *types.SystemContext) (types.ImageDestination, error) {
-	return newImageDestination(ref)
+	return newImageDestination(ctx, ref)
 }
 
 // DeleteImage deletes the named image from the registry, if supported.
@@ -280,9 +280,13 @@ func (ref ociReference) indexPath() string {
 }
 
 // blobPath returns a path for a blob within a directory using OCI image-layout conventions.
-func (ref ociReference) blobPath(digest digest.Digest) (string, error) {
+func (ref ociReference) blobPath(digest digest.Digest, sharedBlobDir string) (string, error) {
 	if err := digest.Validate(); err != nil {
 		return "", errors.Wrapf(err, "unexpected digest reference %s", digest)
 	}
-	return filepath.Join(ref.dir, "blobs", digest.Algorithm().String(), digest.Hex()), nil
+	blobDir := filepath.Join(ref.dir, "blobs")
+	if sharedBlobDir != "" {
+		blobDir = sharedBlobDir
+	}
+	return filepath.Join(blobDir, digest.Algorithm().String(), digest.Hex()), nil
 }

vendor/github.com/containers/image/types/types.go

@@ -316,6 +316,8 @@ type SystemContext struct {
 	OCICertPath string
 	// Allow downloading OCI image layers over HTTP, or HTTPS with failed TLS verification. Note that this does not affect other TLS connections.
 	OCIInsecureSkipTLSVerify bool
+	// If not "", use a shared directory for storing blobs rather than within OCI layouts
+	OCISharedBlobDirPath string
 
 	// === docker.Transport overrides ===
 	// If not "", a directory containing a CA certificate (ending with ".crt"),