github/skopeo
1
0
Fork 0
mirror of https://github.com/containers/skopeo.git synced 2026-02-03 07:48:30 +00:00
Code Issues Projects Releases Wiki Activity

Take a shortcut when writing to local storage

Browse Source 
When writing to local storage, take a couple of shortcuts: instead of
recompressing layers to ensure that the values we store in the image
manifest will be correct for content-addressibility, just pretend that
the layer ID is a blob hash value, and that it's a valid layer diffID.

Local storage doesn't generally care if these values are correct, and we
already have to recompute these values when exporting an image, but this
saves us quite a bit of time.

The image library's Copy() routine actually cares about and
sanity-checks these things, so if we're going to take advantage of the
shortcuts, we need to use its higher-level APIs to write a layer, write
the configuration, and write the manifest, then move those items that it
writes to an image with the right set of layers.

Signed-off-by: Nalin Dahyabhai <nalin@redhat.com>

Closes: #141
Approved by: rhatdan
This commit is contained in:
Nalin Dahyabhai
2017-05-31 13:56:25 -04:00
committed by Atomic Bot
parent ebf3063cf2
commit b2baeb25f4
2 changed files with 230 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

188
commit.go
View File

@@ -1,6 +1,7 @@
package buildah
import (
"bytes"
"io"
"github.com/Sirupsen/logrus"
@@ -10,10 +11,23 @@ import (
"github.com/containers/image/transports"
"github.com/containers/image/types"
"github.com/containers/storage/pkg/archive"
"github.com/containers/storage/pkg/stringid"
"github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/projectatomic/buildah/util"
)
var (
// gzippedEmptyLayer is a gzip-compressed version of an empty tar file (just 1024 zero bytes). This
// comes from github.com/docker/distribution/manifest/schema1/config_builder.go by way of
// github.com/containers/image/image/docker_schema2.go; there is a non-zero embedded timestamp; we could
// zero that, but that would just waste storage space in registries, so let’s use the same values.
gzippedEmptyLayer = []byte{
31, 139, 8, 0, 0, 9, 110, 136, 0, 255, 98, 24, 5, 163, 96, 20, 140, 88,
0, 8, 0, 0, 255, 255, 46, 175, 181, 239, 0, 4, 0, 0,
}
)
// CommitOptions can be used to alter how an image is committed.
type CommitOptions struct {
// PreferredManifestType is the preferred type of image manifest. The
@@ -38,6 +52,159 @@ type CommitOptions struct {
ReportWriter io.Writer
}
// shallowCopy copies the most recent layer, the configuration, and the manifest from one image to another.
// For local storage, which doesn't care about histories and the manifest's contents, that's sufficient, but
// almost any other destination has higher expectations.
// We assume that "dest" is a reference to a local image (specifically, a containers/image/storage.storageReference),
// and will fail if it isn't.
func (b *Builder) shallowCopy(dest types.ImageReference, src types.ImageReference, systemContext *types.SystemContext) error {
// Read the target image name.
if dest.DockerReference() == nil {
return errors.New("can't write to an unnamed image")
}
names, err := util.ExpandTags([]string{dest.DockerReference().String()})
if err != nil {
return err
}
// Make a temporary image reference.
tmpName := stringid.GenerateRandomID() + "-tmp-" + Package + "-commit"
tmpRef, err := storage.Transport.ParseStoreReference(b.store, tmpName)
if err != nil {
return err
}
defer func() {
if err2 := tmpRef.DeleteImage(systemContext); err2 != nil {
logrus.Debugf("error deleting temporary image %q: %v", tmpName, err2)
}
}()
// Open the source for reading and a temporary image for writing.
srcImage, err := src.NewImage(systemContext)
if err != nil {
return errors.Wrapf(err, "error reading configuration to write to image %q", transports.ImageName(dest))
}
defer srcImage.Close()
tmpImage, err := tmpRef.NewImageDestination(systemContext)
if err != nil {
return errors.Wrapf(err, "error opening temporary copy of image %q for writing", transports.ImageName(dest))
}
defer tmpImage.Close()
// Write an empty filesystem layer, because the image layer requires at least one.
_, err = tmpImage.PutBlob(bytes.NewReader(gzippedEmptyLayer), types.BlobInfo{Size: int64(len(gzippedEmptyLayer))})
if err != nil {
return errors.Wrapf(err, "error writing dummy layer for image %q", transports.ImageName(dest))
}
// Read the newly-generated configuration blob.
config, err := srcImage.ConfigBlob()
if err != nil {
return errors.Wrapf(err, "error reading new configuration for image %q", transports.ImageName(dest))
}
if len(config) == 0 {
return errors.Errorf("error reading new configuration for image %q: it's empty", transports.ImageName(dest))
}
logrus.Debugf("read configuration blob %q", string(config))
// Write the configuration to the temporary image.
configBlobInfo := types.BlobInfo{
Digest: digest.Canonical.FromBytes(config),
Size: int64(len(config)),
}
_, err = tmpImage.PutBlob(bytes.NewReader(config), configBlobInfo)
if err != nil && len(config) > 0 {
return errors.Wrapf(err, "error writing image configuration for temporary copy of %q", transports.ImageName(dest))
}
// Read the newly-generated, mostly fake, manifest.
manifest, _, err := srcImage.Manifest()
if err != nil {
return errors.Wrapf(err, "error reading new manifest for image %q", transports.ImageName(dest))
}
// Write the manifest to the temporary image.
err = tmpImage.PutManifest(manifest)
if err != nil {
return errors.Wrapf(err, "error writing new manifest to temporary copy of image %q", transports.ImageName(dest))
}
// Save the temporary image.
err = tmpImage.Commit()
if err != nil {
return errors.Wrapf(err, "error committing new image %q", transports.ImageName(dest))
}
// Locate the temporary image in the lower-level API. Read its item names.
tmpImg, err := storage.Transport.GetStoreImage(b.store, tmpRef)
if err != nil {
return errors.Wrapf(err, "error locating temporary image %q", transports.ImageName(dest))
}
items, err := b.store.ListImageBigData(tmpImg.ID)
if err != nil {
return errors.Wrapf(err, "error reading list of named data for image %q", tmpImg.ID)
}
// Look up the container's read-write layer.
container, err := b.store.Container(b.ContainerID)
if err != nil {
return errors.Wrapf(err, "error reading information about working container %q", b.ContainerID)
}
parentLayer := ""
// Look up the container's source image's layer, if there is a source image.
if container.ImageID != "" {
img, err2 := b.store.Image(container.ImageID)
if err2 != nil {
return errors.Wrapf(err2, "error reading information about working container %q's source image", b.ContainerID)
}
parentLayer = img.TopLayer
}
// Extract the read-write layer's contents.
layerDiff, err := b.store.Diff(parentLayer, container.LayerID)
if err != nil {
return errors.Wrapf(err, "error reading layer from source image %q", transports.ImageName(src))
}
defer layerDiff.Close()
// Write a copy of the layer for the new image to reference.
layer, _, err := b.store.PutLayer("", parentLayer, []string{}, "", false, layerDiff)
if err != nil {
return errors.Wrapf(err, "error creating new read-only layer from container %q", b.ContainerID)
}
// Create a low-level image record that uses the new layer.
image, err := b.store.CreateImage("", []string{}, layer.ID, "", nil)
if err != nil {
err2 := b.store.DeleteLayer(layer.ID)
if err2 != nil {
logrus.Debugf("error removing layer %q: %v", layer, err2)
}
return errors.Wrapf(err, "error creating new low-level image %q", transports.ImageName(dest))
}
logrus.Debugf("created image ID %q", image.ID)
defer func() {
if err != nil {
_, err2 := b.store.DeleteImage(image.ID, true)
if err2 != nil {
logrus.Debugf("error removing image %q: %v", image.ID, err2)
}
}
}()
// Copy the configuration and manifest, which are big data items, along with whatever else is there.
for _, item := range items {
var data []byte
data, err = b.store.ImageBigData(tmpImg.ID, item)
if err != nil {
return errors.Wrapf(err, "error copying data item %q", item)
}
err = b.store.SetImageBigData(image.ID, item, data)
if err != nil {
return errors.Wrapf(err, "error copying data item %q", item)
}
logrus.Debugf("copied data item %q to %q", item, image.ID)
}
// Set low-level metadata in the new image so that the image library will accept it as a real image.
err = b.store.SetMetadata(image.ID, "{}")
if err != nil {
return errors.Wrapf(err, "error assigning metadata to new image %q", transports.ImageName(dest))
}
// Move the target name(s) from the temporary image to the new image.
err = util.AddImageNames(b.store, image, names)
if err != nil {
return errors.Wrapf(err, "error assigning names %v to new image", names)
}
logrus.Debugf("assigned names %v to image %q", names, image.ID)
return nil
}
// Commit writes the contents of the container, along with its updated
// configuration, to a new image in the specified location, and if we know how,
// add any additional tags that were specified.
@@ -50,13 +217,25 @@ func (b *Builder) Commit(dest types.ImageReference, options CommitOptions) error
if err != nil {
return err
}
src, err := b.makeContainerImageRef(options.PreferredManifestType, options.Compression)
// Check if we're keeping everything in local storage. If so, we can take certain shortcuts.
_, destIsStorage := dest.Transport().(storage.StoreTransport)
exporting := !destIsStorage
src, err := b.makeContainerImageRef(options.PreferredManifestType, exporting, options.Compression)
if err != nil {
return errors.Wrapf(err, "error recomputing layer digests and building metadata")
}
err = cp.Image(policyContext, dest, src, getCopyOptions(options.ReportWriter))
if err != nil {
return errors.Wrapf(err, "error copying layers and metadata")
if exporting {
// Copy everything.
err = cp.Image(policyContext, dest, src, getCopyOptions(options.ReportWriter))
if err != nil {
return errors.Wrapf(err, "error copying layers and metadata")
}
} else {
// Copy only the most recent layer, the configuration, and the manifest.
err = b.shallowCopy(dest, src, getSystemContext(options.SignaturePolicyPath))
if err != nil {
return errors.Wrapf(err, "error copying layer and metadata")
}
}
if len(options.AdditionalTags) > 0 {
switch dest.Transport().Name() {
@@ -69,6 +248,7 @@ func (b *Builder) Commit(dest types.ImageReference, options CommitOptions) error
if err != nil {
return errors.Wrapf(err, "error setting image names to %v", append(img.Names, options.AdditionalTags...))
}
logrus.Debugf("assigned names %v to image %q", img.Names, img.ID)
default:
logrus.Warnf("don't know how to add tags to images stored in %q transport", dest.Transport().Name())
}

64
image.go
View File

@@ -46,6 +46,7 @@ type containerImageRef struct {
createdBy string
annotations map[string]string
preferredManifestType string
exporting bool
}
type containerImageSource struct {
@@ -58,6 +59,7 @@ type containerImageSource struct {
configDigest digest.Digest
manifest []byte
manifestType string
exporting bool
}
func (i *containerImageRef) NewImage(sc *types.SystemContext) (types.Image, error) {
@@ -175,6 +177,46 @@ func (i *containerImageRef) NewImageSource(sc *types.SystemContext, manifestType
// Extract each layer and compute its digests, both compressed (if requested) and uncompressed.
for _, layerID := range layers {
omediaType := v1.MediaTypeImageLayer
dmediaType := docker.V2S2MediaTypeUncompressedLayer
// Figure out which media type we want to call this. Assume no compression.
if i.compression != archive.Uncompressed {
switch i.compression {
case archive.Gzip:
omediaType = v1.MediaTypeImageLayerGzip
dmediaType = docker.V2S2MediaTypeLayer
logrus.Debugf("compressing layer %q with gzip", layerID)
case archive.Bzip2:
// Until the image specs define a media type for bzip2-compressed layers, even if we know
// how to decompress them, we can't try to compress layers with bzip2.
return nil, errors.New("media type for bzip2-compressed layers is not defined")
default:
logrus.Debugf("compressing layer %q with unknown compressor(?)", layerID)
}
}
// If we're not re-exporting the data, just fake up layer and diff IDs for the manifest.
if !i.exporting {
fakeLayerDigest := digest.NewDigestFromHex(digest.Canonical.String(), layerID)
// Add a note in the manifest about the layer. The blobs should be identified by their
// possibly-compressed blob digests, but just use the layer IDs here.
olayerDescriptor := v1.Descriptor{
MediaType: omediaType,
Digest: fakeLayerDigest,
Size: -1,
}
omanifest.Layers = append(omanifest.Layers, olayerDescriptor)
dlayerDescriptor := docker.V2S2Descriptor{
MediaType: dmediaType,
Digest: fakeLayerDigest,
Size: -1,
}
dmanifest.Layers = append(dmanifest.Layers, dlayerDescriptor)
// Add a note about the diffID, which should be uncompressed digest of the blob, but
// just use the layer ID here.
oimage.RootFS.DiffIDs = append(oimage.RootFS.DiffIDs, fakeLayerDigest.String())
dimage.RootFS.DiffIDs = append(dimage.RootFS.DiffIDs, fakeLayerDigest)
continue
}
// Start reading the layer.
rc, err := i.store.Diff("", layerID)
if err != nil {
@@ -200,23 +242,7 @@ func (i *containerImageRef) NewImageSource(sc *types.SystemContext, manifestType
destHasher := digest.Canonical.Digester()
counter := ioutils.NewWriteCounter(layerFile)
multiWriter := io.MultiWriter(counter, destHasher.Hash())
// Figure out which media type we want to call this. Assume no compression.
omediaType := v1.MediaTypeImageLayer
dmediaType := docker.V2S2MediaTypeUncompressedLayer
if i.compression != archive.Uncompressed {
switch i.compression {
case archive.Gzip:
omediaType = v1.MediaTypeImageLayerGzip
dmediaType = docker.V2S2MediaTypeLayer
logrus.Debugf("compressing layer %q with gzip", layerID)
case archive.Bzip2:
// Until the image specs define a media type for bzip2-compressed layers, even if we know
// how to decompress them, we can't try to compress layers with bzip2.
return nil, errors.New("media type for bzip2-compressed layers is not defined")
default:
logrus.Debugf("compressing layer %q with unknown compressor(?)", layerID)
}
}
// Compress the layer, if we're compressing it.
writer, err := archive.CompressStream(multiWriter, i.compression)
if err != nil {
return nil, errors.Wrapf(err, "error compressing layer %q", layerID)
@@ -336,6 +362,7 @@ func (i *containerImageRef) NewImageSource(sc *types.SystemContext, manifestType
manifestType: manifestType,
config: config,
configDigest: digest.Canonical.FromBytes(config),
exporting: i.exporting,
}
return src, nil
}
@@ -427,7 +454,7 @@ func (i *containerImageSource) GetBlob(blob types.BlobInfo) (reader io.ReadClose
return ioutils.NewReadCloserWrapper(layerFile, closer), size, nil
}
func (b *Builder) makeContainerImageRef(manifestType string, compress archive.Compression) (types.ImageReference, error) {
func (b *Builder) makeContainerImageRef(manifestType string, exporting bool, compress archive.Compression) (types.ImageReference, error) {
var name reference.Named
if manifestType == "" {
manifestType = OCIv1ImageManifest
@@ -461,6 +488,7 @@ func (b *Builder) makeContainerImageRef(manifestType string, compress archive.Co
createdBy: b.CreatedBy(),
annotations: b.Annotations(),
preferredManifestType: manifestType,
exporting: exporting,
}
return ref, nil
}