From b0648d79d48007203534a38f864608aff8898ed5 Mon Sep 17 00:00:00 2001 From: Nalin Dahyabhai Date: Tue, 27 Jun 2017 17:09:22 -0400 Subject: [PATCH] Bump containers/storage and containers/image Update containers/storage and containers/image to the current-as-of-this-writing versions, 105f7c77aef0c797429e41552743bf5b03b63263 and 23bddaa64cc6bf3f3077cda0dbf1cdd7007434df respectively. Signed-off-by: Nalin Dahyabhai --- vendor/github.com/containers/image/README.md | 74 ------- .../image/docker/reference/README.md | 2 - .../containers/image/pkg/strslice/README.md | 1 - .../containers/image/storage/storage_image.go | 2 +- .../transports/alltransports/alltransports.go | 2 +- .../image/transports/alltransports/ostree.go | 8 + .../transports/alltransports/ostree_stub.go | 9 + .../containers/image/transports/stub.go | 36 +++ .../github.com/containers/image/vendor.conf | 37 ---- .../github.com/containers/storage/README.md | 43 ---- .../containers/storage/containers.go | 25 ++- .../github.com/containers/storage/images.go | 31 ++- .../github.com/containers/storage/layers.go | 207 +++++++++++++++--- vendor/github.com/containers/storage/store.go | 91 +++++++- .../github.com/containers/storage/vendor.conf | 19 -- 15 files changed, 356 insertions(+), 231 deletions(-) delete mode 100644 vendor/github.com/containers/image/README.md delete mode 100644 vendor/github.com/containers/image/docker/reference/README.md delete mode 100644 vendor/github.com/containers/image/pkg/strslice/README.md create mode 100644 vendor/github.com/containers/image/transports/alltransports/ostree.go create mode 100644 vendor/github.com/containers/image/transports/alltransports/ostree_stub.go create mode 100644 vendor/github.com/containers/image/transports/stub.go delete mode 100644 vendor/github.com/containers/image/vendor.conf delete mode 100644 vendor/github.com/containers/storage/README.md delete mode 100644 vendor/github.com/containers/storage/vendor.conf diff --git a/vendor/github.com/containers/image/README.md b/vendor/github.com/containers/image/README.md deleted file mode 100644 index ca8afd4c..00000000 --- a/vendor/github.com/containers/image/README.md +++ /dev/null @@ -1,74 +0,0 @@ -[![GoDoc](https://godoc.org/github.com/containers/image?status.svg)](https://godoc.org/github.com/containers/image) [![Build Status](https://travis-ci.org/containers/image.svg?branch=master)](https://travis-ci.org/containers/image) -= - -`image` is a set of Go libraries aimed at working in various way with -containers' images and container image registries. - -The containers/image library allows application to pull and push images from -container image registries, like the upstream docker registry. It also -implements "simple image signing". - -The containers/image library also allows you to inspect a repository on a -container registry without pulling down the image. This means it fetches the -repository's manifest and it is able to show you a `docker inspect`-like json -output about a whole repository or a tag. This library, in contrast to `docker -inspect`, helps you gather useful information about a repository or a tag -without requiring you to run `docker pull`. - -The containers/image library also allows you to translate from one image format -to another, for example docker container images to OCI images. It also allows -you to copy container images between various registries, possibly converting -them as necessary, and to sign and verify images. - -## Command-line usage - -The containers/image project is only a library with no user interface; -you can either incorporate it into your Go programs, or use the `skopeo` tool: - -The [skopeo](https://github.com/projectatomic/skopeo) tool uses the -containers/image library and takes advantage of many of its features, -e.g. `skopeo copy` exposes the `containers/image/copy.Image` functionality. - -## Dependencies - -This library does not ship a committed version of its dependencies in a `vendor` -subdirectory. This is so you can make well-informed decisions about which -libraries you should use with this package in your own projects, and because -types defined in the `vendor` directory would be impossible to use from your projects. - -What this project tests against dependencies-wise is located -[in vendor.conf](https://github.com/containers/image/blob/master/vendor.conf). - -## Building - -If you want to see what the library can do, or an example of how it is called, -consider starting with the [skopeo](https://github.com/projectatomic/skopeo) tool -instead. - -To integrate this library into your project, put it into `$GOPATH` or use -your preferred vendoring tool to include a copy in your project. -Ensure that the dependencies documented [in vendor.conf](https://github.com/containers/image/blob/master/vendor.conf) -are also available -(using those exact versions or different versions of your choosing). - -This library, by default, also depends on the GpgME C library. Either install it: -```sh -Fedora$ dnf install gpgme-devel libassuan-devel -macOS$ brew install gpgme -``` -or use the `containers_image_openpgp` build tag (e.g. using `go build -tags …`) -This will use a Golang-only OpenPGP implementation for signature verification instead of the default cgo/gpgme-based implementation; -the primary downside is that creating new signatures with the Golang-only implementation is not supported. - -## Contributing - -When developing this library, please use `make` (or `make … BUILDTAGS=…`) to take advantage of the tests and validation. - -## License - -ASL 2.0 - -## Contact - -- Mailing list: [containers-dev](https://groups.google.com/forum/?hl=en#!forum/containers-dev) -- IRC: #[container-projects](irc://irc.freenode.net:6667/#container-projects) on freenode.net diff --git a/vendor/github.com/containers/image/docker/reference/README.md b/vendor/github.com/containers/image/docker/reference/README.md deleted file mode 100644 index 53a88de8..00000000 --- a/vendor/github.com/containers/image/docker/reference/README.md +++ /dev/null @@ -1,2 +0,0 @@ -This is a copy of github.com/docker/distribution/reference as of commit fb0bebc4b64e3881cc52a2478d749845ed76d2a8, -except that ParseAnyReferenceWithSet has been removed to drop the dependency on github.com/docker/distribution/digestset. \ No newline at end of file diff --git a/vendor/github.com/containers/image/pkg/strslice/README.md b/vendor/github.com/containers/image/pkg/strslice/README.md deleted file mode 100644 index ae6097e8..00000000 --- a/vendor/github.com/containers/image/pkg/strslice/README.md +++ /dev/null @@ -1 +0,0 @@ -This package was replicated from [github.com/docker/docker v17.04.0-ce](https://github.com/docker/docker/tree/v17.04.0-ce/api/types/strslice). diff --git a/vendor/github.com/containers/image/storage/storage_image.go b/vendor/github.com/containers/image/storage/storage_image.go index 6ad12b21..5b9e6e20 100644 --- a/vendor/github.com/containers/image/storage/storage_image.go +++ b/vendor/github.com/containers/image/storage/storage_image.go @@ -521,7 +521,7 @@ func diffLayer(store storage.Store, layerID string) (rc io.ReadCloser, n int64, } else { n = layerMeta.CompressedSize } - diff, err := store.Diff("", layer.ID) + diff, err := store.Diff("", layer.ID, nil) if err != nil { return nil, -1, err } diff --git a/vendor/github.com/containers/image/transports/alltransports/alltransports.go b/vendor/github.com/containers/image/transports/alltransports/alltransports.go index dc70fadd..dd80b7f9 100644 --- a/vendor/github.com/containers/image/transports/alltransports/alltransports.go +++ b/vendor/github.com/containers/image/transports/alltransports/alltransports.go @@ -12,7 +12,7 @@ import ( _ "github.com/containers/image/docker/daemon" _ "github.com/containers/image/oci/layout" _ "github.com/containers/image/openshift" - _ "github.com/containers/image/ostree" + // The ostree transport is registered by ostree*.go _ "github.com/containers/image/storage" "github.com/containers/image/transports" "github.com/containers/image/types" diff --git a/vendor/github.com/containers/image/transports/alltransports/ostree.go b/vendor/github.com/containers/image/transports/alltransports/ostree.go new file mode 100644 index 00000000..0fc5d7ef --- /dev/null +++ b/vendor/github.com/containers/image/transports/alltransports/ostree.go @@ -0,0 +1,8 @@ +// +build !containers_image_ostree_stub + +package alltransports + +import ( + // Register the ostree transport + _ "github.com/containers/image/ostree" +) diff --git a/vendor/github.com/containers/image/transports/alltransports/ostree_stub.go b/vendor/github.com/containers/image/transports/alltransports/ostree_stub.go new file mode 100644 index 00000000..8b01afe7 --- /dev/null +++ b/vendor/github.com/containers/image/transports/alltransports/ostree_stub.go @@ -0,0 +1,9 @@ +// +build containers_image_ostree_stub + +package alltransports + +import "github.com/containers/image/transports" + +func init() { + transports.Register(transports.NewStubTransport("ostree")) +} diff --git a/vendor/github.com/containers/image/transports/stub.go b/vendor/github.com/containers/image/transports/stub.go new file mode 100644 index 00000000..087f69b6 --- /dev/null +++ b/vendor/github.com/containers/image/transports/stub.go @@ -0,0 +1,36 @@ +package transports + +import ( + "fmt" + + "github.com/containers/image/types" +) + +// stubTransport is an implementation of types.ImageTransport which has a name, but rejects any references with “the transport $name: is not supported in this build”. +type stubTransport string + +// NewStubTransport returns an implementation of types.ImageTransport which has a name, but rejects any references with “the transport $name: is not supported in this build”. +func NewStubTransport(name string) types.ImageTransport { + return stubTransport(name) +} + +// Name returns the name of the transport, which must be unique among other transports. +func (s stubTransport) Name() string { + return string(s) +} + +// ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an ImageReference. +func (s stubTransport) ParseReference(reference string) (types.ImageReference, error) { + return nil, fmt.Errorf(`The transport "%s:" is not supported in this build`, string(s)) +} + +// ValidatePolicyConfigurationScope checks that scope is a valid name for a signature.PolicyTransportScopes keys +// (i.e. a valid PolicyConfigurationIdentity() or PolicyConfigurationNamespaces() return value). +// It is acceptable to allow an invalid value which will never be matched, it can "only" cause user confusion. +// scope passed to this function will not be "", that value is always allowed. +func (s stubTransport) ValidatePolicyConfigurationScope(scope string) error { + // Allowing any reference in here allows tools with some transports stubbed-out to still + // use signature verification policies which refer to these stubbed-out transports. + // See also the treatment of unknown transports in policyTransportScopesWithTransport.UnmarshalJSON . + return nil +} diff --git a/vendor/github.com/containers/image/vendor.conf b/vendor/github.com/containers/image/vendor.conf deleted file mode 100644 index ffc6e67a..00000000 --- a/vendor/github.com/containers/image/vendor.conf +++ /dev/null @@ -1,37 +0,0 @@ -github.com/Sirupsen/logrus 7f4b1adc791766938c29457bed0703fb9134421a -github.com/containers/storage 989b1c1d85f5dfe2076c67b54289cc13dc836c8c -github.com/davecgh/go-spew 346938d642f2ec3594ed81d874461961cd0faa76 -github.com/docker/distribution df5327f76fb6468b84a87771e361762b8be23fdb -github.com/docker/docker 75843d36aa5c3eaade50da005f9e0ff2602f3d5e -github.com/docker/go-connections 7da10c8c50cad14494ec818dcdfb6506265c0086 -github.com/docker/go-units 0dadbb0345b35ec7ef35e228dabb8de89a65bf52 -github.com/docker/libtrust aabc10ec26b754e797f9028f4589c5b7bd90dc20 -github.com/ghodss/yaml 04f313413ffd65ce25f2541bfd2b2ceec5c0908c -github.com/gorilla/context 08b5f424b9271eedf6f9f0ce86cb9396ed337a42 -github.com/gorilla/mux 94e7d24fd285520f3d12ae998f7fdd6b5393d453 -github.com/imdario/mergo 50d4dbd4eb0e84778abe37cefef140271d96fade -github.com/mattn/go-runewidth 14207d285c6c197daabb5c9793d63e7af9ab2d50 -github.com/mattn/go-shellwords 005a0944d84452842197c2108bd9168ced206f78 -github.com/mistifyio/go-zfs c0224de804d438efd11ea6e52ada8014537d6062 -github.com/mtrmac/gpgme b2432428689ca58c2b8e8dea9449d3295cf96fc9 -github.com/opencontainers/go-digest aa2ec055abd10d26d539eb630a92241b781ce4bc -github.com/opencontainers/image-spec v1.0.0-rc6 -github.com/opencontainers/runc 6b1d0e76f239ffb435445e5ae316d2676c07c6e3 -github.com/pborman/uuid 1b00554d822231195d1babd97ff4a781231955c9 -github.com/pkg/errors 248dadf4e9068a0b3e79f02ed0a610d935de5302 -github.com/pmezard/go-difflib 792786c7400a136282c1664665ae0a8db921c6c2 -github.com/stretchr/testify 4d4bfba8f1d1027c4fdbe371823030df51419987 -github.com/vbatts/tar-split bd4c5d64c3e9297f410025a3b1bd0c58f659e721 -golang.org/x/crypto 453249f01cfeb54c3d549ddb75ff152ca243f9d8 -golang.org/x/net 6b27048ae5e6ad1ef927e72e437531493de612fe -golang.org/x/sys 075e574b89e4c2d22f2286a7e2b919519c6f3547 -gopkg.in/cheggaaa/pb.v1 d7e6ca3010b6f084d8056847f55d7f572f180678 -gopkg.in/yaml.v2 a3f3340b5840cee44f372bddb5880fcbc419b46a -k8s.io/client-go bcde30fb7eaed76fd98a36b4120321b94995ffb6 -github.com/xeipuuv/gojsonschema master -github.com/xeipuuv/gojsonreference master -github.com/xeipuuv/gojsonpointer master -github.com/tchap/go-patricia v2.2.6 -github.com/opencontainers/selinux ba1aefe8057f1d0cfb8e88d0ec1dc85925ef987d -github.com/BurntSushi/toml b26d9c308763d68093482582cea63d69be07a0f0 -github.com/ostreedev/ostree-go 61532f383f1f48e5c27080b0b9c8b022c3706a97 diff --git a/vendor/github.com/containers/storage/README.md b/vendor/github.com/containers/storage/README.md deleted file mode 100644 index 4d100a35..00000000 --- a/vendor/github.com/containers/storage/README.md +++ /dev/null @@ -1,43 +0,0 @@ -`storage` is a Go library which aims to provide methods for storing filesystem -layers, container images, and containers. An `oci-storage` CLI wrapper is also -included for manual and scripting use. - -To build the CLI wrapper, use 'make build-binary'. - -Operations which use VMs expect to launch them using 'vagrant', defaulting to -using its 'libvirt' provider. The boxes used are also available for the -'virtualbox' provider, and can be selected by setting $VAGRANT_PROVIDER to -'virtualbox' before kicking off the build. - -The library manages three types of items: layers, images, and containers. - -A *layer* is a copy-on-write filesystem which is notionally stored as a set of -changes relative to its *parent* layer, if it has one. A given layer can only -have one parent, but any layer can be the parent of multiple layers. Layers -which are parents of other layers should be treated as read-only. - -An *image* is a reference to a particular layer (its _top_ layer), along with -other information which the library can manage for the convenience of its -caller. This information typically includes configuration templates for -running a binary contained within the image's layers, and may include -cryptographic signatures. Multiple images can reference the same layer, as the -differences between two images may not be in their layer contents. - -A *container* is a read-write layer which is a child of an image's top layer, -along with information which the library can manage for the convenience of its -caller. This information typically includes configuration information for -running the specific container. Multiple containers can be derived from a -single image. - -Layers, images, and containers are represented primarily by 32 character -hexadecimal IDs, but items of each kind can also have one or more arbitrary -names attached to them, which the library will automatically resolve to IDs -when they are passed in to API calls which expect IDs. - -The library can store what it calls *metadata* for each of these types of -items. This is expected to be a small piece of data, since it is cached in -memory and stored along with the library's own bookkeeping information. - -Additionally, the library can store one or more of what it calls *big data* for -images and containers. This is a named chunk of larger data, which is only in -memory when it is being read from or being written to its own disk file. diff --git a/vendor/github.com/containers/storage/containers.go b/vendor/github.com/containers/storage/containers.go index a6a4f77b..0908bdd1 100644 --- a/vendor/github.com/containers/storage/containers.go +++ b/vendor/github.com/containers/storage/containers.go @@ -50,6 +50,12 @@ type Container struct { // that has been stored, if they're known. BigDataSizes map[string]int64 `json:"big-data-sizes,omitempty"` + // Created is the datestamp for when this container was created. Older + // versions of the library did not track this information, so callers + // will likely want to use the IsZero() method to verify that a value + // is set before using it. + Created time.Time `json:"created,omitempty"` + Flags map[string]interface{} `json:"flags,omitempty"` } @@ -253,6 +259,7 @@ func (r *containerStore) Create(id string, names []string, image, layer, metadat Metadata: metadata, BigDataNames: []string{}, BigDataSizes: make(map[string]int64), + Created: time.Now().UTC(), Flags: make(map[string]interface{}), } r.containers = append(r.containers, container) @@ -309,10 +316,11 @@ func (r *containerStore) Delete(id string) error { return ErrContainerUnknown } id = container.ID - newContainers := []*Container{} - for _, candidate := range r.containers { - if candidate.ID != id { - newContainers = append(newContainers, candidate) + toDeleteIndex := -1 + for i, candidate := range r.containers { + if candidate.ID == id { + toDeleteIndex = i + break } } delete(r.byid, id) @@ -321,7 +329,14 @@ func (r *containerStore) Delete(id string) error { for _, name := range container.Names { delete(r.byname, name) } - r.containers = newContainers + if toDeleteIndex != -1 { + // delete the container at toDeleteIndex + if toDeleteIndex == len(r.containers)-1 { + r.containers = r.containers[:len(r.containers)-1] + } else { + r.containers = append(r.containers[:toDeleteIndex], r.containers[toDeleteIndex+1:]...) + } + } if err := r.Save(); err != nil { return err } diff --git a/vendor/github.com/containers/storage/images.go b/vendor/github.com/containers/storage/images.go index 4843b213..fe17f631 100644 --- a/vendor/github.com/containers/storage/images.go +++ b/vendor/github.com/containers/storage/images.go @@ -46,6 +46,12 @@ type Image struct { // that has been stored, if they're known. BigDataSizes map[string]int64 `json:"big-data-sizes,omitempty"` + // Created is the datestamp for when this image was created. Older + // versions of the library did not track this information, so callers + // will likely want to use the IsZero() method to verify that a value + // is set before using it. + Created time.Time `json:"created,omitempty"` + Flags map[string]interface{} `json:"flags,omitempty"` } @@ -80,7 +86,7 @@ type ImageStore interface { // Create creates an image that has a specified ID (or a random one) and // optional names, using the specified layer as its topmost (hopefully // read-only) layer. That layer can be referenced by multiple images. - Create(id string, names []string, layer, metadata string) (*Image, error) + Create(id string, names []string, layer, metadata string, created time.Time) (*Image, error) // SetNames replaces the list of names associated with an image with the // supplied values. @@ -254,7 +260,7 @@ func (r *imageStore) SetFlag(id string, flag string, value interface{}) error { return r.Save() } -func (r *imageStore) Create(id string, names []string, layer, metadata string) (image *Image, err error) { +func (r *imageStore) Create(id string, names []string, layer, metadata string, created time.Time) (image *Image, err error) { if !r.IsReadWrite() { return nil, errors.Wrapf(ErrStoreIsReadOnly, "not allowed to create new images at %q", r.imagespath()) } @@ -274,6 +280,9 @@ func (r *imageStore) Create(id string, names []string, layer, metadata string) ( return nil, ErrDuplicateName } } + if created.IsZero() { + created = time.Now().UTC() + } if err == nil { image = &Image{ ID: id, @@ -282,6 +291,7 @@ func (r *imageStore) Create(id string, names []string, layer, metadata string) ( Metadata: metadata, BigDataNames: []string{}, BigDataSizes: make(map[string]int64), + Created: created, Flags: make(map[string]interface{}), } r.images = append(r.images, image) @@ -346,10 +356,10 @@ func (r *imageStore) Delete(id string) error { return ErrImageUnknown } id = image.ID - newImages := []*Image{} - for _, candidate := range r.images { - if candidate.ID != id { - newImages = append(newImages, candidate) + toDeleteIndex := -1 + for i, candidate := range r.images { + if candidate.ID == id { + toDeleteIndex = i } } delete(r.byid, id) @@ -357,7 +367,14 @@ func (r *imageStore) Delete(id string) error { for _, name := range image.Names { delete(r.byname, name) } - r.images = newImages + if toDeleteIndex != -1 { + // delete the image at toDeleteIndex + if toDeleteIndex == len(r.images)-1 { + r.images = r.images[:len(r.images)-1] + } else { + r.images = append(r.images[:toDeleteIndex], r.images[toDeleteIndex+1:]...) + } + } if err := r.Save(); err != nil { return err } diff --git a/vendor/github.com/containers/storage/layers.go b/vendor/github.com/containers/storage/layers.go index 691d84f6..972d3a99 100644 --- a/vendor/github.com/containers/storage/layers.go +++ b/vendor/github.com/containers/storage/layers.go @@ -15,6 +15,7 @@ import ( "github.com/containers/storage/pkg/ioutils" "github.com/containers/storage/pkg/stringid" "github.com/containers/storage/pkg/truncindex" + digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" "github.com/vbatts/tar-split/tar/asm" "github.com/vbatts/tar-split/tar/storage" @@ -66,6 +67,38 @@ type Layer struct { // mounted at the mount point. MountCount int `json:"-"` + // Created is the datestamp for when this layer was created. Older + // versions of the library did not track this information, so callers + // will likely want to use the IsZero() method to verify that a value + // is set before using it. + Created time.Time `json:"created,omitempty"` + + // CompressedDigest is the digest of the blob that was last passed to + // ApplyDiff() or Put(), as it was presented to us. + CompressedDigest digest.Digest `json:"compressed-diff-digest,omitempty"` + + // CompressedSize is the length of the blob that was last passed to + // ApplyDiff() or Put(), as it was presented to us. If + // CompressedDigest is not set, this should be treated as if it were an + // uninitialized value. + CompressedSize int64 `json:"compressed-size,omitempty"` + + // UncompressedDigest is the digest of the blob that was last passed to + // ApplyDiff() or Put(), after we decompressed it. Often referred to + // as a DiffID. + UncompressedDigest digest.Digest `json:"diff-digest,omitempty"` + + // UncompressedSize is the length of the blob that was last passed to + // ApplyDiff() or Put(), after we decompressed it. If + // UncompressedDigest is not set, this should be treated as if it were + // an uninitialized value. + UncompressedSize int64 `json:"diff-size,omitempty"` + + // CompressionType is the type of compression which we detected on the blob + // that was last passed to ApplyDiff() or Put(). + CompressionType archive.Compression `json:"compression,omitempty"` + + // Flags is arbitrary data about the layer. Flags map[string]interface{} `json:"flags,omitempty"` } @@ -75,6 +108,12 @@ type layerMountPoint struct { MountCount int `json:"count"` } +// DiffOptions override the default behavior of Diff() methods. +type DiffOptions struct { + // Compression, if set overrides the default compressor when generating a diff. + Compression *archive.Compression +} + // ROLayerStore wraps a graph driver, adding the ability to refer to layers by // name, and keeping track of parent-child relationships, along with a list of // all known layers. @@ -101,17 +140,31 @@ type ROLayerStore interface { // Diff produces a tarstream which can be applied to a layer with the contents // of the first layer to produce a layer with the contents of the second layer. // By default, the parent of the second layer is used as the first - // layer, so it need not be specified. - Diff(from, to string) (io.ReadCloser, error) + // layer, so it need not be specified. Options can be used to override + // default behavior, but are also not required. + Diff(from, to string, options *DiffOptions) (io.ReadCloser, error) // DiffSize produces an estimate of the length of the tarstream which would be // produced by Diff. DiffSize(from, to string) (int64, error) + // Size produces a cached value for the uncompressed size of the layer, + // if one is known, or -1 if it is not known. If the layer can not be + // found, it returns an error. + Size(name string) (int64, error) + // Lookup attempts to translate a name to an ID. Most methods do this // implicitly. Lookup(name string) (string, error) + // LayersByCompressedDigest returns a slice of the layers with the + // specified compressed digest value recorded for them. + LayersByCompressedDigest(d digest.Digest) ([]Layer, error) + + // LayersByUncompressedDigest returns a slice of the layers with the + // specified uncompressed digest value recorded for them. + LayersByUncompressedDigest(d digest.Digest) ([]Layer, error) + // Layers returns a slice of the known layers. Layers() ([]Layer, error) } @@ -164,15 +217,17 @@ type LayerStore interface { } type layerStore struct { - lockfile Locker - rundir string - driver drivers.Driver - layerdir string - layers []*Layer - idindex *truncindex.TruncIndex - byid map[string]*Layer - byname map[string]*Layer - bymount map[string]*Layer + lockfile Locker + rundir string + driver drivers.Driver + layerdir string + layers []*Layer + idindex *truncindex.TruncIndex + byid map[string]*Layer + byname map[string]*Layer + bymount map[string]*Layer + bycompressedsum map[digest.Digest][]string + byuncompressedsum map[digest.Digest][]string } func (r *layerStore) Layers() ([]Layer, error) { @@ -203,7 +258,8 @@ func (r *layerStore) Load() error { ids := make(map[string]*Layer) names := make(map[string]*Layer) mounts := make(map[string]*Layer) - parents := make(map[string][]*Layer) + compressedsums := make(map[digest.Digest][]string) + uncompressedsums := make(map[digest.Digest][]string) if err = json.Unmarshal(data, &layers); len(data) == 0 || err == nil { for n, layer := range layers { ids[layer.ID] = layers[n] @@ -215,10 +271,11 @@ func (r *layerStore) Load() error { } names[name] = layers[n] } - if pslice, ok := parents[layer.Parent]; ok { - parents[layer.Parent] = append(pslice, layers[n]) - } else { - parents[layer.Parent] = []*Layer{layers[n]} + if layer.CompressedDigest != "" { + compressedsums[layer.CompressedDigest] = append(compressedsums[layer.CompressedDigest], layer.ID) + } + if layer.UncompressedDigest != "" { + uncompressedsums[layer.UncompressedDigest] = append(uncompressedsums[layer.UncompressedDigest], layer.ID) } } } @@ -247,6 +304,8 @@ func (r *layerStore) Load() error { r.byid = ids r.byname = names r.bymount = mounts + r.bycompressedsum = compressedsums + r.byuncompressedsum = uncompressedsums err = nil // Last step: if we're writable, try to remove anything that a previous // user of this storage area marked for deletion but didn't manage to @@ -369,6 +428,20 @@ func (r *layerStore) lookup(id string) (*Layer, bool) { return nil, false } +func (r *layerStore) Size(name string) (int64, error) { + layer, ok := r.lookup(name) + if !ok { + return -1, ErrLayerUnknown + } + // We use the presence of a non-empty digest as an indicator that the size value was intentionally set, and that + // a zero value is not just present because it was never set to anything else (which can happen if the layer was + // created by a version of this library that didn't keep track of digest and size information). + if layer.UncompressedDigest != "" { + return layer.UncompressedSize, nil + } + return -1, nil +} + func (r *layerStore) ClearFlag(id string, flag string) error { if !r.IsReadWrite() { return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to clear flags on layers at %q", r.layerspath()) @@ -440,6 +513,7 @@ func (r *layerStore) Put(id, parent string, names []string, mountLabel string, o Parent: parent, Names: names, MountLabel: mountLabel, + Created: time.Now().UTC(), Flags: make(map[string]interface{}), } r.layers = append(r.layers, layer) @@ -615,13 +689,21 @@ func (r *layerStore) Delete(id string) error { if layer.MountPoint != "" { delete(r.bymount, layer.MountPoint) } - newLayers := []*Layer{} - for _, candidate := range r.layers { - if candidate.ID != id { - newLayers = append(newLayers, candidate) + toDeleteIndex := -1 + for i, candidate := range r.layers { + if candidate.ID == id { + toDeleteIndex = i + break + } + } + if toDeleteIndex != -1 { + // delete the layer at toDeleteIndex + if toDeleteIndex == len(r.layers)-1 { + r.layers = r.layers[:len(r.layers)-1] + } else { + r.layers = append(r.layers[:toDeleteIndex], r.layers[toDeleteIndex+1:]...) } } - r.layers = newLayers if err = r.Save(); err != nil { return err } @@ -726,20 +808,20 @@ func (r *layerStore) newFileGetter(id string) (drivers.FileGetCloser, error) { }, nil } -func (r *layerStore) Diff(from, to string) (io.ReadCloser, error) { +func (r *layerStore) Diff(from, to string, options *DiffOptions) (io.ReadCloser, error) { var metadata storage.Unpacker from, to, toLayer, err := r.findParentAndLayer(from, to) if err != nil { return nil, ErrLayerUnknown } - compression := archive.Uncompressed - if cflag, ok := toLayer.Flags[compressionFlag]; ok { - if ctype, ok := cflag.(float64); ok { - compression = archive.Compression(ctype) - } else if ctype, ok := cflag.(archive.Compression); ok { - compression = archive.Compression(ctype) - } + // Default to applying the type of encryption that we noted was used + // for the layerdiff when it was applied. + compression := toLayer.CompressionType + // If a particular compression type (or no compression) was selected, + // use that instead. + if options != nil && options.Compression != nil { + compression = *options.Compression } if from != toLayer.Parent { diff, err := r.driver.Diff(to, from) @@ -827,6 +909,10 @@ func (r *layerStore) DiffSize(from, to string) (size int64, err error) { } func (r *layerStore) ApplyDiff(to string, diff archive.Reader) (size int64, err error) { + if !r.IsReadWrite() { + return -1, errors.Wrapf(ErrStoreIsReadOnly, "not allowed to modify layer contents at %q", r.layerspath()) + } + layer, ok := r.lookup(to) if !ok { return -1, ErrLayerUnknown @@ -839,7 +925,9 @@ func (r *layerStore) ApplyDiff(to string, diff archive.Reader) (size int64, err } compression := archive.DetectCompression(header[:n]) - defragmented := io.MultiReader(bytes.NewBuffer(header[:n]), diff) + compressedDigest := digest.Canonical.Digester() + compressedCounter := ioutils.NewWriteCounter(compressedDigest.Hash()) + defragmented := io.TeeReader(io.MultiReader(bytes.NewBuffer(header[:n]), diff), compressedCounter) tsdata := bytes.Buffer{} compressor, err := gzip.NewWriterLevel(&tsdata, gzip.BestSpeed) @@ -847,15 +935,20 @@ func (r *layerStore) ApplyDiff(to string, diff archive.Reader) (size int64, err compressor = gzip.NewWriter(&tsdata) } metadata := storage.NewJSONPacker(compressor) - decompressed, err := archive.DecompressStream(defragmented) + uncompressed, err := archive.DecompressStream(defragmented) if err != nil { return -1, err } - payload, err := asm.NewInputTarStream(decompressed, metadata, storage.NewDiscardFilePutter()) + uncompressedDigest := digest.Canonical.Digester() + uncompressedCounter := ioutils.NewWriteCounter(uncompressedDigest.Hash()) + payload, err := asm.NewInputTarStream(io.TeeReader(uncompressed, uncompressedCounter), metadata, storage.NewDiscardFilePutter()) if err != nil { return -1, err } size, err = r.driver.ApplyDiff(layer.ID, layer.Parent, payload) + if err != nil { + return -1, err + } compressor.Close() if err == nil { if err := os.MkdirAll(filepath.Dir(r.tspath(layer.ID)), 0700); err != nil { @@ -866,15 +959,57 @@ func (r *layerStore) ApplyDiff(to string, diff archive.Reader) (size int64, err } } - if compression != archive.Uncompressed { - layer.Flags[compressionFlag] = compression - } else { - delete(layer.Flags, compressionFlag) + updateDigestMap := func(m *map[digest.Digest][]string, oldvalue, newvalue digest.Digest, id string) { + var newList []string + if oldvalue != "" { + for _, value := range (*m)[oldvalue] { + if value != id { + newList = append(newList, value) + } + } + if len(newList) > 0 { + (*m)[oldvalue] = newList + } else { + delete(*m, oldvalue) + } + } + if newvalue != "" { + (*m)[newvalue] = append((*m)[newvalue], id) + } } + updateDigestMap(&r.bycompressedsum, layer.CompressedDigest, compressedDigest.Digest(), layer.ID) + layer.CompressedDigest = compressedDigest.Digest() + layer.CompressedSize = compressedCounter.Count + updateDigestMap(&r.byuncompressedsum, layer.UncompressedDigest, uncompressedDigest.Digest(), layer.ID) + layer.UncompressedDigest = uncompressedDigest.Digest() + layer.UncompressedSize = uncompressedCounter.Count + layer.CompressionType = compression + + err = r.Save() return size, err } +func (r *layerStore) layersByDigestMap(m map[digest.Digest][]string, d digest.Digest) ([]Layer, error) { + var layers []Layer + for _, layerID := range m[d] { + layer, ok := r.lookup(layerID) + if !ok { + return nil, ErrLayerUnknown + } + layers = append(layers, *layer) + } + return layers, nil +} + +func (r *layerStore) LayersByCompressedDigest(d digest.Digest) ([]Layer, error) { + return r.layersByDigestMap(r.bycompressedsum, d) +} + +func (r *layerStore) LayersByUncompressedDigest(d digest.Digest) ([]Layer, error) { + return r.layersByDigestMap(r.byuncompressedsum, d) +} + func (r *layerStore) Lock() { r.lockfile.Lock() } diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index 7a2d29db..2dfcbc37 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -20,6 +20,7 @@ import ( "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/ioutils" "github.com/containers/storage/pkg/stringid" + "github.com/opencontainers/go-digest" "github.com/pkg/errors" ) @@ -298,8 +299,8 @@ type Store interface { DiffSize(from, to string) (int64, error) // Diff returns the tarstream which would specify the changes returned by - // Changes. - Diff(from, to string) (io.ReadCloser, error) + // Changes. If options are passed in, they can override default behaviors. + Diff(from, to string, options *DiffOptions) (io.ReadCloser, error) // ApplyDiff applies a tarstream to a layer. Information about the tarstream // is cached with the layer. Typically, a layer which is populated using a @@ -307,6 +308,18 @@ type Store interface { // before or after the diff is applied. ApplyDiff(to string, diff archive.Reader) (int64, error) + // LayersByCompressedDigest returns a slice of the layers with the + // specified compressed digest value recorded for them. + LayersByCompressedDigest(d digest.Digest) ([]Layer, error) + + // LayersByUncompressedDigest returns a slice of the layers with the + // specified uncompressed digest value recorded for them. + LayersByUncompressedDigest(d digest.Digest) ([]Layer, error) + + // LayerSize returns a cached approximation of the layer's size, or -1 + // if we don't have a value on hand. + LayerSize(id string) (int64, error) + // Layers returns a list of the currently known layers. Layers() ([]Layer, error) @@ -422,6 +435,9 @@ type Store interface { // ImageOptions is used for passing options to a Store's CreateImage() method. type ImageOptions struct { + // CreationDate, if not zero, will override the default behavior of marking the image as having been + // created when CreateImage() was called, recording CreationDate instead. + CreationDate time.Time } // ContainerOptions is used for passing options to a Store's CreateContainer() method. @@ -793,7 +809,13 @@ func (s *store) CreateImage(id string, names []string, layer, metadata string, o if modified, err := ristore.Modified(); modified || err != nil { ristore.Load() } - return ristore.Create(id, names, layer, metadata) + + creationDate := time.Now().UTC() + if options != nil { + creationDate = options.CreationDate + } + + return ristore.Create(id, names, layer, metadata, creationDate) } func (s *store) CreateContainer(id string, names []string, image, layer, metadata string, options *ContainerOptions) (*Container, error) { @@ -1747,7 +1769,7 @@ func (s *store) DiffSize(from, to string) (int64, error) { return -1, ErrLayerUnknown } -func (s *store) Diff(from, to string) (io.ReadCloser, error) { +func (s *store) Diff(from, to string, options *DiffOptions) (io.ReadCloser, error) { rlstore, err := s.LayerStore() if err != nil { return nil, err @@ -1764,7 +1786,7 @@ func (s *store) Diff(from, to string) (io.ReadCloser, error) { rlstore.Load() } if rlstore.Exists(to) { - return rlstore.Diff(from, to) + return rlstore.Diff(from, to, options) } } return nil, ErrLayerUnknown @@ -1786,6 +1808,65 @@ func (s *store) ApplyDiff(to string, diff archive.Reader) (int64, error) { return -1, ErrLayerUnknown } +func (s *store) layersByMappedDigest(m func(ROLayerStore, digest.Digest) ([]Layer, error), d digest.Digest) ([]Layer, error) { + var layers []Layer + rlstore, err := s.LayerStore() + if err != nil { + return nil, err + } + + stores, err := s.ROLayerStores() + if err != nil { + return nil, err + } + stores = append([]ROLayerStore{rlstore}, stores...) + + for _, rlstore := range stores { + rlstore.Lock() + defer rlstore.Unlock() + if modified, err := rlstore.Modified(); modified || err != nil { + rlstore.Load() + } + slayers, err := m(rlstore, d) + if err != nil { + return nil, err + } + layers = append(layers, slayers...) + } + return layers, nil +} + +func (s *store) LayersByCompressedDigest(d digest.Digest) ([]Layer, error) { + return s.layersByMappedDigest(func(r ROLayerStore, d digest.Digest) ([]Layer, error) { return r.LayersByCompressedDigest(d) }, d) +} + +func (s *store) LayersByUncompressedDigest(d digest.Digest) ([]Layer, error) { + return s.layersByMappedDigest(func(r ROLayerStore, d digest.Digest) ([]Layer, error) { return r.LayersByUncompressedDigest(d) }, d) +} + +func (s *store) LayerSize(id string) (int64, error) { + lstore, err := s.LayerStore() + if err != nil { + return -1, err + } + lstores, err := s.ROLayerStores() + if err != nil { + return -1, err + } + lstores = append([]ROLayerStore{lstore}, lstores...) + for _, rlstore := range lstores { + rlstore.Lock() + defer rlstore.Unlock() + if modified, err := rlstore.Modified(); modified || err != nil { + rlstore.Load() + } + if rlstore.Exists(id) { + return rlstore.Size(id) + } + } + return -1, ErrLayerUnknown +} + func (s *store) Layers() ([]Layer, error) { var layers []Layer rlstore, err := s.LayerStore() diff --git a/vendor/github.com/containers/storage/vendor.conf b/vendor/github.com/containers/storage/vendor.conf deleted file mode 100644 index 99895dfd..00000000 --- a/vendor/github.com/containers/storage/vendor.conf +++ /dev/null @@ -1,19 +0,0 @@ -github.com/BurntSushi/toml master -github.com/Microsoft/go-winio 307e919c663683a9000576fdc855acaf9534c165 -github.com/Microsoft/hcsshim 0f615c198a84e0344b4ed49c464d8833d4648dfc -github.com/Sirupsen/logrus 61e43dc76f7ee59a82bdf3d71033dc12bea4c77d -github.com/docker/engine-api 4290f40c056686fcaa5c9caf02eac1dde9315adf -github.com/docker/go-connections eb315e36415380e7c2fdee175262560ff42359da -github.com/docker/go-units 0dadbb0345b35ec7ef35e228dabb8de89a65bf52 -github.com/go-check/check 20d25e2804050c1cd24a7eea1e7a6447dd0e74ec -github.com/mattn/go-shellwords 753a2322a99f87c0eff284980e77f53041555bc6 -github.com/mistifyio/go-zfs c0224de804d438efd11ea6e52ada8014537d6062 -github.com/opencontainers/runc 6c22e77604689db8725fa866f0f2ec0b3e8c3a07 -github.com/opencontainers/selinux ba1aefe8057f1d0cfb8e88d0ec1dc85925ef987d -github.com/pborman/uuid 1b00554d822231195d1babd97ff4a781231955c9 -github.com/pkg/errors master -github.com/tchap/go-patricia v2.2.6 -github.com/vbatts/tar-split bd4c5d64c3e9297f410025a3b1bd0c58f659e721 -github.com/vdemeester/shakers 24d7f1d6a71aa5d9cbe7390e4afb66b7eef9e1b3 -golang.org/x/net f2499483f923065a842d38eb4c7f1927e6fc6e6d -golang.org/x/sys d75a52659825e75fff6158388dddc6a5b04f9ba5