Merge pull request #614 from vrothberg/vendor-storage-image

WIP - Vendor storage image

vendor.conf

@@ -2,13 +2,13 @@
 github.com/urfave/cli v1.20.0
 github.com/kr/pretty v0.1.0
 github.com/kr/text v0.1.0
-github.com/containers/image v1.5
+github.com/containers/image 12382fcf5a3d9aba70999114f3f81ed9e645e551
 github.com/vbauerster/mpb v3.3.4
 github.com/mattn/go-isatty v0.0.4
 github.com/VividCortex/ewma v1.1.1
 golang.org/x/sync 42b317875d0fa942474b76e1b46a6060d720ae6e
 github.com/opencontainers/go-digest c9281466c8b2f606084ac71339773efd177436e7
-github.com/containers/storage v1.11
+github.com/containers/storage 3c88d700b4daf494818633be242f02ccaccc6e72
 github.com/sirupsen/logrus v1.0.0
 github.com/go-check/check v1
 github.com/stretchr/testify v1.1.3

vendor/github.com/containers/image/README.md

@@ -65,7 +65,7 @@ the primary downside is that creating new signatures with the Golang-only implem
 - `containers_image_ostree_stub`: Instead of importing `ostree:` transport in `github.com/containers/image/transports/alltransports`, use a stub which reports that the transport is not supported. This allows building the library without requiring the `libostree` development libraries. The `github.com/containers/image/ostree` package is completely disabled
 and impossible to import when this build tag is in use.
 
-## [Contributing](CONTRIBUTING.md)**
+## [Contributing](CONTRIBUTING.md)
 
 Information about contributing to this project.
 

vendor/github.com/containers/image/docs/containers-certs.d.5.md

@@ -0,0 +1,28 @@
+% containers-certs.d(5)
+
+# NAME
+containers-certs.d - Directory for storing custom container-registry TLS configurations
+
+# DESCRIPTION
+A custom TLS configuration for a container registry can be configured by creating a directory under `/etc/containers/certs.d`.
+The name of the directory must correspond to the `host:port` of the registry (e.g., `my-registry.com:5000`).
+
+## Directory Structure
+A certs directory can contain one or more files with the following extensions:
+
+* `*.crt`  files with this extensions will be interpreted as CA certificates
+* `*.cert` files with this extensions will be interpreted as client certificates
+* `*.key`  files with this extensions will be interpreted as client keys
+
+Note that the client certificate-key pair will be selected by the file name (e.g., `client.{cert,key}`).
+An examplary setup for a registry running at `my-registry.com:5000` may look as follows:
+```
+/etc/containers/certs.d/    <- Certificate directory
+└── my-registry.com:5000    <- Hostname:port
+   ├── client.cert          <- Client certificate
+   ├── client.key           <- Client key
+   └── ca.crt               <- Certificate authority that signed the registry certificate
+```
+
+# HISTORY
+Feb 2019, Originally compiled by Valentin Rothberg <rothberg@redhat.com>

vendor/github.com/containers/image/pkg/blobinfocache/memory.go

@@ -5,7 +5,7 @@ import (
 	"time"
 
 	"github.com/containers/image/types"
-	"github.com/opencontainers/go-digest"
+	digest "github.com/opencontainers/go-digest"
 	"github.com/sirupsen/logrus"
 )
 
@@ -18,7 +18,8 @@ type locationKey struct {
 
 // memoryCache implements an in-memory-only BlobInfoCache
 type memoryCache struct {
-	mutex                 *sync.Mutex // synchronizes concurrent accesses
+	mutex sync.Mutex
+	// The following fields can only be accessed with mutex held.
 	uncompressedDigests   map[digest.Digest]digest.Digest
 	digestsByUncompressed map[digest.Digest]map[digest.Digest]struct{}             // stores a set of digests for each uncompressed digest
 	knownLocations        map[locationKey]map[types.BICLocationReference]time.Time // stores last known existence time for each location reference
@@ -30,7 +31,6 @@ type memoryCache struct {
 // Manual users of types.{ImageSource,ImageDestination} might also use this instead of a persistent cache.
 func NewMemoryCache() types.BlobInfoCache {
 	return &memoryCache{
-		mutex:                 new(sync.Mutex),
 		uncompressedDigests:   map[digest.Digest]digest.Digest{},
 		digestsByUncompressed: map[digest.Digest]map[digest.Digest]struct{}{},
 		knownLocations:        map[locationKey]map[types.BICLocationReference]time.Time{},
@@ -43,13 +43,11 @@ func NewMemoryCache() types.BlobInfoCache {
 func (mem *memoryCache) UncompressedDigest(anyDigest digest.Digest) digest.Digest {
 	mem.mutex.Lock()
 	defer mem.mutex.Unlock()
-	return mem.uncompressedDigest(anyDigest)
+	return mem.uncompressedDigestLocked(anyDigest)
 }
 
-// uncompressedDigest returns an uncompressed digest corresponding to anyDigest.
-// May return anyDigest if it is known to be uncompressed.
-// Returns "" if nothing is known about the digest (it may be compressed or uncompressed).
-func (mem *memoryCache) uncompressedDigest(anyDigest digest.Digest) digest.Digest {
+// uncompressedDigestLocked implements types.BlobInfoCache.UncompressedDigest, but must be called only with mem.mutex held.
+func (mem *memoryCache) uncompressedDigestLocked(anyDigest digest.Digest) digest.Digest {
 	if d, ok := mem.uncompressedDigests[anyDigest]; ok {
 		return d
 	}
@@ -125,7 +123,7 @@ func (mem *memoryCache) CandidateLocations(transport types.ImageTransport, scope
 	res = mem.appendReplacementCandidates(res, transport, scope, primaryDigest)
 	var uncompressedDigest digest.Digest // = ""
 	if canSubstitute {
-		if uncompressedDigest = mem.uncompressedDigest(primaryDigest); uncompressedDigest != "" {
+		if uncompressedDigest = mem.uncompressedDigestLocked(primaryDigest); uncompressedDigest != "" {
 			otherDigests := mem.digestsByUncompressed[uncompressedDigest] // nil if not present in the map
 			for d := range otherDigests {
 				if d != primaryDigest && d != uncompressedDigest {

vendor/github.com/containers/image/signature/policy_config.go

@@ -30,7 +30,7 @@ import (
 // -ldflags '-X github.com/containers/image/signature.systemDefaultPolicyPath=$your_path'
 var systemDefaultPolicyPath = builtinDefaultPolicyPath
 
-// builtinDefaultPolicyPath is the policy pat used for DefaultPolicy().
+// builtinDefaultPolicyPath is the policy path used for DefaultPolicy().
 // DO NOT change this, instead see systemDefaultPolicyPath above.
 const builtinDefaultPolicyPath = "/etc/containers/policy.json"
 

vendor/github.com/containers/image/storage/storage_image.go

@@ -732,7 +732,7 @@ func (s *storageImageDestination) Commit(ctx context.Context) error {
 		if err != nil {
 			return errors.Wrapf(err, "error copying non-layer blob %q to image", blob)
 		}
-		if err := s.imageRef.transport.store.SetImageBigData(img.ID, blob.String(), v); err != nil {
+		if err := s.imageRef.transport.store.SetImageBigData(img.ID, blob.String(), v, manifest.Digest); err != nil {
 			if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil {
 				logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2)
 			}
@@ -765,14 +765,14 @@ func (s *storageImageDestination) Commit(ctx context.Context) error {
 	if err != nil {
 		return errors.Wrapf(err, "error computing manifest digest")
 	}
-	if err := s.imageRef.transport.store.SetImageBigData(img.ID, manifestBigDataKey(manifestDigest), s.manifest); err != nil {
+	if err := s.imageRef.transport.store.SetImageBigData(img.ID, manifestBigDataKey(manifestDigest), s.manifest, manifest.Digest); err != nil {
 		if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil {
 			logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2)
 		}
 		logrus.Debugf("error saving manifest for image %q: %v", img.ID, err)
 		return err
 	}
-	if err := s.imageRef.transport.store.SetImageBigData(img.ID, storage.ImageDigestBigDataKey, s.manifest); err != nil {
+	if err := s.imageRef.transport.store.SetImageBigData(img.ID, storage.ImageDigestBigDataKey, s.manifest, manifest.Digest); err != nil {
 		if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil {
 			logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2)
 		}
@@ -781,7 +781,7 @@ func (s *storageImageDestination) Commit(ctx context.Context) error {
 	}
 	// Save the signatures, if we have any.
 	if len(s.signatures) > 0 {
-		if err := s.imageRef.transport.store.SetImageBigData(img.ID, "signatures", s.signatures); err != nil {
+		if err := s.imageRef.transport.store.SetImageBigData(img.ID, "signatures", s.signatures, manifest.Digest); err != nil {
 			if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil {
 				logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2)
 			}

vendor/github.com/containers/image/version/version.go

@@ -8,10 +8,10 @@ const (
 	// VersionMinor is for functionality in a backwards-compatible manner
 	VersionMinor = 1
 	// VersionPatch is for backwards-compatible bug fixes
-	VersionPatch = 5
+	VersionPatch = 6
 
 	// VersionDev indicates development branch. Releases will be empty string.
-	VersionDev = ""
+	VersionDev = "-dev"
 )
 
 // Version is the specification version that the package types support.

vendor/github.com/containers/storage/containers.go

@@ -71,7 +71,7 @@ type Container struct {
 type ContainerStore interface {
 	FileBasedStore
 	MetadataStore
-	BigDataStore
+	ContainerBigDataStore
 	FlaggableStore
 
 	// Create creates a container that has a specified ID (or generates a
@@ -456,7 +456,7 @@ func (r *containerStore) BigDataSize(id, key string) (int64, error) {
 		return size, nil
 	}
 	if data, err := r.BigData(id, key); err == nil && data != nil {
-		if r.SetBigData(id, key, data) == nil {
+		if err = r.SetBigData(id, key, data); err == nil {
 			c, ok := r.lookup(id)
 			if !ok {
 				return -1, ErrContainerUnknown
@@ -464,6 +464,8 @@ func (r *containerStore) BigDataSize(id, key string) (int64, error) {
 			if size, ok := c.BigDataSizes[key]; ok {
 				return size, nil
 			}
+		} else {
+			return -1, err
 		}
 	}
 	return -1, ErrSizeUnknown
@@ -484,7 +486,7 @@ func (r *containerStore) BigDataDigest(id, key string) (digest.Digest, error) {
 		return d, nil
 	}
 	if data, err := r.BigData(id, key); err == nil && data != nil {
-		if r.SetBigData(id, key, data) == nil {
+		if err = r.SetBigData(id, key, data); err == nil {
 			c, ok := r.lookup(id)
 			if !ok {
 				return "", ErrContainerUnknown
@@ -492,6 +494,8 @@ func (r *containerStore) BigDataDigest(id, key string) (digest.Digest, error) {
 			if d, ok := c.BigDataDigests[key]; ok {
 				return d, nil
 			}
+		} else {
+			return "", err
 		}
 	}
 	return "", ErrDigestUnknown

vendor/github.com/containers/storage/containers_ffjson.go

@@ -1,5 +1,5 @@
 // Code generated by ffjson <https://github.com/pquerna/ffjson>. DO NOT EDIT.
-// source: containers.go
+// source: ./containers.go
 
 package storage
 

vendor/github.com/containers/storage/images.go

@@ -8,7 +8,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/containers/image/manifest"
 	"github.com/containers/storage/pkg/ioutils"
 	"github.com/containers/storage/pkg/stringid"
 	"github.com/containers/storage/pkg/truncindex"
@@ -117,7 +116,7 @@ type ImageStore interface {
 	ROImageStore
 	RWFileBasedStore
 	RWMetadataStore
-	RWBigDataStore
+	RWImageBigDataStore
 	FlaggableStore
 
 	// Create creates an image that has a specified ID (or a random one) and
@@ -595,15 +594,7 @@ func (r *imageStore) BigDataSize(id, key string) (int64, error) {
 		return size, nil
 	}
 	if data, err := r.BigData(id, key); err == nil && data != nil {
-		if r.SetBigData(id, key, data) == nil {
-			image, ok := r.lookup(id)
-			if !ok {
-				return -1, ErrImageUnknown
-			}
-			if size, ok := image.BigDataSizes[key]; ok {
-				return size, nil
-			}
-		}
+		return int64(len(data)), nil
 	}
 	return -1, ErrSizeUnknown
 }
@@ -622,17 +613,6 @@ func (r *imageStore) BigDataDigest(id, key string) (digest.Digest, error) {
 	if d, ok := image.BigDataDigests[key]; ok {
 		return d, nil
 	}
-	if data, err := r.BigData(id, key); err == nil && data != nil {
-		if r.SetBigData(id, key, data) == nil {
-			image, ok := r.lookup(id)
-			if !ok {
-				return "", ErrImageUnknown
-			}
-			if d, ok := image.BigDataDigests[key]; ok {
-				return d, nil
-			}
-		}
-	}
 	return "", ErrDigestUnknown
 }
 
@@ -655,7 +635,7 @@ func imageSliceWithoutValue(slice []*Image, value *Image) []*Image {
 	return modified
 }
 
-func (r *imageStore) SetBigData(id, key string, data []byte) error {
+func (r *imageStore) SetBigData(id, key string, data []byte, digestManifest func([]byte) (digest.Digest, error)) error {
 	if key == "" {
 		return errors.Wrapf(ErrInvalidBigDataName, "can't set empty name for image big data item")
 	}
@@ -672,7 +652,10 @@ func (r *imageStore) SetBigData(id, key string, data []byte) error {
 	}
 	var newDigest digest.Digest
 	if bigDataNameIsManifest(key) {
-		if newDigest, err = manifest.Digest(data); err != nil {
+		if digestManifest == nil {
+			return errors.Wrapf(ErrDigestUnknown, "error digesting manifest: no manifest digest callback provided")
+		}
+		if newDigest, err = digestManifest(data); err != nil {
 			return errors.Wrapf(err, "error digesting manifest")
 		}
 	} else {

vendor/github.com/containers/storage/images_ffjson.go

@@ -1,5 +1,5 @@
 // Code generated by ffjson <https://github.com/pquerna/ffjson>. DO NOT EDIT.
-// source: images.go
+// source: ./images.go
 
 package storage
 

vendor/github.com/containers/storage/lockfile_unix.go

@@ -194,7 +194,7 @@ func (l *lockfile) Modified() (bool, error) {
 		return true, err
 	}
 	if n != len(id) {
-		return true, unix.ENOSPC
+		return true, nil
 	}
 	lw := l.lw
 	l.lw = string(id)

vendor/github.com/containers/storage/pkg/reexec/command_linux.go

@@ -3,6 +3,7 @@
 package reexec
 
 import (
+	"context"
 	"os/exec"
 	"syscall"
 
@@ -20,11 +21,23 @@ func Self() string {
 // This will use the in-memory version (/proc/self/exe) of the current binary,
 // it is thus safe to delete or replace the on-disk binary (os.Args[0]).
 func Command(args ...string) *exec.Cmd {
-	return &exec.Cmd{
-		Path: Self(),
-		Args: args,
-		SysProcAttr: &syscall.SysProcAttr{
-			Pdeathsig: unix.SIGTERM,
-		},
+	cmd := exec.Command(Self())
+	cmd.Args = args
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Pdeathsig: unix.SIGTERM,
 	}
+	return cmd
+}
+
+// CommandContext returns *exec.Cmd which has Path as current binary, and also
+// sets SysProcAttr.Pdeathsig to SIGTERM.
+// This will use the in-memory version (/proc/self/exe) of the current binary,
+// it is thus safe to delete or replace the on-disk binary (os.Args[0]).
+func CommandContext(ctx context.Context, args ...string) *exec.Cmd {
+	cmd := exec.CommandContext(ctx, Self())
+	cmd.Args = args
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Pdeathsig: unix.SIGTERM,
+	}
+	return cmd
 }

vendor/github.com/containers/storage/pkg/reexec/command_unix.go

@@ -3,6 +3,7 @@
 package reexec
 
 import (
+	"context"
 	"os/exec"
 )
 
@@ -16,8 +17,14 @@ func Self() string {
 // For example if current binary is "docker" at "/usr/bin/", then cmd.Path will
 // be set to "/usr/bin/docker".
 func Command(args ...string) *exec.Cmd {
-	return &exec.Cmd{
-		Path: Self(),
-		Args: args,
-	}
+	cmd := exec.Command(Self())
+	cmd.Args = args
+	return cmd
+}
+
+// CommandContext returns *exec.Cmd which has Path as current binary.
+func CommandContext(ctx context.Context, args ...string) *exec.Cmd {
+	cmd := exec.CommandContext(ctx, Self())
+	cmd.Args = args
+	return cmd
 }

vendor/github.com/containers/storage/pkg/reexec/command_unsupported.go

@@ -3,6 +3,7 @@
 package reexec
 
 import (
+	"context"
 	"os/exec"
 )
 
@@ -10,3 +11,8 @@ import (
 func Command(args ...string) *exec.Cmd {
 	return nil
 }
+
+// CommandContext is unsupported on operating systems apart from Linux, Windows, Solaris and Darwin.
+func CommandContext(ctx context.Context, args ...string) *exec.Cmd {
+	return nil
+}

vendor/github.com/containers/storage/pkg/reexec/command_windows.go

@@ -3,6 +3,7 @@
 package reexec
 
 import (
+	"context"
 	"os/exec"
 )
 
@@ -16,8 +17,16 @@ func Self() string {
 // For example if current binary is "docker.exe" at "C:\", then cmd.Path will
 // be set to "C:\docker.exe".
 func Command(args ...string) *exec.Cmd {
-	return &exec.Cmd{
-		Path: Self(),
-		Args: args,
-	}
+	cmd := exec.Command(Self())
+	cmd.Args = args
+	return cmd
+}
+
+// Command returns *exec.Cmd which has Path as current binary.
+// For example if current binary is "docker.exe" at "C:\", then cmd.Path will
+// be set to "C:\docker.exe".
+func CommandContext(ctx context.Context, args ...string) *exec.Cmd {
+	cmd := exec.CommandContext(ctx, Self())
+	cmd.Args = args
+	return cmd
 }

vendor/github.com/containers/storage/store.go

@@ -102,19 +102,21 @@ type ROBigDataStore interface {
 	BigDataNames(id string) ([]string, error)
 }
 
-// A RWBigDataStore wraps up the read-write big-data related methods of the
-// various types of file-based lookaside stores that we implement.
-type RWBigDataStore interface {
-	// SetBigData stores a (potentially large) piece of data associated with this
-	// ID.
-	SetBigData(id, key string, data []byte) error
+// A RWImageBigDataStore wraps up how we store big-data associated with images.
+type RWImageBigDataStore interface {
+	// SetBigData stores a (potentially large) piece of data associated
+	// with this ID.
+	// Pass github.com/containers/image/manifest.Digest as digestManifest
+	// to allow ByDigest to find images by their correct digests.
+	SetBigData(id, key string, data []byte, digestManifest func([]byte) (digest.Digest, error)) error
 }
 
-// A BigDataStore wraps up the most common big-data related methods of the
-// various types of file-based lookaside stores that we implement.
-type BigDataStore interface {
+// A ContainerBigDataStore wraps up how we store big-data associated with containers.
+type ContainerBigDataStore interface {
 	ROBigDataStore
-	RWBigDataStore
+	// SetBigData stores a (potentially large) piece of data associated
+	// with this ID.
+	SetBigData(id, key string, data []byte) error
 }
 
 // A FlaggableStore can have flags set and cleared on items which it manages.
@@ -352,9 +354,11 @@ type Store interface {
 	// of named data associated with an image.
 	ImageBigDataDigest(id, key string) (digest.Digest, error)
 
-	// SetImageBigData stores a (possibly large) chunk of named data associated
-	// with an image.
-	SetImageBigData(id, key string, data []byte) error
+	// SetImageBigData stores a (possibly large) chunk of named data
+	// associated with an image.  Pass
+	// github.com/containers/image/manifest.Digest as digestManifest to
+	// allow ImagesByDigest to find images by their correct digests.
+	SetImageBigData(id, key string, data []byte, digestManifest func([]byte) (digest.Digest, error)) error
 
 	// ImageSize computes the size of the image's layers and ancillary data.
 	ImageSize(id string) (int64, error)
@@ -1485,7 +1489,7 @@ func (s *store) ImageBigData(id, key string) ([]byte, error) {
 	return nil, ErrImageUnknown
 }
 
-func (s *store) SetImageBigData(id, key string, data []byte) error {
+func (s *store) SetImageBigData(id, key string, data []byte, digestManifest func([]byte) (digest.Digest, error)) error {
 	ristore, err := s.ImageStore()
 	if err != nil {
 		return err
@@ -1499,7 +1503,7 @@ func (s *store) SetImageBigData(id, key string, data []byte) error {
 		}
 	}
 
-	return ristore.SetBigData(id, key, data)
+	return ristore.SetBigData(id, key, data, digestManifest)
 }
 
 func (s *store) ImageSize(id string) (int64, error) {

vendor/github.com/containers/storage/vendor.conf

@@ -1,18 +1,15 @@
 github.com/BurntSushi/toml master
 github.com/Microsoft/go-winio 307e919c663683a9000576fdc855acaf9534c165
 github.com/Microsoft/hcsshim a8d9cc56cbce765a7eebdf4792e6ceceeff3edb8
-github.com/containers/image master
 github.com/davecgh/go-spew 346938d642f2ec3594ed81d874461961cd0faa76
 github.com/docker/docker 86f080cff0914e9694068ed78d503701667c4c00
 github.com/docker/go-units 0dadbb0345b35ec7ef35e228dabb8de89a65bf52
-github.com/docker/libtrust master
 github.com/klauspost/compress v1.4.1
 github.com/klauspost/cpuid v1.2.0
 github.com/klauspost/pgzip v1.2.1
 github.com/mattn/go-shellwords 753a2322a99f87c0eff284980e77f53041555bc6
 github.com/mistifyio/go-zfs c0224de804d438efd11ea6e52ada8014537d6062
 github.com/opencontainers/go-digest master
-github.com/opencontainers/image-spec master
 github.com/opencontainers/runc 6c22e77604689db8725fa866f0f2ec0b3e8c3a07
 github.com/opencontainers/selinux v1.1
 github.com/ostreedev/ostree-go master