bump to v0.1.24

Signed-off-by: Antonio Murdaca <runcom@redhat.com>

.travis.yml

@@ -1,10 +1,24 @@
-sudo: required
 
-services:
-  - docker
+matrix:
+  include:
+  -  os: linux
+     sudo: required
+     services:
+       - docker
+  -  os: osx
 
 notifications:
   email: false
 
+install:
+  # NOTE: The (brew update) should not be necessary, and slows things down;
+  # we include it as a workaround for https://github.com/Homebrew/brew/issues/3299
+  # ideally Travis should bake the (brew update) into its images
+  # (https://github.com/travis-ci/travis-ci/issues/8552 ), but that’s only going
+  # to happen around November 2017 per https://blog.travis-ci.com/2017-10-16-a-new-default-os-x-image-is-coming .
+  # Remove the (brew update) at that time.
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update && brew install gpgme ; fi
+
 script:
-  - make check
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]];   then hack/travis_osx.sh ; fi
+  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make check ; fi

Dockerfile

@@ -29,7 +29,7 @@ RUN set -x \
 RUN set -x \
 	&& yum install -y which git tar wget hostname util-linux bsdtar socat ethtool device-mapper iptables tree findutils nmap-ncat e2fsprogs xfsprogs lsof docker iproute \
 	&& export GOPATH=$(mktemp -d) \
-	&& git clone -b v1.5.0-alpha.3 git://github.com/openshift/origin "$GOPATH/src/github.com/openshift/origin" \
+	&& git clone --depth 1 -b v1.5.0-alpha.3 git://github.com/openshift/origin "$GOPATH/src/github.com/openshift/origin" \
 	&& (cd "$GOPATH/src/github.com/openshift/origin" && make clean build && make all WHAT=cmd/dockerregistry) \
 	&& cp -a "$GOPATH/src/github.com/openshift/origin/_output/local/bin/linux"/*/* /usr/local/bin \
 	&& cp "$GOPATH/src/github.com/openshift/origin/images/dockerregistry/config.yml" /atomic-registry-config.yml \

Makefile

@@ -2,7 +2,20 @@
 
 export GO15VENDOREXPERIMENT=1
 
+ifeq ($(shell uname),Darwin)
+PREFIX ?= ${DESTDIR}/usr/local
+DARWIN_BUILD_TAG=containers_image_ostree_stub
+# On macOS, (brew install gpgme) installs it within /usr/local, but /usr/local/include is not in the default search path.
+# Rather than hard-code this directory, use gpgme-config. Sadly that must be done at the top-level user
+# instead of locally in the gpgme subpackage, because cgo supports only pkg-config, not general shell scripts,
+# and gpgme does not install a pkg-config file.
+# If gpgme is not installed or gpgme-config can’t be found for other reasons, the error is silently ignored
+# (and the user will probably find out because the cgo compilation will fail).
+GPGME_ENV := CGO_CFLAGS="$(shell gpgme-config --cflags 2>/dev/null)" CGO_LDFLAGS="$(shell gpgme-config --libs 2>/dev/null)"
+else
 PREFIX ?= ${DESTDIR}/usr
+endif
+
 INSTALLDIR=${PREFIX}/bin
 MANINSTALLDIR=${PREFIX}/share/man
 CONTAINERSSYSCONFIGDIR=${DESTDIR}/etc/containers
@@ -10,6 +23,7 @@ REGISTRIESDDIR=${CONTAINERSSYSCONFIGDIR}/registries.d
 SIGSTOREDIR=${DESTDIR}/var/lib/atomic/sigstore
 BASHINSTALLDIR=${PREFIX}/share/bash-completion/completions
 GO_MD2MAN ?= go-md2man
+GO ?= go
 
 ifeq ($(DEBUG), 1)
   override GOGCFLAGS += -N -l
@@ -34,7 +48,7 @@ MANPAGES_MD = $(wildcard docs/*.md)
 
 BTRFS_BUILD_TAG = $(shell hack/btrfs_tag.sh)
 LIBDM_BUILD_TAG = $(shell hack/libdm_tag.sh)
-LOCAL_BUILD_TAGS = $(BTRFS_BUILD_TAG) $(LIBDM_BUILD_TAG)
+LOCAL_BUILD_TAGS = $(BTRFS_BUILD_TAG) $(LIBDM_BUILD_TAG) $(DARWIN_BUILD_TAG)
 BUILDTAGS += $(LOCAL_BUILD_TAGS)
 
 #   make all DEBUG=1
@@ -57,10 +71,10 @@ binary-static: cmd/skopeo
 
 # Build w/o using Docker containers
 binary-local:
-	go build -ldflags "-X main.gitCommit=${GIT_COMMIT}" -gcflags "$(GOGCFLAGS)" -tags "$(BUILDTAGS)" -o skopeo ./cmd/skopeo
+	$(GPGME_ENV) $(GO) build -ldflags "-X main.gitCommit=${GIT_COMMIT}" -gcflags "$(GOGCFLAGS)" -tags "$(BUILDTAGS)" -o skopeo ./cmd/skopeo
 
 binary-local-static:
-	go build -ldflags "-extldflags \"-static\" -X main.gitCommit=${GIT_COMMIT}" -gcflags "$(GOGCFLAGS)" -tags "$(BUILDTAGS)" -o skopeo ./cmd/skopeo
+	$(GPGME_ENV) $(GO) build -ldflags "-extldflags \"-static\" -X main.gitCommit=${GIT_COMMIT}" -gcflags "$(GOGCFLAGS)" -tags "$(BUILDTAGS)" -o skopeo ./cmd/skopeo
 
 build-container:
 	docker build ${DOCKER_BUILD_ARGS} -t "$(DOCKER_IMAGE)" .
@@ -76,17 +90,22 @@ clean:
 
 install: install-binary install-docs install-completions
 	install -d -m 755 ${SIGSTOREDIR}
-	install -D -m 644 default-policy.json ${CONTAINERSSYSCONFIGDIR}/policy.json
-	install -D -m 644 default.yaml ${REGISTRIESDDIR}/default.yaml
+	install -d -m 755 ${CONTAINERSSYSCONFIGDIR}
+	install -m 644 default-policy.json ${CONTAINERSSYSCONFIGDIR}/policy.json
+	install -d -m 755 ${REGISTRIESDDIR}
+	install -m 644 default.yaml ${REGISTRIESDDIR}/default.yaml
 
 install-binary: ./skopeo
-	install -D -m 755 skopeo ${INSTALLDIR}/skopeo
+	install -d -m 755 ${INSTALLDIR}
+	install -m 755 skopeo ${INSTALLDIR}/skopeo
 
 install-docs: docs/skopeo.1
-	install -D -m 644 docs/skopeo.1 ${MANINSTALLDIR}/man1/skopeo.1
+	install -d -m 755 ${MANINSTALLDIR}/man1
+	install -m 644 docs/skopeo.1 ${MANINSTALLDIR}/man1/skopeo.1
 
 install-completions:
-	install -m 644 -D completions/bash/skopeo ${BASHINSTALLDIR}/skopeo
+	install -m 755 -d ${BASHINSTALLDIR}
+	install -m 644 completions/bash/skopeo ${BASHINSTALLDIR}/skopeo
 
 shell: build-container
 	$(DOCKER_RUN_DOCKER) bash
@@ -111,4 +130,4 @@ validate-local:
 	hack/make.sh validate-git-marks validate-gofmt validate-lint validate-vet
 
 test-unit-local:
-	go test -tags "$(BUILDTAGS)" $$(go list -tags "$(BUILDTAGS)" -e ./... | grep -v '^github\.com/projectatomic/skopeo/\(integration\|vendor/.*\)$$')
+	$(GPGME_ENV) $(GO) test -tags "$(BUILDTAGS)" $$($(GO) list -tags "$(BUILDTAGS)" -e ./... | grep -v '^github\.com/projectatomic/skopeo/\(integration\|vendor/.*\)$$')

README.md

@@ -1,16 +1,44 @@
 skopeo [![Build Status](https://travis-ci.org/projectatomic/skopeo.svg?branch=master)](https://travis-ci.org/projectatomic/skopeo)
 =
 
-_Please be aware `skopeo` is still work in progress and it currently supports only registry API V2_
+`skopeo` is a command line utility that performs various operations on container images and image repositories.  Skopeo works with API V2 registries such as Docker registries, the Atomic registry, private registries, local directories and local OCI-layout directories.  Skopeo does not require a daemon to be running to perform these operations which consist of:
 
-`skopeo` is a command line utility for various operations on container images and image repositories.
+ * Inspecting an image showing its properties including its layers.
+ * Copying an image from and to various storage mechanisms.
+ * Deleting an image from an image repository.
+ * When required by the repository, skopeo can pass the appropriate credentials and certificates for authentication.
+
+ Skopeo operates on the following image and repository types:
+
+ * containers-storage:docker-reference
+         An image located in a local containers/storage image store.  Location and image store specified in /etc/containers/storage.conf
+
+ * dir:path
+         An existing local directory path storing the manifest, layer tarballs and signatures as individual files. This is a non-standardized format, primarily useful for debugging or noninvasive container inspection.
+
+ * docker://docker-reference
+         An image in a registry implementing the "Docker Registry HTTP API V2". By default, uses the authorization state in $HOME/.docker/config.json, which is set e.g. using (docker login).
+
+ * docker-archive:path[:docker-reference]
+         An image is stored in the `docker save` formated file.  docker-reference is only used when creating such a file, and it must not contain a digest.
+
+ * docker-daemon:docker-reference
+         An image docker-reference stored in the docker daemon internal storage.  docker-reference must contain either a tag or a digest.  Alternatively, when reading images, the format can also be docker-daemon:algo:digest (an image ID).
+
+ * oci:path:tag
+         An image tag in a directory compliant with "Open Container Image Layout Specification" at path.
+
+ * ostree:image[@/absolute/repo/path]
+         An image in local OSTree repository.  /absolute/repo/path defaults to /ostree/repo.
 
 Inspecting a repository
 -
 `skopeo` is able to _inspect_ a repository on a Docker registry and fetch images layers.
-By _inspect_ I mean it fetches the repository's manifest and it is able to show you a `docker inspect`-like
+The _inspect_ command fetches the repository's manifest and it is able to show you a `docker inspect`-like
 json output about a whole repository or a tag. This tool, in contrast to `docker inspect`, helps you gather useful information about
-a repository or a tag before pulling it (using disk space) - e.g. - which tags are available for the given repository? which labels the image has?
+a repository or a tag before pulling it (using disk space).  The inspect command can show you which tags are available for the given 
+repository, the labels the image has, the creation date and operating system of the image and more.  
+
 
 Examples:
 ```sh
@@ -115,7 +143,7 @@ Building without a container requires a bit more manual work and setup in your e
 
 Install the necessary dependencies:
 ```sh
-Fedora$ sudo dnf install gpgme-devel libassuan-devel btrfs-progs-devel device-mapper-devel
+Fedora$ sudo dnf install gpgme-devel libassuan-devel btrfs-progs-devel device-mapper-devel ostree-devel
 macOS$ brew install gpgme
 ```
 

cmd/skopeo/cgo_pthread_ordering_workaround.go

@@ -1,3 +1,5 @@
+// +build !containers_image_openpgp
+
 package main
 
 /*

cmd/skopeo/delete.go

@@ -24,10 +24,7 @@ func deleteHandler(context *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	if err := ref.DeleteImage(ctx); err != nil {
-		return err
-	}
-	return nil
+	return ref.DeleteImage(ctx)
 }
 
 var deleteCmd = cli.Command{

cmd/skopeo/inspect.go

@@ -6,12 +6,12 @@ import (
 	"strings"
 	"time"
 
-	"github.com/Sirupsen/logrus"
 	"github.com/containers/image/docker"
 	"github.com/containers/image/manifest"
 	"github.com/containers/image/transports"
 	"github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 

cmd/skopeo/layers.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/containers/image/directory"
 	"github.com/containers/image/image"
-	"github.com/containers/image/manifest"
 	"github.com/containers/image/types"
 	"github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
@@ -25,12 +24,7 @@ var layersCmd = cli.Command{
 		if c.NArg() == 0 {
 			return errors.New("Usage: layers imageReference [layer...]")
 		}
-		rawSource, err := parseImageSource(c, c.Args()[0], []string{
-			// TODO: skopeo layers only supports these now
-			// eventually we'll remove this command altogether...
-			manifest.DockerV2Schema1SignedMediaType,
-			manifest.DockerV2Schema1MediaType,
-		})
+		rawSource, err := parseImageSource(c, c.Args()[0])
 		if err != nil {
 			return err
 		}
@@ -115,10 +109,6 @@ var layersCmd = cli.Command{
 			return err
 		}
 
-		if err := dest.Commit(); err != nil {
-			return err
-		}
-
-		return nil
+		return dest.Commit()
 	},
 }

cmd/skopeo/main.go

@@ -4,10 +4,10 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/Sirupsen/logrus"
 	"github.com/containers/image/signature"
 	"github.com/containers/storage/pkg/reexec"
 	"github.com/projectatomic/skopeo/version"
+	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 

cmd/skopeo/utils.go

@@ -72,9 +72,8 @@ func parseImage(c *cli.Context) (types.Image, error) {
 }
 
 // parseImageSource converts image URL-like string to an ImageSource.
-// requestedManifestMIMETypes is as in types.ImageReference.NewImageSource.
 // The caller must call .Close() on the returned ImageSource.
-func parseImageSource(c *cli.Context, name string, requestedManifestMIMETypes []string) (types.ImageSource, error) {
+func parseImageSource(c *cli.Context, name string) (types.ImageSource, error) {
 	ref, err := alltransports.ParseImageName(name)
 	if err != nil {
 		return nil, err
@@ -83,5 +82,5 @@ func parseImageSource(c *cli.Context, name string, requestedManifestMIMETypes []
 	if err != nil {
 		return nil, err
 	}
-	return ref.NewImageSource(ctx, requestedManifestMIMETypes)
+	return ref.NewImageSource(ctx)
 }

docs/skopeo.1.md

@@ -12,10 +12,7 @@ It also allows you to copy container images between various registries, possibly
 ## IMAGE NAMES
 Most commands refer to container images, using a _transport_`:`_details_ format. The following formats are supported:
 
-  **atomic:**_hostname_**/**_namespace_**/**_stream_**:**_tag_
-  An image served by an OpenShift(Atomic) Registry server. The current OpenShift project and OpenShift Registry instance are by default read from `$HOME/.kube/config`, which is set e.g. using `(oc login)`.
-
-  **containers-storage://**_docker-reference_
+  **containers-storage:**_docker-reference_
   An image located in a local containers/storage image store.  Location and image store specified in /etc/containers/storage.conf
 
   **dir:**_path_

hack/travis_osx.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -e
+
+export GOPATH=$(pwd)/_gopath
+export PATH=$GOPATH/bin:$PATH
+
+_projectatomic="${GOPATH}/src/github.com/projectatomic"
+mkdir -vp ${_projectatomic}
+ln -vsf $(pwd) ${_projectatomic}/skopeo
+
+go get -u github.com/cpuguy83/go-md2man github.com/golang/lint/golint
+
+cd ${_projectatomic}/skopeo
+make validate-local test-unit-local binary-local
+sudo make install
+skopeo -v

integration/copy_test.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -14,6 +15,7 @@ import (
 	"github.com/containers/image/signature"
 	"github.com/go-check/check"
 	"github.com/opencontainers/go-digest"
+	"github.com/opencontainers/image-tools/image"
 )
 
 func init() {
@@ -88,6 +90,7 @@ func (s *CopySuite) TearDownSuite(c *check.C) {
 }
 
 func (s *CopySuite) TestCopyFailsWithManifestList(c *check.C) {
+	c.ExpectFailure("manifest-list-hotfix sacrificed hotfixes for being able to copy images")
 	assertSkopeoFails(c, ".*can not copy docker://estesp/busybox:latest: manifest contains multiple images.*", "copy", "docker://estesp/busybox:latest", "dir:somedir")
 }
 
@@ -135,14 +138,20 @@ func (s *CopySuite) TestCopySimple(c *check.C) {
 	out := combinedOutputOfCommand(c, "diff", "-urN", dir1, dir2)
 	c.Assert(out, check.Equals, "")
 
-	// docker v2s2 -> OCI image layout
+	// docker v2s2 -> OCI image layout with image name
 	// ociDest will be created by oci: if it doesn't exist
 	// so don't create it here to exercise auto-creation
-	ociDest := "busybox-latest"
+	ociDest := "busybox-latest-image"
+	ociImgName := "busybox"
 	defer os.RemoveAll(ociDest)
-	assertSkopeoSucceeds(c, "", "copy", "docker://busybox:latest", "oci:"+ociDest)
+	assertSkopeoSucceeds(c, "", "copy", "docker://busybox:latest", "oci:"+ociDest+":"+ociImgName)
 	_, err = os.Stat(ociDest)
 	c.Assert(err, check.IsNil)
+
+	// docker v2s2 -> OCI image layout without image name
+	ociDest = "busybox-latest-noimage"
+	defer os.RemoveAll(ociDest)
+	assertSkopeoFails(c, ".*Error initializing destination oci:busybox-latest-noimage:: cannot save image with empty image.ref.name.*", "copy", "docker://busybox:latest", "oci:"+ociDest)
 }
 
 // Check whether dir: images in dir1 and dir2 are equal, ignoring schema1 signatures.
@@ -241,13 +250,15 @@ func (s *CopySuite) TestCopyOCIRoundTrip(c *check.C) {
 	c.Assert(out, check.Equals, "")
 
 	// For some silly reason we pass a logger to the OCI library here...
-	//logger := log.New(os.Stderr, "", 0)
+	logger := log.New(os.Stderr, "", 0)
 
-	// TODO: Verify using the upstream OCI image validator.
-	//err = image.ValidateLayout(oci1, nil, logger)
-	//c.Assert(err, check.IsNil)
-	//err = image.ValidateLayout(oci2, nil, logger)
-	//c.Assert(err, check.IsNil)
+	// Verify using the upstream OCI image validator, this should catch most
+	// non-compliance errors. DO NOT REMOVE THIS TEST UNLESS IT'S ABSOLUTELY
+	// NECESSARY.
+	err = image.ValidateLayout(oci1, nil, logger)
+	c.Assert(err, check.IsNil)
+	err = image.ValidateLayout(oci2, nil, logger)
+	c.Assert(err, check.IsNil)
 
 	// Now verify that everything is identical. Currently this is true, but
 	// because we recompute the manifests on-the-fly this doesn't necessarily

vendor.conf

@@ -3,7 +3,8 @@ github.com/containers/image master
 github.com/opencontainers/go-digest master
 gopkg.in/cheggaaa/pb.v1 ad4efe000aa550bb54918c06ebbadc0ff17687b9 https://github.com/cheggaaa/pb
 github.com/containers/storage master
-github.com/Sirupsen/logrus v0.10.0
+github.com/sirupsen/logrus v1.0.0
+github.com/Sirupsen/logrus v1.0.0
 github.com/go-check/check v1
 github.com/stretchr/testify v1.1.3
 github.com/davecgh/go-spew master
@@ -12,7 +13,7 @@ github.com/pkg/errors master
 golang.org/x/crypto master
 # docker deps from https://github.com/docker/docker/blob/v1.11.2/hack/vendor.sh
 github.com/docker/docker v1.13.0
-github.com/docker/go-connections 4ccf312bf1d35e5dbda654e57a9be4c3f3cd0366
+github.com/docker/go-connections 3ede32e2033de7505e6500d6c868c2b9ed9f169d
 github.com/vbatts/tar-split v0.10.1
 github.com/gorilla/context 14f550f51a
 github.com/gorilla/mux e444e69cbd
@@ -22,11 +23,12 @@ golang.org/x/net master
 golang.org/x/text master
 github.com/docker/distribution master
 github.com/docker/libtrust master
+github.com/docker/docker-credential-helpers d68f9aeca33f5fd3f08eeae5e9d175edf4e731d1
 github.com/opencontainers/runc master
 github.com/opencontainers/image-spec v1.0.0
 # -- start OCI image validation requirements.
 github.com/opencontainers/runtime-spec v1.0.0
-github.com/opencontainers/image-tools v0.1.0
+github.com/opencontainers/image-tools da84dc9dddc823a32f543e60323f841d12429c51
 github.com/xeipuuv/gojsonschema master
 github.com/xeipuuv/gojsonreference master
 github.com/xeipuuv/gojsonpointer master

vendor/github.com/Sirupsen/logrus/README.md

@@ -1,4 +1,4 @@
-# Logrus <img src="http://i.imgur.com/hTeVwmJ.png" width="40" height="40" alt=":walrus:" class="emoji" title=":walrus:"/>&nbsp;[![Build Status](https://travis-ci.org/Sirupsen/logrus.svg?branch=master)](https://travis-ci.org/Sirupsen/logrus)&nbsp;[![GoDoc](https://godoc.org/github.com/Sirupsen/logrus?status.svg)](https://godoc.org/github.com/Sirupsen/logrus)
+# Logrus <img src="http://i.imgur.com/hTeVwmJ.png" width="40" height="40" alt=":walrus:" class="emoji" title=":walrus:"/>&nbsp;[![Build Status](https://travis-ci.org/sirupsen/logrus.svg?branch=master)](https://travis-ci.org/sirupsen/logrus)&nbsp;[![GoDoc](https://godoc.org/github.com/sirupsen/logrus?status.svg)](https://godoc.org/github.com/sirupsen/logrus)
 
 Logrus is a structured logger for Go (golang), completely API compatible with
 the standard library logger. [Godoc][godoc]. **Please note the Logrus API is not
@@ -7,6 +7,17 @@ many large deployments. The core API is unlikely to change much but please
 version control your Logrus to make sure you aren't fetching latest `master` on
 every build.**
 
+**Seeing weird case-sensitive problems?** Unfortunately, the author failed to
+realize the consequences of renaming to lower-case. Due to the Go package
+environment, this caused issues. Regretfully, there's no turning back now.
+Everything using `logrus` will need to use the lower-case:
+`github.com/sirupsen/logrus`. Any package that isn't, should be changed.
+
+I am terribly sorry for this inconvenience. Logrus strives hard for backwards
+compatibility, and the author failed to realize the cascading consequences of
+such a name-change. To fix Glide, see [these
+comments](https://github.com/sirupsen/logrus/issues/553#issuecomment-306591437).
+
 Nicely color-coded in development (when a TTY is attached, otherwise just
 plain text):
 
@@ -46,6 +57,12 @@ time="2015-03-26T01:27:38-04:00" level=fatal msg="The ice breaks!" err=&{0x20822
 exit status 1
 ```
 
+#### Case-sensitivity
+
+The organization's name was changed to lower-case--and this will not be changed
+back. If you are getting import conflicts due to case sensitivity, please use
+the lower-case import: `github.com/sirupsen/logrus`.
+
 #### Example
 
 The simplest way to use Logrus is simply the package-level exported logger:
@@ -54,7 +71,7 @@ The simplest way to use Logrus is simply the package-level exported logger:
 package main
 
 import (
-  log "github.com/Sirupsen/logrus"
+  log "github.com/sirupsen/logrus"
 )
 
 func main() {
@@ -65,7 +82,7 @@ func main() {
 ```
 
 Note that it's completely api-compatible with the stdlib logger, so you can
-replace your `log` imports everywhere with `log "github.com/Sirupsen/logrus"`
+replace your `log` imports everywhere with `log "github.com/sirupsen/logrus"`
 and you'll now have the flexibility of Logrus. You can customize it all you
 want:
 
@@ -74,15 +91,16 @@ package main
 
 import (
   "os"
-  log "github.com/Sirupsen/logrus"
+  log "github.com/sirupsen/logrus"
 )
 
 func init() {
   // Log as JSON instead of the default ASCII formatter.
   log.SetFormatter(&log.JSONFormatter{})
 
-  // Output to stderr instead of stdout, could also be a file.
-  log.SetOutput(os.Stderr)
+  // Output to stdout instead of the default stderr
+  // Can be any io.Writer, see below for File example
+  log.SetOutput(os.Stdout)
 
   // Only log the warning severity or above.
   log.SetLevel(log.WarnLevel)
@@ -123,7 +141,8 @@ application, you can also create an instance of the `logrus` Logger:
 package main
 
 import (
-  "github.com/Sirupsen/logrus"
+  "os"
+  "github.com/sirupsen/logrus"
 )
 
 // Create a new instance of the logger. You can have any number of instances.
@@ -132,7 +151,15 @@ var log = logrus.New()
 func main() {
   // The API for setting attributes is a little different than the package level
   // exported logger. See Godoc.
-  log.Out = os.Stderr
+  log.Out = os.Stdout
+
+  // You could set this to any `io.Writer` such as a file
+  // file, err := os.OpenFile("logrus.log", os.O_CREATE|os.O_WRONLY, 0666)
+  // if err == nil {
+  //  log.Out = file
+  // } else {
+  //  log.Info("Failed to log to file, using default stderr")
+  // }
 
   log.WithFields(logrus.Fields{
     "animal": "walrus",
@@ -143,7 +170,7 @@ func main() {
 
 #### Fields
 
-Logrus encourages careful, structured logging though logging fields instead of
+Logrus encourages careful, structured logging through logging fields instead of
 long, unparseable error messages. For example, instead of: `log.Fatalf("Failed
 to send event %s to topic %s with key %d")`, you should log the much more
 discoverable:
@@ -165,6 +192,20 @@ In general, with Logrus using any of the `printf`-family functions should be
 seen as a hint you should add a field, however, you can still use the
 `printf`-family functions with Logrus.
 
+#### Default Fields
+
+Often it's helpful to have fields _always_ attached to log statements in an
+application or parts of one. For example, you may want to always log the
+`request_id` and `user_ip` in the context of a request. Instead of writing
+`log.WithFields(log.Fields{"request_id": request_id, "user_ip": user_ip})` on
+every line, you can create a `logrus.Entry` to pass around instead:
+
+```go
+requestLogger := log.WithFields(log.Fields{"request_id": request_id, "user_ip": user_ip})
+requestLogger.Info("something happened on that request") # will log request_id and user_ip
+requestLogger.Warn("something not great happened")
+```
+
 #### Hooks
 
 You can add hooks for logging levels. For example to send errors to an exception
@@ -176,9 +217,9 @@ Logrus comes with [built-in hooks](hooks/). Add those, or your custom hook, in
 
 ```go
 import (
-  log "github.com/Sirupsen/logrus"
+  log "github.com/sirupsen/logrus"
   "gopkg.in/gemnasium/logrus-airbrake-hook.v2" // the package is named "aibrake"
-  logrus_syslog "github.com/Sirupsen/logrus/hooks/syslog"
+  logrus_syslog "github.com/sirupsen/logrus/hooks/syslog"
   "log/syslog"
 )
 
@@ -200,33 +241,51 @@ Note: Syslog hook also support connecting to local syslog (Ex. "/dev/log" or "/v
 
 | Hook  | Description |
 | ----- | ----------- |
-| [Airbrake](https://github.com/gemnasium/logrus-airbrake-hook) | Send errors to the Airbrake API V3. Uses the official [`gobrake`](https://github.com/airbrake/gobrake) behind the scenes. |
 | [Airbrake "legacy"](https://github.com/gemnasium/logrus-airbrake-legacy-hook) | Send errors to an exception tracking service compatible with the Airbrake API V2. Uses [`airbrake-go`](https://github.com/tobi/airbrake-go) behind the scenes. |
-| [Papertrail](https://github.com/polds/logrus-papertrail-hook) | Send errors to the [Papertrail](https://papertrailapp.com) hosted logging service via UDP. |
-| [Syslog](https://github.com/Sirupsen/logrus/blob/master/hooks/syslog/syslog.go) | Send errors to remote syslog server. Uses standard library `log/syslog` behind the scenes. |
-| [Bugsnag](https://github.com/Shopify/logrus-bugsnag/blob/master/bugsnag.go) | Send errors to the Bugsnag exception tracking service. |
-| [Sentry](https://github.com/evalphobia/logrus_sentry) | Send errors to the Sentry error logging and aggregation service. |
-| [Hiprus](https://github.com/nubo/hiprus) | Send errors to a channel in hipchat. |
-| [Logrusly](https://github.com/sebest/logrusly) | Send logs to [Loggly](https://www.loggly.com/) |
-| [Slackrus](https://github.com/johntdyer/slackrus) | Hook for Slack chat. |
-| [Journalhook](https://github.com/wercker/journalhook) | Hook for logging to `systemd-journald` |
-| [Graylog](https://github.com/gemnasium/logrus-graylog-hook) | Hook for logging to [Graylog](http://graylog2.org/) |
-| [Raygun](https://github.com/squirkle/logrus-raygun-hook) | Hook for logging to [Raygun.io](http://raygun.io/) |
-| [LFShook](https://github.com/rifflock/lfshook) | Hook for logging to the local filesystem |
-| [Honeybadger](https://github.com/agonzalezro/logrus_honeybadger) | Hook for sending exceptions to Honeybadger |
-| [Mail](https://github.com/zbindenren/logrus_mail) | Hook for sending exceptions via mail |
-| [Rollrus](https://github.com/heroku/rollrus) | Hook for sending errors to rollbar |
-| [Fluentd](https://github.com/evalphobia/logrus_fluent) | Hook for logging to fluentd |
-| [Mongodb](https://github.com/weekface/mgorus) | Hook for logging to mongodb |
-| [InfluxDB](https://github.com/Abramovic/logrus_influxdb) | Hook for logging to influxdb |
-| [Octokit](https://github.com/dorajistyle/logrus-octokit-hook) | Hook for logging to github via octokit |
-| [DeferPanic](https://github.com/deferpanic/dp-logrus) | Hook for logging to DeferPanic |
-| [Redis-Hook](https://github.com/rogierlommers/logrus-redis-hook) | Hook for logging to a ELK stack (through Redis) |
+| [Airbrake](https://github.com/gemnasium/logrus-airbrake-hook) | Send errors to the Airbrake API V3. Uses the official [`gobrake`](https://github.com/airbrake/gobrake) behind the scenes. |
+| [Amazon Kinesis](https://github.com/evalphobia/logrus_kinesis) | Hook for logging to [Amazon Kinesis](https://aws.amazon.com/kinesis/) |
 | [Amqp-Hook](https://github.com/vladoatanasov/logrus_amqp) | Hook for logging to Amqp broker (Like RabbitMQ) |
-| [KafkaLogrus](https://github.com/goibibo/KafkaLogrus) | Hook for logging to kafka |
-| [Typetalk](https://github.com/dragon3/logrus-typetalk-hook) | Hook for logging to [Typetalk](https://www.typetalk.in/) |
+| [Bugsnag](https://github.com/Shopify/logrus-bugsnag/blob/master/bugsnag.go) | Send errors to the Bugsnag exception tracking service. |
+| [DeferPanic](https://github.com/deferpanic/dp-logrus) | Hook for logging to DeferPanic |
+| [Discordrus](https://github.com/kz/discordrus) | Hook for logging to [Discord](https://discordapp.com/) |
 | [ElasticSearch](https://github.com/sohlich/elogrus) | Hook for logging to ElasticSearch|
-
+| [Firehose](https://github.com/beaubrewer/logrus_firehose) | Hook for logging to [Amazon Firehose](https://aws.amazon.com/kinesis/firehose/)
+| [Fluentd](https://github.com/evalphobia/logrus_fluent) | Hook for logging to fluentd |
+| [Go-Slack](https://github.com/multiplay/go-slack) | Hook for logging to [Slack](https://slack.com) |
+| [Graylog](https://github.com/gemnasium/logrus-graylog-hook) | Hook for logging to [Graylog](http://graylog2.org/) |
+| [Hiprus](https://github.com/nubo/hiprus) | Send errors to a channel in hipchat. |
+| [Honeybadger](https://github.com/agonzalezro/logrus_honeybadger) | Hook for sending exceptions to Honeybadger |
+| [InfluxDB](https://github.com/Abramovic/logrus_influxdb) | Hook for logging to influxdb |
+| [Influxus](http://github.com/vlad-doru/influxus) | Hook for concurrently logging to [InfluxDB](http://influxdata.com/) |
+| [Journalhook](https://github.com/wercker/journalhook) | Hook for logging to `systemd-journald` |
+| [KafkaLogrus](https://github.com/goibibo/KafkaLogrus) | Hook for logging to kafka |
+| [LFShook](https://github.com/rifflock/lfshook) | Hook for logging to the local filesystem |
+| [Logentries](https://github.com/jcftang/logentriesrus) | Hook for logging to [Logentries](https://logentries.com/) |
+| [Logentrus](https://github.com/puddingfactory/logentrus) | Hook for logging to [Logentries](https://logentries.com/) |
+| [Logmatic.io](https://github.com/logmatic/logmatic-go) | Hook for logging to [Logmatic.io](http://logmatic.io/) |
+| [Logrusly](https://github.com/sebest/logrusly) | Send logs to [Loggly](https://www.loggly.com/) |
+| [Logstash](https://github.com/bshuster-repo/logrus-logstash-hook) | Hook for logging to [Logstash](https://www.elastic.co/products/logstash) |
+| [Mail](https://github.com/zbindenren/logrus_mail) | Hook for sending exceptions via mail |
+| [Mongodb](https://github.com/weekface/mgorus) | Hook for logging to mongodb |
+| [NATS-Hook](https://github.com/rybit/nats_logrus_hook) | Hook for logging to [NATS](https://nats.io) |
+| [Octokit](https://github.com/dorajistyle/logrus-octokit-hook) | Hook for logging to github via octokit |
+| [Papertrail](https://github.com/polds/logrus-papertrail-hook) | Send errors to the [Papertrail](https://papertrailapp.com) hosted logging service via UDP. |
+| [PostgreSQL](https://github.com/gemnasium/logrus-postgresql-hook) | Send logs to [PostgreSQL](http://postgresql.org) |
+| [Pushover](https://github.com/toorop/logrus_pushover) | Send error via [Pushover](https://pushover.net) |
+| [Raygun](https://github.com/squirkle/logrus-raygun-hook) | Hook for logging to [Raygun.io](http://raygun.io/) |
+| [Redis-Hook](https://github.com/rogierlommers/logrus-redis-hook) | Hook for logging to a ELK stack (through Redis) |
+| [Rollrus](https://github.com/heroku/rollrus) | Hook for sending errors to rollbar |
+| [Scribe](https://github.com/sagar8192/logrus-scribe-hook) | Hook for logging to [Scribe](https://github.com/facebookarchive/scribe)|
+| [Sentry](https://github.com/evalphobia/logrus_sentry) | Send errors to the Sentry error logging and aggregation service. |
+| [Slackrus](https://github.com/johntdyer/slackrus) | Hook for Slack chat. |
+| [Stackdriver](https://github.com/knq/sdhook) | Hook for logging to [Google Stackdriver](https://cloud.google.com/logging/) |
+| [Sumorus](https://github.com/doublefree/sumorus) | Hook for logging to [SumoLogic](https://www.sumologic.com/)|
+| [Syslog](https://github.com/Sirupsen/logrus/blob/master/hooks/syslog/syslog.go) | Send errors to remote syslog server. Uses standard library `log/syslog` behind the scenes. |
+| [Syslog TLS](https://github.com/shinji62/logrus-syslog-ng) | Send errors to remote syslog server with TLS support. |
+| [TraceView](https://github.com/evalphobia/logrus_appneta) | Hook for logging to [AppNeta TraceView](https://www.appneta.com/products/traceview/) |
+| [Typetalk](https://github.com/dragon3/logrus-typetalk-hook) | Hook for logging to [Typetalk](https://www.typetalk.in/) |
+| [logz.io](https://github.com/ripcurld00d/logrus-logzio-hook) | Hook for logging to [logz.io](https://logz.io), a Log as a Service using Logstash |
+| [SQS-Hook](https://github.com/tsarpaul/logrus_sqs) | Hook for logging to [Amazon Simple Queue Service (SQS)](https://aws.amazon.com/sqs/) |
 
 #### Level logging
 
@@ -275,7 +334,7 @@ could do:
 
 ```go
 import (
-  log "github.com/Sirupsen/logrus"
+  log "github.com/sirupsen/logrus"
 )
 
 init() {
@@ -302,16 +361,15 @@ The built-in logging formatters are:
   without colors.
   * *Note:* to force colored output when there is no TTY, set the `ForceColors`
     field to `true`.  To force no colored output even if there is a TTY  set the
-    `DisableColors` field to `true`
+    `DisableColors` field to `true`. For Windows, see
+    [github.com/mattn/go-colorable](https://github.com/mattn/go-colorable).
+  * All options are listed in the [generated docs](https://godoc.org/github.com/sirupsen/logrus#TextFormatter).
 * `logrus.JSONFormatter`. Logs fields as JSON.
-* `logrus/formatters/logstash.LogstashFormatter`. Logs fields as [Logstash](http://logstash.net) Events.
-
-    ```go
-      logrus.SetFormatter(&logstash.LogstashFormatter{Type: "application_name"})
-    ```
+  * All options are listed in the [generated docs](https://godoc.org/github.com/sirupsen/logrus#JSONFormatter).
 
 Third party logging formatters:
 
+* [`logstash`](https://github.com/bshuster-repo/logrus-logstash-hook). Logs fields as [Logstash](http://logstash.net) Events.
 * [`prefixed`](https://github.com/x-cray/logrus-prefixed-formatter). Displays log entry source along with alternative layout.
 * [`zalgo`](https://github.com/aybabtme/logzalgo). Invoking the P͉̫o̳̼̊w̖͈̰͎e̬͔̭͂r͚̼̹̲ ̫͓͉̳͈ō̠͕͖̚f̝͍̠ ͕̲̞͖͑Z̖̫̤̫ͪa͉̬͈̗l͖͎g̳̥o̰̥̅!̣͔̲̻͊̄ ̙̘̦̹̦.
 
@@ -356,6 +414,18 @@ srv := http.Server{
 Each line written to that writer will be printed the usual way, using formatters
 and hooks. The level for those entries is `info`.
 
+This means that we can override the standard library logger easily:
+
+```go
+logger := logrus.New()
+logger.Formatter = &logrus.JSONFormatter{}
+
+// Use logrus for standard log output
+// Note that `log` here references stdlib's log
+// Not logrus imported under the name `log`.
+log.SetOutput(logger.Writer())
+```
+
 #### Rotation
 
 Log rotation is not provided with Logrus. Log rotation should be done by an
@@ -367,6 +437,7 @@ entries. It should not be a feature of the application-level logger.
 | Tool | Description |
 | ---- | ----------- |
 |[Logrus Mate](https://github.com/gogap/logrus_mate)|Logrus mate is a tool for Logrus to manage loggers, you can initial logger's level, hook and formatter by config file, the logger will generated with different config at different environment.|
+|[Logrus Viper Helper](https://github.com/heirko/go-contrib/tree/master/logrusHelper)|An Helper around Logrus to wrap with spf13/Viper to load configuration with fangs! And to simplify Logrus configuration use some behavior of [Logrus Mate](https://github.com/gogap/logrus_mate). [sample](https://github.com/heirko/iris-contrib/blob/master/middleware/logrus-logger/example) |
 
 #### Testing
 
@@ -376,13 +447,55 @@ Logrus has a built in facility for asserting the presence of log messages. This
 * a test logger (`test.NewNullLogger`) that just records log messages (and does not output any):
 
 ```go
-logger, hook := NewNullLogger()
-logger.Error("Hello error")
+import(
+  "github.com/sirupsen/logrus"
+  "github.com/sirupsen/logrus/hooks/null"
+  "github.com/stretchr/testify/assert"
+  "testing"
+)
 
-assert.Equal(1, len(hook.Entries))
-assert.Equal(logrus.ErrorLevel, hook.LastEntry().Level)
-assert.Equal("Hello error", hook.LastEntry().Message)
+func TestSomething(t*testing.T){
+  logger, hook := null.NewNullLogger()
+  logger.Error("Helloerror")
 
-hook.Reset()
-assert.Nil(hook.LastEntry())
+  assert.Equal(t, 1, len(hook.Entries))
+  assert.Equal(t, logrus.ErrorLevel, hook.LastEntry().Level)
+  assert.Equal(t, "Helloerror", hook.LastEntry().Message)
+
+  hook.Reset()
+  assert.Nil(t, hook.LastEntry())
+}
 ```
+
+#### Fatal handlers
+
+Logrus can register one or more functions that will be called when any `fatal`
+level message is logged. The registered handlers will be executed before
+logrus performs a `os.Exit(1)`. This behavior may be helpful if callers need
+to gracefully shutdown. Unlike a `panic("Something went wrong...")` call which can be intercepted with a deferred `recover` a call to `os.Exit(1)` can not be intercepted.
+
+```
+...
+handler := func() {
+  // gracefully shutdown something...
+}
+logrus.RegisterExitHandler(handler)
+...
+```
+
+#### Thread safety
+
+By default Logger is protected by mutex for concurrent writes, this mutex is invoked when calling hooks and writing logs.
+If you are sure such locking is not needed, you can call logger.SetNoLock() to disable the locking.
+
+Situation when locking is not needed includes:
+
+* You have no hooks registered, or hooks calling is already thread-safe.
+
+* Writing to logger.Out is already thread-safe, for example:
+
+  1) logger.Out is protected by locks.
+
+  2) logger.Out is a os.File handler opened with `O_APPEND` flag, and every write is smaller than 4k. (This allow multi-thread/multi-process writing)
+
+     (Refer to http://www.notthewizard.com/2014/06/17/are-files-appends-really-atomic/)

vendor/github.com/Sirupsen/logrus/alt_exit.go

@@ -0,0 +1,64 @@
+package logrus
+
+// The following code was sourced and modified from the
+// https://github.com/tebeka/atexit package governed by the following license:
+//
+// Copyright (c) 2012 Miki Tebeka <miki.tebeka@gmail.com>.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy of
+// this software and associated documentation files (the "Software"), to deal in
+// the Software without restriction, including without limitation the rights to
+// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+// the Software, and to permit persons to whom the Software is furnished to do so,
+// subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import (
+	"fmt"
+	"os"
+)
+
+var handlers = []func(){}
+
+func runHandler(handler func()) {
+	defer func() {
+		if err := recover(); err != nil {
+			fmt.Fprintln(os.Stderr, "Error: Logrus exit handler error:", err)
+		}
+	}()
+
+	handler()
+}
+
+func runHandlers() {
+	for _, handler := range handlers {
+		runHandler(handler)
+	}
+}
+
+// Exit runs all the Logrus atexit handlers and then terminates the program using os.Exit(code)
+func Exit(code int) {
+	runHandlers()
+	os.Exit(code)
+}
+
+// RegisterExitHandler adds a Logrus Exit handler, call logrus.Exit to invoke
+// all handlers. The handlers will also be invoked when any Fatal log entry is
+// made.
+//
+// This method is useful when a caller wishes to use logrus to log a fatal
+// message but also needs to gracefully shutdown. An example usecase could be
+// closing database connections, or sending a alert that the application is
+// closing.
+func RegisterExitHandler(handler func()) {
+	handlers = append(handlers, handler)
+}

vendor/github.com/Sirupsen/logrus/doc.go

@@ -7,7 +7,7 @@ The simplest way to use Logrus is simply the package-level exported logger:
   package main
 
   import (
-    log "github.com/Sirupsen/logrus"
+    log "github.com/sirupsen/logrus"
   )
 
   func main() {
@@ -21,6 +21,6 @@ The simplest way to use Logrus is simply the package-level exported logger:
 Output:
   time="2015-09-07T08:48:33Z" level=info msg="A walrus appears" animal=walrus number=1 size=10
 
-For a full guide visit https://github.com/Sirupsen/logrus
+For a full guide visit https://github.com/sirupsen/logrus
 */
 package logrus

vendor/github.com/Sirupsen/logrus/entry.go

@@ -3,11 +3,21 @@ package logrus
 import (
 	"bytes"
 	"fmt"
-	"io"
 	"os"
+	"sync"
 	"time"
 )
 
+var bufferPool *sync.Pool
+
+func init() {
+	bufferPool = &sync.Pool{
+		New: func() interface{} {
+			return new(bytes.Buffer)
+		},
+	}
+}
+
 // Defines the key when adding errors using WithError.
 var ErrorKey = "error"
 
@@ -29,6 +39,9 @@ type Entry struct {
 
 	// Message passed to Debug, Info, Warn, Error, Fatal or Panic
 	Message string
+
+	// When formatter is called in entry.log(), an Buffer may be set to entry
+	Buffer *bytes.Buffer
 }
 
 func NewEntry(logger *Logger) *Entry {
@@ -39,21 +52,15 @@ func NewEntry(logger *Logger) *Entry {
 	}
 }
 
-// Returns a reader for the entry, which is a proxy to the formatter.
-func (entry *Entry) Reader() (*bytes.Buffer, error) {
-	serialized, err := entry.Logger.Formatter.Format(entry)
-	return bytes.NewBuffer(serialized), err
-}
-
 // Returns the string representation from the reader and ultimately the
 // formatter.
 func (entry *Entry) String() (string, error) {
-	reader, err := entry.Reader()
+	serialized, err := entry.Logger.Formatter.Format(entry)
 	if err != nil {
 		return "", err
 	}
-
-	return reader.String(), err
+	str := string(serialized)
+	return str, nil
 }
 
 // Add an error as single field (using the key defined in ErrorKey) to the Entry.
@@ -81,6 +88,7 @@ func (entry *Entry) WithFields(fields Fields) *Entry {
 // This function is not declared with a pointer value because otherwise
 // race conditions will occur when using multiple goroutines
 func (entry Entry) log(level Level, msg string) {
+	var buffer *bytes.Buffer
 	entry.Time = time.Now()
 	entry.Level = level
 	entry.Message = msg
@@ -90,20 +98,23 @@ func (entry Entry) log(level Level, msg string) {
 		fmt.Fprintf(os.Stderr, "Failed to fire hook: %v\n", err)
 		entry.Logger.mu.Unlock()
 	}
-
-	reader, err := entry.Reader()
+	buffer = bufferPool.Get().(*bytes.Buffer)
+	buffer.Reset()
+	defer bufferPool.Put(buffer)
+	entry.Buffer = buffer
+	serialized, err := entry.Logger.Formatter.Format(&entry)
+	entry.Buffer = nil
 	if err != nil {
 		entry.Logger.mu.Lock()
 		fmt.Fprintf(os.Stderr, "Failed to obtain reader, %v\n", err)
 		entry.Logger.mu.Unlock()
-	}
-
-	entry.Logger.mu.Lock()
-	defer entry.Logger.mu.Unlock()
-
-	_, err = io.Copy(entry.Logger.Out, reader)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Failed to write to log, %v\n", err)
+	} else {
+		entry.Logger.mu.Lock()
+		_, err = entry.Logger.Out.Write(serialized)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to write to log, %v\n", err)
+		}
+		entry.Logger.mu.Unlock()
 	}
 
 	// To avoid Entry#log() returning a value that only would make sense for
@@ -115,7 +126,7 @@ func (entry Entry) log(level Level, msg string) {
 }
 
 func (entry *Entry) Debug(args ...interface{}) {
-	if entry.Logger.Level >= DebugLevel {
+	if entry.Logger.level() >= DebugLevel {
 		entry.log(DebugLevel, fmt.Sprint(args...))
 	}
 }
@@ -125,13 +136,13 @@ func (entry *Entry) Print(args ...interface{}) {
 }
 
 func (entry *Entry) Info(args ...interface{}) {
-	if entry.Logger.Level >= InfoLevel {
+	if entry.Logger.level() >= InfoLevel {
 		entry.log(InfoLevel, fmt.Sprint(args...))
 	}
 }
 
 func (entry *Entry) Warn(args ...interface{}) {
-	if entry.Logger.Level >= WarnLevel {
+	if entry.Logger.level() >= WarnLevel {
 		entry.log(WarnLevel, fmt.Sprint(args...))
 	}
 }
@@ -141,20 +152,20 @@ func (entry *Entry) Warning(args ...interface{}) {
 }
 
 func (entry *Entry) Error(args ...interface{}) {
-	if entry.Logger.Level >= ErrorLevel {
+	if entry.Logger.level() >= ErrorLevel {
 		entry.log(ErrorLevel, fmt.Sprint(args...))
 	}
 }
 
 func (entry *Entry) Fatal(args ...interface{}) {
-	if entry.Logger.Level >= FatalLevel {
+	if entry.Logger.level() >= FatalLevel {
 		entry.log(FatalLevel, fmt.Sprint(args...))
 	}
-	os.Exit(1)
+	Exit(1)
 }
 
 func (entry *Entry) Panic(args ...interface{}) {
-	if entry.Logger.Level >= PanicLevel {
+	if entry.Logger.level() >= PanicLevel {
 		entry.log(PanicLevel, fmt.Sprint(args...))
 	}
 	panic(fmt.Sprint(args...))
@@ -163,13 +174,13 @@ func (entry *Entry) Panic(args ...interface{}) {
 // Entry Printf family functions
 
 func (entry *Entry) Debugf(format string, args ...interface{}) {
-	if entry.Logger.Level >= DebugLevel {
+	if entry.Logger.level() >= DebugLevel {
 		entry.Debug(fmt.Sprintf(format, args...))
 	}
 }
 
 func (entry *Entry) Infof(format string, args ...interface{}) {
-	if entry.Logger.Level >= InfoLevel {
+	if entry.Logger.level() >= InfoLevel {
 		entry.Info(fmt.Sprintf(format, args...))
 	}
 }
@@ -179,7 +190,7 @@ func (entry *Entry) Printf(format string, args ...interface{}) {
 }
 
 func (entry *Entry) Warnf(format string, args ...interface{}) {
-	if entry.Logger.Level >= WarnLevel {
+	if entry.Logger.level() >= WarnLevel {
 		entry.Warn(fmt.Sprintf(format, args...))
 	}
 }
@@ -189,20 +200,20 @@ func (entry *Entry) Warningf(format string, args ...interface{}) {
 }
 
 func (entry *Entry) Errorf(format string, args ...interface{}) {
-	if entry.Logger.Level >= ErrorLevel {
+	if entry.Logger.level() >= ErrorLevel {
 		entry.Error(fmt.Sprintf(format, args...))
 	}
 }
 
 func (entry *Entry) Fatalf(format string, args ...interface{}) {
-	if entry.Logger.Level >= FatalLevel {
+	if entry.Logger.level() >= FatalLevel {
 		entry.Fatal(fmt.Sprintf(format, args...))
 	}
-	os.Exit(1)
+	Exit(1)
 }
 
 func (entry *Entry) Panicf(format string, args ...interface{}) {
-	if entry.Logger.Level >= PanicLevel {
+	if entry.Logger.level() >= PanicLevel {
 		entry.Panic(fmt.Sprintf(format, args...))
 	}
 }
@@ -210,13 +221,13 @@ func (entry *Entry) Panicf(format string, args ...interface{}) {
 // Entry Println family functions
 
 func (entry *Entry) Debugln(args ...interface{}) {
-	if entry.Logger.Level >= DebugLevel {
+	if entry.Logger.level() >= DebugLevel {
 		entry.Debug(entry.sprintlnn(args...))
 	}
 }
 
 func (entry *Entry) Infoln(args ...interface{}) {
-	if entry.Logger.Level >= InfoLevel {
+	if entry.Logger.level() >= InfoLevel {
 		entry.Info(entry.sprintlnn(args...))
 	}
 }
@@ -226,7 +237,7 @@ func (entry *Entry) Println(args ...interface{}) {
 }
 
 func (entry *Entry) Warnln(args ...interface{}) {
-	if entry.Logger.Level >= WarnLevel {
+	if entry.Logger.level() >= WarnLevel {
 		entry.Warn(entry.sprintlnn(args...))
 	}
 }
@@ -236,20 +247,20 @@ func (entry *Entry) Warningln(args ...interface{}) {
 }
 
 func (entry *Entry) Errorln(args ...interface{}) {
-	if entry.Logger.Level >= ErrorLevel {
+	if entry.Logger.level() >= ErrorLevel {
 		entry.Error(entry.sprintlnn(args...))
 	}
 }
 
 func (entry *Entry) Fatalln(args ...interface{}) {
-	if entry.Logger.Level >= FatalLevel {
+	if entry.Logger.level() >= FatalLevel {
 		entry.Fatal(entry.sprintlnn(args...))
 	}
-	os.Exit(1)
+	Exit(1)
 }
 
 func (entry *Entry) Panicln(args ...interface{}) {
-	if entry.Logger.Level >= PanicLevel {
+	if entry.Logger.level() >= PanicLevel {
 		entry.Panic(entry.sprintlnn(args...))
 	}
 }

vendor/github.com/Sirupsen/logrus/exported.go

@@ -31,14 +31,14 @@ func SetFormatter(formatter Formatter) {
 func SetLevel(level Level) {
 	std.mu.Lock()
 	defer std.mu.Unlock()
-	std.Level = level
+	std.setLevel(level)
 }
 
 // GetLevel returns the standard logger level.
 func GetLevel() Level {
 	std.mu.Lock()
 	defer std.mu.Unlock()
-	return std.Level
+	return std.level()
 }
 
 // AddHook adds a hook to the standard logger hooks.

vendor/github.com/Sirupsen/logrus/formatter.go

@@ -31,18 +31,15 @@ type Formatter interface {
 // It's not exported because it's still using Data in an opinionated way. It's to
 // avoid code duplication between the two default formatters.
 func prefixFieldClashes(data Fields) {
-	_, ok := data["time"]
-	if ok {
-		data["fields.time"] = data["time"]
+	if t, ok := data["time"]; ok {
+		data["fields.time"] = t
 	}
 
-	_, ok = data["msg"]
-	if ok {
-		data["fields.msg"] = data["msg"]
+	if m, ok := data["msg"]; ok {
+		data["fields.msg"] = m
 	}
 
-	_, ok = data["level"]
-	if ok {
-		data["fields.level"] = data["level"]
+	if l, ok := data["level"]; ok {
+		data["fields.level"] = l
 	}
 }

vendor/github.com/Sirupsen/logrus/json_formatter.go

@@ -5,9 +5,40 @@ import (
 	"fmt"
 )
 
+type fieldKey string
+type FieldMap map[fieldKey]string
+
+const (
+	FieldKeyMsg   = "msg"
+	FieldKeyLevel = "level"
+	FieldKeyTime  = "time"
+)
+
+func (f FieldMap) resolve(key fieldKey) string {
+	if k, ok := f[key]; ok {
+		return k
+	}
+
+	return string(key)
+}
+
 type JSONFormatter struct {
 	// TimestampFormat sets the format used for marshaling timestamps.
 	TimestampFormat string
+
+	// DisableTimestamp allows disabling automatic timestamps in output
+	DisableTimestamp bool
+
+	// FieldMap allows users to customize the names of keys for various fields.
+	// As an example:
+	// formatter := &JSONFormatter{
+	//   	FieldMap: FieldMap{
+	// 		 FieldKeyTime: "@timestamp",
+	// 		 FieldKeyLevel: "@level",
+	// 		 FieldKeyMsg: "@message",
+	//    },
+	// }
+	FieldMap FieldMap
 }
 
 func (f *JSONFormatter) Format(entry *Entry) ([]byte, error) {
@@ -16,7 +47,7 @@ func (f *JSONFormatter) Format(entry *Entry) ([]byte, error) {
 		switch v := v.(type) {
 		case error:
 			// Otherwise errors are ignored by `encoding/json`
-			// https://github.com/Sirupsen/logrus/issues/137
+			// https://github.com/sirupsen/logrus/issues/137
 			data[k] = v.Error()
 		default:
 			data[k] = v
@@ -29,9 +60,11 @@ func (f *JSONFormatter) Format(entry *Entry) ([]byte, error) {
 		timestampFormat = DefaultTimestampFormat
 	}
 
-	data["time"] = entry.Time.Format(timestampFormat)
-	data["msg"] = entry.Message
-	data["level"] = entry.Level.String()
+	if !f.DisableTimestamp {
+		data[f.FieldMap.resolve(FieldKeyTime)] = entry.Time.Format(timestampFormat)
+	}
+	data[f.FieldMap.resolve(FieldKeyMsg)] = entry.Message
+	data[f.FieldMap.resolve(FieldKeyLevel)] = entry.Level.String()
 
 	serialized, err := json.Marshal(data)
 	if err != nil {

vendor/github.com/Sirupsen/logrus/logger.go

@@ -4,6 +4,7 @@ import (
 	"io"
 	"os"
 	"sync"
+	"sync/atomic"
 )
 
 type Logger struct {
@@ -26,8 +27,31 @@ type Logger struct {
 	// to) `logrus.Info`, which allows Info(), Warn(), Error() and Fatal() to be
 	// logged. `logrus.Debug` is useful in
 	Level Level
-	// Used to sync writing to the log.
-	mu sync.Mutex
+	// Used to sync writing to the log. Locking is enabled by Default
+	mu MutexWrap
+	// Reusable empty entry
+	entryPool sync.Pool
+}
+
+type MutexWrap struct {
+	lock     sync.Mutex
+	disabled bool
+}
+
+func (mw *MutexWrap) Lock() {
+	if !mw.disabled {
+		mw.lock.Lock()
+	}
+}
+
+func (mw *MutexWrap) Unlock() {
+	if !mw.disabled {
+		mw.lock.Unlock()
+	}
+}
+
+func (mw *MutexWrap) Disable() {
+	mw.disabled = true
 }
 
 // Creates a new logger. Configuration should be set by changing `Formatter`,
@@ -51,162 +75,243 @@ func New() *Logger {
 	}
 }
 
-// Adds a field to the log entry, note that you it doesn't log until you call
+func (logger *Logger) newEntry() *Entry {
+	entry, ok := logger.entryPool.Get().(*Entry)
+	if ok {
+		return entry
+	}
+	return NewEntry(logger)
+}
+
+func (logger *Logger) releaseEntry(entry *Entry) {
+	logger.entryPool.Put(entry)
+}
+
+// Adds a field to the log entry, note that it doesn't log until you call
 // Debug, Print, Info, Warn, Fatal or Panic. It only creates a log entry.
 // If you want multiple fields, use `WithFields`.
 func (logger *Logger) WithField(key string, value interface{}) *Entry {
-	return NewEntry(logger).WithField(key, value)
+	entry := logger.newEntry()
+	defer logger.releaseEntry(entry)
+	return entry.WithField(key, value)
 }
 
 // Adds a struct of fields to the log entry. All it does is call `WithField` for
 // each `Field`.
 func (logger *Logger) WithFields(fields Fields) *Entry {
-	return NewEntry(logger).WithFields(fields)
+	entry := logger.newEntry()
+	defer logger.releaseEntry(entry)
+	return entry.WithFields(fields)
 }
 
 // Add an error as single field to the log entry.  All it does is call
 // `WithError` for the given `error`.
 func (logger *Logger) WithError(err error) *Entry {
-	return NewEntry(logger).WithError(err)
+	entry := logger.newEntry()
+	defer logger.releaseEntry(entry)
+	return entry.WithError(err)
 }
 
 func (logger *Logger) Debugf(format string, args ...interface{}) {
-	if logger.Level >= DebugLevel {
-		NewEntry(logger).Debugf(format, args...)
+	if logger.level() >= DebugLevel {
+		entry := logger.newEntry()
+		entry.Debugf(format, args...)
+		logger.releaseEntry(entry)
 	}
 }
 
 func (logger *Logger) Infof(format string, args ...interface{}) {
-	if logger.Level >= InfoLevel {
-		NewEntry(logger).Infof(format, args...)
+	if logger.level() >= InfoLevel {
+		entry := logger.newEntry()
+		entry.Infof(format, args...)
+		logger.releaseEntry(entry)
 	}
 }
 
 func (logger *Logger) Printf(format string, args ...interface{}) {
-	NewEntry(logger).Printf(format, args...)
+	entry := logger.newEntry()
+	entry.Printf(format, args...)
+	logger.releaseEntry(entry)
 }
 
 func (logger *Logger) Warnf(format string, args ...interface{}) {
-	if logger.Level >= WarnLevel {
-		NewEntry(logger).Warnf(format, args...)
+	if logger.level() >= WarnLevel {
+		entry := logger.newEntry()
+		entry.Warnf(format, args...)
+		logger.releaseEntry(entry)
 	}
 }
 
 func (logger *Logger) Warningf(format string, args ...interface{}) {
-	if logger.Level >= WarnLevel {
-		NewEntry(logger).Warnf(format, args...)
+	if logger.level() >= WarnLevel {
+		entry := logger.newEntry()
+		entry.Warnf(format, args...)
+		logger.releaseEntry(entry)
 	}
 }
 
 func (logger *Logger) Errorf(format string, args ...interface{}) {
-	if logger.Level >= ErrorLevel {
-		NewEntry(logger).Errorf(format, args...)
+	if logger.level() >= ErrorLevel {
+		entry := logger.newEntry()
+		entry.Errorf(format, args...)
+		logger.releaseEntry(entry)
 	}
 }
 
 func (logger *Logger) Fatalf(format string, args ...interface{}) {
-	if logger.Level >= FatalLevel {
-		NewEntry(logger).Fatalf(format, args...)
+	if logger.level() >= FatalLevel {
+		entry := logger.newEntry()
+		entry.Fatalf(format, args...)
+		logger.releaseEntry(entry)
 	}
-	os.Exit(1)
+	Exit(1)
 }
 
 func (logger *Logger) Panicf(format string, args ...interface{}) {
-	if logger.Level >= PanicLevel {
-		NewEntry(logger).Panicf(format, args...)
+	if logger.level() >= PanicLevel {
+		entry := logger.newEntry()
+		entry.Panicf(format, args...)
+		logger.releaseEntry(entry)
 	}
 }
 
 func (logger *Logger) Debug(args ...interface{}) {
-	if logger.Level >= DebugLevel {
-		NewEntry(logger).Debug(args...)
+	if logger.level() >= DebugLevel {
+		entry := logger.newEntry()
+		entry.Debug(args...)
+		logger.releaseEntry(entry)
 	}
 }
 
 func (logger *Logger) Info(args ...interface{}) {
-	if logger.Level >= InfoLevel {
-		NewEntry(logger).Info(args...)
+	if logger.level() >= InfoLevel {
+		entry := logger.newEntry()
+		entry.Info(args...)
+		logger.releaseEntry(entry)
 	}
 }
 
 func (logger *Logger) Print(args ...interface{}) {
-	NewEntry(logger).Info(args...)
+	entry := logger.newEntry()
+	entry.Info(args...)
+	logger.releaseEntry(entry)
 }
 
 func (logger *Logger) Warn(args ...interface{}) {
-	if logger.Level >= WarnLevel {
-		NewEntry(logger).Warn(args...)
+	if logger.level() >= WarnLevel {
+		entry := logger.newEntry()
+		entry.Warn(args...)
+		logger.releaseEntry(entry)
 	}
 }
 
 func (logger *Logger) Warning(args ...interface{}) {
-	if logger.Level >= WarnLevel {
-		NewEntry(logger).Warn(args...)
+	if logger.level() >= WarnLevel {
+		entry := logger.newEntry()
+		entry.Warn(args...)
+		logger.releaseEntry(entry)
 	}
 }
 
 func (logger *Logger) Error(args ...interface{}) {
-	if logger.Level >= ErrorLevel {
-		NewEntry(logger).Error(args...)
+	if logger.level() >= ErrorLevel {
+		entry := logger.newEntry()
+		entry.Error(args...)
+		logger.releaseEntry(entry)
 	}
 }
 
 func (logger *Logger) Fatal(args ...interface{}) {
-	if logger.Level >= FatalLevel {
-		NewEntry(logger).Fatal(args...)
+	if logger.level() >= FatalLevel {
+		entry := logger.newEntry()
+		entry.Fatal(args...)
+		logger.releaseEntry(entry)
 	}
-	os.Exit(1)
+	Exit(1)
 }
 
 func (logger *Logger) Panic(args ...interface{}) {
-	if logger.Level >= PanicLevel {
-		NewEntry(logger).Panic(args...)
+	if logger.level() >= PanicLevel {
+		entry := logger.newEntry()
+		entry.Panic(args...)
+		logger.releaseEntry(entry)
 	}
 }
 
 func (logger *Logger) Debugln(args ...interface{}) {
-	if logger.Level >= DebugLevel {
-		NewEntry(logger).Debugln(args...)
+	if logger.level() >= DebugLevel {
+		entry := logger.newEntry()
+		entry.Debugln(args...)
+		logger.releaseEntry(entry)
 	}
 }
 
 func (logger *Logger) Infoln(args ...interface{}) {
-	if logger.Level >= InfoLevel {
-		NewEntry(logger).Infoln(args...)
+	if logger.level() >= InfoLevel {
+		entry := logger.newEntry()
+		entry.Infoln(args...)
+		logger.releaseEntry(entry)
 	}
 }
 
 func (logger *Logger) Println(args ...interface{}) {
-	NewEntry(logger).Println(args...)
+	entry := logger.newEntry()
+	entry.Println(args...)
+	logger.releaseEntry(entry)
 }
 
 func (logger *Logger) Warnln(args ...interface{}) {
-	if logger.Level >= WarnLevel {
-		NewEntry(logger).Warnln(args...)
+	if logger.level() >= WarnLevel {
+		entry := logger.newEntry()
+		entry.Warnln(args...)
+		logger.releaseEntry(entry)
 	}
 }
 
 func (logger *Logger) Warningln(args ...interface{}) {
-	if logger.Level >= WarnLevel {
-		NewEntry(logger).Warnln(args...)
+	if logger.level() >= WarnLevel {
+		entry := logger.newEntry()
+		entry.Warnln(args...)
+		logger.releaseEntry(entry)
 	}
 }
 
 func (logger *Logger) Errorln(args ...interface{}) {
-	if logger.Level >= ErrorLevel {
-		NewEntry(logger).Errorln(args...)
+	if logger.level() >= ErrorLevel {
+		entry := logger.newEntry()
+		entry.Errorln(args...)
+		logger.releaseEntry(entry)
 	}
 }
 
 func (logger *Logger) Fatalln(args ...interface{}) {
-	if logger.Level >= FatalLevel {
-		NewEntry(logger).Fatalln(args...)
+	if logger.level() >= FatalLevel {
+		entry := logger.newEntry()
+		entry.Fatalln(args...)
+		logger.releaseEntry(entry)
 	}
-	os.Exit(1)
+	Exit(1)
 }
 
 func (logger *Logger) Panicln(args ...interface{}) {
-	if logger.Level >= PanicLevel {
-		NewEntry(logger).Panicln(args...)
+	if logger.level() >= PanicLevel {
+		entry := logger.newEntry()
+		entry.Panicln(args...)
+		logger.releaseEntry(entry)
 	}
 }
+
+//When file is opened with appending mode, it's safe to
+//write concurrently to a file (within 4k message on Linux).
+//In these cases user can choose to disable the lock.
+func (logger *Logger) SetNoLock() {
+	logger.mu.Disable()
+}
+
+func (logger *Logger) level() Level {
+	return Level(atomic.LoadUint32((*uint32)(&logger.Level)))
+}
+
+func (logger *Logger) setLevel(level Level) {
+	atomic.StoreUint32((*uint32)(&logger.Level), uint32(level))
+}

vendor/github.com/Sirupsen/logrus/logrus.go

@@ -10,7 +10,7 @@ import (
 type Fields map[string]interface{}
 
 // Level type
-type Level uint8
+type Level uint32
 
 // Convert the Level to a string. E.g. PanicLevel becomes "panic".
 func (level Level) String() string {

vendor/github.com/Sirupsen/logrus/terminal_appengine.go

@@ -0,0 +1,10 @@
+// +build appengine
+
+package logrus
+
+import "io"
+
+// IsTerminal returns true if stderr's file descriptor is a terminal.
+func IsTerminal(f io.Writer) bool {
+	return true
+}

vendor/github.com/Sirupsen/logrus/terminal_bsd.go

@@ -1,4 +1,5 @@
 // +build darwin freebsd openbsd netbsd dragonfly
+// +build !appengine
 
 package logrus
 

vendor/github.com/Sirupsen/logrus/terminal_linux.go

@@ -3,6 +3,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build !appengine
+
 package logrus
 
 import "syscall"

vendor/github.com/Sirupsen/logrus/terminal_notwindows.go

@@ -4,18 +4,25 @@
 // license that can be found in the LICENSE file.
 
 // +build linux darwin freebsd openbsd netbsd dragonfly
+// +build !appengine
 
 package logrus
 
 import (
+	"io"
+	"os"
 	"syscall"
 	"unsafe"
 )
 
 // IsTerminal returns true if stderr's file descriptor is a terminal.
-func IsTerminal() bool {
-	fd := syscall.Stderr
+func IsTerminal(f io.Writer) bool {
 	var termios Termios
-	_, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlReadTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0)
-	return err == 0
+	switch v := f.(type) {
+	case *os.File:
+		_, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(v.Fd()), ioctlReadTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0)
+		return err == 0
+	default:
+		return false
+	}
 }

vendor/github.com/Sirupsen/logrus/terminal_solaris.go

@@ -1,15 +1,21 @@
-// +build solaris
+// +build solaris,!appengine
 
 package logrus
 
 import (
+	"io"
 	"os"
 
 	"golang.org/x/sys/unix"
 )
 
 // IsTerminal returns true if the given file descriptor is a terminal.
-func IsTerminal() bool {
-	_, err := unix.IoctlGetTermios(int(os.Stdout.Fd()), unix.TCGETA)
-	return err == nil
+func IsTerminal(f io.Writer) bool {
+	switch v := f.(type) {
+	case *os.File:
+		_, err := unix.IoctlGetTermios(int(v.Fd()), unix.TCGETA)
+		return err == nil
+	default:
+		return false
+	}
 }

vendor/github.com/Sirupsen/logrus/terminal_windows.go

@@ -3,11 +3,18 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build windows
+// +build windows,!appengine
 
 package logrus
 
 import (
+	"bytes"
+	"errors"
+	"io"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
 	"syscall"
 	"unsafe"
 )
@@ -16,12 +23,60 @@ var kernel32 = syscall.NewLazyDLL("kernel32.dll")
 
 var (
 	procGetConsoleMode = kernel32.NewProc("GetConsoleMode")
+	procSetConsoleMode = kernel32.NewProc("SetConsoleMode")
 )
 
-// IsTerminal returns true if stderr's file descriptor is a terminal.
-func IsTerminal() bool {
-	fd := syscall.Stderr
-	var st uint32
-	r, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, uintptr(fd), uintptr(unsafe.Pointer(&st)), 0)
-	return r != 0 && e == 0
+const (
+	enableProcessedOutput           = 0x0001
+	enableWrapAtEolOutput           = 0x0002
+	enableVirtualTerminalProcessing = 0x0004
+)
+
+func getVersion() (float64, error) {
+	stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{}
+	cmd := exec.Command("cmd", "ver")
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+	err := cmd.Run()
+	if err != nil {
+		return -1, err
+	}
+	
+	// The output should be like "Microsoft Windows [Version XX.X.XXXXXX]"
+	version := strings.Replace(stdout.String(), "\n", "", -1)
+	version = strings.Replace(version, "\r\n", "", -1)
+
+	x1 := strings.Index(version, "[Version")
+
+	if x1 == -1 || strings.Index(version, "]") == -1 {
+		return -1, errors.New("Can't determine Windows version")
+	}
+
+	return strconv.ParseFloat(version[x1+9:x1+13], 64)
+}
+
+func init() {
+	ver, err := getVersion()
+	if err != nil {
+		return
+	}
+
+	// Activate Virtual Processing for Windows CMD
+	// Info: https://msdn.microsoft.com/en-us/library/windows/desktop/ms686033(v=vs.85).aspx
+	if ver >= 10 {
+		handle := syscall.Handle(os.Stderr.Fd())
+		procSetConsoleMode.Call(uintptr(handle), enableProcessedOutput|enableWrapAtEolOutput|enableVirtualTerminalProcessing)
+	}
+}
+
+// IsTerminal returns true if stderr's file descriptor is a terminal.
+func IsTerminal(f io.Writer) bool {
+	switch v := f.(type) {
+	case *os.File:
+		var st uint32
+		r, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, uintptr(v.Fd()), uintptr(unsafe.Pointer(&st)), 0)
+		return r != 0 && e == 0
+	default:
+		return false
+	}
 }

vendor/github.com/Sirupsen/logrus/text_formatter.go

@@ -3,9 +3,9 @@ package logrus
 import (
 	"bytes"
 	"fmt"
-	"runtime"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 )
 
@@ -20,16 +20,10 @@ const (
 
 var (
 	baseTimestamp time.Time
-	isTerminal    bool
 )
 
 func init() {
 	baseTimestamp = time.Now()
-	isTerminal = IsTerminal()
-}
-
-func miniTS() int {
-	return int(time.Since(baseTimestamp) / time.Second)
 }
 
 type TextFormatter struct {
@@ -54,10 +48,32 @@ type TextFormatter struct {
 	// that log extremely frequently and don't use the JSON formatter this may not
 	// be desired.
 	DisableSorting bool
+
+	// QuoteEmptyFields will wrap empty fields in quotes if true
+	QuoteEmptyFields bool
+
+	// QuoteCharacter can be set to the override the default quoting character "
+	// with something else. For example: ', or `.
+	QuoteCharacter string
+
+	// Whether the logger's out is to a terminal
+	isTerminal bool
+
+	sync.Once
+}
+
+func (f *TextFormatter) init(entry *Entry) {
+	if len(f.QuoteCharacter) == 0 {
+		f.QuoteCharacter = "\""
+	}
+	if entry.Logger != nil {
+		f.isTerminal = IsTerminal(entry.Logger.Out)
+	}
 }
 
 func (f *TextFormatter) Format(entry *Entry) ([]byte, error) {
-	var keys []string = make([]string, 0, len(entry.Data))
+	var b *bytes.Buffer
+	keys := make([]string, 0, len(entry.Data))
 	for k := range entry.Data {
 		keys = append(keys, k)
 	}
@@ -65,13 +81,17 @@ func (f *TextFormatter) Format(entry *Entry) ([]byte, error) {
 	if !f.DisableSorting {
 		sort.Strings(keys)
 	}
-
-	b := &bytes.Buffer{}
+	if entry.Buffer != nil {
+		b = entry.Buffer
+	} else {
+		b = &bytes.Buffer{}
+	}
 
 	prefixFieldClashes(entry.Data)
 
-	isColorTerminal := isTerminal && (runtime.GOOS != "windows")
-	isColored := (f.ForceColors || isColorTerminal) && !f.DisableColors
+	f.Do(func() { f.init(entry) })
+
+	isColored := (f.ForceColors || f.isTerminal) && !f.DisableColors
 
 	timestampFormat := f.TimestampFormat
 	if timestampFormat == "" {
@@ -111,51 +131,59 @@ func (f *TextFormatter) printColored(b *bytes.Buffer, entry *Entry, keys []strin
 
 	levelText := strings.ToUpper(entry.Level.String())[0:4]
 
-	if !f.FullTimestamp {
-		fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%04d] %-44s ", levelColor, levelText, miniTS(), entry.Message)
+	if f.DisableTimestamp {
+		fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m %-44s ", levelColor, levelText, entry.Message)
+	} else if !f.FullTimestamp {
+		fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%04d] %-44s ", levelColor, levelText, int(entry.Time.Sub(baseTimestamp)/time.Second), entry.Message)
 	} else {
 		fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%s] %-44s ", levelColor, levelText, entry.Time.Format(timestampFormat), entry.Message)
 	}
 	for _, k := range keys {
 		v := entry.Data[k]
-		fmt.Fprintf(b, " \x1b[%dm%s\x1b[0m=%+v", levelColor, k, v)
+		fmt.Fprintf(b, " \x1b[%dm%s\x1b[0m=", levelColor, k)
+		f.appendValue(b, v)
 	}
 }
 
-func needsQuoting(text string) bool {
+func (f *TextFormatter) needsQuoting(text string) bool {
+	if f.QuoteEmptyFields && len(text) == 0 {
+		return true
+	}
 	for _, ch := range text {
 		if !((ch >= 'a' && ch <= 'z') ||
 			(ch >= 'A' && ch <= 'Z') ||
 			(ch >= '0' && ch <= '9') ||
 			ch == '-' || ch == '.') {
-			return false
+			return true
 		}
 	}
-	return true
+	return false
 }
 
 func (f *TextFormatter) appendKeyValue(b *bytes.Buffer, key string, value interface{}) {
 
 	b.WriteString(key)
 	b.WriteByte('=')
+	f.appendValue(b, value)
+	b.WriteByte(' ')
+}
 
+func (f *TextFormatter) appendValue(b *bytes.Buffer, value interface{}) {
 	switch value := value.(type) {
 	case string:
-		if needsQuoting(value) {
+		if !f.needsQuoting(value) {
 			b.WriteString(value)
 		} else {
-			fmt.Fprintf(b, "%q", value)
+			fmt.Fprintf(b, "%s%v%s", f.QuoteCharacter, value, f.QuoteCharacter)
 		}
 	case error:
 		errmsg := value.Error()
-		if needsQuoting(errmsg) {
+		if !f.needsQuoting(errmsg) {
 			b.WriteString(errmsg)
 		} else {
-			fmt.Fprintf(b, "%q", value)
+			fmt.Fprintf(b, "%s%v%s", f.QuoteCharacter, errmsg, f.QuoteCharacter)
 		}
 	default:
 		fmt.Fprint(b, value)
 	}
-
-	b.WriteByte(' ')
 }

vendor/github.com/Sirupsen/logrus/writer.go

@@ -7,21 +7,52 @@ import (
 )
 
 func (logger *Logger) Writer() *io.PipeWriter {
+	return logger.WriterLevel(InfoLevel)
+}
+
+func (logger *Logger) WriterLevel(level Level) *io.PipeWriter {
+	return NewEntry(logger).WriterLevel(level)
+}
+
+func (entry *Entry) Writer() *io.PipeWriter {
+	return entry.WriterLevel(InfoLevel)
+}
+
+func (entry *Entry) WriterLevel(level Level) *io.PipeWriter {
 	reader, writer := io.Pipe()
 
-	go logger.writerScanner(reader)
+	var printFunc func(args ...interface{})
+
+	switch level {
+	case DebugLevel:
+		printFunc = entry.Debug
+	case InfoLevel:
+		printFunc = entry.Info
+	case WarnLevel:
+		printFunc = entry.Warn
+	case ErrorLevel:
+		printFunc = entry.Error
+	case FatalLevel:
+		printFunc = entry.Fatal
+	case PanicLevel:
+		printFunc = entry.Panic
+	default:
+		printFunc = entry.Print
+	}
+
+	go entry.writerScanner(reader, printFunc)
 	runtime.SetFinalizer(writer, writerFinalizer)
 
 	return writer
 }
 
-func (logger *Logger) writerScanner(reader *io.PipeReader) {
+func (entry *Entry) writerScanner(reader *io.PipeReader, printFunc func(args ...interface{})) {
 	scanner := bufio.NewScanner(reader)
 	for scanner.Scan() {
-		logger.Print(scanner.Text())
+		printFunc(scanner.Text())
 	}
 	if err := scanner.Err(); err != nil {
-		logger.Errorf("Error while reading from Writer: %s", err)
+		entry.Errorf("Error while reading from Writer: %s", err)
 	}
 	reader.Close()
 }

vendor/github.com/containers/image/README.md

@@ -62,9 +62,8 @@ or use the build tags described below to avoid the dependencies (e.g. using `go
 
 - `containers_image_openpgp`: Use a Golang-only OpenPGP implementation for signature verification instead of the default cgo/gpgme-based implementation;
 the primary downside is that creating new signatures with the Golang-only implementation is not supported.
-- `containers_image_ostree_stub`: Instead of importing `ostree:` transport in `github.com/containers/image/transports/alltransports`, use a stub which reports that the transport is not supported. This allows building the library without requiring the `libostree` development libraries.
-
-  (Note that explicitly importing `github.com/containers/image/ostree` will still depend on the `libostree` library, this build tag only affects generic users of …`/alltransports`.)
+- `containers_image_ostree_stub`: Instead of importing `ostree:` transport in `github.com/containers/image/transports/alltransports`, use a stub which reports that the transport is not supported. This allows building the library without requiring the `libostree` development libraries. The `github.com/containers/image/ostree` package is completely disabled
+and impossible to import when this build tag is in use.
 
 ## Contributing
 

vendor/github.com/containers/image/copy/copy.go

@@ -3,6 +3,7 @@ package copy
 import (
 	"bytes"
 	"compress/gzip"
+	"context"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -13,7 +14,6 @@ import (
 
 	pb "gopkg.in/cheggaaa/pb.v1"
 
-	"github.com/Sirupsen/logrus"
 	"github.com/containers/image/image"
 	"github.com/containers/image/pkg/compression"
 	"github.com/containers/image/signature"
@@ -21,6 +21,7 @@ import (
 	"github.com/containers/image/types"
 	"github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 type digestingReader struct {
@@ -128,9 +129,7 @@ func Image(policyContext *signature.PolicyContext, destRef, srcRef types.ImageRe
 		}
 	}()
 
-	destSupportedManifestMIMETypes := dest.SupportedManifestMIMETypes()
-
-	rawSource, err := srcRef.NewImageSource(options.SourceCtx, destSupportedManifestMIMETypes)
+	rawSource, err := srcRef.NewImageSource(options.SourceCtx)
 	if err != nil {
 		return errors.Wrapf(err, "Error initializing source %s", transports.ImageName(srcRef))
 	}
@@ -171,7 +170,7 @@ func Image(policyContext *signature.PolicyContext, destRef, srcRef types.ImageRe
 		sigs = [][]byte{}
 	} else {
 		writeReport("Getting image source signatures\n")
-		s, err := src.Signatures()
+		s, err := src.Signatures(context.TODO())
 		if err != nil {
 			return errors.Wrap(err, "Error reading signatures")
 		}
@@ -194,7 +193,7 @@ func Image(policyContext *signature.PolicyContext, destRef, srcRef types.ImageRe
 
 	// We compute preferredManifestMIMEType only to show it in error messages.
 	// Without having to add this context in an error message, we would be happy enough to know only that no conversion is needed.
-	preferredManifestMIMEType, otherManifestMIMETypeCandidates, err := determineManifestConversion(&manifestUpdates, src, destSupportedManifestMIMETypes, canModifyManifest)
+	preferredManifestMIMEType, otherManifestMIMETypeCandidates, err := determineManifestConversion(&manifestUpdates, src, dest.SupportedManifestMIMETypes(), canModifyManifest)
 	if err != nil {
 		return err
 	}
@@ -582,7 +581,7 @@ func (ic *imageCopier) copyBlobFromStream(srcStream io.Reader, srcInfo types.Blo
 	bar.ShowPercent = false
 	bar.Start()
 	destStream = bar.NewProxyReader(destStream)
-	defer fmt.Fprint(ic.reportWriter, "\n")
+	defer bar.Finish()
 
 	// === Send a copy of the original, uncompressed, stream, to a separate path if necessary.
 	var originalLayerReader io.Reader // DO NOT USE this other than to drain the input if no other consumer in the pipeline has done so.

vendor/github.com/containers/image/copy/manifest.go

@@ -3,10 +3,10 @@ package copy
 import (
 	"strings"
 
-	"github.com/Sirupsen/logrus"
 	"github.com/containers/image/manifest"
 	"github.com/containers/image/types"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 // preferredManifestMIMETypes lists manifest MIME types in order of our preference, if we can't use the original manifest and need to convert.

vendor/github.com/containers/image/directory/directory_src.go

@@ -1,6 +1,7 @@
 package directory
 
 import (
+	"context"
 	"io"
 	"io/ioutil"
 	"os"
@@ -59,7 +60,7 @@ func (s *dirImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int64, err
 	return r, fi.Size(), nil
 }
 
-func (s *dirImageSource) GetSignatures() ([][]byte, error) {
+func (s *dirImageSource) GetSignatures(ctx context.Context) ([][]byte, error) {
 	signatures := [][]byte{}
 	for i := 0; ; i++ {
 		signature, err := ioutil.ReadFile(s.ref.signaturePath(i))

vendor/github.com/containers/image/directory/directory_transport.go

@@ -143,11 +143,9 @@ func (ref dirReference) NewImage(ctx *types.SystemContext) (types.Image, error)
 	return image.FromSource(src)
 }
 
-// NewImageSource returns a types.ImageSource for this reference,
-// asking the backend to use a manifest from requestedManifestMIMETypes if possible.
-// nil requestedManifestMIMETypes means manifest.DefaultRequestedManifestMIMETypes.
+// NewImageSource returns a types.ImageSource for this reference.
 // The caller must call .Close() on the returned ImageSource.
-func (ref dirReference) NewImageSource(ctx *types.SystemContext, requestedManifestMIMETypes []string) (types.ImageSource, error) {
+func (ref dirReference) NewImageSource(ctx *types.SystemContext) (types.ImageSource, error) {
 	return newImageSource(ref), nil
 }
 

vendor/github.com/containers/image/docker/archive/dest.go

@@ -19,15 +19,24 @@ func newImageDestination(ctx *types.SystemContext, ref archiveReference) (types.
 	if ref.destinationRef == nil {
 		return nil, errors.Errorf("docker-archive: destination reference not supplied (must be of form <path>:<reference:tag>)")
 	}
-	fh, err := os.OpenFile(ref.path, os.O_WRONLY|os.O_EXCL|os.O_CREATE, 0644)
+
+	// ref.path can be either a pipe or a regular file
+	// in the case of a pipe, we require that we can open it for write
+	// in the case of a regular file, we don't want to overwrite any pre-existing file
+	// so we check for Size() == 0 below (This is racy, but using O_EXCL would also be racy,
+	// only in a different way. Either way, it’s up to the user to not have two writers to the same path.)
+	fh, err := os.OpenFile(ref.path, os.O_WRONLY|os.O_CREATE, 0644)
 	if err != nil {
-		// FIXME: It should be possible to modify archives, but the only really
-		//        sane way of doing it is to create a copy of the image, modify
-		//        it and then do a rename(2).
-		if os.IsExist(err) {
-			err = errors.New("docker-archive doesn't support modifying existing images")
-		}
-		return nil, err
+		return nil, errors.Wrapf(err, "error opening file %q", ref.path)
+	}
+
+	fhStat, err := fh.Stat()
+	if err != nil {
+		return nil, errors.Wrapf(err, "error statting file %q", ref.path)
+	}
+
+	if fhStat.Mode().IsRegular() && fhStat.Size() != 0 {
+		return nil, errors.New("docker-archive doesn't support modifying existing images")
 	}
 
 	return &archiveImageDestination{

vendor/github.com/containers/image/docker/archive/src.go

@@ -1,9 +1,9 @@
 package archive
 
 import (
-	"github.com/Sirupsen/logrus"
 	"github.com/containers/image/docker/tarfile"
 	"github.com/containers/image/types"
+	"github.com/sirupsen/logrus"
 )
 
 type archiveImageSource struct {

vendor/github.com/containers/image/docker/archive/transport.go

@@ -134,11 +134,9 @@ func (ref archiveReference) NewImage(ctx *types.SystemContext) (types.Image, err
 	return ctrImage.FromSource(src)
 }
 
-// NewImageSource returns a types.ImageSource for this reference,
-// asking the backend to use a manifest from requestedManifestMIMETypes if possible.
-// nil requestedManifestMIMETypes means manifest.DefaultRequestedManifestMIMETypes.
+// NewImageSource returns a types.ImageSource for this reference.
 // The caller must call .Close() on the returned ImageSource.
-func (ref archiveReference) NewImageSource(ctx *types.SystemContext, requestedManifestMIMETypes []string) (types.ImageSource, error) {
+func (ref archiveReference) NewImageSource(ctx *types.SystemContext) (types.ImageSource, error) {
 	return newImageSource(ctx, ref), nil
 }
 

vendor/github.com/containers/image/docker/daemon/daemon_dest.go

@@ -3,12 +3,12 @@ package daemon
 import (
 	"io"
 
-	"github.com/Sirupsen/logrus"
 	"github.com/containers/image/docker/reference"
 	"github.com/containers/image/docker/tarfile"
 	"github.com/containers/image/types"
 	"github.com/docker/docker/client"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
 )
 

vendor/github.com/containers/image/docker/daemon/daemon_transport.go

@@ -161,11 +161,9 @@ func (ref daemonReference) NewImage(ctx *types.SystemContext) (types.Image, erro
 	return image.FromSource(src)
 }
 
-// NewImageSource returns a types.ImageSource for this reference,
-// asking the backend to use a manifest from requestedManifestMIMETypes if possible.
-// nil requestedManifestMIMETypes means manifest.DefaultRequestedManifestMIMETypes.
+// NewImageSource returns a types.ImageSource for this reference.
 // The caller must call .Close() on the returned ImageSource.
-func (ref daemonReference) NewImageSource(ctx *types.SystemContext, requestedManifestMIMETypes []string) (types.ImageSource, error) {
+func (ref daemonReference) NewImageSource(ctx *types.SystemContext) (types.ImageSource, error) {
 	return newImageSource(ctx, ref)
 }
 

vendor/github.com/containers/image/docker/docker_client.go

@@ -1,38 +1,31 @@
 package docker
 
 import (
+	"context"
 	"crypto/tls"
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
-	"net"
 	"net/http"
-	"os"
 	"path/filepath"
 	"strings"
 	"time"
 
-	"github.com/Sirupsen/logrus"
 	"github.com/containers/image/docker/reference"
+	"github.com/containers/image/pkg/docker/config"
+	"github.com/containers/image/pkg/tlsclientconfig"
 	"github.com/containers/image/types"
-	"github.com/containers/storage/pkg/homedir"
 	"github.com/docker/distribution/registry/client"
-	"github.com/docker/go-connections/sockets"
 	"github.com/docker/go-connections/tlsconfig"
 	"github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 const (
-	dockerHostname     = "docker.io"
-	dockerRegistry     = "registry-1.docker.io"
-	dockerAuthRegistry = "https://index.docker.io/v1/"
-
-	dockerCfg         = ".docker"
-	dockerCfgFileName = "config.json"
-	dockerCfgObsolete = ".dockercfg"
+	dockerHostname = "docker.io"
+	dockerRegistry = "registry-1.docker.io"
 
 	systemPerHostCertDirPath = "/etc/docker/certs.d"
 
@@ -50,9 +43,13 @@ const (
 	extensionSignatureTypeAtomic    = "atomic" // extensionSignature.Type
 )
 
-// ErrV1NotSupported is returned when we're trying to talk to a
-// docker V1 registry.
-var ErrV1NotSupported = errors.New("can't talk to a V1 docker registry")
+var (
+	// ErrV1NotSupported is returned when we're trying to talk to a
+	// docker V1 registry.
+	ErrV1NotSupported = errors.New("can't talk to a V1 docker registry")
+	// ErrUnauthorizedForCredentials is returned when the status code returned is 401
+	ErrUnauthorizedForCredentials = errors.New("unable to retrieve auth token: invalid username/password")
+)
 
 // extensionSignature and extensionSignatureList come from github.com/openshift/origin/pkg/dockerregistry/server/signaturedispatcher.go:
 // signature represents a Docker image signature.
@@ -111,27 +108,7 @@ func serverDefault() *tls.Config {
 	}
 }
 
-func newTransport() *http.Transport {
-	direct := &net.Dialer{
-		Timeout:   30 * time.Second,
-		KeepAlive: 30 * time.Second,
-		DualStack: true,
-	}
-	tr := &http.Transport{
-		Proxy:               http.ProxyFromEnvironment,
-		Dial:                direct.Dial,
-		TLSHandshakeTimeout: 10 * time.Second,
-		// TODO(dmcgowan): Call close idle connections when complete and use keep alive
-		DisableKeepAlives: true,
-	}
-	proxyDialer, err := sockets.DialerFromEnvironment(direct)
-	if err == nil {
-		tr.Dial = proxyDialer.Dial
-	}
-	return tr
-}
-
-// dockerCertDir returns a path to a directory to be consumed by setupCertificates() depending on ctx and hostPort.
+// dockerCertDir returns a path to a directory to be consumed by tlsclientconfig.SetupCertificates() depending on ctx and hostPort.
 func dockerCertDir(ctx *types.SystemContext, hostPort string) string {
 	if ctx != nil && ctx.DockerCertPath != "" {
 		return ctx.DockerCertPath
@@ -147,131 +124,105 @@ func dockerCertDir(ctx *types.SystemContext, hostPort string) string {
 	return filepath.Join(hostCertDir, hostPort)
 }
 
-func setupCertificates(dir string, tlsc *tls.Config) error {
-	logrus.Debugf("Looking for TLS certificates and private keys in %s", dir)
-	fs, err := ioutil.ReadDir(dir)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return err
-	}
-
-	for _, f := range fs {
-		fullPath := filepath.Join(dir, f.Name())
-		if strings.HasSuffix(f.Name(), ".crt") {
-			systemPool, err := tlsconfig.SystemCertPool()
-			if err != nil {
-				return errors.Wrap(err, "unable to get system cert pool")
-			}
-			tlsc.RootCAs = systemPool
-			logrus.Debugf(" crt: %s", fullPath)
-			data, err := ioutil.ReadFile(fullPath)
-			if err != nil {
-				return err
-			}
-			tlsc.RootCAs.AppendCertsFromPEM(data)
-		}
-		if strings.HasSuffix(f.Name(), ".cert") {
-			certName := f.Name()
-			keyName := certName[:len(certName)-5] + ".key"
-			logrus.Debugf(" cert: %s", fullPath)
-			if !hasFile(fs, keyName) {
-				return errors.Errorf("missing key %s for client certificate %s. Note that CA certificates should use the extension .crt", keyName, certName)
-			}
-			cert, err := tls.LoadX509KeyPair(filepath.Join(dir, certName), filepath.Join(dir, keyName))
-			if err != nil {
-				return err
-			}
-			tlsc.Certificates = append(tlsc.Certificates, cert)
-		}
-		if strings.HasSuffix(f.Name(), ".key") {
-			keyName := f.Name()
-			certName := keyName[:len(keyName)-4] + ".cert"
-			logrus.Debugf(" key: %s", fullPath)
-			if !hasFile(fs, certName) {
-				return errors.Errorf("missing client certificate %s for key %s", certName, keyName)
-			}
-		}
-	}
-	return nil
-}
-
-func hasFile(files []os.FileInfo, name string) bool {
-	for _, f := range files {
-		if f.Name() == name {
-			return true
-		}
-	}
-	return false
-}
-
-// newDockerClient returns a new dockerClient instance for refHostname (a host a specified in the Docker image reference, not canonicalized to dockerRegistry)
+// newDockerClientFromRef returns a new dockerClient instance for refHostname (a host a specified in the Docker image reference, not canonicalized to dockerRegistry)
 // “write” specifies whether the client will be used for "write" access (in particular passed to lookaside.go:toplevelFromSection)
-func newDockerClient(ctx *types.SystemContext, ref dockerReference, write bool, actions string) (*dockerClient, error) {
+func newDockerClientFromRef(ctx *types.SystemContext, ref dockerReference, write bool, actions string) (*dockerClient, error) {
 	registry := reference.Domain(ref.ref)
-	if registry == dockerHostname {
-		registry = dockerRegistry
+	username, password, err := config.GetAuthentication(ctx, reference.Domain(ref.ref))
+	if err != nil {
+		return nil, errors.Wrapf(err, "error getting username and password")
 	}
-	username, password, err := getAuth(ctx, reference.Domain(ref.ref))
+	sigBase, err := configuredSignatureStorageBase(ctx, ref, write)
 	if err != nil {
 		return nil, err
 	}
-	tr := newTransport()
+	remoteName := reference.Path(ref.ref)
+
+	return newDockerClientWithDetails(ctx, registry, username, password, actions, sigBase, remoteName)
+}
+
+// newDockerClientWithDetails returns a new dockerClient instance for the given parameters
+func newDockerClientWithDetails(ctx *types.SystemContext, registry, username, password, actions string, sigBase signatureStorageBase, remoteName string) (*dockerClient, error) {
+	hostName := registry
+	if registry == dockerHostname {
+		registry = dockerRegistry
+	}
+	tr := tlsclientconfig.NewTransport()
 	tr.TLSClientConfig = serverDefault()
+
 	// It is undefined whether the host[:port] string for dockerHostname should be dockerHostname or dockerRegistry,
 	// because docker/docker does not read the certs.d subdirectory at all in that case.  We use the user-visible
 	// dockerHostname here, because it is more symmetrical to read the configuration in that case as well, and because
 	// generally the UI hides the existence of the different dockerRegistry.  But note that this behavior is
 	// undocumented and may change if docker/docker changes.
-	certDir := dockerCertDir(ctx, reference.Domain(ref.ref))
-	if err := setupCertificates(certDir, tr.TLSClientConfig); err != nil {
+	certDir := dockerCertDir(ctx, hostName)
+	if err := tlsclientconfig.SetupCertificates(certDir, tr.TLSClientConfig); err != nil {
 		return nil, err
 	}
+
 	if ctx != nil && ctx.DockerInsecureSkipTLSVerify {
 		tr.TLSClientConfig.InsecureSkipVerify = true
 	}
-	client := &http.Client{Transport: tr}
-
-	sigBase, err := configuredSignatureStorageBase(ctx, ref, write)
-	if err != nil {
-		return nil, err
-	}
 
 	return &dockerClient{
 		ctx:           ctx,
 		registry:      registry,
 		username:      username,
 		password:      password,
-		client:        client,
+		client:        &http.Client{Transport: tr},
 		signatureBase: sigBase,
 		scope: authScope{
 			actions:    actions,
-			remoteName: reference.Path(ref.ref),
+			remoteName: remoteName,
 		},
 	}, nil
 }
 
+// CheckAuth validates the credentials by attempting to log into the registry
+// returns an error if an error occcured while making the http request or the status code received was 401
+func CheckAuth(ctx context.Context, sCtx *types.SystemContext, username, password, registry string) error {
+	newLoginClient, err := newDockerClientWithDetails(sCtx, registry, username, password, "", nil, "")
+	if err != nil {
+		return errors.Wrapf(err, "error creating new docker client")
+	}
+
+	resp, err := newLoginClient.makeRequest(ctx, "GET", "/v2/", nil, nil)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	switch resp.StatusCode {
+	case http.StatusOK:
+		return nil
+	case http.StatusUnauthorized:
+		return ErrUnauthorizedForCredentials
+	default:
+		return errors.Errorf("error occured with status code %q", resp.StatusCode)
+	}
+}
+
 // makeRequest creates and executes a http.Request with the specified parameters, adding authentication and TLS options for the Docker client.
 // The host name and schema is taken from the client or autodetected, and the path is relative to it, i.e. the path usually starts with /v2/.
-func (c *dockerClient) makeRequest(method, path string, headers map[string][]string, stream io.Reader) (*http.Response, error) {
-	if err := c.detectProperties(); err != nil {
+func (c *dockerClient) makeRequest(ctx context.Context, method, path string, headers map[string][]string, stream io.Reader) (*http.Response, error) {
+	if err := c.detectProperties(ctx); err != nil {
 		return nil, err
 	}
 
 	url := fmt.Sprintf("%s://%s%s", c.scheme, c.registry, path)
-	return c.makeRequestToResolvedURL(method, url, headers, stream, -1, true)
+	return c.makeRequestToResolvedURL(ctx, method, url, headers, stream, -1, true)
 }
 
 // makeRequestToResolvedURL creates and executes a http.Request with the specified parameters, adding authentication and TLS options for the Docker client.
 // streamLen, if not -1, specifies the length of the data expected on stream.
 // makeRequest should generally be preferred.
 // TODO(runcom): too many arguments here, use a struct
-func (c *dockerClient) makeRequestToResolvedURL(method, url string, headers map[string][]string, stream io.Reader, streamLen int64, sendAuth bool) (*http.Response, error) {
+func (c *dockerClient) makeRequestToResolvedURL(ctx context.Context, method, url string, headers map[string][]string, stream io.Reader, streamLen int64, sendAuth bool) (*http.Response, error) {
 	req, err := http.NewRequest(method, url, stream)
 	if err != nil {
 		return nil, err
 	}
+	req = req.WithContext(ctx)
 	if streamLen != -1 { // Do not blindly overwrite if streamLen == -1, http.NewRequest above can figure out the length of bytes.Reader and similar objects without us having to compute it.
 		req.ContentLength = streamLen
 	}
@@ -322,8 +273,11 @@ func (c *dockerClient) setupRequestAuth(req *http.Request) error {
 					return errors.Errorf("missing realm in bearer auth challenge")
 				}
 				service, _ := challenge.Parameters["service"] // Will be "" if not present
-				scope := fmt.Sprintf("repository:%s:%s", c.scope.remoteName, c.scope.actions)
-				token, err := c.getBearerToken(realm, service, scope)
+				var scope string
+				if c.scope.remoteName != "" && c.scope.actions != "" {
+					scope = fmt.Sprintf("repository:%s:%s", c.scope.remoteName, c.scope.actions)
+				}
+				token, err := c.getBearerToken(req.Context(), realm, service, scope)
 				if err != nil {
 					return err
 				}
@@ -340,11 +294,12 @@ func (c *dockerClient) setupRequestAuth(req *http.Request) error {
 	return nil
 }
 
-func (c *dockerClient) getBearerToken(realm, service, scope string) (*bearerToken, error) {
+func (c *dockerClient) getBearerToken(ctx context.Context, realm, service, scope string) (*bearerToken, error) {
 	authReq, err := http.NewRequest("GET", realm, nil)
 	if err != nil {
 		return nil, err
 	}
+	authReq = authReq.WithContext(ctx)
 	getParams := authReq.URL.Query()
 	if service != "" {
 		getParams.Add("service", service)
@@ -356,7 +311,7 @@ func (c *dockerClient) getBearerToken(realm, service, scope string) (*bearerToke
 	if c.username != "" && c.password != "" {
 		authReq.SetBasicAuth(c.username, c.password)
 	}
-	tr := newTransport()
+	tr := tlsclientconfig.NewTransport()
 	// TODO(runcom): insecure for now to contact the external token service
 	tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
 	client := &http.Client{Transport: tr}
@@ -367,7 +322,7 @@ func (c *dockerClient) getBearerToken(realm, service, scope string) (*bearerToke
 	defer res.Body.Close()
 	switch res.StatusCode {
 	case http.StatusUnauthorized:
-		return nil, errors.Errorf("unable to retrieve auth token: 401 unauthorized")
+		return nil, ErrUnauthorizedForCredentials
 	case http.StatusOK:
 		break
 	default:
@@ -391,70 +346,16 @@ func (c *dockerClient) getBearerToken(realm, service, scope string) (*bearerToke
 	return &token, nil
 }
 
-func getAuth(ctx *types.SystemContext, registry string) (string, string, error) {
-	if ctx != nil && ctx.DockerAuthConfig != nil {
-		return ctx.DockerAuthConfig.Username, ctx.DockerAuthConfig.Password, nil
-	}
-	var dockerAuth dockerConfigFile
-	dockerCfgPath := filepath.Join(getDefaultConfigDir(".docker"), dockerCfgFileName)
-	if _, err := os.Stat(dockerCfgPath); err == nil {
-		j, err := ioutil.ReadFile(dockerCfgPath)
-		if err != nil {
-			return "", "", err
-		}
-		if err := json.Unmarshal(j, &dockerAuth); err != nil {
-			return "", "", err
-		}
-
-	} else if os.IsNotExist(err) {
-		// try old config path
-		oldDockerCfgPath := filepath.Join(getDefaultConfigDir(dockerCfgObsolete))
-		if _, err := os.Stat(oldDockerCfgPath); err != nil {
-			if os.IsNotExist(err) {
-				return "", "", nil
-			}
-			return "", "", errors.Wrap(err, oldDockerCfgPath)
-		}
-
-		j, err := ioutil.ReadFile(oldDockerCfgPath)
-		if err != nil {
-			return "", "", err
-		}
-		if err := json.Unmarshal(j, &dockerAuth.AuthConfigs); err != nil {
-			return "", "", err
-		}
-
-	} else if err != nil {
-		return "", "", errors.Wrap(err, dockerCfgPath)
-	}
-
-	// I'm feeling lucky
-	if c, exists := dockerAuth.AuthConfigs[registry]; exists {
-		return decodeDockerAuth(c.Auth)
-	}
-
-	// bad luck; let's normalize the entries first
-	registry = normalizeRegistry(registry)
-	normalizedAuths := map[string]dockerAuthConfig{}
-	for k, v := range dockerAuth.AuthConfigs {
-		normalizedAuths[normalizeRegistry(k)] = v
-	}
-	if c, exists := normalizedAuths[registry]; exists {
-		return decodeDockerAuth(c.Auth)
-	}
-	return "", "", nil
-}
-
 // detectProperties detects various properties of the registry.
 // See the dockerClient documentation for members which are affected by this.
-func (c *dockerClient) detectProperties() error {
+func (c *dockerClient) detectProperties(ctx context.Context) error {
 	if c.scheme != "" {
 		return nil
 	}
 
 	ping := func(scheme string) error {
 		url := fmt.Sprintf(resolvedPingV2URL, scheme, c.registry)
-		resp, err := c.makeRequestToResolvedURL("GET", url, nil, nil, -1, true)
+		resp, err := c.makeRequestToResolvedURL(ctx, "GET", url, nil, nil, -1, true)
 		logrus.Debugf("Ping %s err %#v", url, err)
 		if err != nil {
 			return err
@@ -481,7 +382,7 @@ func (c *dockerClient) detectProperties() error {
 		// best effort to understand if we're talking to a V1 registry
 		pingV1 := func(scheme string) bool {
 			url := fmt.Sprintf(resolvedPingV1URL, scheme, c.registry)
-			resp, err := c.makeRequestToResolvedURL("GET", url, nil, nil, -1, true)
+			resp, err := c.makeRequestToResolvedURL(ctx, "GET", url, nil, nil, -1, true)
 			logrus.Debugf("Ping %s err %#v", url, err)
 			if err != nil {
 				return false
@@ -506,9 +407,9 @@ func (c *dockerClient) detectProperties() error {
 
 // getExtensionsSignatures returns signatures from the X-Registry-Supports-Signatures API extension,
 // using the original data structures.
-func (c *dockerClient) getExtensionsSignatures(ref dockerReference, manifestDigest digest.Digest) (*extensionSignatureList, error) {
+func (c *dockerClient) getExtensionsSignatures(ctx context.Context, ref dockerReference, manifestDigest digest.Digest) (*extensionSignatureList, error) {
 	path := fmt.Sprintf(extensionsSignaturePath, reference.Path(ref.ref), manifestDigest)
-	res, err := c.makeRequest("GET", path, nil, nil)
+	res, err := c.makeRequest(ctx, "GET", path, nil, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -527,55 +428,3 @@ func (c *dockerClient) getExtensionsSignatures(ref dockerReference, manifestDige
 	}
 	return &parsedBody, nil
 }
-
-func getDefaultConfigDir(confPath string) string {
-	return filepath.Join(homedir.Get(), confPath)
-}
-
-type dockerAuthConfig struct {
-	Auth string `json:"auth,omitempty"`
-}
-
-type dockerConfigFile struct {
-	AuthConfigs map[string]dockerAuthConfig `json:"auths"`
-}
-
-func decodeDockerAuth(s string) (string, string, error) {
-	decoded, err := base64.StdEncoding.DecodeString(s)
-	if err != nil {
-		return "", "", err
-	}
-	parts := strings.SplitN(string(decoded), ":", 2)
-	if len(parts) != 2 {
-		// if it's invalid just skip, as docker does
-		return "", "", nil
-	}
-	user := parts[0]
-	password := strings.Trim(parts[1], "\x00")
-	return user, password, nil
-}
-
-// convertToHostname converts a registry url which has http|https prepended
-// to just an hostname.
-// Copied from github.com/docker/docker/registry/auth.go
-func convertToHostname(url string) string {
-	stripped := url
-	if strings.HasPrefix(url, "http://") {
-		stripped = strings.TrimPrefix(url, "http://")
-	} else if strings.HasPrefix(url, "https://") {
-		stripped = strings.TrimPrefix(url, "https://")
-	}
-
-	nameParts := strings.SplitN(stripped, "/", 2)
-
-	return nameParts[0]
-}
-
-func normalizeRegistry(registry string) string {
-	normalized := convertToHostname(registry)
-	switch normalized {
-	case "registry-1.docker.io", "docker.io":
-		return "index.docker.io"
-	}
-	return normalized
-}

vendor/github.com/containers/image/docker/docker_image.go

@@ -1,6 +1,7 @@
 package docker
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -22,7 +23,7 @@ type Image struct {
 // a client to the registry hosting the given image.
 // The caller must call .Close() on the returned Image.
 func newImage(ctx *types.SystemContext, ref dockerReference) (types.Image, error) {
-	s, err := newImageSource(ctx, ref, nil)
+	s, err := newImageSource(ctx, ref)
 	if err != nil {
 		return nil, err
 	}
@@ -41,7 +42,8 @@ func (i *Image) SourceRefFullName() string {
 // GetRepositoryTags list all tags available in the repository. Note that this has no connection with the tag(s) used for this specific image, if any.
 func (i *Image) GetRepositoryTags() ([]string, error) {
 	path := fmt.Sprintf(tagsPath, reference.Path(i.src.ref.ref))
-	res, err := i.src.c.makeRequest("GET", path, nil, nil)
+	// FIXME: Pass the context.Context
+	res, err := i.src.c.makeRequest(context.TODO(), "GET", path, nil, nil)
 	if err != nil {
 		return nil, err
 	}

vendor/github.com/containers/image/docker/docker_image_dest.go

@@ -2,6 +2,7 @@ package docker
 
 import (
 	"bytes"
+	"context"
 	"crypto/rand"
 	"encoding/json"
 	"fmt"
@@ -12,7 +13,6 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/Sirupsen/logrus"
 	"github.com/containers/image/docker/reference"
 	"github.com/containers/image/manifest"
 	"github.com/containers/image/types"
@@ -20,24 +20,11 @@ import (
 	"github.com/docker/distribution/registry/api/v2"
 	"github.com/docker/distribution/registry/client"
 	"github.com/opencontainers/go-digest"
+	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
-var manifestMIMETypes = []string{
-	// TODO(runcom): we'll add OCI as part of another PR here
-	manifest.DockerV2Schema2MediaType,
-	manifest.DockerV2Schema1SignedMediaType,
-	manifest.DockerV2Schema1MediaType,
-}
-
-func supportedManifestMIMETypesMap() map[string]bool {
-	m := make(map[string]bool, len(manifestMIMETypes))
-	for _, mt := range manifestMIMETypes {
-		m[mt] = true
-	}
-	return m
-}
-
 type dockerImageDestination struct {
 	ref dockerReference
 	c   *dockerClient
@@ -47,7 +34,7 @@ type dockerImageDestination struct {
 
 // newImageDestination creates a new ImageDestination for the specified image reference.
 func newImageDestination(ctx *types.SystemContext, ref dockerReference) (types.ImageDestination, error) {
-	c, err := newDockerClient(ctx, ref, true, "pull,push")
+	c, err := newDockerClientFromRef(ctx, ref, true, "pull,push")
 	if err != nil {
 		return nil, err
 	}
@@ -69,13 +56,18 @@ func (d *dockerImageDestination) Close() error {
 }
 
 func (d *dockerImageDestination) SupportedManifestMIMETypes() []string {
-	return manifestMIMETypes
+	return []string{
+		imgspecv1.MediaTypeImageManifest,
+		manifest.DockerV2Schema2MediaType,
+		manifest.DockerV2Schema1SignedMediaType,
+		manifest.DockerV2Schema1MediaType,
+	}
 }
 
 // SupportsSignatures returns an error (to be displayed to the user) if the destination certainly can't store signatures.
 // Note: It is still possible for PutSignatures to fail if SupportsSignatures returns nil.
 func (d *dockerImageDestination) SupportsSignatures() error {
-	if err := d.c.detectProperties(); err != nil {
+	if err := d.c.detectProperties(context.TODO()); err != nil {
 		return err
 	}
 	switch {
@@ -132,7 +124,7 @@ func (d *dockerImageDestination) PutBlob(stream io.Reader, inputInfo types.BlobI
 	// FIXME? Chunked upload, progress reporting, etc.
 	uploadPath := fmt.Sprintf(blobUploadPath, reference.Path(d.ref.ref))
 	logrus.Debugf("Uploading %s", uploadPath)
-	res, err := d.c.makeRequest("POST", uploadPath, nil, nil)
+	res, err := d.c.makeRequest(context.TODO(), "POST", uploadPath, nil, nil)
 	if err != nil {
 		return types.BlobInfo{}, err
 	}
@@ -149,7 +141,7 @@ func (d *dockerImageDestination) PutBlob(stream io.Reader, inputInfo types.BlobI
 	digester := digest.Canonical.Digester()
 	sizeCounter := &sizeCounter{}
 	tee := io.TeeReader(stream, io.MultiWriter(digester.Hash(), sizeCounter))
-	res, err = d.c.makeRequestToResolvedURL("PATCH", uploadLocation.String(), map[string][]string{"Content-Type": {"application/octet-stream"}}, tee, inputInfo.Size, true)
+	res, err = d.c.makeRequestToResolvedURL(context.TODO(), "PATCH", uploadLocation.String(), map[string][]string{"Content-Type": {"application/octet-stream"}}, tee, inputInfo.Size, true)
 	if err != nil {
 		logrus.Debugf("Error uploading layer chunked, response %#v", res)
 		return types.BlobInfo{}, err
@@ -168,7 +160,7 @@ func (d *dockerImageDestination) PutBlob(stream io.Reader, inputInfo types.BlobI
 	// TODO: check inputInfo.Digest == computedDigest https://github.com/containers/image/pull/70#discussion_r77646717
 	locationQuery.Set("digest", computedDigest.String())
 	uploadLocation.RawQuery = locationQuery.Encode()
-	res, err = d.c.makeRequestToResolvedURL("PUT", uploadLocation.String(), map[string][]string{"Content-Type": {"application/octet-stream"}}, nil, -1, true)
+	res, err = d.c.makeRequestToResolvedURL(context.TODO(), "PUT", uploadLocation.String(), map[string][]string{"Content-Type": {"application/octet-stream"}}, nil, -1, true)
 	if err != nil {
 		return types.BlobInfo{}, err
 	}
@@ -193,7 +185,7 @@ func (d *dockerImageDestination) HasBlob(info types.BlobInfo) (bool, int64, erro
 	checkPath := fmt.Sprintf(blobsPath, reference.Path(d.ref.ref), info.Digest.String())
 
 	logrus.Debugf("Checking %s", checkPath)
-	res, err := d.c.makeRequest("HEAD", checkPath, nil, nil)
+	res, err := d.c.makeRequest(context.TODO(), "HEAD", checkPath, nil, nil)
 	if err != nil {
 		return false, -1, err
 	}
@@ -239,7 +231,7 @@ func (d *dockerImageDestination) PutManifest(m []byte) error {
 	if mimeType != "" {
 		headers["Content-Type"] = []string{mimeType}
 	}
-	res, err := d.c.makeRequest("PUT", path, headers, bytes.NewReader(m))
+	res, err := d.c.makeRequest(context.TODO(), "PUT", path, headers, bytes.NewReader(m))
 	if err != nil {
 		return err
 	}
@@ -275,7 +267,7 @@ func (d *dockerImageDestination) PutSignatures(signatures [][]byte) error {
 	if len(signatures) == 0 {
 		return nil
 	}
-	if err := d.c.detectProperties(); err != nil {
+	if err := d.c.detectProperties(context.TODO()); err != nil {
 		return err
 	}
 	switch {
@@ -396,7 +388,7 @@ func (d *dockerImageDestination) putSignaturesToAPIExtension(signatures [][]byte
 	// always adds signatures.  Eventually we should also allow removing signatures,
 	// but the X-Registry-Supports-Signatures API extension does not support that yet.
 
-	existingSignatures, err := d.c.getExtensionsSignatures(d.ref, d.manifestDigest)
+	existingSignatures, err := d.c.getExtensionsSignatures(context.TODO(), d.ref, d.manifestDigest)
 	if err != nil {
 		return err
 	}
@@ -438,7 +430,7 @@ sigExists:
 		}
 
 		path := fmt.Sprintf(extensionsSignaturePath, reference.Path(d.ref.ref), d.manifestDigest.String())
-		res, err := d.c.makeRequest("PUT", path, nil, bytes.NewReader(body))
+		res, err := d.c.makeRequest(context.TODO(), "PUT", path, nil, bytes.NewReader(body))
 		if err != nil {
 			return err
 		}

vendor/github.com/containers/image/docker/docker_image_src.go

@@ -1,6 +1,7 @@
 package docker
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -10,51 +11,33 @@ import (
 	"os"
 	"strconv"
 
-	"github.com/Sirupsen/logrus"
 	"github.com/containers/image/docker/reference"
 	"github.com/containers/image/manifest"
 	"github.com/containers/image/types"
 	"github.com/docker/distribution/registry/client"
 	"github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 type dockerImageSource struct {
-	ref                        dockerReference
-	requestedManifestMIMETypes []string
-	c                          *dockerClient
+	ref dockerReference
+	c   *dockerClient
 	// State
 	cachedManifest         []byte // nil if not loaded yet
 	cachedManifestMIMEType string // Only valid if cachedManifest != nil
 }
 
-// newImageSource creates a new ImageSource for the specified image reference,
-// asking the backend to use a manifest from requestedManifestMIMETypes if possible.
-// nil requestedManifestMIMETypes means manifest.DefaultRequestedManifestMIMETypes.
+// newImageSource creates a new ImageSource for the specified image reference.
 // The caller must call .Close() on the returned ImageSource.
-func newImageSource(ctx *types.SystemContext, ref dockerReference, requestedManifestMIMETypes []string) (*dockerImageSource, error) {
-	c, err := newDockerClient(ctx, ref, false, "pull")
+func newImageSource(ctx *types.SystemContext, ref dockerReference) (*dockerImageSource, error) {
+	c, err := newDockerClientFromRef(ctx, ref, false, "pull")
 	if err != nil {
 		return nil, err
 	}
-	if requestedManifestMIMETypes == nil {
-		requestedManifestMIMETypes = manifest.DefaultRequestedManifestMIMETypes
-	}
-	supportedMIMEs := supportedManifestMIMETypesMap()
-	acceptableRequestedMIMEs := false
-	for _, mtrequested := range requestedManifestMIMETypes {
-		if supportedMIMEs[mtrequested] {
-			acceptableRequestedMIMEs = true
-			break
-		}
-	}
-	if !acceptableRequestedMIMEs {
-		requestedManifestMIMETypes = manifest.DefaultRequestedManifestMIMETypes
-	}
 	return &dockerImageSource{
 		ref: ref,
-		requestedManifestMIMETypes: requestedManifestMIMETypes,
-		c: c,
+		c:   c,
 	}, nil
 }
 
@@ -85,18 +68,18 @@ func simplifyContentType(contentType string) string {
 // GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available).
 // It may use a remote (= slow) service.
 func (s *dockerImageSource) GetManifest() ([]byte, string, error) {
-	err := s.ensureManifestIsLoaded()
+	err := s.ensureManifestIsLoaded(context.TODO())
 	if err != nil {
 		return nil, "", err
 	}
 	return s.cachedManifest, s.cachedManifestMIMEType, nil
 }
 
-func (s *dockerImageSource) fetchManifest(tagOrDigest string) ([]byte, string, error) {
+func (s *dockerImageSource) fetchManifest(ctx context.Context, tagOrDigest string) ([]byte, string, error) {
 	path := fmt.Sprintf(manifestPath, reference.Path(s.ref.ref), tagOrDigest)
 	headers := make(map[string][]string)
-	headers["Accept"] = s.requestedManifestMIMETypes
-	res, err := s.c.makeRequest("GET", path, headers, nil)
+	headers["Accept"] = manifest.DefaultRequestedManifestMIMETypes
+	res, err := s.c.makeRequest(ctx, "GET", path, headers, nil)
 	if err != nil {
 		return nil, "", err
 	}
@@ -114,7 +97,7 @@ func (s *dockerImageSource) fetchManifest(tagOrDigest string) ([]byte, string, e
 // GetTargetManifest returns an image's manifest given a digest.
 // This is mainly used to retrieve a single image's manifest out of a manifest list.
 func (s *dockerImageSource) GetTargetManifest(digest digest.Digest) ([]byte, string, error) {
-	return s.fetchManifest(digest.String())
+	return s.fetchManifest(context.TODO(), digest.String())
 }
 
 // ensureManifestIsLoaded sets s.cachedManifest and s.cachedManifestMIMEType
@@ -124,7 +107,7 @@ func (s *dockerImageSource) GetTargetManifest(digest digest.Digest) ([]byte, str
 // we need to ensure that the digest of the manifest returned by GetManifest
 // and used by GetSignatures are consistent, otherwise we would get spurious
 // signature verification failures when pulling while a tag is being updated.
-func (s *dockerImageSource) ensureManifestIsLoaded() error {
+func (s *dockerImageSource) ensureManifestIsLoaded(ctx context.Context) error {
 	if s.cachedManifest != nil {
 		return nil
 	}
@@ -134,7 +117,7 @@ func (s *dockerImageSource) ensureManifestIsLoaded() error {
 		return err
 	}
 
-	manblob, mt, err := s.fetchManifest(reference)
+	manblob, mt, err := s.fetchManifest(ctx, reference)
 	if err != nil {
 		return err
 	}
@@ -150,13 +133,14 @@ func (s *dockerImageSource) getExternalBlob(urls []string) (io.ReadCloser, int64
 		err  error
 	)
 	for _, url := range urls {
-		resp, err = s.c.makeRequestToResolvedURL("GET", url, nil, nil, -1, false)
+		resp, err = s.c.makeRequestToResolvedURL(context.TODO(), "GET", url, nil, nil, -1, false)
 		if err == nil {
 			if resp.StatusCode != http.StatusOK {
 				err = errors.Errorf("error fetching external blob from %q: %d", url, resp.StatusCode)
 				logrus.Debug(err)
 				continue
 			}
+			break
 		}
 	}
 	if resp.Body != nil && err == nil {
@@ -181,7 +165,7 @@ func (s *dockerImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int64,
 
 	path := fmt.Sprintf(blobsPath, reference.Path(s.ref.ref), info.Digest.String())
 	logrus.Debugf("Downloading %s", path)
-	res, err := s.c.makeRequest("GET", path, nil, nil)
+	res, err := s.c.makeRequest(context.TODO(), "GET", path, nil, nil)
 	if err != nil {
 		return nil, 0, err
 	}
@@ -192,27 +176,38 @@ func (s *dockerImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int64,
 	return res.Body, getBlobSize(res), nil
 }
 
-func (s *dockerImageSource) GetSignatures() ([][]byte, error) {
-	if err := s.c.detectProperties(); err != nil {
+func (s *dockerImageSource) GetSignatures(ctx context.Context) ([][]byte, error) {
+	if err := s.c.detectProperties(ctx); err != nil {
 		return nil, err
 	}
 	switch {
 	case s.c.signatureBase != nil:
-		return s.getSignaturesFromLookaside()
+		return s.getSignaturesFromLookaside(ctx)
 	case s.c.supportsSignatures:
-		return s.getSignaturesFromAPIExtension()
+		return s.getSignaturesFromAPIExtension(ctx)
 	default:
 		return [][]byte{}, nil
 	}
 }
 
+// manifestDigest returns a digest of the manifest, either from the supplied reference or from a fetched manifest.
+func (s *dockerImageSource) manifestDigest(ctx context.Context) (digest.Digest, error) {
+	if digested, ok := s.ref.ref.(reference.Digested); ok {
+		d := digested.Digest()
+		if d.Algorithm() == digest.Canonical {
+			return d, nil
+		}
+	}
+	if err := s.ensureManifestIsLoaded(ctx); err != nil {
+		return "", err
+	}
+	return manifest.Digest(s.cachedManifest)
+}
+
 // getSignaturesFromLookaside implements GetSignatures() from the lookaside location configured in s.c.signatureBase,
 // which is not nil.
-func (s *dockerImageSource) getSignaturesFromLookaside() ([][]byte, error) {
-	if err := s.ensureManifestIsLoaded(); err != nil {
-		return nil, err
-	}
-	manifestDigest, err := manifest.Digest(s.cachedManifest)
+func (s *dockerImageSource) getSignaturesFromLookaside(ctx context.Context) ([][]byte, error) {
+	manifestDigest, err := s.manifestDigest(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -224,7 +219,7 @@ func (s *dockerImageSource) getSignaturesFromLookaside() ([][]byte, error) {
 		if url == nil {
 			return nil, errors.Errorf("Internal error: signatureStorageURL with non-nil base returned nil")
 		}
-		signature, missing, err := s.getOneSignature(url)
+		signature, missing, err := s.getOneSignature(ctx, url)
 		if err != nil {
 			return nil, err
 		}
@@ -239,7 +234,7 @@ func (s *dockerImageSource) getSignaturesFromLookaside() ([][]byte, error) {
 // getOneSignature downloads one signature from url.
 // If it successfully determines that the signature does not exist, returns with missing set to true and error set to nil.
 // NOTE: Keep this in sync with docs/signature-protocols.md!
-func (s *dockerImageSource) getOneSignature(url *url.URL) (signature []byte, missing bool, err error) {
+func (s *dockerImageSource) getOneSignature(ctx context.Context, url *url.URL) (signature []byte, missing bool, err error) {
 	switch url.Scheme {
 	case "file":
 		logrus.Debugf("Reading %s", url.Path)
@@ -254,7 +249,12 @@ func (s *dockerImageSource) getOneSignature(url *url.URL) (signature []byte, mis
 
 	case "http", "https":
 		logrus.Debugf("GET %s", url)
-		res, err := s.c.client.Get(url.String())
+		req, err := http.NewRequest("GET", url.String(), nil)
+		if err != nil {
+			return nil, false, err
+		}
+		req = req.WithContext(ctx)
+		res, err := s.c.client.Do(req)
 		if err != nil {
 			return nil, false, err
 		}
@@ -276,16 +276,13 @@ func (s *dockerImageSource) getOneSignature(url *url.URL) (signature []byte, mis
 }
 
 // getSignaturesFromAPIExtension implements GetSignatures() using the X-Registry-Supports-Signatures API extension.
-func (s *dockerImageSource) getSignaturesFromAPIExtension() ([][]byte, error) {
-	if err := s.ensureManifestIsLoaded(); err != nil {
-		return nil, err
-	}
-	manifestDigest, err := manifest.Digest(s.cachedManifest)
+func (s *dockerImageSource) getSignaturesFromAPIExtension(ctx context.Context) ([][]byte, error) {
+	manifestDigest, err := s.manifestDigest(ctx)
 	if err != nil {
 		return nil, err
 	}
 
-	parsedBody, err := s.c.getExtensionsSignatures(s.ref, manifestDigest)
+	parsedBody, err := s.c.getExtensionsSignatures(ctx, s.ref, manifestDigest)
 	if err != nil {
 		return nil, err
 	}
@@ -301,7 +298,7 @@ func (s *dockerImageSource) getSignaturesFromAPIExtension() ([][]byte, error) {
 
 // deleteImage deletes the named image from the registry, if supported.
 func deleteImage(ctx *types.SystemContext, ref dockerReference) error {
-	c, err := newDockerClient(ctx, ref, true, "push")
+	c, err := newDockerClientFromRef(ctx, ref, true, "push")
 	if err != nil {
 		return err
 	}
@@ -316,7 +313,7 @@ func deleteImage(ctx *types.SystemContext, ref dockerReference) error {
 		return err
 	}
 	getPath := fmt.Sprintf(manifestPath, reference.Path(ref.ref), refTail)
-	get, err := c.makeRequest("GET", getPath, headers, nil)
+	get, err := c.makeRequest(context.TODO(), "GET", getPath, headers, nil)
 	if err != nil {
 		return err
 	}
@@ -338,7 +335,7 @@ func deleteImage(ctx *types.SystemContext, ref dockerReference) error {
 
 	// When retrieving the digest from a registry >= 2.3 use the following header:
 	//   "Accept": "application/vnd.docker.distribution.manifest.v2+json"
-	delete, err := c.makeRequest("DELETE", deletePath, headers, nil)
+	delete, err := c.makeRequest(context.TODO(), "DELETE", deletePath, headers, nil)
 	if err != nil {
 		return err
 	}

vendor/github.com/containers/image/docker/docker_transport.go

@@ -130,12 +130,10 @@ func (ref dockerReference) NewImage(ctx *types.SystemContext) (types.Image, erro
 	return newImage(ctx, ref)
 }
 
-// NewImageSource returns a types.ImageSource for this reference,
-// asking the backend to use a manifest from requestedManifestMIMETypes if possible.
-// nil requestedManifestMIMETypes means manifest.DefaultRequestedManifestMIMETypes.
+// NewImageSource returns a types.ImageSource for this reference.
 // The caller must call .Close() on the returned ImageSource.
-func (ref dockerReference) NewImageSource(ctx *types.SystemContext, requestedManifestMIMETypes []string) (types.ImageSource, error) {
-	return newImageSource(ctx, ref, requestedManifestMIMETypes)
+func (ref dockerReference) NewImageSource(ctx *types.SystemContext) (types.ImageSource, error) {
+	return newImageSource(ctx, ref)
 }
 
 // NewImageDestination returns a types.ImageDestination for this reference.

vendor/github.com/containers/image/docker/lookaside.go

@@ -9,12 +9,12 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/Sirupsen/logrus"
 	"github.com/containers/image/docker/reference"
 	"github.com/containers/image/types"
 	"github.com/ghodss/yaml"
 	"github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 // systemRegistriesDirPath is the path to registries.d, used for locating lookaside Docker signature storage.

vendor/github.com/containers/image/docker/tarfile/dest.go

@@ -10,12 +10,12 @@ import (
 	"os"
 	"time"
 
-	"github.com/Sirupsen/logrus"
 	"github.com/containers/image/docker/reference"
 	"github.com/containers/image/manifest"
 	"github.com/containers/image/types"
 	"github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 const temporaryDirectoryForBigFiles = "/var/tmp" // Do not use the system default of os.TempDir(), usually /tmp, because with systemd it could be a tmpfs.

vendor/github.com/containers/image/docker/tarfile/src.go

@@ -3,6 +3,7 @@ package tarfile
 import (
 	"archive/tar"
 	"bytes"
+	"context"
 	"encoding/json"
 	"io"
 	"io/ioutil"
@@ -354,6 +355,6 @@ func (s *Source) GetBlob(info types.BlobInfo) (io.ReadCloser, int64, error) {
 }
 
 // GetSignatures returns the image's signatures.  It may use a remote (= slow) service.
-func (s *Source) GetSignatures() ([][]byte, error) {
+func (s *Source) GetSignatures(ctx context.Context) ([][]byte, error) {
 	return [][]byte{}, nil
 }

vendor/github.com/containers/image/image/docker_schema1.go

@@ -161,14 +161,17 @@ func (m *manifestSchema1) imageInspectInfo() (*types.ImageInspectInfo, error) {
 	if err := json.Unmarshal([]byte(m.History[0].V1Compatibility), v1); err != nil {
 		return nil, err
 	}
-	return &types.ImageInspectInfo{
+	i := &types.ImageInspectInfo{
 		Tag:           m.Tag,
 		DockerVersion: v1.DockerVersion,
 		Created:       v1.Created,
-		Labels:        v1.Config.Labels,
 		Architecture:  v1.Architecture,
 		Os:            v1.OS,
-	}, nil
+	}
+	if v1.Config != nil {
+		i.Labels = v1.Config.Labels
+	}
+	return i, nil
 }
 
 // UpdatedImageNeedsLayerDiffIDs returns true iff UpdatedImage(options) needs InformationOnly.LayerDiffIDs.

vendor/github.com/containers/image/image/docker_schema2.go

@@ -8,13 +8,13 @@ import (
 	"io/ioutil"
 	"strings"
 
-	"github.com/Sirupsen/logrus"
 	"github.com/containers/image/docker/reference"
 	"github.com/containers/image/manifest"
 	"github.com/containers/image/types"
 	"github.com/opencontainers/go-digest"
 	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 // gzippedEmptyLayer is a gzip-compressed version of an empty tar file (1024 NULL bytes)
@@ -157,13 +157,16 @@ func (m *manifestSchema2) imageInspectInfo() (*types.ImageInspectInfo, error) {
 	if err := json.Unmarshal(config, v1); err != nil {
 		return nil, err
 	}
-	return &types.ImageInspectInfo{
+	i := &types.ImageInspectInfo{
 		DockerVersion: v1.DockerVersion,
 		Created:       v1.Created,
-		Labels:        v1.Config.Labels,
 		Architecture:  v1.Architecture,
 		Os:            v1.OS,
-	}, nil
+	}
+	if v1.Config != nil {
+		i.Labels = v1.Config.Labels
+	}
+	return i, nil
 }
 
 // UpdatedImageNeedsLayerDiffIDs returns true iff UpdatedImage(options) needs InformationOnly.LayerDiffIDs.

vendor/github.com/containers/image/image/memory.go

@@ -1,6 +1,8 @@
 package image
 
 import (
+	"context"
+
 	"github.com/pkg/errors"
 
 	"github.com/containers/image/types"
@@ -54,7 +56,7 @@ func (i *memoryImage) Manifest() ([]byte, string, error) {
 }
 
 // Signatures is like ImageSource.GetSignatures, but the result is cached; it is OK to call this however often you need.
-func (i *memoryImage) Signatures() ([][]byte, error) {
+func (i *memoryImage) Signatures(ctx context.Context) ([][]byte, error) {
 	// Modifying an image invalidates signatures; a caller asking the updated image for signatures
 	// is probably confused.
 	return nil, errors.New("Internal error: Image.Signatures() is not supported for images modified in memory")

vendor/github.com/containers/image/image/oci.go

@@ -56,7 +56,7 @@ func (m *manifestOCI1) manifestMIMEType() string {
 // ConfigInfo returns a complete BlobInfo for the separate config object, or a BlobInfo{Digest:""} if there isn't a separate object.
 // Note that the config object may not exist in the underlying storage in the return value of UpdatedImage! Use ConfigBlob() below.
 func (m *manifestOCI1) ConfigInfo() types.BlobInfo {
-	return types.BlobInfo{Digest: m.ConfigDescriptor.Digest, Size: m.ConfigDescriptor.Size}
+	return types.BlobInfo{Digest: m.ConfigDescriptor.Digest, Size: m.ConfigDescriptor.Size, Annotations: m.ConfigDescriptor.Annotations}
 }
 
 // ConfigBlob returns the blob described by ConfigInfo, iff ConfigInfo().Digest != ""; nil otherwise.
@@ -109,7 +109,7 @@ func (m *manifestOCI1) OCIConfig() (*imgspecv1.Image, error) {
 func (m *manifestOCI1) LayerInfos() []types.BlobInfo {
 	blobs := []types.BlobInfo{}
 	for _, layer := range m.LayersDescriptors {
-		blobs = append(blobs, types.BlobInfo{Digest: layer.Digest, Size: layer.Size})
+		blobs = append(blobs, types.BlobInfo{Digest: layer.Digest, Size: layer.Size, Annotations: layer.Annotations, URLs: layer.URLs})
 	}
 	return blobs
 }
@@ -130,13 +130,16 @@ func (m *manifestOCI1) imageInspectInfo() (*types.ImageInspectInfo, error) {
 	if err := json.Unmarshal(config, v1); err != nil {
 		return nil, err
 	}
-	return &types.ImageInspectInfo{
+	i := &types.ImageInspectInfo{
 		DockerVersion: v1.DockerVersion,
 		Created:       v1.Created,
-		Labels:        v1.Config.Labels,
 		Architecture:  v1.Architecture,
 		Os:            v1.OS,
-	}, nil
+	}
+	if v1.Config != nil {
+		i.Labels = v1.Config.Labels
+	}
+	return i, nil
 }
 
 // UpdatedImageNeedsLayerDiffIDs returns true iff UpdatedImage(options) needs InformationOnly.LayerDiffIDs.
@@ -159,6 +162,8 @@ func (m *manifestOCI1) UpdatedImage(options types.ManifestUpdateOptions) (types.
 			copy.LayersDescriptors[i].MediaType = m.LayersDescriptors[i].MediaType
 			copy.LayersDescriptors[i].Digest = info.Digest
 			copy.LayersDescriptors[i].Size = info.Size
+			copy.LayersDescriptors[i].Annotations = info.Annotations
+			copy.LayersDescriptors[i].URLs = info.URLs
 		}
 	}
 	// Ignore options.EmbeddedDockerReference: it may be set when converting from schema1, but we really don't care.

vendor/github.com/containers/image/image/unparsed.go

@@ -1,6 +1,8 @@
 package image
 
 import (
+	"context"
+
 	"github.com/containers/image/docker/reference"
 	"github.com/containers/image/manifest"
 	"github.com/containers/image/types"
@@ -71,9 +73,9 @@ func (i *UnparsedImage) Manifest() ([]byte, string, error) {
 }
 
 // Signatures is like ImageSource.GetSignatures, but the result is cached; it is OK to call this however often you need.
-func (i *UnparsedImage) Signatures() ([][]byte, error) {
+func (i *UnparsedImage) Signatures(ctx context.Context) ([][]byte, error) {
 	if i.cachedSignatures == nil {
-		sigs, err := i.src.GetSignatures()
+		sigs, err := i.src.GetSignatures(ctx)
 		if err != nil {
 			return nil, err
 		}

vendor/github.com/containers/image/manifest/manifest.go

@@ -35,7 +35,7 @@ var DefaultRequestedManifestMIMETypes = []string{
 	DockerV2Schema2MediaType,
 	DockerV2Schema1SignedMediaType,
 	DockerV2Schema1MediaType,
-	DockerV2ListMediaType,
+	// DockerV2ListMediaType, // FIXME: Restore this ASAP
 }
 
 // GuessMIMEType guesses MIME type of a manifest and returns it _if it is recognized_, or "" if unknown or unrecognized.

vendor/github.com/containers/image/oci/archive/oci_dest.go

@@ -0,0 +1,131 @@
+package archive
+
+import (
+	"io"
+	"os"
+
+	"github.com/containers/image/types"
+	"github.com/containers/storage/pkg/archive"
+	"github.com/pkg/errors"
+)
+
+type ociArchiveImageDestination struct {
+	ref          ociArchiveReference
+	unpackedDest types.ImageDestination
+	tempDirRef   tempDirOCIRef
+}
+
+// newImageDestination returns an ImageDestination for writing to an existing directory.
+func newImageDestination(ctx *types.SystemContext, ref ociArchiveReference) (types.ImageDestination, error) {
+	tempDirRef, err := createOCIRef(ref.image)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error creating oci reference")
+	}
+	unpackedDest, err := tempDirRef.ociRefExtracted.NewImageDestination(ctx)
+	if err != nil {
+		if err := tempDirRef.deleteTempDir(); err != nil {
+			return nil, errors.Wrapf(err, "error deleting temp directory", tempDirRef.tempDirectory)
+		}
+		return nil, err
+	}
+	return &ociArchiveImageDestination{ref: ref,
+		unpackedDest: unpackedDest,
+		tempDirRef:   tempDirRef}, nil
+}
+
+// Reference returns the reference used to set up this destination.
+func (d *ociArchiveImageDestination) Reference() types.ImageReference {
+	return d.ref
+}
+
+// Close removes resources associated with an initialized ImageDestination, if any
+// Close deletes the temp directory of the oci-archive image
+func (d *ociArchiveImageDestination) Close() error {
+	defer d.tempDirRef.deleteTempDir()
+	return d.unpackedDest.Close()
+}
+
+func (d *ociArchiveImageDestination) SupportedManifestMIMETypes() []string {
+	return d.unpackedDest.SupportedManifestMIMETypes()
+}
+
+// SupportsSignatures returns an error (to be displayed to the user) if the destination certainly can't store signatures
+func (d *ociArchiveImageDestination) SupportsSignatures() error {
+	return d.unpackedDest.SupportsSignatures()
+}
+
+// ShouldCompressLayers returns true iff it is desirable to compress layer blobs written to this destination
+func (d *ociArchiveImageDestination) ShouldCompressLayers() bool {
+	return d.unpackedDest.ShouldCompressLayers()
+}
+
+// AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually
+// uploaded to the image destination, true otherwise.
+func (d *ociArchiveImageDestination) AcceptsForeignLayerURLs() bool {
+	return d.unpackedDest.AcceptsForeignLayerURLs()
+}
+
+// MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime OS. False otherwise
+func (d *ociArchiveImageDestination) MustMatchRuntimeOS() bool {
+	return d.unpackedDest.MustMatchRuntimeOS()
+}
+
+// PutBlob writes contents of stream and returns data representing the result (with all data filled in).
+// inputInfo.Digest can be optionally provided if known; it is not mandatory for the implementation to verify it.
+// inputInfo.Size is the expected length of stream, if known.
+func (d *ociArchiveImageDestination) PutBlob(stream io.Reader, inputInfo types.BlobInfo) (types.BlobInfo, error) {
+	return d.unpackedDest.PutBlob(stream, inputInfo)
+}
+
+// HasBlob returns true iff the image destination already contains a blob with the matching digest which can be reapplied using ReapplyBlob
+func (d *ociArchiveImageDestination) HasBlob(info types.BlobInfo) (bool, int64, error) {
+	return d.unpackedDest.HasBlob(info)
+}
+
+func (d *ociArchiveImageDestination) ReapplyBlob(info types.BlobInfo) (types.BlobInfo, error) {
+	return d.unpackedDest.ReapplyBlob(info)
+}
+
+// PutManifest writes manifest to the destination
+func (d *ociArchiveImageDestination) PutManifest(m []byte) error {
+	return d.unpackedDest.PutManifest(m)
+}
+
+func (d *ociArchiveImageDestination) PutSignatures(signatures [][]byte) error {
+	return d.unpackedDest.PutSignatures(signatures)
+}
+
+// Commit marks the process of storing the image as successful and asks for the image to be persisted
+// after the directory is made, it is tarred up into a file and the directory is deleted
+func (d *ociArchiveImageDestination) Commit() error {
+	if err := d.unpackedDest.Commit(); err != nil {
+		return errors.Wrapf(err, "error storing image %q", d.ref.image)
+	}
+
+	// path of directory to tar up
+	src := d.tempDirRef.tempDirectory
+	// path to save tarred up file
+	dst := d.ref.resolvedFile
+	return tarDirectory(src, dst)
+}
+
+// tar converts the directory at src and saves it to dst
+func tarDirectory(src, dst string) error {
+	// input is a stream of bytes from the archive of the directory at path
+	input, err := archive.Tar(src, archive.Uncompressed)
+	if err != nil {
+		return errors.Wrapf(err, "error retrieving stream of bytes from %q", src)
+	}
+
+	// creates the tar file
+	outFile, err := os.Create(dst)
+	if err != nil {
+		return errors.Wrapf(err, "error creating tar file %q", dst)
+	}
+	defer outFile.Close()
+
+	// copies the contents of the directory to the tar file
+	_, err = io.Copy(outFile, input)
+
+	return err
+}

vendor/github.com/containers/image/oci/archive/oci_src.go

@@ -0,0 +1,88 @@
+package archive
+
+import (
+	"context"
+	"io"
+
+	ocilayout "github.com/containers/image/oci/layout"
+	"github.com/containers/image/types"
+	digest "github.com/opencontainers/go-digest"
+	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+type ociArchiveImageSource struct {
+	ref         ociArchiveReference
+	unpackedSrc types.ImageSource
+	tempDirRef  tempDirOCIRef
+}
+
+// newImageSource returns an ImageSource for reading from an existing directory.
+// newImageSource untars the file and saves it in a temp directory
+func newImageSource(ctx *types.SystemContext, ref ociArchiveReference) (types.ImageSource, error) {
+	tempDirRef, err := createUntarTempDir(ref)
+	if err != nil {
+		return nil, errors.Wrap(err, "error creating temp directory")
+	}
+
+	unpackedSrc, err := tempDirRef.ociRefExtracted.NewImageSource(ctx)
+	if err != nil {
+		if err := tempDirRef.deleteTempDir(); err != nil {
+			return nil, errors.Wrapf(err, "error deleting temp directory", tempDirRef.tempDirectory)
+		}
+		return nil, err
+	}
+	return &ociArchiveImageSource{ref: ref,
+		unpackedSrc: unpackedSrc,
+		tempDirRef:  tempDirRef}, nil
+}
+
+// LoadManifestDescriptor loads the manifest
+func LoadManifestDescriptor(imgRef types.ImageReference) (imgspecv1.Descriptor, error) {
+	ociArchRef, ok := imgRef.(ociArchiveReference)
+	if !ok {
+		return imgspecv1.Descriptor{}, errors.Errorf("error typecasting, need type ociArchiveReference")
+	}
+	tempDirRef, err := createUntarTempDir(ociArchRef)
+	if err != nil {
+		return imgspecv1.Descriptor{}, errors.Wrap(err, "error creating temp directory")
+	}
+	defer tempDirRef.deleteTempDir()
+
+	descriptor, err := ocilayout.LoadManifestDescriptor(tempDirRef.ociRefExtracted)
+	if err != nil {
+		return imgspecv1.Descriptor{}, errors.Wrap(err, "error loading index")
+	}
+	return descriptor, nil
+}
+
+// Reference returns the reference used to set up this source.
+func (s *ociArchiveImageSource) Reference() types.ImageReference {
+	return s.ref
+}
+
+// Close removes resources associated with an initialized ImageSource, if any.
+// Close deletes the temporary directory at dst
+func (s *ociArchiveImageSource) Close() error {
+	defer s.tempDirRef.deleteTempDir()
+	return s.unpackedSrc.Close()
+}
+
+// GetManifest returns the image's manifest along with its MIME type
+// (which may be empty when it can't be determined but the manifest is available).
+func (s *ociArchiveImageSource) GetManifest() ([]byte, string, error) {
+	return s.unpackedSrc.GetManifest()
+}
+
+func (s *ociArchiveImageSource) GetTargetManifest(digest digest.Digest) ([]byte, string, error) {
+	return s.unpackedSrc.GetTargetManifest(digest)
+}
+
+// GetBlob returns a stream for the specified blob, and the blob's size.
+func (s *ociArchiveImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int64, error) {
+	return s.unpackedSrc.GetBlob(info)
+}
+
+func (s *ociArchiveImageSource) GetSignatures(c context.Context) ([][]byte, error) {
+	return s.unpackedSrc.GetSignatures(c)
+}

vendor/github.com/containers/image/oci/archive/oci_transport.go

@@ -0,0 +1,225 @@
+package archive
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/containers/image/directory/explicitfilepath"
+	"github.com/containers/image/docker/reference"
+	"github.com/containers/image/image"
+	ocilayout "github.com/containers/image/oci/layout"
+	"github.com/containers/image/transports"
+	"github.com/containers/image/types"
+	"github.com/containers/storage/pkg/archive"
+	"github.com/pkg/errors"
+)
+
+func init() {
+	transports.Register(Transport)
+}
+
+// Transport is an ImageTransport for OCI archive
+// it creates an oci-archive tar file by calling into the OCI transport
+// tarring the directory created by oci and deleting the directory
+var Transport = ociArchiveTransport{}
+
+type ociArchiveTransport struct{}
+
+// ociArchiveReference is an ImageReference for OCI Archive paths
+type ociArchiveReference struct {
+	file         string
+	resolvedFile string
+	image        string
+}
+
+func (t ociArchiveTransport) Name() string {
+	return "oci-archive"
+}
+
+// ParseReference converts a string, which should not start with the ImageTransport.Name prefix
+// into an ImageReference.
+func (t ociArchiveTransport) ParseReference(reference string) (types.ImageReference, error) {
+	return ParseReference(reference)
+}
+
+// ValidatePolicyConfigurationScope checks that scope is a valid name for a signature.PolicyTransportScopes keys
+func (t ociArchiveTransport) ValidatePolicyConfigurationScope(scope string) error {
+	var file string
+	sep := strings.SplitN(scope, ":", 2)
+	file = sep[0]
+
+	if len(sep) == 2 {
+		image := sep[1]
+		if !refRegexp.MatchString(image) {
+			return errors.Errorf("Invalid image %s", image)
+		}
+	}
+
+	if !strings.HasPrefix(file, "/") {
+		return errors.Errorf("Invalid scope %s: must be an absolute path", scope)
+	}
+	// Refuse also "/", otherwise "/" and "" would have the same semantics,
+	// and "" could be unexpectedly shadowed by the "/" entry.
+	// (Note: we do allow "/:someimage", a bit ridiculous but why refuse it?)
+	if scope == "/" {
+		return errors.New(`Invalid scope "/": Use the generic default scope ""`)
+	}
+	cleaned := filepath.Clean(file)
+	if cleaned != file {
+		return errors.Errorf(`Invalid scope %s: Uses non-canonical path format, perhaps try with path %s`, scope, cleaned)
+	}
+	return nil
+}
+
+// annotation spex from https://github.com/opencontainers/image-spec/blob/master/annotations.md#pre-defined-annotation-keys
+const (
+	separator = `(?:[-._:@+]|--)`
+	alphanum  = `(?:[A-Za-z0-9]+)`
+	component = `(?:` + alphanum + `(?:` + separator + alphanum + `)*)`
+)
+
+var refRegexp = regexp.MustCompile(`^` + component + `(?:/` + component + `)*$`)
+
+// ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an OCI ImageReference.
+func ParseReference(reference string) (types.ImageReference, error) {
+	var file, image string
+	sep := strings.SplitN(reference, ":", 2)
+	file = sep[0]
+
+	if len(sep) == 2 {
+		image = sep[1]
+	}
+	return NewReference(file, image)
+}
+
+// NewReference returns an OCI reference for a file and a image.
+func NewReference(file, image string) (types.ImageReference, error) {
+	resolved, err := explicitfilepath.ResolvePathToFullyExplicit(file)
+	if err != nil {
+		return nil, err
+	}
+	// This is necessary to prevent directory paths returned by PolicyConfigurationNamespaces
+	// from being ambiguous with values of PolicyConfigurationIdentity.
+	if strings.Contains(resolved, ":") {
+		return nil, errors.Errorf("Invalid OCI reference %s:%s: path %s contains a colon", file, image, resolved)
+	}
+	if len(image) > 0 && !refRegexp.MatchString(image) {
+		return nil, errors.Errorf("Invalid image %s", image)
+	}
+	return ociArchiveReference{file: file, resolvedFile: resolved, image: image}, nil
+}
+
+func (ref ociArchiveReference) Transport() types.ImageTransport {
+	return Transport
+}
+
+// StringWithinTransport returns a string representation of the reference, which MUST be such that
+// reference.Transport().ParseReference(reference.StringWithinTransport()) returns an equivalent reference.
+func (ref ociArchiveReference) StringWithinTransport() string {
+	return fmt.Sprintf("%s:%s", ref.file, ref.image)
+}
+
+// DockerReference returns a Docker reference associated with this reference
+func (ref ociArchiveReference) DockerReference() reference.Named {
+	return nil
+}
+
+// PolicyConfigurationIdentity returns a string representation of the reference, suitable for policy lookup.
+func (ref ociArchiveReference) PolicyConfigurationIdentity() string {
+	// NOTE: ref.image is not a part of the image identity, because "$dir:$someimage" and "$dir:" may mean the
+	// same image and the two can’t be statically disambiguated.  Using at least the repository directory is
+	// less granular but hopefully still useful.
+	return fmt.Sprintf("%s", ref.resolvedFile)
+}
+
+// PolicyConfigurationNamespaces returns a list of other policy configuration namespaces to search
+// for if explicit configuration for PolicyConfigurationIdentity() is not set
+func (ref ociArchiveReference) PolicyConfigurationNamespaces() []string {
+	res := []string{}
+	path := ref.resolvedFile
+	for {
+		lastSlash := strings.LastIndex(path, "/")
+		// Note that we do not include "/"; it is redundant with the default "" global default,
+		// and rejected by ociTransport.ValidatePolicyConfigurationScope above.
+		if lastSlash == -1 || path == "/" {
+			break
+		}
+		res = append(res, path)
+		path = path[:lastSlash]
+	}
+	return res
+}
+
+// NewImage returns a types.Image for this reference, possibly specialized for this ImageTransport.
+// The caller must call .Close() on the returned Image.
+func (ref ociArchiveReference) NewImage(ctx *types.SystemContext) (types.Image, error) {
+	src, err := newImageSource(ctx, ref)
+	if err != nil {
+		return nil, err
+	}
+	return image.FromSource(src)
+}
+
+// NewImageSource returns a types.ImageSource for this reference.
+// The caller must call .Close() on the returned ImageSource.
+func (ref ociArchiveReference) NewImageSource(ctx *types.SystemContext) (types.ImageSource, error) {
+	return newImageSource(ctx, ref)
+}
+
+// NewImageDestination returns a types.ImageDestination for this reference.
+// The caller must call .Close() on the returned ImageDestination.
+func (ref ociArchiveReference) NewImageDestination(ctx *types.SystemContext) (types.ImageDestination, error) {
+	return newImageDestination(ctx, ref)
+}
+
+// DeleteImage deletes the named image from the registry, if supported.
+func (ref ociArchiveReference) DeleteImage(ctx *types.SystemContext) error {
+	return errors.Errorf("Deleting images not implemented for oci: images")
+}
+
+// struct to store the ociReference and temporary directory returned by createOCIRef
+type tempDirOCIRef struct {
+	tempDirectory   string
+	ociRefExtracted types.ImageReference
+}
+
+// deletes the temporary directory created
+func (t *tempDirOCIRef) deleteTempDir() error {
+	return os.RemoveAll(t.tempDirectory)
+}
+
+// createOCIRef creates the oci reference of the image
+func createOCIRef(image string) (tempDirOCIRef, error) {
+	dir, err := ioutil.TempDir("/var/tmp", "oci")
+	if err != nil {
+		return tempDirOCIRef{}, errors.Wrapf(err, "error creating temp directory")
+	}
+	ociRef, err := ocilayout.NewReference(dir, image)
+	if err != nil {
+		return tempDirOCIRef{}, err
+	}
+
+	tempDirRef := tempDirOCIRef{tempDirectory: dir, ociRefExtracted: ociRef}
+	return tempDirRef, nil
+}
+
+// creates the temporary directory and copies the tarred content to it
+func createUntarTempDir(ref ociArchiveReference) (tempDirOCIRef, error) {
+	tempDirRef, err := createOCIRef(ref.image)
+	if err != nil {
+		return tempDirOCIRef{}, errors.Wrap(err, "error creating oci reference")
+	}
+	src := ref.resolvedFile
+	dst := tempDirRef.tempDirectory
+	if err := archive.UntarPath(src, dst); err != nil {
+		if err := tempDirRef.deleteTempDir(); err != nil {
+			return tempDirOCIRef{}, errors.Wrapf(err, "error deleting temp directory %q", tempDirRef.tempDirectory)
+		}
+		return tempDirOCIRef{}, errors.Wrapf(err, "error untarring file %q", tempDirRef.tempDirectory)
+	}
+	return tempDirRef, nil
+}

vendor/github.com/containers/image/oci/layout/oci_dest.go

@@ -23,13 +23,16 @@ type ociImageDestination struct {
 }
 
 // newImageDestination returns an ImageDestination for writing to an existing directory.
-func newImageDestination(ref ociReference) types.ImageDestination {
+func newImageDestination(ref ociReference) (types.ImageDestination, error) {
+	if ref.image == "" {
+		return nil, errors.Errorf("cannot save image with empty image.ref.name")
+	}
 	index := imgspecv1.Index{
 		Versioned: imgspec.Versioned{
 			SchemaVersion: 2,
 		},
 	}
-	return &ociImageDestination{ref: ref, index: index}
+	return &ociImageDestination{ref: ref, index: index}, nil
 }
 
 // Reference returns the reference used to set up this destination.  Note that this should directly correspond to user's intent,
@@ -63,7 +66,7 @@ func (d *ociImageDestination) ShouldCompressLayers() bool {
 // AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually
 // uploaded to the image destination, true otherwise.
 func (d *ociImageDestination) AcceptsForeignLayerURLs() bool {
-	return false
+	return true
 }
 
 // MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime OS. False otherwise.
@@ -177,8 +180,12 @@ func (d *ociImageDestination) PutManifest(m []byte) error {
 		return err
 	}
 
+	if d.ref.image == "" {
+		return errors.Errorf("cannot save image with empyt image.ref.name")
+	}
+
 	annotations := make(map[string]string)
-	annotations["org.opencontainers.image.ref.name"] = d.ref.tag
+	annotations["org.opencontainers.image.ref.name"] = d.ref.image
 	desc.Annotations = annotations
 	desc.Platform = &imgspecv1.Platform{
 		Architecture: runtime.GOARCH,

vendor/github.com/containers/image/oci/layout/oci_src.go

@@ -1,27 +1,46 @@
 package layout
 
 import (
+	"context"
 	"io"
 	"io/ioutil"
+	"net/http"
 	"os"
+	"strconv"
 
+	"github.com/containers/image/pkg/tlsclientconfig"
 	"github.com/containers/image/types"
+	"github.com/docker/go-connections/tlsconfig"
 	"github.com/opencontainers/go-digest"
 	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
 )
 
 type ociImageSource struct {
 	ref        ociReference
 	descriptor imgspecv1.Descriptor
+	client     *http.Client
 }
 
 // newImageSource returns an ImageSource for reading from an existing directory.
-func newImageSource(ref ociReference) (types.ImageSource, error) {
+func newImageSource(ctx *types.SystemContext, ref ociReference) (types.ImageSource, error) {
+	tr := tlsclientconfig.NewTransport()
+	tr.TLSClientConfig = tlsconfig.ServerDefault()
+
+	if ctx != nil && ctx.OCICertPath != "" {
+		if err := tlsclientconfig.SetupCertificates(ctx.OCICertPath, tr.TLSClientConfig); err != nil {
+			return nil, err
+		}
+		tr.TLSClientConfig.InsecureSkipVerify = ctx.OCIInsecureSkipTLSVerify
+	}
+
+	client := &http.Client{}
+	client.Transport = tr
 	descriptor, err := ref.getManifestDescriptor()
 	if err != nil {
 		return nil, err
 	}
-	return &ociImageSource{ref: ref, descriptor: descriptor}, nil
+	return &ociImageSource{ref: ref, descriptor: descriptor, client: client}, nil
 }
 
 // Reference returns the reference used to set up this source.
@@ -69,6 +88,10 @@ func (s *ociImageSource) GetTargetManifest(digest digest.Digest) ([]byte, string
 
 // GetBlob returns a stream for the specified blob, and the blob's size.
 func (s *ociImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int64, error) {
+	if len(info.URLs) != 0 {
+		return s.getExternalBlob(info.URLs)
+	}
+
 	path, err := s.ref.blobPath(info.Digest)
 	if err != nil {
 		return nil, 0, err
@@ -85,6 +108,35 @@ func (s *ociImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int64, err
 	return r, fi.Size(), nil
 }
 
-func (s *ociImageSource) GetSignatures() ([][]byte, error) {
+func (s *ociImageSource) GetSignatures(context.Context) ([][]byte, error) {
 	return [][]byte{}, nil
 }
+
+func (s *ociImageSource) getExternalBlob(urls []string) (io.ReadCloser, int64, error) {
+	errWrap := errors.New("failed fetching external blob from all urls")
+	for _, url := range urls {
+		resp, err := s.client.Get(url)
+		if err != nil {
+			errWrap = errors.Wrapf(errWrap, "fetching %s failed %s", url, err.Error())
+			continue
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			resp.Body.Close()
+			errWrap = errors.Wrapf(errWrap, "fetching %s failed, response code not 200", url)
+			continue
+		}
+
+		return resp.Body, getBlobSize(resp), nil
+	}
+
+	return nil, 0, errWrap
+}
+
+func getBlobSize(resp *http.Response) int64 {
+	size, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64)
+	if err != nil {
+		size = -1
+	}
+	return size
+}

vendor/github.com/containers/image/oci/layout/oci_transport.go

@@ -36,7 +36,14 @@ func (t ociTransport) ParseReference(reference string) (types.ImageReference, er
 	return ParseReference(reference)
 }
 
-var refRegexp = regexp.MustCompile(`^([A-Za-z0-9._-]+)+$`)
+// annotation spex from https://github.com/opencontainers/image-spec/blob/master/annotations.md#pre-defined-annotation-keys
+const (
+	separator = `(?:[-._:@+]|--)`
+	alphanum  = `(?:[A-Za-z0-9]+)`
+	component = `(?:` + alphanum + `(?:` + separator + alphanum + `)*)`
+)
+
+var refRegexp = regexp.MustCompile(`^` + component + `(?:/` + component + `)*$`)
 
 // ValidatePolicyConfigurationScope checks that scope is a valid name for a signature.PolicyTransportScopes keys
 // (i.e. a valid PolicyConfigurationIdentity() or PolicyConfigurationNamespaces() return value).
@@ -44,19 +51,14 @@ var refRegexp = regexp.MustCompile(`^([A-Za-z0-9._-]+)+$`)
 // scope passed to this function will not be "", that value is always allowed.
 func (t ociTransport) ValidatePolicyConfigurationScope(scope string) error {
 	var dir string
-	sep := strings.LastIndex(scope, ":")
-	if sep == -1 {
-		dir = scope
-	} else {
-		dir = scope[:sep]
-		tag := scope[sep+1:]
-		if !refRegexp.MatchString(tag) {
-			return errors.Errorf("Invalid tag %s", tag)
-		}
-	}
+	sep := strings.SplitN(scope, ":", 2)
+	dir = sep[0]
 
-	if strings.Contains(dir, ":") {
-		return errors.Errorf("Invalid OCI reference %s: path contains a colon", scope)
+	if len(sep) == 2 {
+		image := sep[1]
+		if !refRegexp.MatchString(image) {
+			return errors.Errorf("Invalid image %s", image)
+		}
 	}
 
 	if !strings.HasPrefix(dir, "/") {
@@ -64,7 +66,7 @@ func (t ociTransport) ValidatePolicyConfigurationScope(scope string) error {
 	}
 	// Refuse also "/", otherwise "/" and "" would have the same semantics,
 	// and "" could be unexpectedly shadowed by the "/" entry.
-	// (Note: we do allow "/:sometag", a bit ridiculous but why refuse it?)
+	// (Note: we do allow "/:someimage", a bit ridiculous but why refuse it?)
 	if scope == "/" {
 		return errors.New(`Invalid scope "/": Use the generic default scope ""`)
 	}
@@ -85,28 +87,26 @@ type ociReference struct {
 	// (But in general, we make no attempt to be completely safe against concurrent hostile filesystem modifications.)
 	dir         string // As specified by the user. May be relative, contain symlinks, etc.
 	resolvedDir string // Absolute path with no symlinks, at least at the time of its creation. Primarily used for policy namespaces.
-	tag         string
+	image       string // If image=="", it means the only image in the index.json is used
 }
 
 // ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an OCI ImageReference.
 func ParseReference(reference string) (types.ImageReference, error) {
-	var dir, tag string
-	sep := strings.LastIndex(reference, ":")
-	if sep == -1 {
-		dir = reference
-		tag = "latest"
-	} else {
-		dir = reference[:sep]
-		tag = reference[sep+1:]
+	var dir, image string
+	sep := strings.SplitN(reference, ":", 2)
+	dir = sep[0]
+
+	if len(sep) == 2 {
+		image = sep[1]
 	}
-	return NewReference(dir, tag)
+	return NewReference(dir, image)
 }
 
-// NewReference returns an OCI reference for a directory and a tag.
+// NewReference returns an OCI reference for a directory and a image.
 //
 // We do not expose an API supplying the resolvedDir; we could, but recomputing it
 // is generally cheap enough that we prefer being confident about the properties of resolvedDir.
-func NewReference(dir, tag string) (types.ImageReference, error) {
+func NewReference(dir, image string) (types.ImageReference, error) {
 	resolved, err := explicitfilepath.ResolvePathToFullyExplicit(dir)
 	if err != nil {
 		return nil, err
@@ -114,12 +114,12 @@ func NewReference(dir, tag string) (types.ImageReference, error) {
 	// This is necessary to prevent directory paths returned by PolicyConfigurationNamespaces
 	// from being ambiguous with values of PolicyConfigurationIdentity.
 	if strings.Contains(resolved, ":") {
-		return nil, errors.Errorf("Invalid OCI reference %s:%s: path %s contains a colon", dir, tag, resolved)
+		return nil, errors.Errorf("Invalid OCI reference %s:%s: path %s contains a colon", dir, image, resolved)
 	}
-	if !refRegexp.MatchString(tag) {
-		return nil, errors.Errorf("Invalid tag %s", tag)
+	if len(image) > 0 && !refRegexp.MatchString(image) {
+		return nil, errors.Errorf("Invalid image %s", image)
 	}
-	return ociReference{dir: dir, resolvedDir: resolved, tag: tag}, nil
+	return ociReference{dir: dir, resolvedDir: resolved, image: image}, nil
 }
 
 func (ref ociReference) Transport() types.ImageTransport {
@@ -132,7 +132,7 @@ func (ref ociReference) Transport() types.ImageTransport {
 // e.g. default attribute values omitted by the user may be filled in in the return value, or vice versa.
 // WARNING: Do not use the return value in the UI to describe an image, it does not contain the Transport().Name() prefix.
 func (ref ociReference) StringWithinTransport() string {
-	return fmt.Sprintf("%s:%s", ref.dir, ref.tag)
+	return fmt.Sprintf("%s:%s", ref.dir, ref.image)
 }
 
 // DockerReference returns a Docker reference associated with this reference
@@ -150,7 +150,10 @@ func (ref ociReference) DockerReference() reference.Named {
 // not required/guaranteed that it will be a valid input to Transport().ParseReference().
 // Returns "" if configuration identities for these references are not supported.
 func (ref ociReference) PolicyConfigurationIdentity() string {
-	return fmt.Sprintf("%s:%s", ref.resolvedDir, ref.tag)
+	// NOTE: ref.image is not a part of the image identity, because "$dir:$someimage" and "$dir:" may mean the
+	// same image and the two can’t be statically disambiguated.  Using at least the repository directory is
+	// less granular but hopefully still useful.
+	return fmt.Sprintf("%s", ref.resolvedDir)
 }
 
 // PolicyConfigurationNamespaces returns a list of other policy configuration namespaces to search
@@ -179,7 +182,7 @@ func (ref ociReference) PolicyConfigurationNamespaces() []string {
 // NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource,
 // verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage.
 func (ref ociReference) NewImage(ctx *types.SystemContext) (types.Image, error) {
-	src, err := newImageSource(ref)
+	src, err := newImageSource(ctx, ref)
 	if err != nil {
 		return nil, err
 	}
@@ -196,38 +199,58 @@ func (ref ociReference) getManifestDescriptor() (imgspecv1.Descriptor, error) {
 	if err := json.NewDecoder(indexJSON).Decode(&index); err != nil {
 		return imgspecv1.Descriptor{}, err
 	}
+
 	var d *imgspecv1.Descriptor
-	for _, md := range index.Manifests {
-		if md.MediaType != imgspecv1.MediaTypeImageManifest {
-			continue
+	if ref.image == "" {
+		// return manifest if only one image is in the oci directory
+		if len(index.Manifests) == 1 {
+			d = &index.Manifests[0]
+		} else {
+			// ask user to choose image when more than one image in the oci directory
+			return imgspecv1.Descriptor{}, errors.Wrapf(err, "more than one image in oci, choose an image")
 		}
-		refName, ok := md.Annotations["org.opencontainers.image.ref.name"]
-		if !ok {
-			continue
-		}
-		if refName == ref.tag {
-			d = &md
-			break
+	} else {
+		// if image specified, look through all manifests for a match
+		for _, md := range index.Manifests {
+			if md.MediaType != imgspecv1.MediaTypeImageManifest {
+				continue
+			}
+			refName, ok := md.Annotations["org.opencontainers.image.ref.name"]
+			if !ok {
+				continue
+			}
+			if refName == ref.image {
+				d = &md
+				break
+			}
 		}
 	}
 	if d == nil {
-		return imgspecv1.Descriptor{}, fmt.Errorf("no descriptor found for reference %q", ref.tag)
+		return imgspecv1.Descriptor{}, fmt.Errorf("no descriptor found for reference %q", ref.image)
 	}
 	return *d, nil
 }
 
-// NewImageSource returns a types.ImageSource for this reference,
-// asking the backend to use a manifest from requestedManifestMIMETypes if possible.
-// nil requestedManifestMIMETypes means manifest.DefaultRequestedManifestMIMETypes.
+// LoadManifestDescriptor loads the manifest descriptor to be used to retrieve the image name
+// when pulling an image
+func LoadManifestDescriptor(imgRef types.ImageReference) (imgspecv1.Descriptor, error) {
+	ociRef, ok := imgRef.(ociReference)
+	if !ok {
+		return imgspecv1.Descriptor{}, errors.Errorf("error typecasting, need type ociRef")
+	}
+	return ociRef.getManifestDescriptor()
+}
+
+// NewImageSource returns a types.ImageSource for this reference.
 // The caller must call .Close() on the returned ImageSource.
-func (ref ociReference) NewImageSource(ctx *types.SystemContext, requestedManifestMIMETypes []string) (types.ImageSource, error) {
-	return newImageSource(ref)
+func (ref ociReference) NewImageSource(ctx *types.SystemContext) (types.ImageSource, error) {
+	return newImageSource(ctx, ref)
 }
 
 // NewImageDestination returns a types.ImageDestination for this reference.
 // The caller must call .Close() on the returned ImageDestination.
 func (ref ociReference) NewImageDestination(ctx *types.SystemContext) (types.ImageDestination, error) {
-	return newImageDestination(ref), nil
+	return newImageDestination(ref)
 }
 
 // DeleteImage deletes the named image from the registry, if supported.

vendor/github.com/containers/image/openshift/openshift.go

@@ -2,6 +2,7 @@ package openshift
 
 import (
 	"bytes"
+	"context"
 	"crypto/rand"
 	"encoding/json"
 	"fmt"
@@ -11,7 +12,6 @@ import (
 	"net/url"
 	"strings"
 
-	"github.com/Sirupsen/logrus"
 	"github.com/containers/image/docker"
 	"github.com/containers/image/docker/reference"
 	"github.com/containers/image/manifest"
@@ -19,6 +19,7 @@ import (
 	"github.com/containers/image/version"
 	"github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 // openshiftClient is configuration for dealing with a single image stream, for reading or writing.
@@ -70,7 +71,7 @@ func newOpenshiftClient(ref openshiftReference) (*openshiftClient, error) {
 }
 
 // doRequest performs a correctly authenticated request to a specified path, and returns response body or an error object.
-func (c *openshiftClient) doRequest(method, path string, requestBody []byte) ([]byte, error) {
+func (c *openshiftClient) doRequest(ctx context.Context, method, path string, requestBody []byte) ([]byte, error) {
 	url := *c.baseURL
 	url.Path = path
 	var requestBodyReader io.Reader
@@ -82,6 +83,7 @@ func (c *openshiftClient) doRequest(method, path string, requestBody []byte) ([]
 	if err != nil {
 		return nil, err
 	}
+	req = req.WithContext(ctx)
 
 	if len(c.bearerToken) != 0 {
 		req.Header.Set("Authorization", "Bearer "+c.bearerToken)
@@ -132,10 +134,10 @@ func (c *openshiftClient) doRequest(method, path string, requestBody []byte) ([]
 }
 
 // getImage loads the specified image object.
-func (c *openshiftClient) getImage(imageStreamImageName string) (*image, error) {
+func (c *openshiftClient) getImage(ctx context.Context, imageStreamImageName string) (*image, error) {
 	// FIXME: validate components per validation.IsValidPathSegmentName?
 	path := fmt.Sprintf("/oapi/v1/namespaces/%s/imagestreamimages/%s@%s", c.ref.namespace, c.ref.stream, imageStreamImageName)
-	body, err := c.doRequest("GET", path, nil)
+	body, err := c.doRequest(ctx, "GET", path, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -160,18 +162,15 @@ func (c *openshiftClient) convertDockerImageReference(ref string) (string, error
 type openshiftImageSource struct {
 	client *openshiftClient
 	// Values specific to this image
-	ctx                        *types.SystemContext
-	requestedManifestMIMETypes []string
+	ctx *types.SystemContext
 	// State
 	docker               types.ImageSource // The Docker Registry endpoint, or nil if not resolved yet
 	imageStreamImageName string            // Resolved image identifier, or "" if not known yet
 }
 
-// newImageSource creates a new ImageSource for the specified reference,
-// asking the backend to use a manifest from requestedManifestMIMETypes if possible.
-// nil requestedManifestMIMETypes means manifest.DefaultRequestedManifestMIMETypes.
+// newImageSource creates a new ImageSource for the specified reference.
 // The caller must call .Close() on the returned ImageSource.
-func newImageSource(ctx *types.SystemContext, ref openshiftReference, requestedManifestMIMETypes []string) (types.ImageSource, error) {
+func newImageSource(ctx *types.SystemContext, ref openshiftReference) (types.ImageSource, error) {
 	client, err := newOpenshiftClient(ref)
 	if err != nil {
 		return nil, err
@@ -180,7 +179,6 @@ func newImageSource(ctx *types.SystemContext, ref openshiftReference, requestedM
 	return &openshiftImageSource{
 		client: client,
 		ctx:    ctx,
-		requestedManifestMIMETypes: requestedManifestMIMETypes,
 	}, nil
 }
 
@@ -203,7 +201,7 @@ func (s *openshiftImageSource) Close() error {
 }
 
 func (s *openshiftImageSource) GetTargetManifest(digest digest.Digest) ([]byte, string, error) {
-	if err := s.ensureImageIsResolved(); err != nil {
+	if err := s.ensureImageIsResolved(context.TODO()); err != nil {
 		return nil, "", err
 	}
 	return s.docker.GetTargetManifest(digest)
@@ -212,7 +210,7 @@ func (s *openshiftImageSource) GetTargetManifest(digest digest.Digest) ([]byte,
 // GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available).
 // It may use a remote (= slow) service.
 func (s *openshiftImageSource) GetManifest() ([]byte, string, error) {
-	if err := s.ensureImageIsResolved(); err != nil {
+	if err := s.ensureImageIsResolved(context.TODO()); err != nil {
 		return nil, "", err
 	}
 	return s.docker.GetManifest()
@@ -220,18 +218,18 @@ func (s *openshiftImageSource) GetManifest() ([]byte, string, error) {
 
 // GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown).
 func (s *openshiftImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int64, error) {
-	if err := s.ensureImageIsResolved(); err != nil {
+	if err := s.ensureImageIsResolved(context.TODO()); err != nil {
 		return nil, 0, err
 	}
 	return s.docker.GetBlob(info)
 }
 
-func (s *openshiftImageSource) GetSignatures() ([][]byte, error) {
-	if err := s.ensureImageIsResolved(); err != nil {
+func (s *openshiftImageSource) GetSignatures(ctx context.Context) ([][]byte, error) {
+	if err := s.ensureImageIsResolved(ctx); err != nil {
 		return nil, err
 	}
 
-	image, err := s.client.getImage(s.imageStreamImageName)
+	image, err := s.client.getImage(ctx, s.imageStreamImageName)
 	if err != nil {
 		return nil, err
 	}
@@ -245,14 +243,14 @@ func (s *openshiftImageSource) GetSignatures() ([][]byte, error) {
 }
 
 // ensureImageIsResolved sets up s.docker and s.imageStreamImageName
-func (s *openshiftImageSource) ensureImageIsResolved() error {
+func (s *openshiftImageSource) ensureImageIsResolved(ctx context.Context) error {
 	if s.docker != nil {
 		return nil
 	}
 
 	// FIXME: validate components per validation.IsValidPathSegmentName?
 	path := fmt.Sprintf("/oapi/v1/namespaces/%s/imagestreams/%s", s.client.ref.namespace, s.client.ref.stream)
-	body, err := s.client.doRequest("GET", path, nil)
+	body, err := s.client.doRequest(ctx, "GET", path, nil)
 	if err != nil {
 		return err
 	}
@@ -284,7 +282,7 @@ func (s *openshiftImageSource) ensureImageIsResolved() error {
 	if err != nil {
 		return err
 	}
-	d, err := dockerRef.NewImageSource(s.ctx, s.requestedManifestMIMETypes)
+	d, err := dockerRef.NewImageSource(s.ctx)
 	if err != nil {
 		return err
 	}
@@ -410,7 +408,7 @@ func (d *openshiftImageDestination) PutSignatures(signatures [][]byte) error {
 		return nil // No need to even read the old state.
 	}
 
-	image, err := d.client.getImage(d.imageStreamImageName)
+	image, err := d.client.getImage(context.TODO(), d.imageStreamImageName)
 	if err != nil {
 		return err
 	}
@@ -451,7 +449,7 @@ sigExists:
 			Content:    newSig,
 		}
 		body, err := json.Marshal(sig)
-		_, err = d.client.doRequest("POST", "/oapi/v1/imagesignatures", body)
+		_, err = d.client.doRequest(context.TODO(), "POST", "/oapi/v1/imagesignatures", body)
 		if err != nil {
 			return err
 		}

vendor/github.com/containers/image/openshift/openshift_transport.go

@@ -130,19 +130,17 @@ func (ref openshiftReference) PolicyConfigurationNamespaces() []string {
 // NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource,
 // verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage.
 func (ref openshiftReference) NewImage(ctx *types.SystemContext) (types.Image, error) {
-	src, err := newImageSource(ctx, ref, nil)
+	src, err := newImageSource(ctx, ref)
 	if err != nil {
 		return nil, err
 	}
 	return genericImage.FromSource(src)
 }
 
-// NewImageSource returns a types.ImageSource for this reference,
-// asking the backend to use a manifest from requestedManifestMIMETypes if possible.
-// nil requestedManifestMIMETypes means manifest.DefaultRequestedManifestMIMETypes.
+// NewImageSource returns a types.ImageSource for this reference.
 // The caller must call .Close() on the returned ImageSource.
-func (ref openshiftReference) NewImageSource(ctx *types.SystemContext, requestedManifestMIMETypes []string) (types.ImageSource, error) {
-	return newImageSource(ctx, ref, requestedManifestMIMETypes)
+func (ref openshiftReference) NewImageSource(ctx *types.SystemContext) (types.ImageSource, error) {
+	return newImageSource(ctx, ref)
 }
 
 // NewImageDestination returns a types.ImageDestination for this reference.

vendor/github.com/containers/image/ostree/ostree_dest.go

@@ -1,3 +1,5 @@
+// +build !containers_image_ostree_stub
+
 package ostree
 
 import (
@@ -44,6 +46,7 @@ type ostreeImageDestination struct {
 	schema     manifestSchema
 	tmpDirPath string
 	blobs      map[string]*blobToImport
+	digest     digest.Digest
 }
 
 // newImageDestination returns an ImageDestination for writing to an existing ostree.
@@ -52,7 +55,7 @@ func newImageDestination(ref ostreeReference, tmpDirPath string) (types.ImageDes
 	if err := ensureDirectoryExists(tmpDirPath); err != nil {
 		return nil, err
 	}
-	return &ostreeImageDestination{ref, "", manifestSchema{}, tmpDirPath, map[string]*blobToImport{}}, nil
+	return &ostreeImageDestination{ref, "", manifestSchema{}, tmpDirPath, map[string]*blobToImport{}, ""}, nil
 }
 
 // Reference returns the reference used to set up this destination.  Note that this should directly correspond to user's intent,
@@ -151,7 +154,7 @@ func fixFiles(dir string, usermode bool) error {
 			if err != nil {
 				return err
 			}
-		} else if usermode && (info.Mode().IsRegular() || (info.Mode()&os.ModeSymlink) != 0) {
+		} else if usermode && (info.Mode().IsRegular()) {
 			if err := os.Chmod(fullpath, info.Mode()|0600); err != nil {
 				return err
 			}
@@ -236,10 +239,10 @@ func (d *ostreeImageDestination) ReapplyBlob(info types.BlobInfo) (types.BlobInf
 // FIXME? This should also receive a MIME type if known, to differentiate between schema versions.
 // If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema),
 // but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError.
-func (d *ostreeImageDestination) PutManifest(manifest []byte) error {
-	d.manifest = string(manifest)
+func (d *ostreeImageDestination) PutManifest(manifestBlob []byte) error {
+	d.manifest = string(manifestBlob)
 
-	if err := json.Unmarshal(manifest, &d.schema); err != nil {
+	if err := json.Unmarshal(manifestBlob, &d.schema); err != nil {
 		return err
 	}
 
@@ -248,7 +251,13 @@ func (d *ostreeImageDestination) PutManifest(manifest []byte) error {
 		return err
 	}
 
-	return ioutil.WriteFile(manifestPath, manifest, 0644)
+	digest, err := manifest.Digest(manifestBlob)
+	if err != nil {
+		return err
+	}
+	d.digest = digest
+
+	return ioutil.WriteFile(manifestPath, manifestBlob, 0644)
 }
 
 func (d *ostreeImageDestination) PutSignatures(signatures [][]byte) error {
@@ -302,7 +311,7 @@ func (d *ostreeImageDestination) Commit() error {
 
 	manifestPath := filepath.Join(d.tmpDirPath, "manifest")
 
-	metadata := []string{fmt.Sprintf("docker.manifest=%s", string(d.manifest))}
+	metadata := []string{fmt.Sprintf("docker.manifest=%s", string(d.manifest)), fmt.Sprintf("docker.digest=%s", string(d.digest))}
 	err = d.ostreeCommit(repo, fmt.Sprintf("ociimage/%s", d.ref.branchName), manifestPath, metadata)
 
 	_, err = repo.CommitTransaction()

vendor/github.com/containers/image/ostree/ostree_transport.go

@@ -1,3 +1,5 @@
+// +build !containers_image_ostree_stub
+
 package ostree
 
 import (
@@ -82,24 +84,15 @@ func NewReference(image string, repo string) (types.ImageReference, error) {
 	// image is not _really_ in a containers/image/docker/reference format;
 	// as far as the libOSTree ociimage/* namespace is concerned, it is more or
 	// less an arbitrary string with an implied tag.
-	// We use the reference.* parsers basically for the default tag name in
-	// reference.TagNameOnly, and incidentally for some character set and length
-	// restrictions.
-	var ostreeImage reference.Named
-	s := strings.SplitN(image, ":", 2)
-
-	named, err := reference.WithName(s[0])
+	// Parse the image using reference.ParseNormalizedNamed so that we can
+	// check whether the images has a tag specified and we can add ":latest" if needed
+	ostreeImage, err := reference.ParseNormalizedNamed(image)
 	if err != nil {
 		return nil, err
 	}
 
-	if len(s) == 1 {
-		ostreeImage = reference.TagNameOnly(named)
-	} else {
-		ostreeImage, err = reference.WithTag(named, s[1])
-		if err != nil {
-			return nil, err
-		}
+	if reference.IsNameOnly(ostreeImage) {
+		image = image + ":latest"
 	}
 
 	resolved, err := explicitfilepath.ResolvePathToFullyExplicit(repo)
@@ -121,8 +114,8 @@ func NewReference(image string, repo string) (types.ImageReference, error) {
 	}
 
 	return ostreeReference{
-		image:      ostreeImage.String(),
-		branchName: encodeOStreeRef(ostreeImage.String()),
+		image:      image,
+		branchName: encodeOStreeRef(image),
 		repo:       resolved,
 	}, nil
 }
@@ -183,11 +176,9 @@ func (ref ostreeReference) NewImage(ctx *types.SystemContext) (types.Image, erro
 	return nil, errors.New("Reading ostree: images is currently not supported")
 }
 
-// NewImageSource returns a types.ImageSource for this reference,
-// asking the backend to use a manifest from requestedManifestMIMETypes if possible.
-// nil requestedManifestMIMETypes means manifest.DefaultRequestedManifestMIMETypes.
+// NewImageSource returns a types.ImageSource for this reference.
 // The caller must call .Close() on the returned ImageSource.
-func (ref ostreeReference) NewImageSource(ctx *types.SystemContext, requestedManifestMIMETypes []string) (types.ImageSource, error) {
+func (ref ostreeReference) NewImageSource(ctx *types.SystemContext) (types.ImageSource, error) {
 	return nil, errors.New("Reading ostree: images is currently not supported")
 }
 

vendor/github.com/containers/image/pkg/compression/compression.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/pkg/errors"
 
-	"github.com/Sirupsen/logrus"
+	"github.com/sirupsen/logrus"
 )
 
 // DecompressorFunc returns the decompressed stream, given a compressed stream.

vendor/github.com/containers/image/pkg/docker/config/config.go

@@ -0,0 +1,295 @@
+package config
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/containers/image/types"
+	helperclient "github.com/docker/docker-credential-helpers/client"
+	"github.com/docker/docker-credential-helpers/credentials"
+	"github.com/docker/docker/pkg/homedir"
+	"github.com/pkg/errors"
+)
+
+type dockerAuthConfig struct {
+	Auth string `json:"auth,omitempty"`
+}
+
+type dockerConfigFile struct {
+	AuthConfigs map[string]dockerAuthConfig `json:"auths"`
+	CredHelpers map[string]string           `json:"credHelpers,omitempty"`
+}
+
+const (
+	defaultPath       = "/run/user"
+	authCfg           = "containers"
+	authCfgFileName   = "auth.json"
+	dockerCfg         = ".docker"
+	dockerCfgFileName = "config.json"
+	dockerLegacyCfg   = ".dockercfg"
+)
+
+var (
+	// ErrNotLoggedIn is returned for users not logged into a registry
+	// that they are trying to logout of
+	ErrNotLoggedIn = errors.New("not logged in")
+)
+
+// SetAuthentication stores the username and password in the auth.json file
+func SetAuthentication(ctx *types.SystemContext, registry, username, password string) error {
+	return modifyJSON(ctx, func(auths *dockerConfigFile) (bool, error) {
+		if ch, exists := auths.CredHelpers[registry]; exists {
+			return false, setAuthToCredHelper(ch, registry, username, password)
+		}
+
+		creds := base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
+		newCreds := dockerAuthConfig{Auth: creds}
+		auths.AuthConfigs[registry] = newCreds
+		return true, nil
+	})
+}
+
+// GetAuthentication returns the registry credentials stored in
+// either auth.json file or .docker/config.json
+// If an entry is not found empty strings are returned for the username and password
+func GetAuthentication(ctx *types.SystemContext, registry string) (string, string, error) {
+	if ctx != nil && ctx.DockerAuthConfig != nil {
+		return ctx.DockerAuthConfig.Username, ctx.DockerAuthConfig.Password, nil
+	}
+
+	dockerLegacyPath := filepath.Join(homedir.Get(), dockerLegacyCfg)
+	paths := [3]string{getPathToAuth(ctx), filepath.Join(homedir.Get(), dockerCfg, dockerCfgFileName), dockerLegacyPath}
+
+	for _, path := range paths {
+		legacyFormat := path == dockerLegacyPath
+		username, password, err := findAuthentication(registry, path, legacyFormat)
+		if err != nil {
+			return "", "", err
+		}
+		if username != "" && password != "" {
+			return username, password, nil
+		}
+	}
+	return "", "", nil
+}
+
+// GetUserLoggedIn returns the username logged in to registry from either
+// auth.json or XDG_RUNTIME_DIR
+// Used to tell the user if someone is logged in to the registry when logging in
+func GetUserLoggedIn(ctx *types.SystemContext, registry string) string {
+	path := getPathToAuth(ctx)
+	username, _, _ := findAuthentication(registry, path, false)
+	if username != "" {
+		return username
+	}
+	return ""
+}
+
+// RemoveAuthentication deletes the credentials stored in auth.json
+func RemoveAuthentication(ctx *types.SystemContext, registry string) error {
+	return modifyJSON(ctx, func(auths *dockerConfigFile) (bool, error) {
+		// First try cred helpers.
+		if ch, exists := auths.CredHelpers[registry]; exists {
+			return false, deleteAuthFromCredHelper(ch, registry)
+		}
+
+		if _, ok := auths.AuthConfigs[registry]; ok {
+			delete(auths.AuthConfigs, registry)
+		} else if _, ok := auths.AuthConfigs[normalizeRegistry(registry)]; ok {
+			delete(auths.AuthConfigs, normalizeRegistry(registry))
+		} else {
+			return false, ErrNotLoggedIn
+		}
+		return true, nil
+	})
+}
+
+// RemoveAllAuthentication deletes all the credentials stored in auth.json
+func RemoveAllAuthentication(ctx *types.SystemContext) error {
+	return modifyJSON(ctx, func(auths *dockerConfigFile) (bool, error) {
+		auths.CredHelpers = make(map[string]string)
+		auths.AuthConfigs = make(map[string]dockerAuthConfig)
+		return true, nil
+	})
+}
+
+// getPath gets the path of the auth.json file
+// The path can be overriden by the user if the overwrite-path flag is set
+// If the flag is not set and XDG_RUNTIME_DIR is ser, the auth.json file is saved in XDG_RUNTIME_DIR/containers
+// Otherwise, the auth.json file is stored in /run/user/UID/containers
+func getPathToAuth(ctx *types.SystemContext) string {
+	if ctx != nil {
+		if ctx.AuthFilePath != "" {
+			return ctx.AuthFilePath
+		}
+		if ctx.RootForImplicitAbsolutePaths != "" {
+			return filepath.Join(ctx.RootForImplicitAbsolutePaths, defaultPath, strconv.Itoa(os.Getuid()), authCfg, authCfgFileName)
+		}
+	}
+	runtimeDir := os.Getenv("XDG_RUNTIME_DIR")
+	if runtimeDir == "" {
+		runtimeDir = filepath.Join(defaultPath, strconv.Itoa(os.Getuid()))
+	}
+	return filepath.Join(runtimeDir, authCfg, authCfgFileName)
+}
+
+// readJSONFile unmarshals the authentications stored in the auth.json file and returns it
+// or returns an empty dockerConfigFile data structure if auth.json does not exist
+// if the file exists and is empty, readJSONFile returns an error
+func readJSONFile(path string, legacyFormat bool) (dockerConfigFile, error) {
+	var auths dockerConfigFile
+
+	raw, err := ioutil.ReadFile(path)
+	if os.IsNotExist(err) {
+		auths.AuthConfigs = map[string]dockerAuthConfig{}
+		return auths, nil
+	}
+
+	if legacyFormat {
+		if err = json.Unmarshal(raw, &auths.AuthConfigs); err != nil {
+			return dockerConfigFile{}, errors.Wrapf(err, "error unmarshaling JSON at %q", path)
+		}
+		return auths, nil
+	}
+
+	if err = json.Unmarshal(raw, &auths); err != nil {
+		return dockerConfigFile{}, errors.Wrapf(err, "error unmarshaling JSON at %q", path)
+	}
+
+	return auths, nil
+}
+
+// modifyJSON writes to auth.json if the dockerConfigFile has been updated
+func modifyJSON(ctx *types.SystemContext, editor func(auths *dockerConfigFile) (bool, error)) error {
+	path := getPathToAuth(ctx)
+	dir := filepath.Dir(path)
+	if _, err := os.Stat(dir); os.IsNotExist(err) {
+		if err = os.Mkdir(dir, 0700); err != nil {
+			return errors.Wrapf(err, "error creating directory %q", dir)
+		}
+	}
+
+	auths, err := readJSONFile(path, false)
+	if err != nil {
+		return errors.Wrapf(err, "error reading JSON file %q", path)
+	}
+
+	updated, err := editor(&auths)
+	if err != nil {
+		return errors.Wrapf(err, "error updating %q", path)
+	}
+	if updated {
+		newData, err := json.MarshalIndent(auths, "", "\t")
+		if err != nil {
+			return errors.Wrapf(err, "error marshaling JSON %q", path)
+		}
+
+		if err = ioutil.WriteFile(path, newData, 0755); err != nil {
+			return errors.Wrapf(err, "error writing to file %q", path)
+		}
+	}
+
+	return nil
+}
+
+func getAuthFromCredHelper(credHelper, registry string) (string, string, error) {
+	helperName := fmt.Sprintf("docker-credential-%s", credHelper)
+	p := helperclient.NewShellProgramFunc(helperName)
+	creds, err := helperclient.Get(p, registry)
+	if err != nil {
+		return "", "", err
+	}
+	return creds.Username, creds.Secret, nil
+}
+
+func setAuthToCredHelper(credHelper, registry, username, password string) error {
+	helperName := fmt.Sprintf("docker-credential-%s", credHelper)
+	p := helperclient.NewShellProgramFunc(helperName)
+	creds := &credentials.Credentials{
+		ServerURL: registry,
+		Username:  username,
+		Secret:    password,
+	}
+	return helperclient.Store(p, creds)
+}
+
+func deleteAuthFromCredHelper(credHelper, registry string) error {
+	helperName := fmt.Sprintf("docker-credential-%s", credHelper)
+	p := helperclient.NewShellProgramFunc(helperName)
+	return helperclient.Erase(p, registry)
+}
+
+// findAuthentication looks for auth of registry in path
+func findAuthentication(registry, path string, legacyFormat bool) (string, string, error) {
+	auths, err := readJSONFile(path, legacyFormat)
+	if err != nil {
+		return "", "", errors.Wrapf(err, "error reading JSON file %q", path)
+	}
+
+	// First try cred helpers. They should always be normalized.
+	if ch, exists := auths.CredHelpers[registry]; exists {
+		return getAuthFromCredHelper(ch, registry)
+	}
+
+	// I'm feeling lucky
+	if val, exists := auths.AuthConfigs[registry]; exists {
+		return decodeDockerAuth(val.Auth)
+	}
+
+	// bad luck; let's normalize the entries first
+	registry = normalizeRegistry(registry)
+	normalizedAuths := map[string]dockerAuthConfig{}
+	for k, v := range auths.AuthConfigs {
+		normalizedAuths[normalizeRegistry(k)] = v
+	}
+	if val, exists := normalizedAuths[registry]; exists {
+		return decodeDockerAuth(val.Auth)
+	}
+	return "", "", nil
+}
+
+func decodeDockerAuth(s string) (string, string, error) {
+	decoded, err := base64.StdEncoding.DecodeString(s)
+	if err != nil {
+		return "", "", err
+	}
+	parts := strings.SplitN(string(decoded), ":", 2)
+	if len(parts) != 2 {
+		// if it's invalid just skip, as docker does
+		return "", "", nil
+	}
+	user := parts[0]
+	password := strings.Trim(parts[1], "\x00")
+	return user, password, nil
+}
+
+// convertToHostname converts a registry url which has http|https prepended
+// to just an hostname.
+// Copied from github.com/docker/docker/registry/auth.go
+func convertToHostname(url string) string {
+	stripped := url
+	if strings.HasPrefix(url, "http://") {
+		stripped = strings.TrimPrefix(url, "http://")
+	} else if strings.HasPrefix(url, "https://") {
+		stripped = strings.TrimPrefix(url, "https://")
+	}
+
+	nameParts := strings.SplitN(stripped, "/", 2)
+
+	return nameParts[0]
+}
+
+func normalizeRegistry(registry string) string {
+	normalized := convertToHostname(registry)
+	switch normalized {
+	case "registry-1.docker.io", "docker.io":
+		return "index.docker.io"
+	}
+	return normalized
+}

vendor/github.com/containers/image/pkg/tlsclientconfig/tlsclientconfig.go

@@ -0,0 +1,102 @@
+package tlsclientconfig
+
+import (
+	"crypto/tls"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/docker/go-connections/sockets"
+	"github.com/docker/go-connections/tlsconfig"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+// SetupCertificates opens all .crt, .cert, and .key files in dir and appends / loads certs and key pairs as appropriate to tlsc
+func SetupCertificates(dir string, tlsc *tls.Config) error {
+	logrus.Debugf("Looking for TLS certificates and private keys in %s", dir)
+	fs, err := ioutil.ReadDir(dir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		if os.IsPermission(err) {
+			logrus.Debugf("Skipping scan of %s due to permission error: %v", dir, err)
+			return nil
+		}
+		return err
+	}
+
+	for _, f := range fs {
+		fullPath := filepath.Join(dir, f.Name())
+		if strings.HasSuffix(f.Name(), ".crt") {
+			systemPool, err := tlsconfig.SystemCertPool()
+			if err != nil {
+				return errors.Wrap(err, "unable to get system cert pool")
+			}
+			tlsc.RootCAs = systemPool
+			logrus.Debugf(" crt: %s", fullPath)
+			data, err := ioutil.ReadFile(fullPath)
+			if err != nil {
+				return err
+			}
+			tlsc.RootCAs.AppendCertsFromPEM(data)
+		}
+		if strings.HasSuffix(f.Name(), ".cert") {
+			certName := f.Name()
+			keyName := certName[:len(certName)-5] + ".key"
+			logrus.Debugf(" cert: %s", fullPath)
+			if !hasFile(fs, keyName) {
+				return errors.Errorf("missing key %s for client certificate %s. Note that CA certificates should use the extension .crt", keyName, certName)
+			}
+			cert, err := tls.LoadX509KeyPair(filepath.Join(dir, certName), filepath.Join(dir, keyName))
+			if err != nil {
+				return err
+			}
+			tlsc.Certificates = append(tlsc.Certificates, cert)
+		}
+		if strings.HasSuffix(f.Name(), ".key") {
+			keyName := f.Name()
+			certName := keyName[:len(keyName)-4] + ".cert"
+			logrus.Debugf(" key: %s", fullPath)
+			if !hasFile(fs, certName) {
+				return errors.Errorf("missing client certificate %s for key %s", certName, keyName)
+			}
+		}
+	}
+	return nil
+}
+
+func hasFile(files []os.FileInfo, name string) bool {
+	for _, f := range files {
+		if f.Name() == name {
+			return true
+		}
+	}
+	return false
+}
+
+// NewTransport Creates a default transport
+func NewTransport() *http.Transport {
+	direct := &net.Dialer{
+		Timeout:   30 * time.Second,
+		KeepAlive: 30 * time.Second,
+		DualStack: true,
+	}
+	tr := &http.Transport{
+		Proxy:               http.ProxyFromEnvironment,
+		Dial:                direct.Dial,
+		TLSHandshakeTimeout: 10 * time.Second,
+		// TODO(dmcgowan): Call close idle connections when complete and use keep alive
+		DisableKeepAlives: true,
+	}
+	proxyDialer, err := sockets.DialerFromEnvironment(direct)
+	if err == nil {
+		tr.Dial = proxyDialer.Dial
+	}
+	return tr
+}

vendor/github.com/containers/image/signature/mechanism_openpgp.go

@@ -132,11 +132,17 @@ func (m *openpgpSigningMechanism) Verify(unverifiedSignature []byte) (contents [
 	if md.SignedBy == nil {
 		return nil, "", InvalidSignatureError{msg: fmt.Sprintf("Invalid GPG signature: %#v", md.Signature)}
 	}
-	if md.Signature.SigLifetimeSecs != nil {
-		expiry := md.Signature.CreationTime.Add(time.Duration(*md.Signature.SigLifetimeSecs) * time.Second)
-		if time.Now().After(expiry) {
-			return nil, "", InvalidSignatureError{msg: fmt.Sprintf("Signature expired on %s", expiry)}
+	if md.Signature != nil {
+		if md.Signature.SigLifetimeSecs != nil {
+			expiry := md.Signature.CreationTime.Add(time.Duration(*md.Signature.SigLifetimeSecs) * time.Second)
+			if time.Now().After(expiry) {
+				return nil, "", InvalidSignatureError{msg: fmt.Sprintf("Signature expired on %s", expiry)}
+			}
 		}
+	} else if md.SignatureV3 == nil {
+		// Coverage: If md.SignedBy != nil, the final md.UnverifiedBody.Read() either sets one of md.Signature or md.SignatureV3,
+		// or sets md.SignatureError.
+		return nil, "", InvalidSignatureError{msg: "Unexpected openpgp.MessageDetails: neither Signature nor SignatureV3 is set"}
 	}
 
 	// Uppercase the fingerprint to be compatible with gpgme

vendor/github.com/containers/image/signature/policy_eval.go

@@ -6,9 +6,11 @@
 package signature
 
 import (
-	"github.com/Sirupsen/logrus"
+	"context"
+
 	"github.com/containers/image/types"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 // PolicyRequirementError is an explanatory text for rejecting a signature or an image.
@@ -188,7 +190,8 @@ func (pc *PolicyContext) GetSignaturesWithAcceptedAuthor(image types.UnparsedIma
 	reqs := pc.requirementsForImageRef(image.Reference())
 
 	// FIXME: rename Signatures to UnverifiedSignatures
-	unverifiedSignatures, err := image.Signatures()
+	// FIXME: pass context.Context
+	unverifiedSignatures, err := image.Signatures(context.TODO())
 	if err != nil {
 		return nil, err
 	}

vendor/github.com/containers/image/signature/policy_eval_baselayer.go

@@ -3,8 +3,8 @@
 package signature
 
 import (
-	"github.com/Sirupsen/logrus"
 	"github.com/containers/image/types"
+	"github.com/sirupsen/logrus"
 )
 
 func (pr *prSignedBaseLayer) isSignatureAuthorAccepted(image types.UnparsedImage, sig []byte) (signatureAcceptanceResult, *Signature, error) {

vendor/github.com/containers/image/signature/policy_eval_signedby.go

@@ -3,6 +3,7 @@
 package signature
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"strings"
@@ -90,7 +91,8 @@ func (pr *prSignedBy) isSignatureAuthorAccepted(image types.UnparsedImage, sig [
 }
 
 func (pr *prSignedBy) isRunningImageAllowed(image types.UnparsedImage) (bool, error) {
-	sigs, err := image.Signatures()
+	// FIXME: pass context.Context
+	sigs, err := image.Signatures(context.TODO())
 	if err != nil {
 		return false, err
 	}

vendor/github.com/containers/image/signature/signature.go

@@ -180,13 +180,9 @@ func (s *untrustedSignature) strictUnmarshalJSON(data []byte) error {
 	}
 	s.UntrustedDockerManifestDigest = digest.Digest(digestString)
 
-	if err := paranoidUnmarshalJSONObjectExactFields(identity, map[string]interface{}{
+	return paranoidUnmarshalJSONObjectExactFields(identity, map[string]interface{}{
 		"docker-reference": &s.UntrustedDockerReference,
-	}); err != nil {
-		return err
-	}
-
-	return nil
+	})
 }
 
 // Sign formats the signature and returns a blob signed using mech and keyIdentity

vendor/github.com/containers/image/storage/storage_image.go

@@ -2,6 +2,7 @@ package storage
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"io"
 	"io/ioutil"
@@ -9,7 +10,6 @@ import (
 
 	"github.com/pkg/errors"
 
-	"github.com/Sirupsen/logrus"
 	"github.com/containers/image/image"
 	"github.com/containers/image/manifest"
 	"github.com/containers/image/types"
@@ -17,6 +17,7 @@ import (
 	"github.com/containers/storage/pkg/archive"
 	"github.com/containers/storage/pkg/ioutils"
 	ddigest "github.com/opencontainers/go-digest"
+	"github.com/sirupsen/logrus"
 )
 
 var (
@@ -537,7 +538,7 @@ func (s *storageImageSource) GetTargetManifest(digest ddigest.Digest) (manifestB
 	return nil, "", ErrNoManifestLists
 }
 
-func (s *storageImageSource) GetSignatures() (signatures [][]byte, err error) {
+func (s *storageImageSource) GetSignatures(ctx context.Context) (signatures [][]byte, err error) {
 	var offset int
 	signature, err := s.imageRef.transport.store.ImageBigData(s.ID, "signatures")
 	if err != nil {

vendor/github.com/containers/image/storage/storage_reference.go

@@ -3,11 +3,11 @@ package storage
 import (
 	"strings"
 
-	"github.com/Sirupsen/logrus"
 	"github.com/containers/image/docker/reference"
 	"github.com/containers/image/types"
 	"github.com/containers/storage"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 // A storageReference holds an arbitrary name and/or an ID, which is a 32-byte
@@ -154,7 +154,7 @@ func (s storageReference) DeleteImage(ctx *types.SystemContext) error {
 	return err
 }
 
-func (s storageReference) NewImageSource(ctx *types.SystemContext, requestedManifestMIMETypes []string) (types.ImageSource, error) {
+func (s storageReference) NewImageSource(ctx *types.SystemContext) (types.ImageSource, error) {
 	return newImageSource(s)
 }
 

vendor/github.com/containers/image/storage/storage_transport.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/pkg/errors"
 
-	"github.com/Sirupsen/logrus"
 	"github.com/containers/image/docker/reference"
 	"github.com/containers/image/transports"
 	"github.com/containers/image/types"
@@ -14,6 +13,7 @@ import (
 	"github.com/containers/storage/pkg/idtools"
 	"github.com/opencontainers/go-digest"
 	ddigest "github.com/opencontainers/go-digest"
+	"github.com/sirupsen/logrus"
 )
 
 func init() {

vendor/github.com/containers/image/transports/alltransports/alltransports.go

@@ -10,6 +10,7 @@ import (
 	_ "github.com/containers/image/docker"
 	_ "github.com/containers/image/docker/archive"
 	_ "github.com/containers/image/docker/daemon"
+	_ "github.com/containers/image/oci/archive"
 	_ "github.com/containers/image/oci/layout"
 	_ "github.com/containers/image/openshift"
 	// The ostree transport is registered by ostree*.go

vendor/github.com/containers/image/transports/transports.go

@@ -71,13 +71,19 @@ func ImageName(ref types.ImageReference) string {
 	return ref.Transport().Name() + ":" + ref.StringWithinTransport()
 }
 
-// ListNames returns a list of transport names
+// ListNames returns a list of non deprecated transport names.
+// Deprecated transports can be used, but are not presented to users.
 func ListNames() []string {
 	kt.mu.Lock()
 	defer kt.mu.Unlock()
+	deprecated := map[string]bool{
+		"atomic": true,
+	}
 	var names []string
 	for _, transport := range kt.transports {
-		names = append(names, transport.Name())
+		if !deprecated[transport.Name()] {
+			names = append(names, transport.Name())
+		}
 	}
 	sort.Strings(names)
 	return names

vendor/github.com/containers/image/types/types.go

@@ -1,6 +1,7 @@
 package types
 
 import (
+	"context"
 	"io"
 	"time"
 
@@ -77,11 +78,9 @@ type ImageReference interface {
 	// NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource,
 	// verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage.
 	NewImage(ctx *SystemContext) (Image, error)
-	// NewImageSource returns a types.ImageSource for this reference,
-	// asking the backend to use a manifest from requestedManifestMIMETypes if possible.
-	// nil requestedManifestMIMETypes means manifest.DefaultRequestedManifestMIMETypes.
+	// NewImageSource returns a types.ImageSource for this reference.
 	// The caller must call .Close() on the returned ImageSource.
-	NewImageSource(ctx *SystemContext, requestedManifestMIMETypes []string) (ImageSource, error)
+	NewImageSource(ctx *SystemContext) (ImageSource, error)
 	// NewImageDestination returns a types.ImageDestination for this reference.
 	// The caller must call .Close() on the returned ImageDestination.
 	NewImageDestination(ctx *SystemContext) (ImageDestination, error)
@@ -93,9 +92,10 @@ type ImageReference interface {
 // BlobInfo collects known information about a blob (layer/config).
 // In some situations, some fields may be unknown, in others they may be mandatory; documenting an “unknown” value here does not override that.
 type BlobInfo struct {
-	Digest digest.Digest // "" if unknown.
-	Size   int64         // -1 if unknown
-	URLs   []string
+	Digest      digest.Digest // "" if unknown.
+	Size        int64         // -1 if unknown
+	URLs        []string
+	Annotations map[string]string
 }
 
 // ImageSource is a service, possibly remote (= slow), to download components of a single image.
@@ -121,7 +121,7 @@ type ImageSource interface {
 	// The Digest field in BlobInfo is guaranteed to be provided; Size may be -1.
 	GetBlob(BlobInfo) (io.ReadCloser, int64, error)
 	// GetSignatures returns the image's signatures.  It may use a remote (= slow) service.
-	GetSignatures() ([][]byte, error)
+	GetSignatures(context.Context) ([][]byte, error)
 }
 
 // ImageDestination is a service, possibly remote (= slow), to store components of a single image.
@@ -204,7 +204,7 @@ type UnparsedImage interface {
 	// Manifest is like ImageSource.GetManifest, but the result is cached; it is OK to call this however often you need.
 	Manifest() ([]byte, string, error)
 	// Signatures is like ImageSource.GetSignatures, but the result is cached; it is OK to call this however often you need.
-	Signatures() ([][]byte, error)
+	Signatures(ctx context.Context) ([][]byte, error)
 }
 
 // Image is the primary API for inspecting properties of images.
@@ -302,6 +302,18 @@ type SystemContext struct {
 	SignaturePolicyPath string
 	// If not "", overrides the system's default path for registries.d (Docker signature storage configuration)
 	RegistriesDirPath string
+	// Path to the system-wide registries configuration file
+	SystemRegistriesConfPath string
+	// If not "", overrides the default path for the authentication file
+	AuthFilePath string
+
+	// === OCI.Transport overrides ===
+	// If not "", a directory containing a CA certificate (ending with ".crt"),
+	// a client certificate (ending with ".cert") and a client ceritificate key
+	// (ending with ".key") used when downloading OCI image layers.
+	OCICertPath string
+	// Allow downloading OCI image layers over HTTP, or HTTPS with failed TLS verification. Note that this does not affect other TLS connections.
+	OCIInsecureSkipTLSVerify bool
 
 	// === docker.Transport overrides ===
 	// If not "", a directory containing a CA certificate (ending with ".crt"),

vendor/github.com/containers/image/vendor.conf

@@ -1,9 +1,10 @@
-github.com/Sirupsen/logrus 7f4b1adc791766938c29457bed0703fb9134421a
-github.com/containers/storage 105f7c77aef0c797429e41552743bf5b03b63263
+github.com/sirupsen/logrus v1.0.0
+github.com/containers/storage 47536c89fcc545a87745e1a1573addc439409165
 github.com/davecgh/go-spew 346938d642f2ec3594ed81d874461961cd0faa76
-github.com/docker/distribution df5327f76fb6468b84a87771e361762b8be23fdb
-github.com/docker/docker 75843d36aa5c3eaade50da005f9e0ff2602f3d5e
-github.com/docker/go-connections 7da10c8c50cad14494ec818dcdfb6506265c0086
+github.com/docker/docker-credential-helpers d68f9aeca33f5fd3f08eeae5e9d175edf4e731d1
+github.com/docker/distribution 5f6282db7d65e6d72ad7c2cc66310724a57be716
+github.com/docker/docker 30eb4d8cdc422b023d5f11f29a82ecb73554183b
+github.com/docker/go-connections 3ede32e2033de7505e6500d6c868c2b9ed9f169d
 github.com/docker/go-units 0dadbb0345b35ec7ef35e228dabb8de89a65bf52
 github.com/docker/libtrust aabc10ec26b754e797f9028f4589c5b7bd90dc20
 github.com/ghodss/yaml 04f313413ffd65ce25f2541bfd2b2ceec5c0908c
@@ -24,7 +25,7 @@ github.com/stretchr/testify 4d4bfba8f1d1027c4fdbe371823030df51419987
 github.com/vbatts/tar-split bd4c5d64c3e9297f410025a3b1bd0c58f659e721
 golang.org/x/crypto 453249f01cfeb54c3d549ddb75ff152ca243f9d8
 golang.org/x/net 6b27048ae5e6ad1ef927e72e437531493de612fe
-golang.org/x/sys 075e574b89e4c2d22f2286a7e2b919519c6f3547
+golang.org/x/sys 43e60d72a8e2bd92ee98319ba9a384a0e9837c08
 gopkg.in/cheggaaa/pb.v1 d7e6ca3010b6f084d8056847f55d7f572f180678
 gopkg.in/yaml.v2 a3f3340b5840cee44f372bddb5880fcbc419b46a
 k8s.io/client-go bcde30fb7eaed76fd98a36b4120321b94995ffb6
@@ -35,3 +36,4 @@ github.com/tchap/go-patricia v2.2.6
 github.com/opencontainers/selinux ba1aefe8057f1d0cfb8e88d0ec1dc85925ef987d
 github.com/BurntSushi/toml b26d9c308763d68093482582cea63d69be07a0f0
 github.com/ostreedev/ostree-go aeb02c6b6aa2889db3ef62f7855650755befd460
+github.com/gogo/protobuf/proto fcdc5011193ff531a548e9b0301828d5a5b97fd8

vendor/github.com/containers/storage/containers.go

@@ -2,7 +2,6 @@ package storage
 
 import (
 	"encoding/json"
-	"errors"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -13,11 +12,6 @@ import (
 	"github.com/containers/storage/pkg/truncindex"
 )
 
-var (
-	// ErrContainerUnknown indicates that there was no container with the specified name or ID
-	ErrContainerUnknown = errors.New("container not known")
-)
-
 // A Container is a reference to a read-write layer with metadata.
 type Container struct {
 	// ID is either one which was specified at create-time, or a random
@@ -245,6 +239,7 @@ func (r *containerStore) Create(id string, names []string, image, layer, metadat
 	if _, idInUse := r.byid[id]; idInUse {
 		return nil, ErrDuplicateID
 	}
+	names = dedupeNames(names)
 	for _, name := range names {
 		if _, nameInUse := r.byname[name]; nameInUse {
 			return nil, ErrDuplicateName
@@ -294,6 +289,7 @@ func (r *containerStore) removeName(container *Container, name string) {
 }
 
 func (r *containerStore) SetNames(id string, names []string) error {
+	names = dedupeNames(names)
 	if container, ok := r.lookup(id); ok {
 		for _, name := range container.Names {
 			delete(r.byname, name)

vendor/github.com/containers/storage/drivers/aufs/aufs.go

@@ -25,6 +25,7 @@ package aufs
 import (
 	"bufio"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -32,22 +33,22 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
-	"syscall"
-
-	"github.com/Sirupsen/logrus"
-	"github.com/vbatts/tar-split/tar/storage"
+	"time"
 
 	"github.com/containers/storage/drivers"
 	"github.com/containers/storage/pkg/archive"
 	"github.com/containers/storage/pkg/chrootarchive"
 	"github.com/containers/storage/pkg/directory"
 	"github.com/containers/storage/pkg/idtools"
+	"github.com/containers/storage/pkg/locker"
 	mountpk "github.com/containers/storage/pkg/mount"
-	"github.com/containers/storage/pkg/stringid"
-
+	"github.com/containers/storage/pkg/system"
 	rsystem "github.com/opencontainers/runc/libcontainer/system"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"github.com/vbatts/tar-split/tar/storage"
+	"golang.org/x/sys/unix"
 )
 
 var (
@@ -74,6 +75,8 @@ type Driver struct {
 	ctr           *graphdriver.RefCounter
 	pathCacheLock sync.Mutex
 	pathCache     map[string]string
+	naiveDiff     graphdriver.DiffDriver
+	locker        *locker.Locker
 }
 
 // Init returns a new AUFS driver.
@@ -83,6 +86,7 @@ func Init(root string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 	// Try to load the aufs kernel module
 	if err := supportsAufs(); err != nil {
 		return nil, errors.Wrap(graphdriver.ErrNotSupported, "kernel does not support aufs")
+
 	}
 
 	fsMagic, err := graphdriver.GetFSMagic(root)
@@ -111,6 +115,7 @@ func Init(root string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		gidMaps:   gidMaps,
 		pathCache: make(map[string]string),
 		ctr:       graphdriver.NewRefCounter(graphdriver.NewFsChecker(graphdriver.FsMagicAufs)),
+		locker:    locker.New(),
 	}
 
 	rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)
@@ -137,6 +142,32 @@ func Init(root string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 			return nil, err
 		}
 	}
+	logger := logrus.WithFields(logrus.Fields{
+		"module": "graphdriver",
+		"driver": "aufs",
+	})
+
+	for _, path := range []string{"mnt", "diff"} {
+		p := filepath.Join(root, path)
+		entries, err := ioutil.ReadDir(p)
+		if err != nil {
+			logger.WithError(err).WithField("dir", p).Error("error reading dir entries")
+			continue
+		}
+		for _, entry := range entries {
+			if !entry.IsDir() {
+				continue
+			}
+			if strings.HasSuffix(entry.Name(), "-removing") {
+				logger.WithField("dir", entry.Name()).Debug("Cleaning up stale layer dir")
+				if err := system.EnsureRemoveAll(filepath.Join(p, entry.Name())); err != nil {
+					logger.WithField("dir", entry.Name()).WithError(err).Error("Error removing stale layer dir")
+				}
+			}
+		}
+	}
+
+	a.naiveDiff = graphdriver.NewNaiveDiffDriver(a, uidMaps, gidMaps)
 	return a, nil
 }
 
@@ -200,17 +231,22 @@ func (a *Driver) Exists(id string) bool {
 	return true
 }
 
+// AdditionalImageStores returns additional image stores supported by the driver
+func (a *Driver) AdditionalImageStores() []string {
+	return nil
+}
+
 // CreateReadWrite creates a layer that is writable for use as a container
 // file system.
-func (a *Driver) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error {
-	return a.Create(id, parent, mountLabel, storageOpt)
+func (a *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error {
+	return a.Create(id, parent, opts)
 }
 
 // Create three folders for each id
 // mnt, layers, and diff
-func (a *Driver) Create(id, parent, mountLabel string, storageOpt map[string]string) error {
+func (a *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error {
 
-	if len(storageOpt) != 0 {
+	if opts != nil && len(opts.StorageOpt) != 0 {
 		return fmt.Errorf("--storage-opt is not supported for aufs")
 	}
 
@@ -225,7 +261,7 @@ func (a *Driver) Create(id, parent, mountLabel string, storageOpt map[string]str
 	defer f.Close()
 
 	if parent != "" {
-		ids, err := getParentIds(a.rootPath(), parent)
+		ids, err := getParentIDs(a.rootPath(), parent)
 		if err != nil {
 			return err
 		}
@@ -268,35 +304,68 @@ func (a *Driver) createDirsFor(id string) error {
 
 // Remove will unmount and remove the given id.
 func (a *Driver) Remove(id string) error {
+	a.locker.Lock(id)
+	defer a.locker.Unlock(id)
 	a.pathCacheLock.Lock()
 	mountpoint, exists := a.pathCache[id]
 	a.pathCacheLock.Unlock()
 	if !exists {
 		mountpoint = a.getMountpoint(id)
 	}
-	if err := a.unmount(mountpoint); err != nil {
-		// no need to return here, we can still try to remove since the `Rename` will fail below if still mounted
-		logrus.Debugf("aufs: error while unmounting %s: %v", mountpoint, err)
-	}
 
-	// Atomically remove each directory in turn by first moving it out of the
-	// way (so that container runtimes don't find it anymore) before doing removal of
-	// the whole tree.
-	tmpMntPath := path.Join(a.mntPath(), fmt.Sprintf("%s-removing", id))
-	if err := os.Rename(mountpoint, tmpMntPath); err != nil && !os.IsNotExist(err) {
-		return err
-	}
-	defer os.RemoveAll(tmpMntPath)
+	logger := logrus.WithFields(logrus.Fields{
+		"module": "graphdriver",
+		"driver": "aufs",
+		"layer":  id,
+	})
 
-	tmpDiffpath := path.Join(a.diffPath(), fmt.Sprintf("%s-removing", id))
-	if err := os.Rename(a.getDiffPath(id), tmpDiffpath); err != nil && !os.IsNotExist(err) {
-		return err
+	var retries int
+	for {
+		mounted, err := a.mounted(mountpoint)
+		if err != nil {
+			if os.IsNotExist(err) {
+				break
+			}
+			return err
+		}
+		if !mounted {
+			break
+		}
+
+		err = a.unmount(mountpoint)
+		if err == nil {
+			break
+		}
+
+		if err != unix.EBUSY {
+			return errors.Wrapf(err, "aufs: unmount error: %s", mountpoint)
+		}
+		if retries >= 5 {
+			return errors.Wrapf(err, "aufs: unmount error after retries: %s", mountpoint)
+		}
+		// If unmount returns EBUSY, it could be a transient error. Sleep and retry.
+		retries++
+		logger.Warnf("unmount failed due to EBUSY: retry count: %d", retries)
+		time.Sleep(100 * time.Millisecond)
 	}
-	defer os.RemoveAll(tmpDiffpath)
 
 	// Remove the layers file for the id
 	if err := os.Remove(path.Join(a.rootPath(), "layers", id)); err != nil && !os.IsNotExist(err) {
-		return err
+		return errors.Wrapf(err, "error removing layers dir for %s", id)
+	}
+
+	if err := atomicRemove(a.getDiffPath(id)); err != nil {
+		return errors.Wrapf(err, "could not remove diff path for id %s", id)
+	}
+
+	// Atomically remove each directory in turn by first moving it out of the
+	// way (so that container runtime doesn't find it anymore) before doing removal of
+	// the whole tree.
+	if err := atomicRemove(mountpoint); err != nil {
+		if errors.Cause(err) == unix.EBUSY {
+			logger.WithField("dir", mountpoint).WithError(err).Warn("error performing atomic remove due to EBUSY")
+		}
+		return errors.Wrapf(err, "could not remove mountpoint for id %s", id)
 	}
 
 	a.pathCacheLock.Lock()
@@ -305,9 +374,29 @@ func (a *Driver) Remove(id string) error {
 	return nil
 }
 
+func atomicRemove(source string) error {
+	target := source + "-removing"
+
+	err := os.Rename(source, target)
+	switch {
+	case err == nil, os.IsNotExist(err):
+	case os.IsExist(err):
+		// Got error saying the target dir already exists, maybe the source doesn't exist due to a previous (failed) remove
+		if _, e := os.Stat(source); !os.IsNotExist(e) {
+			return errors.Wrapf(err, "target rename dir '%s' exists but should not, this needs to be manually cleaned up")
+		}
+	default:
+		return errors.Wrapf(err, "error preparing atomic delete")
+	}
+
+	return system.EnsureRemoveAll(target)
+}
+
 // Get returns the rootfs path for the id.
-// This will mount the dir at it's given path
+// This will mount the dir at its given path
 func (a *Driver) Get(id, mountLabel string) (string, error) {
+	a.locker.Lock(id)
+	defer a.locker.Unlock(id)
 	parents, err := a.getParentLayerPaths(id)
 	if err != nil && !os.IsNotExist(err) {
 		return "", err
@@ -343,6 +432,8 @@ func (a *Driver) Get(id, mountLabel string) (string, error) {
 
 // Put unmounts and updates list of active mounts.
 func (a *Driver) Put(id string) error {
+	a.locker.Lock(id)
+	defer a.locker.Unlock(id)
 	a.pathCacheLock.Lock()
 	m, exists := a.pathCache[id]
 	if !exists {
@@ -361,9 +452,22 @@ func (a *Driver) Put(id string) error {
 	return err
 }
 
+// isParent returns if the passed in parent is the direct parent of the passed in layer
+func (a *Driver) isParent(id, parent string) bool {
+	parents, _ := getParentIDs(a.rootPath(), id)
+	if parent == "" && len(parents) > 0 {
+		return false
+	}
+	return !(len(parents) > 0 && parent != parents[0])
+}
+
 // Diff produces an archive of the changes between the specified
 // layer and its parent layer which may be "".
-func (a *Driver) Diff(id, parent string) (archive.Archive, error) {
+func (a *Driver) Diff(id, parent string) (io.ReadCloser, error) {
+	if !a.isParent(id, parent) {
+		return a.naiveDiff.Diff(id, parent)
+	}
+
 	// AUFS doesn't need the parent layer to produce a diff.
 	return archive.TarWithOptions(path.Join(a.rootPath(), "diff", id), &archive.TarOptions{
 		Compression:     archive.Uncompressed,
@@ -373,12 +477,6 @@ func (a *Driver) Diff(id, parent string) (archive.Archive, error) {
 	})
 }
 
-// AdditionalImageStores returns additional image stores supported by the driver
-func (a *Driver) AdditionalImageStores() []string {
-	var imageStores []string
-	return imageStores
-}
-
 type fileGetNilCloser struct {
 	storage.FileGetter
 }
@@ -394,7 +492,7 @@ func (a *Driver) DiffGetter(id string) (graphdriver.FileGetCloser, error) {
 	return fileGetNilCloser{storage.NewPathFileGetter(p)}, nil
 }
 
-func (a *Driver) applyDiff(id string, diff archive.Reader) error {
+func (a *Driver) applyDiff(id string, diff io.Reader) error {
 	return chrootarchive.UntarUncompressed(diff, path.Join(a.rootPath(), "diff", id), &archive.TarOptions{
 		UIDMaps: a.uidMaps,
 		GIDMaps: a.gidMaps,
@@ -405,6 +503,9 @@ func (a *Driver) applyDiff(id string, diff archive.Reader) error {
 // and its parent and returns the size in bytes of the changes
 // relative to its base filesystem directory.
 func (a *Driver) DiffSize(id, parent string) (size int64, err error) {
+	if !a.isParent(id, parent) {
+		return a.naiveDiff.DiffSize(id, parent)
+	}
 	// AUFS doesn't need the parent layer to calculate the diff size.
 	return directory.Size(path.Join(a.rootPath(), "diff", id))
 }
@@ -412,8 +513,12 @@ func (a *Driver) DiffSize(id, parent string) (size int64, err error) {
 // ApplyDiff extracts the changeset from the given diff into the
 // layer with the specified id and parent, returning the size of the
 // new layer in bytes.
-func (a *Driver) ApplyDiff(id, parent string, diff archive.Reader) (size int64, err error) {
-	// AUFS doesn't need the parent id to apply the diff.
+func (a *Driver) ApplyDiff(id, parent string, diff io.Reader) (size int64, err error) {
+	if !a.isParent(id, parent) {
+		return a.naiveDiff.ApplyDiff(id, parent, diff)
+	}
+
+	// AUFS doesn't need the parent id to apply the diff if it is the direct parent.
 	if err = a.applyDiff(id, diff); err != nil {
 		return
 	}
@@ -424,6 +529,10 @@ func (a *Driver) ApplyDiff(id, parent string, diff archive.Reader) (size int64,
 // Changes produces a list of changes between the specified layer
 // and its parent layer. If parent is "", then all changes will be ADD changes.
 func (a *Driver) Changes(id, parent string) ([]archive.Change, error) {
+	if !a.isParent(id, parent) {
+		return a.naiveDiff.Changes(id, parent)
+	}
+
 	// AUFS doesn't have snapshots, so we need to get changes from all parent
 	// layers.
 	layers, err := a.getParentLayerPaths(id)
@@ -434,7 +543,7 @@ func (a *Driver) Changes(id, parent string) ([]archive.Change, error) {
 }
 
 func (a *Driver) getParentLayerPaths(id string) ([]string, error) {
-	parentIds, err := getParentIds(a.rootPath(), id)
+	parentIds, err := getParentIDs(a.rootPath(), id)
 	if err != nil {
 		return nil, err
 	}
@@ -499,7 +608,7 @@ func (a *Driver) Cleanup() error {
 
 	for _, m := range dirs {
 		if err := a.unmount(m); err != nil {
-			logrus.Debugf("aufs error unmounting %s: %s", stringid.TruncateID(m), err)
+			logrus.Debugf("aufs error unmounting %s: %s", m, err)
 		}
 	}
 	return mountpk.Unmount(a.root)
@@ -517,46 +626,35 @@ func (a *Driver) aufsMount(ro []string, rw, target, mountLabel string) (err erro
 
 	offset := 54
 	if useDirperm() {
-		offset += len("dirperm1")
+		offset += len(",dirperm1")
 	}
-	b := make([]byte, syscall.Getpagesize()-len(mountLabel)-offset) // room for xino & mountLabel
+	b := make([]byte, unix.Getpagesize()-len(mountLabel)-offset) // room for xino & mountLabel
 	bp := copy(b, fmt.Sprintf("br:%s=rw", rw))
 
-	firstMount := true
-	i := 0
-
-	for {
-		for ; i < len(ro); i++ {
-			layer := fmt.Sprintf(":%s=ro+wh", ro[i])
-
-			if firstMount {
-				if bp+len(layer) > len(b) {
-					break
-				}
-				bp += copy(b[bp:], layer)
-			} else {
-				data := label.FormatMountLabel(fmt.Sprintf("append%s", layer), mountLabel)
-				if err = mount("none", target, "aufs", syscall.MS_REMOUNT, data); err != nil {
-					return
-				}
-			}
-		}
-
-		if firstMount {
-			opts := "dio,xino=/dev/shm/aufs.xino"
-			if useDirperm() {
-				opts += ",dirperm1"
-			}
-			data := label.FormatMountLabel(fmt.Sprintf("%s,%s", string(b[:bp]), opts), mountLabel)
-			if err = mount("none", target, "aufs", 0, data); err != nil {
-				return
-			}
-			firstMount = false
-		}
-
-		if i == len(ro) {
+	index := 0
+	for ; index < len(ro); index++ {
+		layer := fmt.Sprintf(":%s=ro+wh", ro[index])
+		if bp+len(layer) > len(b) {
 			break
 		}
+		bp += copy(b[bp:], layer)
+	}
+
+	opts := "dio,xino=/dev/shm/aufs.xino"
+	if useDirperm() {
+		opts += ",dirperm1"
+	}
+	data := label.FormatMountLabel(fmt.Sprintf("%s,%s", string(b[:bp]), opts), mountLabel)
+	if err = mount("none", target, "aufs", 0, data); err != nil {
+		return
+	}
+
+	for ; index < len(ro); index++ {
+		layer := fmt.Sprintf(":%s=ro+wh", ro[index])
+		data := label.FormatMountLabel(fmt.Sprintf("append%s", layer), mountLabel)
+		if err = mount("none", target, "aufs", unix.MS_REMOUNT, data); err != nil {
+			return
+		}
 	}
 
 	return

vendor/github.com/containers/storage/drivers/aufs/dirs.go

@@ -29,7 +29,7 @@ func loadIds(root string) ([]string, error) {
 //
 // If there are no lines in the file then the id has no parent
 // and an empty slice is returned.
-func getParentIds(root, id string) ([]string, error) {
+func getParentIDs(root, id string) ([]string, error) {
 	f, err := os.Open(path.Join(root, "layers", id))
 	if err != nil {
 		return nil, err

vendor/github.com/containers/storage/drivers/aufs/mount.go

@@ -4,9 +4,9 @@ package aufs
 
 import (
 	"os/exec"
-	"syscall"
 
-	"github.com/Sirupsen/logrus"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
 )
 
 // Unmount the target specified.
@@ -14,7 +14,7 @@ func Unmount(target string) error {
 	if err := exec.Command("auplink", target, "flush").Run(); err != nil {
 		logrus.Warnf("Couldn't run auplink before unmount %s: %s", target, err)
 	}
-	if err := syscall.Unmount(target, 0); err != nil {
+	if err := unix.Unmount(target, 0); err != nil {
 		return err
 	}
 	return nil

vendor/github.com/containers/storage/drivers/aufs/mount_linux.go

@@ -1,7 +1,7 @@
 package aufs
 
-import "syscall"
+import "golang.org/x/sys/unix"
 
 func mount(source string, target string, fstype string, flags uintptr, data string) error {
-	return syscall.Mount(source, target, fstype, flags, data)
+	return unix.Mount(source, target, fstype, flags, data)
 }

vendor/github.com/containers/storage/drivers/aufs/mount_unsupported.go

@@ -2,7 +2,7 @@
 
 package aufs
 
-import "github.com/pkg/errors"
+import "errors"
 
 // MsRemount declared to specify a non-linux system mount.
 const MsRemount = 0

vendor/github.com/containers/storage/drivers/btrfs/btrfs.go

@@ -16,31 +16,32 @@ import "C"
 
 import (
 	"fmt"
+	"io/ioutil"
+	"math"
 	"os"
 	"path"
 	"path/filepath"
+	"strconv"
 	"strings"
-	"syscall"
+	"sync"
 	"unsafe"
 
 	"github.com/containers/storage/drivers"
 	"github.com/containers/storage/pkg/idtools"
 	"github.com/containers/storage/pkg/mount"
 	"github.com/containers/storage/pkg/parsers"
+	"github.com/containers/storage/pkg/system"
 	"github.com/docker/go-units"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
 )
 
 func init() {
 	graphdriver.Register("btrfs", Init)
 }
 
-var (
-	quotaEnabled  = false
-	userDiskQuota = false
-)
-
 type btrfsOptions struct {
 	minSpace uint64
 	size     uint64
@@ -71,18 +72,11 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		return nil, err
 	}
 
-	opt, err := parseOptions(options)
+	opt, userDiskQuota, err := parseOptions(options)
 	if err != nil {
 		return nil, err
 	}
 
-	if userDiskQuota {
-		if err := subvolEnableQuota(home); err != nil {
-			return nil, err
-		}
-		quotaEnabled = true
-	}
-
 	driver := &Driver{
 		home:    home,
 		uidMaps: uidMaps,
@@ -90,39 +84,48 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		options: opt,
 	}
 
+	if userDiskQuota {
+		if err := driver.subvolEnableQuota(); err != nil {
+			return nil, err
+		}
+	}
+
 	return graphdriver.NewNaiveDiffDriver(driver, uidMaps, gidMaps), nil
 }
 
-func parseOptions(opt []string) (btrfsOptions, error) {
+func parseOptions(opt []string) (btrfsOptions, bool, error) {
 	var options btrfsOptions
+	userDiskQuota := false
 	for _, option := range opt {
 		key, val, err := parsers.ParseKeyValueOpt(option)
 		if err != nil {
-			return options, err
+			return options, userDiskQuota, err
 		}
 		key = strings.ToLower(key)
 		switch key {
 		case "btrfs.min_space":
 			minSpace, err := units.RAMInBytes(val)
 			if err != nil {
-				return options, err
+				return options, userDiskQuota, err
 			}
 			userDiskQuota = true
 			options.minSpace = uint64(minSpace)
 		default:
-			return options, fmt.Errorf("Unknown option %s", key)
+			return options, userDiskQuota, fmt.Errorf("Unknown option %s", key)
 		}
 	}
-	return options, nil
+	return options, userDiskQuota, nil
 }
 
 // Driver contains information about the filesystem mounted.
 type Driver struct {
 	//root of the file system
-	home    string
-	uidMaps []idtools.IDMap
-	gidMaps []idtools.IDMap
-	options btrfsOptions
+	home         string
+	uidMaps      []idtools.IDMap
+	gidMaps      []idtools.IDMap
+	options      btrfsOptions
+	quotaEnabled bool
+	once         sync.Once
 }
 
 // String prints the name of the driver (btrfs).
@@ -151,10 +154,8 @@ func (d *Driver) Metadata(id string) (map[string]string, error) {
 
 // Cleanup unmounts the home directory.
 func (d *Driver) Cleanup() error {
-	if quotaEnabled {
-		if err := subvolDisableQuota(d.home); err != nil {
-			return err
-		}
+	if err := d.subvolDisableQuota(); err != nil {
+		return err
 	}
 
 	return mount.Unmount(d.home)
@@ -197,7 +198,7 @@ func subvolCreate(path, name string) error {
 		args.name[i] = C.char(c)
 	}
 
-	_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_SUBVOL_CREATE,
+	_, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_SUBVOL_CREATE,
 		uintptr(unsafe.Pointer(&args)))
 	if errno != 0 {
 		return fmt.Errorf("Failed to create btrfs subvolume: %v", errno.Error())
@@ -225,7 +226,7 @@ func subvolSnapshot(src, dest, name string) error {
 	C.set_name_btrfs_ioctl_vol_args_v2(&args, cs)
 	C.free(unsafe.Pointer(cs))
 
-	_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(destDir), C.BTRFS_IOC_SNAP_CREATE_V2,
+	_, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(destDir), C.BTRFS_IOC_SNAP_CREATE_V2,
 		uintptr(unsafe.Pointer(&args)))
 	if errno != 0 {
 		return fmt.Errorf("Failed to create btrfs snapshot: %v", errno.Error())
@@ -234,8 +235,8 @@ func subvolSnapshot(src, dest, name string) error {
 }
 
 func isSubvolume(p string) (bool, error) {
-	var bufStat syscall.Stat_t
-	if err := syscall.Lstat(p, &bufStat); err != nil {
+	var bufStat unix.Stat_t
+	if err := unix.Lstat(p, &bufStat); err != nil {
 		return false, err
 	}
 
@@ -243,7 +244,7 @@ func isSubvolume(p string) (bool, error) {
 	return bufStat.Ino == C.BTRFS_FIRST_FREE_OBJECTID, nil
 }
 
-func subvolDelete(dirpath, name string) error {
+func subvolDelete(dirpath, name string, quotaEnabled bool) error {
 	dir, err := openDir(dirpath)
 	if err != nil {
 		return err
@@ -271,7 +272,7 @@ func subvolDelete(dirpath, name string) error {
 				return fmt.Errorf("Failed to test if %s is a btrfs subvolume: %v", p, err)
 			}
 			if sv {
-				if err := subvolDelete(path.Dir(p), f.Name()); err != nil {
+				if err := subvolDelete(path.Dir(p), f.Name(), quotaEnabled); err != nil {
 					return fmt.Errorf("Failed to destroy btrfs child subvolume (%s) of parent (%s): %v", p, dirpath, err)
 				}
 			}
@@ -282,12 +283,27 @@ func subvolDelete(dirpath, name string) error {
 		return fmt.Errorf("Recursively walking subvolumes for %s failed: %v", dirpath, err)
 	}
 
+	if quotaEnabled {
+		if qgroupid, err := subvolLookupQgroup(fullPath); err == nil {
+			var args C.struct_btrfs_ioctl_qgroup_create_args
+			args.qgroupid = C.__u64(qgroupid)
+
+			_, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QGROUP_CREATE,
+				uintptr(unsafe.Pointer(&args)))
+			if errno != 0 {
+				logrus.Errorf("Failed to delete btrfs qgroup %v for %s: %v", qgroupid, fullPath, errno.Error())
+			}
+		} else {
+			logrus.Errorf("Failed to lookup btrfs qgroup for %s: %v", fullPath, err.Error())
+		}
+	}
+
 	// all subvolumes have been removed
 	// now remove the one originally passed in
 	for i, c := range []byte(name) {
 		args.name[i] = C.char(c)
 	}
-	_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_SNAP_DESTROY,
+	_, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_SNAP_DESTROY,
 		uintptr(unsafe.Pointer(&args)))
 	if errno != 0 {
 		return fmt.Errorf("Failed to destroy btrfs snapshot %s for %s: %v", dirpath, name, errno.Error())
@@ -295,8 +311,27 @@ func subvolDelete(dirpath, name string) error {
 	return nil
 }
 
-func subvolEnableQuota(path string) error {
-	dir, err := openDir(path)
+func (d *Driver) updateQuotaStatus() {
+	d.once.Do(func() {
+		if !d.quotaEnabled {
+			// In case quotaEnabled is not set, check qgroup and update quotaEnabled as needed
+			if err := subvolQgroupStatus(d.home); err != nil {
+				// quota is still not enabled
+				return
+			}
+			d.quotaEnabled = true
+		}
+	})
+}
+
+func (d *Driver) subvolEnableQuota() error {
+	d.updateQuotaStatus()
+
+	if d.quotaEnabled {
+		return nil
+	}
+
+	dir, err := openDir(d.home)
 	if err != nil {
 		return err
 	}
@@ -304,17 +339,25 @@ func subvolEnableQuota(path string) error {
 
 	var args C.struct_btrfs_ioctl_quota_ctl_args
 	args.cmd = C.BTRFS_QUOTA_CTL_ENABLE
-	_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QUOTA_CTL,
+	_, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QUOTA_CTL,
 		uintptr(unsafe.Pointer(&args)))
 	if errno != 0 {
 		return fmt.Errorf("Failed to enable btrfs quota for %s: %v", dir, errno.Error())
 	}
 
+	d.quotaEnabled = true
+
 	return nil
 }
 
-func subvolDisableQuota(path string) error {
-	dir, err := openDir(path)
+func (d *Driver) subvolDisableQuota() error {
+	d.updateQuotaStatus()
+
+	if !d.quotaEnabled {
+		return nil
+	}
+
+	dir, err := openDir(d.home)
 	if err != nil {
 		return err
 	}
@@ -322,24 +365,32 @@ func subvolDisableQuota(path string) error {
 
 	var args C.struct_btrfs_ioctl_quota_ctl_args
 	args.cmd = C.BTRFS_QUOTA_CTL_DISABLE
-	_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QUOTA_CTL,
+	_, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QUOTA_CTL,
 		uintptr(unsafe.Pointer(&args)))
 	if errno != 0 {
 		return fmt.Errorf("Failed to disable btrfs quota for %s: %v", dir, errno.Error())
 	}
 
+	d.quotaEnabled = false
+
 	return nil
 }
 
-func subvolRescanQuota(path string) error {
-	dir, err := openDir(path)
+func (d *Driver) subvolRescanQuota() error {
+	d.updateQuotaStatus()
+
+	if !d.quotaEnabled {
+		return nil
+	}
+
+	dir, err := openDir(d.home)
 	if err != nil {
 		return err
 	}
 	defer closeDir(dir)
 
 	var args C.struct_btrfs_ioctl_quota_rescan_args
-	_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QUOTA_RESCAN_WAIT,
+	_, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QUOTA_RESCAN_WAIT,
 		uintptr(unsafe.Pointer(&args)))
 	if errno != 0 {
 		return fmt.Errorf("Failed to rescan btrfs quota for %s: %v", dir, errno.Error())
@@ -358,7 +409,7 @@ func subvolLimitQgroup(path string, size uint64) error {
 	var args C.struct_btrfs_ioctl_qgroup_limit_args
 	args.lim.max_referenced = C.__u64(size)
 	args.lim.flags = C.BTRFS_QGROUP_LIMIT_MAX_RFER
-	_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QGROUP_LIMIT,
+	_, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QGROUP_LIMIT,
 		uintptr(unsafe.Pointer(&args)))
 	if errno != 0 {
 		return fmt.Errorf("Failed to limit qgroup for %s: %v", dir, errno.Error())
@@ -367,6 +418,60 @@ func subvolLimitQgroup(path string, size uint64) error {
 	return nil
 }
 
+// subvolQgroupStatus performs a BTRFS_IOC_TREE_SEARCH on the root path
+// with search key of BTRFS_QGROUP_STATUS_KEY.
+// In case qgroup is enabled, the retuned key type will match BTRFS_QGROUP_STATUS_KEY.
+// For more details please see https://github.com/kdave/btrfs-progs/blob/v4.9/qgroup.c#L1035
+func subvolQgroupStatus(path string) error {
+	dir, err := openDir(path)
+	if err != nil {
+		return err
+	}
+	defer closeDir(dir)
+
+	var args C.struct_btrfs_ioctl_search_args
+	args.key.tree_id = C.BTRFS_QUOTA_TREE_OBJECTID
+	args.key.min_type = C.BTRFS_QGROUP_STATUS_KEY
+	args.key.max_type = C.BTRFS_QGROUP_STATUS_KEY
+	args.key.max_objectid = C.__u64(math.MaxUint64)
+	args.key.max_offset = C.__u64(math.MaxUint64)
+	args.key.max_transid = C.__u64(math.MaxUint64)
+	args.key.nr_items = 4096
+
+	_, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_TREE_SEARCH,
+		uintptr(unsafe.Pointer(&args)))
+	if errno != 0 {
+		return fmt.Errorf("Failed to search qgroup for %s: %v", path, errno.Error())
+	}
+	sh := (*C.struct_btrfs_ioctl_search_header)(unsafe.Pointer(&args.buf))
+	if sh._type != C.BTRFS_QGROUP_STATUS_KEY {
+		return fmt.Errorf("Invalid qgroup search header type for %s: %v", path, sh._type)
+	}
+	return nil
+}
+
+func subvolLookupQgroup(path string) (uint64, error) {
+	dir, err := openDir(path)
+	if err != nil {
+		return 0, err
+	}
+	defer closeDir(dir)
+
+	var args C.struct_btrfs_ioctl_ino_lookup_args
+	args.objectid = C.BTRFS_FIRST_FREE_OBJECTID
+
+	_, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_INO_LOOKUP,
+		uintptr(unsafe.Pointer(&args)))
+	if errno != 0 {
+		return 0, fmt.Errorf("Failed to lookup qgroup for %s: %v", dir, errno.Error())
+	}
+	if args.treeid == 0 {
+		return 0, fmt.Errorf("Invalid qgroup id for %s: 0", dir)
+	}
+
+	return uint64(args.treeid), nil
+}
+
 func (d *Driver) subvolumesDir() string {
 	return path.Join(d.home, "subvolumes")
 }
@@ -375,14 +480,23 @@ func (d *Driver) subvolumesDirID(id string) string {
 	return path.Join(d.subvolumesDir(), id)
 }
 
+func (d *Driver) quotasDir() string {
+	return path.Join(d.home, "quotas")
+}
+
+func (d *Driver) quotasDirID(id string) string {
+	return path.Join(d.quotasDir(), id)
+}
+
 // CreateReadWrite creates a layer that is writable for use as a container
 // file system.
-func (d *Driver) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error {
-	return d.Create(id, parent, mountLabel, storageOpt)
+func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error {
+	return d.Create(id, parent, opts)
 }
 
 // Create the filesystem with given id.
-func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]string) error {
+func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error {
+	quotas := path.Join(d.home, "quotas")
 	subvolumes := path.Join(d.home, "subvolumes")
 	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
 	if err != nil {
@@ -409,14 +523,26 @@ func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]str
 		}
 	}
 
+	var storageOpt map[string]string
+	if opts != nil {
+		storageOpt = opts.StorageOpt
+	}
+
 	if _, ok := storageOpt["size"]; ok {
 		driver := &Driver{}
 		if err := d.parseStorageOpt(storageOpt, driver); err != nil {
 			return err
 		}
+
 		if err := d.setStorageSize(path.Join(subvolumes, id), driver); err != nil {
 			return err
 		}
+		if err := idtools.MkdirAllAs(quotas, 0700, rootUID, rootGID); err != nil {
+			return err
+		}
+		if err := ioutil.WriteFile(path.Join(quotas, id), []byte(fmt.Sprint(driver.options.size)), 0644); err != nil {
+			return err
+		}
 	}
 
 	// if we have a remapped root (user namespaces enabled), change the created snapshot
@@ -427,6 +553,11 @@ func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]str
 		}
 	}
 
+	mountLabel := ""
+	if opts != nil {
+		mountLabel = opts.MountLabel
+	}
+
 	return label.Relabel(path.Join(subvolumes, id), mountLabel, false)
 }
 
@@ -459,11 +590,8 @@ func (d *Driver) setStorageSize(dir string, driver *Driver) error {
 		return fmt.Errorf("btrfs: storage size cannot be less than %s", units.HumanSize(float64(d.options.minSpace)))
 	}
 
-	if !quotaEnabled {
-		if err := subvolEnableQuota(d.home); err != nil {
-			return err
-		}
-		quotaEnabled = true
+	if err := d.subvolEnableQuota(); err != nil {
+		return err
 	}
 
 	if err := subvolLimitQgroup(dir, driver.options.size); err != nil {
@@ -479,13 +607,25 @@ func (d *Driver) Remove(id string) error {
 	if _, err := os.Stat(dir); err != nil {
 		return err
 	}
-	if err := subvolDelete(d.subvolumesDir(), id); err != nil {
+	quotasDir := d.quotasDirID(id)
+	if _, err := os.Stat(quotasDir); err == nil {
+		if err := os.Remove(quotasDir); err != nil {
+			return err
+		}
+	} else if !os.IsNotExist(err) {
 		return err
 	}
-	if err := os.RemoveAll(dir); err != nil && !os.IsNotExist(err) {
+
+	// Call updateQuotaStatus() to invoke status update
+	d.updateQuotaStatus()
+
+	if err := subvolDelete(d.subvolumesDir(), id, d.quotaEnabled); err != nil {
 		return err
 	}
-	if err := subvolRescanQuota(d.home); err != nil {
+	if err := system.EnsureRemoveAll(dir); err != nil {
+		return err
+	}
+	if err := d.subvolRescanQuota(); err != nil {
 		return err
 	}
 	return nil
@@ -503,6 +643,17 @@ func (d *Driver) Get(id, mountLabel string) (string, error) {
 		return "", fmt.Errorf("%s: not a directory", dir)
 	}
 
+	if quota, err := ioutil.ReadFile(d.quotasDirID(id)); err == nil {
+		if size, err := strconv.ParseUint(string(quota), 10, 64); err == nil && size >= d.options.minSpace {
+			if err := d.subvolEnableQuota(); err != nil {
+				return "", err
+			}
+			if err := subvolLimitQgroup(dir, size); err != nil {
+				return "", err
+			}
+		}
+	}
+
 	return dir, nil
 }
 
@@ -522,6 +673,5 @@ func (d *Driver) Exists(id string) bool {
 
 // AdditionalImageStores returns additional image stores supported by the driver
 func (d *Driver) AdditionalImageStores() []string {
-	var imageStores []string
-	return imageStores
+	return nil
 }

vendor/github.com/containers/storage/drivers/counter.go

@@ -22,30 +22,21 @@ func NewRefCounter(c Checker) *RefCounter {
 	}
 }
 
-// Increment increaes the ref count for the given id and returns the current count
+// Increment increases the ref count for the given id and returns the current count
 func (c *RefCounter) Increment(path string) int {
-	c.mu.Lock()
-	m := c.counts[path]
-	if m == nil {
-		m = &minfo{}
-		c.counts[path] = m
-	}
-	// if we are checking this path for the first time check to make sure
-	// if it was already mounted on the system and make sure we have a correct ref
-	// count if it is mounted as it is in use.
-	if !m.check {
-		m.check = true
-		if c.checker.IsMounted(path) {
-			m.count++
-		}
-	}
-	m.count++
-	c.mu.Unlock()
-	return m.count
+	return c.incdec(path, func(minfo *minfo) {
+		minfo.count++
+	})
 }
 
 // Decrement decreases the ref count for the given id and returns the current count
 func (c *RefCounter) Decrement(path string) int {
+	return c.incdec(path, func(minfo *minfo) {
+		minfo.count--
+	})
+}
+
+func (c *RefCounter) incdec(path string, infoOp func(minfo *minfo)) int {
 	c.mu.Lock()
 	m := c.counts[path]
 	if m == nil {
@@ -61,7 +52,8 @@ func (c *RefCounter) Decrement(path string) int {
 			m.count++
 		}
 	}
-	m.count--
+	infoOp(m)
+	count := m.count
 	c.mu.Unlock()
-	return m.count
+	return count
 }

vendor/github.com/containers/storage/drivers/devmapper/device_setup.go

@@ -0,0 +1,236 @@
+package devmapper
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"reflect"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+type directLVMConfig struct {
+	Device              string
+	ThinpPercent        uint64
+	ThinpMetaPercent    uint64
+	AutoExtendPercent   uint64
+	AutoExtendThreshold uint64
+}
+
+var (
+	errThinpPercentMissing = errors.New("must set both `dm.thinp_percent` and `dm.thinp_metapercent` if either is specified")
+	errThinpPercentTooBig  = errors.New("combined `dm.thinp_percent` and `dm.thinp_metapercent` must not be greater than 100")
+	errMissingSetupDevice  = errors.New("must provide device path in `dm.setup_device` in order to configure direct-lvm")
+)
+
+func validateLVMConfig(cfg directLVMConfig) error {
+	if reflect.DeepEqual(cfg, directLVMConfig{}) {
+		return nil
+	}
+	if cfg.Device == "" {
+		return errMissingSetupDevice
+	}
+	if (cfg.ThinpPercent > 0 && cfg.ThinpMetaPercent == 0) || cfg.ThinpMetaPercent > 0 && cfg.ThinpPercent == 0 {
+		return errThinpPercentMissing
+	}
+
+	if cfg.ThinpPercent+cfg.ThinpMetaPercent > 100 {
+		return errThinpPercentTooBig
+	}
+	return nil
+}
+
+func checkDevAvailable(dev string) error {
+	lvmScan, err := exec.LookPath("lvmdiskscan")
+	if err != nil {
+		logrus.Debug("could not find lvmdiskscan")
+		return nil
+	}
+
+	out, err := exec.Command(lvmScan).CombinedOutput()
+	if err != nil {
+		logrus.WithError(err).Error(string(out))
+		return nil
+	}
+
+	if !bytes.Contains(out, []byte(dev)) {
+		return errors.Errorf("%s is not available for use with devicemapper", dev)
+	}
+	return nil
+}
+
+func checkDevInVG(dev string) error {
+	pvDisplay, err := exec.LookPath("pvdisplay")
+	if err != nil {
+		logrus.Debug("could not find pvdisplay")
+		return nil
+	}
+
+	out, err := exec.Command(pvDisplay, dev).CombinedOutput()
+	if err != nil {
+		logrus.WithError(err).Error(string(out))
+		return nil
+	}
+
+	scanner := bufio.NewScanner(bytes.NewReader(bytes.TrimSpace(out)))
+	for scanner.Scan() {
+		fields := strings.SplitAfter(strings.TrimSpace(scanner.Text()), "VG Name")
+		if len(fields) > 1 {
+			// got "VG Name" line"
+			vg := strings.TrimSpace(fields[1])
+			if len(vg) > 0 {
+				return errors.Errorf("%s is already part of a volume group %q: must remove this device from any volume group or provide a different device", dev, vg)
+			}
+			logrus.Error(fields)
+			break
+		}
+	}
+	return nil
+}
+
+func checkDevHasFS(dev string) error {
+	blkid, err := exec.LookPath("blkid")
+	if err != nil {
+		logrus.Debug("could not find blkid")
+		return nil
+	}
+
+	out, err := exec.Command(blkid, dev).CombinedOutput()
+	if err != nil {
+		logrus.WithError(err).Error(string(out))
+		return nil
+	}
+
+	fields := bytes.Fields(out)
+	for _, f := range fields {
+		kv := bytes.Split(f, []byte{'='})
+		if bytes.Equal(kv[0], []byte("TYPE")) {
+			v := bytes.Trim(kv[1], "\"")
+			if len(v) > 0 {
+				return errors.Errorf("%s has a filesystem already, use dm.directlvm_device_force=true if you want to wipe the device", dev)
+			}
+			return nil
+		}
+	}
+	return nil
+}
+
+func verifyBlockDevice(dev string, force bool) error {
+	if err := checkDevAvailable(dev); err != nil {
+		return err
+	}
+	if err := checkDevInVG(dev); err != nil {
+		return err
+	}
+
+	if force {
+		return nil
+	}
+
+	if err := checkDevHasFS(dev); err != nil {
+		return err
+	}
+	return nil
+}
+
+func readLVMConfig(root string) (directLVMConfig, error) {
+	var cfg directLVMConfig
+
+	p := filepath.Join(root, "setup-config.json")
+	b, err := ioutil.ReadFile(p)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return cfg, nil
+		}
+		return cfg, errors.Wrap(err, "error reading existing setup config")
+	}
+
+	// check if this is just an empty file, no need to produce a json error later if so
+	if len(b) == 0 {
+		return cfg, nil
+	}
+
+	err = json.Unmarshal(b, &cfg)
+	return cfg, errors.Wrap(err, "error unmarshaling previous device setup config")
+}
+
+func writeLVMConfig(root string, cfg directLVMConfig) error {
+	p := filepath.Join(root, "setup-config.json")
+	b, err := json.Marshal(cfg)
+	if err != nil {
+		return errors.Wrap(err, "error marshalling direct lvm config")
+	}
+	err = ioutil.WriteFile(p, b, 0600)
+	return errors.Wrap(err, "error writing direct lvm config to file")
+}
+
+func setupDirectLVM(cfg directLVMConfig) error {
+	lvmProfileDir := "/etc/lvm/profile"
+	binaries := []string{"pvcreate", "vgcreate", "lvcreate", "lvconvert", "lvchange", "thin_check"}
+
+	for _, bin := range binaries {
+		if _, err := exec.LookPath(bin); err != nil {
+			return errors.Wrap(err, "error looking up command `"+bin+"` while setting up direct lvm")
+		}
+	}
+
+	err := os.MkdirAll(lvmProfileDir, 0755)
+	if err != nil {
+		return errors.Wrap(err, "error creating lvm profile directory")
+	}
+
+	if cfg.AutoExtendPercent == 0 {
+		cfg.AutoExtendPercent = 20
+	}
+
+	if cfg.AutoExtendThreshold == 0 {
+		cfg.AutoExtendThreshold = 80
+	}
+
+	if cfg.ThinpPercent == 0 {
+		cfg.ThinpPercent = 95
+	}
+	if cfg.ThinpMetaPercent == 0 {
+		cfg.ThinpMetaPercent = 1
+	}
+
+	out, err := exec.Command("pvcreate", "-f", cfg.Device).CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, string(out))
+	}
+
+	out, err = exec.Command("vgcreate", "storage", cfg.Device).CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, string(out))
+	}
+
+	out, err = exec.Command("lvcreate", "--wipesignatures", "y", "-n", "thinpool", "storage", "--extents", fmt.Sprintf("%d%%VG", cfg.ThinpPercent)).CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, string(out))
+	}
+	out, err = exec.Command("lvcreate", "--wipesignatures", "y", "-n", "thinpoolmeta", "storage", "--extents", fmt.Sprintf("%d%%VG", cfg.ThinpMetaPercent)).CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, string(out))
+	}
+
+	out, err = exec.Command("lvconvert", "-y", "--zero", "n", "-c", "512K", "--thinpool", "storage/thinpool", "--poolmetadata", "storage/thinpoolmeta").CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, string(out))
+	}
+
+	profile := fmt.Sprintf("activation{\nthin_pool_autoextend_threshold=%d\nthin_pool_autoextend_percent=%d\n}", cfg.AutoExtendThreshold, cfg.AutoExtendPercent)
+	err = ioutil.WriteFile(lvmProfileDir+"/storage-thinpool.profile", []byte(profile), 0600)
+	if err != nil {
+		return errors.Wrap(err, "error writing storage thinp autoextend profile")
+	}
+
+	out, err = exec.Command("lvchange", "--metadataprofile", "storage-thinpool", "storage/thinpool").CombinedOutput()
+	return errors.Wrap(err, string(out))
+}

vendor/github.com/containers/storage/drivers/devmapper/deviceset.go

@@ -12,44 +12,41 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"reflect"
 	"strconv"
 	"strings"
 	"sync"
-	"syscall"
 	"time"
 
-	"github.com/Sirupsen/logrus"
-
 	"github.com/containers/storage/drivers"
 	"github.com/containers/storage/pkg/devicemapper"
+	"github.com/containers/storage/pkg/dmesg"
 	"github.com/containers/storage/pkg/idtools"
-	"github.com/containers/storage/pkg/ioutils"
 	"github.com/containers/storage/pkg/loopback"
 	"github.com/containers/storage/pkg/mount"
 	"github.com/containers/storage/pkg/parsers"
-	"github.com/docker/go-units"
-
+	"github.com/containers/storage/pkg/parsers/kernel"
+	units "github.com/docker/go-units"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
 )
 
 var (
-	defaultDataLoopbackSize     int64  = 100 * 1024 * 1024 * 1024
-	defaultMetaDataLoopbackSize int64  = 2 * 1024 * 1024 * 1024
-	defaultBaseFsSize           uint64 = 10 * 1024 * 1024 * 1024
-	defaultThinpBlockSize       uint32 = 128 // 64K = 128 512b sectors
-	defaultUdevSyncOverride            = false
-	maxDeviceID                        = 0xffffff // 24 bit, pool limit
-	deviceIDMapSz                      = (maxDeviceID + 1) / 8
-	// We retry device removal so many a times that even error messages
-	// will fill up console during normal operation. So only log Fatal
-	// messages by default.
-	logLevel                            = devicemapper.LogLevelFatal
+	defaultDataLoopbackSize      int64  = 100 * 1024 * 1024 * 1024
+	defaultMetaDataLoopbackSize  int64  = 2 * 1024 * 1024 * 1024
+	defaultBaseFsSize            uint64 = 10 * 1024 * 1024 * 1024
+	defaultThinpBlockSize        uint32 = 128 // 64K = 128 512b sectors
+	defaultUdevSyncOverride             = false
+	maxDeviceID                         = 0xffffff // 24 bit, pool limit
+	deviceIDMapSz                       = (maxDeviceID + 1) / 8
 	driverDeferredRemovalSupport        = false
 	enableDeferredRemoval               = false
 	enableDeferredDeletion              = false
 	userBaseSize                        = false
 	defaultMinFreeSpacePercent   uint32 = 10
+	lvmSetupConfigForce          bool
 )
 
 const deviceSetMetaFile string = "deviceset-metadata"
@@ -122,6 +119,8 @@ type DeviceSet struct {
 	uidMaps               []idtools.IDMap
 	gidMaps               []idtools.IDMap
 	minFreeSpacePercent   uint32 //min free space percentage in thinpool
+	xfsNospaceRetries     string // max retries when xfs receives ENOSPC
+	lvmSetupConfig        directLVMConfig
 }
 
 // DiskUsage contains information about disk usage and is used when reporting Status of a device.
@@ -170,7 +169,7 @@ type Status struct {
 	MinFreeSpace               uint64
 }
 
-// Structure used to export image/container metadata in docker inspect.
+// Structure used to export image/container metadata in inspect.
 type deviceMetadata struct {
 	deviceID   int
 	deviceSize uint64 // size in bytes
@@ -379,10 +378,7 @@ func (devices *DeviceSet) isDeviceIDFree(deviceID int) bool {
 	var mask byte
 	i := deviceID % 8
 	mask = (1 << uint(i))
-	if (devices.deviceIDMap[deviceID/8] & mask) != 0 {
-		return false
-	}
-	return true
+	return (devices.deviceIDMap[deviceID/8] & mask) == 0
 }
 
 // Should be called with devices.Lock() held.
@@ -409,8 +405,8 @@ func (devices *DeviceSet) lookupDeviceWithLock(hash string) (*devInfo, error) {
 // This function relies on that device hash map has been loaded in advance.
 // Should be called with devices.Lock() held.
 func (devices *DeviceSet) constructDeviceIDMap() {
-	logrus.Debugf("devmapper: constructDeviceIDMap()")
-	defer logrus.Debugf("devmapper: constructDeviceIDMap() END")
+	logrus.Debug("devmapper: constructDeviceIDMap()")
+	defer logrus.Debug("devmapper: constructDeviceIDMap() END")
 
 	for _, info := range devices.Devices {
 		devices.markDeviceIDUsed(info.DeviceID)
@@ -458,8 +454,8 @@ func (devices *DeviceSet) deviceFileWalkFunction(path string, finfo os.FileInfo)
 }
 
 func (devices *DeviceSet) loadDeviceFilesOnStart() error {
-	logrus.Debugf("devmapper: loadDeviceFilesOnStart()")
-	defer logrus.Debugf("devmapper: loadDeviceFilesOnStart() END")
+	logrus.Debug("devmapper: loadDeviceFilesOnStart()")
+	defer logrus.Debug("devmapper: loadDeviceFilesOnStart() END")
 
 	var scan = func(path string, info os.FileInfo, err error) error {
 		if err != nil {
@@ -479,11 +475,10 @@ func (devices *DeviceSet) loadDeviceFilesOnStart() error {
 }
 
 // Should be called with devices.Lock() held.
-func (devices *DeviceSet) unregisterDevice(id int, hash string) error {
-	logrus.Debugf("devmapper: unregisterDevice(%v, %v)", id, hash)
+func (devices *DeviceSet) unregisterDevice(hash string) error {
+	logrus.Debugf("devmapper: unregisterDevice(%v)", hash)
 	info := &devInfo{
-		Hash:     hash,
-		DeviceID: id,
+		Hash: hash,
 	}
 
 	delete(devices.Devices, hash)
@@ -528,7 +523,7 @@ func (devices *DeviceSet) activateDeviceIfNeeded(info *devInfo, ignoreDeleted bo
 
 	// Make sure deferred removal on device is canceled, if one was
 	// scheduled.
-	if err := devices.cancelDeferredRemoval(info); err != nil {
+	if err := devices.cancelDeferredRemovalIfNeeded(info); err != nil {
 		return fmt.Errorf("devmapper: Device Deferred Removal Cancellation Failed: %s", err)
 	}
 
@@ -539,11 +534,11 @@ func (devices *DeviceSet) activateDeviceIfNeeded(info *devInfo, ignoreDeleted bo
 	return devicemapper.ActivateDevice(devices.getPoolDevName(), info.Name(), info.DeviceID, info.Size)
 }
 
-// Return true only if kernel supports xfs and mkfs.xfs is available
-func xfsSupported() bool {
+// xfsSupported checks if xfs is supported, returns nil if it is, otherwise an error
+func xfsSupported() error {
 	// Make sure mkfs.xfs is available
 	if _, err := exec.LookPath("mkfs.xfs"); err != nil {
-		return false
+		return err // error text is descriptive enough
 	}
 
 	// Check if kernel supports xfs filesystem or not.
@@ -551,43 +546,48 @@ func xfsSupported() bool {
 
 	f, err := os.Open("/proc/filesystems")
 	if err != nil {
-		logrus.Warnf("devmapper: Could not check if xfs is supported: %v", err)
-		return false
+		return errors.Wrapf(err, "error checking for xfs support")
 	}
 	defer f.Close()
 
 	s := bufio.NewScanner(f)
 	for s.Scan() {
 		if strings.HasSuffix(s.Text(), "\txfs") {
-			return true
+			return nil
 		}
 	}
 
 	if err := s.Err(); err != nil {
-		logrus.Warnf("devmapper: Could not check if xfs is supported: %v", err)
+		return errors.Wrapf(err, "error checking for xfs support")
 	}
-	return false
+
+	return errors.New(`kernel does not support xfs, or "modprobe xfs" failed`)
 }
 
 func determineDefaultFS() string {
-	if xfsSupported() {
+	err := xfsSupported()
+	if err == nil {
 		return "xfs"
 	}
 
-	logrus.Warn("devmapper: XFS is not supported in your system. Either the kernel doesn't support it or mkfs.xfs is not in your PATH. Defaulting to ext4 filesystem")
+	logrus.Warnf("devmapper: XFS is not supported in your system (%v). Defaulting to ext4 filesystem", err)
 	return "ext4"
 }
 
+// mkfsOptions tries to figure out whether some additional mkfs options are required
+func mkfsOptions(fs string) []string {
+	if fs == "xfs" && !kernel.CheckKernelVersion(3, 16, 0) {
+		// For kernels earlier than 3.16 (and newer xfsutils),
+		// some xfs features need to be explicitly disabled.
+		return []string{"-m", "crc=0,finobt=0"}
+	}
+
+	return []string{}
+}
+
 func (devices *DeviceSet) createFilesystem(info *devInfo) (err error) {
 	devname := info.DevName()
 
-	args := []string{}
-	for _, arg := range devices.mkfsArgs {
-		args = append(args, arg)
-	}
-
-	args = append(args, devname)
-
 	if devices.filesystem == "" {
 		devices.filesystem = determineDefaultFS()
 	}
@@ -595,7 +595,11 @@ func (devices *DeviceSet) createFilesystem(info *devInfo) (err error) {
 		return err
 	}
 
-	logrus.Infof("devmapper: Creating filesystem %s on device %s", devices.filesystem, info.Name())
+	args := mkfsOptions(devices.filesystem)
+	args = append(args, devices.mkfsArgs...)
+	args = append(args, devname)
+
+	logrus.Infof("devmapper: Creating filesystem %s on device %s, mkfs args: %v", devices.filesystem, info.Name(), args)
 	defer func() {
 		if err != nil {
 			logrus.Infof("devmapper: Error while creating filesystem %s on device %s: %v", devices.filesystem, info.Name(), err)
@@ -833,7 +837,7 @@ func (devices *DeviceSet) createRegisterDevice(hash string) (*devInfo, error) {
 	}
 
 	if err := devices.closeTransaction(); err != nil {
-		devices.unregisterDevice(deviceID, hash)
+		devices.unregisterDevice(hash)
 		devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID)
 		devices.markDeviceIDFree(deviceID)
 		return nil, err
@@ -841,11 +845,57 @@ func (devices *DeviceSet) createRegisterDevice(hash string) (*devInfo, error) {
 	return info, nil
 }
 
-func (devices *DeviceSet) createRegisterSnapDevice(hash string, baseInfo *devInfo, size uint64) error {
-	if err := devices.poolHasFreeSpace(); err != nil {
+func (devices *DeviceSet) takeSnapshot(hash string, baseInfo *devInfo, size uint64) error {
+	var (
+		devinfo *devicemapper.Info
+		err     error
+	)
+
+	if err = devices.poolHasFreeSpace(); err != nil {
 		return err
 	}
 
+	if devices.deferredRemove {
+		devinfo, err = devicemapper.GetInfoWithDeferred(baseInfo.Name())
+		if err != nil {
+			return err
+		}
+		if devinfo != nil && devinfo.DeferredRemove != 0 {
+			err = devices.cancelDeferredRemoval(baseInfo)
+			if err != nil {
+				// If Error is ErrEnxio. Device is probably already gone. Continue.
+				if errors.Cause(err) != devicemapper.ErrEnxio {
+					return err
+				}
+				devinfo = nil
+			} else {
+				defer devices.deactivateDevice(baseInfo)
+			}
+		}
+	} else {
+		devinfo, err = devicemapper.GetInfo(baseInfo.Name())
+		if err != nil {
+			return err
+		}
+	}
+
+	doSuspend := devinfo != nil && devinfo.Exists != 0
+
+	if doSuspend {
+		if err = devicemapper.SuspendDevice(baseInfo.Name()); err != nil {
+			return err
+		}
+		defer devicemapper.ResumeDevice(baseInfo.Name())
+	}
+
+	if err = devices.createRegisterSnapDevice(hash, baseInfo, size); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (devices *DeviceSet) createRegisterSnapDevice(hash string, baseInfo *devInfo, size uint64) error {
 	deviceID, err := devices.getNextFreeDeviceID()
 	if err != nil {
 		return err
@@ -858,7 +908,7 @@ func (devices *DeviceSet) createRegisterSnapDevice(hash string, baseInfo *devInf
 	}
 
 	for {
-		if err := devicemapper.CreateSnapDevice(devices.getPoolDevName(), deviceID, baseInfo.Name(), baseInfo.DeviceID); err != nil {
+		if err := devicemapper.CreateSnapDeviceRaw(devices.getPoolDevName(), deviceID, baseInfo.DeviceID); err != nil {
 			if devicemapper.DeviceIDExists(err) {
 				// Device ID already exists. This should not
 				// happen. Now we have a mechanism to find
@@ -888,7 +938,7 @@ func (devices *DeviceSet) createRegisterSnapDevice(hash string, baseInfo *devInf
 	}
 
 	if err := devices.closeTransaction(); err != nil {
-		devices.unregisterDevice(deviceID, hash)
+		devices.unregisterDevice(hash)
 		devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID)
 		devices.markDeviceIDFree(deviceID)
 		return err
@@ -1134,7 +1184,7 @@ func (devices *DeviceSet) growFS(info *devInfo) error {
 
 	defer devices.deactivateDevice(info)
 
-	fsMountPoint := "/run/containers/mnt"
+	fsMountPoint := "/run/containers/storage/mnt"
 	if _, err := os.Stat(fsMountPoint); os.IsNotExist(err) {
 		if err := os.MkdirAll(fsMountPoint, 0700); err != nil {
 			return err
@@ -1150,10 +1200,10 @@ func (devices *DeviceSet) growFS(info *devInfo) error {
 	options = joinMountOptions(options, devices.mountOptions)
 
 	if err := mount.Mount(info.DevName(), fsMountPoint, devices.BaseDeviceFilesystem, options); err != nil {
-		return fmt.Errorf("Error mounting '%s' on '%s': %s", info.DevName(), fsMountPoint, err)
+		return fmt.Errorf("Error mounting '%s' on '%s': %s\n%v", info.DevName(), fsMountPoint, err, string(dmesg.Dmesg(256)))
 	}
 
-	defer syscall.Unmount(fsMountPoint, syscall.MNT_DETACH)
+	defer unix.Unmount(fsMountPoint, unix.MNT_DETACH)
 
 	switch devices.BaseDeviceFilesystem {
 	case "ext4":
@@ -1216,39 +1266,18 @@ func (devices *DeviceSet) setupBaseImage() error {
 }
 
 func setCloseOnExec(name string) {
-	if fileInfos, _ := ioutil.ReadDir("/proc/self/fd"); fileInfos != nil {
-		for _, i := range fileInfos {
-			link, _ := os.Readlink(filepath.Join("/proc/self/fd", i.Name()))
-			if link == name {
-				fd, err := strconv.Atoi(i.Name())
-				if err == nil {
-					syscall.CloseOnExec(fd)
-				}
+	fileInfos, _ := ioutil.ReadDir("/proc/self/fd")
+	for _, i := range fileInfos {
+		link, _ := os.Readlink(filepath.Join("/proc/self/fd", i.Name()))
+		if link == name {
+			fd, err := strconv.Atoi(i.Name())
+			if err == nil {
+				unix.CloseOnExec(fd)
 			}
 		}
 	}
 }
 
-// DMLog implements logging using DevMapperLogger interface.
-func (devices *DeviceSet) DMLog(level int, file string, line int, dmError int, message string) {
-	// By default libdm sends us all the messages including debug ones.
-	// We need to filter out messages here and figure out which one
-	// should be printed.
-	if level > logLevel {
-		return
-	}
-
-	// FIXME(vbatts) push this back into ./pkg/devicemapper/
-	if level <= devicemapper.LogLevelErr {
-		logrus.Errorf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message)
-	} else if level <= devicemapper.LogLevelInfo {
-		logrus.Infof("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message)
-	} else {
-		// FIXME(vbatts) push this back into ./pkg/devicemapper/
-		logrus.Debugf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message)
-	}
-}
-
 func major(device uint64) uint64 {
 	return (device >> 8) & 0xfff
 }
@@ -1356,10 +1385,7 @@ func (devices *DeviceSet) saveTransactionMetaData() error {
 }
 
 func (devices *DeviceSet) removeTransactionMetaData() error {
-	if err := os.RemoveAll(devices.transactionMetaFile()); err != nil {
-		return err
-	}
-	return nil
+	return os.RemoveAll(devices.transactionMetaFile())
 }
 
 func (devices *DeviceSet) rollbackTransaction() error {
@@ -1464,12 +1490,9 @@ func (devices *DeviceSet) closeTransaction() error {
 }
 
 func determineDriverCapabilities(version string) error {
-	/*
-	 * Driver version 4.27.0 and greater support deferred activation
-	 * feature.
-	 */
+	// Kernel driver version >= 4.27.0 support deferred removal
 
-	logrus.Debugf("devicemapper: driver version is %s", version)
+	logrus.Debugf("devicemapper: kernel dm driver version is %s", version)
 
 	versionSplit := strings.Split(version, ".")
 	major, err := strconv.Atoi(versionSplit[0])
@@ -1505,12 +1528,13 @@ func determineDriverCapabilities(version string) error {
 
 // Determine the major and minor number of loopback device
 func getDeviceMajorMinor(file *os.File) (uint64, uint64, error) {
-	stat, err := file.Stat()
+	var stat unix.Stat_t
+	err := unix.Stat(file.Name(), &stat)
 	if err != nil {
 		return 0, 0, err
 	}
 
-	dev := stat.Sys().(*syscall.Stat_t).Rdev
+	dev := stat.Rdev
 	majorNum := major(dev)
 	minorNum := minor(dev)
 
@@ -1648,36 +1672,19 @@ func (devices *DeviceSet) enableDeferredRemovalDeletion() error {
 	return nil
 }
 
-func (devices *DeviceSet) initDevmapper(doInit bool) error {
-	// give ourselves to libdm as a log handler
-	devicemapper.LogInit(devices)
-
-	version, err := devicemapper.GetDriverVersion()
-	if err != nil {
-		// Can't even get driver version, assume not supported
-		return errors.Wrap(graphdriver.ErrNotSupported, "unable to determine version of device mapper")
-	}
-
-	if err := determineDriverCapabilities(version); err != nil {
-		return errors.Wrap(graphdriver.ErrNotSupported, "unable to determine device mapper driver capabilities")
-	}
-
+func (devices *DeviceSet) initDevmapper(doInit bool) (retErr error) {
 	if err := devices.enableDeferredRemovalDeletion(); err != nil {
 		return err
 	}
 
 	// https://github.com/docker/docker/issues/4036
-	// if supported := devicemapper.UdevSetSyncSupport(true); !supported {
-	// 	if storageversion.IAmStatic == "true" {
-	// 		logrus.Errorf("devmapper: Udev sync is not supported. This will lead to data loss and unexpected behavior. Install a dynamic binary to use devicemapper or select a different storage driver. For more information, see https://docs.docker.com/engine/reference/commandline/daemon/#daemon-storage-driver-option")
-	// 	} else {
-	// 		logrus.Errorf("devmapper: Udev sync is not supported. This will lead to data loss and unexpected behavior. Install a more recent version of libdevmapper or select a different storage driver. For more information, see https://docs.docker.com/engine/reference/commandline/daemon/#daemon-storage-driver-option")
-	// 	}
-	//
-	// 	if !devices.overrideUdevSyncCheck {
-	// 		return graphdriver.ErrNotSupported
-	// 	}
-	// }
+	if supported := devicemapper.UdevSetSyncSupport(true); !supported {
+		logrus.Error("devmapper: Udev sync is not supported. This will lead to data loss and unexpected behavior. Install a more recent version of libdevmapper or select a different storage driver. For more information, see https://docs.docker.com/engine/reference/commandline/dockerd/#storage-driver-options")
+
+		if !devices.overrideUdevSyncCheck {
+			return graphdriver.ErrNotSupported
+		}
+	}
 
 	//create the root dir of the devmapper driver ownership to match this
 	//daemon's remapped root uid/gid so containers can start properly
@@ -1692,20 +1699,47 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error {
 		return err
 	}
 
-	// Set the device prefix from the device id and inode of the container root dir
-
-	st, err := os.Stat(devices.root)
+	prevSetupConfig, err := readLVMConfig(devices.root)
 	if err != nil {
+		return err
+	}
+
+	if !reflect.DeepEqual(devices.lvmSetupConfig, directLVMConfig{}) {
+		if devices.thinPoolDevice != "" {
+			return errors.New("cannot setup direct-lvm when `dm.thinpooldev` is also specified")
+		}
+
+		if !reflect.DeepEqual(prevSetupConfig, devices.lvmSetupConfig) {
+			if !reflect.DeepEqual(prevSetupConfig, directLVMConfig{}) {
+				return errors.New("changing direct-lvm config is not supported")
+			}
+			logrus.WithField("storage-driver", "devicemapper").WithField("direct-lvm-config", devices.lvmSetupConfig).Debugf("Setting up direct lvm mode")
+			if err := verifyBlockDevice(devices.lvmSetupConfig.Device, lvmSetupConfigForce); err != nil {
+				return err
+			}
+			if err := setupDirectLVM(devices.lvmSetupConfig); err != nil {
+				return err
+			}
+			if err := writeLVMConfig(devices.root, devices.lvmSetupConfig); err != nil {
+				return err
+			}
+		}
+		devices.thinPoolDevice = "storage-thinpool"
+		logrus.WithField("storage-driver", "devicemapper").Debugf("Setting dm.thinpooldev to %q", devices.thinPoolDevice)
+	}
+
+	// Set the device prefix from the device id and inode of the storage root dir
+	var st unix.Stat_t
+	if err := unix.Stat(devices.root, &st); err != nil {
 		return fmt.Errorf("devmapper: Error looking up dir %s: %s", devices.root, err)
 	}
-	sysSt := st.Sys().(*syscall.Stat_t)
 	// "reg-" stands for "regular file".
 	// In the future we might use "dev-" for "device file", etc.
 	// container-maj,min[-inode] stands for:
 	//	- Managed by container storage
 	//	- The target of this device is at major <maj> and minor <min>
 	//	- If <inode> is defined, use that file inside the device as a loopback image. Otherwise use the device itself.
-	devices.devicePrefix = fmt.Sprintf("container-%d:%d-%d", major(sysSt.Dev), minor(sysSt.Dev), sysSt.Ino)
+	devices.devicePrefix = fmt.Sprintf("container-%d:%d-%d", major(st.Dev), minor(st.Dev), st.Ino)
 	logrus.Debugf("devmapper: Generated prefix: %s", devices.devicePrefix)
 
 	// Check for the existence of the thin-pool device
@@ -1748,7 +1782,7 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error {
 			hasData := devices.hasImage("data")
 
 			if !doInit && !hasData {
-				return errors.New("Loopback data file not found")
+				return errors.New("loopback data file not found")
 			}
 
 			if !hasData {
@@ -1781,7 +1815,7 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error {
 			hasMetadata := devices.hasImage("metadata")
 
 			if !doInit && !hasMetadata {
-				return errors.New("Loopback metadata file not found")
+				return errors.New("loopback metadata file not found")
 			}
 
 			if !hasMetadata {
@@ -1811,6 +1845,14 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error {
 		if err := devicemapper.CreatePool(devices.getPoolName(), dataFile, metadataFile, devices.thinpBlockSize); err != nil {
 			return err
 		}
+		defer func() {
+			if retErr != nil {
+				err = devices.deactivatePool()
+				if err != nil {
+					logrus.Warnf("devmapper: Failed to deactivatePool: %v", err)
+				}
+			}
+		}()
 	}
 
 	// Pool already exists and caller did not pass us a pool. That means
@@ -1857,8 +1899,8 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error {
 
 // AddDevice adds a device and registers in the hash.
 func (devices *DeviceSet) AddDevice(hash, baseHash string, storageOpt map[string]string) error {
-	logrus.Debugf("devmapper: AddDevice(hash=%s basehash=%s)", hash, baseHash)
-	defer logrus.Debugf("devmapper: AddDevice(hash=%s basehash=%s) END", hash, baseHash)
+	logrus.Debugf("devmapper: AddDevice START(hash=%s basehash=%s)", hash, baseHash)
+	defer logrus.Debugf("devmapper: AddDevice END(hash=%s basehash=%s)", hash, baseHash)
 
 	// If a deleted device exists, return error.
 	baseInfo, err := devices.lookupDeviceWithLock(baseHash)
@@ -1895,7 +1937,7 @@ func (devices *DeviceSet) AddDevice(hash, baseHash string, storageOpt map[string
 		return fmt.Errorf("devmapper: Container size cannot be smaller than %s", units.HumanSize(float64(baseInfo.Size)))
 	}
 
-	if err := devices.createRegisterSnapDevice(hash, baseInfo, size); err != nil {
+	if err := devices.takeSnapshot(hash, baseInfo, size); err != nil {
 		return err
 	}
 
@@ -1975,7 +2017,7 @@ func (devices *DeviceSet) deleteTransaction(info *devInfo, syncDelete bool) erro
 	}
 
 	if err == nil {
-		if err := devices.unregisterDevice(info.DeviceID, info.Hash); err != nil {
+		if err := devices.unregisterDevice(info.Hash); err != nil {
 			return err
 		}
 		// If device was already in deferred delete state that means
@@ -1996,8 +2038,8 @@ func (devices *DeviceSet) deleteTransaction(info *devInfo, syncDelete bool) erro
 
 // Issue discard only if device open count is zero.
 func (devices *DeviceSet) issueDiscard(info *devInfo) error {
-	logrus.Debugf("devmapper: issueDiscard(device: %s). START", info.Hash)
-	defer logrus.Debugf("devmapper: issueDiscard(device: %s). END", info.Hash)
+	logrus.Debugf("devmapper: issueDiscard START(device: %s).", info.Hash)
+	defer logrus.Debugf("devmapper: issueDiscard END(device: %s).", info.Hash)
 	// This is a workaround for the kernel not discarding block so
 	// on the thin pool when we remove a thinp device, so we do it
 	// manually.
@@ -2030,7 +2072,16 @@ func (devices *DeviceSet) deleteDevice(info *devInfo, syncDelete bool) error {
 	}
 
 	// Try to deactivate device in case it is active.
-	if err := devices.deactivateDevice(info); err != nil {
+	// If deferred removal is enabled and deferred deletion is disabled
+	// then make sure device is removed synchronously. There have been
+	// some cases of device being busy for short duration and we would
+	// rather busy wait for device removal to take care of these cases.
+	deferredRemove := devices.deferredRemove
+	if !devices.deferredDelete {
+		deferredRemove = false
+	}
+
+	if err := devices.deactivateDeviceMode(info, deferredRemove); err != nil {
 		logrus.Debugf("devmapper: Error deactivating device: %s", err)
 		return err
 	}
@@ -2046,8 +2097,8 @@ func (devices *DeviceSet) deleteDevice(info *devInfo, syncDelete bool) error {
 // removal. If one wants to override that and want DeleteDevice() to fail if
 // device was busy and could not be deleted, set syncDelete=true.
 func (devices *DeviceSet) DeleteDevice(hash string, syncDelete bool) error {
-	logrus.Debugf("devmapper: DeleteDevice(hash=%v syncDelete=%v) START", hash, syncDelete)
-	defer logrus.Debugf("devmapper: DeleteDevice(hash=%v syncDelete=%v) END", hash, syncDelete)
+	logrus.Debugf("devmapper: DeleteDevice START(hash=%v syncDelete=%v)", hash, syncDelete)
+	defer logrus.Debugf("devmapper: DeleteDevice END(hash=%v syncDelete=%v)", hash, syncDelete)
 	info, err := devices.lookupDeviceWithLock(hash)
 	if err != nil {
 		return err
@@ -2063,8 +2114,8 @@ func (devices *DeviceSet) DeleteDevice(hash string, syncDelete bool) error {
 }
 
 func (devices *DeviceSet) deactivatePool() error {
-	logrus.Debug("devmapper: deactivatePool()")
-	defer logrus.Debug("devmapper: deactivatePool END")
+	logrus.Debug("devmapper: deactivatePool() START")
+	defer logrus.Debug("devmapper: deactivatePool() END")
 	devname := devices.getPoolDevName()
 
 	devinfo, err := devicemapper.GetInfo(devname)
@@ -2087,7 +2138,12 @@ func (devices *DeviceSet) deactivatePool() error {
 }
 
 func (devices *DeviceSet) deactivateDevice(info *devInfo) error {
-	logrus.Debugf("devmapper: deactivateDevice(%s)", info.Hash)
+	return devices.deactivateDeviceMode(info, devices.deferredRemove)
+}
+
+func (devices *DeviceSet) deactivateDeviceMode(info *devInfo, deferredRemove bool) error {
+	var err error
+	logrus.Debugf("devmapper: deactivateDevice START(%s)", info.Hash)
 	defer logrus.Debugf("devmapper: deactivateDevice END(%s)", info.Hash)
 
 	devinfo, err := devicemapper.GetInfo(info.Name())
@@ -2099,14 +2155,17 @@ func (devices *DeviceSet) deactivateDevice(info *devInfo) error {
 		return nil
 	}
 
-	if devices.deferredRemove {
-		if err := devicemapper.RemoveDeviceDeferred(info.Name()); err != nil {
-			return err
-		}
+	if deferredRemove {
+		err = devicemapper.RemoveDeviceDeferred(info.Name())
 	} else {
-		if err := devices.removeDevice(info.Name()); err != nil {
-			return err
-		}
+		err = devices.removeDevice(info.Name())
+	}
+
+	// This function's semantics is such that it does not return an
+	// error if device does not exist. So if device went away by
+	// the time we actually tried to remove it, do not return error.
+	if errors.Cause(err) != devicemapper.ErrEnxio {
+		return err
 	}
 	return nil
 }
@@ -2137,41 +2196,53 @@ func (devices *DeviceSet) removeDevice(devname string) error {
 	return err
 }
 
-func (devices *DeviceSet) cancelDeferredRemoval(info *devInfo) error {
+func (devices *DeviceSet) cancelDeferredRemovalIfNeeded(info *devInfo) error {
 	if !devices.deferredRemove {
 		return nil
 	}
 
-	logrus.Debugf("devmapper: cancelDeferredRemoval START(%s)", info.Name())
-	defer logrus.Debugf("devmapper: cancelDeferredRemoval END(%s)", info.Name())
+	logrus.Debugf("devmapper: cancelDeferredRemovalIfNeeded START(%s)", info.Name())
+	defer logrus.Debugf("devmapper: cancelDeferredRemovalIfNeeded END(%s)", info.Name())
 
 	devinfo, err := devicemapper.GetInfoWithDeferred(info.Name())
+	if err != nil {
+		return err
+	}
 
 	if devinfo != nil && devinfo.DeferredRemove == 0 {
 		return nil
 	}
 
 	// Cancel deferred remove
-	for i := 0; i < 100; i++ {
-		err = devicemapper.CancelDeferredRemove(info.Name())
-		if err == nil {
-			break
-		}
-
-		if errors.Cause(err) == devicemapper.ErrEnxio {
-			// Device is probably already gone. Return success.
-			return nil
-		}
-
+	if err := devices.cancelDeferredRemoval(info); err != nil {
+		// If Error is ErrEnxio. Device is probably already gone. Continue.
 		if errors.Cause(err) != devicemapper.ErrBusy {
 			return err
 		}
+	}
+	return nil
+}
 
-		// If we see EBUSY it may be a transient error,
-		// sleep a bit a retry a few times.
-		devices.Unlock()
-		time.Sleep(100 * time.Millisecond)
-		devices.Lock()
+func (devices *DeviceSet) cancelDeferredRemoval(info *devInfo) error {
+	logrus.Debugf("devmapper: cancelDeferredRemoval START(%s)", info.Name())
+	defer logrus.Debugf("devmapper: cancelDeferredRemoval END(%s)", info.Name())
+
+	var err error
+
+	// Cancel deferred remove
+	for i := 0; i < 100; i++ {
+		err = devicemapper.CancelDeferredRemove(info.Name())
+		if err != nil {
+			if errors.Cause(err) != devicemapper.ErrBusy {
+				// If we see EBUSY it may be a transient error,
+				// sleep a bit a retry a few times.
+				devices.Unlock()
+				time.Sleep(100 * time.Millisecond)
+				devices.Lock()
+				continue
+			}
+		}
+		break
 	}
 	return err
 }
@@ -2209,9 +2280,6 @@ func (devices *DeviceSet) Shutdown(home string) error {
 		if err != nil {
 			return err
 		}
-		if p == path.Join(home, "mnt") {
-			return nil
-		}
 		if !info.IsDir() {
 			return nil
 		}
@@ -2220,7 +2288,7 @@ func (devices *DeviceSet) Shutdown(home string) error {
 			// We use MNT_DETACH here in case it is still busy in some running
 			// container. This means it'll go away from the global scope directly,
 			// and the device will be released when that container dies.
-			if err := syscall.Unmount(p, syscall.MNT_DETACH); err != nil {
+			if err := unix.Unmount(p, unix.MNT_DETACH); err != nil {
 				logrus.Debugf("devmapper: Shutdown unmounting %s, error: %s", p, err)
 			}
 		}
@@ -2263,6 +2331,34 @@ func (devices *DeviceSet) Shutdown(home string) error {
 	return nil
 }
 
+// Recent XFS changes allow changing behavior of filesystem in case of errors.
+// When thin pool gets full and XFS gets ENOSPC error, currently it tries
+// IO infinitely and sometimes it can block the container process
+// and process can't be killWith 0 value, XFS will not retry upon error
+// and instead will shutdown filesystem.
+
+func (devices *DeviceSet) xfsSetNospaceRetries(info *devInfo) error {
+	dmDevicePath, err := os.Readlink(info.DevName())
+	if err != nil {
+		return fmt.Errorf("devmapper: readlink failed for device %v:%v", info.DevName(), err)
+	}
+
+	dmDeviceName := path.Base(dmDevicePath)
+	filePath := "/sys/fs/xfs/" + dmDeviceName + "/error/metadata/ENOSPC/max_retries"
+	maxRetriesFile, err := os.OpenFile(filePath, os.O_WRONLY, 0)
+	if err != nil {
+		return fmt.Errorf("devmapper: user specified daemon option dm.xfs_nospace_max_retries but it does not seem to be supported on this system :%v", err)
+	}
+	defer maxRetriesFile.Close()
+
+	// Set max retries to 0
+	_, err = maxRetriesFile.WriteString(devices.xfsNospaceRetries)
+	if err != nil {
+		return fmt.Errorf("devmapper: Failed to write string %v to file %v:%v", devices.xfsNospaceRetries, filePath, err)
+	}
+	return nil
+}
+
 // MountDevice mounts the device if not already mounted.
 func (devices *DeviceSet) MountDevice(hash, path, mountLabel string) error {
 	info, err := devices.lookupDeviceWithLock(hash)
@@ -2300,7 +2396,15 @@ func (devices *DeviceSet) MountDevice(hash, path, mountLabel string) error {
 	options = joinMountOptions(options, label.FormatMountLabel("", mountLabel))
 
 	if err := mount.Mount(info.DevName(), path, fstype, options); err != nil {
-		return fmt.Errorf("devmapper: Error mounting '%s' on '%s': %s", info.DevName(), path, err)
+		return fmt.Errorf("devmapper: Error mounting '%s' on '%s': %s\n%v", info.DevName(), path, err, string(dmesg.Dmesg(256)))
+	}
+
+	if fstype == "xfs" && devices.xfsNospaceRetries != "" {
+		if err := devices.xfsSetNospaceRetries(info); err != nil {
+			unix.Unmount(path, unix.MNT_DETACH)
+			devices.deactivateDevice(info)
+			return err
+		}
 	}
 
 	return nil
@@ -2308,8 +2412,8 @@ func (devices *DeviceSet) MountDevice(hash, path, mountLabel string) error {
 
 // UnmountDevice unmounts the device and removes it from hash.
 func (devices *DeviceSet) UnmountDevice(hash, mountPath string) error {
-	logrus.Debugf("devmapper: UnmountDevice(hash=%s)", hash)
-	defer logrus.Debugf("devmapper: UnmountDevice(hash=%s) END", hash)
+	logrus.Debugf("devmapper: UnmountDevice START(hash=%s)", hash)
+	defer logrus.Debugf("devmapper: UnmountDevice END(hash=%s)", hash)
 
 	info, err := devices.lookupDeviceWithLock(hash)
 	if err != nil {
@@ -2323,16 +2427,12 @@ func (devices *DeviceSet) UnmountDevice(hash, mountPath string) error {
 	defer devices.Unlock()
 
 	logrus.Debugf("devmapper: Unmount(%s)", mountPath)
-	if err := syscall.Unmount(mountPath, syscall.MNT_DETACH); err != nil {
+	if err := unix.Unmount(mountPath, unix.MNT_DETACH); err != nil {
 		return err
 	}
 	logrus.Debug("devmapper: Unmount done")
 
-	if err := devices.deactivateDevice(info); err != nil {
-		return err
-	}
-
-	return nil
+	return devices.deactivateDevice(info)
 }
 
 // HasDevice returns true if the device metadata exists.
@@ -2424,8 +2524,8 @@ func (devices *DeviceSet) MetadataDevicePath() string {
 }
 
 func (devices *DeviceSet) getUnderlyingAvailableSpace(loopFile string) (uint64, error) {
-	buf := new(syscall.Statfs_t)
-	if err := syscall.Statfs(loopFile, buf); err != nil {
+	buf := new(unix.Statfs_t)
+	if err := unix.Statfs(loopFile, buf); err != nil {
 		logrus.Warnf("devmapper: Couldn't stat loopfile filesystem %v: %v", loopFile, err)
 		return 0, err
 	}
@@ -2534,22 +2634,25 @@ func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps [
 		minFreeSpacePercent:   defaultMinFreeSpacePercent,
 	}
 
-	// Pick up initialization settings, if any were saved before
-	defaultsFile := path.Join(root, "defaults")
-	defaultsBytes, err := ioutil.ReadFile(defaultsFile)
-	defaults := []string{}
-	settings := map[string]string{}
-	if err == nil && len(defaultsBytes) > 0 {
-		defaults = strings.Split(string(defaultsBytes), "\n")
+	version, err := devicemapper.GetDriverVersion()
+	if err != nil {
+		// Can't even get driver version, assume not supported
+		return nil, graphdriver.ErrNotSupported
+	}
+
+	if err := determineDriverCapabilities(version); err != nil {
+		return nil, graphdriver.ErrNotSupported
+	}
+
+	if driverDeferredRemovalSupport && devicemapper.LibraryDeferredRemovalSupport {
+		// enable deferred stuff by default
+		enableDeferredDeletion = true
+		enableDeferredRemoval = true
 	}
 
 	foundBlkDiscard := false
-	nthOption := 0
-	for _, option := range append(defaults, options...) {
-		nthOption = nthOption + 1
-		if len(option) == 0 {
-			continue
-		}
+	var lvmSetupConfig directLVMConfig
+	for _, option := range options {
 		key, val, err := parsers.ParseKeyValueOpt(option)
 		if err != nil {
 			return nil, err
@@ -2637,15 +2740,78 @@ func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps [
 			}
 
 			devices.minFreeSpacePercent = uint32(minFreeSpacePercent)
-		default:
-			if nthOption > len(defaults) {
-				return nil, fmt.Errorf("devmapper: Unknown option %s", key)
+		case "dm.xfs_nospace_max_retries":
+			_, err := strconv.ParseUint(val, 10, 64)
+			if err != nil {
+				return nil, err
 			}
-			logrus.Errorf("devmapper: Unknown option %s, ignoring", key)
+			devices.xfsNospaceRetries = val
+		case "dm.directlvm_device":
+			lvmSetupConfig.Device = val
+		case "dm.directlvm_device_force":
+			lvmSetupConfigForce, err = strconv.ParseBool(val)
+			if err != nil {
+				return nil, err
+			}
+		case "dm.thinp_percent":
+			per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
+			if err != nil {
+				return nil, errors.Wrapf(err, "could not parse `dm.thinp_percent=%s`", val)
+			}
+			if per >= 100 {
+				return nil, errors.New("dm.thinp_percent must be greater than 0 and less than 100")
+			}
+			lvmSetupConfig.ThinpPercent = per
+		case "dm.thinp_metapercent":
+			per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
+			if err != nil {
+				return nil, errors.Wrapf(err, "could not parse `dm.thinp_metapercent=%s`", val)
+			}
+			if per >= 100 {
+				return nil, errors.New("dm.thinp_metapercent must be greater than 0 and less than 100")
+			}
+			lvmSetupConfig.ThinpMetaPercent = per
+		case "dm.thinp_autoextend_percent":
+			per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
+			if err != nil {
+				return nil, errors.Wrapf(err, "could not parse `dm.thinp_autoextend_percent=%s`", val)
+			}
+			if per > 100 {
+				return nil, errors.New("dm.thinp_autoextend_percent must be greater than 0 and less than 100")
+			}
+			lvmSetupConfig.AutoExtendPercent = per
+		case "dm.thinp_autoextend_threshold":
+			per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32)
+			if err != nil {
+				return nil, errors.Wrapf(err, "could not parse `dm.thinp_autoextend_threshold=%s`", val)
+			}
+			if per > 100 {
+				return nil, errors.New("dm.thinp_autoextend_threshold must be greater than 0 and less than 100")
+			}
+			lvmSetupConfig.AutoExtendThreshold = per
+		case "dm.libdm_log_level":
+			level, err := strconv.ParseInt(val, 10, 32)
+			if err != nil {
+				return nil, errors.Wrapf(err, "could not parse `dm.libdm_log_level=%s`", val)
+			}
+			if level < devicemapper.LogLevelFatal || level > devicemapper.LogLevelDebug {
+				return nil, errors.Errorf("dm.libdm_log_level must be in range [%d,%d]", devicemapper.LogLevelFatal, devicemapper.LogLevelDebug)
+			}
+			// Register a new logging callback with the specified level.
+			devicemapper.LogInit(devicemapper.DefaultLogger{
+				Level: int(level),
+			})
+		default:
+			return nil, fmt.Errorf("devmapper: Unknown option %s", key)
 		}
-		settings[key] = val
 	}
 
+	if err := validateLVMConfig(lvmSetupConfig); err != nil {
+		return nil, err
+	}
+
+	devices.lvmSetupConfig = lvmSetupConfig
+
 	// By default, don't do blk discard hack on raw devices, its rarely useful and is expensive
 	if !foundBlkDiscard && (devices.dataDevice != "" || devices.thinPoolDevice != "") {
 		devices.doBlkDiscard = false
@@ -2655,15 +2821,5 @@ func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps [
 		return nil, err
 	}
 
-	// Save these settings along with the other metadata
-	defaults = []string{}
-	for key, val := range settings {
-		defaults = append(defaults, key+"="+val)
-	}
-	defaultsBytes = []byte(strings.Join(defaults, "\n") + "\n")
-	if err := ioutils.AtomicWriteFile(defaultsFile, defaultsBytes, 0600); err != nil {
-		return nil, err
-	}
-
 	return devices, nil
 }

vendor/github.com/containers/storage/drivers/devmapper/driver.go

@@ -9,13 +9,15 @@ import (
 	"path"
 	"strconv"
 
-	"github.com/Sirupsen/logrus"
+	"github.com/sirupsen/logrus"
 
 	"github.com/containers/storage/drivers"
 	"github.com/containers/storage/pkg/devicemapper"
 	"github.com/containers/storage/pkg/idtools"
+	"github.com/containers/storage/pkg/locker"
 	"github.com/containers/storage/pkg/mount"
-	"github.com/docker/go-units"
+	"github.com/containers/storage/pkg/system"
+	units "github.com/docker/go-units"
 )
 
 func init() {
@@ -29,6 +31,7 @@ type Driver struct {
 	uidMaps []idtools.IDMap
 	gidMaps []idtools.IDMap
 	ctr     *graphdriver.RefCounter
+	locker  *locker.Locker
 }
 
 // Init creates a driver with the given home and the set of options.
@@ -48,6 +51,7 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		uidMaps:   uidMaps,
 		gidMaps:   gidMaps,
 		ctr:       graphdriver.NewRefCounter(graphdriver.NewDefaultChecker()),
+		locker:    locker.New(),
 	}
 
 	return graphdriver.NewNaiveDiffDriver(d, uidMaps, gidMaps), nil
@@ -65,18 +69,18 @@ func (d *Driver) Status() [][2]string {
 
 	status := [][2]string{
 		{"Pool Name", s.PoolName},
-		{"Pool Blocksize", fmt.Sprintf("%s", units.HumanSize(float64(s.SectorSize)))},
-		{"Base Device Size", fmt.Sprintf("%s", units.HumanSize(float64(s.BaseDeviceSize)))},
+		{"Pool Blocksize", units.HumanSize(float64(s.SectorSize))},
+		{"Base Device Size", units.HumanSize(float64(s.BaseDeviceSize))},
 		{"Backing Filesystem", s.BaseDeviceFS},
 		{"Data file", s.DataFile},
 		{"Metadata file", s.MetadataFile},
-		{"Data Space Used", fmt.Sprintf("%s", units.HumanSize(float64(s.Data.Used)))},
-		{"Data Space Total", fmt.Sprintf("%s", units.HumanSize(float64(s.Data.Total)))},
-		{"Data Space Available", fmt.Sprintf("%s", units.HumanSize(float64(s.Data.Available)))},
-		{"Metadata Space Used", fmt.Sprintf("%s", units.HumanSize(float64(s.Metadata.Used)))},
-		{"Metadata Space Total", fmt.Sprintf("%s", units.HumanSize(float64(s.Metadata.Total)))},
-		{"Metadata Space Available", fmt.Sprintf("%s", units.HumanSize(float64(s.Metadata.Available)))},
-		{"Thin Pool Minimum Free Space", fmt.Sprintf("%s", units.HumanSize(float64(s.MinFreeSpace)))},
+		{"Data Space Used", units.HumanSize(float64(s.Data.Used))},
+		{"Data Space Total", units.HumanSize(float64(s.Data.Total))},
+		{"Data Space Available", units.HumanSize(float64(s.Data.Available))},
+		{"Metadata Space Used", units.HumanSize(float64(s.Metadata.Used))},
+		{"Metadata Space Total", units.HumanSize(float64(s.Metadata.Total))},
+		{"Metadata Space Available", units.HumanSize(float64(s.Metadata.Available))},
+		{"Thin Pool Minimum Free Space", units.HumanSize(float64(s.MinFreeSpace))},
 		{"Udev Sync Supported", fmt.Sprintf("%v", s.UdevSyncSupported)},
 		{"Deferred Removal Enabled", fmt.Sprintf("%v", s.DeferredRemoveEnabled)},
 		{"Deferred Deletion Enabled", fmt.Sprintf("%v", s.DeferredDeleteEnabled)},
@@ -122,12 +126,17 @@ func (d *Driver) Cleanup() error {
 
 // CreateReadWrite creates a layer that is writable for use as a container
 // file system.
-func (d *Driver) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error {
-	return d.Create(id, parent, mountLabel, storageOpt)
+func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error {
+	return d.Create(id, parent, opts)
 }
 
 // Create adds a device with a given id and the parent.
-func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]string) error {
+func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error {
+	var storageOpt map[string]string
+	if opts != nil {
+		storageOpt = opts.StorageOpt
+	}
+
 	if err := d.DeviceSet.AddDevice(id, parent, storageOpt); err != nil {
 		return err
 	}
@@ -137,6 +146,8 @@ func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]str
 
 // Remove removes a device with a given id, unmounts the filesystem.
 func (d *Driver) Remove(id string) error {
+	d.locker.Lock(id)
+	defer d.locker.Unlock(id)
 	if !d.DeviceSet.HasDevice(id) {
 		// Consider removing a non-existing device a no-op
 		// This is useful to be able to progress on container removal
@@ -146,19 +157,15 @@ func (d *Driver) Remove(id string) error {
 
 	// This assumes the device has been properly Get/Put:ed and thus is unmounted
 	if err := d.DeviceSet.DeleteDevice(id, false); err != nil {
-		return err
+		return fmt.Errorf("failed to remove device %s: %v", id, err)
 	}
-
-	mp := path.Join(d.home, "mnt", id)
-	if err := os.RemoveAll(mp); err != nil && !os.IsNotExist(err) {
-		return err
-	}
-
-	return nil
+	return system.EnsureRemoveAll(path.Join(d.home, "mnt", id))
 }
 
 // Get mounts a device with given id into the root filesystem
 func (d *Driver) Get(id, mountLabel string) (string, error) {
+	d.locker.Lock(id)
+	defer d.locker.Unlock(id)
 	mp := path.Join(d.home, "mnt", id)
 	rootFs := path.Join(mp, "rootfs")
 	if count := d.ctr.Increment(mp); count > 1 {
@@ -209,6 +216,8 @@ func (d *Driver) Get(id, mountLabel string) (string, error) {
 
 // Put unmounts a device and removes it.
 func (d *Driver) Put(id string) error {
+	d.locker.Lock(id)
+	defer d.locker.Unlock(id)
 	mp := path.Join(d.home, "mnt", id)
 	if count := d.ctr.Decrement(mp); count > 0 {
 		return nil
@@ -227,6 +236,5 @@ func (d *Driver) Exists(id string) bool {
 
 // AdditionalImageStores returns additional image stores supported by the driver
 func (d *Driver) AdditionalImageStores() []string {
-	var imageStores []string
-	return imageStores
+	return nil
 }

vendor/github.com/containers/storage/drivers/devmapper/mount.go

@@ -7,7 +7,8 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"syscall"
+
+	"golang.org/x/sys/unix"
 )
 
 // FIXME: this is copy-pasted from the aufs driver.
@@ -15,19 +16,17 @@ import (
 
 // Mounted returns true if a mount point exists.
 func Mounted(mountpoint string) (bool, error) {
-	mntpoint, err := os.Stat(mountpoint)
-	if err != nil {
+	var mntpointSt unix.Stat_t
+	if err := unix.Stat(mountpoint, &mntpointSt); err != nil {
 		if os.IsNotExist(err) {
 			return false, nil
 		}
 		return false, err
 	}
-	parent, err := os.Stat(filepath.Join(mountpoint, ".."))
-	if err != nil {
+	var parentSt unix.Stat_t
+	if err := unix.Stat(filepath.Join(mountpoint, ".."), &parentSt); err != nil {
 		return false, err
 	}
-	mntpointSt := mntpoint.Sys().(*syscall.Stat_t)
-	parentSt := parent.Sys().(*syscall.Stat_t)
 	return mntpointSt.Dev != parentSt.Dev, nil
 }
 

vendor/github.com/containers/storage/drivers/driver.go

@@ -2,12 +2,13 @@ package graphdriver
 
 import (
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"strings"
 
-	"github.com/Sirupsen/logrus"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 	"github.com/vbatts/tar-split/tar/storage"
 
 	"github.com/containers/storage/pkg/archive"
@@ -28,12 +29,19 @@ var (
 
 	// ErrNotSupported returned when driver is not supported.
 	ErrNotSupported = errors.New("driver not supported")
-	// ErrPrerequisites retuned when driver does not meet prerequisites.
+	// ErrPrerequisites returned when driver does not meet prerequisites.
 	ErrPrerequisites = errors.New("prerequisites for driver not satisfied (wrong filesystem?)")
 	// ErrIncompatibleFS returned when file system is not supported.
 	ErrIncompatibleFS = fmt.Errorf("backing file system is unsupported for this graph driver")
 )
 
+//CreateOpts contains optional arguments for Create() and CreateReadWrite()
+// methods.
+type CreateOpts struct {
+	MountLabel string
+	StorageOpt map[string]string
+}
+
 // InitFunc initializes the storage driver.
 type InitFunc func(root string, options []string, uidMaps, gidMaps []idtools.IDMap) (Driver, error)
 
@@ -47,11 +55,13 @@ type ProtoDriver interface {
 	// String returns a string representation of this driver.
 	String() string
 	// CreateReadWrite creates a new, empty filesystem layer that is ready
-	// to be used as the storage for a container.
-	CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error
+	// to be used as the storage for a container. Additional options can
+	// be passed in opts. parent may be "" and opts may be nil.
+	CreateReadWrite(id, parent string, opts *CreateOpts) error
 	// Create creates a new, empty, filesystem layer with the
-	// specified id and parent and mountLabel. Parent and mountLabel may be "".
-	Create(id, parent, mountLabel string, storageOpt map[string]string) error
+	// specified id and parent and options passed in opts. Parent
+	// may be "" and opts may be nil.
+	Create(id, parent string, opts *CreateOpts) error
 	// Remove attempts to remove the filesystem layer with this id.
 	Remove(id string) error
 	// Get returns the mountpoint for the layered filesystem referred
@@ -78,26 +88,48 @@ type ProtoDriver interface {
 	AdditionalImageStores() []string
 }
 
-// Driver is the interface for layered/snapshot file system drivers.
-type Driver interface {
-	ProtoDriver
+// DiffDriver is the interface to use to implement graph diffs
+type DiffDriver interface {
 	// Diff produces an archive of the changes between the specified
 	// layer and its parent layer which may be "".
-	Diff(id, parent string) (archive.Archive, error)
+	Diff(id, parent string) (io.ReadCloser, error)
 	// Changes produces a list of changes between the specified layer
 	// and its parent layer. If parent is "", then all changes will be ADD changes.
 	Changes(id, parent string) ([]archive.Change, error)
 	// ApplyDiff extracts the changeset from the given diff into the
 	// layer with the specified id and parent, returning the size of the
 	// new layer in bytes.
-	// The archive.Reader must be an uncompressed stream.
-	ApplyDiff(id, parent string, diff archive.Reader) (size int64, err error)
+	// The io.Reader must be an uncompressed stream.
+	ApplyDiff(id, parent string, diff io.Reader) (size int64, err error)
 	// DiffSize calculates the changes between the specified id
 	// and its parent and returns the size in bytes of the changes
 	// relative to its base filesystem directory.
 	DiffSize(id, parent string) (size int64, err error)
 }
 
+// Driver is the interface for layered/snapshot file system drivers.
+type Driver interface {
+	ProtoDriver
+	DiffDriver
+}
+
+// Capabilities defines a list of capabilities a driver may implement.
+// These capabilities are not required; however, they do determine how a
+// graphdriver can be used.
+type Capabilities struct {
+	// Flags that this driver is capable of reproducing exactly equivalent
+	// diffs for read-only layers. If set, clients can rely on the driver
+	// for consistent tar streams, and avoid extra processing to account
+	// for potential differences (eg: the layer store's use of tar-split).
+	ReproducesExactDiffs bool
+}
+
+// CapabilityDriver is the interface for layered file system drivers that
+// can report on their Capabilities.
+type CapabilityDriver interface {
+	Capabilities() Capabilities
+}
+
 // DiffGetterDriver is the interface for layered file system drivers that
 // provide a specialized function for getting file contents for tar-split.
 type DiffGetterDriver interface {
@@ -136,15 +168,13 @@ func Register(name string, initFunc InitFunc) error {
 }
 
 // GetDriver initializes and returns the registered driver
-func GetDriver(name, home string, options []string, uidMaps, gidMaps []idtools.IDMap) (Driver, error) {
+func GetDriver(name string, config Options) (Driver, error) {
 	if initFunc, exists := drivers[name]; exists {
-		return initFunc(filepath.Join(home, name), options, uidMaps, gidMaps)
+		return initFunc(filepath.Join(config.Root, name), config.DriverOptions, config.UIDMaps, config.GIDMaps)
 	}
-	if pluginDriver, err := lookupPlugin(name, home, options); err == nil {
-		return pluginDriver, nil
-	}
-	logrus.Errorf("Failed to GetDriver graph %s %s", name, home)
-	return nil, errors.Wrapf(ErrNotSupported, "failed to GetDriver graph %s %s", name, home)
+
+	logrus.Errorf("Failed to GetDriver graph %s %s", name, config.Root)
+	return nil, errors.Wrapf(ErrNotSupported, "failed to GetDriver graph %s %s", name, config.Root)
 }
 
 // getBuiltinDriver initializes and returns the registered driver, but does not try to load from plugins
@@ -156,15 +186,24 @@ func getBuiltinDriver(name, home string, options []string, uidMaps, gidMaps []id
 	return nil, errors.Wrapf(ErrNotSupported, "failed to built-in GetDriver graph %s %s", name, home)
 }
 
+// Options is used to initialize a graphdriver
+type Options struct {
+	Root                string
+	DriverOptions       []string
+	UIDMaps             []idtools.IDMap
+	GIDMaps             []idtools.IDMap
+	ExperimentalEnabled bool
+}
+
 // New creates the driver and initializes it at the specified root.
-func New(root string, name string, options []string, uidMaps, gidMaps []idtools.IDMap) (Driver, error) {
+func New(name string, config Options) (Driver, error) {
 	if name != "" {
 		logrus.Debugf("[graphdriver] trying provided driver %q", name) // so the logs show specified driver
-		return GetDriver(name, root, options, uidMaps, gidMaps)
+		return GetDriver(name, config)
 	}
 
 	// Guess for prior driver
-	driversMap := scanPriorDrivers(root)
+	driversMap := scanPriorDrivers(config.Root)
 	for _, name := range priority {
 		if name == "vfs" {
 			// don't use vfs even if there is state present.
@@ -173,13 +212,13 @@ func New(root string, name string, options []string, uidMaps, gidMaps []idtools.
 		if _, prior := driversMap[name]; prior {
 			// of the state found from prior drivers, check in order of our priority
 			// which we would prefer
-			driver, err := getBuiltinDriver(name, root, options, uidMaps, gidMaps)
+			driver, err := getBuiltinDriver(name, config.Root, config.DriverOptions, config.UIDMaps, config.GIDMaps)
 			if err != nil {
 				// unlike below, we will return error here, because there is prior
 				// state, and now it is no longer supported/prereq/compatible, so
 				// something changed and needs attention. Otherwise the daemon's
 				// images would just "disappear".
-				logrus.Errorf("[graphdriver] prior storage driver %q failed: %s", name, err)
+				logrus.Errorf("[graphdriver] prior storage driver %s failed: %s", name, err)
 				return nil, err
 			}
 
@@ -191,17 +230,17 @@ func New(root string, name string, options []string, uidMaps, gidMaps []idtools.
 					driversSlice = append(driversSlice, name)
 				}
 
-				return nil, fmt.Errorf("%q contains several valid graphdrivers: %s; Please cleanup or explicitly choose storage driver (-s <DRIVER>)", root, strings.Join(driversSlice, ", "))
+				return nil, fmt.Errorf("%s contains several valid graphdrivers: %s; Please cleanup or explicitly choose storage driver (-s <DRIVER>)", config.Root, strings.Join(driversSlice, ", "))
 			}
 
-			logrus.Infof("[graphdriver] using prior storage driver %q", name)
+			logrus.Infof("[graphdriver] using prior storage driver: %s", name)
 			return driver, nil
 		}
 	}
 
 	// Check for priority drivers first
 	for _, name := range priority {
-		driver, err := getBuiltinDriver(name, root, options, uidMaps, gidMaps)
+		driver, err := getBuiltinDriver(name, config.Root, config.DriverOptions, config.UIDMaps, config.GIDMaps)
 		if err != nil {
 			if isDriverNotSupported(err) {
 				continue
@@ -213,7 +252,7 @@ func New(root string, name string, options []string, uidMaps, gidMaps []idtools.
 
 	// Check all registered drivers if no priority driver is found
 	for name, initFunc := range drivers {
-		driver, err := initFunc(filepath.Join(root, name), options, uidMaps, gidMaps)
+		driver, err := initFunc(filepath.Join(config.Root, name), config.DriverOptions, config.UIDMaps, config.GIDMaps)
 		if err != nil {
 			if isDriverNotSupported(err) {
 				continue

vendor/github.com/containers/storage/drivers/driver_freebsd.go

@@ -1,6 +1,10 @@
 package graphdriver
 
-import "syscall"
+import (
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
 
 var (
 	// Slice of drivers that should be used in an order
@@ -11,7 +15,7 @@ var (
 
 // Mounted checks if the given path is mounted as the fs type
 func Mounted(fsType FsMagic, mountPath string) (bool, error) {
-	var buf syscall.Statfs_t
+	var buf unix.Statfs_t
 	if err := syscall.Statfs(mountPath, &buf); err != nil {
 		return false, err
 	}

vendor/github.com/containers/storage/drivers/driver_linux.go

@@ -4,9 +4,9 @@ package graphdriver
 
 import (
 	"path/filepath"
-	"syscall"
 
 	"github.com/containers/storage/pkg/mount"
+	"golang.org/x/sys/unix"
 )
 
 const (
@@ -66,13 +66,14 @@ var (
 		FsMagicAufs:        "aufs",
 		FsMagicBtrfs:       "btrfs",
 		FsMagicCramfs:      "cramfs",
+		FsMagicEcryptfs:    "ecryptfs",
 		FsMagicExtfs:       "extfs",
 		FsMagicF2fs:        "f2fs",
 		FsMagicGPFS:        "gpfs",
 		FsMagicJffs2Fs:     "jffs2",
 		FsMagicJfs:         "jfs",
 		FsMagicNfsFs:       "nfs",
-		FsMagicOverlay:     "overlay",
+		FsMagicOverlay:     "overlayfs",
 		FsMagicRAMFs:       "ramfs",
 		FsMagicReiserFs:    "reiserfs",
 		FsMagicSmbFs:       "smb",
@@ -87,14 +88,14 @@ var (
 
 // GetFSMagic returns the filesystem id given the path.
 func GetFSMagic(rootpath string) (FsMagic, error) {
-	var buf syscall.Statfs_t
-	if err := syscall.Statfs(filepath.Dir(rootpath), &buf); err != nil {
+	var buf unix.Statfs_t
+	if err := unix.Statfs(filepath.Dir(rootpath), &buf); err != nil {
 		return 0, err
 	}
 	return FsMagic(buf.Type), nil
 }
 
-// NewFsChecker returns a checker configured for the provied FsMagic
+// NewFsChecker returns a checker configured for the provided FsMagic
 func NewFsChecker(t FsMagic) Checker {
 	return &fsChecker{
 		t: t,
@@ -126,8 +127,8 @@ func (c *defaultChecker) IsMounted(path string) bool {
 
 // Mounted checks if the given path is mounted as the fs type
 func Mounted(fsType FsMagic, mountPath string) (bool, error) {
-	var buf syscall.Statfs_t
-	if err := syscall.Statfs(mountPath, &buf); err != nil {
+	var buf unix.Statfs_t
+	if err := unix.Statfs(mountPath, &buf); err != nil {
 		return false, err
 	}
 	return FsMagic(buf.Type) == fsType, nil

vendor/github.com/containers/storage/drivers/driver_solaris.go

@@ -19,8 +19,8 @@ import (
 	"path/filepath"
 	"unsafe"
 
-	log "github.com/Sirupsen/logrus"
-	"github.com/pkg/errors"
+	"github.com/containers/storage/pkg/mount"
+	"github.com/sirupsen/logrus"
 )
 
 const (
@@ -45,22 +45,52 @@ func GetFSMagic(rootpath string) (FsMagic, error) {
 	return 0, nil
 }
 
+type fsChecker struct {
+	t FsMagic
+}
+
+func (c *fsChecker) IsMounted(path string) bool {
+	m, _ := Mounted(c.t, path)
+	return m
+}
+
+// NewFsChecker returns a checker configured for the provided FsMagic
+func NewFsChecker(t FsMagic) Checker {
+	return &fsChecker{
+		t: t,
+	}
+}
+
+// NewDefaultChecker returns a check that parses /proc/mountinfo to check
+// if the specified path is mounted.
+// No-op on Solaris.
+func NewDefaultChecker() Checker {
+	return &defaultChecker{}
+}
+
+type defaultChecker struct {
+}
+
+func (c *defaultChecker) IsMounted(path string) bool {
+	m, _ := mount.Mounted(path)
+	return m
+}
+
 // Mounted checks if the given path is mounted as the fs type
 //Solaris supports only ZFS for now
 func Mounted(fsType FsMagic, mountPath string) (bool, error) {
 
 	cs := C.CString(filepath.Dir(mountPath))
+	defer C.free(unsafe.Pointer(cs))
 	buf := C.getstatfs(cs)
+	defer C.free(unsafe.Pointer(buf))
 
 	// on Solaris buf.f_basetype contains ['z', 'f', 's', 0 ... ]
 	if (buf.f_basetype[0] != 122) || (buf.f_basetype[1] != 102) || (buf.f_basetype[2] != 115) ||
 		(buf.f_basetype[3] != 0) {
-		log.Debugf("[zfs] no zfs dataset found for rootdir '%s'", mountPath)
-		C.free(unsafe.Pointer(buf))
-		return false, errors.Wrapf(graphdriver.ErrPrerequisites, "no zfs dataset found for rootdir '%s'", mountPath)
+		logrus.Debugf("[zfs] no zfs dataset found for rootdir '%s'", mountPath)
+		return false, ErrPrerequisites
 	}
 
-	C.free(unsafe.Pointer(buf))
-	C.free(unsafe.Pointer(cs))
 	return true, nil
 }

vendor/github.com/containers/storage/drivers/fsdiff.go

@@ -1,14 +1,14 @@
 package graphdriver
 
 import (
+	"io"
 	"time"
 
-	"github.com/Sirupsen/logrus"
-
 	"github.com/containers/storage/pkg/archive"
 	"github.com/containers/storage/pkg/chrootarchive"
 	"github.com/containers/storage/pkg/idtools"
 	"github.com/containers/storage/pkg/ioutils"
+	"github.com/sirupsen/logrus"
 )
 
 var (
@@ -31,30 +31,30 @@ type NaiveDiffDriver struct {
 // NewNaiveDiffDriver returns a fully functional driver that wraps the
 // given ProtoDriver and adds the capability of the following methods which
 // it may or may not support on its own:
-//     Diff(id, parent string) (archive.Archive, error)
+//     Diff(id, parent string) (io.ReadCloser, error)
 //     Changes(id, parent string) ([]archive.Change, error)
-//     ApplyDiff(id, parent string, diff archive.Reader) (size int64, err error)
+//     ApplyDiff(id, parent string, diff io.Reader) (size int64, err error)
 //     DiffSize(id, parent string) (size int64, err error)
 func NewNaiveDiffDriver(driver ProtoDriver, uidMaps, gidMaps []idtools.IDMap) Driver {
-	gdw := &NaiveDiffDriver{
-		ProtoDriver: driver,
-		uidMaps:     uidMaps,
-		gidMaps:     gidMaps,
-	}
-	return gdw
+	return &NaiveDiffDriver{ProtoDriver: driver,
+		uidMaps: uidMaps,
+		gidMaps: gidMaps}
 }
 
 // Diff produces an archive of the changes between the specified
 // layer and its parent layer which may be "".
-func (gdw *NaiveDiffDriver) Diff(id, parent string) (arch archive.Archive, err error) {
-	layerFs, err := gdw.Get(id, "")
+func (gdw *NaiveDiffDriver) Diff(id, parent string) (arch io.ReadCloser, err error) {
+	startTime := time.Now()
+	driver := gdw.ProtoDriver
+
+	layerFs, err := driver.Get(id, "")
 	if err != nil {
 		return nil, err
 	}
 
 	defer func() {
 		if err != nil {
-			gdw.Put(id)
+			driver.Put(id)
 		}
 	}()
 
@@ -65,16 +65,16 @@ func (gdw *NaiveDiffDriver) Diff(id, parent string) (arch archive.Archive, err e
 		}
 		return ioutils.NewReadCloserWrapper(archive, func() error {
 			err := archive.Close()
-			gdw.Put(id)
+			driver.Put(id)
 			return err
 		}), nil
 	}
 
-	parentFs, err := gdw.Get(parent, "")
+	parentFs, err := driver.Get(parent, "")
 	if err != nil {
 		return nil, err
 	}
-	defer gdw.Put(parent)
+	defer driver.Put(parent)
 
 	changes, err := archive.ChangesDirs(layerFs, parentFs)
 	if err != nil {
@@ -88,7 +88,13 @@ func (gdw *NaiveDiffDriver) Diff(id, parent string) (arch archive.Archive, err e
 
 	return ioutils.NewReadCloserWrapper(archive, func() error {
 		err := archive.Close()
-		gdw.Put(id)
+		driver.Put(id)
+
+		// NaiveDiffDriver compares file metadata with parent layers. Parent layers
+		// are extracted from tar's with full second precision on modified time.
+		// We need this hack here to make sure calls within same second receive
+		// correct result.
+		time.Sleep(startTime.Truncate(time.Second).Add(time.Second).Sub(time.Now()))
 		return err
 	}), nil
 }
@@ -96,20 +102,22 @@ func (gdw *NaiveDiffDriver) Diff(id, parent string) (arch archive.Archive, err e
 // Changes produces a list of changes between the specified layer
 // and its parent layer. If parent is "", then all changes will be ADD changes.
 func (gdw *NaiveDiffDriver) Changes(id, parent string) ([]archive.Change, error) {
-	layerFs, err := gdw.Get(id, "")
+	driver := gdw.ProtoDriver
+
+	layerFs, err := driver.Get(id, "")
 	if err != nil {
 		return nil, err
 	}
-	defer gdw.Put(id)
+	defer driver.Put(id)
 
 	parentFs := ""
 
 	if parent != "" {
-		parentFs, err = gdw.Get(parent, "")
+		parentFs, err = driver.Get(parent, "")
 		if err != nil {
 			return nil, err
 		}
-		defer gdw.Put(parent)
+		defer driver.Put(parent)
 	}
 
 	return archive.ChangesDirs(layerFs, parentFs)
@@ -118,13 +126,15 @@ func (gdw *NaiveDiffDriver) Changes(id, parent string) ([]archive.Change, error)
 // ApplyDiff extracts the changeset from the given diff into the
 // layer with the specified id and parent, returning the size of the
 // new layer in bytes.
-func (gdw *NaiveDiffDriver) ApplyDiff(id, parent string, diff archive.Reader) (size int64, err error) {
+func (gdw *NaiveDiffDriver) ApplyDiff(id, parent string, diff io.Reader) (size int64, err error) {
+	driver := gdw.ProtoDriver
+
 	// Mount the root filesystem so we can apply the diff/layer.
-	layerFs, err := gdw.Get(id, "")
+	layerFs, err := driver.Get(id, "")
 	if err != nil {
 		return
 	}
-	defer gdw.Put(id)
+	defer driver.Put(id)
 
 	options := &archive.TarOptions{UIDMaps: gdw.uidMaps,
 		GIDMaps: gdw.gidMaps}
@@ -142,16 +152,18 @@ func (gdw *NaiveDiffDriver) ApplyDiff(id, parent string, diff archive.Reader) (s
 // and its parent and returns the size in bytes of the changes
 // relative to its base filesystem directory.
 func (gdw *NaiveDiffDriver) DiffSize(id, parent string) (size int64, err error) {
+	driver := gdw.ProtoDriver
+
 	changes, err := gdw.Changes(id, parent)
 	if err != nil {
 		return
 	}
 
-	layerFs, err := gdw.Get(id, "")
+	layerFs, err := driver.Get(id, "")
 	if err != nil {
 		return
 	}
-	defer gdw.Put(id)
+	defer driver.Put(id)
 
 	return archive.ChangesSize(layerFs, changes), nil
 }

vendor/github.com/containers/storage/drivers/overlay/check.go

@@ -0,0 +1,79 @@
+// +build linux
+
+package overlay
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+
+	"github.com/containers/storage/pkg/system"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
+)
+
+// hasOpaqueCopyUpBug checks whether the filesystem has a bug
+// which copies up the opaque flag when copying up an opaque
+// directory. When this bug exists naive diff should be used.
+func hasOpaqueCopyUpBug(d string) error {
+	td, err := ioutil.TempDir(d, "opaque-bug-check")
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := os.RemoveAll(td); err != nil {
+			logrus.Warnf("Failed to remove check directory %v: %v", td, err)
+		}
+	}()
+
+	// Make directories l1/d, l2/d, l3, work, merged
+	if err := os.MkdirAll(filepath.Join(td, "l1", "d"), 0755); err != nil {
+		return err
+	}
+	if err := os.MkdirAll(filepath.Join(td, "l2", "d"), 0755); err != nil {
+		return err
+	}
+	if err := os.Mkdir(filepath.Join(td, "l3"), 0755); err != nil {
+		return err
+	}
+	if err := os.Mkdir(filepath.Join(td, "work"), 0755); err != nil {
+		return err
+	}
+	if err := os.Mkdir(filepath.Join(td, "merged"), 0755); err != nil {
+		return err
+	}
+
+	// Mark l2/d as opaque
+	if err := system.Lsetxattr(filepath.Join(td, "l2", "d"), "trusted.overlay.opaque", []byte("y"), 0); err != nil {
+		return errors.Wrap(err, "failed to set opaque flag on middle layer")
+	}
+
+	opts := fmt.Sprintf("lowerdir=%s:%s,upperdir=%s,workdir=%s", path.Join(td, "l2"), path.Join(td, "l1"), path.Join(td, "l3"), path.Join(td, "work"))
+	if err := unix.Mount("overlay", filepath.Join(td, "merged"), "overlay", 0, opts); err != nil {
+		return errors.Wrap(err, "failed to mount overlay")
+	}
+	defer func() {
+		if err := unix.Unmount(filepath.Join(td, "merged"), 0); err != nil {
+			logrus.Warnf("Failed to unmount check directory %v: %v", filepath.Join(td, "merged"), err)
+		}
+	}()
+
+	// Touch file in d to force copy up of opaque directory "d" from "l2" to "l3"
+	if err := ioutil.WriteFile(filepath.Join(td, "merged", "d", "f"), []byte{}, 0644); err != nil {
+		return errors.Wrap(err, "failed to write to merged directory")
+	}
+
+	// Check l3/d does not have opaque flag
+	xattrOpaque, err := system.Lgetxattr(filepath.Join(td, "l3", "d"), "trusted.overlay.opaque")
+	if err != nil {
+		return errors.Wrap(err, "failed to read opaque flag on upper layer")
+	}
+	if string(xattrOpaque) == "y" {
+		return errors.New("opaque flag erroneously copied up, consider update to kernel 4.8 or later to fix")
+	}
+
+	return nil
+}

vendor/github.com/containers/storage/drivers/overlay/mount.go

@@ -9,9 +9,9 @@ import (
 	"fmt"
 	"os"
 	"runtime"
-	"syscall"
 
 	"github.com/containers/storage/pkg/reexec"
+	"golang.org/x/sys/unix"
 )
 
 func init() {
@@ -31,12 +31,12 @@ type mountOptions struct {
 	Flag   uint32
 }
 
-func mountFrom(dir, device, target, mType, label string) error {
+func mountFrom(dir, device, target, mType string, flags uintptr, label string) error {
 	options := &mountOptions{
 		Device: device,
 		Target: target,
 		Type:   mType,
-		Flag:   0,
+		Flag:   uint32(flags),
 		Label:  label,
 	}
 
@@ -51,16 +51,18 @@ func mountFrom(dir, device, target, mType, label string) error {
 	cmd.Stderr = output
 
 	if err := cmd.Start(); err != nil {
+		w.Close()
 		return fmt.Errorf("mountfrom error on re-exec cmd: %v", err)
 	}
 	//write the options to the pipe for the untar exec to read
 	if err := json.NewEncoder(w).Encode(options); err != nil {
+		w.Close()
 		return fmt.Errorf("mountfrom json encode to pipe failed: %v", err)
 	}
 	w.Close()
 
 	if err := cmd.Wait(); err != nil {
-		return fmt.Errorf("mountfrom re-exec error: %v: output: %s", err, output)
+		return fmt.Errorf("mountfrom re-exec error: %v: output: %v", err, output)
 	}
 	return nil
 }
@@ -80,7 +82,7 @@ func mountFromMain() {
 		fatal(err)
 	}
 
-	if err := syscall.Mount(options.Device, options.Target, options.Type, uintptr(options.Flag), options.Label); err != nil {
+	if err := unix.Mount(options.Device, options.Target, options.Type, uintptr(options.Flag), options.Label); err != nil {
 		fatal(err)
 	}
 

vendor/github.com/containers/storage/drivers/overlay/overlay.go

@@ -5,6 +5,7 @@ package overlay
 import (
 	"bufio"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -12,21 +13,26 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
-	"syscall"
-
-	"github.com/Sirupsen/logrus"
+	"sync"
 
 	"github.com/containers/storage/drivers"
+	"github.com/containers/storage/drivers/overlayutils"
+	"github.com/containers/storage/drivers/quota"
 	"github.com/containers/storage/pkg/archive"
 	"github.com/containers/storage/pkg/chrootarchive"
 	"github.com/containers/storage/pkg/directory"
+	"github.com/containers/storage/pkg/fsutils"
 	"github.com/containers/storage/pkg/idtools"
+	"github.com/containers/storage/pkg/locker"
 	"github.com/containers/storage/pkg/mount"
 	"github.com/containers/storage/pkg/parsers"
 	"github.com/containers/storage/pkg/parsers/kernel"
-
+	"github.com/containers/storage/pkg/system"
+	units "github.com/docker/go-units"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
 )
 
 var (
@@ -42,7 +48,7 @@ var (
 
 // Each container/image has at least a "diff" directory and "link" file.
 // If there is also a "lower" file when there are diff layers
-// below  as well as "merged" and "work" directories. The "diff" directory
+// below as well as "merged" and "work" directories. The "diff" directory
 // has the upper layer of the overlay and is used to capture any
 // changes to the layer. The "lower" file contains all the lower layer
 // mounts separated by ":" and ordered from uppermost to lowermost
@@ -76,26 +82,43 @@ const (
 	idLength = 26
 )
 
+type overlayOptions struct {
+	overrideKernelCheck bool
+	imageStores         []string
+	quota               quota.Quota
+}
+
 // Driver contains information about the home directory and the list of active mounts that are created using this driver.
 type Driver struct {
-	name    string
-	home    string
-	uidMaps []idtools.IDMap
-	gidMaps []idtools.IDMap
-	ctr     *graphdriver.RefCounter
-	opts    *overlayOptions
+	name          string
+	home          string
+	uidMaps       []idtools.IDMap
+	gidMaps       []idtools.IDMap
+	ctr           *graphdriver.RefCounter
+	quotaCtl      *quota.Control
+	options       overlayOptions
+	naiveDiff     graphdriver.DiffDriver
+	supportsDType bool
+	locker        *locker.Locker
 }
 
-var backingFs = "<unknown>"
+var (
+	backingFs             = "<unknown>"
+	projectQuotaSupported = false
+
+	useNaiveDiffLock sync.Once
+	useNaiveDiffOnly bool
+)
 
 func init() {
-	graphdriver.Register("overlay", InitAsOverlay)
-	graphdriver.Register("overlay2", InitAsOverlay2)
+	graphdriver.Register("overlay", Init)
+	graphdriver.Register("overlay2", Init)
 }
 
-// InitWithName returns the a naive diff driver for the overlay filesystem,
-// which returns the passed-in name when asked which driver it is.
-func InitWithName(name, home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) {
+// Init returns the a native diff driver for overlay filesystem.
+// If overlay filesystem is not supported on the host, graphdriver.ErrNotSupported is returned as error.
+// If an overlay filesystem is not supported over an existing filesystem then error graphdriver.ErrIncompatibleFS is returned.
+func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) {
 	opts, err := parseOptions(options)
 	if err != nil {
 		return nil, err
@@ -114,7 +137,7 @@ func InitWithName(name, home string, options []string, uidMaps, gidMaps []idtool
 		if !opts.overrideKernelCheck {
 			return nil, errors.Wrap(graphdriver.ErrNotSupported, "kernel too old to provide multiple lowers feature for overlay")
 		}
-		logrus.Warnf("Using pre-4.0.0 kernel for overlay, mount failures may require kernel update")
+		logrus.Warn("Using pre-4.0.0 kernel for overlay, mount failures may require kernel update")
 	}
 
 	fsMagic, err := graphdriver.GetFSMagic(home)
@@ -127,9 +150,19 @@ func InitWithName(name, home string, options []string, uidMaps, gidMaps []idtool
 
 	// check if they are running over btrfs, aufs, zfs, overlay, or ecryptfs
 	switch fsMagic {
-	case graphdriver.FsMagicBtrfs, graphdriver.FsMagicAufs, graphdriver.FsMagicZfs, graphdriver.FsMagicOverlay, graphdriver.FsMagicEcryptfs:
+	case graphdriver.FsMagicAufs, graphdriver.FsMagicZfs, graphdriver.FsMagicOverlay, graphdriver.FsMagicEcryptfs:
 		logrus.Errorf("'overlay' is not supported over %s", backingFs)
 		return nil, errors.Wrapf(graphdriver.ErrIncompatibleFS, "'overlay' is not supported over %s", backingFs)
+	case graphdriver.FsMagicBtrfs:
+		// Support for OverlayFS on BTRFS was added in kernel 4.7
+		// See https://btrfs.wiki.kernel.org/index.php/Changelog
+		if kernel.CompareKernelVersion(*v, kernel.VersionInfo{Kernel: 4, Major: 7, Minor: 0}) < 0 {
+			if !opts.overrideKernelCheck {
+				logrus.Errorf("'overlay' requires kernel 4.7 to use on %s", backingFs)
+				return nil, errors.Wrapf(graphdriver.ErrIncompatibleFS, "'overlay' requires kernel 4.7 to use on %s", backingFs)
+			}
+			logrus.Warn("Using pre-4.7.0 kernel for overlay on btrfs, may require kernel update")
+		}
 	}
 
 	rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)
@@ -145,37 +178,46 @@ func InitWithName(name, home string, options []string, uidMaps, gidMaps []idtool
 		return nil, err
 	}
 
-	d := &Driver{
-		name:    name,
-		home:    home,
-		uidMaps: uidMaps,
-		gidMaps: gidMaps,
-		ctr:     graphdriver.NewRefCounter(graphdriver.NewFsChecker(graphdriver.FsMagicOverlay)),
-		opts:    opts,
+	supportsDType, err := fsutils.SupportsDType(home)
+	if err != nil {
+		return nil, err
+	}
+	if !supportsDType {
+		logrus.Warn(overlayutils.ErrDTypeNotSupported("overlay", backingFs))
+		// TODO: Will make fatal when CRI-O Has AMI built on RHEL7.4
+		// return nil, overlayutils.ErrDTypeNotSupported("overlay", backingFs)
 	}
 
+	d := &Driver{
+		name:          "overlay",
+		home:          home,
+		uidMaps:       uidMaps,
+		gidMaps:       gidMaps,
+		ctr:           graphdriver.NewRefCounter(graphdriver.NewFsChecker(graphdriver.FsMagicOverlay)),
+		supportsDType: supportsDType,
+		locker:        locker.New(),
+		options:       *opts,
+	}
+
+	d.naiveDiff = graphdriver.NewNaiveDiffDriver(d, uidMaps, gidMaps)
+
+	if backingFs == "xfs" {
+		// Try to enable project quota support over xfs.
+		if d.quotaCtl, err = quota.NewControl(home); err == nil {
+			projectQuotaSupported = true
+		} else if opts.quota.Size > 0 {
+			return nil, fmt.Errorf("Storage option overlay.size not supported. Filesystem does not support Project Quota: %v", err)
+		}
+	} else if opts.quota.Size > 0 {
+		// if xfs is not the backing fs then error out if the storage-opt overlay.size is used.
+		return nil, fmt.Errorf("Storage Option overlay.size only supported for backingFS XFS. Found %v", backingFs)
+	}
+
+	logrus.Debugf("backingFs=%s,  projectQuotaSupported=%v", backingFs, projectQuotaSupported)
+
 	return d, nil
 }
 
-// InitAsOverlay returns the a naive diff driver for overlay filesystem.
-// If overlay filesystem is not supported on the host, graphdriver.ErrNotSupported is returned as error.
-// If a overlay filesystem is not supported over a existing filesystem then error graphdriver.ErrIncompatibleFS is returned.
-func InitAsOverlay(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) {
-	return InitWithName("overlay", home, options, uidMaps, gidMaps)
-}
-
-// InitAsOverlay2 returns the a naive diff driver for overlay filesystem.
-// If overlay filesystem is not supported on the host, graphdriver.ErrNotSupported is returned as error.
-// If a overlay filesystem is not supported over a existing filesystem then error graphdriver.ErrIncompatibleFS is returned.
-func InitAsOverlay2(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) {
-	return InitWithName("overlay2", home, options, uidMaps, gidMaps)
-}
-
-type overlayOptions struct {
-	overrideKernelCheck bool
-	imageStores         []string
-}
-
 func parseOptions(options []string) (*overlayOptions, error) {
 	o := &overlayOptions{}
 	for _, option := range options {
@@ -186,11 +228,20 @@ func parseOptions(options []string) (*overlayOptions, error) {
 		key = strings.ToLower(key)
 		switch key {
 		case "overlay.override_kernel_check", "overlay2.override_kernel_check":
+			logrus.Debugf("overlay: overide_kernelcheck=%s", val)
 			o.overrideKernelCheck, err = strconv.ParseBool(val)
 			if err != nil {
 				return nil, err
 			}
-		case "overlay.imagestore":
+		case "overlay.size", "overlay2.size":
+			logrus.Debugf("overlay: size=%s", val)
+			size, err := units.RAMInBytes(val)
+			if err != nil {
+				return nil, err
+			}
+			o.quota.Size = uint64(size)
+		case "overlay.imagestore", "overlay2.imagestore":
+			logrus.Debugf("overlay: imagestore=%s", val)
 			// Additional read only image stores to use for lower paths
 			for _, store := range strings.Split(val, ",") {
 				store = filepath.Clean(store)
@@ -199,7 +250,7 @@ func parseOptions(options []string) (*overlayOptions, error) {
 				}
 				st, err := os.Stat(store)
 				if err != nil {
-					return nil, fmt.Errorf("overlay: Can't stat imageStore dir %s: %v", store, err)
+					return nil, fmt.Errorf("overlay: can't stat imageStore dir %s: %v", store, err)
 				}
 				if !st.IsDir() {
 					return nil, fmt.Errorf("overlay: image path %q must be a directory", store)
@@ -234,6 +285,16 @@ func supportsOverlay() error {
 	return errors.Wrap(graphdriver.ErrNotSupported, "'overlay' not found as a supported filesystem on this host. Please ensure kernel is new enough and has overlay support loaded.")
 }
 
+func useNaiveDiff(home string) bool {
+	useNaiveDiffLock.Do(func() {
+		if err := hasOpaqueCopyUpBug(home); err != nil {
+			logrus.Warnf("Not using native diff for overlay: %v", err)
+			useNaiveDiffOnly = true
+		}
+	})
+	return useNaiveDiffOnly
+}
+
 func (d *Driver) String() string {
 	return d.name
 }
@@ -243,6 +304,8 @@ func (d *Driver) String() string {
 func (d *Driver) Status() [][2]string {
 	return [][2]string{
 		{"Backing Filesystem", backingFs},
+		{"Supports d_type", strconv.FormatBool(d.supportsDType)},
+		{"Native Overlay Diff", strconv.FormatBool(!useNaiveDiff(d.home))},
 	}
 }
 
@@ -280,18 +343,39 @@ func (d *Driver) Cleanup() error {
 
 // CreateReadWrite creates a layer that is writable for use as a container
 // file system.
-func (d *Driver) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error {
-	return d.Create(id, parent, mountLabel, storageOpt)
+func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error {
+	if opts != nil && len(opts.StorageOpt) != 0 && !projectQuotaSupported {
+		return fmt.Errorf("--storage-opt is supported only for overlay over xfs with 'pquota' mount option")
+	}
+
+	if opts == nil {
+		opts = &graphdriver.CreateOpts{
+			StorageOpt: map[string]string{},
+		}
+	}
+
+	if _, ok := opts.StorageOpt["size"]; !ok {
+		if opts.StorageOpt == nil {
+			opts.StorageOpt = map[string]string{}
+		}
+		opts.StorageOpt["size"] = strconv.FormatUint(d.options.quota.Size, 10)
+	}
+
+	return d.create(id, parent, opts)
 }
 
 // Create is used to create the upper, lower, and merge directories required for overlay fs for a given id.
 // The parent filesystem is used to configure these directories for the overlay.
-func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]string) (retErr error) {
-
-	if len(storageOpt) != 0 {
-		return fmt.Errorf("--storage-opt is not supported for overlay")
+func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) (retErr error) {
+	if opts != nil && len(opts.StorageOpt) != 0 {
+		if _, ok := opts.StorageOpt["size"]; ok {
+			return fmt.Errorf("--storage-opt size is only supported for ReadWrite Layers")
+		}
 	}
+	return d.create(id, parent, opts)
+}
 
+func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts) (retErr error) {
 	dir := d.dir(id)
 
 	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
@@ -312,6 +396,20 @@ func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]str
 		}
 	}()
 
+	if opts != nil && len(opts.StorageOpt) > 0 {
+		driver := &Driver{}
+		if err := d.parseStorageOpt(opts.StorageOpt, driver); err != nil {
+			return err
+		}
+
+		if driver.options.quota.Size > 0 {
+			// Set container disk quota limit
+			if err := d.quotaCtl.SetQuota(dir, driver.options.quota); err != nil {
+				return err
+			}
+		}
+	}
+
 	if err := idtools.MkdirAs(path.Join(dir, "diff"), 0755, rootUID, rootGID); err != nil {
 		return err
 	}
@@ -351,6 +449,26 @@ func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]str
 	return nil
 }
 
+// Parse overlay storage options
+func (d *Driver) parseStorageOpt(storageOpt map[string]string, driver *Driver) error {
+	// Read size to set the disk project quota per container
+	for key, val := range storageOpt {
+		key := strings.ToLower(key)
+		switch key {
+		case "size":
+			size, err := units.RAMInBytes(val)
+			if err != nil {
+				return err
+			}
+			driver.options.quota.Size = uint64(size)
+		default:
+			return fmt.Errorf("Unknown option %s", key)
+		}
+	}
+
+	return nil
+}
+
 func (d *Driver) getLower(parent string) (string, error) {
 	parentDir := d.dir(parent)
 
@@ -377,11 +495,11 @@ func (d *Driver) getLower(parent string) (string, error) {
 	return strings.Join(lowers, ":"), nil
 }
 
-func (d *Driver) dir(val string) string {
-	newpath := path.Join(d.home, val)
+func (d *Driver) dir(id string) string {
+	newpath := path.Join(d.home, id)
 	if _, err := os.Stat(newpath); err != nil {
 		for _, p := range d.AdditionalImageStores() {
-			l := path.Join(p, d.name, val)
+			l := path.Join(p, d.name, id)
 			_, err = os.Stat(l)
 			if err == nil {
 				return l
@@ -411,6 +529,8 @@ func (d *Driver) getLowerDirs(id string) ([]string, error) {
 
 // Remove cleans the directories that are created for this id.
 func (d *Driver) Remove(id string) error {
+	d.locker.Lock(id)
+	defer d.locker.Unlock(id)
 	dir := d.dir(id)
 	lid, err := ioutil.ReadFile(path.Join(dir, "link"))
 	if err == nil {
@@ -419,14 +539,16 @@ func (d *Driver) Remove(id string) error {
 		}
 	}
 
-	if err := os.RemoveAll(dir); err != nil && !os.IsNotExist(err) {
+	if err := system.EnsureRemoveAll(dir); err != nil && !os.IsNotExist(err) {
 		return err
 	}
 	return nil
 }
 
 // Get creates and mounts the required file system for the given id and returns the mount path.
-func (d *Driver) Get(id string, mountLabel string) (s string, err error) {
+func (d *Driver) Get(id, mountLabel string) (_ string, retErr error) {
+	d.locker.Lock(id)
+	defer d.locker.Unlock(id)
 	dir := d.dir(id)
 	if _, err := os.Stat(dir); err != nil {
 		return "", err
@@ -458,7 +580,7 @@ func (d *Driver) Get(id string, mountLabel string) (s string, err error) {
 				return "", fmt.Errorf("Can't stat lower layer %q: %v", newpath, err)
 			}
 		} else {
-			lower = l
+			lower = newpath
 		}
 		if newlowers == "" {
 			newlowers = lower
@@ -472,22 +594,42 @@ func (d *Driver) Get(id string, mountLabel string) (s string, err error) {
 		return mergedDir, nil
 	}
 	defer func() {
-		if err != nil {
+		if retErr != nil {
 			if c := d.ctr.Decrement(mergedDir); c <= 0 {
-				syscall.Unmount(mergedDir, 0)
+				if mntErr := unix.Unmount(mergedDir, 0); mntErr != nil {
+					logrus.Errorf("error unmounting %v: %v", mergedDir, mntErr)
+				}
 			}
 		}
 	}()
 
 	workDir := path.Join(dir, "work")
-	opts := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", newlowers, path.Join(id, "diff"), path.Join(id, "work"))
-	mountLabel = label.FormatMountLabel(opts, mountLabel)
-	if len(mountLabel) > syscall.Getpagesize() {
-		return "", fmt.Errorf("cannot mount layer, mount label too large %d", len(mountLabel))
-	}
+	opts := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", newlowers, diffDir, workDir)
+	mountData := label.FormatMountLabel(opts, mountLabel)
+	mount := unix.Mount
+	mountTarget := mergedDir
 
-	if err := mountFrom(d.home, "overlay", path.Join(id, "merged"), "overlay", mountLabel); err != nil {
-		return "", fmt.Errorf("error creating overlay mount to %s: %v", mergedDir, err)
+	pageSize := unix.Getpagesize()
+
+	// Use relative paths and mountFrom when the mount data has exceeded
+	// the page size. The mount syscall fails if the mount data cannot
+	// fit within a page and relative links make the mount data much
+	// smaller at the expense of requiring a fork exec to chroot.
+	if len(mountData) > pageSize {
+		//FIXME: We need to figure out to get this to work with additional stores
+		opts = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", string(lowers), path.Join(id, "diff"), path.Join(id, "work"))
+		mountData = label.FormatMountLabel(opts, mountLabel)
+		if len(mountData) > pageSize {
+			return "", fmt.Errorf("cannot mount layer, mount label too large %d", len(mountData))
+		}
+
+		mount = func(source string, target string, mType string, flags uintptr, label string) error {
+			return mountFrom(d.home, source, target, mType, flags, label)
+		}
+		mountTarget = path.Join(id, "merged")
+	}
+	if err := mount("overlay", mountTarget, "overlay", 0, mountData); err != nil {
+		return "", fmt.Errorf("error creating overlay mount to %s: %v", mountTarget, err)
 	}
 
 	// chown "workdir/work" to the remapped root UID/GID. Overlay fs inside a
@@ -506,19 +648,17 @@ func (d *Driver) Get(id string, mountLabel string) (s string, err error) {
 
 // Put unmounts the mount path created for the give id.
 func (d *Driver) Put(id string) error {
+	d.locker.Lock(id)
+	defer d.locker.Unlock(id)
 	mountpoint := path.Join(d.dir(id), "merged")
 	if count := d.ctr.Decrement(mountpoint); count > 0 {
 		return nil
 	}
-	err := syscall.Unmount(mountpoint, 0)
+	err := unix.Unmount(mountpoint, unix.MNT_DETACH)
 	if err != nil {
-		if _, err := ioutil.ReadFile(path.Join(d.dir(id), lowerFile)); err != nil {
-			// We didn't have a "lower" directory, so we weren't mounting a "merged" directory anyway
-			return nil
-		}
-		logrus.Debugf("Failed to unmount %s overlay: %v", id, err)
+		logrus.Debugf("Failed to unmount %s overlay: %s - %v", id, mountpoint, err)
 	}
-	return err
+	return nil
 }
 
 // Exists checks to see if the id is already mounted.
@@ -527,8 +667,33 @@ func (d *Driver) Exists(id string) bool {
 	return err == nil
 }
 
+// isParent returns if the passed in parent is the direct parent of the passed in layer
+func (d *Driver) isParent(id, parent string) bool {
+	lowers, err := d.getLowerDirs(id)
+	if err != nil {
+		return false
+	}
+	if parent == "" && len(lowers) > 0 {
+		return false
+	}
+
+	parentDir := d.dir(parent)
+	var ld string
+	if len(lowers) > 0 {
+		ld = filepath.Dir(lowers[0])
+	}
+	if ld == "" && parent == "" {
+		return true
+	}
+	return ld == parentDir
+}
+
 // ApplyDiff applies the new layer into a root
-func (d *Driver) ApplyDiff(id string, parent string, diff archive.Reader) (size int64, err error) {
+func (d *Driver) ApplyDiff(id string, parent string, diff io.Reader) (size int64, err error) {
+	if !d.isParent(id, parent) {
+		return d.naiveDiff.ApplyDiff(id, parent, diff)
+	}
+
 	applyDir := d.getDiffPath(id)
 
 	logrus.Debugf("Applying tar in %s", applyDir)
@@ -541,7 +706,7 @@ func (d *Driver) ApplyDiff(id string, parent string, diff archive.Reader) (size
 		return 0, err
 	}
 
-	return d.DiffSize(id, parent)
+	return directory.Size(applyDir)
 }
 
 func (d *Driver) getDiffPath(id string) string {
@@ -554,12 +719,19 @@ func (d *Driver) getDiffPath(id string) string {
 // and its parent and returns the size in bytes of the changes
 // relative to its base filesystem directory.
 func (d *Driver) DiffSize(id, parent string) (size int64, err error) {
+	if useNaiveDiff(d.home) || !d.isParent(id, parent) {
+		return d.naiveDiff.DiffSize(id, parent)
+	}
 	return directory.Size(d.getDiffPath(id))
 }
 
 // Diff produces an archive of the changes between the specified
 // layer and its parent layer which may be "".
-func (d *Driver) Diff(id, parent string) (archive.Archive, error) {
+func (d *Driver) Diff(id, parent string) (io.ReadCloser, error) {
+	if useNaiveDiff(d.home) || !d.isParent(id, parent) {
+		return d.naiveDiff.Diff(id, parent)
+	}
+
 	diffPath := d.getDiffPath(id)
 	logrus.Debugf("Tar with options on %s", diffPath)
 	return archive.TarWithOptions(diffPath, &archive.TarOptions{
@@ -573,6 +745,9 @@ func (d *Driver) Diff(id, parent string) (archive.Archive, error) {
 // Changes produces a list of changes between the specified layer
 // and its parent layer. If parent is "", then all changes will be ADD changes.
 func (d *Driver) Changes(id, parent string) ([]archive.Change, error) {
+	if useNaiveDiff(d.home) || !d.isParent(id, parent) {
+		return d.naiveDiff.Changes(id, parent)
+	}
 	// Overlay doesn't have snapshots, so we need to get changes from all parent
 	// layers.
 	diffPath := d.getDiffPath(id)
@@ -586,5 +761,5 @@ func (d *Driver) Changes(id, parent string) ([]archive.Change, error) {
 
 // AdditionalImageStores returns additional image stores supported by the driver
 func (d *Driver) AdditionalImageStores() []string {
-	return d.opts.imageStores
+	return d.options.imageStores
 }