Bump github.com/containers/storage from 1.26.0 to 1.29.0

Bumps [github.com/containers/storage](https://github.com/containers/storage) from 1.26.0 to 1.29.0.
- [Release notes](https://github.com/containers/storage/releases)
- [Changelog](https://github.com/containers/storage/blob/master/docs/containers-storage-changes.md)
- [Commits](https://github.com/containers/storage/compare/v1.26.0...v1.29.0)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
Signed-off-by: Daniel J Walsh <dwalsh@redhat.com>

vendor/github.com/containers/storage/drivers/overlay/overlay.go

@@ -4,6 +4,7 @@ package overlay
 
 import (
 	"bytes"
+	"encoding/base64"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -30,7 +31,10 @@ import (
 	"github.com/containers/storage/pkg/system"
 	"github.com/containers/storage/pkg/unshare"
 	units "github.com/docker/go-units"
+	"github.com/hashicorp/go-multierror"
+	digest "github.com/opencontainers/go-digest"
 	rsystem "github.com/opencontainers/runc/libcontainer/system"
+	"github.com/opencontainers/selinux/go-selinux"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -43,7 +47,10 @@ var (
 	untar = chrootarchive.UntarUncompressed
 )
 
-const defaultPerms = os.FileMode(0555)
+const (
+	defaultPerms     = os.FileMode(0555)
+	selinuxLabelTest = "system_u:object_r:container_file_t:s0"
+)
 
 // This backend uses the overlay union filesystem for containers
 // with diff directories for each layer.
@@ -89,6 +96,7 @@ const (
 
 type overlayOptions struct {
 	imageStores       []string
+	layerStores       []additionalLayerStore
 	quota             quota.Quota
 	mountProgram      string
 	skipMountHome     bool
@@ -114,6 +122,17 @@ type Driver struct {
 	locker           *locker.Locker
 }
 
+type additionalLayerStore struct {
+
+	// path is the directory where this store is available on the host.
+	path string
+
+	// withReference is true when the store contains image reference information (base64-encoded)
+	// in its layer search path so the path to the diff will be
+	//  <path>/base64(reference)/<layerdigest>/
+	withReference bool
+}
+
 var (
 	backingFs             = "<unknown>"
 	projectQuotaSupported = false
@@ -145,6 +164,10 @@ func hasVolatileOption(opts []string) bool {
 	return false
 }
 
+func getMountProgramFlagFile(path string) string {
+	return filepath.Join(path, ".has-mount-program")
+}
+
 func checkSupportVolatile(home, runhome string) (bool, error) {
 	feature := fmt.Sprintf("volatile")
 	volatileCacheResult, _, err := cachedFeatureCheck(runhome, feature)
@@ -172,6 +195,47 @@ func checkSupportVolatile(home, runhome string) (bool, error) {
 	return usingVolatile, nil
 }
 
+func checkAndRecordOverlaySupport(fsMagic graphdriver.FsMagic, home, runhome string) (bool, error) {
+	var supportsDType bool
+
+	if os.Geteuid() != 0 {
+		return false, nil
+	}
+
+	feature := "overlay"
+	overlayCacheResult, overlayCacheText, err := cachedFeatureCheck(runhome, feature)
+	if err == nil {
+		if overlayCacheResult {
+			logrus.Debugf("cached value indicated that overlay is supported")
+		} else {
+			logrus.Debugf("cached value indicated that overlay is not supported")
+		}
+		supportsDType = overlayCacheResult
+		if !supportsDType {
+			return false, errors.New(overlayCacheText)
+		}
+	} else {
+		supportsDType, err = supportsOverlay(home, fsMagic, 0, 0)
+		if err != nil {
+			os.Remove(filepath.Join(home, linkDir))
+			os.Remove(home)
+			patherr, ok := err.(*os.PathError)
+			if ok && patherr.Err == syscall.ENOSPC {
+				return false, err
+			}
+			err = errors.Wrap(err, "kernel does not support overlay fs")
+			if err2 := cachedFeatureRecord(runhome, feature, false, err.Error()); err2 != nil {
+				return false, errors.Wrapf(err2, "error recording overlay not being supported (%v)", err)
+			}
+			return false, err
+		}
+		if err = cachedFeatureRecord(runhome, feature, supportsDType, ""); err != nil {
+			return false, errors.Wrap(err, "error recording overlay support status")
+		}
+	}
+	return supportsDType, nil
+}
+
 // Init returns the a native diff driver for overlay filesystem.
 // If overlay filesystem is not supported on the host, a wrapped graphdriver.ErrNotSupported is returned as error.
 // If an overlay filesystem is not supported over an existing filesystem then a wrapped graphdriver.ErrIncompatibleFS is returned.
@@ -189,8 +253,13 @@ func Init(home string, options graphdriver.Options) (graphdriver.Driver, error)
 		backingFs = fsName
 	}
 
-	// check if they are running over btrfs, aufs, zfs, overlay, or ecryptfs
-	if opts.mountProgram == "" {
+	if opts.mountProgram != "" {
+		f, err := os.Create(getMountProgramFlagFile(home))
+		if err == nil {
+			f.Close()
+		}
+	} else {
+		// check if they are running over btrfs, aufs, zfs, overlay, or ecryptfs
 		if opts.forceMask != nil {
 			return nil, errors.New("'force_mask' is supported only with 'mount_program'")
 		}
@@ -221,39 +290,11 @@ func Init(home string, options graphdriver.Options) (graphdriver.Driver, error)
 		supportsDType = true
 		supportsVolatile = true
 	} else {
-		feature := "overlay"
-		overlayCacheResult, overlayCacheText, err := cachedFeatureCheck(runhome, feature)
-		if err == nil {
-			if overlayCacheResult {
-				logrus.Debugf("cached value indicated that overlay is supported")
-			} else {
-				logrus.Debugf("cached value indicated that overlay is not supported")
-			}
-			supportsDType = overlayCacheResult
-			if !supportsDType {
-				return nil, errors.New(overlayCacheText)
-			}
-		} else {
-			supportsDType, err = supportsOverlay(home, fsMagic, rootUID, rootGID)
-			if err != nil {
-				os.Remove(filepath.Join(home, linkDir))
-				os.Remove(home)
-				patherr, ok := err.(*os.PathError)
-				if ok && patherr.Err == syscall.ENOSPC {
-					return nil, err
-				}
-				err = errors.Wrap(err, "kernel does not support overlay fs")
-				if err2 := cachedFeatureRecord(runhome, feature, false, err.Error()); err2 != nil {
-					return nil, errors.Wrapf(err2, "error recording overlay not being supported (%v)", err)
-				}
-				return nil, err
-			}
-			if err = cachedFeatureRecord(runhome, feature, supportsDType, ""); err != nil {
-				return nil, errors.Wrap(err, "error recording overlay support status")
-			}
+		supportsDType, err = checkAndRecordOverlaySupport(fsMagic, home, runhome)
+		if err != nil {
+			return nil, err
 		}
-
-		feature = fmt.Sprintf("metacopy(%s)", opts.mountOptions)
+		feature := fmt.Sprintf("metacopy(%s)", opts.mountOptions)
 		metacopyCacheResult, _, err := cachedFeatureCheck(runhome, feature)
 		if err == nil {
 			if metacopyCacheResult {
@@ -370,11 +411,49 @@ func parseOptions(options []string) (*overlayOptions, error) {
 				}
 				o.imageStores = append(o.imageStores, store)
 			}
+		case "additionallayerstore":
+			logrus.Debugf("overlay: additionallayerstore=%s", val)
+			// Additional read only layer stores to use for lower paths
+			if val == "" {
+				continue
+			}
+			for _, lstore := range strings.Split(val, ",") {
+				elems := strings.Split(lstore, ":")
+				lstore = filepath.Clean(elems[0])
+				if !filepath.IsAbs(lstore) {
+					return nil, fmt.Errorf("overlay: additionallayerstore path %q is not absolute.  Can not be relative", lstore)
+				}
+				st, err := os.Stat(lstore)
+				if err != nil {
+					return nil, errors.Wrap(err, "overlay: can't stat additionallayerstore dir")
+				}
+				if !st.IsDir() {
+					return nil, fmt.Errorf("overlay: additionallayerstore path %q must be a directory", lstore)
+				}
+				var withReference bool
+				for _, e := range elems[1:] {
+					switch e {
+					case "ref":
+						if withReference {
+							return nil, fmt.Errorf("overlay: additionallayerstore config of %q contains %q option twice", lstore, e)
+						}
+						withReference = true
+					default:
+						return nil, fmt.Errorf("overlay: additionallayerstore config %q contains unknown option %q", lstore, e)
+					}
+				}
+				o.layerStores = append(o.layerStores, additionalLayerStore{
+					path:          lstore,
+					withReference: withReference,
+				})
+			}
 		case "mount_program":
 			logrus.Debugf("overlay: mount_program=%s", val)
-			_, err := os.Stat(val)
-			if err != nil {
-				return nil, fmt.Errorf("overlay: can't stat program %s: %v", val, err)
+			if val != "" {
+				_, err := os.Stat(val)
+				if err != nil {
+					return nil, errors.Wrapf(err, "overlay: can't stat program %q", val)
+				}
 			}
 			o.mountProgram = val
 		case "skip_mount_home":
@@ -439,11 +518,44 @@ func cachedFeatureRecord(runhome, feature string, supported bool, text string) (
 	return err
 }
 
+func SupportsNativeOverlay(graphroot, rundir string) (bool, error) {
+	if os.Geteuid() != 0 || graphroot == "" || rundir == "" {
+		return false, nil
+	}
+
+	home := filepath.Join(graphroot, "overlay")
+	runhome := filepath.Join(rundir, "overlay")
+
+	if _, err := os.Stat(getMountProgramFlagFile(home)); err == nil {
+		logrus.Debugf("overlay storage already configured with a mount-program")
+		return false, nil
+	}
+
+	for _, dir := range []string{home, runhome} {
+		if _, err := os.Stat(dir); err != nil {
+			_ = idtools.MkdirAllAs(dir, 0700, 0, 0)
+		}
+	}
+
+	fsMagic, err := graphdriver.GetFSMagic(home)
+	if err != nil {
+		return false, err
+	}
+
+	supportsDType, _ := checkAndRecordOverlaySupport(fsMagic, home, runhome)
+	return supportsDType, nil
+}
+
 func supportsOverlay(home string, homeMagic graphdriver.FsMagic, rootUID, rootGID int) (supportsDType bool, err error) {
 	// We can try to modprobe overlay first
 
 	exec.Command("modprobe", "overlay").Run()
 
+	logLevel := logrus.ErrorLevel
+	if unshare.IsRootless() {
+		logLevel = logrus.DebugLevel
+	}
+
 	layerDir, err := ioutil.TempDir(home, "compat")
 	if err != nil {
 		patherr, ok := err.(*os.PathError)
@@ -482,8 +594,18 @@ func supportsOverlay(home string, homeMagic graphdriver.FsMagic, rootUID, rootGI
 		_ = idtools.MkdirAs(upperDir, 0700, rootUID, rootGID)
 		_ = idtools.MkdirAs(workDir, 0700, rootUID, rootGID)
 		flags := fmt.Sprintf("lowerdir=%s:%s,upperdir=%s,workdir=%s", lower1Dir, lower2Dir, upperDir, workDir)
+		if selinux.GetEnabled() {
+			// Linux 5.11 introduced unprivileged overlay mounts but it has an issue
+			// when used together with selinux labels.
+			// Check that overlay supports selinux labels as well.
+			flags = label.FormatMountLabel(flags, selinuxLabelTest)
+		}
+		if unshare.IsRootless() {
+			flags = fmt.Sprintf("%s,userxattr", flags)
+		}
+
 		if len(flags) < unix.Getpagesize() {
-			err := mountFrom(filepath.Dir(home), "overlay", mergedDir, "overlay", 0, flags)
+			err := unix.Mount("overlay", mergedDir, "overlay", 0, flags)
 			if err == nil {
 				logrus.Debugf("overlay test mount with multiple lowers succeeded")
 				return supportsDType, nil
@@ -491,19 +613,22 @@ func supportsOverlay(home string, homeMagic graphdriver.FsMagic, rootUID, rootGI
 			logrus.Debugf("overlay test mount with multiple lowers failed %v", err)
 		}
 		flags = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", lower1Dir, upperDir, workDir)
+		if selinux.GetEnabled() {
+			flags = label.FormatMountLabel(flags, selinuxLabelTest)
+		}
 		if len(flags) < unix.Getpagesize() {
-			err := mountFrom(filepath.Dir(home), "overlay", mergedDir, "overlay", 0, flags)
+			err := unix.Mount("overlay", mergedDir, "overlay", 0, flags)
 			if err == nil {
 				logrus.Errorf("overlay test mount with multiple lowers failed, but succeeded with a single lower")
 				return supportsDType, errors.Wrap(graphdriver.ErrNotSupported, "kernel too old to provide multiple lowers feature for overlay")
 			}
 			logrus.Debugf("overlay test mount with a single lower failed %v", err)
 		}
-		logrus.Errorf("'overlay' is not supported over %s at %q", backingFs, home)
+		logrus.StandardLogger().Logf(logLevel, "'overlay' is not supported over %s at %q", backingFs, home)
 		return supportsDType, errors.Wrapf(graphdriver.ErrIncompatibleFS, "'overlay' is not supported over %s at %q", backingFs, home)
 	}
 
-	logrus.Error("'overlay' not found as a supported filesystem on this host. Please ensure kernel is new enough and has overlay support loaded.")
+	logrus.StandardLogger().Logf(logLevel, "'overlay' not found as a supported filesystem on this host. Please ensure kernel is new enough and has overlay support loaded.")
 	return supportsDType, errors.Wrap(graphdriver.ErrNotSupported, "'overlay' not found as a supported filesystem on this host. Please ensure kernel is new enough and has overlay support loaded.")
 }
 
@@ -575,18 +700,6 @@ func (d *Driver) Metadata(id string) (map[string]string, error) {
 	return metadata, nil
 }
 
-// ReadWriteDiskUsage returns the disk usage of the writable directory for the ID.
-// For Overlay, it attempts to check the XFS quota for size, and falls back to
-// finding the size of the "diff" directory.
-func (d *Driver) ReadWriteDiskUsage(id string) (*directory.DiskUsage, error) {
-	usage := &directory.DiskUsage{}
-	if d.quotaCtl != nil {
-		err := d.quotaCtl.GetDiskUsage(d.dir(id), usage)
-		return usage, err
-	}
-	return directory.Usage(path.Join(d.dir(id), "diff"))
-}
-
 // Cleanup any state created by overlay which should be cleaned when daemon
 // is being shutdown. For now, we just have to unmount the bind mounted
 // we had created.
@@ -594,6 +707,24 @@ func (d *Driver) Cleanup() error {
 	return mount.Unmount(d.home)
 }
 
+// LookupAdditionalLayer looks up additional layer store by the specified
+// digest and ref and returns an object representing that layer.
+// This API is experimental and can be changed without bumping the major version number.
+func (d *Driver) LookupAdditionalLayer(dgst digest.Digest, ref string) (graphdriver.AdditionalLayer, error) {
+	l, err := d.getAdditionalLayerPath(dgst, ref)
+	if err != nil {
+		return nil, err
+	}
+	// Tell the additional layer store that we use this layer.
+	// This will increase reference counter on the store's side.
+	// This will be decreased on Release() method.
+	notifyUseAdditionalLayer(l)
+	return &additionalLayer{
+		path: l,
+		d:    d,
+	}, nil
+}
+
 // CreateFromTemplate creates a layer with the same contents and parent as another layer.
 func (d *Driver) CreateFromTemplate(id, template string, templateIDMappings *idtools.IDMappings, parent string, parentIDMappings *idtools.IDMappings, opts *graphdriver.CreateOpts, readWrite bool) error {
 	if readWrite {
@@ -779,7 +910,17 @@ func (d *Driver) getLower(parent string) (string, error) {
 	// Read Parent link fileA
 	parentLink, err := ioutil.ReadFile(path.Join(parentDir, "link"))
 	if err != nil {
-		return "", err
+		if !os.IsNotExist(err) {
+			return "", err
+		}
+		logrus.Warnf("Can't read parent link %q because it does not exist. Going through storage to recreate the missing links.", path.Join(parentDir, "link"))
+		if err := d.recreateSymlinks(); err != nil {
+			return "", errors.Wrap(err, "error recreating the links")
+		}
+		parentLink, err = ioutil.ReadFile(path.Join(parentDir, "link"))
+		if err != nil {
+			return "", err
+		}
 	}
 	lowers := []string{path.Join(linkDir, string(parentLink))}
 
@@ -816,7 +957,7 @@ func (d *Driver) getLowerDirs(id string) ([]string, error) {
 			// Let's go ahead and recreate those symlinks.
 			if err != nil {
 				if os.IsNotExist(err) {
-					logrus.Warnf("Can't read link %q because it does not exist. Going through storage to recreate the missing symlinks.", lower)
+					logrus.Warnf("Can't read link %q because it does not exist. A storage corruption might have occurred, attempting to recreate the missing symlinks. It might be best wipe the storage to avoid further errors due to storage corruption.", lower)
 					if err := d.recreateSymlinks(); err != nil {
 						return nil, fmt.Errorf("error recreating the missing symlinks: %v", err)
 					}
@@ -886,6 +1027,8 @@ func (d *Driver) Remove(id string) error {
 		}
 	}
 
+	d.releaseAdditionalLayerByID(id)
+
 	if err := system.EnsureRemoveAll(dir); err != nil && !os.IsNotExist(err) {
 		return err
 	}
@@ -901,6 +1044,7 @@ func (d *Driver) recreateSymlinks() error {
 	if err != nil {
 		return fmt.Errorf("error reading driver home directory %q: %v", d.home, err)
 	}
+	linksDir := filepath.Join(d.home, "l")
 	// This makes the link directory if it doesn't exist
 	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
 	if err != nil {
@@ -909,28 +1053,80 @@ func (d *Driver) recreateSymlinks() error {
 	if err := idtools.MkdirAllAs(path.Join(d.home, linkDir), 0700, rootUID, rootGID); err != nil {
 		return err
 	}
-	for _, dir := range dirs {
-		// Skip over the linkDir and anything that is not a directory
-		if dir.Name() == linkDir || !dir.Mode().IsDir() {
+	// Keep looping as long as we take some corrective action in each iteration
+	var errs *multierror.Error
+	madeProgress := true
+	for madeProgress {
+		errs = nil
+		madeProgress = false
+		// Check that for each layer, there's a link in "l" with the name in
+		// the layer's "link" file that points to the layer's "diff" directory.
+		for _, dir := range dirs {
+			// Skip over the linkDir and anything that is not a directory
+			if dir.Name() == linkDir || !dir.Mode().IsDir() {
+				continue
+			}
+			// Read the "link" file under each layer to get the name of the symlink
+			data, err := ioutil.ReadFile(path.Join(d.dir(dir.Name()), "link"))
+			if err != nil {
+				errs = multierror.Append(errs, errors.Wrapf(err, "error reading name of symlink for %q", dir))
+				continue
+			}
+			linkPath := path.Join(d.home, linkDir, strings.Trim(string(data), "\n"))
+			// Check if the symlink exists, and if it doesn't, create it again with the
+			// name we got from the "link" file
+			_, err = os.Lstat(linkPath)
+			if err != nil && os.IsNotExist(err) {
+				if err := os.Symlink(path.Join("..", dir.Name(), "diff"), linkPath); err != nil {
+					errs = multierror.Append(errs, err)
+					continue
+				}
+				madeProgress = true
+			} else if err != nil {
+				errs = multierror.Append(errs, errors.Wrapf(err, "error trying to stat %q", linkPath))
+				continue
+			}
+		}
+		// Now check if we somehow lost a "link" file, by making sure
+		// that each symlink we have corresponds to one.
+		links, err := ioutil.ReadDir(linksDir)
+		if err != nil {
+			errs = multierror.Append(errs, errors.Wrapf(err, "error reading links directory %q", linksDir))
 			continue
 		}
-		// Read the "link" file under each layer to get the name of the symlink
-		data, err := ioutil.ReadFile(path.Join(d.dir(dir.Name()), "link"))
-		if err != nil {
-			return fmt.Errorf("error reading name of symlink for %q: %v", dir, err)
-		}
-		linkPath := path.Join(d.home, linkDir, strings.Trim(string(data), "\n"))
-		// Check if the symlink exists, and if it doesn't create it again with the name we
-		// got from the "link" file
-		_, err = os.Stat(linkPath)
-		if err != nil && os.IsNotExist(err) {
-			if err := os.Symlink(path.Join("..", dir.Name(), "diff"), linkPath); err != nil {
-				return err
+		// Go through all of the symlinks in the "l" directory
+		for _, link := range links {
+			// Read the symlink's target, which should be "../$layer/diff"
+			target, err := os.Readlink(filepath.Join(linksDir, link.Name()))
+			if err != nil {
+				errs = multierror.Append(errs, errors.Wrapf(err, "error reading target of link %q", link))
+				continue
+			}
+			targetComponents := strings.Split(target, string(os.PathSeparator))
+			if len(targetComponents) != 3 || targetComponents[0] != ".." || targetComponents[2] != "diff" {
+				errs = multierror.Append(errs, errors.Errorf("link target of %q looks weird: %q", link, target))
+				// force the link to be recreated on the next pass
+				os.Remove(filepath.Join(linksDir, link.Name()))
+				madeProgress = true
+				continue
+			}
+			// Reconstruct the name of the target's link file and check that
+			// it has the basename of our symlink in it.
+			targetID := targetComponents[1]
+			linkFile := filepath.Join(d.dir(targetID), "link")
+			data, err := ioutil.ReadFile(linkFile)
+			if err != nil || string(data) != link.Name() {
+				if err := ioutil.WriteFile(linkFile, []byte(link.Name()), 0644); err != nil {
+					errs = multierror.Append(errs, errors.Wrapf(err, "error correcting link for layer %q", targetID))
+					continue
+				}
+				madeProgress = true
 			}
-		} else if err != nil {
-			return fmt.Errorf("error trying to stat %q: %v", linkPath, err)
 		}
 	}
+	if errs != nil {
+		return errs.ErrorOrNil()
+	}
 	return nil
 }
 
@@ -987,7 +1183,17 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO
 	// lists that we're building.  "diff" itself is the upper, so it won't be in the lists.
 	link, err := ioutil.ReadFile(path.Join(dir, "link"))
 	if err != nil {
-		return "", err
+		if !os.IsNotExist(err) {
+			return "", err
+		}
+		logrus.Warnf("Can't read parent link %q because it does not exist. Going through storage to recreate the missing links.", path.Join(dir, "link"))
+		if err := d.recreateSymlinks(); err != nil {
+			return "", errors.Wrap(err, "error recreating the links")
+		}
+		link, err = ioutil.ReadFile(path.Join(dir, "link"))
+		if err != nil {
+			return "", err
+		}
 	}
 	diffN := 1
 	perms := defaultPerms
@@ -1094,9 +1300,11 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO
 		}
 	}()
 
+	workdir := path.Join(dir, "work")
+
 	var opts string
 	if readWrite {
-		opts = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", strings.Join(absLowers, ":"), diffDir, path.Join(dir, "work"))
+		opts = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", strings.Join(absLowers, ":"), diffDir, workdir)
 	} else {
 		opts = fmt.Sprintf("lowerdir=%s:%s", diffDir, strings.Join(absLowers, ":"))
 	}
@@ -1108,12 +1316,6 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO
 		opts = fmt.Sprintf("%s,userxattr", opts)
 	}
 
-	// overlay has a check in place to prevent mounting the same file system twice
-	// if volatile was already specified.
-	err = os.RemoveAll(filepath.Join(dir, "work", "incompat/volatile"))
-	if err != nil && !os.IsNotExist(err) {
-		return "", err
-	}
 	// If "volatile" is not supported by the file system, just ignore the request
 	if d.supportsVolatile && options.Volatile && !hasVolatileOption(strings.Split(opts, ",")) {
 		opts = fmt.Sprintf("%s,volatile", opts)
@@ -1150,26 +1352,35 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO
 			return nil
 		}
 	} else if len(mountData) > pageSize {
+		workdir = path.Join(id, "work")
 		//FIXME: We need to figure out to get this to work with additional stores
 		if readWrite {
 			diffDir := path.Join(id, "diff")
-			opts = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", strings.Join(relLowers, ":"), diffDir, path.Join(id, "work"))
+			opts = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", strings.Join(relLowers, ":"), diffDir, workdir)
 		} else {
 			opts = fmt.Sprintf("lowerdir=%s", strings.Join(absLowers, ":"))
 		}
 		mountData = label.FormatMountLabel(opts, options.MountLabel)
 		if len(mountData) > pageSize {
-			return "", fmt.Errorf("cannot mount layer, mount label too large %d", len(mountData))
+			return "", fmt.Errorf("cannot mount layer, mount label %q too large %d > page size %d", options.MountLabel, len(mountData), pageSize)
 		}
 		mountFunc = func(source string, target string, mType string, flags uintptr, label string) error {
 			return mountFrom(d.home, source, target, mType, flags, label)
 		}
 		mountTarget = path.Join(id, "merged")
 	}
+
+	// overlay has a check in place to prevent mounting the same file system twice
+	// if volatile was already specified.
+	err = os.RemoveAll(filepath.Join(workdir, "work/incompat/volatile"))
+	if err != nil && !os.IsNotExist(err) {
+		return "", err
+	}
+
 	flags, data := mount.ParseOptions(mountData)
 	logrus.Debugf("overlay: mount_data=%s", mountData)
 	if err := mountFunc("overlay", mountTarget, "overlay", uintptr(flags), data); err != nil {
-		return "", fmt.Errorf("error creating overlay mount to %s: %v", mountTarget, err)
+		return "", fmt.Errorf("error creating overlay mount to %s, mount_data=%q: %v", mountTarget, mountData, err)
 	}
 
 	return mergedDir, nil
@@ -1281,7 +1492,10 @@ func (f fileGetNilCloser) Close() error {
 // DiffGetter returns a FileGetCloser that can read files from the directory that
 // contains files for the layer differences. Used for direct access for tar-split.
 func (d *Driver) DiffGetter(id string) (graphdriver.FileGetCloser, error) {
-	p := d.getDiffPath(id)
+	p, err := d.getDiffPath(id)
+	if err != nil {
+		return nil, err
+	}
 	return fileGetNilCloser{storage.NewPathFileGetter(p)}, nil
 }
 
@@ -1303,7 +1517,10 @@ func (d *Driver) ApplyDiff(id, parent string, options graphdriver.ApplyDiffOpts)
 		idMappings = &idtools.IDMappings{}
 	}
 
-	applyDir := d.getDiffPath(id)
+	applyDir, err := d.getDiffPath(id)
+	if err != nil {
+		return 0, err
+	}
 
 	logrus.Debugf("Applying tar in %s", applyDir)
 	// Overlay doesn't need the parent id to apply the diff
@@ -1321,10 +1538,23 @@ func (d *Driver) ApplyDiff(id, parent string, options graphdriver.ApplyDiffOpts)
 	return directory.Size(applyDir)
 }
 
-func (d *Driver) getDiffPath(id string) string {
+func (d *Driver) getDiffPath(id string) (string, error) {
 	dir := d.dir(id)
+	return redirectDiffIfAdditionalLayer(path.Join(dir, "diff"))
+}
 
-	return path.Join(dir, "diff")
+func (d *Driver) getLowerDiffPaths(id string) ([]string, error) {
+	layers, err := d.getLowerDirs(id)
+	if err != nil {
+		return nil, err
+	}
+	for i, l := range layers {
+		layers[i], err = redirectDiffIfAdditionalLayer(l)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return layers, nil
 }
 
 // DiffSize calculates the changes between the specified id
@@ -1335,7 +1565,11 @@ func (d *Driver) DiffSize(id string, idMappings *idtools.IDMappings, parent stri
 		return d.naiveDiff.DiffSize(id, idMappings, parent, parentMappings, mountLabel)
 	}
 
-	return directory.Size(d.getDiffPath(id))
+	p, err := d.getDiffPath(id)
+	if err != nil {
+		return 0, err
+	}
+	return directory.Size(p)
 }
 
 // Diff produces an archive of the changes between the specified
@@ -1349,12 +1583,15 @@ func (d *Driver) Diff(id string, idMappings *idtools.IDMappings, parent string,
 		idMappings = &idtools.IDMappings{}
 	}
 
-	lowerDirs, err := d.getLowerDirs(id)
+	lowerDirs, err := d.getLowerDiffPaths(id)
 	if err != nil {
 		return nil, err
 	}
 
-	diffPath := d.getDiffPath(id)
+	diffPath, err := d.getDiffPath(id)
+	if err != nil {
+		return nil, err
+	}
 	logrus.Debugf("Tar with options on %s", diffPath)
 	return archive.TarWithOptions(diffPath, &archive.TarOptions{
 		Compression:    archive.Uncompressed,
@@ -1373,8 +1610,11 @@ func (d *Driver) Changes(id string, idMappings *idtools.IDMappings, parent strin
 	}
 	// Overlay doesn't have snapshots, so we need to get changes from all parent
 	// layers.
-	diffPath := d.getDiffPath(id)
-	layers, err := d.getLowerDirs(id)
+	diffPath, err := d.getDiffPath(id)
+	if err != nil {
+		return nil, err
+	}
+	layers, err := d.getLowerDiffPaths(id)
 	if err != nil {
 		return nil, err
 	}
@@ -1486,3 +1726,139 @@ func nameWithSuffix(name string, number int) string {
 	}
 	return fmt.Sprintf("%s%d", name, number)
 }
+
+func (d *Driver) getAdditionalLayerPath(dgst digest.Digest, ref string) (string, error) {
+	refElem := base64.StdEncoding.EncodeToString([]byte(ref))
+	for _, ls := range d.options.layerStores {
+		ref := ""
+		if ls.withReference {
+			ref = refElem
+		}
+		target := path.Join(ls.path, ref, dgst.String())
+		// Check if all necessary files exist
+		for _, p := range []string{
+			filepath.Join(target, "diff"),
+			filepath.Join(target, "info"),
+			// TODO(ktock): We should have an API to expose the stream data of this layer
+			//              to enable the client to retrieve the entire contents of this
+			//              layer when it exports this layer.
+		} {
+			if _, err := os.Stat(p); err != nil {
+				return "", errors.Wrapf(graphdriver.ErrLayerUnknown,
+					"failed to stat additional layer %q: %v", p, err)
+			}
+		}
+		return target, nil
+	}
+
+	return "", errors.Wrapf(graphdriver.ErrLayerUnknown,
+		"additional layer (%q, %q) not found", dgst, ref)
+}
+
+func (d *Driver) releaseAdditionalLayerByID(id string) {
+	if al, err := ioutil.ReadFile(path.Join(d.dir(id), "additionallayer")); err == nil {
+		notifyReleaseAdditionalLayer(string(al))
+	} else if !os.IsNotExist(err) {
+		logrus.Warnf("unexpected error on reading Additional Layer Store pointer %v", err)
+	}
+}
+
+// additionalLayer represents a layer in Additional Layer Store.
+type additionalLayer struct {
+	path        string
+	d           *Driver
+	releaseOnce sync.Once
+}
+
+// Info returns arbitrary information stored along with this layer (i.e. `info` file).
+// This API is experimental and can be changed without bumping the major version number.
+func (al *additionalLayer) Info() (io.ReadCloser, error) {
+	return os.Open(filepath.Join(al.path, "info"))
+}
+
+// CreateAs creates a new layer from this additional layer.
+// This API is experimental and can be changed without bumping the major version number.
+func (al *additionalLayer) CreateAs(id, parent string) error {
+	// TODO: support opts
+	if err := al.d.Create(id, parent, nil); err != nil {
+		return err
+	}
+	dir := al.d.dir(id)
+	diffDir := path.Join(dir, "diff")
+	if err := os.RemoveAll(diffDir); err != nil {
+		return err
+	}
+	// tell the additional layer store that we use this layer.
+	// mark this layer as "additional layer"
+	if err := ioutil.WriteFile(path.Join(dir, "additionallayer"), []byte(al.path), 0644); err != nil {
+		return err
+	}
+	notifyUseAdditionalLayer(al.path)
+	return os.Symlink(filepath.Join(al.path, "diff"), diffDir)
+}
+
+// Release tells the additional layer store that we don't use this handler.
+// This API is experimental and can be changed without bumping the major version number.
+func (al *additionalLayer) Release() {
+	// Tell the additional layer store that we don't use this layer handler.
+	// This will decrease the reference counter on the store's side, which was
+	// increased in LookupAdditionalLayer (so this must be called only once).
+	al.releaseOnce.Do(func() {
+		notifyReleaseAdditionalLayer(al.path)
+	})
+}
+
+// notifyUseAdditionalLayer notifies Additional Layer Store that we use the specified layer.
+// This is done by creating "use" file in the layer directory. This is useful for
+// Additional Layer Store to consider when to perform GC. Notification-aware Additional
+// Layer Store must return ENOENT.
+func notifyUseAdditionalLayer(al string) {
+	if !path.IsAbs(al) {
+		logrus.Warnf("additionallayer must be absolute (got: %v)", al)
+		return
+	}
+	useFile := path.Join(al, "use")
+	f, err := os.Create(useFile)
+	if os.IsNotExist(err) {
+		return
+	} else if err == nil {
+		f.Close()
+		if err := os.Remove(useFile); err != nil {
+			logrus.Warnf("failed to remove use file")
+		}
+	}
+	logrus.Warnf("unexpected error by Additional Layer Store %v during use; GC doesn't seem to be supported", err)
+}
+
+// notifyReleaseAdditionalLayer notifies Additional Layer Store that we don't use the specified
+// layer anymore. This is done by rmdir-ing the layer directory. This is useful for
+// Additional Layer Store to consider when to perform GC. Notification-aware Additional
+// Layer Store must return ENOENT.
+func notifyReleaseAdditionalLayer(al string) {
+	if !path.IsAbs(al) {
+		logrus.Warnf("additionallayer must be absolute (got: %v)", al)
+		return
+	}
+	// tell the additional layer store that we don't use this layer anymore.
+	err := unix.Rmdir(al)
+	if os.IsNotExist(err) {
+		return
+	}
+	logrus.Warnf("unexpected error by Additional Layer Store %v during release; GC doesn't seem to be supported", err)
+}
+
+// redirectDiffIfAdditionalLayer checks if the passed diff path is Additional Layer and
+// returns the redirected path. If the passed diff is not the one in Additional Layer
+// Store, it returns the original path without changes.
+func redirectDiffIfAdditionalLayer(diffPath string) (string, error) {
+	if ld, err := os.Readlink(diffPath); err == nil {
+		// diff is the link to Additional Layer Store
+		if !path.IsAbs(ld) {
+			return "", fmt.Errorf("linkpath must be absolute (got: %q)", ld)
+		}
+		diffPath = ld
+	} else if err.(*os.PathError).Err != syscall.EINVAL {
+		return "", err
+	}
+	return diffPath, nil
+}