Merge pull request #506 from rhatdan/storage

Vendor in latest containers-storage to add devmapper support

vendor/github.com/containers/storage/containers.go

@@ -304,8 +304,9 @@ func (r *containerStore) Create(id string, names []string, image, layer, metadat
 			r.byname[name] = container
 		}
 		err = r.Save()
+		container = copyContainer(container)
 	}
-	return copyContainer(container), err
+	return container, err
 }
 
 func (r *containerStore) Metadata(id string) (string, error) {

vendor/github.com/containers/storage/containers_ffjson.go

@@ -1,5 +1,5 @@
 // Code generated by ffjson <https://github.com/pquerna/ffjson>. DO NOT EDIT.
-// source: containers.go
+// source: ./containers.go
 
 package storage
 

vendor/github.com/containers/storage/drivers/devmapper/device_setup.go

@@ -9,7 +9,6 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"reflect"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -31,9 +30,6 @@ var (
 )
 
 func validateLVMConfig(cfg directLVMConfig) error {
-	if reflect.DeepEqual(cfg, directLVMConfig{}) {
-		return nil
-	}
 	if cfg.Device == "" {
 		return errMissingSetupDevice
 	}

vendor/github.com/containers/storage/drivers/devmapper/deviceset.go

@@ -703,6 +703,10 @@ func (devices *DeviceSet) startDeviceDeletionWorker() {
 		return
 	}
 
+	// Cleanup right away if there are any leaked devices.  Note this
+	// could cause some slowdown for process startup, if there were
+	// Leaked devices
+	devices.cleanupDeletedDevices()
 	logrus.Debug("devmapper: Worker to cleanup deleted devices started")
 	for range devices.deletionWorkerTicker.C {
 		devices.cleanupDeletedDevices()
@@ -2652,6 +2656,7 @@ func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps [
 
 	foundBlkDiscard := false
 	var lvmSetupConfig directLVMConfig
+	testMode := false
 	for _, option := range options {
 		key, val, err := parsers.ParseKeyValueOpt(option)
 		if err != nil {
@@ -2801,13 +2806,20 @@ func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps [
 			devicemapper.LogInit(devicemapper.DefaultLogger{
 				Level: int(level),
 			})
+		case "test":
+			testMode, err = strconv.ParseBool(val)
+			if err != nil {
+				return nil, err
+			}
 		default:
 			return nil, fmt.Errorf("devmapper: Unknown option %s", key)
 		}
 	}
 
-	if err := validateLVMConfig(lvmSetupConfig); err != nil {
+	if !testMode {
-		return nil, err
+		if err := validateLVMConfig(lvmSetupConfig); err != nil {
+			return nil, err
+		}
 	}
 
 	devices.lvmSetupConfig = lvmSetupConfig

vendor/github.com/containers/storage/images.go

@@ -357,8 +357,9 @@ func (r *imageStore) Create(id string, names []string, layer, metadata string, c
 			r.byname[name] = image
 		}
 		err = r.Save()
+		image = copyImage(image)
 	}
-	return copyImage(image), err
+	return image, err
 }
 
 func (r *imageStore) Metadata(id string) (string, error) {

vendor/github.com/containers/storage/images_ffjson.go

@@ -1,5 +1,5 @@
 // Code generated by ffjson <https://github.com/pquerna/ffjson>. DO NOT EDIT.
-// source: images.go
+// source: ./images.go
 
 package storage
 

vendor/github.com/containers/storage/layers.go

@@ -610,8 +610,9 @@ func (r *layerStore) Put(id string, parentLayer *Layer, names []string, mountLab
 			r.driver.Remove(id)
 			return nil, -1, err
 		}
+		layer = copyLayer(layer)
 	}
-	return copyLayer(layer), size, err
+	return layer, size, err
 }
 
 func (r *layerStore) CreateWithFlags(id string, parent *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, flags map[string]interface{}) (layer *Layer, err error) {

vendor/github.com/containers/storage/layers_ffjson.go

@@ -1,5 +1,5 @@
 // Code generated by ffjson <https://github.com/pquerna/ffjson>. DO NOT EDIT.
-// source: layers.go
+// source: ./layers.go
 
 package storage
 

vendor/github.com/containers/storage/pkg/archive/archive.go

@@ -945,7 +945,8 @@ loop:
 		}
 		trBuf.Reset(tr)
 
-		if err := remapIDs(nil, idMappings, options.ChownOpts, hdr); err != nil {
+		chownOpts := options.ChownOpts
+		if err := remapIDs(nil, idMappings, chownOpts, hdr); err != nil {
 			return err
 		}
 
@@ -959,7 +960,11 @@ loop:
 			}
 		}
 
-		if err := createTarFile(path, dest, hdr, trBuf, !options.NoLchown, options.ChownOpts, options.InUserNS); err != nil {
+		if chownOpts != nil {
+			chownOpts = &idtools.IDPair{UID: hdr.Uid, GID: hdr.Gid}
+		}
+
+		if err := createTarFile(path, dest, hdr, trBuf, !options.NoLchown, chownOpts, options.InUserNS); err != nil {
 			return err
 		}
 
@@ -1141,7 +1146,7 @@ func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) {
 		hdr.Name = filepath.Base(dst)
 		hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode)))
 
-		if err := remapIDs(archiver.TarIDMappings, archiver.UntarIDMappings, archiver.ChownOpts, hdr); err != nil {
+		if err := remapIDs(archiver.TarIDMappings, nil, archiver.ChownOpts, hdr); err != nil {
 			return err
 		}
 
@@ -1161,7 +1166,12 @@ func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) {
 		}
 	}()
 
-	err = archiver.Untar(r, filepath.Dir(dst), nil)
+	options := &TarOptions{
+		UIDMaps:   archiver.UntarIDMappings.UIDs(),
+		GIDMaps:   archiver.UntarIDMappings.GIDs(),
+		ChownOpts: archiver.ChownOpts,
+	}
+	err = archiver.Untar(r, filepath.Dir(dst), options)
 	if err != nil {
 		r.CloseWithError(err)
 	}

vendor/github.com/containers/storage/pkg/archive/archive_ffjson.go

@@ -1,5 +1,5 @@
 // Code generated by ffjson <https://github.com/pquerna/ffjson>. DO NOT EDIT.
-// source: pkg/archive/archive.go
+// source: ./pkg/archive/archive.go
 
 package archive
 

vendor/github.com/containers/storage/store.go

@@ -173,7 +173,7 @@ type Store interface {
 	// process's main() function needs to import our pkg/reexec package and
 	// should begin with something like this in order to allow us to
 	// properly start that child process:
-	//   if reexec.Init {
+	//   if reexec.Init() {
 	//       return
 	//   }
 	PutLayer(id, parent string, names []string, mountLabel string, writeable bool, options *LayerOptions, diff io.Reader) (*Layer, int64, error)
@@ -253,7 +253,7 @@ type Store interface {
 	// process's main() function needs to import our pkg/reexec package and
 	// should begin with something like this in order to allow us to
 	// properly start that child process:
-	//   if reexec.Init {
+	//   if reexec.Init() {
 	//       return
 	//   }
 	Mount(id, mountLabel string) (string, error)
@@ -288,7 +288,7 @@ type Store interface {
 	// process's main() function needs to import our pkg/reexec package and
 	// should begin with something like this in order to allow us to
 	// properly start that child process:
-	//   if reexec.Init {
+	//   if reexec.Init() {
 	//       return
 	//   }
 	ApplyDiff(to string, diff io.Reader) (int64, error)
@@ -512,6 +512,14 @@ type store struct {
 // These defaults observe environment variables:
 //  * `STORAGE_DRIVER` for the name of the storage driver to attempt to use
 //  * `STORAGE_OPTS` for the string of options to pass to the driver
+//
+// Note that we do some of this work in a child process.  The calling process's
+// main() function needs to import our pkg/reexec package and should begin with
+// something like this in order to allow us to properly start that child
+// process:
+//   if reexec.Init() {
+//       return
+//   }
 func GetStore(options StoreOptions) (Store, error) {
 	if options.RunRoot == "" && options.GraphRoot == "" && options.GraphDriverName == "" && len(options.GraphDriverOptions) == 0 {
 		options = DefaultStoreOptions
@@ -2587,6 +2595,65 @@ func copyStringInterfaceMap(m map[string]interface{}) map[string]interface{} {
 
 const configFile = "/etc/containers/storage.conf"
 
+// ThinpoolOptionsConfig represents the "storage.options.thinpool"
+// TOML config table.
+type ThinpoolOptionsConfig struct {
+	// AutoExtendPercent determines the amount by which pool needs to be
+	// grown. This is specified in terms of % of pool size. So a value of
+	// 20 means that when threshold is hit, pool will be grown by 20% of
+	// existing pool size.
+	AutoExtendPercent string `toml:"autoextend_percent"`
+
+	// AutoExtendThreshold determines the pool extension threshold in terms
+	// of percentage of pool size. For example, if threshold is 60, that
+	// means when pool is 60% full, threshold has been hit.
+	AutoExtendThreshold string `toml:"autoextend_threshold"`
+
+	// BaseSize specifies the size to use when creating the base device,
+	// which limits the size of images and containers.
+	BaseSize string `toml:"basesize"`
+
+	// BlockSize specifies a custom blocksize to use for the thin pool.
+	BlockSize string `toml:"blocksize"`
+
+	// DirectLvmDevice specifies a custom block storage device to use for
+	// the thin pool.
+	DirectLvmDevice string `toml:"directlvm_device"`
+
+	// DirectLvmDeviceForcewipes device even if device already has a
+	// filesystem
+	DirectLvmDeviceForce string `toml:"directlvm_device_force"`
+
+	// Fs specifies the filesystem type to use for the base device.
+	Fs string `toml:"fs"`
+
+	// log_level sets the log level of devicemapper.
+	LogLevel string `toml:"log_level"`
+
+	// MinFreeSpace specifies the min free space percent in a thin pool
+	// require for new device creation to
+	MinFreeSpace string `toml:"min_free_space"`
+
+	// MkfsArg specifies extra mkfs arguments to be used when creating the
+	// basedevice.
+	MkfsArg string `toml:"mkfsarg"`
+
+	// MountOpt specifies extra mount options used when mounting the thin
+	// devices.
+	MountOpt string `toml:"mountopt"`
+
+	// UseDeferredDeletion marks device for deferred deletion
+	UseDeferredDeletion string `toml:"use_deferred_deletion"`
+
+	// UseDeferredRemoval marks device for deferred removal
+	UseDeferredRemoval string `toml:"use_deferred_removal"`
+
+	// XfsNoSpaceMaxRetriesFreeSpace specifies the maximum number of
+	// retries XFS should attempt to complete IO when ENOSPC (no space)
+	// error is returned by underlying storage device.
+	XfsNoSpaceMaxRetries string `toml:"xfs_nospace_max_retries"`
+}
+
 // OptionsConfig represents the "storage.options" TOML config table.
 type OptionsConfig struct {
 	// AdditionalImagesStores is the location of additional read/only
@@ -2611,6 +2678,8 @@ type OptionsConfig struct {
 	// RemapGroup is the name of one or more entries in /etc/subgid which
 	// should be used to set up default GID mappings.
 	RemapGroup string `toml:"remap-group"`
+	// Thinpool container options to be handed to thinpool drivers
+	Thinpool struct{ ThinpoolOptionsConfig } `toml:"thinpool"`
 }
 
 // TOML-friendly explicit tables used for conversions.
@@ -2651,6 +2720,50 @@ func init() {
 	if config.Storage.GraphRoot != "" {
 		DefaultStoreOptions.GraphRoot = config.Storage.GraphRoot
 	}
+	if config.Storage.Options.Thinpool.AutoExtendPercent != "" {
+		DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("dm.thinp_autoextend_percent=%s", config.Storage.Options.Thinpool.AutoExtendPercent))
+	}
+
+	if config.Storage.Options.Thinpool.AutoExtendThreshold != "" {
+		DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("dm.thinp_autoextend_threshold=%s", config.Storage.Options.Thinpool.AutoExtendThreshold))
+	}
+
+	if config.Storage.Options.Thinpool.BaseSize != "" {
+		DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("dm.basesize=%s", config.Storage.Options.Thinpool.BaseSize))
+	}
+	if config.Storage.Options.Thinpool.BlockSize != "" {
+		DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("dm.blocksize=%s", config.Storage.Options.Thinpool.BlockSize))
+	}
+	if config.Storage.Options.Thinpool.DirectLvmDevice != "" {
+		DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("dm.directlvm_device=%s", config.Storage.Options.Thinpool.DirectLvmDevice))
+	}
+	if config.Storage.Options.Thinpool.DirectLvmDeviceForce != "" {
+		DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("dm.directlvm_device_force=%s", config.Storage.Options.Thinpool.DirectLvmDeviceForce))
+	}
+	if config.Storage.Options.Thinpool.Fs != "" {
+		DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("dm.fs=%s", config.Storage.Options.Thinpool.Fs))
+	}
+	if config.Storage.Options.Thinpool.LogLevel != "" {
+		DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("dm.libdm_log_level=%s", config.Storage.Options.Thinpool.LogLevel))
+	}
+	if config.Storage.Options.Thinpool.MinFreeSpace != "" {
+		DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("dm.min_free_space=%s", config.Storage.Options.Thinpool.MinFreeSpace))
+	}
+	if config.Storage.Options.Thinpool.MkfsArg != "" {
+		DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("dm.mkfsarg=%s", config.Storage.Options.Thinpool.MkfsArg))
+	}
+	if config.Storage.Options.Thinpool.MountOpt != "" {
+		DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("dm.mountopt=%s", config.Storage.Options.Thinpool.MountOpt))
+	}
+	if config.Storage.Options.Thinpool.UseDeferredDeletion != "" {
+		DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("dm.use_deferred_deletion=%s", config.Storage.Options.Thinpool.UseDeferredDeletion))
+	}
+	if config.Storage.Options.Thinpool.UseDeferredRemoval != "" {
+		DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("dm.use_deferred_removal=%s", config.Storage.Options.Thinpool.UseDeferredRemoval))
+	}
+	if config.Storage.Options.Thinpool.XfsNoSpaceMaxRetries != "" {
+		DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("dm.xfs_nospace_max_retries=%s", config.Storage.Options.Thinpool.XfsNoSpaceMaxRetries))
+	}
 	for _, s := range config.Storage.Options.AdditionalImageStores {
 		DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("%s.imagestore=%s", config.Storage.Driver, s))
 	}