From 9f2491694d538cf4e059484b3a704403841c725b Mon Sep 17 00:00:00 2001 From: Nalin Dahyabhai Date: Wed, 13 Dec 2017 11:03:37 -0500 Subject: [PATCH] Bump containers/storage and containers/image Re-vendor containers/storage to current revision 0d32dfce498e06c132c60dac945081bf44c22464, and containers/image to current revision c8bcd6aa11c62637c5a7da1420f43dd6a15f0e8d. Signed-off-by: Nalin Dahyabhai --- .../image/docker/daemon/daemon_dest.go | 21 ++- .../image/docker/daemon/daemon_src.go | 5 +- .../containers/image/docker/tarfile/dest.go | 5 +- .../image/internal/tmpdir/tmpdir.go | 19 +++ .../image/oci/archive/oci_transport.go | 62 ++------- .../containers/image/oci/internal/oci_util.go | 126 ++++++++++++++++++ .../containers/image/oci/layout/oci_dest.go | 20 ++- .../image/oci/layout/oci_transport.go | 58 ++------ .../image/storage/storage_transport.go | 6 +- .../storage/drivers/overlay/overlay.go | 110 +++++++-------- .../github.com/containers/storage/images.go | 35 ++++- .../containers/storage/images_ffjson.go | 52 ++++++++ vendor/github.com/containers/storage/store.go | 11 +- 13 files changed, 348 insertions(+), 182 deletions(-) create mode 100644 vendor/github.com/containers/image/internal/tmpdir/tmpdir.go create mode 100644 vendor/github.com/containers/image/oci/internal/oci_util.go diff --git a/vendor/github.com/containers/image/docker/daemon/daemon_dest.go b/vendor/github.com/containers/image/docker/daemon/daemon_dest.go index 4202b89f..f73ac233 100644 --- a/vendor/github.com/containers/image/docker/daemon/daemon_dest.go +++ b/vendor/github.com/containers/image/docker/daemon/daemon_dest.go @@ -14,6 +14,7 @@ import ( type daemonImageDestination struct { ref daemonReference + mustMatchRuntimeOS bool *tarfile.Destination // Implements most of types.ImageDestination // For talking to imageLoadGoroutine goroutineCancel context.CancelFunc @@ -33,6 +34,11 @@ func newImageDestination(ctx *types.SystemContext, ref daemonReference) (types.I return nil, errors.Errorf("Invalid destination docker-daemon:%s: a destination must be a name:tag", ref.StringWithinTransport()) } + var mustMatchRuntimeOS = true + if ctx != nil && ctx.DockerDaemonHost != client.DefaultDockerHost { + mustMatchRuntimeOS = false + } + c, err := newDockerClient(ctx) if err != nil { return nil, errors.Wrap(err, "Error initializing docker engine client") @@ -46,12 +52,13 @@ func newImageDestination(ctx *types.SystemContext, ref daemonReference) (types.I go imageLoadGoroutine(goroutineContext, c, reader, statusChannel) return &daemonImageDestination{ - ref: ref, - Destination: tarfile.NewDestination(writer, namedTaggedRef), - goroutineCancel: goroutineCancel, - statusChannel: statusChannel, - writer: writer, - committed: false, + ref: ref, + mustMatchRuntimeOS: mustMatchRuntimeOS, + Destination: tarfile.NewDestination(writer, namedTaggedRef), + goroutineCancel: goroutineCancel, + statusChannel: statusChannel, + writer: writer, + committed: false, }, nil } @@ -80,7 +87,7 @@ func imageLoadGoroutine(ctx context.Context, c *client.Client, reader *io.PipeRe // MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime OS. False otherwise. func (d *daemonImageDestination) MustMatchRuntimeOS() bool { - return true + return d.mustMatchRuntimeOS } // Close removes resources associated with an initialized ImageDestination, if any. diff --git a/vendor/github.com/containers/image/docker/daemon/daemon_src.go b/vendor/github.com/containers/image/docker/daemon/daemon_src.go index c08640f2..3bd4ad26 100644 --- a/vendor/github.com/containers/image/docker/daemon/daemon_src.go +++ b/vendor/github.com/containers/image/docker/daemon/daemon_src.go @@ -6,13 +6,12 @@ import ( "os" "github.com/containers/image/docker/tarfile" + "github.com/containers/image/internal/tmpdir" "github.com/containers/image/types" "github.com/pkg/errors" "golang.org/x/net/context" ) -const temporaryDirectoryForBigFiles = "/var/tmp" // Do not use the system default of os.TempDir(), usually /tmp, because with systemd it could be a tmpfs. - type daemonImageSource struct { ref daemonReference *tarfile.Source // Implements most of types.ImageSource @@ -47,7 +46,7 @@ func newImageSource(ctx *types.SystemContext, ref daemonReference) (types.ImageS defer inputStream.Close() // FIXME: use SystemContext here. - tarCopyFile, err := ioutil.TempFile(temporaryDirectoryForBigFiles, "docker-daemon-tar") + tarCopyFile, err := ioutil.TempFile(tmpdir.TemporaryDirectoryForBigFiles(), "docker-daemon-tar") if err != nil { return nil, err } diff --git a/vendor/github.com/containers/image/docker/tarfile/dest.go b/vendor/github.com/containers/image/docker/tarfile/dest.go index 72c85c70..6e042582 100644 --- a/vendor/github.com/containers/image/docker/tarfile/dest.go +++ b/vendor/github.com/containers/image/docker/tarfile/dest.go @@ -11,6 +11,7 @@ import ( "time" "github.com/containers/image/docker/reference" + "github.com/containers/image/internal/tmpdir" "github.com/containers/image/manifest" "github.com/containers/image/types" "github.com/opencontainers/go-digest" @@ -18,8 +19,6 @@ import ( "github.com/sirupsen/logrus" ) -const temporaryDirectoryForBigFiles = "/var/tmp" // Do not use the system default of os.TempDir(), usually /tmp, because with systemd it could be a tmpfs. - // Destination is a partial implementation of types.ImageDestination for writing to an io.Writer. type Destination struct { writer io.Writer @@ -107,7 +106,7 @@ func (d *Destination) PutBlob(stream io.Reader, inputInfo types.BlobInfo) (types if inputInfo.Size == -1 { // Ouch, we need to stream the blob into a temporary file just to determine the size. logrus.Debugf("docker tarfile: input with unknown size, streaming to disk first ...") - streamCopy, err := ioutil.TempFile(temporaryDirectoryForBigFiles, "docker-tarfile-blob") + streamCopy, err := ioutil.TempFile(tmpdir.TemporaryDirectoryForBigFiles(), "docker-tarfile-blob") if err != nil { return types.BlobInfo{}, err } diff --git a/vendor/github.com/containers/image/internal/tmpdir/tmpdir.go b/vendor/github.com/containers/image/internal/tmpdir/tmpdir.go new file mode 100644 index 00000000..a28020ed --- /dev/null +++ b/vendor/github.com/containers/image/internal/tmpdir/tmpdir.go @@ -0,0 +1,19 @@ +package tmpdir + +import ( + "os" + "runtime" +) + +// TemporaryDirectoryForBigFiles returns a directory for temporary (big) files. +// On non Windows systems it avoids the use of os.TempDir(), because the default temporary directory usually falls under /tmp +// which on systemd based systems could be the unsuitable tmpfs filesystem. +func TemporaryDirectoryForBigFiles() string { + var temporaryDirectoryForBigFiles string + if runtime.GOOS == "windows" { + temporaryDirectoryForBigFiles = os.TempDir() + } else { + temporaryDirectoryForBigFiles = "/var/tmp" + } + return temporaryDirectoryForBigFiles +} diff --git a/vendor/github.com/containers/image/oci/archive/oci_transport.go b/vendor/github.com/containers/image/oci/archive/oci_transport.go index 24a1502f..c4a4fa71 100644 --- a/vendor/github.com/containers/image/oci/archive/oci_transport.go +++ b/vendor/github.com/containers/image/oci/archive/oci_transport.go @@ -4,13 +4,13 @@ import ( "fmt" "io/ioutil" "os" - "path/filepath" - "regexp" "strings" "github.com/containers/image/directory/explicitfilepath" "github.com/containers/image/docker/reference" "github.com/containers/image/image" + "github.com/containers/image/internal/tmpdir" + "github.com/containers/image/oci/internal" ocilayout "github.com/containers/image/oci/layout" "github.com/containers/image/transports" "github.com/containers/image/types" @@ -48,51 +48,12 @@ func (t ociArchiveTransport) ParseReference(reference string) (types.ImageRefere // ValidatePolicyConfigurationScope checks that scope is a valid name for a signature.PolicyTransportScopes keys func (t ociArchiveTransport) ValidatePolicyConfigurationScope(scope string) error { - var file string - sep := strings.SplitN(scope, ":", 2) - file = sep[0] - - if len(sep) == 2 { - image := sep[1] - if !refRegexp.MatchString(image) { - return errors.Errorf("Invalid image %s", image) - } - } - - if !strings.HasPrefix(file, "/") { - return errors.Errorf("Invalid scope %s: must be an absolute path", scope) - } - // Refuse also "/", otherwise "/" and "" would have the same semantics, - // and "" could be unexpectedly shadowed by the "/" entry. - // (Note: we do allow "/:someimage", a bit ridiculous but why refuse it?) - if scope == "/" { - return errors.New(`Invalid scope "/": Use the generic default scope ""`) - } - cleaned := filepath.Clean(file) - if cleaned != file { - return errors.Errorf(`Invalid scope %s: Uses non-canonical path format, perhaps try with path %s`, scope, cleaned) - } - return nil + return internal.ValidateScope(scope) } -// annotation spex from https://github.com/opencontainers/image-spec/blob/master/annotations.md#pre-defined-annotation-keys -const ( - separator = `(?:[-._:@+]|--)` - alphanum = `(?:[A-Za-z0-9]+)` - component = `(?:` + alphanum + `(?:` + separator + alphanum + `)*)` -) - -var refRegexp = regexp.MustCompile(`^` + component + `(?:/` + component + `)*$`) - // ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an OCI ImageReference. func ParseReference(reference string) (types.ImageReference, error) { - var file, image string - sep := strings.SplitN(reference, ":", 2) - file = sep[0] - - if len(sep) == 2 { - image = sep[1] - } + file, image := internal.SplitPathAndImage(reference) return NewReference(file, image) } @@ -102,14 +63,15 @@ func NewReference(file, image string) (types.ImageReference, error) { if err != nil { return nil, err } - // This is necessary to prevent directory paths returned by PolicyConfigurationNamespaces - // from being ambiguous with values of PolicyConfigurationIdentity. - if strings.Contains(resolved, ":") { - return nil, errors.Errorf("Invalid OCI reference %s:%s: path %s contains a colon", file, image, resolved) + + if err := internal.ValidateOCIPath(file); err != nil { + return nil, err } - if len(image) > 0 && !refRegexp.MatchString(image) { - return nil, errors.Errorf("Invalid image %s", image) + + if err := internal.ValidateImageName(image); err != nil { + return nil, err } + return ociArchiveReference{file: file, resolvedFile: resolved, image: image}, nil } @@ -197,7 +159,7 @@ func (t *tempDirOCIRef) deleteTempDir() error { // createOCIRef creates the oci reference of the image func createOCIRef(image string) (tempDirOCIRef, error) { - dir, err := ioutil.TempDir("/var/tmp", "oci") + dir, err := ioutil.TempDir(tmpdir.TemporaryDirectoryForBigFiles(), "oci") if err != nil { return tempDirOCIRef{}, errors.Wrapf(err, "error creating temp directory") } diff --git a/vendor/github.com/containers/image/oci/internal/oci_util.go b/vendor/github.com/containers/image/oci/internal/oci_util.go new file mode 100644 index 00000000..c2012e50 --- /dev/null +++ b/vendor/github.com/containers/image/oci/internal/oci_util.go @@ -0,0 +1,126 @@ +package internal + +import ( + "github.com/pkg/errors" + "path/filepath" + "regexp" + "runtime" + "strings" +) + +// annotation spex from https://github.com/opencontainers/image-spec/blob/master/annotations.md#pre-defined-annotation-keys +const ( + separator = `(?:[-._:@+]|--)` + alphanum = `(?:[A-Za-z0-9]+)` + component = `(?:` + alphanum + `(?:` + separator + alphanum + `)*)` +) + +var refRegexp = regexp.MustCompile(`^` + component + `(?:/` + component + `)*$`) +var windowsRefRegexp = regexp.MustCompile(`^([a-zA-Z]:\\.+?):(.*)$`) + +// ValidateImageName returns nil if the image name is empty or matches the open-containers image name specs. +// In any other case an error is returned. +func ValidateImageName(image string) error { + if len(image) == 0 { + return nil + } + + var err error + if !refRegexp.MatchString(image) { + err = errors.Errorf("Invalid image %s", image) + } + return err +} + +// SplitPathAndImage tries to split the provided OCI reference into the OCI path and image. +// Neither path nor image parts are validated at this stage. +func SplitPathAndImage(reference string) (string, string) { + if runtime.GOOS == "windows" { + return splitPathAndImageWindows(reference) + } + return splitPathAndImageNonWindows(reference) +} + +func splitPathAndImageWindows(reference string) (string, string) { + groups := windowsRefRegexp.FindStringSubmatch(reference) + // nil group means no match + if groups == nil { + return reference, "" + } + + // we expect three elements. First one full match, second the capture group for the path and + // the third the capture group for the image + if len(groups) != 3 { + return reference, "" + } + return groups[1], groups[2] +} + +func splitPathAndImageNonWindows(reference string) (string, string) { + sep := strings.SplitN(reference, ":", 2) + path := sep[0] + + var image string + if len(sep) == 2 { + image = sep[1] + } + return path, image +} + +// ValidateOCIPath takes the OCI path and validates it. +func ValidateOCIPath(path string) error { + if runtime.GOOS == "windows" { + // On Windows we must allow for a ':' as part of the path + if strings.Count(path, ":") > 1 { + return errors.Errorf("Invalid OCI reference: path %s contains more than one colon", path) + } + } else { + if strings.Contains(path, ":") { + return errors.Errorf("Invalid OCI reference: path %s contains a colon", path) + } + } + return nil +} + +// ValidateScope validates a policy configuration scope for an OCI transport. +func ValidateScope(scope string) error { + var err error + if runtime.GOOS == "windows" { + err = validateScopeWindows(scope) + } else { + err = validateScopeNonWindows(scope) + } + if err != nil { + return err + } + + cleaned := filepath.Clean(scope) + if cleaned != scope { + return errors.Errorf(`Invalid scope %s: Uses non-canonical path format, perhaps try with path %s`, scope, cleaned) + } + + return nil +} + +func validateScopeWindows(scope string) error { + matched, _ := regexp.Match(`^[a-zA-Z]:\\`, []byte(scope)) + if !matched { + return errors.Errorf("Invalid scope '%s'. Must be an absolute path", scope) + } + + return nil +} + +func validateScopeNonWindows(scope string) error { + if !strings.HasPrefix(scope, "/") { + return errors.Errorf("Invalid scope %s: must be an absolute path", scope) + } + + // Refuse also "/", otherwise "/" and "" would have the same semantics, + // and "" could be unexpectedly shadowed by the "/" entry. + if scope == "/" { + return errors.New(`Invalid scope "/": Use the generic default scope ""`) + } + + return nil +} diff --git a/vendor/github.com/containers/image/oci/layout/oci_dest.go b/vendor/github.com/containers/image/oci/layout/oci_dest.go index aec3a5f1..e95f6516 100644 --- a/vendor/github.com/containers/image/oci/layout/oci_dest.go +++ b/vendor/github.com/containers/image/oci/layout/oci_dest.go @@ -112,8 +112,11 @@ func (d *ociImageDestination) PutBlob(stream io.Reader, inputInfo types.BlobInfo return types.BlobInfo{}, err } succeeded := false + explicitClosed := false defer func() { - blobFile.Close() + if !explicitClosed { + blobFile.Close() + } if !succeeded { os.Remove(blobFile.Name()) } @@ -133,8 +136,15 @@ func (d *ociImageDestination) PutBlob(stream io.Reader, inputInfo types.BlobInfo if err := blobFile.Sync(); err != nil { return types.BlobInfo{}, err } - if err := blobFile.Chmod(0644); err != nil { - return types.BlobInfo{}, err + + // On POSIX systems, blobFile was created with mode 0600, so we need to make it readable. + // On Windows, the “permissions of newly created files” argument to syscall.Open is + // ignored and the file is already readable; besides, blobFile.Chmod, i.e. syscall.Fchmod, + // always fails on Windows. + if runtime.GOOS != "windows" { + if err := blobFile.Chmod(0644); err != nil { + return types.BlobInfo{}, err + } } blobPath, err := d.ref.blobPath(computedDigest, d.sharedBlobDir) @@ -144,6 +154,10 @@ func (d *ociImageDestination) PutBlob(stream io.Reader, inputInfo types.BlobInfo if err := ensureParentDirectoryExists(blobPath); err != nil { return types.BlobInfo{}, err } + + // need to explicitly close the file, since a rename won't otherwise not work on Windows + blobFile.Close() + explicitClosed = true if err := os.Rename(blobFile.Name(), blobPath); err != nil { return types.BlobInfo{}, err } diff --git a/vendor/github.com/containers/image/oci/layout/oci_transport.go b/vendor/github.com/containers/image/oci/layout/oci_transport.go index 80f35eee..c181c4c7 100644 --- a/vendor/github.com/containers/image/oci/layout/oci_transport.go +++ b/vendor/github.com/containers/image/oci/layout/oci_transport.go @@ -5,12 +5,12 @@ import ( "fmt" "os" "path/filepath" - "regexp" "strings" "github.com/containers/image/directory/explicitfilepath" "github.com/containers/image/docker/reference" "github.com/containers/image/image" + "github.com/containers/image/oci/internal" "github.com/containers/image/transports" "github.com/containers/image/types" "github.com/opencontainers/go-digest" @@ -36,45 +36,12 @@ func (t ociTransport) ParseReference(reference string) (types.ImageReference, er return ParseReference(reference) } -// annotation spex from https://github.com/opencontainers/image-spec/blob/master/annotations.md#pre-defined-annotation-keys -const ( - separator = `(?:[-._:@+]|--)` - alphanum = `(?:[A-Za-z0-9]+)` - component = `(?:` + alphanum + `(?:` + separator + alphanum + `)*)` -) - -var refRegexp = regexp.MustCompile(`^` + component + `(?:/` + component + `)*$`) - // ValidatePolicyConfigurationScope checks that scope is a valid name for a signature.PolicyTransportScopes keys // (i.e. a valid PolicyConfigurationIdentity() or PolicyConfigurationNamespaces() return value). // It is acceptable to allow an invalid value which will never be matched, it can "only" cause user confusion. // scope passed to this function will not be "", that value is always allowed. func (t ociTransport) ValidatePolicyConfigurationScope(scope string) error { - var dir string - sep := strings.SplitN(scope, ":", 2) - dir = sep[0] - - if len(sep) == 2 { - image := sep[1] - if !refRegexp.MatchString(image) { - return errors.Errorf("Invalid image %s", image) - } - } - - if !strings.HasPrefix(dir, "/") { - return errors.Errorf("Invalid scope %s: must be an absolute path", scope) - } - // Refuse also "/", otherwise "/" and "" would have the same semantics, - // and "" could be unexpectedly shadowed by the "/" entry. - // (Note: we do allow "/:someimage", a bit ridiculous but why refuse it?) - if scope == "/" { - return errors.New(`Invalid scope "/": Use the generic default scope ""`) - } - cleaned := filepath.Clean(dir) - if cleaned != dir { - return errors.Errorf(`Invalid scope %s: Uses non-canonical path format, perhaps try with path %s`, scope, cleaned) - } - return nil + return internal.ValidateScope(scope) } // ociReference is an ImageReference for OCI directory paths. @@ -92,13 +59,7 @@ type ociReference struct { // ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an OCI ImageReference. func ParseReference(reference string) (types.ImageReference, error) { - var dir, image string - sep := strings.SplitN(reference, ":", 2) - dir = sep[0] - - if len(sep) == 2 { - image = sep[1] - } + dir, image := internal.SplitPathAndImage(reference) return NewReference(dir, image) } @@ -111,14 +72,15 @@ func NewReference(dir, image string) (types.ImageReference, error) { if err != nil { return nil, err } - // This is necessary to prevent directory paths returned by PolicyConfigurationNamespaces - // from being ambiguous with values of PolicyConfigurationIdentity. - if strings.Contains(resolved, ":") { - return nil, errors.Errorf("Invalid OCI reference %s:%s: path %s contains a colon", dir, image, resolved) + + if err := internal.ValidateOCIPath(dir); err != nil { + return nil, err } - if len(image) > 0 && !refRegexp.MatchString(image) { - return nil, errors.Errorf("Invalid image %s", image) + + if err = internal.ValidateImageName(image); err != nil { + return nil, err } + return ociReference{dir: dir, resolvedDir: resolved, image: image}, nil } diff --git a/vendor/github.com/containers/image/storage/storage_transport.go b/vendor/github.com/containers/image/storage/storage_transport.go index 4d98449b..df4578a8 100644 --- a/vendor/github.com/containers/image/storage/storage_transport.go +++ b/vendor/github.com/containers/image/storage/storage_transport.go @@ -159,11 +159,11 @@ func (s storageTransport) ParseStoreReference(store storage.Store, ref string) ( refname = verboseName(name) } if refname == "" { - logrus.Debugf("parsed reference into %q", storeSpec+"@"+id) + logrus.Debugf("parsed reference to id into %q", storeSpec+"@"+id) } else if id == "" { - logrus.Debugf("parsed reference into %q", storeSpec+refname) + logrus.Debugf("parsed reference to refname into %q", storeSpec+refname) } else { - logrus.Debugf("parsed reference into %q", storeSpec+refname+"@"+id) + logrus.Debugf("parsed reference to refname@id into %q", storeSpec+refname+"@"+id) } return newReference(storageTransport{store: store, defaultUIDMap: s.defaultUIDMap, defaultGIDMap: s.defaultGIDMap}, refname, id, name), nil } diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index d224406e..4974a94e 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -3,7 +3,6 @@ package overlay import ( - "bufio" "fmt" "io" "io/ioutil" @@ -26,7 +25,6 @@ import ( "github.com/containers/storage/pkg/locker" "github.com/containers/storage/pkg/mount" "github.com/containers/storage/pkg/parsers" - "github.com/containers/storage/pkg/parsers/kernel" "github.com/containers/storage/pkg/system" units "github.com/docker/go-units" "github.com/opencontainers/selinux/go-selinux/label" @@ -124,22 +122,6 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap return nil, err } - if err := supportsOverlay(); err != nil { - return nil, errors.Wrap(graphdriver.ErrNotSupported, "kernel does not support overlay fs") - } - - // require kernel 4.0.0 to ensure multiple lower dirs are supported - v, err := kernel.GetKernelVersion() - if err != nil { - return nil, err - } - if kernel.CompareKernelVersion(*v, kernel.VersionInfo{Kernel: 4, Major: 0, Minor: 0}) < 0 { - if !opts.overrideKernelCheck { - return nil, errors.Wrap(graphdriver.ErrNotSupported, "kernel too old to provide multiple lowers feature for overlay") - } - logrus.Warn("Using pre-4.0.0 kernel for overlay, mount failures may require kernel update") - } - fsMagic, err := graphdriver.GetFSMagic(home) if err != nil { return nil, err @@ -153,22 +135,18 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap case graphdriver.FsMagicAufs, graphdriver.FsMagicZfs, graphdriver.FsMagicOverlay, graphdriver.FsMagicEcryptfs: logrus.Errorf("'overlay' is not supported over %s", backingFs) return nil, errors.Wrapf(graphdriver.ErrIncompatibleFS, "'overlay' is not supported over %s", backingFs) - case graphdriver.FsMagicBtrfs: - // Support for OverlayFS on BTRFS was added in kernel 4.7 - // See https://btrfs.wiki.kernel.org/index.php/Changelog - if kernel.CompareKernelVersion(*v, kernel.VersionInfo{Kernel: 4, Major: 7, Minor: 0}) < 0 { - if !opts.overrideKernelCheck { - logrus.Errorf("'overlay' requires kernel 4.7 to use on %s", backingFs) - return nil, errors.Wrapf(graphdriver.ErrIncompatibleFS, "'overlay' requires kernel 4.7 to use on %s", backingFs) - } - logrus.Warn("Using pre-4.7.0 kernel for overlay on btrfs, may require kernel update") - } } rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps) if err != nil { return nil, err } + + supportsDType, err := supportsOverlay(home, fsMagic, rootUID, rootGID) + if err != nil { + return nil, errors.Wrap(graphdriver.ErrNotSupported, "kernel does not support overlay fs") + } + // Create the driver home dir if err := idtools.MkdirAllAs(path.Join(home, linkDir), 0700, rootUID, rootGID); err != nil && !os.IsExist(err) { return nil, err @@ -178,16 +156,6 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap return nil, err } - supportsDType, err := fsutils.SupportsDType(home) - if err != nil { - return nil, err - } - if !supportsDType { - logrus.Warn(overlayutils.ErrDTypeNotSupported("overlay", backingFs)) - // TODO: Will make fatal when CRI-O Has AMI built on RHEL7.4 - // return nil, overlayutils.ErrDTypeNotSupported("overlay", backingFs) - } - d := &Driver{ name: "overlay", home: home, @@ -210,10 +178,10 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap } } else if opts.quota.Size > 0 { // if xfs is not the backing fs then error out if the storage-opt overlay.size is used. - return nil, fmt.Errorf("Storage Option overlay.size only supported for backingFS XFS. Found %v", backingFs) + return nil, fmt.Errorf("Storage option overlay.size only supported for backingFS XFS. Found %v", backingFs) } - logrus.Debugf("backingFs=%s, projectQuotaSupported=%v", backingFs, projectQuotaSupported) + logrus.Debugf("backingFs=%s, projectQuotaSupported=%v", backingFs, projectQuotaSupported) return d, nil } @@ -264,25 +232,59 @@ func parseOptions(options []string) (*overlayOptions, error) { return o, nil } -func supportsOverlay() error { - // We can try to modprobe overlay first before looking at - // proc/filesystems for when overlay is supported +func supportsOverlay(home string, homeMagic graphdriver.FsMagic, rootUID, rootGID int) (supportsDType bool, err error) { + // We can try to modprobe overlay first exec.Command("modprobe", "overlay").Run() - f, err := os.Open("/proc/filesystems") - if err != nil { - return err - } - defer f.Close() - - s := bufio.NewScanner(f) - for s.Scan() { - if s.Text() == "nodev\toverlay" { - return nil + layerDir, err := ioutil.TempDir(home, "compat") + if err == nil { + // Check if reading the directory's contents populates the d_type field, which is required + // for proper operation of the overlay filesystem. + supportsDType, err = fsutils.SupportsDType(layerDir) + if err != nil { + return false, err } + if !supportsDType { + logrus.Warn(overlayutils.ErrDTypeNotSupported("overlay", backingFs)) + // TODO: Will make fatal when CRI-O Has AMI built on RHEL7.4 + // return nil, overlayutils.ErrDTypeNotSupported("overlay", backingFs) + } + + // Try a test mount in the specific location we're looking at using. + mergedDir := filepath.Join(layerDir, "merged") + lower1Dir := filepath.Join(layerDir, "lower1") + lower2Dir := filepath.Join(layerDir, "lower2") + defer func() { + // Permitted to fail, since the various subdirectories + // can be empty or not even there, and the home might + // legitimately be not empty + _ = unix.Unmount(mergedDir, unix.MNT_DETACH) + _ = os.RemoveAll(layerDir) + _ = os.Remove(home) + }() + _ = idtools.MkdirAs(mergedDir, 0700, rootUID, rootGID) + _ = idtools.MkdirAs(lower1Dir, 0700, rootUID, rootGID) + _ = idtools.MkdirAs(lower2Dir, 0700, rootUID, rootGID) + flags := fmt.Sprintf("lowerdir=%s:%s", lower1Dir, lower2Dir) + if len(flags) < unix.Getpagesize() { + if mountFrom(filepath.Dir(home), "overlay", mergedDir, "overlay", 0, flags) == nil { + logrus.Debugf("overlay test mount with multiple lowers succeeded") + return supportsDType, nil + } + } + flags = fmt.Sprintf("lowerdir=%s", lower1Dir) + if len(flags) < unix.Getpagesize() { + if mountFrom(filepath.Dir(home), "overlay", mergedDir, "overlay", 0, flags) == nil { + logrus.Errorf("overlay test mount with multiple lowers failed, but succeeded with a single lower") + return supportsDType, errors.Wrap(graphdriver.ErrNotSupported, "kernel too old to provide multiple lowers feature for overlay") + } + } + logrus.Errorf("'overlay' is not supported over %s at %q", backingFs, home) + return supportsDType, errors.Wrapf(graphdriver.ErrIncompatibleFS, "'overlay' is not supported over %s at %q", backingFs, home) } + logrus.Error("'overlay' not found as a supported filesystem on this host. Please ensure kernel is new enough and has overlay support loaded.") - return errors.Wrap(graphdriver.ErrNotSupported, "'overlay' not found as a supported filesystem on this host. Please ensure kernel is new enough and has overlay support loaded.") + return supportsDType, errors.Wrap(graphdriver.ErrNotSupported, "'overlay' not found as a supported filesystem on this host. Please ensure kernel is new enough and has overlay support loaded.") } func useNaiveDiff(home string) bool { diff --git a/vendor/github.com/containers/storage/images.go b/vendor/github.com/containers/storage/images.go index 8e8f1149..962e1bb7 100644 --- a/vendor/github.com/containers/storage/images.go +++ b/vendor/github.com/containers/storage/images.go @@ -27,6 +27,9 @@ type Image struct { // value which was generated by the library. ID string `json:"id"` + // Digest is a digest value that we can use to locate the image. + Digest digest.Digest `json:"digest,omitempty"` + // Names is an optional set of user-defined convenience values. The // image can be referred to by its ID or any of its names. Names are // unique among images. @@ -98,7 +101,7 @@ type ImageStore interface { // Create creates an image that has a specified ID (or a random one) and // optional names, using the specified layer as its topmost (hopefully // read-only) layer. That layer can be referenced by multiple images. - Create(id string, names []string, layer, metadata string, created time.Time) (*Image, error) + Create(id string, names []string, layer, metadata string, created time.Time, searchableDigest digest.Digest) (*Image, error) // SetNames replaces the list of names associated with an image with the // supplied values. @@ -165,9 +168,16 @@ func (r *imageStore) Load() error { } names[name] = images[n] } + // Implicit digest if digest, ok := image.BigDataDigests[ImageDigestBigDataKey]; ok { digests[digest] = append(digests[digest], images[n]) } + // Explicit digest + if image.Digest == "" { + image.Digest = image.BigDataDigests[ImageDigestBigDataKey] + } else if image.Digest != image.BigDataDigests[ImageDigestBigDataKey] { + digests[image.Digest] = append(digests[image.Digest], images[n]) + } } } if shouldSave && !r.IsReadWrite() { @@ -284,7 +294,7 @@ func (r *imageStore) SetFlag(id string, flag string, value interface{}) error { return r.Save() } -func (r *imageStore) Create(id string, names []string, layer, metadata string, created time.Time) (image *Image, err error) { +func (r *imageStore) Create(id string, names []string, layer, metadata string, created time.Time, searchableDigest digest.Digest) (image *Image, err error) { if !r.IsReadWrite() { return nil, errors.Wrapf(ErrStoreIsReadOnly, "not allowed to create new images at %q", r.imagespath()) } @@ -311,6 +321,7 @@ func (r *imageStore) Create(id string, names []string, layer, metadata string, c if err == nil { image = &Image{ ID: id, + Digest: searchableDigest, Names: names, TopLayer: layer, Metadata: metadata, @@ -323,6 +334,10 @@ func (r *imageStore) Create(id string, names []string, layer, metadata string, c r.images = append(r.images, image) r.idindex.Add(id) r.byid[id] = image + if searchableDigest != "" { + list := r.bydigest[searchableDigest] + r.bydigest[searchableDigest] = append(list, image) + } for _, name := range names { r.byname[name] = image } @@ -413,6 +428,17 @@ func (r *imageStore) Delete(id string) error { } } } + if image.Digest != "" { + // remove the image's hard-coded digest from the digest-based index + if list, ok := r.bydigest[image.Digest]; ok { + prunedList := imageSliceWithoutValue(list, image) + if len(prunedList) == 0 { + delete(r.bydigest, image.Digest) + } else { + r.bydigest[image.Digest] = prunedList + } + } + } if err := r.Save(); err != nil { return err } @@ -577,9 +603,10 @@ func (r *imageStore) SetBigData(id, key string, data []byte) error { save = true } if key == ImageDigestBigDataKey { - if oldDigest != "" && oldDigest != newDigest { + if oldDigest != "" && oldDigest != newDigest && oldDigest != image.Digest { // remove the image from the list of images in the digest-based - // index which corresponds to the old digest for this item + // index which corresponds to the old digest for this item, unless + // it's also the hard-coded digest if list, ok := r.bydigest[oldDigest]; ok { prunedList := imageSliceWithoutValue(list, image) if len(prunedList) == 0 { diff --git a/vendor/github.com/containers/storage/images_ffjson.go b/vendor/github.com/containers/storage/images_ffjson.go index 59df7805..f6a8b065 100644 --- a/vendor/github.com/containers/storage/images_ffjson.go +++ b/vendor/github.com/containers/storage/images_ffjson.go @@ -38,6 +38,11 @@ func (j *Image) MarshalJSONBuf(buf fflib.EncodingBuffer) error { buf.WriteString(`{ "id":`) fflib.WriteJsonString(buf, string(j.ID)) buf.WriteByte(',') + if len(j.Digest) != 0 { + buf.WriteString(`"digest":`) + fflib.WriteJsonString(buf, string(j.Digest)) + buf.WriteByte(',') + } if len(j.Names) != 0 { buf.WriteString(`"names":`) if j.Names != nil { @@ -146,6 +151,8 @@ const ( ffjtImageID + ffjtImageDigest + ffjtImageNames ffjtImageTopLayer @@ -165,6 +172,8 @@ const ( var ffjKeyImageID = []byte("id") +var ffjKeyImageDigest = []byte("digest") + var ffjKeyImageNames = []byte("names") var ffjKeyImageTopLayer = []byte("layer") @@ -268,6 +277,14 @@ mainparse: goto mainparse } + case 'd': + + if bytes.Equal(ffjKeyImageDigest, kn) { + currentKey = ffjtImageDigest + state = fflib.FFParse_want_colon + goto mainparse + } + case 'f': if bytes.Equal(ffjKeyImageFlags, kn) { @@ -358,6 +375,12 @@ mainparse: goto mainparse } + if fflib.EqualFoldRight(ffjKeyImageDigest, kn) { + currentKey = ffjtImageDigest + state = fflib.FFParse_want_colon + goto mainparse + } + if fflib.SimpleLetterEqualFold(ffjKeyImageID, kn) { currentKey = ffjtImageID state = fflib.FFParse_want_colon @@ -384,6 +407,9 @@ mainparse: case ffjtImageID: goto handle_ID + case ffjtImageDigest: + goto handle_Digest + case ffjtImageNames: goto handle_Names @@ -448,6 +474,32 @@ handle_ID: state = fflib.FFParse_after_value goto mainparse +handle_Digest: + + /* handler: j.Digest type=digest.Digest kind=string quoted=false*/ + + { + + { + if tok != fflib.FFTok_string && tok != fflib.FFTok_null { + return fs.WrapErr(fmt.Errorf("cannot unmarshal %s into Go value for Digest", tok)) + } + } + + if tok == fflib.FFTok_null { + + } else { + + outBuf := fs.Output.Bytes() + + j.Digest = digest.Digest(string(outBuf)) + + } + } + + state = fflib.FFParse_after_value + goto mainparse + handle_Names: /* handler: j.Names type=[]string kind=slice quoted=false*/ diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index 59a6cbb1..de605432 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -434,6 +434,8 @@ type ImageOptions struct { // CreationDate, if not zero, will override the default behavior of marking the image as having been // created when CreateImage() was called, recording CreationDate instead. CreationDate time.Time + // Digest is a hard-coded digest value that we can use to look up the image. It is optional. + Digest digest.Digest } // ContainerOptions is used for passing options to a Store's CreateContainer() method. @@ -491,11 +493,6 @@ func GetStore(options StoreOptions) (Store, error) { if err := os.MkdirAll(options.RunRoot, 0700); err != nil && !os.IsExist(err) { return nil, err } - for _, subdir := range []string{} { - if err := os.MkdirAll(filepath.Join(options.RunRoot, subdir), 0700); err != nil && !os.IsExist(err) { - return nil, err - } - } if err := os.MkdirAll(options.GraphRoot, 0700); err != nil && !os.IsExist(err) { return nil, err } @@ -838,11 +835,11 @@ func (s *store) CreateImage(id string, names []string, layer, metadata string, o } creationDate := time.Now().UTC() - if options != nil { + if options != nil && !options.CreationDate.IsZero() { creationDate = options.CreationDate } - return ristore.Create(id, names, layer, metadata, creationDate) + return ristore.Create(id, names, layer, metadata, creationDate, options.Digest) } func (s *store) CreateContainer(id string, names []string, image, layer, metadata string, options *ContainerOptions) (*Container, error) {