fix(deps): update module github.com/containers/storage to v1.57.1

Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/containers/common v0.61.1
 	github.com/containers/image/v5 v5.33.1
 	github.com/containers/ocicrypt v1.2.1
-	github.com/containers/storage v1.57.0
+	github.com/containers/storage v1.57.1
 	github.com/docker/distribution v2.8.3+incompatible
 	github.com/moby/sys/capability v0.4.0
 	github.com/opencontainers/go-digest v1.0.0

go.sum

@@ -49,8 +49,8 @@ github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYgle
 github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY=
 github.com/containers/ocicrypt v1.2.1 h1:0qIOTT9DoYwcKmxSt8QJt+VzMY18onl9jUXsxpVhSmM=
 github.com/containers/ocicrypt v1.2.1/go.mod h1:aD0AAqfMp0MtwqWgHM1bUwe1anx0VazI108CRrSKINQ=
-github.com/containers/storage v1.57.0 h1:ANTF3oOC62wym590lkFuGCg1IIedobfHIUg7zw61awc=
+github.com/containers/storage v1.57.1 h1:hKPoFsuBcB3qTzBxa4IFpZMRzUuL5Xhv/BE44W0XHx8=
-github.com/containers/storage v1.57.0/go.mod h1:i/Hb4lu7YgFr9G0K6BMjqW0BLJO1sFsnWQwj2UoWCUM=
+github.com/containers/storage v1.57.1/go.mod h1:i/Hb4lu7YgFr9G0K6BMjqW0BLJO1sFsnWQwj2UoWCUM=
 github.com/coreos/go-oidc/v3 v3.11.0 h1:Ia3MxdwpSw702YW0xgfmP1GVCMA9aEFWu12XUZ3/OtI=
 github.com/coreos/go-oidc/v3 v3.11.0/go.mod h1:gE3LgjOgFoHi9a4ce4/tJczr0Ai2/BoDhf0r5lltWI0=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=

vendor/github.com/containers/storage/VERSION

@@ -1 +1 @@
-1.57.0
+1.57.1

vendor/github.com/containers/storage/pkg/chunked/compression_linux.go

@@ -23,7 +23,7 @@ import (
 const (
 	// maxTocSize is the maximum size of a blob that we will attempt to process.
 	// It is used to prevent DoS attacks from layers that embed a very large TOC file.
-	maxTocSize = (1 << 20) * 50
+	maxTocSize = (1 << 20) * 150
 )
 
 var typesToTar = map[string]byte{
@@ -44,6 +44,8 @@ func typeToTarType(t string) (byte, error) {
 	return r, nil
 }
 
+// readEstargzChunkedManifest reads the estargz manifest from the seekable stream blobStream.
+// It may return an error matching ErrFallbackToOrdinaryLayerDownload / errFallbackCanConvert.
 func readEstargzChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, tocDigest digest.Digest) ([]byte, int64, error) {
 	// information on the format here https://github.com/containerd/stargz-snapshotter/blob/main/docs/stargz-estargz.md
 	footerSize := int64(51)
@@ -54,6 +56,10 @@ func readEstargzChunkedManifest(blobStream ImageSourceSeekable, blobSize int64,
 	footer := make([]byte, footerSize)
 	streamsOrErrors, err := getBlobAt(blobStream, ImageSourceChunk{Offset: uint64(blobSize - footerSize), Length: uint64(footerSize)})
 	if err != nil {
+		var badRequestErr ErrBadRequest
+		if errors.As(err, &badRequestErr) {
+			err = errFallbackCanConvert{newErrFallbackToOrdinaryLayerDownload(err)}
+		}
 		return nil, 0, err
 	}
 
@@ -84,11 +90,16 @@ func readEstargzChunkedManifest(blobStream ImageSourceSeekable, blobSize int64,
 	size := int64(blobSize - footerSize - tocOffset)
 	// set a reasonable limit
 	if size > maxTocSize {
-		return nil, 0, errors.New("manifest too big")
+		// Not errFallbackCanConvert: we would still use too much memory.
+		return nil, 0, newErrFallbackToOrdinaryLayerDownload(fmt.Errorf("estargz manifest too big to process in memory (%d bytes)", size))
 	}
 
 	streamsOrErrors, err = getBlobAt(blobStream, ImageSourceChunk{Offset: uint64(tocOffset), Length: uint64(size)})
 	if err != nil {
+		var badRequestErr ErrBadRequest
+		if errors.As(err, &badRequestErr) {
+			err = errFallbackCanConvert{newErrFallbackToOrdinaryLayerDownload(err)}
+		}
 		return nil, 0, err
 	}
 
@@ -148,6 +159,7 @@ func readEstargzChunkedManifest(blobStream ImageSourceSeekable, blobSize int64,
 
 // readZstdChunkedManifest reads the zstd:chunked manifest from the seekable stream blobStream.
 // Returns (manifest blob, parsed manifest, tar-split blob or nil, manifest offset).
+// It may return an error matching ErrFallbackToOrdinaryLayerDownload / errFallbackCanConvert.
 func readZstdChunkedManifest(blobStream ImageSourceSeekable, tocDigest digest.Digest, annotations map[string]string) (_ []byte, _ *minimal.TOC, _ []byte, _ int64, retErr error) {
 	offsetMetadata := annotations[minimal.ManifestInfoKey]
 	if offsetMetadata == "" {
@@ -173,10 +185,12 @@ func readZstdChunkedManifest(blobStream ImageSourceSeekable, tocDigest digest.Di
 
 	// set a reasonable limit
 	if manifestChunk.Length > maxTocSize {
-		return nil, nil, nil, 0, errors.New("manifest too big")
+		// Not errFallbackCanConvert: we would still use too much memory.
+		return nil, nil, nil, 0, newErrFallbackToOrdinaryLayerDownload(fmt.Errorf("zstd:chunked manifest too big to process in memory (%d bytes compressed)", manifestChunk.Length))
 	}
 	if manifestLengthUncompressed > maxTocSize {
-		return nil, nil, nil, 0, errors.New("manifest too big")
+		// Not errFallbackCanConvert: we would still use too much memory.
+		return nil, nil, nil, 0, newErrFallbackToOrdinaryLayerDownload(fmt.Errorf("zstd:chunked manifest too big to process in memory (%d bytes uncompressed)", manifestLengthUncompressed))
 	}
 
 	chunks := []ImageSourceChunk{manifestChunk}
@@ -186,6 +200,10 @@ func readZstdChunkedManifest(blobStream ImageSourceSeekable, tocDigest digest.Di
 
 	streamsOrErrors, err := getBlobAt(blobStream, chunks...)
 	if err != nil {
+		var badRequestErr ErrBadRequest
+		if errors.As(err, &badRequestErr) {
+			err = errFallbackCanConvert{newErrFallbackToOrdinaryLayerDownload(err)}
+		}
 		return nil, nil, nil, 0, err
 	}
 

vendor/github.com/containers/storage/pkg/chunked/storage_linux.go

@@ -62,46 +62,53 @@ const (
 type compressedFileType int
 
 type chunkedDiffer struct {
+	// Initial parameters, used throughout and never modified
+	// ==========
+	pullOptions pullOptions
 	stream      ImageSourceSeekable
-	manifest    []byte
+	// blobDigest is the digest of the whole compressed layer.  It is used if
-	toc         *minimal.TOC // The parsed contents of manifest, or nil if not yet available
+	// convertToZstdChunked to validate a layer when it is converted since there
-	tarSplit    []byte
+	// is no TOC referenced by the manifest.
-	layersCache *layersCache
+	blobDigest digest.Digest
-	tocOffset   int64
+	blobSize   int64
-	fileType    compressedFileType
-
-	copyBuffer []byte
-
-	gzipReader *pgzip.Reader
-	zstdReader *zstd.Decoder
-	rawReader  io.Reader
-
-	// tocDigest is the digest of the TOC document when the layer
-	// is partially pulled.
-	tocDigest digest.Digest
 
+	// Input format
+	// ==========
+	fileType compressedFileType
 	// convertedToZstdChunked is set to true if the layer needs to
 	// be converted to the zstd:chunked format before it can be
 	// handled.
 	convertToZstdChunked bool
 
+	// Chunked metadata
+	// This is usually set in GetDiffer, but if convertToZstdChunked, it is only computed in chunkedDiffer.ApplyDiff
+	// ==========
+	// tocDigest is the digest of the TOC document when the layer
+	// is partially pulled, or "" if not relevant to consumers.
+	tocDigest           digest.Digest
+	tocOffset           int64
+	manifest            []byte
+	toc                 *minimal.TOC // The parsed contents of manifest, or nil if not yet available
+	tarSplit            []byte
+	uncompressedTarSize int64 // -1 if unknown
 	// skipValidation is set to true if the individual files in
 	// the layer are trusted and should not be validated.
 	skipValidation bool
 
-	// blobDigest is the digest of the whole compressed layer.  It is used if
+	// Long-term caches
-	// convertToZstdChunked to validate a layer when it is converted since there
+	// This is set in GetDiffer, when the caller must not hold any storage locks, and later consumed in .ApplyDiff()
-	// is no TOC referenced by the manifest.
+	// ==========
-	blobDigest digest.Digest
+	layersCache     *layersCache
-
+	copyBuffer      []byte
-	blobSize            int64
+	fsVerityMutex   sync.Mutex // protects fsVerityDigests
-	uncompressedTarSize int64 // -1 if unknown
-
-	pullOptions pullOptions
-
-	useFsVerity     graphdriver.DifferFsVerity
 	fsVerityDigests map[string]string
-	fsVerityMutex   sync.Mutex
+
+	// Private state of .ApplyDiff
+	// ==========
+	gzipReader  *pgzip.Reader
+	zstdReader  *zstd.Decoder
+	rawReader   io.Reader
+	useFsVerity graphdriver.DifferFsVerity
 }
 
 var xattrsToIgnore = map[string]interface{}{
@@ -185,7 +192,7 @@ func (c *chunkedDiffer) convertTarToZstdChunked(destDirectory string, payload *o
 }
 
 // GetDiffer returns a differ than can be used with ApplyDiffWithDiffer.
-// If it returns an error that implements IsErrFallbackToOrdinaryLayerDownload, the caller can
+// If it returns an error that matches ErrFallbackToOrdinaryLayerDownload, the caller can
 // retry the operation with a different method.
 func GetDiffer(ctx context.Context, store storage.Store, blobDigest digest.Digest, blobSize int64, annotations map[string]string, iss ImageSourceSeekable) (graphdriver.Differ, error) {
 	pullOptions := parsePullOptions(store)
@@ -208,65 +215,94 @@ func GetDiffer(ctx context.Context, store storage.Store, blobDigest digest.Diges
 		return nil, newErrFallbackToOrdinaryLayerDownload(fmt.Errorf("graph driver %s does not support partial pull", graphDriver.String()))
 	}
 
-	differ, canFallback, err := getProperDiffer(store, blobDigest, blobSize, annotations, iss, pullOptions)
+	differ, err := getProperDiffer(store, blobDigest, blobSize, annotations, iss, pullOptions)
 	if err != nil {
-		if !canFallback {
+		var fallbackErr ErrFallbackToOrdinaryLayerDownload
+		if !errors.As(err, &fallbackErr) {
 			return nil, err
 		}
 		// If convert_images is enabled, always attempt to convert it instead of returning an error or falling back to a different method.
-		if pullOptions.convertImages {
+		if !pullOptions.convertImages {
-			logrus.Debugf("Created differ to convert blob %q", blobDigest)
+			return nil, err
-			return makeConvertFromRawDiffer(store, blobDigest, blobSize, iss, pullOptions)
 		}
-		return nil, newErrFallbackToOrdinaryLayerDownload(err)
+		var canConvertErr errFallbackCanConvert
+		if !errors.As(err, &canConvertErr) {
+			// We are supposed to use makeConvertFromRawDiffer, but that would not work.
+			// Fail, and make sure the error does _not_ match ErrFallbackToOrdinaryLayerDownload: use only the error text,
+			// discard all type information.
+			return nil, fmt.Errorf("neither a partial pull nor convert_images is possible: %s", err.Error())
+		}
+		logrus.Debugf("Created differ to convert blob %q", blobDigest)
+		return makeConvertFromRawDiffer(store, blobDigest, blobSize, iss, pullOptions)
 	}
 
 	return differ, nil
 }
 
+// errFallbackCanConvert is an an error type _accompanying_ ErrFallbackToOrdinaryLayerDownload
+// within getProperDiffer, to mark that using makeConvertFromRawDiffer makes sense.
+// This is used to distinguish between cases where the environment does not support partial pulls
+// (e.g. a registry does not support range requests) and convert_images is still possible,
+// from cases where the image content is unacceptable for partial pulls (e.g. exceeds memory limits)
+// and convert_images would not help.
+type errFallbackCanConvert struct {
+	err error
+}
+
+func (e errFallbackCanConvert) Error() string {
+	return e.err.Error()
+}
+
+func (e errFallbackCanConvert) Unwrap() error {
+	return e.err
+}
+
 // getProperDiffer is an implementation detail of GetDiffer.
 // It returns a “proper” differ (not a convert_images one) if possible.
-// On error, the second return value is true if a fallback to an alternative (either the makeConverToRaw differ, or a non-partial pull)
+// May return an error matching ErrFallbackToOrdinaryLayerDownload if a fallback to an alternative
-// is permissible.
+// (either makeConvertFromRawDiffer, or a non-partial pull) is permissible.
-func getProperDiffer(store storage.Store, blobDigest digest.Digest, blobSize int64, annotations map[string]string, iss ImageSourceSeekable, pullOptions pullOptions) (graphdriver.Differ, bool, error) {
+func getProperDiffer(store storage.Store, blobDigest digest.Digest, blobSize int64, annotations map[string]string, iss ImageSourceSeekable, pullOptions pullOptions) (graphdriver.Differ, error) {
 	zstdChunkedTOCDigestString, hasZstdChunkedTOC := annotations[minimal.ManifestChecksumKey]
 	estargzTOCDigestString, hasEstargzTOC := annotations[estargz.TOCJSONDigestAnnotation]
 
 	switch {
 	case hasZstdChunkedTOC && hasEstargzTOC:
-		return nil, false, errors.New("both zstd:chunked and eStargz TOC found")
+		return nil, errors.New("both zstd:chunked and eStargz TOC found")
 
 	case hasZstdChunkedTOC:
 		zstdChunkedTOCDigest, err := digest.Parse(zstdChunkedTOCDigestString)
 		if err != nil {
-			return nil, false, err
+			return nil, err
 		}
-		differ, canFallback, err := makeZstdChunkedDiffer(store, blobSize, zstdChunkedTOCDigest, annotations, iss, pullOptions)
+		differ, err := makeZstdChunkedDiffer(store, blobSize, zstdChunkedTOCDigest, annotations, iss, pullOptions)
 		if err != nil {
 			logrus.Debugf("Could not create zstd:chunked differ for blob %q: %v", blobDigest, err)
-			return nil, canFallback, err
+			return nil, err
 		}
 		logrus.Debugf("Created zstd:chunked differ for blob %q", blobDigest)
-		return differ, false, nil
+		return differ, nil
 
 	case hasEstargzTOC:
 		estargzTOCDigest, err := digest.Parse(estargzTOCDigestString)
 		if err != nil {
-			return nil, false, err
+			return nil, err
 		}
-		differ, canFallback, err := makeEstargzChunkedDiffer(store, blobSize, estargzTOCDigest, iss, pullOptions)
+		differ, err := makeEstargzChunkedDiffer(store, blobSize, estargzTOCDigest, iss, pullOptions)
 		if err != nil {
 			logrus.Debugf("Could not create estargz differ for blob %q: %v", blobDigest, err)
-			return nil, canFallback, err
+			return nil, err
 		}
 		logrus.Debugf("Created eStargz differ for blob %q", blobDigest)
-		return differ, false, nil
+		return differ, nil
 
 	default: // no TOC
+		message := "no TOC found"
 		if !pullOptions.convertImages {
-			return nil, true, errors.New("no TOC found and convert_images is not configured")
+			message = "no TOC found and convert_images is not configured"
+		}
+		return nil, errFallbackCanConvert{
+			newErrFallbackToOrdinaryLayerDownload(errors.New(message)),
 		}
-		return nil, true, errors.New("no TOC found")
 	}
 }
 
@@ -277,95 +313,100 @@ func makeConvertFromRawDiffer(store storage.Store, blobDigest digest.Digest, blo
 	}
 
 	return &chunkedDiffer{
-		fsVerityDigests:      make(map[string]string),
+		pullOptions: pullOptions,
-		blobDigest:           blobDigest,
+		stream:      iss,
-		blobSize:             blobSize,
+		blobDigest:  blobDigest,
-		uncompressedTarSize:  -1, // Will be computed later
+		blobSize:    blobSize,
+
 		convertToZstdChunked: true,
-		copyBuffer:           makeCopyBuffer(),
+
-		layersCache:          layersCache,
+		uncompressedTarSize: -1, // Will be computed later
-		pullOptions:          pullOptions,
+
-		stream:               iss,
+		layersCache:     layersCache,
+		copyBuffer:      makeCopyBuffer(),
+		fsVerityDigests: make(map[string]string),
 	}, nil
 }
 
 // makeZstdChunkedDiffer sets up a chunkedDiffer for a zstd:chunked layer.
-//
+// It may return an error matching ErrFallbackToOrdinaryLayerDownload / errFallbackCanConvert.
-// On error, the second return value is true if a fallback to an alternative (either the makeConverToRaw differ, or a non-partial pull)
+func makeZstdChunkedDiffer(store storage.Store, blobSize int64, tocDigest digest.Digest, annotations map[string]string, iss ImageSourceSeekable, pullOptions pullOptions) (*chunkedDiffer, error) {
-// is permissible.
-func makeZstdChunkedDiffer(store storage.Store, blobSize int64, tocDigest digest.Digest, annotations map[string]string, iss ImageSourceSeekable, pullOptions pullOptions) (*chunkedDiffer, bool, error) {
 	manifest, toc, tarSplit, tocOffset, err := readZstdChunkedManifest(iss, tocDigest, annotations)
-	if err != nil {
+	if err != nil { // May be ErrFallbackToOrdinaryLayerDownload / errFallbackCanConvert
-		// If the error is a bad request to the server, then signal to the caller that it can try a different method.
+		return nil, fmt.Errorf("read zstd:chunked manifest: %w", err)
-		var badRequestErr ErrBadRequest
-		return nil, errors.As(err, &badRequestErr), fmt.Errorf("read zstd:chunked manifest: %w", err)
 	}
 
 	var uncompressedTarSize int64 = -1
 	if tarSplit != nil {
 		uncompressedTarSize, err = tarSizeFromTarSplit(tarSplit)
 		if err != nil {
-			return nil, false, fmt.Errorf("computing size from tar-split: %w", err)
+			return nil, fmt.Errorf("computing size from tar-split: %w", err)
 		}
 	} else if !pullOptions.insecureAllowUnpredictableImageContents { // With no tar-split, we can't compute the traditional UncompressedDigest.
-		return nil, true, fmt.Errorf("zstd:chunked layers without tar-split data don't support partial pulls with guaranteed consistency with non-partial pulls")
+		return nil, errFallbackCanConvert{
+			newErrFallbackToOrdinaryLayerDownload(fmt.Errorf("zstd:chunked layers without tar-split data don't support partial pulls with guaranteed consistency with non-partial pulls")),
+		}
 	}
 
 	layersCache, err := getLayersCache(store)
 	if err != nil {
-		return nil, false, err
+		return nil, err
 	}
 
 	return &chunkedDiffer{
-		fsVerityDigests:     make(map[string]string),
+		pullOptions: pullOptions,
-		blobSize:            blobSize,
+		stream:      iss,
-		uncompressedTarSize: uncompressedTarSize,
+		blobSize:    blobSize,
+
+		fileType: fileTypeZstdChunked,
+
 		tocDigest:           tocDigest,
-		copyBuffer:          makeCopyBuffer(),
+		tocOffset:           tocOffset,
-		fileType:            fileTypeZstdChunked,
-		layersCache:         layersCache,
 		manifest:            manifest,
 		toc:                 toc,
-		pullOptions:         pullOptions,
-		stream:              iss,
 		tarSplit:            tarSplit,
-		tocOffset:           tocOffset,
+		uncompressedTarSize: uncompressedTarSize,
-	}, false, nil
+
+		layersCache:     layersCache,
+		copyBuffer:      makeCopyBuffer(),
+		fsVerityDigests: make(map[string]string),
+	}, nil
 }
 
-// makeZstdChunkedDiffer sets up a chunkedDiffer for an estargz layer.
+// makeEstargzChunkedDiffer sets up a chunkedDiffer for an estargz layer.
-//
+// It may return an error matching ErrFallbackToOrdinaryLayerDownload / errFallbackCanConvert.
-// On error, the second return value is true if a fallback to an alternative (either the makeConverToRaw differ, or a non-partial pull)
+func makeEstargzChunkedDiffer(store storage.Store, blobSize int64, tocDigest digest.Digest, iss ImageSourceSeekable, pullOptions pullOptions) (*chunkedDiffer, error) {
-// is permissible.
-func makeEstargzChunkedDiffer(store storage.Store, blobSize int64, tocDigest digest.Digest, iss ImageSourceSeekable, pullOptions pullOptions) (*chunkedDiffer, bool, error) {
 	if !pullOptions.insecureAllowUnpredictableImageContents { // With no tar-split, we can't compute the traditional UncompressedDigest.
-		return nil, true, fmt.Errorf("estargz layers don't support partial pulls with guaranteed consistency with non-partial pulls")
+		return nil, errFallbackCanConvert{
+			newErrFallbackToOrdinaryLayerDownload(fmt.Errorf("estargz layers don't support partial pulls with guaranteed consistency with non-partial pulls")),
+		}
 	}
 
 	manifest, tocOffset, err := readEstargzChunkedManifest(iss, blobSize, tocDigest)
-	if err != nil {
+	if err != nil { // May be ErrFallbackToOrdinaryLayerDownload / errFallbackCanConvert
-		// If the error is a bad request to the server, then signal to the caller that it can try a different method.
+		return nil, fmt.Errorf("read zstd:chunked manifest: %w", err)
-		var badRequestErr ErrBadRequest
-		return nil, errors.As(err, &badRequestErr), fmt.Errorf("read zstd:chunked manifest: %w", err)
 	}
 	layersCache, err := getLayersCache(store)
 	if err != nil {
-		return nil, false, err
+		return nil, err
 	}
 
 	return &chunkedDiffer{
-		fsVerityDigests:     make(map[string]string),
+		pullOptions: pullOptions,
-		blobSize:            blobSize,
+		stream:      iss,
-		uncompressedTarSize: -1, // We would have to read and decompress the whole layer
+		blobSize:    blobSize,
+
+		fileType: fileTypeEstargz,
+
 		tocDigest:           tocDigest,
-		copyBuffer:          makeCopyBuffer(),
-		fileType:            fileTypeEstargz,
-		layersCache:         layersCache,
-		manifest:            manifest,
-		pullOptions:         pullOptions,
-		stream:              iss,
 		tocOffset:           tocOffset,
-	}, false, nil
+		manifest:            manifest,
+		uncompressedTarSize: -1, // We would have to read and decompress the whole layer
+
+		layersCache:     layersCache,
+		copyBuffer:      makeCopyBuffer(),
+		fsVerityDigests: make(map[string]string),
+	}, nil
 }
 
 func makeCopyBuffer() []byte {

vendor/modules.txt

@@ -173,7 +173,7 @@ github.com/containers/ocicrypt/keywrap/pkcs7
 github.com/containers/ocicrypt/spec
 github.com/containers/ocicrypt/utils
 github.com/containers/ocicrypt/utils/keyprovider
-# github.com/containers/storage v1.57.0
+# github.com/containers/storage v1.57.1
 ## explicit; go 1.22.0
 github.com/containers/storage
 github.com/containers/storage/drivers