Merge pull request #2243 from containers/renovate/github.com-containers-storage-1.x

fix(deps): update module github.com/containers/storage to v1.53.0

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/containers/common v0.57.4
 	github.com/containers/image/v5 v5.29.3-0.20240301163503-faa4f4fd0e4f
 	github.com/containers/ocicrypt v1.1.9
-	github.com/containers/storage v1.52.1-0.20240301111729-226cffb1c4d2
+	github.com/containers/storage v1.53.0
 	github.com/docker/distribution v2.8.3+incompatible
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.1.0

go.sum

@@ -42,8 +42,8 @@ github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYgle
 github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY=
 github.com/containers/ocicrypt v1.1.9 h1:2Csfba4jse85Raxk5HIyEk8OwZNjRvfkhEGijOjIdEM=
 github.com/containers/ocicrypt v1.1.9/go.mod h1:dTKx1918d8TDkxXvarscpNVY+lyPakPNFN4jwA9GBys=
-github.com/containers/storage v1.52.1-0.20240301111729-226cffb1c4d2 h1:9+rUow7EtVgtlcn8oFValad/M548QnSpSCSBctduPbw=
+github.com/containers/storage v1.53.0 h1:VSES3C/u1pxjTJIXvLrSmyP7OBtDky04oGu07UvdTEA=
-github.com/containers/storage v1.52.1-0.20240301111729-226cffb1c4d2/go.mod h1:mFA6QpUoT9qTa3q2DD1CvSo3Az3syNkw1P9X+4nUYdY=
+github.com/containers/storage v1.53.0/go.mod h1:pujcoOSc+upx15Jirdkebhtd8uJiLwbSd/mYT6zDJK8=
 github.com/coreos/go-oidc/v3 v3.9.0 h1:0J/ogVOd4y8P0f0xUh8l9t07xRP/d8tccvjHl2dcsSo=
 github.com/coreos/go-oidc/v3 v3.9.0/go.mod h1:rTKz2PYwftcrtoCzV5g5kvfJoWcm0Mk8AF8y1iAQro4=
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=

vendor/github.com/containers/storage/VERSION

@@ -1 +1 @@
-1.52.1-dev
+1.53.0

vendor/github.com/containers/storage/pkg/chunked/storage_linux.go

@@ -41,6 +41,7 @@ import (
 
 const (
 	maxNumberMissingChunks  = 1024
+	autoMergePartsThreshold = 128 // if the gap between two ranges is below this threshold, automatically merge them.
 	newFileFlags            = (unix.O_CREAT | unix.O_TRUNC | unix.O_EXCL | unix.O_WRONLY)
 	containersOverrideXattr = "user.containers.override_stat"
 	bigDataKey              = "zstd-chunked-manifest"
@@ -1180,22 +1181,12 @@ func (c *chunkedDiffer) storeMissingFiles(streams chan io.ReadCloser, errs chan
 }
 
 func mergeMissingChunks(missingParts []missingPart, target int) []missingPart {
-	getGap := func(missingParts []missingPart, i int) int {
+	getGap := func(missingParts []missingPart, i int) uint64 {
 		prev := missingParts[i-1].SourceChunk.Offset + missingParts[i-1].SourceChunk.Length
-		return int(missingParts[i].SourceChunk.Offset - prev)
+		return missingParts[i].SourceChunk.Offset - prev
-	}
-	getCost := func(missingParts []missingPart, i int) int {
-		cost := getGap(missingParts, i)
-		if missingParts[i-1].OriginFile != nil {
-			cost += int(missingParts[i-1].SourceChunk.Length)
-		}
-		if missingParts[i].OriginFile != nil {
-			cost += int(missingParts[i].SourceChunk.Length)
-		}
-		return cost
 	}
 
-	// simple case: merge chunks from the same file.
+	// simple case: merge chunks from the same file.  Useful to reduce the number of parts to work with later.
 	newMissingParts := missingParts[0:1]
 	prevIndex := 0
 	for i := 1; i < len(missingParts); i++ {
@@ -1215,28 +1206,50 @@ func mergeMissingChunks(missingParts []missingPart, target int) []missingPart {
 	}
 	missingParts = newMissingParts
 
-	if len(missingParts) <= target {
+	type gap struct {
-		return missingParts
+		from int
+		to   int
+		cost uint64
 	}
-
+	var requestGaps []gap
-	// this implementation doesn't account for duplicates, so it could merge
+	lastOffset := int(-1)
-	// more than necessary to reach the specified target.  Since target itself
+	numberSourceChunks := 0
-	// is a heuristic value, it doesn't matter.
+	for i, c := range missingParts {
-	costs := make([]int, len(missingParts)-1)
+		if c.OriginFile != nil || c.Hole {
-	for i := 1; i < len(missingParts); i++ {
+			// it does not require a network request
-		costs[i-1] = getCost(missingParts, i)
+			continue
 		}
-	sort.Ints(costs)
+		numberSourceChunks++
-
+		if lastOffset >= 0 {
-	toShrink := len(missingParts) - target
+			prevEnd := missingParts[lastOffset].SourceChunk.Offset + missingParts[lastOffset].SourceChunk.Length
-	if toShrink >= len(costs) {
+			cost := c.SourceChunk.Offset - prevEnd
-		toShrink = len(costs) - 1
+			g := gap{
+				from: lastOffset,
+				to:   i,
+				cost: cost,
+			}
+			requestGaps = append(requestGaps, g)
+		}
+		lastOffset = i
+	}
+	sort.Slice(requestGaps, func(i, j int) bool {
+		return requestGaps[i].cost < requestGaps[j].cost
+	})
+	toMergeMap := make([]bool, len(missingParts))
+	remainingToMerge := numberSourceChunks - target
+	for _, g := range requestGaps {
+		if remainingToMerge < 0 && g.cost > autoMergePartsThreshold {
+			continue
+		}
+		for i := g.from + 1; i <= g.to; i++ {
+			toMergeMap[i] = true
+		}
+		remainingToMerge--
 	}
-	targetValue := costs[toShrink]
 
 	newMissingParts = missingParts[0:1]
 	for i := 1; i < len(missingParts); i++ {
-		if getCost(missingParts, i) > targetValue {
+		if !toMergeMap[i] {
 			newMissingParts = append(newMissingParts, missingParts[i])
 		} else {
 			gap := getGap(missingParts, i)
@@ -1268,6 +1281,7 @@ func (c *chunkedDiffer) retrieveMissingFiles(stream ImageSourceSeekable, dest st
 		}
 	}
 
+	missingParts = mergeMissingChunks(missingParts, maxNumberMissingChunks)
 	calculateChunksToRequest()
 
 	// There are some missing files.  Prepare a multirange request for the missing chunks.
@@ -1281,14 +1295,13 @@ func (c *chunkedDiffer) retrieveMissingFiles(stream ImageSourceSeekable, dest st
 		}
 
 		if _, ok := err.(ErrBadRequest); ok {
-			requested := len(missingParts)
 			// If the server cannot handle at least 64 chunks in a single request, just give up.
-			if requested < 64 {
+			if len(chunksToRequest) < 64 {
 				return err
 			}
 
 			// Merge more chunks to request
-			missingParts = mergeMissingChunks(missingParts, requested/2)
+			missingParts = mergeMissingChunks(missingParts, len(chunksToRequest)/2)
 			calculateChunksToRequest()
 			continue
 		}
@@ -1999,7 +2012,6 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
 	}
 	// There are some missing files.  Prepare a multirange request for the missing chunks.
 	if len(missingParts) > 0 {
-		missingParts = mergeMissingChunks(missingParts, maxNumberMissingChunks)
 		if err := c.retrieveMissingFiles(stream, dest, dirfd, missingParts, options); err != nil {
 			return output, err
 		}

vendor/modules.txt

@@ -164,7 +164,7 @@ github.com/containers/ocicrypt/keywrap/pkcs7
 github.com/containers/ocicrypt/spec
 github.com/containers/ocicrypt/utils
 github.com/containers/ocicrypt/utils/keyprovider
-# github.com/containers/storage v1.52.1-0.20240301111729-226cffb1c4d2
+# github.com/containers/storage v1.53.0
 ## explicit; go 1.20
 github.com/containers/storage
 github.com/containers/storage/drivers