fix(deps): update module github.com/containers/storage to v1.50.1

Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
This commit is contained in:
renovate[bot]
2023-09-12 13:54:25 +00:00
committed by GitHub
parent ea0a627e64
commit 9563e3b84b
45 changed files with 970 additions and 238 deletions

View File

@@ -2,8 +2,6 @@ package chunked
import (
archivetar "archive/tar"
"bytes"
"encoding/binary"
"errors"
"fmt"
"io"
@@ -35,13 +33,6 @@ func typeToTarType(t string) (byte, error) {
return r, nil
}
func isZstdChunkedFrameMagic(data []byte) bool {
if len(data) < 8 {
return false
}
return bytes.Equal(internal.ZstdChunkedFrameMagic, data[:8])
}
func readEstargzChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, annotations map[string]string) ([]byte, int64, error) {
// information on the format here https://github.com/containerd/stargz-snapshotter/blob/main/docs/stargz-estargz.md
footerSize := int64(51)
@@ -155,27 +146,14 @@ func readZstdChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, ann
return nil, nil, 0, errors.New("blob too small")
}
manifestChecksumAnnotation := annotations[internal.ManifestChecksumKey]
if manifestChecksumAnnotation == "" {
return nil, nil, 0, fmt.Errorf("manifest checksum annotation %q not found", internal.ManifestChecksumKey)
}
var offset, length, lengthUncompressed, manifestType uint64
var offsetTarSplit, lengthTarSplit, lengthUncompressedTarSplit uint64
tarSplitChecksumAnnotation := ""
var footerData internal.ZstdChunkedFooterData
if offsetMetadata := annotations[internal.ManifestInfoKey]; offsetMetadata != "" {
if _, err := fmt.Sscanf(offsetMetadata, "%d:%d:%d:%d", &offset, &length, &lengthUncompressed, &manifestType); err != nil {
var err error
footerData, err = internal.ReadFooterDataFromAnnotations(annotations)
if err != nil {
return nil, nil, 0, err
}
if tarSplitInfoKeyAnnotation, found := annotations[internal.TarSplitInfoKey]; found {
if _, err := fmt.Sscanf(tarSplitInfoKeyAnnotation, "%d:%d:%d", &offsetTarSplit, &lengthTarSplit, &lengthUncompressedTarSplit); err != nil {
return nil, nil, 0, err
}
tarSplitChecksumAnnotation = annotations[internal.TarSplitChecksumKey]
}
} else {
chunk := ImageSourceChunk{
Offset: uint64(blobSize - footerSize),
@@ -197,38 +175,35 @@ func readZstdChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, ann
return nil, nil, 0, err
}
offset = binary.LittleEndian.Uint64(footer[0:8])
length = binary.LittleEndian.Uint64(footer[8:16])
lengthUncompressed = binary.LittleEndian.Uint64(footer[16:24])
manifestType = binary.LittleEndian.Uint64(footer[24:32])
if !isZstdChunkedFrameMagic(footer[48:56]) {
return nil, nil, 0, errors.New("invalid magic number")
footerData, err = internal.ReadFooterDataFromBlob(footer)
if err != nil {
return nil, nil, 0, err
}
}
if manifestType != internal.ManifestTypeCRFS {
if footerData.ManifestType != internal.ManifestTypeCRFS {
return nil, nil, 0, errors.New("invalid manifest type")
}
// set a reasonable limit
if length > (1<<20)*50 {
if footerData.LengthCompressed > (1<<20)*50 {
return nil, nil, 0, errors.New("manifest too big")
}
if lengthUncompressed > (1<<20)*50 {
if footerData.LengthUncompressed > (1<<20)*50 {
return nil, nil, 0, errors.New("manifest too big")
}
chunk := ImageSourceChunk{
Offset: offset,
Length: length,
Offset: footerData.Offset,
Length: footerData.LengthCompressed,
}
chunks := []ImageSourceChunk{chunk}
if offsetTarSplit > 0 {
if footerData.OffsetTarSplit > 0 {
chunkTarSplit := ImageSourceChunk{
Offset: offsetTarSplit,
Length: lengthTarSplit,
Offset: footerData.OffsetTarSplit,
Length: footerData.LengthCompressedTarSplit,
}
chunks = append(chunks, chunkTarSplit)
}
@@ -258,28 +233,28 @@ func readZstdChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, ann
return blob, nil
}
manifest, err := readBlob(length)
manifest, err := readBlob(footerData.LengthCompressed)
if err != nil {
return nil, nil, 0, err
}
decodedBlob, err := decodeAndValidateBlob(manifest, lengthUncompressed, manifestChecksumAnnotation)
decodedBlob, err := decodeAndValidateBlob(manifest, footerData.LengthUncompressed, footerData.ChecksumAnnotation)
if err != nil {
return nil, nil, 0, err
}
decodedTarSplit := []byte{}
if offsetTarSplit > 0 {
tarSplit, err := readBlob(lengthTarSplit)
if footerData.OffsetTarSplit > 0 {
tarSplit, err := readBlob(footerData.LengthCompressedTarSplit)
if err != nil {
return nil, nil, 0, err
}
decodedTarSplit, err = decodeAndValidateBlob(tarSplit, lengthUncompressedTarSplit, tarSplitChecksumAnnotation)
decodedTarSplit, err = decodeAndValidateBlob(tarSplit, footerData.LengthUncompressedTarSplit, footerData.ChecksumAnnotationTarSplit)
if err != nil {
return nil, nil, 0, err
}
}
return decodedBlob, decodedTarSplit, int64(offset), err
return decodedBlob, decodedTarSplit, int64(footerData.Offset), err
}
func decodeAndValidateBlob(blob []byte, lengthUncompressed uint64, expectedUncompressedChecksum string) ([]byte, error) {

View File

@@ -8,6 +8,7 @@ import (
"archive/tar"
"bytes"
"encoding/binary"
"errors"
"fmt"
"io"
"time"
@@ -99,7 +100,7 @@ const (
// FooterSizeSupported is the footer size supported by this implementation.
// Newer versions of the image format might increase this value, so reject
// any version that is not supported.
FooterSizeSupported = 56
FooterSizeSupported = 64
)
var (
@@ -108,7 +109,7 @@ var (
// https://tools.ietf.org/html/rfc8478#section-3.1.2
skippableFrameMagic = []byte{0x50, 0x2a, 0x4d, 0x18}
ZstdChunkedFrameMagic = []byte{0x47, 0x6e, 0x55, 0x6c, 0x49, 0x6e, 0x55, 0x78}
ZstdChunkedFrameMagic = []byte{0x47, 0x4e, 0x55, 0x6c, 0x49, 0x6e, 0x55, 0x78}
)
func appendZstdSkippableFrame(dest io.Writer, data []byte) error {
@@ -183,13 +184,19 @@ func WriteZstdChunkedManifest(dest io.Writer, outMetadata map[string]string, off
return err
}
// Store the offset to the manifest and its size in LE order
manifestDataLE := make([]byte, FooterSizeSupported)
binary.LittleEndian.PutUint64(manifestDataLE, manifestOffset)
binary.LittleEndian.PutUint64(manifestDataLE[8*1:], uint64(len(compressedManifest)))
binary.LittleEndian.PutUint64(manifestDataLE[8*2:], uint64(len(manifest)))
binary.LittleEndian.PutUint64(manifestDataLE[8*3:], uint64(ManifestTypeCRFS))
copy(manifestDataLE[8*4:], ZstdChunkedFrameMagic)
footer := ZstdChunkedFooterData{
ManifestType: uint64(ManifestTypeCRFS),
Offset: manifestOffset,
LengthCompressed: uint64(len(compressedManifest)),
LengthUncompressed: uint64(len(manifest)),
ChecksumAnnotation: "", // unused
OffsetTarSplit: uint64(tarSplitOffset),
LengthCompressedTarSplit: uint64(len(tarSplitData.Data)),
LengthUncompressedTarSplit: uint64(tarSplitData.UncompressedSize),
ChecksumAnnotationTarSplit: "", // unused
}
manifestDataLE := footerDataToBlob(footer)
return appendZstdSkippableFrame(dest, manifestDataLE)
}
@@ -198,3 +205,79 @@ func ZstdWriterWithLevel(dest io.Writer, level int) (*zstd.Encoder, error) {
el := zstd.EncoderLevelFromZstd(level)
return zstd.NewWriter(dest, zstd.WithEncoderLevel(el))
}
// ZstdChunkedFooterData contains all the data stored in the zstd:chunked footer.
type ZstdChunkedFooterData struct {
ManifestType uint64
Offset uint64
LengthCompressed uint64
LengthUncompressed uint64
ChecksumAnnotation string // Only used when reading a layer, not when creating it
OffsetTarSplit uint64
LengthCompressedTarSplit uint64
LengthUncompressedTarSplit uint64
ChecksumAnnotationTarSplit string // Only used when reading a layer, not when creating it
}
func footerDataToBlob(footer ZstdChunkedFooterData) []byte {
// Store the offset to the manifest and its size in LE order
manifestDataLE := make([]byte, FooterSizeSupported)
binary.LittleEndian.PutUint64(manifestDataLE[8*0:], footer.Offset)
binary.LittleEndian.PutUint64(manifestDataLE[8*1:], footer.LengthCompressed)
binary.LittleEndian.PutUint64(manifestDataLE[8*2:], footer.LengthUncompressed)
binary.LittleEndian.PutUint64(manifestDataLE[8*3:], footer.ManifestType)
binary.LittleEndian.PutUint64(manifestDataLE[8*4:], footer.OffsetTarSplit)
binary.LittleEndian.PutUint64(manifestDataLE[8*5:], footer.LengthCompressedTarSplit)
binary.LittleEndian.PutUint64(manifestDataLE[8*6:], footer.LengthUncompressedTarSplit)
copy(manifestDataLE[8*7:], ZstdChunkedFrameMagic)
return manifestDataLE
}
// ReadFooterDataFromAnnotations reads the zstd:chunked footer data from the given annotations.
func ReadFooterDataFromAnnotations(annotations map[string]string) (ZstdChunkedFooterData, error) {
var footerData ZstdChunkedFooterData
footerData.ChecksumAnnotation = annotations[ManifestChecksumKey]
if footerData.ChecksumAnnotation == "" {
return footerData, fmt.Errorf("manifest checksum annotation %q not found", ManifestChecksumKey)
}
offsetMetadata := annotations[ManifestInfoKey]
if _, err := fmt.Sscanf(offsetMetadata, "%d:%d:%d:%d", &footerData.Offset, &footerData.LengthCompressed, &footerData.LengthUncompressed, &footerData.ManifestType); err != nil {
return footerData, err
}
if tarSplitInfoKeyAnnotation, found := annotations[TarSplitInfoKey]; found {
if _, err := fmt.Sscanf(tarSplitInfoKeyAnnotation, "%d:%d:%d", &footerData.OffsetTarSplit, &footerData.LengthCompressedTarSplit, &footerData.LengthUncompressedTarSplit); err != nil {
return footerData, err
}
footerData.ChecksumAnnotationTarSplit = annotations[TarSplitChecksumKey]
}
return footerData, nil
}
// ReadFooterDataFromBlob reads the zstd:chunked footer from the binary buffer.
func ReadFooterDataFromBlob(footer []byte) (ZstdChunkedFooterData, error) {
var footerData ZstdChunkedFooterData
if len(footer) < FooterSizeSupported {
return footerData, errors.New("blob too small")
}
footerData.Offset = binary.LittleEndian.Uint64(footer[0:8])
footerData.LengthCompressed = binary.LittleEndian.Uint64(footer[8:16])
footerData.LengthUncompressed = binary.LittleEndian.Uint64(footer[16:24])
footerData.ManifestType = binary.LittleEndian.Uint64(footer[24:32])
footerData.OffsetTarSplit = binary.LittleEndian.Uint64(footer[32:40])
footerData.LengthCompressedTarSplit = binary.LittleEndian.Uint64(footer[40:48])
footerData.LengthUncompressedTarSplit = binary.LittleEndian.Uint64(footer[48:56])
// the magic number is stored in the last 8 bytes
if !bytes.Equal(ZstdChunkedFrameMagic, footer[len(footer)-len(ZstdChunkedFrameMagic):]) {
return footerData, errors.New("invalid magic number")
}
return footerData, nil
}