github/skopeo
1
0
Fork 0
mirror of https://github.com/containers/skopeo.git synced 2025-09-04 08:04:56 +00:00
Code Issues Projects Releases Wiki Activity

fix(deps): update module github.com/containers/storage to v1.45.0

Browse Source 
Signed-off-by: Renovate Bot <bot@renovateapp.com>
This commit is contained in:
renovate[bot]
2023-01-13 01:15:34 +00:00
committed by GitHub
parent 28175104d7
commit 1133a2a395
118 changed files with 3801 additions and 1741 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
vendor/github.com/containerd/stargz-snapshotter/estargz/build.go generated vendored
View File

@@ -49,6 +49,7 @@ type options struct {
missedPrioritizedFiles *[]string
compression Compression
ctx context.Context
minChunkSize int
}
type Option func(o *options) error
@@ -63,6 +64,7 @@ func WithChunkSize(chunkSize int) Option {
// WithCompressionLevel option specifies the gzip compression level.
// The default is gzip.BestCompression.
// This option will be ignored if WithCompression option is used.
// See also: https://godoc.org/compress/gzip#pkg-constants
func WithCompressionLevel(level int) Option {
return func(o *options) error {
@@ -113,6 +115,18 @@ func WithContext(ctx context.Context) Option {
}
}
// WithMinChunkSize option specifies the minimal number of bytes of data
// must be written in one gzip stream.
// By increasing this number, one gzip stream can contain multiple files
// and it hopefully leads to smaller result blob.
// NOTE: This adds a TOC property that old reader doesn't understand.
func WithMinChunkSize(minChunkSize int) Option {
return func(o *options) error {
o.minChunkSize = minChunkSize
return nil
}
}
// Blob is an eStargz blob.
type Blob struct {
io.ReadCloser
@@ -180,7 +194,14 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) {
if err != nil {
return nil, err
}
tarParts := divideEntries(entries, runtime.GOMAXPROCS(0))
var tarParts [][]*entry
if opts.minChunkSize > 0 {
// Each entry needs to know the size of the current gzip stream so they
// cannot be processed in parallel.
tarParts = [][]*entry{entries}
} else {
tarParts = divideEntries(entries, runtime.GOMAXPROCS(0))
}
writers := make([]*Writer, len(tarParts))
payloads := make([]*os.File, len(tarParts))
var mu sync.Mutex
@@ -195,6 +216,13 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) {
}
sw := NewWriterWithCompressor(esgzFile, opts.compression)
sw.ChunkSize = opts.chunkSize
sw.MinChunkSize = opts.minChunkSize
if sw.needsOpenGzEntries == nil {
sw.needsOpenGzEntries = make(map[string]struct{})
}
for _, f := range []string{PrefetchLandmark, NoPrefetchLandmark} {
sw.needsOpenGzEntries[f] = struct{}{}
}
if err := sw.AppendTar(readerFromEntries(parts...)); err != nil {
return err
}
@@ -209,7 +237,7 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) {
rErr = err
return nil, err
}
tocAndFooter, tocDgst, err := closeWithCombine(opts.compressionLevel, writers...)
tocAndFooter, tocDgst, err := closeWithCombine(writers...)
if err != nil {
rErr = err
return nil, err
@@ -252,7 +280,7 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) {
// Writers doesn't write TOC and footer to the underlying writers so they can be
// combined into a single eStargz and tocAndFooter returned by this function can
// be appended at the tail of that combined blob.
func closeWithCombine(compressionLevel int, ws ...*Writer) (tocAndFooterR io.Reader, tocDgst digest.Digest, err error) {
func closeWithCombine(ws ...*Writer) (tocAndFooterR io.Reader, tocDgst digest.Digest, err error) {
if len(ws) == 0 {
return nil, "", fmt.Errorf("at least one writer must be passed")
}
@@ -395,7 +423,7 @@ func readerFromEntries(entries ...*entry) io.Reader {
func importTar(in io.ReaderAt) (*tarFile, error) {
tf := &tarFile{}
pw, err := newCountReader(in)
pw, err := newCountReadSeeker(in)
if err != nil {
return nil, fmt.Errorf("failed to make position watcher: %w", err)
}
@@ -571,19 +599,19 @@ func (tf *tempFiles) cleanupAll() error {
return errorutil.Aggregate(allErr)
}
func newCountReader(r io.ReaderAt) (*countReader, error) {
func newCountReadSeeker(r io.ReaderAt) (*countReadSeeker, error) {
pos := int64(0)
return &countReader{r: r, cPos: &pos}, nil
return &countReadSeeker{r: r, cPos: &pos}, nil
}
type countReader struct {
type countReadSeeker struct {
r io.ReaderAt
cPos *int64
mu sync.Mutex
}
func (cr *countReader) Read(p []byte) (int, error) {
func (cr *countReadSeeker) Read(p []byte) (int, error) {
cr.mu.Lock()
defer cr.mu.Unlock()
@@ -594,7 +622,7 @@ func (cr *countReader) Read(p []byte) (int, error) {
return n, err
}
func (cr *countReader) Seek(offset int64, whence int) (int64, error) {
func (cr *countReadSeeker) Seek(offset int64, whence int) (int64, error) {
cr.mu.Lock()
defer cr.mu.Unlock()
@@ -615,7 +643,7 @@ func (cr *countReader) Seek(offset int64, whence int) (int64, error) {
return offset, nil
}
func (cr *countReader) currentPos() int64 {
func (cr *countReadSeeker) currentPos() int64 {
cr.mu.Lock()
defer cr.mu.Unlock()

224
vendor/github.com/containerd/stargz-snapshotter/estargz/estargz.go generated vendored
View File

@@ -150,10 +150,10 @@ func Open(sr *io.SectionReader, opt ...OpenOption) (*Reader, error) {
allErr = append(allErr, err)
continue
}
if tocSize <= 0 {
if tocOffset >= 0 && tocSize <= 0 {
tocSize = sr.Size() - tocOffset - fSize
}
if tocSize < int64(len(maybeTocBytes)) {
if tocOffset >= 0 && tocSize < int64(len(maybeTocBytes)) {
maybeTocBytes = maybeTocBytes[:tocSize]
}
r, err = parseTOC(d, sr, tocOffset, tocSize, maybeTocBytes, opts)
@@ -207,8 +207,16 @@ func (r *Reader) initFields() error {
uname := map[int]string{}
gname := map[int]string{}
var lastRegEnt *TOCEntry
for _, ent := range r.toc.Entries {
var chunkTopIndex int
for i, ent := range r.toc.Entries {
ent.Name = cleanEntryName(ent.Name)
switch ent.Type {
case "reg", "chunk":
if ent.Offset != r.toc.Entries[chunkTopIndex].Offset {
chunkTopIndex = i
}
ent.chunkTopIndex = chunkTopIndex
}
if ent.Type == "reg" {
lastRegEnt = ent
}
@@ -294,7 +302,7 @@ func (r *Reader) initFields() error {
if e.isDataType() {
e.nextOffset = lastOffset
}
if e.Offset != 0 {
if e.Offset != 0 && e.InnerOffset == 0 {
lastOffset = e.Offset
}
}
@@ -488,6 +496,14 @@ func (r *Reader) Lookup(path string) (e *TOCEntry, ok bool) {
//
// Name must be absolute path or one that is relative to root.
func (r *Reader) OpenFile(name string) (*io.SectionReader, error) {
fr, err := r.newFileReader(name)
if err != nil {
return nil, err
}
return io.NewSectionReader(fr, 0, fr.size), nil
}
func (r *Reader) newFileReader(name string) (*fileReader, error) {
name = cleanEntryName(name)
ent, ok := r.Lookup(name)
if !ok {
@@ -505,11 +521,19 @@ func (r *Reader) OpenFile(name string) (*io.SectionReader, error) {
Err: errors.New("not a regular file"),
}
}
fr := &fileReader{
return &fileReader{
r: r,
size: ent.Size,
ents: r.getChunks(ent),
}, nil
}
func (r *Reader) OpenFileWithPreReader(name string, preRead func(*TOCEntry, io.Reader) error) (*io.SectionReader, error) {
fr, err := r.newFileReader(name)
if err != nil {
return nil, err
}
fr.preRead = preRead
return io.NewSectionReader(fr, 0, fr.size), nil
}
@@ -521,9 +545,10 @@ func (r *Reader) getChunks(ent *TOCEntry) []*TOCEntry {
}
type fileReader struct {
r *Reader
size int64
ents []*TOCEntry // 1 or more reg/chunk entries
r *Reader
size int64
ents []*TOCEntry // 1 or more reg/chunk entries
preRead func(*TOCEntry, io.Reader) error
}
func (fr *fileReader) ReadAt(p []byte, off int64) (n int, err error) {
@@ -578,10 +603,48 @@ func (fr *fileReader) ReadAt(p []byte, off int64) (n int, err error) {
return 0, fmt.Errorf("fileReader.ReadAt.decompressor.Reader: %v", err)
}
defer dr.Close()
if n, err := io.CopyN(io.Discard, dr, off); n != off || err != nil {
return 0, fmt.Errorf("discard of %d bytes = %v, %v", off, n, err)
if fr.preRead == nil {
if n, err := io.CopyN(io.Discard, dr, ent.InnerOffset+off); n != ent.InnerOffset+off || err != nil {
return 0, fmt.Errorf("discard of %d bytes != %v, %v", ent.InnerOffset+off, n, err)
}
return io.ReadFull(dr, p)
}
return io.ReadFull(dr, p)
var retN int
var retErr error
var found bool
var nr int64
for _, e := range fr.r.toc.Entries[ent.chunkTopIndex:] {
if !e.isDataType() {
continue
}
if e.Offset != fr.r.toc.Entries[ent.chunkTopIndex].Offset {
break
}
if in, err := io.CopyN(io.Discard, dr, e.InnerOffset-nr); err != nil || in != e.InnerOffset-nr {
return 0, fmt.Errorf("discard of remaining %d bytes != %v, %v", e.InnerOffset-nr, in, err)
}
nr = e.InnerOffset
if e == ent {
found = true
if n, err := io.CopyN(io.Discard, dr, off); n != off || err != nil {
return 0, fmt.Errorf("discard of offset %d bytes != %v, %v", off, n, err)
}
retN, retErr = io.ReadFull(dr, p)
nr += off + int64(retN)
continue
}
cr := &countReader{r: io.LimitReader(dr, e.ChunkSize)}
if err := fr.preRead(e, cr); err != nil {
return 0, fmt.Errorf("failed to pre read: %w", err)
}
nr += cr.n
}
if !found {
return 0, fmt.Errorf("fileReader.ReadAt: target entry not found")
}
return retN, retErr
}
// A Writer writes stargz files.
@@ -599,11 +662,20 @@ type Writer struct {
lastGroupname map[int]string
compressor Compressor
uncompressedCounter *countWriteFlusher
// ChunkSize optionally controls the maximum number of bytes
// of data of a regular file that can be written in one gzip
// stream before a new gzip stream is started.
// Zero means to use a default, currently 4 MiB.
ChunkSize int
// MinChunkSize optionally controls the minimum number of bytes
// of data must be written in one gzip stream before a new gzip
// NOTE: This adds a TOC property that stargz snapshotter < v0.13.0 doesn't understand.
MinChunkSize int
needsOpenGzEntries map[string]struct{}
}
// currentCompressionWriter writes to the current w.gz field, which can
@@ -646,6 +718,9 @@ func Unpack(sr *io.SectionReader, c Decompressor) (io.ReadCloser, error) {
if err != nil {
return nil, fmt.Errorf("failed to parse footer: %w", err)
}
if blobPayloadSize < 0 {
blobPayloadSize = sr.Size()
}
return c.Reader(io.LimitReader(sr, blobPayloadSize))
}
@@ -672,11 +747,12 @@ func NewWriterWithCompressor(w io.Writer, c Compressor) *Writer {
bw := bufio.NewWriter(w)
cw := &countWriter{w: bw}
return &Writer{
bw: bw,
cw: cw,
toc: &JTOC{Version: 1},
diffHash: sha256.New(),
compressor: c,
bw: bw,
cw: cw,
toc: &JTOC{Version: 1},
diffHash: sha256.New(),
compressor: c,
uncompressedCounter: &countWriteFlusher{},
}
}
@@ -717,6 +793,20 @@ func (w *Writer) closeGz() error {
return nil
}
func (w *Writer) flushGz() error {
if w.closed {
return errors.New("flush on closed Writer")
}
if w.gz != nil {
if f, ok := w.gz.(interface {
Flush() error
}); ok {
return f.Flush()
}
}
return nil
}
// nameIfChanged returns name, unless it was the already the value of (*mp)[id],
// in which case it returns the empty string.
func (w *Writer) nameIfChanged(mp *map[int]string, id int, name string) string {
@@ -736,6 +826,9 @@ func (w *Writer) nameIfChanged(mp *map[int]string, id int, name string) string {
func (w *Writer) condOpenGz() (err error) {
if w.gz == nil {
w.gz, err = w.compressor.Writer(w.cw)
if w.gz != nil {
w.gz = w.uncompressedCounter.register(w.gz)
}
}
return
}
@@ -784,6 +877,8 @@ func (w *Writer) appendTar(r io.Reader, lossless bool) error {
if lossless {
tr.RawAccounting = true
}
prevOffset := w.cw.n
var prevOffsetUncompressed int64
for {
h, err := tr.Next()
if err == io.EOF {
@@ -883,10 +978,6 @@ func (w *Writer) appendTar(r io.Reader, lossless bool) error {
totalSize := ent.Size // save it before we destroy ent
tee := io.TeeReader(tr, payloadDigest.Hash())
for written < totalSize {
if err := w.closeGz(); err != nil {
return err
}
chunkSize := int64(w.chunkSize())
remain := totalSize - written
if remain < chunkSize {
@@ -894,7 +985,23 @@ func (w *Writer) appendTar(r io.Reader, lossless bool) error {
} else {
ent.ChunkSize = chunkSize
}
ent.Offset = w.cw.n
// We flush the underlying compression writer here to correctly calculate "w.cw.n".
if err := w.flushGz(); err != nil {
return err
}
if w.needsOpenGz(ent) || w.cw.n-prevOffset >= int64(w.MinChunkSize) {
if err := w.closeGz(); err != nil {
return err
}
ent.Offset = w.cw.n
prevOffset = ent.Offset
prevOffsetUncompressed = w.uncompressedCounter.n
} else {
ent.Offset = prevOffset
ent.InnerOffset = w.uncompressedCounter.n - prevOffsetUncompressed
}
ent.ChunkOffset = written
chunkDigest := digest.Canonical.Digester()
@@ -940,6 +1047,17 @@ func (w *Writer) appendTar(r io.Reader, lossless bool) error {
return err
}
func (w *Writer) needsOpenGz(ent *TOCEntry) bool {
if ent.Type != "reg" {
return false
}
if w.needsOpenGzEntries == nil {
return false
}
_, ok := w.needsOpenGzEntries[ent.Name]
return ok
}
// DiffID returns the SHA-256 of the uncompressed tar bytes.
// It is only valid to call DiffID after Close.
func (w *Writer) DiffID() string {
@@ -956,6 +1074,28 @@ func maxFooterSize(blobSize int64, decompressors ...Decompressor) (res int64) {
}
func parseTOC(d Decompressor, sr *io.SectionReader, tocOff, tocSize int64, tocBytes []byte, opts openOpts) (*Reader, error) {
if tocOff < 0 {
// This means that TOC isn't contained in the blob.
// We pass nil reader to ParseTOC and expect that ParseTOC acquire TOC from
// the external location.
start := time.Now()
toc, tocDgst, err := d.ParseTOC(nil)
if err != nil {
return nil, err
}
if opts.telemetry != nil && opts.telemetry.GetTocLatency != nil {
opts.telemetry.GetTocLatency(start)
}
if opts.telemetry != nil && opts.telemetry.DeserializeTocLatency != nil {
opts.telemetry.DeserializeTocLatency(start)
}
return &Reader{
sr: sr,
toc: toc,
tocDigest: tocDgst,
decompressor: d,
}, nil
}
if len(tocBytes) > 0 {
start := time.Now()
toc, tocDgst, err := d.ParseTOC(bytes.NewReader(tocBytes))
@@ -1021,6 +1161,37 @@ func (cw *countWriter) Write(p []byte) (n int, err error) {
return
}
type countWriteFlusher struct {
io.WriteCloser
n int64
}
func (wc *countWriteFlusher) register(w io.WriteCloser) io.WriteCloser {
wc.WriteCloser = w
return wc
}
func (wc *countWriteFlusher) Write(p []byte) (n int, err error) {
n, err = wc.WriteCloser.Write(p)
wc.n += int64(n)
return
}
func (wc *countWriteFlusher) Flush() error {
if f, ok := wc.WriteCloser.(interface {
Flush() error
}); ok {
return f.Flush()
}
return nil
}
func (wc *countWriteFlusher) Close() error {
err := wc.WriteCloser.Close()
wc.WriteCloser = nil
return err
}
// isGzip reports whether br is positioned right before an upcoming gzip stream.
// It does not consume any bytes from br.
func isGzip(br *bufio.Reader) bool {
@@ -1039,3 +1210,14 @@ func positive(n int64) int64 {
}
return n
}
type countReader struct {
r io.Reader
n int64
}
func (cr *countReader) Read(p []byte) (n int, err error) {
n, err = cr.r.Read(p)
cr.n += int64(n)
return
}

2
vendor/github.com/containerd/stargz-snapshotter/estargz/gzip.go generated vendored
View File

@@ -60,7 +60,7 @@ type GzipCompressor struct {
compressionLevel int
}
func (gc *GzipCompressor) Writer(w io.Writer) (io.WriteCloser, error) {
func (gc *GzipCompressor) Writer(w io.Writer) (WriteFlushCloser, error) {
return gzip.NewWriterLevel(w, gc.compressionLevel)
}

627
vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go generated vendored
View File

@@ -31,6 +31,7 @@ import (
"errors"
"fmt"
"io"
"math/rand"
"os"
"path/filepath"
"reflect"
@@ -44,21 +45,27 @@ import (
digest "github.com/opencontainers/go-digest"
)
func init() {
rand.Seed(time.Now().UnixNano())
}
// TestingController is Compression with some helper methods necessary for testing.
type TestingController interface {
Compression
CountStreams(*testing.T, []byte) int
TestStreams(t *testing.T, b []byte, streams []int64)
DiffIDOf(*testing.T, []byte) string
String() string
}
// CompressionTestSuite tests this pkg with controllers can build valid eStargz blobs and parse them.
func CompressionTestSuite(t *testing.T, controllers ...TestingController) {
func CompressionTestSuite(t *testing.T, controllers ...TestingControllerFactory) {
t.Run("testBuild", func(t *testing.T) { t.Parallel(); testBuild(t, controllers...) })
t.Run("testDigestAndVerify", func(t *testing.T) { t.Parallel(); testDigestAndVerify(t, controllers...) })
t.Run("testWriteAndOpen", func(t *testing.T) { t.Parallel(); testWriteAndOpen(t, controllers...) })
}
type TestingControllerFactory func() TestingController
const (
uncompressedType int = iota
gzipType
@@ -75,11 +82,12 @@ var allowedPrefix = [4]string{"", "./", "/", "../"}
// testBuild tests the resulting stargz blob built by this pkg has the same
// contents as the normal stargz blob.
func testBuild(t *testing.T, controllers ...TestingController) {
func testBuild(t *testing.T, controllers ...TestingControllerFactory) {
tests := []struct {
name string
chunkSize int
in []tarEntry
name string
chunkSize int
minChunkSize []int
in []tarEntry
}{
{
name: "regfiles and directories",
@@ -108,11 +116,14 @@ func testBuild(t *testing.T, controllers ...TestingController) {
),
},
{
name: "various files",
chunkSize: 4,
name: "various files",
chunkSize: 4,
minChunkSize: []int{0, 64000},
in: tarOf(
file("baz.txt", "bazbazbazbazbazbazbaz"),
file("foo.txt", "a"),
file("foo1.txt", "a"),
file("bar/foo2.txt", "b"),
file("foo3.txt", "c"),
symlink("barlink", "test/bar.txt"),
dir("test/"),
dir("dev/"),
@@ -144,99 +155,112 @@ func testBuild(t *testing.T, controllers ...TestingController) {
},
}
for _, tt := range tests {
if len(tt.minChunkSize) == 0 {
tt.minChunkSize = []int{0}
}
for _, srcCompression := range srcCompressions {
srcCompression := srcCompression
for _, cl := range controllers {
cl := cl
for _, newCL := range controllers {
newCL := newCL
for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} {
srcTarFormat := srcTarFormat
for _, prefix := range allowedPrefix {
prefix := prefix
t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,src=%d,format=%s", cl, prefix, srcCompression, srcTarFormat), func(t *testing.T) {
tarBlob := buildTar(t, tt.in, prefix, srcTarFormat)
// Test divideEntries()
entries, err := sortEntries(tarBlob, nil, nil) // identical order
if err != nil {
t.Fatalf("failed to parse tar: %v", err)
}
var merged []*entry
for _, part := range divideEntries(entries, 4) {
merged = append(merged, part...)
}
if !reflect.DeepEqual(entries, merged) {
for _, e := range entries {
t.Logf("Original: %v", e.header)
for _, minChunkSize := range tt.minChunkSize {
minChunkSize := minChunkSize
t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,src=%d,format=%s,minChunkSize=%d", newCL(), prefix, srcCompression, srcTarFormat, minChunkSize), func(t *testing.T) {
tarBlob := buildTar(t, tt.in, prefix, srcTarFormat)
// Test divideEntries()
entries, err := sortEntries(tarBlob, nil, nil) // identical order
if err != nil {
t.Fatalf("failed to parse tar: %v", err)
}
for _, e := range merged {
t.Logf("Merged: %v", e.header)
var merged []*entry
for _, part := range divideEntries(entries, 4) {
merged = append(merged, part...)
}
if !reflect.DeepEqual(entries, merged) {
for _, e := range entries {
t.Logf("Original: %v", e.header)
}
for _, e := range merged {
t.Logf("Merged: %v", e.header)
}
t.Errorf("divided entries couldn't be merged")
return
}
t.Errorf("divided entries couldn't be merged")
return
}
// Prepare sample data
wantBuf := new(bytes.Buffer)
sw := NewWriterWithCompressor(wantBuf, cl)
sw.ChunkSize = tt.chunkSize
if err := sw.AppendTar(tarBlob); err != nil {
t.Fatalf("failed to append tar to want stargz: %v", err)
}
if _, err := sw.Close(); err != nil {
t.Fatalf("failed to prepare want stargz: %v", err)
}
wantData := wantBuf.Bytes()
want, err := Open(io.NewSectionReader(
bytes.NewReader(wantData), 0, int64(len(wantData))),
WithDecompressors(cl),
)
if err != nil {
t.Fatalf("failed to parse the want stargz: %v", err)
}
// Prepare sample data
cl1 := newCL()
wantBuf := new(bytes.Buffer)
sw := NewWriterWithCompressor(wantBuf, cl1)
sw.MinChunkSize = minChunkSize
sw.ChunkSize = tt.chunkSize
if err := sw.AppendTar(tarBlob); err != nil {
t.Fatalf("failed to append tar to want stargz: %v", err)
}
if _, err := sw.Close(); err != nil {
t.Fatalf("failed to prepare want stargz: %v", err)
}
wantData := wantBuf.Bytes()
want, err := Open(io.NewSectionReader(
bytes.NewReader(wantData), 0, int64(len(wantData))),
WithDecompressors(cl1),
)
if err != nil {
t.Fatalf("failed to parse the want stargz: %v", err)
}
// Prepare testing data
rc, err := Build(compressBlob(t, tarBlob, srcCompression),
WithChunkSize(tt.chunkSize), WithCompression(cl))
if err != nil {
t.Fatalf("failed to build stargz: %v", err)
}
defer rc.Close()
gotBuf := new(bytes.Buffer)
if _, err := io.Copy(gotBuf, rc); err != nil {
t.Fatalf("failed to copy built stargz blob: %v", err)
}
gotData := gotBuf.Bytes()
got, err := Open(io.NewSectionReader(
bytes.NewReader(gotBuf.Bytes()), 0, int64(len(gotData))),
WithDecompressors(cl),
)
if err != nil {
t.Fatalf("failed to parse the got stargz: %v", err)
}
// Prepare testing data
var opts []Option
if minChunkSize > 0 {
opts = append(opts, WithMinChunkSize(minChunkSize))
}
cl2 := newCL()
rc, err := Build(compressBlob(t, tarBlob, srcCompression),
append(opts, WithChunkSize(tt.chunkSize), WithCompression(cl2))...)
if err != nil {
t.Fatalf("failed to build stargz: %v", err)
}
defer rc.Close()
gotBuf := new(bytes.Buffer)
if _, err := io.Copy(gotBuf, rc); err != nil {
t.Fatalf("failed to copy built stargz blob: %v", err)
}
gotData := gotBuf.Bytes()
got, err := Open(io.NewSectionReader(
bytes.NewReader(gotBuf.Bytes()), 0, int64(len(gotData))),
WithDecompressors(cl2),
)
if err != nil {
t.Fatalf("failed to parse the got stargz: %v", err)
}
// Check DiffID is properly calculated
rc.Close()
diffID := rc.DiffID()
wantDiffID := cl.DiffIDOf(t, gotData)
if diffID.String() != wantDiffID {
t.Errorf("DiffID = %q; want %q", diffID, wantDiffID)
}
// Check DiffID is properly calculated
rc.Close()
diffID := rc.DiffID()
wantDiffID := cl2.DiffIDOf(t, gotData)
if diffID.String() != wantDiffID {
t.Errorf("DiffID = %q; want %q", diffID, wantDiffID)
}
// Compare as stargz
if !isSameVersion(t, cl, wantData, gotData) {
t.Errorf("built stargz hasn't same json")
return
}
if !isSameEntries(t, want, got) {
t.Errorf("built stargz isn't same as the original")
return
}
// Compare as stargz
if !isSameVersion(t, cl1, wantData, cl2, gotData) {
t.Errorf("built stargz hasn't same json")
return
}
if !isSameEntries(t, want, got) {
t.Errorf("built stargz isn't same as the original")
return
}
// Compare as tar.gz
if !isSameTarGz(t, cl, wantData, gotData) {
t.Errorf("built stargz isn't same tar.gz")
return
}
})
// Compare as tar.gz
if !isSameTarGz(t, cl1, wantData, cl2, gotData) {
t.Errorf("built stargz isn't same tar.gz")
return
}
})
}
}
}
}
@@ -244,13 +268,13 @@ func testBuild(t *testing.T, controllers ...TestingController) {
}
}
func isSameTarGz(t *testing.T, controller TestingController, a, b []byte) bool {
aGz, err := controller.Reader(bytes.NewReader(a))
func isSameTarGz(t *testing.T, cla TestingController, a []byte, clb TestingController, b []byte) bool {
aGz, err := cla.Reader(bytes.NewReader(a))
if err != nil {
t.Fatalf("failed to read A")
}
defer aGz.Close()
bGz, err := controller.Reader(bytes.NewReader(b))
bGz, err := clb.Reader(bytes.NewReader(b))
if err != nil {
t.Fatalf("failed to read B")
}
@@ -304,12 +328,12 @@ func isSameTarGz(t *testing.T, controller TestingController, a, b []byte) bool {
return true
}
func isSameVersion(t *testing.T, controller TestingController, a, b []byte) bool {
aJTOC, _, err := parseStargz(io.NewSectionReader(bytes.NewReader(a), 0, int64(len(a))), controller)
func isSameVersion(t *testing.T, cla TestingController, a []byte, clb TestingController, b []byte) bool {
aJTOC, _, err := parseStargz(io.NewSectionReader(bytes.NewReader(a), 0, int64(len(a))), cla)
if err != nil {
t.Fatalf("failed to parse A: %v", err)
}
bJTOC, _, err := parseStargz(io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b))), controller)
bJTOC, _, err := parseStargz(io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b))), clb)
if err != nil {
t.Fatalf("failed to parse B: %v", err)
}
@@ -463,7 +487,7 @@ func equalEntry(a, b *TOCEntry) bool {
a.GID == b.GID &&
a.Uname == b.Uname &&
a.Gname == b.Gname &&
(a.Offset > 0) == (b.Offset > 0) &&
(a.Offset >= 0) == (b.Offset >= 0) &&
(a.NextOffset() > 0) == (b.NextOffset() > 0) &&
a.DevMajor == b.DevMajor &&
a.DevMinor == b.DevMinor &&
@@ -510,14 +534,15 @@ func dumpTOCJSON(t *testing.T, tocJSON *JTOC) string {
const chunkSize = 3
// type check func(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, compressionLevel int)
type check func(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController)
type check func(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory)
// testDigestAndVerify runs specified checks against sample stargz blobs.
func testDigestAndVerify(t *testing.T, controllers ...TestingController) {
func testDigestAndVerify(t *testing.T, controllers ...TestingControllerFactory) {
tests := []struct {
name string
tarInit func(t *testing.T, dgstMap map[string]digest.Digest) (blob []tarEntry)
checks []check
name string
tarInit func(t *testing.T, dgstMap map[string]digest.Digest) (blob []tarEntry)
checks []check
minChunkSize []int
}{
{
name: "no-regfile",
@@ -544,6 +569,7 @@ func testDigestAndVerify(t *testing.T, controllers ...TestingController) {
regDigest(t, "test/bar.txt", "bbb", dgstMap),
)
},
minChunkSize: []int{0, 64000},
checks: []check{
checkStargzTOC,
checkVerifyTOC,
@@ -581,11 +607,14 @@ func testDigestAndVerify(t *testing.T, controllers ...TestingController) {
},
},
{
name: "with-non-regfiles",
name: "with-non-regfiles",
minChunkSize: []int{0, 64000},
tarInit: func(t *testing.T, dgstMap map[string]digest.Digest) (blob []tarEntry) {
return tarOf(
regDigest(t, "baz.txt", "bazbazbazbazbazbazbaz", dgstMap),
regDigest(t, "foo.txt", "a", dgstMap),
regDigest(t, "bar/foo2.txt", "b", dgstMap),
regDigest(t, "foo3.txt", "c", dgstMap),
symlink("barlink", "test/bar.txt"),
dir("test/"),
regDigest(t, "test/bar.txt", "testbartestbar", dgstMap),
@@ -599,6 +628,8 @@ func testDigestAndVerify(t *testing.T, controllers ...TestingController) {
checkVerifyInvalidStargzFail(buildTar(t, tarOf(
file("baz.txt", "bazbazbazbazbazbazbaz"),
file("foo.txt", "a"),
file("bar/foo2.txt", "b"),
file("foo3.txt", "c"),
symlink("barlink", "test/bar.txt"),
dir("test/"),
file("test/bar.txt", "testbartestbar"),
@@ -612,38 +643,45 @@ func testDigestAndVerify(t *testing.T, controllers ...TestingController) {
}
for _, tt := range tests {
if len(tt.minChunkSize) == 0 {
tt.minChunkSize = []int{0}
}
for _, srcCompression := range srcCompressions {
srcCompression := srcCompression
for _, cl := range controllers {
cl := cl
for _, newCL := range controllers {
newCL := newCL
for _, prefix := range allowedPrefix {
prefix := prefix
for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} {
srcTarFormat := srcTarFormat
t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,format=%s", cl, prefix, srcTarFormat), func(t *testing.T) {
// Get original tar file and chunk digests
dgstMap := make(map[string]digest.Digest)
tarBlob := buildTar(t, tt.tarInit(t, dgstMap), prefix, srcTarFormat)
for _, minChunkSize := range tt.minChunkSize {
minChunkSize := minChunkSize
t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,format=%s,minChunkSize=%d", newCL(), prefix, srcTarFormat, minChunkSize), func(t *testing.T) {
// Get original tar file and chunk digests
dgstMap := make(map[string]digest.Digest)
tarBlob := buildTar(t, tt.tarInit(t, dgstMap), prefix, srcTarFormat)
rc, err := Build(compressBlob(t, tarBlob, srcCompression),
WithChunkSize(chunkSize), WithCompression(cl))
if err != nil {
t.Fatalf("failed to convert stargz: %v", err)
}
tocDigest := rc.TOCDigest()
defer rc.Close()
buf := new(bytes.Buffer)
if _, err := io.Copy(buf, rc); err != nil {
t.Fatalf("failed to copy built stargz blob: %v", err)
}
newStargz := buf.Bytes()
// NoPrefetchLandmark is added during `Bulid`, which is expected behaviour.
dgstMap[chunkID(NoPrefetchLandmark, 0, int64(len([]byte{landmarkContents})))] = digest.FromBytes([]byte{landmarkContents})
cl := newCL()
rc, err := Build(compressBlob(t, tarBlob, srcCompression),
WithChunkSize(chunkSize), WithCompression(cl))
if err != nil {
t.Fatalf("failed to convert stargz: %v", err)
}
tocDigest := rc.TOCDigest()
defer rc.Close()
buf := new(bytes.Buffer)
if _, err := io.Copy(buf, rc); err != nil {
t.Fatalf("failed to copy built stargz blob: %v", err)
}
newStargz := buf.Bytes()
// NoPrefetchLandmark is added during `Bulid`, which is expected behaviour.
dgstMap[chunkID(NoPrefetchLandmark, 0, int64(len([]byte{landmarkContents})))] = digest.FromBytes([]byte{landmarkContents})
for _, check := range tt.checks {
check(t, newStargz, tocDigest, dgstMap, cl)
}
})
for _, check := range tt.checks {
check(t, newStargz, tocDigest, dgstMap, cl, newCL)
}
})
}
}
}
}
@@ -654,7 +692,7 @@ func testDigestAndVerify(t *testing.T, controllers ...TestingController) {
// checkStargzTOC checks the TOC JSON of the passed stargz has the expected
// digest and contains valid chunks. It walks all entries in the stargz and
// checks all chunk digests stored to the TOC JSON match the actual contents.
func checkStargzTOC(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController) {
func checkStargzTOC(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) {
sgz, err := Open(
io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData))),
WithDecompressors(controller),
@@ -765,7 +803,7 @@ func checkStargzTOC(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstM
// checkVerifyTOC checks the verification works for the TOC JSON of the passed
// stargz. It walks all entries in the stargz and checks the verifications for
// all chunks work.
func checkVerifyTOC(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController) {
func checkVerifyTOC(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) {
sgz, err := Open(
io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData))),
WithDecompressors(controller),
@@ -846,7 +884,7 @@ func checkVerifyTOC(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstM
// checkVerifyInvalidTOCEntryFail checks if misconfigured TOC JSON can be
// detected during the verification and the verification returns an error.
func checkVerifyInvalidTOCEntryFail(filename string) check {
return func(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController) {
return func(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) {
funcs := map[string]rewriteFunc{
"lost digest in a entry": func(t *testing.T, toc *JTOC, sgz *io.SectionReader) {
var found bool
@@ -920,8 +958,9 @@ func checkVerifyInvalidTOCEntryFail(filename string) check {
// checkVerifyInvalidStargzFail checks if the verification detects that the
// given stargz file doesn't match to the expected digest and returns error.
func checkVerifyInvalidStargzFail(invalid *io.SectionReader) check {
return func(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController) {
rc, err := Build(invalid, WithChunkSize(chunkSize), WithCompression(controller))
return func(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) {
cl := newController()
rc, err := Build(invalid, WithChunkSize(chunkSize), WithCompression(cl))
if err != nil {
t.Fatalf("failed to convert stargz: %v", err)
}
@@ -934,7 +973,7 @@ func checkVerifyInvalidStargzFail(invalid *io.SectionReader) check {
sgz, err := Open(
io.NewSectionReader(bytes.NewReader(mStargz), 0, int64(len(mStargz))),
WithDecompressors(controller),
WithDecompressors(cl),
)
if err != nil {
t.Fatalf("failed to parse converted stargz: %v", err)
@@ -951,7 +990,7 @@ func checkVerifyInvalidStargzFail(invalid *io.SectionReader) check {
// checkVerifyBrokenContentFail checks if the verifier detects broken contents
// that doesn't match to the expected digest and returns error.
func checkVerifyBrokenContentFail(filename string) check {
return func(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController) {
return func(t *testing.T, sgzData []byte, tocDigest digest.Digest, dgstMap map[string]digest.Digest, controller TestingController, newController TestingControllerFactory) {
// Parse stargz file
sgz, err := Open(
io.NewSectionReader(bytes.NewReader(sgzData), 0, int64(len(sgzData))),
@@ -1070,7 +1109,10 @@ func parseStargz(sgz *io.SectionReader, controller TestingController) (decodedJT
}
// Decode the TOC JSON
tocReader := io.NewSectionReader(sgz, tocOffset, sgz.Size()-tocOffset-fSize)
var tocReader io.Reader
if tocOffset >= 0 {
tocReader = io.NewSectionReader(sgz, tocOffset, sgz.Size()-tocOffset-fSize)
}
decodedJTOC, _, err = controller.ParseTOC(tocReader)
if err != nil {
return nil, 0, fmt.Errorf("failed to parse TOC: %w", err)
@@ -1078,28 +1120,31 @@ func parseStargz(sgz *io.SectionReader, controller TestingController) (decodedJT
return decodedJTOC, tocOffset, nil
}
func testWriteAndOpen(t *testing.T, controllers ...TestingController) {
func testWriteAndOpen(t *testing.T, controllers ...TestingControllerFactory) {
const content = "Some contents"
invalidUtf8 := "\xff\xfe\xfd"
xAttrFile := xAttr{"foo": "bar", "invalid-utf8": invalidUtf8}
sampleOwner := owner{uid: 50, gid: 100}
data64KB := randomContents(64000)
tests := []struct {
name string
chunkSize int
in []tarEntry
want []stargzCheck
wantNumGz int // expected number of streams
name string
chunkSize int
minChunkSize int
in []tarEntry
want []stargzCheck
wantNumGz int // expected number of streams
wantNumGzLossLess int // expected number of streams (> 0) in lossless mode if it's different from wantNumGz
wantFailOnLossLess bool
wantTOCVersion int // default = 1
}{
{
name: "empty",
in: tarOf(),
wantNumGz: 2, // empty tar + TOC + footer
wantNumGzLossLess: 3, // empty tar + TOC + footer
name: "empty",
in: tarOf(),
wantNumGz: 2, // (empty tar) + TOC + footer
want: checks(
numTOCEntries(0),
),
@@ -1195,7 +1240,7 @@ func testWriteAndOpen(t *testing.T, controllers ...TestingController) {
dir("foo/"),
file("foo/big.txt", "This "+"is s"+"uch "+"a bi"+"g fi"+"le"),
),
wantNumGz: 9,
wantNumGz: 9, // dir + big.txt(6 chunks) + TOC + footer
want: checks(
numTOCEntries(7), // 1 for foo dir, 6 for the foo/big.txt file
hasDir("foo/"),
@@ -1326,23 +1371,108 @@ func testWriteAndOpen(t *testing.T, controllers ...TestingController) {
mustSameEntry("foo/foo1", "foolink"),
),
},
{
name: "several_files_in_chunk",
minChunkSize: 8000,
in: tarOf(
dir("foo/"),
file("foo/foo1", data64KB),
file("foo2", "bb"),
file("foo22", "ccc"),
dir("bar/"),
file("bar/bar.txt", "aaa"),
file("foo3", data64KB),
),
// NOTE: we assume that the compressed "data64KB" is still larger than 8KB
wantNumGz: 4, // dir+foo1, foo2+foo22+dir+bar.txt+foo3, TOC, footer
want: checks(
numTOCEntries(7), // dir, foo1, foo2, foo22, dir, bar.txt, foo3
hasDir("foo/"),
hasDir("bar/"),
hasFileLen("foo/foo1", len(data64KB)),
hasFileLen("foo2", len("bb")),
hasFileLen("foo22", len("ccc")),
hasFileLen("bar/bar.txt", len("aaa")),
hasFileLen("foo3", len(data64KB)),
hasFileDigest("foo/foo1", digestFor(data64KB)),
hasFileDigest("foo2", digestFor("bb")),
hasFileDigest("foo22", digestFor("ccc")),
hasFileDigest("bar/bar.txt", digestFor("aaa")),
hasFileDigest("foo3", digestFor(data64KB)),
hasFileContentsWithPreRead("foo22", 0, "ccc", chunkInfo{"foo2", "bb"}, chunkInfo{"bar/bar.txt", "aaa"}, chunkInfo{"foo3", data64KB}),
hasFileContentsRange("foo/foo1", 0, data64KB),
hasFileContentsRange("foo2", 0, "bb"),
hasFileContentsRange("foo2", 1, "b"),
hasFileContentsRange("foo22", 0, "ccc"),
hasFileContentsRange("foo22", 1, "cc"),
hasFileContentsRange("foo22", 2, "c"),
hasFileContentsRange("bar/bar.txt", 0, "aaa"),
hasFileContentsRange("bar/bar.txt", 1, "aa"),
hasFileContentsRange("bar/bar.txt", 2, "a"),
hasFileContentsRange("foo3", 0, data64KB),
hasFileContentsRange("foo3", 1, data64KB[1:]),
hasFileContentsRange("foo3", 2, data64KB[2:]),
hasFileContentsRange("foo3", len(data64KB)/2, data64KB[len(data64KB)/2:]),
hasFileContentsRange("foo3", len(data64KB)-1, data64KB[len(data64KB)-1:]),
),
},
{
name: "several_files_in_chunk_chunked",
minChunkSize: 8000,
chunkSize: 32000,
in: tarOf(
dir("foo/"),
file("foo/foo1", data64KB),
file("foo2", "bb"),
dir("bar/"),
file("foo3", data64KB),
),
// NOTE: we assume that the compressed chunk of "data64KB" is still larger than 8KB
wantNumGz: 6, // dir+foo1(1), foo1(2), foo2+dir+foo3(1), foo3(2), TOC, footer
want: checks(
numTOCEntries(7), // dir, foo1(2 chunks), foo2, dir, foo3(2 chunks)
hasDir("foo/"),
hasDir("bar/"),
hasFileLen("foo/foo1", len(data64KB)),
hasFileLen("foo2", len("bb")),
hasFileLen("foo3", len(data64KB)),
hasFileDigest("foo/foo1", digestFor(data64KB)),
hasFileDigest("foo2", digestFor("bb")),
hasFileDigest("foo3", digestFor(data64KB)),
hasFileContentsWithPreRead("foo2", 0, "bb", chunkInfo{"foo3", data64KB[:32000]}),
hasFileContentsRange("foo/foo1", 0, data64KB),
hasFileContentsRange("foo/foo1", 1, data64KB[1:]),
hasFileContentsRange("foo/foo1", 2, data64KB[2:]),
hasFileContentsRange("foo/foo1", len(data64KB)/2, data64KB[len(data64KB)/2:]),
hasFileContentsRange("foo/foo1", len(data64KB)-1, data64KB[len(data64KB)-1:]),
hasFileContentsRange("foo2", 0, "bb"),
hasFileContentsRange("foo2", 1, "b"),
hasFileContentsRange("foo3", 0, data64KB),
hasFileContentsRange("foo3", 1, data64KB[1:]),
hasFileContentsRange("foo3", 2, data64KB[2:]),
hasFileContentsRange("foo3", len(data64KB)/2, data64KB[len(data64KB)/2:]),
hasFileContentsRange("foo3", len(data64KB)-1, data64KB[len(data64KB)-1:]),
),
},
}
for _, tt := range tests {
for _, cl := range controllers {
cl := cl
for _, newCL := range controllers {
newCL := newCL
for _, prefix := range allowedPrefix {
prefix := prefix
for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} {
srcTarFormat := srcTarFormat
for _, lossless := range []bool{true, false} {
t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,lossless=%v,format=%s", cl, prefix, lossless, srcTarFormat), func(t *testing.T) {
t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,lossless=%v,format=%s", newCL(), prefix, lossless, srcTarFormat), func(t *testing.T) {
var tr io.Reader = buildTar(t, tt.in, prefix, srcTarFormat)
origTarDgstr := digest.Canonical.Digester()
tr = io.TeeReader(tr, origTarDgstr.Hash())
var stargzBuf bytes.Buffer
w := NewWriterWithCompressor(&stargzBuf, cl)
cl1 := newCL()
w := NewWriterWithCompressor(&stargzBuf, cl1)
w.ChunkSize = tt.chunkSize
w.MinChunkSize = tt.minChunkSize
if lossless {
err := w.AppendTarLossLess(tr)
if tt.wantFailOnLossLess {
@@ -1366,7 +1496,7 @@ func testWriteAndOpen(t *testing.T, controllers ...TestingController) {
if lossless {
// Check if the result blob reserves original tar metadata
rc, err := Unpack(io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b))), cl)
rc, err := Unpack(io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b))), cl1)
if err != nil {
t.Errorf("failed to decompress blob: %v", err)
return
@@ -1385,32 +1515,71 @@ func testWriteAndOpen(t *testing.T, controllers ...TestingController) {
}
diffID := w.DiffID()
wantDiffID := cl.DiffIDOf(t, b)
wantDiffID := cl1.DiffIDOf(t, b)
if diffID != wantDiffID {
t.Errorf("DiffID = %q; want %q", diffID, wantDiffID)
}
got := cl.CountStreams(t, b)
wantNumGz := tt.wantNumGz
if lossless && tt.wantNumGzLossLess > 0 {
wantNumGz = tt.wantNumGzLossLess
}
if got != wantNumGz {
t.Errorf("number of streams = %d; want %d", got, wantNumGz)
}
telemetry, checkCalled := newCalledTelemetry()
sr := io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b)))
r, err := Open(
io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b))),
WithDecompressors(cl),
sr,
WithDecompressors(cl1),
WithTelemetry(telemetry),
)
if err != nil {
t.Fatalf("stargz.Open: %v", err)
}
if err := checkCalled(); err != nil {
wantTOCVersion := 1
if tt.wantTOCVersion > 0 {
wantTOCVersion = tt.wantTOCVersion
}
if r.toc.Version != wantTOCVersion {
t.Fatalf("invalid TOC Version %d; wanted %d", r.toc.Version, wantTOCVersion)
}
footerSize := cl1.FooterSize()
footerOffset := sr.Size() - footerSize
footer := make([]byte, footerSize)
if _, err := sr.ReadAt(footer, footerOffset); err != nil {
t.Errorf("failed to read footer: %v", err)
}
_, tocOffset, _, err := cl1.ParseFooter(footer)
if err != nil {
t.Errorf("failed to parse footer: %v", err)
}
if err := checkCalled(tocOffset >= 0); err != nil {
t.Errorf("telemetry failure: %v", err)
}
wantNumGz := tt.wantNumGz
if lossless && tt.wantNumGzLossLess > 0 {
wantNumGz = tt.wantNumGzLossLess
}
streamOffsets := []int64{0}
prevOffset := int64(-1)
streams := 0
for _, e := range r.toc.Entries {
if e.Offset > prevOffset {
streamOffsets = append(streamOffsets, e.Offset)
prevOffset = e.Offset
streams++
}
}
streams++ // TOC
if tocOffset >= 0 {
// toc is in the blob
streamOffsets = append(streamOffsets, tocOffset)
}
streams++ // footer
streamOffsets = append(streamOffsets, footerOffset)
if streams != wantNumGz {
t.Errorf("number of streams in TOC = %d; want %d", streams, wantNumGz)
}
t.Logf("testing streams: %+v", streamOffsets)
cl1.TestStreams(t, b, streamOffsets)
for _, want := range tt.want {
want.check(t, r)
}
@@ -1422,7 +1591,12 @@ func testWriteAndOpen(t *testing.T, controllers ...TestingController) {
}
}
func newCalledTelemetry() (telemetry *Telemetry, check func() error) {
type chunkInfo struct {
name string
data string
}
func newCalledTelemetry() (telemetry *Telemetry, check func(needsGetTOC bool) error) {
var getFooterLatencyCalled bool
var getTocLatencyCalled bool
var deserializeTocLatencyCalled bool
@@ -1430,13 +1604,15 @@ func newCalledTelemetry() (telemetry *Telemetry, check func() error) {
func(time.Time) { getFooterLatencyCalled = true },
func(time.Time) { getTocLatencyCalled = true },
func(time.Time) { deserializeTocLatencyCalled = true },
}, func() error {
}, func(needsGetTOC bool) error {
var allErr []error
if !getFooterLatencyCalled {
allErr = append(allErr, fmt.Errorf("metrics GetFooterLatency isn't called"))
}
if !getTocLatencyCalled {
allErr = append(allErr, fmt.Errorf("metrics GetTocLatency isn't called"))
if needsGetTOC {
if !getTocLatencyCalled {
allErr = append(allErr, fmt.Errorf("metrics GetTocLatency isn't called"))
}
}
if !deserializeTocLatencyCalled {
allErr = append(allErr, fmt.Errorf("metrics DeserializeTocLatency isn't called"))
@@ -1573,6 +1749,53 @@ func hasFileDigest(file string, digest string) stargzCheck {
})
}
func hasFileContentsWithPreRead(file string, offset int, want string, extra ...chunkInfo) stargzCheck {
return stargzCheckFn(func(t *testing.T, r *Reader) {
extraMap := make(map[string]chunkInfo)
for _, e := range extra {
extraMap[e.name] = e
}
var extraNames []string
for n := range extraMap {
extraNames = append(extraNames, n)
}
f, err := r.OpenFileWithPreReader(file, func(e *TOCEntry, cr io.Reader) error {
t.Logf("On %q: got preread of %q", file, e.Name)
ex, ok := extraMap[e.Name]
if !ok {
t.Fatalf("fail on %q: unexpected entry %q: %+v, %+v", file, e.Name, e, extraNames)
}
got, err := io.ReadAll(cr)
if err != nil {
t.Fatalf("fail on %q: failed to read %q: %v", file, e.Name, err)
}
if ex.data != string(got) {
t.Fatalf("fail on %q: unexpected contents of %q: len=%d; want=%d", file, e.Name, len(got), len(ex.data))
}
delete(extraMap, e.Name)
return nil
})
if err != nil {
t.Fatal(err)
}
got := make([]byte, len(want))
n, err := f.ReadAt(got, int64(offset))
if err != nil {
t.Fatalf("ReadAt(len %d, offset %d, size %d) = %v, %v", len(got), offset, f.Size(), n, err)
}
if string(got) != want {
t.Fatalf("ReadAt(len %d, offset %d) = %q, want %q", len(got), offset, viewContent(got), viewContent([]byte(want)))
}
if len(extraMap) != 0 {
var exNames []string
for _, ex := range extraMap {
exNames = append(exNames, ex.name)
}
t.Fatalf("fail on %q: some entries aren't read: %+v", file, exNames)
}
})
}
func hasFileContentsRange(file string, offset int, want string) stargzCheck {
return stargzCheckFn(func(t *testing.T, r *Reader) {
f, err := r.OpenFile(file)
@@ -1585,7 +1808,7 @@ func hasFileContentsRange(file string, offset int, want string) stargzCheck {
t.Fatalf("ReadAt(len %d, offset %d) = %v, %v", len(got), offset, n, err)
}
if string(got) != want {
t.Fatalf("ReadAt(len %d, offset %d) = %q, want %q", len(got), offset, got, want)
t.Fatalf("ReadAt(len %d, offset %d) = %q, want %q", len(got), offset, viewContent(got), viewContent([]byte(want)))
}
})
}
@@ -1797,6 +2020,13 @@ func mustSameEntry(files ...string) stargzCheck {
})
}
func viewContent(c []byte) string {
if len(c) < 100 {
return string(c)
}
return string(c[:50]) + "...(omit)..." + string(c[50:100])
}
func tarOf(s ...tarEntry) []tarEntry { return s }
type tarEntry interface {
@@ -2056,6 +2286,16 @@ func regDigest(t *testing.T, name string, contentStr string, digestMap map[strin
})
}
var runes = []rune("1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
func randomContents(n int) string {
b := make([]rune, n)
for i := range b {
b[i] = runes[rand.Intn(len(runes))]
}
return string(b)
}
func fileModeToTarMode(mode os.FileMode) (int64, error) {
h, err := tar.FileInfoHeader(fileInfoOnlyMode(mode), "")
if err != nil {
@@ -2073,3 +2313,54 @@ func (f fileInfoOnlyMode) Mode() os.FileMode { return os.FileMode(f) }
func (f fileInfoOnlyMode) ModTime() time.Time { return time.Now() }
func (f fileInfoOnlyMode) IsDir() bool { return os.FileMode(f).IsDir() }
func (f fileInfoOnlyMode) Sys() interface{} { return nil }
func CheckGzipHasStreams(t *testing.T, b []byte, streams []int64) {
if len(streams) == 0 {
return // nop
}
wants := map[int64]struct{}{}
for _, s := range streams {
wants[s] = struct{}{}
}
len0 := len(b)
br := bytes.NewReader(b)
zr := new(gzip.Reader)
t.Logf("got gzip streams:")
numStreams := 0
for {
zoff := len0 - br.Len()
if err := zr.Reset(br); err != nil {
if err == io.EOF {
return
}
t.Fatalf("countStreams(gzip), Reset: %v", err)
}
zr.Multistream(false)
n, err := io.Copy(io.Discard, zr)
if err != nil {
t.Fatalf("countStreams(gzip), Copy: %v", err)
}
var extra string
if len(zr.Header.Extra) > 0 {
extra = fmt.Sprintf("; extra=%q", zr.Header.Extra)
}
t.Logf(" [%d] at %d in stargz, uncompressed length %d%s", numStreams, zoff, n, extra)
delete(wants, int64(zoff))
numStreams++
}
}
func GzipDiffIDOf(t *testing.T, b []byte) string {
h := sha256.New()
zr, err := gzip.NewReader(bytes.NewReader(b))
if err != nil {
t.Fatalf("diffIDOf(gzip): %v", err)
}
defer zr.Close()
if _, err := io.Copy(h, zr); err != nil {
t.Fatalf("diffIDOf(gzip).Copy: %v", err)
}
return fmt.Sprintf("sha256:%x", h.Sum(nil))
}

31
vendor/github.com/containerd/stargz-snapshotter/estargz/types.go generated vendored
View File

@@ -149,6 +149,12 @@ type TOCEntry struct {
// ChunkSize.
Offset int64 `json:"offset,omitempty"`
// InnerOffset is an optional field indicates uncompressed offset
// of this "reg" or "chunk" payload in a stream starts from Offset.
// This field enables to put multiple "reg" or "chunk" payloads
// in one chunk with having the same Offset but different InnerOffset.
InnerOffset int64 `json:"innerOffset,omitempty"`
nextOffset int64 // the Offset of the next entry with a non-zero Offset
// DevMajor is the major device number for "char" and "block" types.
@@ -186,6 +192,9 @@ type TOCEntry struct {
ChunkDigest string `json:"chunkDigest,omitempty"`
children map[string]*TOCEntry
// chunkTopIndex is index of the entry where Offset starts in the blob.
chunkTopIndex int
}
// ModTime returns the entry's modification time.
@@ -279,7 +288,10 @@ type Compressor interface {
// Writer returns WriteCloser to be used for writing a chunk to eStargz.
// Everytime a chunk is written, the WriteCloser is closed and Writer is
// called again for writing the next chunk.
Writer(w io.Writer) (io.WriteCloser, error)
//
// The returned writer should implement "Flush() error" function that flushes
// any pending compressed data to the underlying writer.
Writer(w io.Writer) (WriteFlushCloser, error)
// WriteTOCAndFooter is called to write JTOC to the passed Writer.
// diffHash calculates the DiffID (uncompressed sha256 hash) of the blob
@@ -303,8 +315,12 @@ type Decompressor interface {
// payloadBlobSize is the (compressed) size of the blob payload (i.e. the size between
// the top until the TOC JSON).
//
// Here, tocSize is optional. If tocSize <= 0, it's by default the size of the range
// from tocOffset until the beginning of the footer (blob size - tocOff - FooterSize).
// If tocOffset < 0, we assume that TOC isn't contained in the blob and pass nil reader
// to ParseTOC. We expect that ParseTOC acquire TOC from the external location and return it.
//
// tocSize is optional. If tocSize <= 0, it's by default the size of the range from tocOffset until the beginning of the
// footer (blob size - tocOff - FooterSize).
// If blobPayloadSize < 0, blobPayloadSize become the blob size.
ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error)
// ParseTOC parses TOC from the passed reader. The reader provides the partial contents
@@ -313,5 +329,14 @@ type Decompressor interface {
// This function returns tocDgst that represents the digest of TOC that will be used
// to verify this blob. This must match to the value returned from
// Compressor.WriteTOCAndFooter that is used when creating this blob.
//
// If tocOffset returned by ParseFooter is < 0, we assume that TOC isn't contained in the blob.
// Pass nil reader to ParseTOC then we expect that ParseTOC acquire TOC from the external location
// and return it.
ParseTOC(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error)
}
type WriteFlushCloser interface {
io.WriteCloser
Flush() error
}