Merge pull request #2140 from containers/renovate/go-github.com/docker/docker-vulnerability

chore(deps): update module github.com/docker/docker to v24.0.7+incompatible [security]
This commit is contained in:
Miloslav Trmač 2023-10-30 19:40:31 +01:00 committed by GitHub
commit aec071dd9f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 1155 additions and 229 deletions

4
go.mod
View File

@ -38,7 +38,7 @@ require (
github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect
github.com/distribution/reference v0.5.0 // indirect github.com/distribution/reference v0.5.0 // indirect
github.com/docker/docker v24.0.6+incompatible // indirect github.com/docker/docker v24.0.7+incompatible // indirect
github.com/docker/docker-credential-helpers v0.8.0 // indirect github.com/docker/docker-credential-helpers v0.8.0 // indirect
github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect
@ -70,7 +70,7 @@ require (
github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/josharian/intern v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.16.7 // indirect github.com/klauspost/compress v1.17.2 // indirect
github.com/klauspost/pgzip v1.2.6 // indirect github.com/klauspost/pgzip v1.2.6 // indirect
github.com/letsencrypt/boulder v0.0.0-20230213213521-fdfea0d469b6 // indirect github.com/letsencrypt/boulder v0.0.0-20230213213521-fdfea0d469b6 // indirect
github.com/mailru/easyjson v0.7.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect

8
go.sum
View File

@ -55,8 +55,8 @@ github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK
github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker v24.0.6+incompatible h1:hceabKCtUgDqPu+qm0NgsaXf28Ljf4/pWFL7xjWWDgE= github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
github.com/docker/docker v24.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker-credential-helpers v0.8.0 h1:YQFtbBQb4VrpoPxhFuzEBPQ9E16qz5SpHLS+uswaCp8= github.com/docker/docker-credential-helpers v0.8.0 h1:YQFtbBQb4VrpoPxhFuzEBPQ9E16qz5SpHLS+uswaCp8=
github.com/docker/docker-credential-helpers v0.8.0/go.mod h1:UGFXcuoQ5TxPiB54nHOZ32AWRqQdECoh/Mg0AlEYb40= github.com/docker/docker-credential-helpers v0.8.0/go.mod h1:UGFXcuoQ5TxPiB54nHOZ32AWRqQdECoh/Mg0AlEYb40=
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
@ -217,8 +217,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=

View File

@ -16,11 +16,11 @@ func compare(v1, v2 string) int {
otherTab = strings.Split(v2, ".") otherTab = strings.Split(v2, ".")
) )
max := len(currTab) maxVer := len(currTab)
if len(otherTab) > max { if len(otherTab) > maxVer {
max = len(otherTab) maxVer = len(otherTab)
} }
for i := 0; i < max; i++ { for i := 0; i < maxVer; i++ {
var currInt, otherInt int var currInt, otherInt int
if len(currTab) > i { if len(currTab) > i {

View File

@ -3,7 +3,7 @@
before: before:
hooks: hooks:
- ./gen.sh - ./gen.sh
- go install mvdan.cc/garble@v0.9.3 - go install mvdan.cc/garble@v0.10.1
builds: builds:
- -
@ -92,16 +92,7 @@ builds:
archives: archives:
- -
id: s2-binaries id: s2-binaries
name_template: "s2-{{ .Os }}_{{ .Arch }}_{{ .Version }}" name_template: "s2-{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
replacements:
aix: AIX
darwin: OSX
linux: Linux
windows: Windows
386: i386
amd64: x86_64
freebsd: FreeBSD
netbsd: NetBSD
format_overrides: format_overrides:
- goos: windows - goos: windows
format: zip format: zip
@ -125,7 +116,7 @@ changelog:
nfpms: nfpms:
- -
file_name_template: "s2_package_{{ .Version }}_{{ .Os }}_{{ .Arch }}" file_name_template: "s2_package__{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
vendor: Klaus Post vendor: Klaus Post
homepage: https://github.com/klauspost/compress homepage: https://github.com/klauspost/compress
maintainer: Klaus Post <klauspost@gmail.com> maintainer: Klaus Post <klauspost@gmail.com>
@ -134,8 +125,3 @@ nfpms:
formats: formats:
- deb - deb
- rpm - rpm
replacements:
darwin: Darwin
linux: Linux
freebsd: FreeBSD
amd64: x86_64

View File

@ -16,6 +16,18 @@ This package provides various compression algorithms.
# changelog # changelog
* Sept 19th, 2023 - [v1.17.0](https://github.com/klauspost/compress/releases/tag/v1.17.0)
* Add experimental dictionary builder https://github.com/klauspost/compress/pull/853
* Add xerial snappy read/writer https://github.com/klauspost/compress/pull/838
* flate: Add limited window compression https://github.com/klauspost/compress/pull/843
* s2: Do 2 overlapping match checks https://github.com/klauspost/compress/pull/839
* flate: Add amd64 assembly matchlen https://github.com/klauspost/compress/pull/837
* gzip: Copy bufio.Reader on Reset by @thatguystone in https://github.com/klauspost/compress/pull/860
* July 1st, 2023 - [v1.16.7](https://github.com/klauspost/compress/releases/tag/v1.16.7)
* zstd: Fix default level first dictionary encode https://github.com/klauspost/compress/pull/829
* s2: add GetBufferCapacity() method by @GiedriusS in https://github.com/klauspost/compress/pull/832
* June 13, 2023 - [v1.16.6](https://github.com/klauspost/compress/releases/tag/v1.16.6) * June 13, 2023 - [v1.16.6](https://github.com/klauspost/compress/releases/tag/v1.16.6)
* zstd: correctly ignore WithEncoderPadding(1) by @ianlancetaylor in https://github.com/klauspost/compress/pull/806 * zstd: correctly ignore WithEncoderPadding(1) by @ianlancetaylor in https://github.com/klauspost/compress/pull/806
* zstd: Add amd64 match length assembly https://github.com/klauspost/compress/pull/824 * zstd: Add amd64 match length assembly https://github.com/klauspost/compress/pull/824
@ -50,6 +62,9 @@ This package provides various compression algorithms.
* s2: Support io.ReaderAt in ReadSeeker. https://github.com/klauspost/compress/pull/747 * s2: Support io.ReaderAt in ReadSeeker. https://github.com/klauspost/compress/pull/747
* s2c/s2sx: Use concurrent decoding. https://github.com/klauspost/compress/pull/746 * s2c/s2sx: Use concurrent decoding. https://github.com/klauspost/compress/pull/746
<details>
<summary>See changes to v1.15.x</summary>
* Jan 21st, 2023 (v1.15.15) * Jan 21st, 2023 (v1.15.15)
* deflate: Improve level 7-9 by @klauspost in https://github.com/klauspost/compress/pull/739 * deflate: Improve level 7-9 by @klauspost in https://github.com/klauspost/compress/pull/739
* zstd: Add delta encoding support by @greatroar in https://github.com/klauspost/compress/pull/728 * zstd: Add delta encoding support by @greatroar in https://github.com/klauspost/compress/pull/728
@ -176,6 +191,8 @@ Stream decompression is now faster on asynchronous, since the goroutine allocati
While the release has been extensively tested, it is recommended to testing when upgrading. While the release has been extensively tested, it is recommended to testing when upgrading.
</details>
<details> <details>
<summary>See changes to v1.14.x</summary> <summary>See changes to v1.14.x</summary>
@ -636,6 +653,8 @@ Here are other packages of good quality and pure Go (no cgo wrappers or autoconv
* [github.com/dsnet/compress](https://github.com/dsnet/compress) - brotli decompression, bzip2 writer. * [github.com/dsnet/compress](https://github.com/dsnet/compress) - brotli decompression, bzip2 writer.
* [github.com/ronanh/intcomp](https://github.com/ronanh/intcomp) - Integer compression. * [github.com/ronanh/intcomp](https://github.com/ronanh/intcomp) - Integer compression.
* [github.com/spenczar/fpc](https://github.com/spenczar/fpc) - Float compression. * [github.com/spenczar/fpc](https://github.com/spenczar/fpc) - Float compression.
* [github.com/minio/zipindex](https://github.com/minio/zipindex) - External ZIP directory index.
* [github.com/ybirader/pzip](https://github.com/ybirader/pzip) - Fast concurrent zip archiver and extractor.
# license # license

View File

@ -7,6 +7,7 @@ package flate
import ( import (
"encoding/binary" "encoding/binary"
"errors"
"fmt" "fmt"
"io" "io"
"math" "math"
@ -833,6 +834,12 @@ func (d *compressor) init(w io.Writer, level int) (err error) {
d.initDeflate() d.initDeflate()
d.fill = (*compressor).fillDeflate d.fill = (*compressor).fillDeflate
d.step = (*compressor).deflateLazy d.step = (*compressor).deflateLazy
case -level >= MinCustomWindowSize && -level <= MaxCustomWindowSize:
d.w.logNewTablePenalty = 7
d.fast = &fastEncL5Window{maxOffset: int32(-level), cur: maxStoreBlockSize}
d.window = make([]byte, maxStoreBlockSize)
d.fill = (*compressor).fillBlock
d.step = (*compressor).storeFast
default: default:
return fmt.Errorf("flate: invalid compression level %d: want value in range [-2, 9]", level) return fmt.Errorf("flate: invalid compression level %d: want value in range [-2, 9]", level)
} }
@ -929,6 +936,28 @@ func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) {
return zw, err return zw, err
} }
// MinCustomWindowSize is the minimum window size that can be sent to NewWriterWindow.
const MinCustomWindowSize = 32
// MaxCustomWindowSize is the maximum custom window that can be sent to NewWriterWindow.
const MaxCustomWindowSize = windowSize
// NewWriterWindow returns a new Writer compressing data with a custom window size.
// windowSize must be from MinCustomWindowSize to MaxCustomWindowSize.
func NewWriterWindow(w io.Writer, windowSize int) (*Writer, error) {
if windowSize < MinCustomWindowSize {
return nil, errors.New("flate: requested window size less than MinWindowSize")
}
if windowSize > MaxCustomWindowSize {
return nil, errors.New("flate: requested window size bigger than MaxCustomWindowSize")
}
var dw Writer
if err := dw.d.init(w, -windowSize); err != nil {
return nil, err
}
return &dw, nil
}
// A Writer takes data written to it and writes the compressed // A Writer takes data written to it and writes the compressed
// form of that data to an underlying writer (see NewWriter). // form of that data to an underlying writer (see NewWriter).
type Writer struct { type Writer struct {

View File

@ -8,7 +8,6 @@ package flate
import ( import (
"encoding/binary" "encoding/binary"
"fmt" "fmt"
"math/bits"
) )
type fastEnc interface { type fastEnc interface {
@ -192,25 +191,3 @@ func (e *fastGen) Reset() {
} }
e.hist = e.hist[:0] e.hist = e.hist[:0]
} }
// matchLen returns the maximum length.
// 'a' must be the shortest of the two.
func matchLen(a, b []byte) int {
var checked int
for len(a) >= 8 {
if diff := binary.LittleEndian.Uint64(a) ^ binary.LittleEndian.Uint64(b); diff != 0 {
return checked + (bits.TrailingZeros64(diff) >> 3)
}
checked += 8
a = a[8:]
b = b[8:]
}
b = b[:len(a)]
for i := range a {
if a[i] != b[i] {
return i + checked
}
}
return len(a) + checked
}

View File

@ -120,8 +120,9 @@ func (h *huffmanDecoder) init(lengths []int) bool {
const sanity = false const sanity = false
if h.chunks == nil { if h.chunks == nil {
h.chunks = &[huffmanNumChunks]uint16{} h.chunks = new([huffmanNumChunks]uint16)
} }
if h.maxRead != 0 { if h.maxRead != 0 {
*h = huffmanDecoder{chunks: h.chunks, links: h.links} *h = huffmanDecoder{chunks: h.chunks, links: h.links}
} }
@ -175,6 +176,7 @@ func (h *huffmanDecoder) init(lengths []int) bool {
} }
h.maxRead = min h.maxRead = min
chunks := h.chunks[:] chunks := h.chunks[:]
for i := range chunks { for i := range chunks {
chunks[i] = 0 chunks[i] = 0
@ -202,8 +204,7 @@ func (h *huffmanDecoder) init(lengths []int) bool {
if cap(h.links[off]) < numLinks { if cap(h.links[off]) < numLinks {
h.links[off] = make([]uint16, numLinks) h.links[off] = make([]uint16, numLinks)
} else { } else {
links := h.links[off][:0] h.links[off] = h.links[off][:numLinks]
h.links[off] = links[:numLinks]
} }
} }
} else { } else {
@ -277,7 +278,7 @@ func (h *huffmanDecoder) init(lengths []int) bool {
return true return true
} }
// The actual read interface needed by NewReader. // Reader is the actual read interface needed by NewReader.
// If the passed in io.Reader does not also have ReadByte, // If the passed in io.Reader does not also have ReadByte,
// the NewReader will introduce its own buffering. // the NewReader will introduce its own buffering.
type Reader interface { type Reader interface {
@ -285,6 +286,18 @@ type Reader interface {
io.ByteReader io.ByteReader
} }
type step uint8
const (
copyData step = iota + 1
nextBlock
huffmanBytesBuffer
huffmanBytesReader
huffmanBufioReader
huffmanStringsReader
huffmanGenericReader
)
// Decompress state. // Decompress state.
type decompressor struct { type decompressor struct {
// Input source. // Input source.
@ -303,7 +316,7 @@ type decompressor struct {
// Next step in the decompression, // Next step in the decompression,
// and decompression state. // and decompression state.
step func(*decompressor) step step
stepState int stepState int
err error err error
toRead []byte toRead []byte
@ -342,7 +355,7 @@ func (f *decompressor) nextBlock() {
// compressed, fixed Huffman tables // compressed, fixed Huffman tables
f.hl = &fixedHuffmanDecoder f.hl = &fixedHuffmanDecoder
f.hd = nil f.hd = nil
f.huffmanBlockDecoder()() f.huffmanBlockDecoder()
if debugDecode { if debugDecode {
fmt.Println("predefinied huffman block") fmt.Println("predefinied huffman block")
} }
@ -353,7 +366,7 @@ func (f *decompressor) nextBlock() {
} }
f.hl = &f.h1 f.hl = &f.h1
f.hd = &f.h2 f.hd = &f.h2
f.huffmanBlockDecoder()() f.huffmanBlockDecoder()
if debugDecode { if debugDecode {
fmt.Println("dynamic huffman block") fmt.Println("dynamic huffman block")
} }
@ -379,14 +392,16 @@ func (f *decompressor) Read(b []byte) (int, error) {
if f.err != nil { if f.err != nil {
return 0, f.err return 0, f.err
} }
f.step(f)
f.doStep()
if f.err != nil && len(f.toRead) == 0 { if f.err != nil && len(f.toRead) == 0 {
f.toRead = f.dict.readFlush() // Flush what's left in case of error f.toRead = f.dict.readFlush() // Flush what's left in case of error
} }
} }
} }
// Support the io.WriteTo interface for io.Copy and friends. // WriteTo implements the io.WriteTo interface for io.Copy and friends.
func (f *decompressor) WriteTo(w io.Writer) (int64, error) { func (f *decompressor) WriteTo(w io.Writer) (int64, error) {
total := int64(0) total := int64(0)
flushed := false flushed := false
@ -410,7 +425,7 @@ func (f *decompressor) WriteTo(w io.Writer) (int64, error) {
return total, f.err return total, f.err
} }
if f.err == nil { if f.err == nil {
f.step(f) f.doStep()
} }
if len(f.toRead) == 0 && f.err != nil && !flushed { if len(f.toRead) == 0 && f.err != nil && !flushed {
f.toRead = f.dict.readFlush() // Flush what's left in case of error f.toRead = f.dict.readFlush() // Flush what's left in case of error
@ -631,7 +646,7 @@ func (f *decompressor) copyData() {
if f.dict.availWrite() == 0 || f.copyLen > 0 { if f.dict.availWrite() == 0 || f.copyLen > 0 {
f.toRead = f.dict.readFlush() f.toRead = f.dict.readFlush()
f.step = (*decompressor).copyData f.step = copyData
return return
} }
f.finishBlock() f.finishBlock()
@ -644,7 +659,28 @@ func (f *decompressor) finishBlock() {
} }
f.err = io.EOF f.err = io.EOF
} }
f.step = (*decompressor).nextBlock f.step = nextBlock
}
func (f *decompressor) doStep() {
switch f.step {
case copyData:
f.copyData()
case nextBlock:
f.nextBlock()
case huffmanBytesBuffer:
f.huffmanBytesBuffer()
case huffmanBytesReader:
f.huffmanBytesReader()
case huffmanBufioReader:
f.huffmanBufioReader()
case huffmanStringsReader:
f.huffmanStringsReader()
case huffmanGenericReader:
f.huffmanGenericReader()
default:
panic("BUG: unexpected step state")
}
} }
// noEOF returns err, unless err == io.EOF, in which case it returns io.ErrUnexpectedEOF. // noEOF returns err, unless err == io.EOF, in which case it returns io.ErrUnexpectedEOF.
@ -747,7 +783,7 @@ func (f *decompressor) Reset(r io.Reader, dict []byte) error {
h1: f.h1, h1: f.h1,
h2: f.h2, h2: f.h2,
dict: f.dict, dict: f.dict,
step: (*decompressor).nextBlock, step: nextBlock,
} }
f.dict.init(maxMatchOffset, dict) f.dict.init(maxMatchOffset, dict)
return nil return nil
@ -768,7 +804,7 @@ func NewReader(r io.Reader) io.ReadCloser {
f.r = makeReader(r) f.r = makeReader(r)
f.bits = new([maxNumLit + maxNumDist]int) f.bits = new([maxNumLit + maxNumDist]int)
f.codebits = new([numCodes]int) f.codebits = new([numCodes]int)
f.step = (*decompressor).nextBlock f.step = nextBlock
f.dict.init(maxMatchOffset, nil) f.dict.init(maxMatchOffset, nil)
return &f return &f
} }
@ -787,7 +823,7 @@ func NewReaderDict(r io.Reader, dict []byte) io.ReadCloser {
f.r = makeReader(r) f.r = makeReader(r)
f.bits = new([maxNumLit + maxNumDist]int) f.bits = new([maxNumLit + maxNumDist]int)
f.codebits = new([numCodes]int) f.codebits = new([numCodes]int)
f.step = (*decompressor).nextBlock f.step = nextBlock
f.dict.init(maxMatchOffset, dict) f.dict.init(maxMatchOffset, dict)
return &f return &f
} }

View File

@ -85,7 +85,7 @@ readLiteral:
dict.writeByte(byte(v)) dict.writeByte(byte(v))
if dict.availWrite() == 0 { if dict.availWrite() == 0 {
f.toRead = dict.readFlush() f.toRead = dict.readFlush()
f.step = (*decompressor).huffmanBytesBuffer f.step = huffmanBytesBuffer
f.stepState = stateInit f.stepState = stateInit
f.b, f.nb = fb, fnb f.b, f.nb = fb, fnb
return return
@ -251,7 +251,7 @@ copyHistory:
if dict.availWrite() == 0 || f.copyLen > 0 { if dict.availWrite() == 0 || f.copyLen > 0 {
f.toRead = dict.readFlush() f.toRead = dict.readFlush()
f.step = (*decompressor).huffmanBytesBuffer // We need to continue this work f.step = huffmanBytesBuffer // We need to continue this work
f.stepState = stateDict f.stepState = stateDict
f.b, f.nb = fb, fnb f.b, f.nb = fb, fnb
return return
@ -336,7 +336,7 @@ readLiteral:
dict.writeByte(byte(v)) dict.writeByte(byte(v))
if dict.availWrite() == 0 { if dict.availWrite() == 0 {
f.toRead = dict.readFlush() f.toRead = dict.readFlush()
f.step = (*decompressor).huffmanBytesReader f.step = huffmanBytesReader
f.stepState = stateInit f.stepState = stateInit
f.b, f.nb = fb, fnb f.b, f.nb = fb, fnb
return return
@ -502,7 +502,7 @@ copyHistory:
if dict.availWrite() == 0 || f.copyLen > 0 { if dict.availWrite() == 0 || f.copyLen > 0 {
f.toRead = dict.readFlush() f.toRead = dict.readFlush()
f.step = (*decompressor).huffmanBytesReader // We need to continue this work f.step = huffmanBytesReader // We need to continue this work
f.stepState = stateDict f.stepState = stateDict
f.b, f.nb = fb, fnb f.b, f.nb = fb, fnb
return return
@ -587,7 +587,7 @@ readLiteral:
dict.writeByte(byte(v)) dict.writeByte(byte(v))
if dict.availWrite() == 0 { if dict.availWrite() == 0 {
f.toRead = dict.readFlush() f.toRead = dict.readFlush()
f.step = (*decompressor).huffmanBufioReader f.step = huffmanBufioReader
f.stepState = stateInit f.stepState = stateInit
f.b, f.nb = fb, fnb f.b, f.nb = fb, fnb
return return
@ -753,7 +753,7 @@ copyHistory:
if dict.availWrite() == 0 || f.copyLen > 0 { if dict.availWrite() == 0 || f.copyLen > 0 {
f.toRead = dict.readFlush() f.toRead = dict.readFlush()
f.step = (*decompressor).huffmanBufioReader // We need to continue this work f.step = huffmanBufioReader // We need to continue this work
f.stepState = stateDict f.stepState = stateDict
f.b, f.nb = fb, fnb f.b, f.nb = fb, fnb
return return
@ -838,7 +838,7 @@ readLiteral:
dict.writeByte(byte(v)) dict.writeByte(byte(v))
if dict.availWrite() == 0 { if dict.availWrite() == 0 {
f.toRead = dict.readFlush() f.toRead = dict.readFlush()
f.step = (*decompressor).huffmanStringsReader f.step = huffmanStringsReader
f.stepState = stateInit f.stepState = stateInit
f.b, f.nb = fb, fnb f.b, f.nb = fb, fnb
return return
@ -1004,7 +1004,7 @@ copyHistory:
if dict.availWrite() == 0 || f.copyLen > 0 { if dict.availWrite() == 0 || f.copyLen > 0 {
f.toRead = dict.readFlush() f.toRead = dict.readFlush()
f.step = (*decompressor).huffmanStringsReader // We need to continue this work f.step = huffmanStringsReader // We need to continue this work
f.stepState = stateDict f.stepState = stateDict
f.b, f.nb = fb, fnb f.b, f.nb = fb, fnb
return return
@ -1089,7 +1089,7 @@ readLiteral:
dict.writeByte(byte(v)) dict.writeByte(byte(v))
if dict.availWrite() == 0 { if dict.availWrite() == 0 {
f.toRead = dict.readFlush() f.toRead = dict.readFlush()
f.step = (*decompressor).huffmanGenericReader f.step = huffmanGenericReader
f.stepState = stateInit f.stepState = stateInit
f.b, f.nb = fb, fnb f.b, f.nb = fb, fnb
return return
@ -1255,7 +1255,7 @@ copyHistory:
if dict.availWrite() == 0 || f.copyLen > 0 { if dict.availWrite() == 0 || f.copyLen > 0 {
f.toRead = dict.readFlush() f.toRead = dict.readFlush()
f.step = (*decompressor).huffmanGenericReader // We need to continue this work f.step = huffmanGenericReader // We need to continue this work
f.stepState = stateDict f.stepState = stateDict
f.b, f.nb = fb, fnb f.b, f.nb = fb, fnb
return return
@ -1265,19 +1265,19 @@ copyHistory:
// Not reached // Not reached
} }
func (f *decompressor) huffmanBlockDecoder() func() { func (f *decompressor) huffmanBlockDecoder() {
switch f.r.(type) { switch f.r.(type) {
case *bytes.Buffer: case *bytes.Buffer:
return f.huffmanBytesBuffer f.huffmanBytesBuffer()
case *bytes.Reader: case *bytes.Reader:
return f.huffmanBytesReader f.huffmanBytesReader()
case *bufio.Reader: case *bufio.Reader:
return f.huffmanBufioReader f.huffmanBufioReader()
case *strings.Reader: case *strings.Reader:
return f.huffmanStringsReader f.huffmanStringsReader()
case Reader: case Reader:
return f.huffmanGenericReader f.huffmanGenericReader()
default: default:
return f.huffmanGenericReader f.huffmanGenericReader()
} }
} }

View File

@ -308,3 +308,401 @@ emitRemainder:
emitLiteral(dst, src[nextEmit:]) emitLiteral(dst, src[nextEmit:])
} }
} }
// fastEncL5Window is a level 5 encoder,
// but with a custom window size.
type fastEncL5Window struct {
hist []byte
cur int32
maxOffset int32
table [tableSize]tableEntry
bTable [tableSize]tableEntryPrev
}
func (e *fastEncL5Window) Encode(dst *tokens, src []byte) {
const (
inputMargin = 12 - 1
minNonLiteralBlockSize = 1 + 1 + inputMargin
hashShortBytes = 4
)
maxMatchOffset := e.maxOffset
if debugDeflate && e.cur < 0 {
panic(fmt.Sprint("e.cur < 0: ", e.cur))
}
// Protect against e.cur wraparound.
for e.cur >= bufferReset {
if len(e.hist) == 0 {
for i := range e.table[:] {
e.table[i] = tableEntry{}
}
for i := range e.bTable[:] {
e.bTable[i] = tableEntryPrev{}
}
e.cur = maxMatchOffset
break
}
// Shift down everything in the table that isn't already too far away.
minOff := e.cur + int32(len(e.hist)) - maxMatchOffset
for i := range e.table[:] {
v := e.table[i].offset
if v <= minOff {
v = 0
} else {
v = v - e.cur + maxMatchOffset
}
e.table[i].offset = v
}
for i := range e.bTable[:] {
v := e.bTable[i]
if v.Cur.offset <= minOff {
v.Cur.offset = 0
v.Prev.offset = 0
} else {
v.Cur.offset = v.Cur.offset - e.cur + maxMatchOffset
if v.Prev.offset <= minOff {
v.Prev.offset = 0
} else {
v.Prev.offset = v.Prev.offset - e.cur + maxMatchOffset
}
}
e.bTable[i] = v
}
e.cur = maxMatchOffset
}
s := e.addBlock(src)
// This check isn't in the Snappy implementation, but there, the caller
// instead of the callee handles this case.
if len(src) < minNonLiteralBlockSize {
// We do not fill the token table.
// This will be picked up by caller.
dst.n = uint16(len(src))
return
}
// Override src
src = e.hist
nextEmit := s
// sLimit is when to stop looking for offset/length copies. The inputMargin
// lets us use a fast path for emitLiteral in the main loop, while we are
// looking for copies.
sLimit := int32(len(src) - inputMargin)
// nextEmit is where in src the next emitLiteral should start from.
cv := load6432(src, s)
for {
const skipLog = 6
const doEvery = 1
nextS := s
var l int32
var t int32
for {
nextHashS := hashLen(cv, tableBits, hashShortBytes)
nextHashL := hash7(cv, tableBits)
s = nextS
nextS = s + doEvery + (s-nextEmit)>>skipLog
if nextS > sLimit {
goto emitRemainder
}
// Fetch a short+long candidate
sCandidate := e.table[nextHashS]
lCandidate := e.bTable[nextHashL]
next := load6432(src, nextS)
entry := tableEntry{offset: s + e.cur}
e.table[nextHashS] = entry
eLong := &e.bTable[nextHashL]
eLong.Cur, eLong.Prev = entry, eLong.Cur
nextHashS = hashLen(next, tableBits, hashShortBytes)
nextHashL = hash7(next, tableBits)
t = lCandidate.Cur.offset - e.cur
if s-t < maxMatchOffset {
if uint32(cv) == load3232(src, lCandidate.Cur.offset-e.cur) {
// Store the next match
e.table[nextHashS] = tableEntry{offset: nextS + e.cur}
eLong := &e.bTable[nextHashL]
eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur
t2 := lCandidate.Prev.offset - e.cur
if s-t2 < maxMatchOffset && uint32(cv) == load3232(src, lCandidate.Prev.offset-e.cur) {
l = e.matchlen(s+4, t+4, src) + 4
ml1 := e.matchlen(s+4, t2+4, src) + 4
if ml1 > l {
t = t2
l = ml1
break
}
}
break
}
t = lCandidate.Prev.offset - e.cur
if s-t < maxMatchOffset && uint32(cv) == load3232(src, lCandidate.Prev.offset-e.cur) {
// Store the next match
e.table[nextHashS] = tableEntry{offset: nextS + e.cur}
eLong := &e.bTable[nextHashL]
eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur
break
}
}
t = sCandidate.offset - e.cur
if s-t < maxMatchOffset && uint32(cv) == load3232(src, sCandidate.offset-e.cur) {
// Found a 4 match...
l = e.matchlen(s+4, t+4, src) + 4
lCandidate = e.bTable[nextHashL]
// Store the next match
e.table[nextHashS] = tableEntry{offset: nextS + e.cur}
eLong := &e.bTable[nextHashL]
eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur
// If the next long is a candidate, use that...
t2 := lCandidate.Cur.offset - e.cur
if nextS-t2 < maxMatchOffset {
if load3232(src, lCandidate.Cur.offset-e.cur) == uint32(next) {
ml := e.matchlen(nextS+4, t2+4, src) + 4
if ml > l {
t = t2
s = nextS
l = ml
break
}
}
// If the previous long is a candidate, use that...
t2 = lCandidate.Prev.offset - e.cur
if nextS-t2 < maxMatchOffset && load3232(src, lCandidate.Prev.offset-e.cur) == uint32(next) {
ml := e.matchlen(nextS+4, t2+4, src) + 4
if ml > l {
t = t2
s = nextS
l = ml
break
}
}
}
break
}
cv = next
}
// A 4-byte match has been found. We'll later see if more than 4 bytes
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
// them as literal bytes.
if l == 0 {
// Extend the 4-byte match as long as possible.
l = e.matchlenLong(s+4, t+4, src) + 4
} else if l == maxMatchLength {
l += e.matchlenLong(s+l, t+l, src)
}
// Try to locate a better match by checking the end of best match...
if sAt := s + l; l < 30 && sAt < sLimit {
// Allow some bytes at the beginning to mismatch.
// Sweet spot is 2/3 bytes depending on input.
// 3 is only a little better when it is but sometimes a lot worse.
// The skipped bytes are tested in Extend backwards,
// and still picked up as part of the match if they do.
const skipBeginning = 2
eLong := e.bTable[hash7(load6432(src, sAt), tableBits)].Cur.offset
t2 := eLong - e.cur - l + skipBeginning
s2 := s + skipBeginning
off := s2 - t2
if t2 >= 0 && off < maxMatchOffset && off > 0 {
if l2 := e.matchlenLong(s2, t2, src); l2 > l {
t = t2
l = l2
s = s2
}
}
}
// Extend backwards
for t > 0 && s > nextEmit && src[t-1] == src[s-1] {
s--
t--
l++
}
if nextEmit < s {
if false {
emitLiteral(dst, src[nextEmit:s])
} else {
for _, v := range src[nextEmit:s] {
dst.tokens[dst.n] = token(v)
dst.litHist[v]++
dst.n++
}
}
}
if debugDeflate {
if t >= s {
panic(fmt.Sprintln("s-t", s, t))
}
if (s - t) > maxMatchOffset {
panic(fmt.Sprintln("mmo", s-t))
}
if l < baseMatchLength {
panic("bml")
}
}
dst.AddMatchLong(l, uint32(s-t-baseMatchOffset))
s += l
nextEmit = s
if nextS >= s {
s = nextS + 1
}
if s >= sLimit {
goto emitRemainder
}
// Store every 3rd hash in-between.
if true {
const hashEvery = 3
i := s - l + 1
if i < s-1 {
cv := load6432(src, i)
t := tableEntry{offset: i + e.cur}
e.table[hashLen(cv, tableBits, hashShortBytes)] = t
eLong := &e.bTable[hash7(cv, tableBits)]
eLong.Cur, eLong.Prev = t, eLong.Cur
// Do an long at i+1
cv >>= 8
t = tableEntry{offset: t.offset + 1}
eLong = &e.bTable[hash7(cv, tableBits)]
eLong.Cur, eLong.Prev = t, eLong.Cur
// We only have enough bits for a short entry at i+2
cv >>= 8
t = tableEntry{offset: t.offset + 1}
e.table[hashLen(cv, tableBits, hashShortBytes)] = t
// Skip one - otherwise we risk hitting 's'
i += 4
for ; i < s-1; i += hashEvery {
cv := load6432(src, i)
t := tableEntry{offset: i + e.cur}
t2 := tableEntry{offset: t.offset + 1}
eLong := &e.bTable[hash7(cv, tableBits)]
eLong.Cur, eLong.Prev = t, eLong.Cur
e.table[hashLen(cv>>8, tableBits, hashShortBytes)] = t2
}
}
}
// We could immediately start working at s now, but to improve
// compression we first update the hash table at s-1 and at s.
x := load6432(src, s-1)
o := e.cur + s - 1
prevHashS := hashLen(x, tableBits, hashShortBytes)
prevHashL := hash7(x, tableBits)
e.table[prevHashS] = tableEntry{offset: o}
eLong := &e.bTable[prevHashL]
eLong.Cur, eLong.Prev = tableEntry{offset: o}, eLong.Cur
cv = x >> 8
}
emitRemainder:
if int(nextEmit) < len(src) {
// If nothing was added, don't encode literals.
if dst.n == 0 {
return
}
emitLiteral(dst, src[nextEmit:])
}
}
// Reset the encoding table.
func (e *fastEncL5Window) Reset() {
// We keep the same allocs, since we are compressing the same block sizes.
if cap(e.hist) < allocHistory {
e.hist = make([]byte, 0, allocHistory)
}
// We offset current position so everything will be out of reach.
// If we are above the buffer reset it will be cleared anyway since len(hist) == 0.
if e.cur <= int32(bufferReset) {
e.cur += e.maxOffset + int32(len(e.hist))
}
e.hist = e.hist[:0]
}
func (e *fastEncL5Window) addBlock(src []byte) int32 {
// check if we have space already
maxMatchOffset := e.maxOffset
if len(e.hist)+len(src) > cap(e.hist) {
if cap(e.hist) == 0 {
e.hist = make([]byte, 0, allocHistory)
} else {
if cap(e.hist) < int(maxMatchOffset*2) {
panic("unexpected buffer size")
}
// Move down
offset := int32(len(e.hist)) - maxMatchOffset
copy(e.hist[0:maxMatchOffset], e.hist[offset:])
e.cur += offset
e.hist = e.hist[:maxMatchOffset]
}
}
s := int32(len(e.hist))
e.hist = append(e.hist, src...)
return s
}
// matchlen will return the match length between offsets and t in src.
// The maximum length returned is maxMatchLength - 4.
// It is assumed that s > t, that t >=0 and s < len(src).
func (e *fastEncL5Window) matchlen(s, t int32, src []byte) int32 {
if debugDecode {
if t >= s {
panic(fmt.Sprint("t >=s:", t, s))
}
if int(s) >= len(src) {
panic(fmt.Sprint("s >= len(src):", s, len(src)))
}
if t < 0 {
panic(fmt.Sprint("t < 0:", t))
}
if s-t > e.maxOffset {
panic(fmt.Sprint(s, "-", t, "(", s-t, ") > maxMatchLength (", maxMatchOffset, ")"))
}
}
s1 := int(s) + maxMatchLength - 4
if s1 > len(src) {
s1 = len(src)
}
// Extend the match to be as long as possible.
return int32(matchLen(src[s:s1], src[t:]))
}
// matchlenLong will return the match length between offsets and t in src.
// It is assumed that s > t, that t >=0 and s < len(src).
func (e *fastEncL5Window) matchlenLong(s, t int32, src []byte) int32 {
if debugDeflate {
if t >= s {
panic(fmt.Sprint("t >=s:", t, s))
}
if int(s) >= len(src) {
panic(fmt.Sprint("s >= len(src):", s, len(src)))
}
if t < 0 {
panic(fmt.Sprint("t < 0:", t))
}
if s-t > e.maxOffset {
panic(fmt.Sprint(s, "-", t, "(", s-t, ") > maxMatchLength (", maxMatchOffset, ")"))
}
}
// Extend the match to be as long as possible.
return int32(matchLen(src[s:], src[t:]))
}

View File

@ -0,0 +1,16 @@
//go:build amd64 && !appengine && !noasm && gc
// +build amd64,!appengine,!noasm,gc
// Copyright 2019+ Klaus Post. All rights reserved.
// License information can be found in the LICENSE file.
package flate
// matchLen returns how many bytes match in a and b
//
// It assumes that:
//
// len(a) <= len(b) and len(a) > 0
//
//go:noescape
func matchLen(a []byte, b []byte) int

View File

@ -0,0 +1,68 @@
// Copied from S2 implementation.
//go:build !appengine && !noasm && gc && !noasm
#include "textflag.h"
// func matchLen(a []byte, b []byte) int
// Requires: BMI
TEXT ·matchLen(SB), NOSPLIT, $0-56
MOVQ a_base+0(FP), AX
MOVQ b_base+24(FP), CX
MOVQ a_len+8(FP), DX
// matchLen
XORL SI, SI
CMPL DX, $0x08
JB matchlen_match4_standalone
matchlen_loopback_standalone:
MOVQ (AX)(SI*1), BX
XORQ (CX)(SI*1), BX
TESTQ BX, BX
JZ matchlen_loop_standalone
#ifdef GOAMD64_v3
TZCNTQ BX, BX
#else
BSFQ BX, BX
#endif
SARQ $0x03, BX
LEAL (SI)(BX*1), SI
JMP gen_match_len_end
matchlen_loop_standalone:
LEAL -8(DX), DX
LEAL 8(SI), SI
CMPL DX, $0x08
JAE matchlen_loopback_standalone
matchlen_match4_standalone:
CMPL DX, $0x04
JB matchlen_match2_standalone
MOVL (AX)(SI*1), BX
CMPL (CX)(SI*1), BX
JNE matchlen_match2_standalone
LEAL -4(DX), DX
LEAL 4(SI), SI
matchlen_match2_standalone:
CMPL DX, $0x02
JB matchlen_match1_standalone
MOVW (AX)(SI*1), BX
CMPW (CX)(SI*1), BX
JNE matchlen_match1_standalone
LEAL -2(DX), DX
LEAL 2(SI), SI
matchlen_match1_standalone:
CMPL DX, $0x01
JB gen_match_len_end
MOVB (AX)(SI*1), BL
CMPB (CX)(SI*1), BL
JNE gen_match_len_end
INCL SI
gen_match_len_end:
MOVQ SI, ret+48(FP)
RET

View File

@ -0,0 +1,33 @@
//go:build !amd64 || appengine || !gc || noasm
// +build !amd64 appengine !gc noasm
// Copyright 2019+ Klaus Post. All rights reserved.
// License information can be found in the LICENSE file.
package flate
import (
"encoding/binary"
"math/bits"
)
// matchLen returns the maximum common prefix length of a and b.
// a must be the shortest of the two.
func matchLen(a, b []byte) (n int) {
for ; len(a) >= 8 && len(b) >= 8; a, b = a[8:], b[8:] {
diff := binary.LittleEndian.Uint64(a) ^ binary.LittleEndian.Uint64(b)
if diff != 0 {
return n + bits.TrailingZeros64(diff)>>3
}
n += 8
}
for i := range a {
if a[i] != b[i] {
break
}
n++
}
return n
}

View File

@ -152,12 +152,11 @@ func (b *bitWriter) flushAlign() {
// close will write the alignment bit and write the final byte(s) // close will write the alignment bit and write the final byte(s)
// to the output. // to the output.
func (b *bitWriter) close() error { func (b *bitWriter) close() {
// End mark // End mark
b.addBits16Clean(1, 1) b.addBits16Clean(1, 1)
// flush until next byte. // flush until next byte.
b.flushAlign() b.flushAlign()
return nil
} }
// reset and continue writing by appending to out. // reset and continue writing by appending to out.

View File

@ -199,7 +199,8 @@ func (s *Scratch) compress(src []byte) error {
c2.flush(s.actualTableLog) c2.flush(s.actualTableLog)
c1.flush(s.actualTableLog) c1.flush(s.actualTableLog)
return s.bw.close() s.bw.close()
return nil
} }
// writeCount will write the normalized histogram count to header. // writeCount will write the normalized histogram count to header.

View File

@ -94,10 +94,9 @@ func (b *bitWriter) flushAlign() {
// close will write the alignment bit and write the final byte(s) // close will write the alignment bit and write the final byte(s)
// to the output. // to the output.
func (b *bitWriter) close() error { func (b *bitWriter) close() {
// End mark // End mark
b.addBits16Clean(1, 1) b.addBits16Clean(1, 1)
// flush until next byte. // flush until next byte.
b.flushAlign() b.flushAlign()
return nil
} }

View File

@ -227,10 +227,10 @@ func EstimateSizes(in []byte, s *Scratch) (tableSz, dataSz, reuseSz int, err err
} }
func (s *Scratch) compress1X(src []byte) ([]byte, error) { func (s *Scratch) compress1X(src []byte) ([]byte, error) {
return s.compress1xDo(s.Out, src) return s.compress1xDo(s.Out, src), nil
} }
func (s *Scratch) compress1xDo(dst, src []byte) ([]byte, error) { func (s *Scratch) compress1xDo(dst, src []byte) []byte {
var bw = bitWriter{out: dst} var bw = bitWriter{out: dst}
// N is length divisible by 4. // N is length divisible by 4.
@ -260,8 +260,8 @@ func (s *Scratch) compress1xDo(dst, src []byte) ([]byte, error) {
bw.encTwoSymbols(cTable, tmp[1], tmp[0]) bw.encTwoSymbols(cTable, tmp[1], tmp[0])
} }
} }
err := bw.close() bw.close()
return bw.out, err return bw.out
} }
var sixZeros [6]byte var sixZeros [6]byte
@ -283,12 +283,8 @@ func (s *Scratch) compress4X(src []byte) ([]byte, error) {
} }
src = src[len(toDo):] src = src[len(toDo):]
var err error
idx := len(s.Out) idx := len(s.Out)
s.Out, err = s.compress1xDo(s.Out, toDo) s.Out = s.compress1xDo(s.Out, toDo)
if err != nil {
return nil, err
}
if len(s.Out)-idx > math.MaxUint16 { if len(s.Out)-idx > math.MaxUint16 {
// We cannot store the size in the jump table // We cannot store the size in the jump table
return nil, ErrIncompressible return nil, ErrIncompressible
@ -315,7 +311,6 @@ func (s *Scratch) compress4Xp(src []byte) ([]byte, error) {
segmentSize := (len(src) + 3) / 4 segmentSize := (len(src) + 3) / 4
var wg sync.WaitGroup var wg sync.WaitGroup
var errs [4]error
wg.Add(4) wg.Add(4)
for i := 0; i < 4; i++ { for i := 0; i < 4; i++ {
toDo := src toDo := src
@ -326,15 +321,12 @@ func (s *Scratch) compress4Xp(src []byte) ([]byte, error) {
// Separate goroutine for each block. // Separate goroutine for each block.
go func(i int) { go func(i int) {
s.tmpOut[i], errs[i] = s.compress1xDo(s.tmpOut[i][:0], toDo) s.tmpOut[i] = s.compress1xDo(s.tmpOut[i][:0], toDo)
wg.Done() wg.Done()
}(i) }(i)
} }
wg.Wait() wg.Wait()
for i := 0; i < 4; i++ { for i := 0; i < 4; i++ {
if errs[i] != nil {
return nil, errs[i]
}
o := s.tmpOut[i] o := s.tmpOut[i]
if len(o) > math.MaxUint16 { if len(o) > math.MaxUint16 {
// We cannot store the size in the jump table // We cannot store the size in the jump table

View File

@ -17,7 +17,6 @@ import (
// for aligning the input. // for aligning the input.
type bitReader struct { type bitReader struct {
in []byte in []byte
off uint // next byte to read is at in[off - 1]
value uint64 // Maybe use [16]byte, but shifting is awkward. value uint64 // Maybe use [16]byte, but shifting is awkward.
bitsRead uint8 bitsRead uint8
} }
@ -28,7 +27,6 @@ func (b *bitReader) init(in []byte) error {
return errors.New("corrupt stream: too short") return errors.New("corrupt stream: too short")
} }
b.in = in b.in = in
b.off = uint(len(in))
// The highest bit of the last byte indicates where to start // The highest bit of the last byte indicates where to start
v := in[len(in)-1] v := in[len(in)-1]
if v == 0 { if v == 0 {
@ -69,21 +67,19 @@ func (b *bitReader) fillFast() {
if b.bitsRead < 32 { if b.bitsRead < 32 {
return return
} }
// 2 bounds checks. v := b.in[len(b.in)-4:]
v := b.in[b.off-4:] b.in = b.in[:len(b.in)-4]
v = v[:4]
low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
b.value = (b.value << 32) | uint64(low) b.value = (b.value << 32) | uint64(low)
b.bitsRead -= 32 b.bitsRead -= 32
b.off -= 4
} }
// fillFastStart() assumes the bitreader is empty and there is at least 8 bytes to read. // fillFastStart() assumes the bitreader is empty and there is at least 8 bytes to read.
func (b *bitReader) fillFastStart() { func (b *bitReader) fillFastStart() {
// Do single re-slice to avoid bounds checks. v := b.in[len(b.in)-8:]
b.value = binary.LittleEndian.Uint64(b.in[b.off-8:]) b.in = b.in[:len(b.in)-8]
b.value = binary.LittleEndian.Uint64(v)
b.bitsRead = 0 b.bitsRead = 0
b.off -= 8
} }
// fill() will make sure at least 32 bits are available. // fill() will make sure at least 32 bits are available.
@ -91,25 +87,25 @@ func (b *bitReader) fill() {
if b.bitsRead < 32 { if b.bitsRead < 32 {
return return
} }
if b.off >= 4 { if len(b.in) >= 4 {
v := b.in[b.off-4:] v := b.in[len(b.in)-4:]
v = v[:4] b.in = b.in[:len(b.in)-4]
low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
b.value = (b.value << 32) | uint64(low) b.value = (b.value << 32) | uint64(low)
b.bitsRead -= 32 b.bitsRead -= 32
b.off -= 4
return return
} }
for b.off > 0 {
b.value = (b.value << 8) | uint64(b.in[b.off-1]) b.bitsRead -= uint8(8 * len(b.in))
b.bitsRead -= 8 for len(b.in) > 0 {
b.off-- b.value = (b.value << 8) | uint64(b.in[len(b.in)-1])
b.in = b.in[:len(b.in)-1]
} }
} }
// finished returns true if all bits have been read from the bit stream. // finished returns true if all bits have been read from the bit stream.
func (b *bitReader) finished() bool { func (b *bitReader) finished() bool {
return b.off == 0 && b.bitsRead >= 64 return len(b.in) == 0 && b.bitsRead >= 64
} }
// overread returns true if more bits have been requested than is on the stream. // overread returns true if more bits have been requested than is on the stream.
@ -119,7 +115,7 @@ func (b *bitReader) overread() bool {
// remain returns the number of bits remaining. // remain returns the number of bits remaining.
func (b *bitReader) remain() uint { func (b *bitReader) remain() uint {
return b.off*8 + 64 - uint(b.bitsRead) return 8*uint(len(b.in)) + 64 - uint(b.bitsRead)
} }
// close the bitstream and returns an error if out-of-buffer reads occurred. // close the bitstream and returns an error if out-of-buffer reads occurred.

View File

@ -97,12 +97,11 @@ func (b *bitWriter) flushAlign() {
// close will write the alignment bit and write the final byte(s) // close will write the alignment bit and write the final byte(s)
// to the output. // to the output.
func (b *bitWriter) close() error { func (b *bitWriter) close() {
// End mark // End mark
b.addBits16Clean(1, 1) b.addBits16Clean(1, 1)
// flush until next byte. // flush until next byte.
b.flushAlign() b.flushAlign()
return nil
} }
// reset and continue writing by appending to out. // reset and continue writing by appending to out.

View File

@ -361,14 +361,21 @@ func (b *blockEnc) encodeLits(lits []byte, raw bool) error {
if len(lits) >= 1024 { if len(lits) >= 1024 {
// Use 4 Streams. // Use 4 Streams.
out, reUsed, err = huff0.Compress4X(lits, b.litEnc) out, reUsed, err = huff0.Compress4X(lits, b.litEnc)
} else if len(lits) > 32 { } else if len(lits) > 16 {
// Use 1 stream // Use 1 stream
single = true single = true
out, reUsed, err = huff0.Compress1X(lits, b.litEnc) out, reUsed, err = huff0.Compress1X(lits, b.litEnc)
} else { } else {
err = huff0.ErrIncompressible err = huff0.ErrIncompressible
} }
if err == nil && len(out)+5 > len(lits) {
// If we are close, we may still be worse or equal to raw.
var lh literalsHeader
lh.setSizes(len(out), len(lits), single)
if len(out)+lh.size() >= len(lits) {
err = huff0.ErrIncompressible
}
}
switch err { switch err {
case huff0.ErrIncompressible: case huff0.ErrIncompressible:
if debugEncoder { if debugEncoder {
@ -503,7 +510,7 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error {
if len(b.literals) >= 1024 && !raw { if len(b.literals) >= 1024 && !raw {
// Use 4 Streams. // Use 4 Streams.
out, reUsed, err = huff0.Compress4X(b.literals, b.litEnc) out, reUsed, err = huff0.Compress4X(b.literals, b.litEnc)
} else if len(b.literals) > 32 && !raw { } else if len(b.literals) > 16 && !raw {
// Use 1 stream // Use 1 stream
single = true single = true
out, reUsed, err = huff0.Compress1X(b.literals, b.litEnc) out, reUsed, err = huff0.Compress1X(b.literals, b.litEnc)
@ -511,6 +518,17 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error {
err = huff0.ErrIncompressible err = huff0.ErrIncompressible
} }
if err == nil && len(out)+5 > len(b.literals) {
// If we are close, we may still be worse or equal to raw.
var lh literalsHeader
lh.setSize(len(b.literals))
szRaw := lh.size()
lh.setSizes(len(out), len(b.literals), single)
szComp := lh.size()
if len(out)+szComp >= len(b.literals)+szRaw {
err = huff0.ErrIncompressible
}
}
switch err { switch err {
case huff0.ErrIncompressible: case huff0.ErrIncompressible:
lh.setType(literalsBlockRaw) lh.setType(literalsBlockRaw)
@ -773,10 +791,7 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error {
ml.flush(mlEnc.actualTableLog) ml.flush(mlEnc.actualTableLog)
of.flush(ofEnc.actualTableLog) of.flush(ofEnc.actualTableLog)
ll.flush(llEnc.actualTableLog) ll.flush(llEnc.actualTableLog)
err = wr.close() wr.close()
if err != nil {
return err
}
b.output = wr.out b.output = wr.out
// Maybe even add a bigger margin. // Maybe even add a bigger margin.

View File

@ -1,10 +1,13 @@
package zstd package zstd
import ( import (
"bytes"
"encoding/binary" "encoding/binary"
"errors" "errors"
"fmt" "fmt"
"io" "io"
"math"
"sort"
"github.com/klauspost/compress/huff0" "github.com/klauspost/compress/huff0"
) )
@ -14,7 +17,6 @@ type dict struct {
litEnc *huff0.Scratch litEnc *huff0.Scratch
llDec, ofDec, mlDec sequenceDec llDec, ofDec, mlDec sequenceDec
//llEnc, ofEnc, mlEnc []*fseEncoder
offsets [3]int offsets [3]int
content []byte content []byte
} }
@ -159,3 +161,374 @@ func InspectDictionary(b []byte) (interface {
d, err := loadDict(b) d, err := loadDict(b)
return d, err return d, err
} }
type BuildDictOptions struct {
// Dictionary ID.
ID uint32
// Content to use to create dictionary tables.
Contents [][]byte
// History to use for all blocks.
History []byte
// Offsets to use.
Offsets [3]int
// CompatV155 will make the dictionary compatible with Zstd v1.5.5 and earlier.
// See https://github.com/facebook/zstd/issues/3724
CompatV155 bool
// Use the specified encoder level.
// The dictionary will be built using the specified encoder level,
// which will reflect speed and make the dictionary tailored for that level.
// If not set SpeedBestCompression will be used.
Level EncoderLevel
// DebugOut will write stats and other details here if set.
DebugOut io.Writer
}
func BuildDict(o BuildDictOptions) ([]byte, error) {
initPredefined()
hist := o.History
contents := o.Contents
debug := o.DebugOut != nil
println := func(args ...interface{}) {
if o.DebugOut != nil {
fmt.Fprintln(o.DebugOut, args...)
}
}
printf := func(s string, args ...interface{}) {
if o.DebugOut != nil {
fmt.Fprintf(o.DebugOut, s, args...)
}
}
print := func(args ...interface{}) {
if o.DebugOut != nil {
fmt.Fprint(o.DebugOut, args...)
}
}
if int64(len(hist)) > dictMaxLength {
return nil, fmt.Errorf("dictionary of size %d > %d", len(hist), int64(dictMaxLength))
}
if len(hist) < 8 {
return nil, fmt.Errorf("dictionary of size %d < %d", len(hist), 8)
}
if len(contents) == 0 {
return nil, errors.New("no content provided")
}
d := dict{
id: o.ID,
litEnc: nil,
llDec: sequenceDec{},
ofDec: sequenceDec{},
mlDec: sequenceDec{},
offsets: o.Offsets,
content: hist,
}
block := blockEnc{lowMem: false}
block.init()
enc := encoder(&bestFastEncoder{fastBase: fastBase{maxMatchOff: int32(maxMatchLen), bufferReset: math.MaxInt32 - int32(maxMatchLen*2), lowMem: false}})
if o.Level != 0 {
eOpts := encoderOptions{
level: o.Level,
blockSize: maxMatchLen,
windowSize: maxMatchLen,
dict: &d,
lowMem: false,
}
enc = eOpts.encoder()
} else {
o.Level = SpeedBestCompression
}
var (
remain [256]int
ll [256]int
ml [256]int
of [256]int
)
addValues := func(dst *[256]int, src []byte) {
for _, v := range src {
dst[v]++
}
}
addHist := func(dst *[256]int, src *[256]uint32) {
for i, v := range src {
dst[i] += int(v)
}
}
seqs := 0
nUsed := 0
litTotal := 0
newOffsets := make(map[uint32]int, 1000)
for _, b := range contents {
block.reset(nil)
if len(b) < 8 {
continue
}
nUsed++
enc.Reset(&d, true)
enc.Encode(&block, b)
addValues(&remain, block.literals)
litTotal += len(block.literals)
seqs += len(block.sequences)
block.genCodes()
addHist(&ll, block.coders.llEnc.Histogram())
addHist(&ml, block.coders.mlEnc.Histogram())
addHist(&of, block.coders.ofEnc.Histogram())
for i, seq := range block.sequences {
if i > 3 {
break
}
offset := seq.offset
if offset == 0 {
continue
}
if offset > 3 {
newOffsets[offset-3]++
} else {
newOffsets[uint32(o.Offsets[offset-1])]++
}
}
}
// Find most used offsets.
var sortedOffsets []uint32
for k := range newOffsets {
sortedOffsets = append(sortedOffsets, k)
}
sort.Slice(sortedOffsets, func(i, j int) bool {
a, b := sortedOffsets[i], sortedOffsets[j]
if a == b {
// Prefer the longer offset
return sortedOffsets[i] > sortedOffsets[j]
}
return newOffsets[sortedOffsets[i]] > newOffsets[sortedOffsets[j]]
})
if len(sortedOffsets) > 3 {
if debug {
print("Offsets:")
for i, v := range sortedOffsets {
if i > 20 {
break
}
printf("[%d: %d],", v, newOffsets[v])
}
println("")
}
sortedOffsets = sortedOffsets[:3]
}
for i, v := range sortedOffsets {
o.Offsets[i] = int(v)
}
if debug {
println("New repeat offsets", o.Offsets)
}
if nUsed == 0 || seqs == 0 {
return nil, fmt.Errorf("%d blocks, %d sequences found", nUsed, seqs)
}
if debug {
println("Sequences:", seqs, "Blocks:", nUsed, "Literals:", litTotal)
}
if seqs/nUsed < 512 {
// Use 512 as minimum.
nUsed = seqs / 512
}
copyHist := func(dst *fseEncoder, src *[256]int) ([]byte, error) {
hist := dst.Histogram()
var maxSym uint8
var maxCount int
var fakeLength int
for i, v := range src {
if v > 0 {
v = v / nUsed
if v == 0 {
v = 1
}
}
if v > maxCount {
maxCount = v
}
if v != 0 {
maxSym = uint8(i)
}
fakeLength += v
hist[i] = uint32(v)
}
dst.HistogramFinished(maxSym, maxCount)
dst.reUsed = false
dst.useRLE = false
err := dst.normalizeCount(fakeLength)
if err != nil {
return nil, err
}
if debug {
println("RAW:", dst.count[:maxSym+1], "NORM:", dst.norm[:maxSym+1], "LEN:", fakeLength)
}
return dst.writeCount(nil)
}
if debug {
print("Literal lengths: ")
}
llTable, err := copyHist(block.coders.llEnc, &ll)
if err != nil {
return nil, err
}
if debug {
print("Match lengths: ")
}
mlTable, err := copyHist(block.coders.mlEnc, &ml)
if err != nil {
return nil, err
}
if debug {
print("Offsets: ")
}
ofTable, err := copyHist(block.coders.ofEnc, &of)
if err != nil {
return nil, err
}
// Literal table
avgSize := litTotal
if avgSize > huff0.BlockSizeMax/2 {
avgSize = huff0.BlockSizeMax / 2
}
huffBuff := make([]byte, 0, avgSize)
// Target size
div := litTotal / avgSize
if div < 1 {
div = 1
}
if debug {
println("Huffman weights:")
}
for i, n := range remain[:] {
if n > 0 {
n = n / div
// Allow all entries to be represented.
if n == 0 {
n = 1
}
huffBuff = append(huffBuff, bytes.Repeat([]byte{byte(i)}, n)...)
if debug {
printf("[%d: %d], ", i, n)
}
}
}
if o.CompatV155 && remain[255]/div == 0 {
huffBuff = append(huffBuff, 255)
}
scratch := &huff0.Scratch{TableLog: 11}
for tries := 0; tries < 255; tries++ {
scratch = &huff0.Scratch{TableLog: 11}
_, _, err = huff0.Compress1X(huffBuff, scratch)
if err == nil {
break
}
if debug {
printf("Try %d: Huffman error: %v\n", tries+1, err)
}
huffBuff = huffBuff[:0]
if tries == 250 {
if debug {
println("Huffman: Bailing out with predefined table")
}
// Bail out.... Just generate something
huffBuff = append(huffBuff, bytes.Repeat([]byte{255}, 10000)...)
for i := 0; i < 128; i++ {
huffBuff = append(huffBuff, byte(i))
}
continue
}
if errors.Is(err, huff0.ErrIncompressible) {
// Try truncating least common.
for i, n := range remain[:] {
if n > 0 {
n = n / (div * (i + 1))
if n > 0 {
huffBuff = append(huffBuff, bytes.Repeat([]byte{byte(i)}, n)...)
}
}
}
if o.CompatV155 && len(huffBuff) > 0 && huffBuff[len(huffBuff)-1] != 255 {
huffBuff = append(huffBuff, 255)
}
if len(huffBuff) == 0 {
huffBuff = append(huffBuff, 0, 255)
}
}
if errors.Is(err, huff0.ErrUseRLE) {
for i, n := range remain[:] {
n = n / (div * (i + 1))
// Allow all entries to be represented.
if n == 0 {
n = 1
}
huffBuff = append(huffBuff, bytes.Repeat([]byte{byte(i)}, n)...)
}
}
}
var out bytes.Buffer
out.Write([]byte(dictMagic))
out.Write(binary.LittleEndian.AppendUint32(nil, o.ID))
out.Write(scratch.OutTable)
if debug {
println("huff table:", len(scratch.OutTable), "bytes")
println("of table:", len(ofTable), "bytes")
println("ml table:", len(mlTable), "bytes")
println("ll table:", len(llTable), "bytes")
}
out.Write(ofTable)
out.Write(mlTable)
out.Write(llTable)
out.Write(binary.LittleEndian.AppendUint32(nil, uint32(o.Offsets[0])))
out.Write(binary.LittleEndian.AppendUint32(nil, uint32(o.Offsets[1])))
out.Write(binary.LittleEndian.AppendUint32(nil, uint32(o.Offsets[2])))
out.Write(hist)
if debug {
_, err := loadDict(out.Bytes())
if err != nil {
panic(err)
}
i, err := InspectDictionary(out.Bytes())
if err != nil {
panic(err)
}
println("ID:", i.ID())
println("Content size:", i.ContentSize())
println("Encoder:", i.LitEncoder() != nil)
println("Offsets:", i.Offsets())
var totalSize int
for _, b := range contents {
totalSize += len(b)
}
encWith := func(opts ...EOption) int {
enc, err := NewWriter(nil, opts...)
if err != nil {
panic(err)
}
defer enc.Close()
var dst []byte
var totalSize int
for _, b := range contents {
dst = enc.EncodeAll(b, dst[:0])
totalSize += len(dst)
}
return totalSize
}
plain := encWith(WithEncoderLevel(o.Level))
withDict := encWith(WithEncoderLevel(o.Level), WithEncoderDict(out.Bytes()))
println("Input size:", totalSize)
println("Plain Compressed:", plain)
println("Dict Compressed:", withDict)
println("Saved:", plain-withDict, (plain-withDict)/len(contents), "bytes per input (rounded down)")
}
return out.Bytes(), nil
}

View File

@ -197,12 +197,13 @@ encodeLoop:
// Set m to a match at offset if it looks like that will improve compression. // Set m to a match at offset if it looks like that will improve compression.
improve := func(m *match, offset int32, s int32, first uint32, rep int32) { improve := func(m *match, offset int32, s int32, first uint32, rep int32) {
if s-offset >= e.maxMatchOff || load3232(src, offset) != first { delta := s - offset
if delta >= e.maxMatchOff || delta <= 0 || load3232(src, offset) != first {
return return
} }
if debugAsserts { if debugAsserts {
if offset <= 0 { if offset >= s {
panic(offset) panic(fmt.Sprintf("offset: %d - s:%d - rep: %d - cur :%d - max: %d", offset, s, rep, e.cur, e.maxMatchOff))
} }
if !bytes.Equal(src[s:s+4], src[offset:offset+4]) { if !bytes.Equal(src[s:s+4], src[offset:offset+4]) {
panic(fmt.Sprintf("first match mismatch: %v != %v, first: %08x", src[s:s+4], src[offset:offset+4], first)) panic(fmt.Sprintf("first match mismatch: %v != %v, first: %08x", src[s:s+4], src[offset:offset+4], first))
@ -343,8 +344,8 @@ encodeLoop:
if best.rep > 0 { if best.rep > 0 {
var seq seq var seq seq
seq.matchLen = uint32(best.length - zstdMinMatch) seq.matchLen = uint32(best.length - zstdMinMatch)
if debugAsserts && s <= nextEmit { if debugAsserts && s < nextEmit {
panic("s <= nextEmit") panic("s < nextEmit")
} }
addLiterals(&seq, best.s) addLiterals(&seq, best.s)

View File

@ -227,10 +227,7 @@ func (e *Encoder) nextBlock(final bool) error {
DictID: e.o.dict.ID(), DictID: e.o.dict.ID(),
} }
dst, err := fh.appendTo(tmp[:0]) dst := fh.appendTo(tmp[:0])
if err != nil {
return err
}
s.headerWritten = true s.headerWritten = true
s.wWg.Wait() s.wWg.Wait()
var n2 int var n2 int
@ -483,7 +480,7 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte {
Checksum: false, Checksum: false,
DictID: 0, DictID: 0,
} }
dst, _ = fh.appendTo(dst) dst = fh.appendTo(dst)
// Write raw block as last one only. // Write raw block as last one only.
var blk blockHeader var blk blockHeader
@ -518,10 +515,7 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte {
if len(dst) == 0 && cap(dst) == 0 && len(src) < 1<<20 && !e.o.lowMem { if len(dst) == 0 && cap(dst) == 0 && len(src) < 1<<20 && !e.o.lowMem {
dst = make([]byte, 0, len(src)) dst = make([]byte, 0, len(src))
} }
dst, err := fh.appendTo(dst) dst = fh.appendTo(dst)
if err != nil {
panic(err)
}
// If we can do everything in one block, prefer that. // If we can do everything in one block, prefer that.
if len(src) <= e.o.blockSize { if len(src) <= e.o.blockSize {
@ -581,6 +575,7 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte {
// Add padding with content from crypto/rand.Reader // Add padding with content from crypto/rand.Reader
if e.o.pad > 0 { if e.o.pad > 0 {
add := calcSkippableFrame(int64(len(dst)), int64(e.o.pad)) add := calcSkippableFrame(int64(len(dst)), int64(e.o.pad))
var err error
dst, err = skippableFrame(dst, add, rand.Reader) dst, err = skippableFrame(dst, add, rand.Reader)
if err != nil { if err != nil {
panic(err) panic(err)

View File

@ -22,7 +22,7 @@ type frameHeader struct {
const maxHeaderSize = 14 const maxHeaderSize = 14
func (f frameHeader) appendTo(dst []byte) ([]byte, error) { func (f frameHeader) appendTo(dst []byte) []byte {
dst = append(dst, frameMagic...) dst = append(dst, frameMagic...)
var fhd uint8 var fhd uint8
if f.Checksum { if f.Checksum {
@ -88,7 +88,7 @@ func (f frameHeader) appendTo(dst []byte) ([]byte, error) {
default: default:
panic("invalid fcs") panic("invalid fcs")
} }
return dst, nil return dst
} }
const skippableFrameHeader = 4 + 4 const skippableFrameHeader = 4 + 4

View File

@ -245,7 +245,7 @@ func (s *sequenceDecs) decodeSync(hist []byte) error {
return io.ErrUnexpectedEOF return io.ErrUnexpectedEOF
} }
var ll, mo, ml int var ll, mo, ml int
if br.off > 4+((maxOffsetBits+16+16)>>3) { if len(br.in) > 4+((maxOffsetBits+16+16)>>3) {
// inlined function: // inlined function:
// ll, mo, ml = s.nextFast(br, llState, mlState, ofState) // ll, mo, ml = s.nextFast(br, llState, mlState, ofState)
@ -452,18 +452,13 @@ func (s *sequenceDecs) next(br *bitReader, llState, mlState, ofState decSymbol)
// extra bits are stored in reverse order. // extra bits are stored in reverse order.
br.fill() br.fill()
if s.maxBits <= 32 {
mo += br.getBits(moB)
ml += br.getBits(mlB)
ll += br.getBits(llB)
} else {
mo += br.getBits(moB) mo += br.getBits(moB)
if s.maxBits > 32 {
br.fill() br.fill()
}
// matchlength+literal length, max 32 bits // matchlength+literal length, max 32 bits
ml += br.getBits(mlB) ml += br.getBits(mlB)
ll += br.getBits(llB) ll += br.getBits(llB)
}
mo = s.adjustOffset(mo, ll, moB) mo = s.adjustOffset(mo, ll, moB)
return return
} }

View File

@ -5,11 +5,11 @@
// func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
// Requires: CMOV // Requires: CMOV
TEXT ·sequenceDecs_decode_amd64(SB), $8-32 TEXT ·sequenceDecs_decode_amd64(SB), $8-32
MOVQ br+8(FP), AX MOVQ br+8(FP), CX
MOVQ 32(AX), DX MOVQ 24(CX), DX
MOVBQZX 40(AX), BX MOVBQZX 32(CX), BX
MOVQ 24(AX), SI MOVQ (CX), AX
MOVQ (AX), AX MOVQ 8(CX), SI
ADDQ SI, AX ADDQ SI, AX
MOVQ AX, (SP) MOVQ AX, (SP)
MOVQ ctx+16(FP), AX MOVQ ctx+16(FP), AX
@ -301,9 +301,9 @@ sequenceDecs_decode_amd64_match_len_ofs_ok:
MOVQ R12, 152(AX) MOVQ R12, 152(AX)
MOVQ R13, 160(AX) MOVQ R13, 160(AX)
MOVQ br+8(FP), AX MOVQ br+8(FP), AX
MOVQ DX, 32(AX) MOVQ DX, 24(AX)
MOVB BL, 40(AX) MOVB BL, 32(AX)
MOVQ SI, 24(AX) MOVQ SI, 8(AX)
// Return success // Return success
MOVQ $0x00000000, ret+24(FP) MOVQ $0x00000000, ret+24(FP)
@ -336,11 +336,11 @@ error_overread:
// func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
// Requires: CMOV // Requires: CMOV
TEXT ·sequenceDecs_decode_56_amd64(SB), $8-32 TEXT ·sequenceDecs_decode_56_amd64(SB), $8-32
MOVQ br+8(FP), AX MOVQ br+8(FP), CX
MOVQ 32(AX), DX MOVQ 24(CX), DX
MOVBQZX 40(AX), BX MOVBQZX 32(CX), BX
MOVQ 24(AX), SI MOVQ (CX), AX
MOVQ (AX), AX MOVQ 8(CX), SI
ADDQ SI, AX ADDQ SI, AX
MOVQ AX, (SP) MOVQ AX, (SP)
MOVQ ctx+16(FP), AX MOVQ ctx+16(FP), AX
@ -603,9 +603,9 @@ sequenceDecs_decode_56_amd64_match_len_ofs_ok:
MOVQ R12, 152(AX) MOVQ R12, 152(AX)
MOVQ R13, 160(AX) MOVQ R13, 160(AX)
MOVQ br+8(FP), AX MOVQ br+8(FP), AX
MOVQ DX, 32(AX) MOVQ DX, 24(AX)
MOVB BL, 40(AX) MOVB BL, 32(AX)
MOVQ SI, 24(AX) MOVQ SI, 8(AX)
// Return success // Return success
MOVQ $0x00000000, ret+24(FP) MOVQ $0x00000000, ret+24(FP)
@ -638,11 +638,11 @@ error_overread:
// func sequenceDecs_decode_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // func sequenceDecs_decode_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
// Requires: BMI, BMI2, CMOV // Requires: BMI, BMI2, CMOV
TEXT ·sequenceDecs_decode_bmi2(SB), $8-32 TEXT ·sequenceDecs_decode_bmi2(SB), $8-32
MOVQ br+8(FP), CX MOVQ br+8(FP), BX
MOVQ 32(CX), AX MOVQ 24(BX), AX
MOVBQZX 40(CX), DX MOVBQZX 32(BX), DX
MOVQ 24(CX), BX MOVQ (BX), CX
MOVQ (CX), CX MOVQ 8(BX), BX
ADDQ BX, CX ADDQ BX, CX
MOVQ CX, (SP) MOVQ CX, (SP)
MOVQ ctx+16(FP), CX MOVQ ctx+16(FP), CX
@ -892,9 +892,9 @@ sequenceDecs_decode_bmi2_match_len_ofs_ok:
MOVQ R11, 152(CX) MOVQ R11, 152(CX)
MOVQ R12, 160(CX) MOVQ R12, 160(CX)
MOVQ br+8(FP), CX MOVQ br+8(FP), CX
MOVQ AX, 32(CX) MOVQ AX, 24(CX)
MOVB DL, 40(CX) MOVB DL, 32(CX)
MOVQ BX, 24(CX) MOVQ BX, 8(CX)
// Return success // Return success
MOVQ $0x00000000, ret+24(FP) MOVQ $0x00000000, ret+24(FP)
@ -927,11 +927,11 @@ error_overread:
// func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int // func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
// Requires: BMI, BMI2, CMOV // Requires: BMI, BMI2, CMOV
TEXT ·sequenceDecs_decode_56_bmi2(SB), $8-32 TEXT ·sequenceDecs_decode_56_bmi2(SB), $8-32
MOVQ br+8(FP), CX MOVQ br+8(FP), BX
MOVQ 32(CX), AX MOVQ 24(BX), AX
MOVBQZX 40(CX), DX MOVBQZX 32(BX), DX
MOVQ 24(CX), BX MOVQ (BX), CX
MOVQ (CX), CX MOVQ 8(BX), BX
ADDQ BX, CX ADDQ BX, CX
MOVQ CX, (SP) MOVQ CX, (SP)
MOVQ ctx+16(FP), CX MOVQ ctx+16(FP), CX
@ -1152,9 +1152,9 @@ sequenceDecs_decode_56_bmi2_match_len_ofs_ok:
MOVQ R11, 152(CX) MOVQ R11, 152(CX)
MOVQ R12, 160(CX) MOVQ R12, 160(CX)
MOVQ br+8(FP), CX MOVQ br+8(FP), CX
MOVQ AX, 32(CX) MOVQ AX, 24(CX)
MOVB DL, 40(CX) MOVB DL, 32(CX)
MOVQ BX, 24(CX) MOVQ BX, 8(CX)
// Return success // Return success
MOVQ $0x00000000, ret+24(FP) MOVQ $0x00000000, ret+24(FP)
@ -1797,11 +1797,11 @@ empty_seqs:
// func sequenceDecs_decodeSync_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // func sequenceDecs_decodeSync_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int
// Requires: CMOV, SSE // Requires: CMOV, SSE
TEXT ·sequenceDecs_decodeSync_amd64(SB), $64-32 TEXT ·sequenceDecs_decodeSync_amd64(SB), $64-32
MOVQ br+8(FP), AX MOVQ br+8(FP), CX
MOVQ 32(AX), DX MOVQ 24(CX), DX
MOVBQZX 40(AX), BX MOVBQZX 32(CX), BX
MOVQ 24(AX), SI MOVQ (CX), AX
MOVQ (AX), AX MOVQ 8(CX), SI
ADDQ SI, AX ADDQ SI, AX
MOVQ AX, (SP) MOVQ AX, (SP)
MOVQ ctx+16(FP), AX MOVQ ctx+16(FP), AX
@ -2295,9 +2295,9 @@ handle_loop:
loop_finished: loop_finished:
MOVQ br+8(FP), AX MOVQ br+8(FP), AX
MOVQ DX, 32(AX) MOVQ DX, 24(AX)
MOVB BL, 40(AX) MOVB BL, 32(AX)
MOVQ SI, 24(AX) MOVQ SI, 8(AX)
// Update the context // Update the context
MOVQ ctx+16(FP), AX MOVQ ctx+16(FP), AX
@ -2362,11 +2362,11 @@ error_not_enough_space:
// func sequenceDecs_decodeSync_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // func sequenceDecs_decodeSync_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int
// Requires: BMI, BMI2, CMOV, SSE // Requires: BMI, BMI2, CMOV, SSE
TEXT ·sequenceDecs_decodeSync_bmi2(SB), $64-32 TEXT ·sequenceDecs_decodeSync_bmi2(SB), $64-32
MOVQ br+8(FP), CX MOVQ br+8(FP), BX
MOVQ 32(CX), AX MOVQ 24(BX), AX
MOVBQZX 40(CX), DX MOVBQZX 32(BX), DX
MOVQ 24(CX), BX MOVQ (BX), CX
MOVQ (CX), CX MOVQ 8(BX), BX
ADDQ BX, CX ADDQ BX, CX
MOVQ CX, (SP) MOVQ CX, (SP)
MOVQ ctx+16(FP), CX MOVQ ctx+16(FP), CX
@ -2818,9 +2818,9 @@ handle_loop:
loop_finished: loop_finished:
MOVQ br+8(FP), CX MOVQ br+8(FP), CX
MOVQ AX, 32(CX) MOVQ AX, 24(CX)
MOVB DL, 40(CX) MOVB DL, 32(CX)
MOVQ BX, 24(CX) MOVQ BX, 8(CX)
// Update the context // Update the context
MOVQ ctx+16(FP), AX MOVQ ctx+16(FP), AX
@ -2885,11 +2885,11 @@ error_not_enough_space:
// func sequenceDecs_decodeSync_safe_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // func sequenceDecs_decodeSync_safe_amd64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int
// Requires: CMOV, SSE // Requires: CMOV, SSE
TEXT ·sequenceDecs_decodeSync_safe_amd64(SB), $64-32 TEXT ·sequenceDecs_decodeSync_safe_amd64(SB), $64-32
MOVQ br+8(FP), AX MOVQ br+8(FP), CX
MOVQ 32(AX), DX MOVQ 24(CX), DX
MOVBQZX 40(AX), BX MOVBQZX 32(CX), BX
MOVQ 24(AX), SI MOVQ (CX), AX
MOVQ (AX), AX MOVQ 8(CX), SI
ADDQ SI, AX ADDQ SI, AX
MOVQ AX, (SP) MOVQ AX, (SP)
MOVQ ctx+16(FP), AX MOVQ ctx+16(FP), AX
@ -3485,9 +3485,9 @@ handle_loop:
loop_finished: loop_finished:
MOVQ br+8(FP), AX MOVQ br+8(FP), AX
MOVQ DX, 32(AX) MOVQ DX, 24(AX)
MOVB BL, 40(AX) MOVB BL, 32(AX)
MOVQ SI, 24(AX) MOVQ SI, 8(AX)
// Update the context // Update the context
MOVQ ctx+16(FP), AX MOVQ ctx+16(FP), AX
@ -3552,11 +3552,11 @@ error_not_enough_space:
// func sequenceDecs_decodeSync_safe_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int // func sequenceDecs_decodeSync_safe_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int
// Requires: BMI, BMI2, CMOV, SSE // Requires: BMI, BMI2, CMOV, SSE
TEXT ·sequenceDecs_decodeSync_safe_bmi2(SB), $64-32 TEXT ·sequenceDecs_decodeSync_safe_bmi2(SB), $64-32
MOVQ br+8(FP), CX MOVQ br+8(FP), BX
MOVQ 32(CX), AX MOVQ 24(BX), AX
MOVBQZX 40(CX), DX MOVBQZX 32(BX), DX
MOVQ 24(CX), BX MOVQ (BX), CX
MOVQ (CX), CX MOVQ 8(BX), BX
ADDQ BX, CX ADDQ BX, CX
MOVQ CX, (SP) MOVQ CX, (SP)
MOVQ ctx+16(FP), CX MOVQ ctx+16(FP), CX
@ -4110,9 +4110,9 @@ handle_loop:
loop_finished: loop_finished:
MOVQ br+8(FP), CX MOVQ br+8(FP), CX
MOVQ AX, 32(CX) MOVQ AX, 24(CX)
MOVB DL, 40(CX) MOVB DL, 32(CX)
MOVQ BX, 24(CX) MOVQ BX, 8(CX)
// Update the context // Update the context
MOVQ ctx+16(FP), AX MOVQ ctx+16(FP), AX

View File

@ -29,7 +29,7 @@ func (s *sequenceDecs) decode(seqs []seqVals) error {
} }
for i := range seqs { for i := range seqs {
var ll, mo, ml int var ll, mo, ml int
if br.off > 4+((maxOffsetBits+16+16)>>3) { if len(br.in) > 4+((maxOffsetBits+16+16)>>3) {
// inlined function: // inlined function:
// ll, mo, ml = s.nextFast(br, llState, mlState, ofState) // ll, mo, ml = s.nextFast(br, llState, mlState, ofState)

View File

@ -95,10 +95,9 @@ func (r *SnappyConverter) Convert(in io.Reader, w io.Writer) (int64, error) {
var written int64 var written int64
var readHeader bool var readHeader bool
{ {
var header []byte header := frameHeader{WindowSize: snappyMaxBlockSize}.appendTo(r.buf[:0])
var n int
header, r.err = frameHeader{WindowSize: snappyMaxBlockSize}.appendTo(r.buf[:0])
var n int
n, r.err = w.Write(header) n, r.err = w.Write(header)
if r.err != nil { if r.err != nil {
return written, r.err return written, r.err

4
vendor/modules.txt vendored
View File

@ -231,7 +231,7 @@ github.com/docker/distribution/reference
github.com/docker/distribution/registry/api/errcode github.com/docker/distribution/registry/api/errcode
github.com/docker/distribution/registry/api/v2 github.com/docker/distribution/registry/api/v2
github.com/docker/distribution/registry/client/auth/challenge github.com/docker/distribution/registry/client/auth/challenge
# github.com/docker/docker v24.0.6+incompatible # github.com/docker/docker v24.0.7+incompatible
## explicit ## explicit
github.com/docker/docker/api github.com/docker/docker/api
github.com/docker/docker/api/types github.com/docker/docker/api/types
@ -371,7 +371,7 @@ github.com/josharian/intern
# github.com/json-iterator/go v1.1.12 # github.com/json-iterator/go v1.1.12
## explicit; go 1.12 ## explicit; go 1.12
github.com/json-iterator/go github.com/json-iterator/go
# github.com/klauspost/compress v1.16.7 # github.com/klauspost/compress v1.17.2
## explicit; go 1.18 ## explicit; go 1.18
github.com/klauspost/compress github.com/klauspost/compress
github.com/klauspost/compress/flate github.com/klauspost/compress/flate