github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-23 03:41:45 +00:00
Code Issues Projects Releases Wiki Activity

fix bug when using big yaml on the scanner

Browse Source
This commit is contained in:
tanjunchen 2020-04-09 10:08:04 +08:00
parent 4bdb72ed11
commit 11804c34b6
2 changed files with 31 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder.go
View File

@ -92,6 +92,10 @@ type YAMLDecoder struct {
// the caller in framing the chunk.
func NewDocumentDecoder(r io.ReadCloser) io.ReadCloser {
scanner := bufio.NewScanner(r)
// the size of initial allocation for buffer 4k
buf := make([]byte, 4*1024)
// the maximum size used to buffer a token 5M
scanner.Buffer(buf, 5*1024*1024)
scanner.Split(splitYAMLDocument)
return &YAMLDecoder{
r: r,

27
staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder_test.go
View File

@ -54,6 +54,33 @@ stuff: 1
}
}
func TestBigYAML(t *testing.T) {
d := `
stuff: 1
`
maxLen := 5 * 1024 * 1024
bufferLen := 4 * 1024
// maxLen 5 M
dd := strings.Repeat(d, 512*1024)
r := NewDocumentDecoder(ioutil.NopCloser(bytes.NewReader([]byte(dd[:maxLen-1]))))
b := make([]byte, bufferLen)
n, err := r.Read(b)
if err != io.ErrShortBuffer {
t.Fatalf("expected ErrShortBuffer: %d / %v", n, err)
}
b = make([]byte, maxLen)
n, err = r.Read(b)
if err != nil {
t.Fatalf("expected nil: %d / %v", n, err)
}
r = NewDocumentDecoder(ioutil.NopCloser(bytes.NewReader([]byte(dd))))
b = make([]byte, maxLen)
n, err = r.Read(b)
if err != bufio.ErrTooLong {
t.Fatalf("bufio.Scanner: token too long: %d / %v", n, err)
}
}
func TestYAMLDecoderCallsAfterErrShortBufferRestOfFrame(t *testing.T) {
d := `---
stuff: 1