From 9f288610eba61dd7ea0eb85e6d8b755b2bccf92b Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Sun, 31 Jan 2021 00:19:10 -0500 Subject: [PATCH] serializer: Recognizer interface should take a buffer The original intent of the Recognizer interface was to take a stream and peek into the contents. Over time all of the callers migrated to instead performing their own peek and then passing the buffer as a bytes.NewBuffer to the interface as a stream, which was then implemented as a read on the other side. This resulted in an excess buffer allocation in general decoder usage. Update the interface to take a []bytes slice and change all callers to stop creating buffered readers, then update the JSON serializer to check the buffer directly for "json-ness". --- .../pkg/runtime/serializer/json/json.go | 5 ++--- .../pkg/runtime/serializer/protobuf/protobuf.go | 17 +++-------------- .../runtime/serializer/recognizer/recognizer.go | 13 ++++--------- ...runtime_serializer_protobuf_protobuf_test.go | 6 +++--- .../apimachinery/pkg/util/yaml/decoder.go | 6 ++++++ 5 files changed, 18 insertions(+), 29 deletions(-) diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json.go b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json.go index 83b2e139311..48f0777b24e 100644 --- a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json.go +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json.go @@ -333,13 +333,12 @@ func (s *Serializer) Identifier() runtime.Identifier { } // RecognizesData implements the RecognizingDecoder interface. -func (s *Serializer) RecognizesData(peek io.Reader) (ok, unknown bool, err error) { +func (s *Serializer) RecognizesData(data []byte) (ok, unknown bool, err error) { if s.options.Yaml { // we could potentially look for '---' return false, true, nil } - _, _, ok = utilyaml.GuessJSONStream(peek, 2048) - return ok, false, nil + return utilyaml.IsJSONBuffer(data), false, nil } // Framer is the default JSON framing behavior, with newlines delimiting individual objects. diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/protobuf/protobuf.go b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/protobuf/protobuf.go index 404fb1b7e5f..8358d77c39e 100644 --- a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/protobuf/protobuf.go +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/protobuf/protobuf.go @@ -120,7 +120,7 @@ func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, i if intoUnknown, ok := into.(*runtime.Unknown); ok && intoUnknown != nil { *intoUnknown = unk - if ok, _, _ := s.RecognizesData(bytes.NewBuffer(unk.Raw)); ok { + if ok, _, _ := s.RecognizesData(unk.Raw); ok { intoUnknown.ContentType = runtime.ContentTypeProtobuf } return intoUnknown, &actual, nil @@ -245,19 +245,8 @@ func (s *Serializer) Identifier() runtime.Identifier { } // RecognizesData implements the RecognizingDecoder interface. -func (s *Serializer) RecognizesData(peek io.Reader) (bool, bool, error) { - prefix := make([]byte, 4) - n, err := peek.Read(prefix) - if err != nil { - if err == io.EOF { - return false, false, nil - } - return false, false, err - } - if n != 4 { - return false, false, nil - } - return bytes.Equal(s.prefix, prefix), false, nil +func (s *Serializer) RecognizesData(data []byte) (bool, bool, error) { + return bytes.HasPrefix(data, s.prefix), false, nil } // copyKindDefaults defaults dst to the value in src if dst does not have a value set. diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/recognizer/recognizer.go b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/recognizer/recognizer.go index 38497ab533e..709f8529115 100644 --- a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/recognizer/recognizer.go +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/recognizer/recognizer.go @@ -17,10 +17,7 @@ limitations under the License. package recognizer import ( - "bufio" - "bytes" "fmt" - "io" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -35,7 +32,7 @@ type RecognizingDecoder interface { // provides) and may return unknown if the data provided is not sufficient to make a // a determination. When peek returns EOF that may mean the end of the input or the // end of buffered input - recognizers should return the best guess at that time. - RecognizesData(peek io.Reader) (ok, unknown bool, err error) + RecognizesData(peek []byte) (ok, unknown bool, err error) } // NewDecoder creates a decoder that will attempt multiple decoders in an order defined @@ -57,16 +54,15 @@ type decoder struct { var _ RecognizingDecoder = &decoder{} -func (d *decoder) RecognizesData(peek io.Reader) (bool, bool, error) { +func (d *decoder) RecognizesData(data []byte) (bool, bool, error) { var ( lastErr error anyUnknown bool ) - data, _ := bufio.NewReaderSize(peek, 1024).Peek(1024) for _, r := range d.decoders { switch t := r.(type) { case RecognizingDecoder: - ok, unknown, err := t.RecognizesData(bytes.NewBuffer(data)) + ok, unknown, err := t.RecognizesData(data) if err != nil { lastErr = err continue @@ -91,8 +87,7 @@ func (d *decoder) Decode(data []byte, gvk *schema.GroupVersionKind, into runtime for _, r := range d.decoders { switch t := r.(type) { case RecognizingDecoder: - buf := bytes.NewBuffer(data) - ok, unknown, err := t.RecognizesData(buf) + ok, unknown, err := t.RecognizesData(data) if err != nil { lastErr = err continue diff --git a/staging/src/k8s.io/apimachinery/pkg/test/runtime_serializer_protobuf_protobuf_test.go b/staging/src/k8s.io/apimachinery/pkg/test/runtime_serializer_protobuf_protobuf_test.go index 5349031aeba..17bcfc17a8a 100644 --- a/staging/src/k8s.io/apimachinery/pkg/test/runtime_serializer_protobuf_protobuf_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/test/runtime_serializer_protobuf_protobuf_test.go @@ -92,7 +92,7 @@ func TestRecognize(t *testing.T) { {0x6b, 0x38, 0x73, 0x01}, } for i, data := range ignores { - if ok, _, err := s.RecognizesData(bytes.NewBuffer(data)); err != nil || ok { + if ok, _, err := s.RecognizesData(data); err != nil || ok { t.Errorf("%d: should not recognize data: %v", i, err) } } @@ -101,7 +101,7 @@ func TestRecognize(t *testing.T) { {0x6b, 0x38, 0x73, 0x00, 0x01}, } for i, data := range recognizes { - if ok, _, err := s.RecognizesData(bytes.NewBuffer(data)); err != nil || !ok { + if ok, _, err := s.RecognizesData(data); err != nil || !ok { t.Errorf("%d: should recognize data: %v", i, err) } } @@ -197,7 +197,7 @@ func TestEncode(t *testing.T) { continue } - if ok, _, err := s.RecognizesData(bytes.NewBuffer(data)); !ok || err != nil { + if ok, _, err := s.RecognizesData(data); !ok || err != nil { t.Errorf("%d: did not recognize data generated by call: %v", i, err) } } diff --git a/staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder.go b/staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder.go index 5b218c98ebe..612d63a6948 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder.go @@ -343,6 +343,12 @@ func GuessJSONStream(r io.Reader, size int) (io.Reader, []byte, bool) { return buffer, b, hasJSONPrefix(b) } +// IsJSONBuffer scans the provided buffer, looking +// for an open brace indicating this is JSON. +func IsJSONBuffer(buf []byte) bool { + return hasJSONPrefix(buf) +} + var jsonPrefix = []byte("{") // hasJSONPrefix returns true if the provided buffer appears to start with