Switch from json-iterator to utiljson

This commit is contained in:
Jordan Liggitt 2021-09-14 17:54:37 -04:00
parent d5de03f0d3
commit bba877d3a6
19 changed files with 86 additions and 331 deletions

View File

@ -1133,7 +1133,7 @@ profiles:
ConfigFile: unknownFieldConfig, ConfigFile: unknownFieldConfig,
Logs: logs.NewOptions(), Logs: logs.NewOptions(),
}, },
expectedError: "found unknown field: foo", expectedError: `unknown field "foo"`,
checkErrFn: runtime.IsStrictDecodingError, checkErrFn: runtime.IsStrictDecodingError,
}, },
{ {

View File

@ -26,7 +26,6 @@ import (
"testing" "testing"
"github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp"
jsoniter "github.com/json-iterator/go"
appsv1 "k8s.io/api/apps/v1" appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
@ -38,7 +37,6 @@ import (
"k8s.io/apimachinery/pkg/conversion" "k8s.io/apimachinery/pkg/conversion"
"k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer/json"
"k8s.io/apimachinery/pkg/runtime/serializer/streaming" "k8s.io/apimachinery/pkg/runtime/serializer/streaming"
"k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/watch" "k8s.io/apimachinery/pkg/watch"
@ -589,59 +587,6 @@ func BenchmarkDecodeIntoJSON(b *testing.B) {
b.StopTimer() b.StopTimer()
} }
// BenchmarkDecodeIntoJSONCodecGenConfigFast provides a baseline
// for JSON decode performance with jsoniter.ConfigFast
func BenchmarkDecodeIntoJSONCodecGenConfigFast(b *testing.B) {
kcodec := legacyscheme.Codecs.LegacyCodec(v1.SchemeGroupVersion)
items := benchmarkItems(b)
width := len(items)
encoded := make([][]byte, width)
for i := range items {
data, err := runtime.Encode(kcodec, &items[i])
if err != nil {
b.Fatal(err)
}
encoded[i] = data
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
obj := v1.Pod{}
if err := jsoniter.ConfigFastest.Unmarshal(encoded[i%width], &obj); err != nil {
b.Fatal(err)
}
}
b.StopTimer()
}
// BenchmarkDecodeIntoJSONCodecGenConfigCompatibleWithStandardLibrary provides a
// baseline for JSON decode performance with
// jsoniter.ConfigCompatibleWithStandardLibrary, but with case sensitivity set
// to true
func BenchmarkDecodeIntoJSONCodecGenConfigCompatibleWithStandardLibrary(b *testing.B) {
kcodec := legacyscheme.Codecs.LegacyCodec(v1.SchemeGroupVersion)
items := benchmarkItems(b)
width := len(items)
encoded := make([][]byte, width)
for i := range items {
data, err := runtime.Encode(kcodec, &items[i])
if err != nil {
b.Fatal(err)
}
encoded[i] = data
}
b.ResetTimer()
iter := json.CaseSensitiveJSONIterator()
for i := 0; i < b.N; i++ {
obj := v1.Pod{}
if err := iter.Unmarshal(encoded[i%width], &obj); err != nil {
b.Fatal(err)
}
}
b.StopTimer()
}
// BenchmarkEncodeYAMLMarshal provides a baseline for regular YAML encode performance // BenchmarkEncodeYAMLMarshal provides a baseline for regular YAML encode performance
func BenchmarkEncodeYAMLMarshal(b *testing.B) { func BenchmarkEncodeYAMLMarshal(b *testing.B) {
items := benchmarkItems(b) items := benchmarkItems(b)

View File

@ -19,12 +19,10 @@ package testing
import ( import (
"math/rand" "math/rand"
"reflect" "reflect"
"sort"
"testing" "testing"
"github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp"
fuzz "github.com/google/gofuzz" fuzz "github.com/google/gofuzz"
jsoniter "github.com/json-iterator/go"
v1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/apitesting/fuzzer" "k8s.io/apimachinery/pkg/api/apitesting/fuzzer"
@ -260,75 +258,3 @@ func BenchmarkFromUnstructuredViaJSON(b *testing.B) {
} }
b.StopTimer() b.StopTimer()
} }
func BenchmarkToUnstructuredViaJSONIter(b *testing.B) {
items := benchmarkItems(b)
size := len(items)
var keys []string
for k := range jsonIterConfig {
keys = append(keys, k)
}
sort.Strings(keys)
for _, name := range keys {
c := jsonIterConfig[name]
b.Run(name, func(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
data, err := c.Marshal(&items[i%size])
if err != nil {
b.Fatalf("unexpected error: %v", err)
}
unstr := map[string]interface{}{}
if err := c.Unmarshal(data, &unstr); err != nil {
b.Fatalf("unexpected error: %v", err)
}
}
b.StopTimer()
})
}
}
var jsonIterConfig = map[string]jsoniter.API{
"default": jsoniter.ConfigDefault,
"fastest": jsoniter.ConfigFastest,
"compat": jsoniter.ConfigCompatibleWithStandardLibrary,
}
func BenchmarkFromUnstructuredViaJSONIter(b *testing.B) {
items := benchmarkItems(b)
var unstr []map[string]interface{}
for i := range items {
data, err := jsoniter.Marshal(&items[i])
if err != nil {
b.Fatalf("unexpected error: %v", err)
}
item := map[string]interface{}{}
if err := json.Unmarshal(data, &item); err != nil {
b.Fatalf("unexpected error: %v", err)
}
unstr = append(unstr, item)
}
size := len(items)
var keys []string
for k := range jsonIterConfig {
keys = append(keys, k)
}
sort.Strings(keys)
for _, name := range keys {
c := jsonIterConfig[name]
b.Run(name, func(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
item, err := c.Marshal(unstr[i%size])
if err != nil {
b.Fatalf("unexpected error: %v", err)
}
obj := v1.Pod{}
if err := c.Unmarshal(item, &obj); err != nil {
b.Fatalf("unexpected error: %v", err)
}
}
b.StopTimer()
})
}
}

View File

@ -263,7 +263,7 @@ profiles:
- Score: 2 - Score: 2
Utilization: 1 Utilization: 1
`), `),
wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoder error for {"scoringStrategy":{"requestedToCapacityRatio":{"shape":[{"Score":2,"Utilization":1}]}}}: v1beta2.NodeResourcesFitArgs.ScoringStrategy: v1beta2.ScoringStrategy.RequestedToCapacityRatio: v1beta2.RequestedToCapacityRatioParam.Shape: []v1beta2.UtilizationShapePoint: v1beta2.UtilizationShapePoint.ReadObject: found unknown field: Score, error found in #10 byte of ...|:[{"Score":2,"Utiliz|..., bigger context ...|gy":{"requestedToCapacityRatio":{"shape":[{"Score":2,"Utilization":1}]}}}|...`, wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoding error: unknown field "Score", unknown field "Utilization"`,
}, },
{ {
name: "v1beta2 NodeResourcesFitArgs resources encoding is strict", name: "v1beta2 NodeResourcesFitArgs resources encoding is strict",
@ -279,7 +279,7 @@ profiles:
- Name: cpu - Name: cpu
Weight: 1 Weight: 1
`), `),
wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoder error for {"scoringStrategy":{"resources":[{"Name":"cpu","Weight":1}]}}: v1beta2.NodeResourcesFitArgs.ScoringStrategy: v1beta2.ScoringStrategy.Resources: []v1beta2.ResourceSpec: v1beta2.ResourceSpec.ReadObject: found unknown field: Name, error found in #10 byte of ...|":[{"Name":"cpu","We|..., bigger context ...|{"scoringStrategy":{"resources":[{"Name":"cpu","Weight":1}]}}|...`, wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoding error: unknown field "Name", unknown field "Weight"`,
}, },
{ {
name: "out-of-tree plugin args", name: "out-of-tree plugin args",
@ -604,7 +604,7 @@ profiles:
- Score: 2 - Score: 2
Utilization: 1 Utilization: 1
`), `),
wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoder error for {"scoringStrategy":{"requestedToCapacityRatio":{"shape":[{"Score":2,"Utilization":1}]}}}: v1beta3.NodeResourcesFitArgs.ScoringStrategy: v1beta3.ScoringStrategy.RequestedToCapacityRatio: v1beta3.RequestedToCapacityRatioParam.Shape: []v1beta3.UtilizationShapePoint: v1beta3.UtilizationShapePoint.ReadObject: found unknown field: Score, error found in #10 byte of ...|:[{"Score":2,"Utiliz|..., bigger context ...|gy":{"requestedToCapacityRatio":{"shape":[{"Score":2,"Utilization":1}]}}}|...`, wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoding error: unknown field "Score", unknown field "Utilization"`,
}, },
{ {
name: "v1beta3 NodeResourcesFitArgs resources encoding is strict", name: "v1beta3 NodeResourcesFitArgs resources encoding is strict",
@ -620,7 +620,7 @@ profiles:
- Name: cpu - Name: cpu
Weight: 1 Weight: 1
`), `),
wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoder error for {"scoringStrategy":{"resources":[{"Name":"cpu","Weight":1}]}}: v1beta3.NodeResourcesFitArgs.ScoringStrategy: v1beta3.ScoringStrategy.Resources: []v1beta3.ResourceSpec: v1beta3.ResourceSpec.ReadObject: found unknown field: Name, error found in #10 byte of ...|":[{"Name":"cpu","We|..., bigger context ...|{"scoringStrategy":{"resources":[{"Name":"cpu","Weight":1}]}}|...`, wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoding error: unknown field "Name", unknown field "Weight"`,
}, },
{ {
name: "out-of-tree plugin args", name: "out-of-tree plugin args",

View File

@ -22,11 +22,9 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer/json" utiljson "k8s.io/apimachinery/pkg/util/json"
) )
var encodingjson = json.CaseSensitiveJSONIterator()
// GetObjectMeta does conversion of JSON to ObjectMeta. It first tries json.Unmarshal into a metav1.ObjectMeta // GetObjectMeta does conversion of JSON to ObjectMeta. It first tries json.Unmarshal into a metav1.ObjectMeta
// type. If that does not work and dropMalformedFields is true, it does field-by-field best-effort conversion // type. If that does not work and dropMalformedFields is true, it does field-by-field best-effort conversion
// throwing away fields which lead to errors. // throwing away fields which lead to errors.
@ -38,11 +36,11 @@ func GetObjectMeta(obj map[string]interface{}, dropMalformedFields bool) (*metav
// round-trip through JSON first, hoping that unmarshalling just works // round-trip through JSON first, hoping that unmarshalling just works
objectMeta := &metav1.ObjectMeta{} objectMeta := &metav1.ObjectMeta{}
metadataBytes, err := encodingjson.Marshal(metadata) metadataBytes, err := utiljson.Marshal(metadata)
if err != nil { if err != nil {
return nil, false, err return nil, false, err
} }
if err = encodingjson.Unmarshal(metadataBytes, objectMeta); err == nil { if err = utiljson.Unmarshal(metadataBytes, objectMeta); err == nil {
// if successful, return // if successful, return
return objectMeta, true, nil return objectMeta, true, nil
} }
@ -63,11 +61,11 @@ func GetObjectMeta(obj map[string]interface{}, dropMalformedFields bool) (*metav
testObjectMeta := &metav1.ObjectMeta{} testObjectMeta := &metav1.ObjectMeta{}
for k, v := range metadataMap { for k, v := range metadataMap {
// serialize a single field // serialize a single field
if singleFieldBytes, err := encodingjson.Marshal(map[string]interface{}{k: v}); err == nil { if singleFieldBytes, err := utiljson.Marshal(map[string]interface{}{k: v}); err == nil {
// do a test unmarshal // do a test unmarshal
if encodingjson.Unmarshal(singleFieldBytes, testObjectMeta) == nil { if utiljson.Unmarshal(singleFieldBytes, testObjectMeta) == nil {
// if that succeeds, unmarshal for real // if that succeeds, unmarshal for real
encodingjson.Unmarshal(singleFieldBytes, accumulatedObjectMeta) utiljson.Unmarshal(singleFieldBytes, accumulatedObjectMeta)
} }
} }
} }

View File

@ -31,6 +31,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/runtime/serializer/json" "k8s.io/apimachinery/pkg/runtime/serializer/json"
"k8s.io/apimachinery/pkg/util/diff" "k8s.io/apimachinery/pkg/util/diff"
utiljson "k8s.io/apimachinery/pkg/util/json"
clientgoscheme "k8s.io/client-go/kubernetes/scheme" clientgoscheme "k8s.io/client-go/kubernetes/scheme"
) )
@ -114,12 +115,12 @@ func TestMalformedObjectMetaFields(t *testing.T) {
} }
// See if it can unmarshal to object meta // See if it can unmarshal to object meta
spuriousJSON, err := encodingjson.Marshal(spuriousMetaMap) spuriousJSON, err := utiljson.Marshal(spuriousMetaMap)
if err != nil { if err != nil {
t.Fatalf("error on %v=%#v: %v", pth, v, err) t.Fatalf("error on %v=%#v: %v", pth, v, err)
} }
expectedObjectMeta := &metav1.ObjectMeta{} expectedObjectMeta := &metav1.ObjectMeta{}
if err := encodingjson.Unmarshal(spuriousJSON, expectedObjectMeta); err != nil { if err := utiljson.Unmarshal(spuriousJSON, expectedObjectMeta); err != nil {
// if standard json unmarshal would fail decoding this field, drop the field entirely // if standard json unmarshal would fail decoding this field, drop the field entirely
truncatedMetaMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(fuzzedObjectMeta.DeepCopy()) truncatedMetaMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(fuzzedObjectMeta.DeepCopy())
if err != nil { if err != nil {
@ -133,12 +134,12 @@ func TestMalformedObjectMetaFields(t *testing.T) {
DeleteJSONPath(truncatedMetaMap, pth[:1], 0) DeleteJSONPath(truncatedMetaMap, pth[:1], 0)
} }
truncatedJSON, err := encodingjson.Marshal(truncatedMetaMap) truncatedJSON, err := utiljson.Marshal(truncatedMetaMap)
if err != nil { if err != nil {
t.Fatalf("error on %v=%#v: %v", pth, v, err) t.Fatalf("error on %v=%#v: %v", pth, v, err)
} }
expectedObjectMeta = &metav1.ObjectMeta{} expectedObjectMeta = &metav1.ObjectMeta{}
if err := encodingjson.Unmarshal(truncatedJSON, expectedObjectMeta); err != nil { if err := utiljson.Unmarshal(truncatedJSON, expectedObjectMeta); err != nil {
t.Fatalf("error on %v=%#v: %v", pth, v, err) t.Fatalf("error on %v=%#v: %v", pth, v, err)
} }
} }
@ -190,7 +191,7 @@ func TestGetObjectMetaNils(t *testing.T) {
} }
// double check this what the kube JSON decode is doing // double check this what the kube JSON decode is doing
bs, _ := encodingjson.Marshal(u.UnstructuredContent()) bs, _ := utiljson.Marshal(u.UnstructuredContent())
kubeObj, _, err := clientgoscheme.Codecs.UniversalDecoder(corev1.SchemeGroupVersion).Decode(bs, nil, nil) kubeObj, _, err := clientgoscheme.Codecs.UniversalDecoder(corev1.SchemeGroupVersion).Decode(bs, nil, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)

View File

@ -21,7 +21,7 @@ import (
"reflect" "reflect"
"testing" "testing"
"k8s.io/apimachinery/pkg/runtime/serializer/json" utiljson "k8s.io/apimachinery/pkg/util/json"
) )
type GroupVersionHolder struct { type GroupVersionHolder struct {
@ -46,13 +46,12 @@ func TestGroupVersionUnmarshalJSON(t *testing.T) {
if !reflect.DeepEqual(result.GV, c.expect) { if !reflect.DeepEqual(result.GV, c.expect) {
t.Errorf("JSON codec failed to unmarshal input '%s': expected %+v, got %+v", c.input, c.expect, result.GV) t.Errorf("JSON codec failed to unmarshal input '%s': expected %+v, got %+v", c.input, c.expect, result.GV)
} }
// test the json-iterator codec // test the utiljson codec
iter := json.CaseSensitiveJSONIterator() if err := utiljson.Unmarshal(c.input, &result); err != nil {
if err := iter.Unmarshal(c.input, &result); err != nil { t.Errorf("util/json codec failed to unmarshal input '%v': %v", c.input, err)
t.Errorf("json-iterator codec failed to unmarshal input '%v': %v", c.input, err)
} }
if !reflect.DeepEqual(result.GV, c.expect) { if !reflect.DeepEqual(result.GV, c.expect) {
t.Errorf("json-iterator codec failed to unmarshal input '%s': expected %+v, got %+v", c.input, c.expect, result.GV) t.Errorf("util/json codec failed to unmarshal input '%s': expected %+v, got %+v", c.input, c.expect, result.GV)
} }
} }
} }

View File

@ -21,7 +21,7 @@ import (
"reflect" "reflect"
"testing" "testing"
"k8s.io/apimachinery/pkg/runtime/serializer/json" utiljson "k8s.io/apimachinery/pkg/util/json"
) )
func TestVerbsMarshalJSON(t *testing.T) { func TestVerbsMarshalJSON(t *testing.T) {
@ -56,10 +56,9 @@ func TestVerbsJsonIterUnmarshalJSON(t *testing.T) {
{`{"verbs":["delete"]}`, APIResource{Verbs: Verbs([]string{"delete"})}}, {`{"verbs":["delete"]}`, APIResource{Verbs: Verbs([]string{"delete"})}},
} }
iter := json.CaseSensitiveJSONIterator()
for i, c := range cases { for i, c := range cases {
var result APIResource var result APIResource
if err := iter.Unmarshal([]byte(c.input), &result); err != nil { if err := utiljson.Unmarshal([]byte(c.input), &result); err != nil {
t.Errorf("[%d] Failed to unmarshal input '%v': %v", i, c.input, err) t.Errorf("[%d] Failed to unmarshal input '%v': %v", i, c.input, err)
} }
if !reflect.DeepEqual(result, c.result) { if !reflect.DeepEqual(result, c.result) {

View File

@ -274,7 +274,7 @@ func TestRoundTrip(t *testing.T) {
{ {
// Test slice of interface{} with different values. // Test slice of interface{} with different values.
obj: &D{ obj: &D{
A: []interface{}{3.0, "3.0", nil}, A: []interface{}{float64(3.5), int64(4), "3.0", nil},
}, },
}, },
} }
@ -322,11 +322,11 @@ func TestUnrecognized(t *testing.T) {
err error err error
}{ }{
{ {
data: "{\"da\":[3.0,\"3.0\",null]}", data: "{\"da\":[3.5,4,\"3.0\",null]}",
obj: &D{}, obj: &D{},
}, },
{ {
data: "{\"ea\":[3.0,\"3.0\",null]}", data: "{\"ea\":[3.5,4,\"3.0\",null]}",
obj: &E{}, obj: &E{},
}, },
{ {

View File

@ -19,6 +19,7 @@ package runtime
import ( import (
"fmt" "fmt"
"reflect" "reflect"
"strings"
"k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/schema"
) )
@ -124,20 +125,26 @@ func IsMissingVersion(err error) bool {
// strictDecodingError is a base error type that is returned by a strict Decoder such // strictDecodingError is a base error type that is returned by a strict Decoder such
// as UniversalStrictDecoder. // as UniversalStrictDecoder.
type strictDecodingError struct { type strictDecodingError struct {
message string errors []error
data string
} }
// NewStrictDecodingError creates a new strictDecodingError object. // NewStrictDecodingError creates a new strictDecodingError object.
func NewStrictDecodingError(message string, data string) error { func NewStrictDecodingError(errors []error) error {
return &strictDecodingError{ return &strictDecodingError{
message: message, errors: errors,
data: data,
} }
} }
func (e *strictDecodingError) Error() string { func (e *strictDecodingError) Error() string {
return fmt.Sprintf("strict decoder error for %s: %s", e.data, e.message) var s strings.Builder
s.WriteString("strict decoding error: ")
for i, err := range e.errors {
if i != 0 {
s.WriteString(", ")
}
s.WriteString(err.Error())
}
return s.String()
} }
// IsStrictDecodingError returns true if the error indicates that the provided object // IsStrictDecodingError returns true if the error indicates that the provided object

View File

@ -20,10 +20,8 @@ import (
"encoding/json" "encoding/json"
"io" "io"
"strconv" "strconv"
"unsafe"
jsoniter "github.com/json-iterator/go" kjson "sigs.k8s.io/json"
"github.com/modern-go/reflect2"
"sigs.k8s.io/yaml" "sigs.k8s.io/yaml"
"k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime"
@ -110,79 +108,6 @@ type Serializer struct {
var _ runtime.Serializer = &Serializer{} var _ runtime.Serializer = &Serializer{}
var _ recognizer.RecognizingDecoder = &Serializer{} var _ recognizer.RecognizingDecoder = &Serializer{}
type customNumberExtension struct {
jsoniter.DummyExtension
}
func (cne *customNumberExtension) CreateDecoder(typ reflect2.Type) jsoniter.ValDecoder {
if typ.String() == "interface {}" {
return customNumberDecoder{}
}
return nil
}
type customNumberDecoder struct {
}
func (customNumberDecoder) Decode(ptr unsafe.Pointer, iter *jsoniter.Iterator) {
switch iter.WhatIsNext() {
case jsoniter.NumberValue:
var number jsoniter.Number
iter.ReadVal(&number)
i64, err := strconv.ParseInt(string(number), 10, 64)
if err == nil {
*(*interface{})(ptr) = i64
return
}
f64, err := strconv.ParseFloat(string(number), 64)
if err == nil {
*(*interface{})(ptr) = f64
return
}
iter.ReportError("DecodeNumber", err.Error())
default:
*(*interface{})(ptr) = iter.Read()
}
}
// CaseSensitiveJSONIterator returns a jsoniterator API that's configured to be
// case-sensitive when unmarshalling, and otherwise compatible with
// the encoding/json standard library.
func CaseSensitiveJSONIterator() jsoniter.API {
config := jsoniter.Config{
EscapeHTML: true,
SortMapKeys: true,
ValidateJsonRawMessage: true,
CaseSensitive: true,
}.Froze()
// Force jsoniter to decode number to interface{} via int64/float64, if possible.
config.RegisterExtension(&customNumberExtension{})
return config
}
// StrictCaseSensitiveJSONIterator returns a jsoniterator API that's configured to be
// case-sensitive, but also disallows unknown fields when unmarshalling. It is compatible with
// the encoding/json standard library.
func StrictCaseSensitiveJSONIterator() jsoniter.API {
config := jsoniter.Config{
EscapeHTML: true,
SortMapKeys: true,
ValidateJsonRawMessage: true,
CaseSensitive: true,
DisallowUnknownFields: true,
}.Froze()
// Force jsoniter to decode number to interface{} via int64/float64, if possible.
config.RegisterExtension(&customNumberExtension{})
return config
}
// Private copies of jsoniter to try to shield against possible mutations
// from outside. Still does not protect from package level jsoniter.Register*() functions - someone calling them
// in some other library will mess with every usage of the jsoniter library in the whole program.
// See https://github.com/json-iterator/go/issues/265
var caseSensitiveJSONIterator = CaseSensitiveJSONIterator()
var strictCaseSensitiveJSONIterator = StrictCaseSensitiveJSONIterator()
// gvkWithDefaults returns group kind and version defaulting from provided default // gvkWithDefaults returns group kind and version defaulting from provided default
func gvkWithDefaults(actual, defaultGVK schema.GroupVersionKind) schema.GroupVersionKind { func gvkWithDefaults(actual, defaultGVK schema.GroupVersionKind) schema.GroupVersionKind {
if len(actual.Kind) == 0 { if len(actual.Kind) == 0 {
@ -237,7 +162,7 @@ func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, i
types, _, err := s.typer.ObjectKinds(into) types, _, err := s.typer.ObjectKinds(into)
switch { switch {
case runtime.IsNotRegisteredError(err), isUnstructured: case runtime.IsNotRegisteredError(err), isUnstructured:
if err := caseSensitiveJSONIterator.Unmarshal(data, into); err != nil { if err := kjson.UnmarshalCaseSensitivePreserveInts(data, into); err != nil {
return nil, actual, err return nil, actual, err
} }
return into, actual, nil return into, actual, nil
@ -261,35 +186,35 @@ func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, i
return nil, actual, err return nil, actual, err
} }
if err := caseSensitiveJSONIterator.Unmarshal(data, obj); err != nil { // If the deserializer is non-strict, return here.
return nil, actual, err
}
// If the deserializer is non-strict, return successfully here.
if !s.options.Strict { if !s.options.Strict {
if err := kjson.UnmarshalCaseSensitivePreserveInts(data, obj); err != nil {
return nil, actual, err
}
return obj, actual, nil return obj, actual, nil
} }
// In strict mode pass the data trough the YAMLToJSONStrict converter. var allStrictErrs []error
// This is done to catch duplicate fields regardless of encoding (JSON or YAML). For JSON data, if s.options.Yaml {
// the output would equal the input, unless there is a parsing error such as duplicate fields. // In strict mode pass the original data through the YAMLToJSONStrict converter.
// As we know this was successful in the non-strict case, the only error that may be returned here // This is done to catch duplicate fields in YAML that would have been dropped in the original YAMLToJSON conversion.
// is because of the newly-added strictness. hence we know we can return the typed strictDecoderError // TODO: rework YAMLToJSONStrict to return warnings about duplicate fields without terminating so we don't have to do this twice.
// the actual error is that the object contains duplicate fields. _, err := yaml.YAMLToJSONStrict(originalData)
altered, err := yaml.YAMLToJSONStrict(originalData) if err != nil {
allStrictErrs = append(allStrictErrs, err)
}
}
strictJSONErrs, err := kjson.UnmarshalStrict(data, obj)
if err != nil { if err != nil {
return nil, actual, runtime.NewStrictDecodingError(err.Error(), string(originalData)) // fatal decoding error, not due to strictness
return nil, actual, err
} }
// As performance is not an issue for now for the strict deserializer (one has regardless to do allStrictErrs = append(allStrictErrs, strictJSONErrs...)
// the unmarshal twice), we take the sanitized, altered data that is guaranteed to have no duplicated if len(allStrictErrs) > 0 {
// fields, and unmarshal this into a copy of the already-populated obj. Any error that occurs here is // return the successfully decoded object along with the strict errors
// due to that a matching field doesn't exist in the object. hence we can return a typed strictDecoderError, return obj, actual, runtime.NewStrictDecodingError(allStrictErrs)
// the actual error is that the object contains unknown field.
strictObj := obj.DeepCopyObject()
if err := strictCaseSensitiveJSONIterator.Unmarshal(altered, strictObj); err != nil {
return nil, actual, runtime.NewStrictDecodingError(err.Error(), string(originalData))
} }
// Always return the same object as the non-strict serializer to avoid any deviations.
return obj, actual, nil return obj, actual, nil
} }

View File

@ -123,7 +123,6 @@ func testcases() []testcase {
var decoders = map[string]func([]byte, interface{}) error{ var decoders = map[string]func([]byte, interface{}) error{
"gojson": gojson.Unmarshal, "gojson": gojson.Unmarshal,
"utiljson": utiljson.Unmarshal, "utiljson": utiljson.Unmarshal,
"jsoniter": CaseSensitiveJSONIterator().Unmarshal,
} }
func TestJSONLimits(t *testing.T) { func TestJSONLimits(t *testing.T) {

View File

@ -39,8 +39,7 @@ type testDecodable struct {
Interface interface{} `json:"interface"` Interface interface{} `json:"interface"`
} }
// DecodableSpec has 15 fields. json-iterator treats struct with more than 10 // DecodableSpec has 15 fields.
// fields differently from struct that has less than 10 fields.
type DecodableSpec struct { type DecodableSpec struct {
A int `json:"A"` A int `json:"A"`
B int `json:"B"` B int `json:"B"`
@ -264,7 +263,7 @@ func TestDecode(t *testing.T) {
creater: &mockCreater{obj: &testDecodable{}}, creater: &mockCreater{obj: &testDecodable{}},
expectedGVK: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}, expectedGVK: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"},
errFn: func(err error) bool { errFn: func(err error) bool {
return strings.Contains(err.Error(), `json_test.testDecodable.Interface: DecodeNumber: strconv.ParseFloat: parsing "1e1000": value out of range`) return strings.Contains(err.Error(), `json: cannot unmarshal number 1e1000 into Go struct field testDecodable.interface of type float64`)
}, },
}, },
// Unmarshalling is case-sensitive // Unmarshalling is case-sensitive
@ -298,7 +297,7 @@ func TestDecode(t *testing.T) {
typer: &mockTyper{gvk: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}}, typer: &mockTyper{gvk: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}},
expectedGVK: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}, expectedGVK: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"},
errFn: func(err error) bool { errFn: func(err error) bool {
return strings.Contains(err.Error(), "found unknown field") return strings.Contains(err.Error(), `unknown field "unknown"`)
}, },
strict: true, strict: true,
}, },
@ -309,7 +308,7 @@ func TestDecode(t *testing.T) {
typer: &mockTyper{gvk: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}}, typer: &mockTyper{gvk: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}},
expectedGVK: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}, expectedGVK: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"},
errFn: func(err error) bool { errFn: func(err error) bool {
return strings.Contains(err.Error(), "found unknown field: unknown") return strings.Contains(err.Error(), `unknown field "unknown"`)
}, },
yaml: true, yaml: true,
strict: true, strict: true,
@ -321,7 +320,7 @@ func TestDecode(t *testing.T) {
typer: &mockTyper{gvk: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}}, typer: &mockTyper{gvk: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}},
expectedGVK: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}, expectedGVK: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"},
errFn: func(err error) bool { errFn: func(err error) bool {
return strings.Contains(err.Error(), `"value" already set in map`) return strings.Contains(err.Error(), `duplicate field "value"`)
}, },
strict: true, strict: true,
}, },
@ -526,7 +525,7 @@ func TestDecode(t *testing.T) {
if !test.errFn(err) { if !test.errFn(err) {
t.Errorf("%d: failed: %v", i, err) t.Errorf("%d: failed: %v", i, err)
} }
if obj != nil { if !runtime.IsStrictDecodingError(err) && obj != nil {
t.Errorf("%d: should have returned nil object", i) t.Errorf("%d: should have returned nil object", i)
} }
continue continue

View File

@ -17,10 +17,11 @@ limitations under the License.
package json package json
import ( import (
"bytes"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
kjson "sigs.k8s.io/json"
) )
// NewEncoder delegates to json.NewEncoder // NewEncoder delegates to json.NewEncoder
@ -38,50 +39,11 @@ func Marshal(v interface{}) ([]byte, error) {
// limit recursive depth to prevent stack overflow errors // limit recursive depth to prevent stack overflow errors
const maxDepth = 10000 const maxDepth = 10000
// Unmarshal unmarshals the given data // Unmarshal unmarshals the given data.
// If v is a *map[string]interface{}, *[]interface{}, or *interface{} numbers // Object keys are case-sensitive.
// are converted to int64 or float64 // Numbers decoded into interface{} fields are converted to int64 or float64.
func Unmarshal(data []byte, v interface{}) error { func Unmarshal(data []byte, v interface{}) error {
switch v := v.(type) { return kjson.UnmarshalCaseSensitivePreserveInts(data, v)
case *map[string]interface{}:
// Build a decoder from the given data
decoder := json.NewDecoder(bytes.NewBuffer(data))
// Preserve numbers, rather than casting to float64 automatically
decoder.UseNumber()
// Run the decode
if err := decoder.Decode(v); err != nil {
return err
}
// If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64
return ConvertMapNumbers(*v, 0)
case *[]interface{}:
// Build a decoder from the given data
decoder := json.NewDecoder(bytes.NewBuffer(data))
// Preserve numbers, rather than casting to float64 automatically
decoder.UseNumber()
// Run the decode
if err := decoder.Decode(v); err != nil {
return err
}
// If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64
return ConvertSliceNumbers(*v, 0)
case *interface{}:
// Build a decoder from the given data
decoder := json.NewDecoder(bytes.NewBuffer(data))
// Preserve numbers, rather than casting to float64 automatically
decoder.UseNumber()
// Run the decode
if err := decoder.Decode(v); err != nil {
return err
}
// If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64
return ConvertInterfaceNumbers(v, 0)
default:
return json.Unmarshal(data, v)
}
} }
// ConvertInterfaceNumbers converts any json.Number values to int64 or float64. // ConvertInterfaceNumbers converts any json.Number values to int64 or float64.

View File

@ -34,6 +34,7 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer/json" "k8s.io/apimachinery/pkg/runtime/serializer/json"
utiljson "k8s.io/apimachinery/pkg/util/json"
utilruntime "k8s.io/apimachinery/pkg/util/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/admission"
admissionmetrics "k8s.io/apiserver/pkg/admission/metrics" admissionmetrics "k8s.io/apiserver/pkg/admission/metrics"
@ -61,8 +62,6 @@ const (
MutationAuditAnnotationFailedOpenKeyPrefix string = "failed-open." + MutationAuditAnnotationPrefix MutationAuditAnnotationFailedOpenKeyPrefix string = "failed-open." + MutationAuditAnnotationPrefix
) )
var encodingjson = json.CaseSensitiveJSONIterator()
type mutatingDispatcher struct { type mutatingDispatcher struct {
cm *webhookutil.ClientManager cm *webhookutil.ClientManager
plugin *Plugin plugin *Plugin
@ -444,7 +443,7 @@ func mutationAnnotationValue(configuration, webhook string, mutated bool) (strin
Webhook: webhook, Webhook: webhook,
Mutated: mutated, Mutated: mutated,
} }
bytes, err := encodingjson.Marshal(m) bytes, err := utiljson.Marshal(m)
return string(bytes), err return string(bytes), err
} }
@ -455,6 +454,6 @@ func jsonPatchAnnotationValue(configuration, webhook string, patch interface{})
Patch: patch, Patch: patch,
PatchType: string(admissionv1.PatchTypeJSONPatch), PatchType: string(admissionv1.PatchTypeJSONPatch),
} }
bytes, err := encodingjson.Marshal(p) bytes, err := utiljson.Marshal(p)
return string(bytes), err return string(bytes), err
} }

View File

@ -50,7 +50,7 @@ import (
utiltesting "k8s.io/client-go/util/testing" utiltesting "k8s.io/client-go/util/testing"
// TODO we need to remove this linkage and create our own scheme // TODO we need to remove this linkage and create our own scheme
"k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/kubernetes/scheme"
) )
@ -1762,7 +1762,7 @@ func TestUnstructured(t *testing.T) {
{ {
name: "badpod", name: "badpod",
file: "badpod.json", file: "badpod.json",
expectedError: "v1.ObjectMeta.Annotations", expectedError: "ObjectMeta.",
}, },
} }

View File

@ -20,12 +20,9 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer/json" utiljson "k8s.io/apimachinery/pkg/util/json"
) )
// hold a single instance of the case-sensitive decoder
var caseSensitiveJsonIterator = json.CaseSensitiveJSONIterator()
// metadataValidatingDecoder wraps a decoder and additionally ensures metadata schema fields decode before returning an unstructured object // metadataValidatingDecoder wraps a decoder and additionally ensures metadata schema fields decode before returning an unstructured object
type metadataValidatingDecoder struct { type metadataValidatingDecoder struct {
decoder runtime.Decoder decoder runtime.Decoder
@ -47,7 +44,7 @@ func (m *metadataValidatingDecoder) Decode(data []byte, defaults *schema.GroupVe
// make sure the data can decode into ObjectMeta before we return, // make sure the data can decode into ObjectMeta before we return,
// so we don't silently truncate schema errors in metadata later with accesser get/set calls // so we don't silently truncate schema errors in metadata later with accesser get/set calls
v := &metadataOnlyObject{} v := &metadataOnlyObject{}
if typedErr := caseSensitiveJsonIterator.Unmarshal(data, v); typedErr != nil { if typedErr := utiljson.Unmarshal(data, v); typedErr != nil {
return obj, gvk, typedErr return obj, gvk, typedErr
} }
return obj, gvk, err return obj, gvk, err

View File

@ -22,8 +22,7 @@ import (
"net/http" "net/http"
"strings" "strings"
jsoniter "github.com/json-iterator/go" utiljson "k8s.io/apimachinery/pkg/util/json"
"k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/kube-openapi/pkg/validation/spec" "k8s.io/kube-openapi/pkg/validation/spec"
@ -80,7 +79,7 @@ func (s *Downloader) Download(handler http.Handler, etag string) (returnSpec *sp
return nil, "", http.StatusNotFound, nil return nil, "", http.StatusNotFound, nil
case http.StatusOK: case http.StatusOK:
openAPISpec := &spec.Swagger{} openAPISpec := &spec.Swagger{}
if err := jsoniter.ConfigCompatibleWithStandardLibrary.Unmarshal(writer.data, openAPISpec); err != nil { if err := utiljson.Unmarshal(writer.data, openAPISpec); err != nil {
return nil, "", 0, err return nil, "", 0, err
} }
newEtag = writer.Header().Get("Etag") newEtag = writer.Header().Get("Etag")

View File

@ -277,7 +277,7 @@ exemptions:
"apiVersion":"pod-security.admission.config.k8s.io/v1alpha1", "apiVersion":"pod-security.admission.config.k8s.io/v1alpha1",
"kind":"PodSecurityConfiguration", "kind":"PodSecurityConfiguration",
"deflaults":{"enforce":"baseline"}}`), "deflaults":{"enforce":"baseline"}}`),
expectErr: `unknown field: deflaults`, expectErr: `unknown field "deflaults"`,
}, },
} }