github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-25 12:43:23 +00:00
Code Issues Projects Releases Wiki Activity

Switch from json-iterator to utiljson

Browse Source
This commit is contained in:
Jordan Liggitt 2021-09-14 17:54:37 -04:00
parent d5de03f0d3
commit bba877d3a6
19 changed files with 86 additions and 331 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/kube-scheduler/app/options/options_test.go
View File

@ -1133,7 +1133,7 @@ profiles:
ConfigFile: unknownFieldConfig,
Logs: logs.NewOptions(),
},
expectedError: "found unknown field: foo",
expectedError: `unknown field "foo"`,
checkErrFn: runtime.IsStrictDecodingError,
},
{

55
pkg/api/testing/serialization_test.go
View File

@ -26,7 +26,6 @@ import (
"testing"
"github.com/google/go-cmp/cmp"
jsoniter "github.com/json-iterator/go"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
@ -38,7 +37,6 @@ import (
"k8s.io/apimachinery/pkg/conversion"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer/json"
"k8s.io/apimachinery/pkg/runtime/serializer/streaming"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/watch"
@ -589,59 +587,6 @@ func BenchmarkDecodeIntoJSON(b *testing.B) {
b.StopTimer()
}
// BenchmarkDecodeIntoJSONCodecGenConfigFast provides a baseline
// for JSON decode performance with jsoniter.ConfigFast
func BenchmarkDecodeIntoJSONCodecGenConfigFast(b *testing.B) {
kcodec := legacyscheme.Codecs.LegacyCodec(v1.SchemeGroupVersion)
items := benchmarkItems(b)
width := len(items)
encoded := make([][]byte, width)
for i := range items {
data, err := runtime.Encode(kcodec, &items[i])
if err != nil {
b.Fatal(err)
}
encoded[i] = data
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
obj := v1.Pod{}
if err := jsoniter.ConfigFastest.Unmarshal(encoded[i%width], &obj); err != nil {
b.Fatal(err)
}
}
b.StopTimer()
}
// BenchmarkDecodeIntoJSONCodecGenConfigCompatibleWithStandardLibrary provides a
// baseline for JSON decode performance with
// jsoniter.ConfigCompatibleWithStandardLibrary, but with case sensitivity set
// to true
func BenchmarkDecodeIntoJSONCodecGenConfigCompatibleWithStandardLibrary(b *testing.B) {
kcodec := legacyscheme.Codecs.LegacyCodec(v1.SchemeGroupVersion)
items := benchmarkItems(b)
width := len(items)
encoded := make([][]byte, width)
for i := range items {
data, err := runtime.Encode(kcodec, &items[i])
if err != nil {
b.Fatal(err)
}
encoded[i] = data
}
b.ResetTimer()
iter := json.CaseSensitiveJSONIterator()
for i := 0; i < b.N; i++ {
obj := v1.Pod{}
if err := iter.Unmarshal(encoded[i%width], &obj); err != nil {
b.Fatal(err)
}
}
b.StopTimer()
}
// BenchmarkEncodeYAMLMarshal provides a baseline for regular YAML encode performance
func BenchmarkEncodeYAMLMarshal(b *testing.B) {
items := benchmarkItems(b)

74
pkg/api/testing/unstructured_test.go
View File

@ -19,12 +19,10 @@ package testing
import (
"math/rand"
"reflect"
"sort"
"testing"
"github.com/google/go-cmp/cmp"
fuzz "github.com/google/gofuzz"
jsoniter "github.com/json-iterator/go"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/apitesting/fuzzer"
@ -260,75 +258,3 @@ func BenchmarkFromUnstructuredViaJSON(b *testing.B) {
}
b.StopTimer()
}
func BenchmarkToUnstructuredViaJSONIter(b *testing.B) {
items := benchmarkItems(b)
size := len(items)
var keys []string
for k := range jsonIterConfig {
keys = append(keys, k)
}
sort.Strings(keys)
for _, name := range keys {
c := jsonIterConfig[name]
b.Run(name, func(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
data, err := c.Marshal(&items[i%size])
if err != nil {
b.Fatalf("unexpected error: %v", err)
}
unstr := map[string]interface{}{}
if err := c.Unmarshal(data, &unstr); err != nil {
b.Fatalf("unexpected error: %v", err)
}
}
b.StopTimer()
})
}
}
var jsonIterConfig = map[string]jsoniter.API{
"default": jsoniter.ConfigDefault,
"fastest": jsoniter.ConfigFastest,
"compat": jsoniter.ConfigCompatibleWithStandardLibrary,
}
func BenchmarkFromUnstructuredViaJSONIter(b *testing.B) {
items := benchmarkItems(b)
var unstr []map[string]interface{}
for i := range items {
data, err := jsoniter.Marshal(&items[i])
if err != nil {
b.Fatalf("unexpected error: %v", err)
}
item := map[string]interface{}{}
if err := json.Unmarshal(data, &item); err != nil {
b.Fatalf("unexpected error: %v", err)
}
unstr = append(unstr, item)
}
size := len(items)
var keys []string
for k := range jsonIterConfig {
keys = append(keys, k)
}
sort.Strings(keys)
for _, name := range keys {
c := jsonIterConfig[name]
b.Run(name, func(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
item, err := c.Marshal(unstr[i%size])
if err != nil {
b.Fatalf("unexpected error: %v", err)
}
obj := v1.Pod{}
if err := c.Unmarshal(item, &obj); err != nil {
b.Fatalf("unexpected error: %v", err)
}
}
b.StopTimer()
})
}
}

8
pkg/scheduler/apis/config/scheme/scheme_test.go
View File

@ -263,7 +263,7 @@ profiles:
- Score: 2
Utilization: 1
`),
wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoder error for {"scoringStrategy":{"requestedToCapacityRatio":{"shape":[{"Score":2,"Utilization":1}]}}}: v1beta2.NodeResourcesFitArgs.ScoringStrategy: v1beta2.ScoringStrategy.RequestedToCapacityRatio: v1beta2.RequestedToCapacityRatioParam.Shape: []v1beta2.UtilizationShapePoint: v1beta2.UtilizationShapePoint.ReadObject: found unknown field: Score, error found in #10 byte of ...|:[{"Score":2,"Utiliz|..., bigger context ...|gy":{"requestedToCapacityRatio":{"shape":[{"Score":2,"Utilization":1}]}}}|...`,
wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoding error: unknown field "Score", unknown field "Utilization"`,
},
{
name: "v1beta2 NodeResourcesFitArgs resources encoding is strict",
@ -279,7 +279,7 @@ profiles:
- Name: cpu
Weight: 1
`),
wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoder error for {"scoringStrategy":{"resources":[{"Name":"cpu","Weight":1}]}}: v1beta2.NodeResourcesFitArgs.ScoringStrategy: v1beta2.ScoringStrategy.Resources: []v1beta2.ResourceSpec: v1beta2.ResourceSpec.ReadObject: found unknown field: Name, error found in #10 byte of ...|":[{"Name":"cpu","We|..., bigger context ...|{"scoringStrategy":{"resources":[{"Name":"cpu","Weight":1}]}}|...`,
wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoding error: unknown field "Name", unknown field "Weight"`,
},
{
name: "out-of-tree plugin args",
@ -604,7 +604,7 @@ profiles:
- Score: 2
Utilization: 1
`),
wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoder error for {"scoringStrategy":{"requestedToCapacityRatio":{"shape":[{"Score":2,"Utilization":1}]}}}: v1beta3.NodeResourcesFitArgs.ScoringStrategy: v1beta3.ScoringStrategy.RequestedToCapacityRatio: v1beta3.RequestedToCapacityRatioParam.Shape: []v1beta3.UtilizationShapePoint: v1beta3.UtilizationShapePoint.ReadObject: found unknown field: Score, error found in #10 byte of ...|:[{"Score":2,"Utiliz|..., bigger context ...|gy":{"requestedToCapacityRatio":{"shape":[{"Score":2,"Utilization":1}]}}}|...`,
wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoding error: unknown field "Score", unknown field "Utilization"`,
},
{
name: "v1beta3 NodeResourcesFitArgs resources encoding is strict",
@ -620,7 +620,7 @@ profiles:
- Name: cpu
Weight: 1
`),
wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoder error for {"scoringStrategy":{"resources":[{"Name":"cpu","Weight":1}]}}: v1beta3.NodeResourcesFitArgs.ScoringStrategy: v1beta3.ScoringStrategy.Resources: []v1beta3.ResourceSpec: v1beta3.ResourceSpec.ReadObject: found unknown field: Name, error found in #10 byte of ...|":[{"Name":"cpu","We|..., bigger context ...|{"scoringStrategy":{"resources":[{"Name":"cpu","Weight":1}]}}|...`,
wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoding error: unknown field "Name", unknown field "Weight"`,
},
{
name: "out-of-tree plugin args",

14
staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/objectmeta/coerce.go
View File

@ -22,11 +22,9 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer/json"
utiljson "k8s.io/apimachinery/pkg/util/json"
)
var encodingjson = json.CaseSensitiveJSONIterator()
// GetObjectMeta does conversion of JSON to ObjectMeta. It first tries json.Unmarshal into a metav1.ObjectMeta
// type. If that does not work and dropMalformedFields is true, it does field-by-field best-effort conversion
// throwing away fields which lead to errors.
@ -38,11 +36,11 @@ func GetObjectMeta(obj map[string]interface{}, dropMalformedFields bool) (*metav
// round-trip through JSON first, hoping that unmarshalling just works
objectMeta := &metav1.ObjectMeta{}
metadataBytes, err := encodingjson.Marshal(metadata)
metadataBytes, err := utiljson.Marshal(metadata)
if err != nil {
return nil, false, err
}
if err = encodingjson.Unmarshal(metadataBytes, objectMeta); err == nil {
if err = utiljson.Unmarshal(metadataBytes, objectMeta); err == nil {
// if successful, return
return objectMeta, true, nil
}
@ -63,11 +61,11 @@ func GetObjectMeta(obj map[string]interface{}, dropMalformedFields bool) (*metav
testObjectMeta := &metav1.ObjectMeta{}
for k, v := range metadataMap {
// serialize a single field
if singleFieldBytes, err := encodingjson.Marshal(map[string]interface{}{k: v}); err == nil {
if singleFieldBytes, err := utiljson.Marshal(map[string]interface{}{k: v}); err == nil {
// do a test unmarshal
if encodingjson.Unmarshal(singleFieldBytes, testObjectMeta) == nil {
if utiljson.Unmarshal(singleFieldBytes, testObjectMeta) == nil {
// if that succeeds, unmarshal for real
encodingjson.Unmarshal(singleFieldBytes, accumulatedObjectMeta)
utiljson.Unmarshal(singleFieldBytes, accumulatedObjectMeta)
}
}
}

11
staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/objectmeta/coerce_test.go
View File

@ -31,6 +31,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/runtime/serializer/json"
"k8s.io/apimachinery/pkg/util/diff"
utiljson "k8s.io/apimachinery/pkg/util/json"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
)
@ -114,12 +115,12 @@ func TestMalformedObjectMetaFields(t *testing.T) {
}
// See if it can unmarshal to object meta
spuriousJSON, err := encodingjson.Marshal(spuriousMetaMap)
spuriousJSON, err := utiljson.Marshal(spuriousMetaMap)
if err != nil {
t.Fatalf("error on %v=%#v: %v", pth, v, err)
}
expectedObjectMeta := &metav1.ObjectMeta{}
if err := encodingjson.Unmarshal(spuriousJSON, expectedObjectMeta); err != nil {
if err := utiljson.Unmarshal(spuriousJSON, expectedObjectMeta); err != nil {
// if standard json unmarshal would fail decoding this field, drop the field entirely
truncatedMetaMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(fuzzedObjectMeta.DeepCopy())
if err != nil {
@ -133,12 +134,12 @@ func TestMalformedObjectMetaFields(t *testing.T) {
DeleteJSONPath(truncatedMetaMap, pth[:1], 0)
}
truncatedJSON, err := encodingjson.Marshal(truncatedMetaMap)
truncatedJSON, err := utiljson.Marshal(truncatedMetaMap)
if err != nil {
t.Fatalf("error on %v=%#v: %v", pth, v, err)
}
expectedObjectMeta = &metav1.ObjectMeta{}
if err := encodingjson.Unmarshal(truncatedJSON, expectedObjectMeta); err != nil {
if err := utiljson.Unmarshal(truncatedJSON, expectedObjectMeta); err != nil {
t.Fatalf("error on %v=%#v: %v", pth, v, err)
}
}
@ -190,7 +191,7 @@ func TestGetObjectMetaNils(t *testing.T) {
}
// double check this what the kube JSON decode is doing
bs, _ := encodingjson.Marshal(u.UnstructuredContent())
bs, _ := utiljson.Marshal(u.UnstructuredContent())
kubeObj, _, err := clientgoscheme.Codecs.UniversalDecoder(corev1.SchemeGroupVersion).Decode(bs, nil, nil)
if err != nil {
t.Fatal(err)

11
staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/group_version_test.go
View File

@ -21,7 +21,7 @@ import (
"reflect"
"testing"
"k8s.io/apimachinery/pkg/runtime/serializer/json"
utiljson "k8s.io/apimachinery/pkg/util/json"
)
type GroupVersionHolder struct {
@ -46,13 +46,12 @@ func TestGroupVersionUnmarshalJSON(t *testing.T) {
if !reflect.DeepEqual(result.GV, c.expect) {
t.Errorf("JSON codec failed to unmarshal input '%s': expected %+v, got %+v", c.input, c.expect, result.GV)
}
// test the json-iterator codec
iter := json.CaseSensitiveJSONIterator()
if err := iter.Unmarshal(c.input, &result); err != nil {
t.Errorf("json-iterator codec failed to unmarshal input '%v': %v", c.input, err)
// test the utiljson codec
if err := utiljson.Unmarshal(c.input, &result); err != nil {
t.Errorf("util/json codec failed to unmarshal input '%v': %v", c.input, err)
}
if !reflect.DeepEqual(result.GV, c.expect) {
t.Errorf("json-iterator codec failed to unmarshal input '%s': expected %+v, got %+v", c.input, c.expect, result.GV)
t.Errorf("util/json codec failed to unmarshal input '%s': expected %+v, got %+v", c.input, c.expect, result.GV)
}
}
}

5
staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types_test.go
View File

@ -21,7 +21,7 @@ import (
"reflect"
"testing"
"k8s.io/apimachinery/pkg/runtime/serializer/json"
utiljson "k8s.io/apimachinery/pkg/util/json"
)
func TestVerbsMarshalJSON(t *testing.T) {
@ -56,10 +56,9 @@ func TestVerbsJsonIterUnmarshalJSON(t *testing.T) {
{`{"verbs":["delete"]}`, APIResource{Verbs: Verbs([]string{"delete"})}},
}
iter := json.CaseSensitiveJSONIterator()
for i, c := range cases {
var result APIResource
if err := iter.Unmarshal([]byte(c.input), &result); err != nil {
if err := utiljson.Unmarshal([]byte(c.input), &result); err != nil {
t.Errorf("[%d] Failed to unmarshal input '%v': %v", i, c.input, err)
}
if !reflect.DeepEqual(result, c.result) {

6
staging/src/k8s.io/apimachinery/pkg/runtime/converter_test.go
View File

@ -274,7 +274,7 @@ func TestRoundTrip(t *testing.T) {
{
// Test slice of interface{} with different values.
obj: &D{
A: []interface{}{3.0, "3.0", nil},
A: []interface{}{float64(3.5), int64(4), "3.0", nil},
},
},
}
@ -322,11 +322,11 @@ func TestUnrecognized(t *testing.T) {
err error
}{
{
data: "{\"da\":[3.0,\"3.0\",null]}",
data: "{\"da\":[3.5,4,\"3.0\",null]}",
obj: &D{},
},
{
data: "{\"ea\":[3.0,\"3.0\",null]}",
data: "{\"ea\":[3.5,4,\"3.0\",null]}",
obj: &E{},
},
{

19
staging/src/k8s.io/apimachinery/pkg/runtime/error.go
View File

@ -19,6 +19,7 @@ package runtime
import (
"fmt"
"reflect"
"strings"
"k8s.io/apimachinery/pkg/runtime/schema"
)
@ -124,20 +125,26 @@ func IsMissingVersion(err error) bool {
// strictDecodingError is a base error type that is returned by a strict Decoder such
// as UniversalStrictDecoder.
type strictDecodingError struct {
message string
data string
errors []error
}
// NewStrictDecodingError creates a new strictDecodingError object.
func NewStrictDecodingError(message string, data string) error {
func NewStrictDecodingError(errors []error) error {
return &strictDecodingError{
message: message,
data: data,
errors: errors,
}
}
func (e *strictDecodingError) Error() string {
return fmt.Sprintf("strict decoder error for %s: %s", e.data, e.message)
var s strings.Builder
s.WriteString("strict decoding error: ")
for i, err := range e.errors {
if i != 0 {
s.WriteString(", ")
}
s.WriteString(err.Error())
}
return s.String()
}
// IsStrictDecodingError returns true if the error indicates that the provided object

123
staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json.go
View File

@ -20,10 +20,8 @@ import (
"encoding/json"
"io"
"strconv"
"unsafe"
jsoniter "github.com/json-iterator/go"
"github.com/modern-go/reflect2"
kjson "sigs.k8s.io/json"
"sigs.k8s.io/yaml"
"k8s.io/apimachinery/pkg/runtime"
@ -110,79 +108,6 @@ type Serializer struct {
var _ runtime.Serializer = &Serializer{}
var _ recognizer.RecognizingDecoder = &Serializer{}
type customNumberExtension struct {
jsoniter.DummyExtension
}
func (cne *customNumberExtension) CreateDecoder(typ reflect2.Type) jsoniter.ValDecoder {
if typ.String() == "interface {}" {
return customNumberDecoder{}
}
return nil
}
type customNumberDecoder struct {
}
func (customNumberDecoder) Decode(ptr unsafe.Pointer, iter *jsoniter.Iterator) {
switch iter.WhatIsNext() {
case jsoniter.NumberValue:
var number jsoniter.Number
iter.ReadVal(&number)
i64, err := strconv.ParseInt(string(number), 10, 64)
if err == nil {
*(*interface{})(ptr) = i64
return
}
f64, err := strconv.ParseFloat(string(number), 64)
if err == nil {
*(*interface{})(ptr) = f64
return
}
iter.ReportError("DecodeNumber", err.Error())
default:
*(*interface{})(ptr) = iter.Read()
}
}
// CaseSensitiveJSONIterator returns a jsoniterator API that's configured to be
// case-sensitive when unmarshalling, and otherwise compatible with
// the encoding/json standard library.
func CaseSensitiveJSONIterator() jsoniter.API {
config := jsoniter.Config{
EscapeHTML: true,
SortMapKeys: true,
ValidateJsonRawMessage: true,
CaseSensitive: true,
}.Froze()
// Force jsoniter to decode number to interface{} via int64/float64, if possible.
config.RegisterExtension(&customNumberExtension{})
return config
}
// StrictCaseSensitiveJSONIterator returns a jsoniterator API that's configured to be
// case-sensitive, but also disallows unknown fields when unmarshalling. It is compatible with
// the encoding/json standard library.
func StrictCaseSensitiveJSONIterator() jsoniter.API {
config := jsoniter.Config{
EscapeHTML: true,
SortMapKeys: true,
ValidateJsonRawMessage: true,
CaseSensitive: true,
DisallowUnknownFields: true,
}.Froze()
// Force jsoniter to decode number to interface{} via int64/float64, if possible.
config.RegisterExtension(&customNumberExtension{})
return config
}
// Private copies of jsoniter to try to shield against possible mutations
// from outside. Still does not protect from package level jsoniter.Register*() functions - someone calling them
// in some other library will mess with every usage of the jsoniter library in the whole program.
// See https://github.com/json-iterator/go/issues/265
var caseSensitiveJSONIterator = CaseSensitiveJSONIterator()
var strictCaseSensitiveJSONIterator = StrictCaseSensitiveJSONIterator()
// gvkWithDefaults returns group kind and version defaulting from provided default
func gvkWithDefaults(actual, defaultGVK schema.GroupVersionKind) schema.GroupVersionKind {
if len(actual.Kind) == 0 {
@ -237,7 +162,7 @@ func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, i
types, _, err := s.typer.ObjectKinds(into)
switch {
case runtime.IsNotRegisteredError(err), isUnstructured:
if err := caseSensitiveJSONIterator.Unmarshal(data, into); err != nil {
if err := kjson.UnmarshalCaseSensitivePreserveInts(data, into); err != nil {
return nil, actual, err
}
return into, actual, nil
@ -261,35 +186,35 @@ func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, i
return nil, actual, err
}
if err := caseSensitiveJSONIterator.Unmarshal(data, obj); err != nil {
return nil, actual, err
}
// If the deserializer is non-strict, return successfully here.
// If the deserializer is non-strict, return here.
if !s.options.Strict {
if err := kjson.UnmarshalCaseSensitivePreserveInts(data, obj); err != nil {
return nil, actual, err
}
return obj, actual, nil
}
// In strict mode pass the data trough the YAMLToJSONStrict converter.
// This is done to catch duplicate fields regardless of encoding (JSON or YAML). For JSON data,
// the output would equal the input, unless there is a parsing error such as duplicate fields.
// As we know this was successful in the non-strict case, the only error that may be returned here
// is because of the newly-added strictness. hence we know we can return the typed strictDecoderError
// the actual error is that the object contains duplicate fields.
altered, err := yaml.YAMLToJSONStrict(originalData)
var allStrictErrs []error
if s.options.Yaml {
// In strict mode pass the original data through the YAMLToJSONStrict converter.
// This is done to catch duplicate fields in YAML that would have been dropped in the original YAMLToJSON conversion.
// TODO: rework YAMLToJSONStrict to return warnings about duplicate fields without terminating so we don't have to do this twice.
_, err := yaml.YAMLToJSONStrict(originalData)
if err != nil {
allStrictErrs = append(allStrictErrs, err)
}
}
strictJSONErrs, err := kjson.UnmarshalStrict(data, obj)
if err != nil {
return nil, actual, runtime.NewStrictDecodingError(err.Error(), string(originalData))
// fatal decoding error, not due to strictness
return nil, actual, err
}
// As performance is not an issue for now for the strict deserializer (one has regardless to do
// the unmarshal twice), we take the sanitized, altered data that is guaranteed to have no duplicated
// fields, and unmarshal this into a copy of the already-populated obj. Any error that occurs here is
// due to that a matching field doesn't exist in the object. hence we can return a typed strictDecoderError,
// the actual error is that the object contains unknown field.
strictObj := obj.DeepCopyObject()
if err := strictCaseSensitiveJSONIterator.Unmarshal(altered, strictObj); err != nil {
return nil, actual, runtime.NewStrictDecodingError(err.Error(), string(originalData))
allStrictErrs = append(allStrictErrs, strictJSONErrs...)
if len(allStrictErrs) > 0 {
// return the successfully decoded object along with the strict errors
return obj, actual, runtime.NewStrictDecodingError(allStrictErrs)
}
// Always return the same object as the non-strict serializer to avoid any deviations.
return obj, actual, nil
}

1
staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json_limit_test.go
View File

@ -123,7 +123,6 @@ func testcases() []testcase {
var decoders = map[string]func([]byte, interface{}) error{
"gojson": gojson.Unmarshal,
"utiljson": utiljson.Unmarshal,
"jsoniter": CaseSensitiveJSONIterator().Unmarshal,
}
func TestJSONLimits(t *testing.T) {

13
staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json_test.go
View File

@ -39,8 +39,7 @@ type testDecodable struct {
Interface interface{} `json:"interface"`
}
// DecodableSpec has 15 fields. json-iterator treats struct with more than 10
// fields differently from struct that has less than 10 fields.
// DecodableSpec has 15 fields.
type DecodableSpec struct {
A int `json:"A"`
B int `json:"B"`
@ -264,7 +263,7 @@ func TestDecode(t *testing.T) {
creater: &mockCreater{obj: &testDecodable{}},
expectedGVK: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"},
errFn: func(err error) bool {
return strings.Contains(err.Error(), `json_test.testDecodable.Interface: DecodeNumber: strconv.ParseFloat: parsing "1e1000": value out of range`)
return strings.Contains(err.Error(), `json: cannot unmarshal number 1e1000 into Go struct field testDecodable.interface of type float64`)
},
},
// Unmarshalling is case-sensitive
@ -298,7 +297,7 @@ func TestDecode(t *testing.T) {
typer: &mockTyper{gvk: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}},
expectedGVK: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"},
errFn: func(err error) bool {
return strings.Contains(err.Error(), "found unknown field")
return strings.Contains(err.Error(), `unknown field "unknown"`)
},
strict: true,
},
@ -309,7 +308,7 @@ func TestDecode(t *testing.T) {
typer: &mockTyper{gvk: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}},
expectedGVK: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"},
errFn: func(err error) bool {
return strings.Contains(err.Error(), "found unknown field: unknown")
return strings.Contains(err.Error(), `unknown field "unknown"`)
},
yaml: true,
strict: true,
@ -321,7 +320,7 @@ func TestDecode(t *testing.T) {
typer: &mockTyper{gvk: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}},
expectedGVK: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"},
errFn: func(err error) bool {
return strings.Contains(err.Error(), `"value" already set in map`)
return strings.Contains(err.Error(), `duplicate field "value"`)
},
strict: true,
},
@ -526,7 +525,7 @@ func TestDecode(t *testing.T) {
if !test.errFn(err) {
t.Errorf("%d: failed: %v", i, err)
}
if obj != nil {
if !runtime.IsStrictDecodingError(err) && obj != nil {
t.Errorf("%d: should have returned nil object", i)
}
continue

50
staging/src/k8s.io/apimachinery/pkg/util/json/json.go
View File

@ -17,10 +17,11 @@ limitations under the License.
package json
import (
"bytes"
"encoding/json"
"fmt"
"io"
kjson "sigs.k8s.io/json"
)
// NewEncoder delegates to json.NewEncoder
@ -38,50 +39,11 @@ func Marshal(v interface{}) ([]byte, error) {
// limit recursive depth to prevent stack overflow errors
const maxDepth = 10000
// Unmarshal unmarshals the given data
// If v is a *map[string]interface{}, *[]interface{}, or *interface{} numbers
// are converted to int64 or float64
// Unmarshal unmarshals the given data.
// Object keys are case-sensitive.
// Numbers decoded into interface{} fields are converted to int64 or float64.
func Unmarshal(data []byte, v interface{}) error {
switch v := v.(type) {
case *map[string]interface{}:
// Build a decoder from the given data
decoder := json.NewDecoder(bytes.NewBuffer(data))
// Preserve numbers, rather than casting to float64 automatically
decoder.UseNumber()
// Run the decode
if err := decoder.Decode(v); err != nil {
return err
}
// If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64
return ConvertMapNumbers(*v, 0)
case *[]interface{}:
// Build a decoder from the given data
decoder := json.NewDecoder(bytes.NewBuffer(data))
// Preserve numbers, rather than casting to float64 automatically
decoder.UseNumber()
// Run the decode
if err := decoder.Decode(v); err != nil {
return err
}
// If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64
return ConvertSliceNumbers(*v, 0)
case *interface{}:
// Build a decoder from the given data
decoder := json.NewDecoder(bytes.NewBuffer(data))
// Preserve numbers, rather than casting to float64 automatically
decoder.UseNumber()
// Run the decode
if err := decoder.Decode(v); err != nil {
return err
}
// If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64
return ConvertInterfaceNumbers(v, 0)
default:
return json.Unmarshal(data, v)
}
return kjson.UnmarshalCaseSensitivePreserveInts(data, v)
}
// ConvertInterfaceNumbers converts any json.Number values to int64 or float64.

7
staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating/dispatcher.go
View File

@ -34,6 +34,7 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer/json"
utiljson "k8s.io/apimachinery/pkg/util/json"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apiserver/pkg/admission"
admissionmetrics "k8s.io/apiserver/pkg/admission/metrics"
@ -61,8 +62,6 @@ const (
MutationAuditAnnotationFailedOpenKeyPrefix string = "failed-open." + MutationAuditAnnotationPrefix
)
var encodingjson = json.CaseSensitiveJSONIterator()
type mutatingDispatcher struct {
cm *webhookutil.ClientManager
plugin *Plugin
@ -444,7 +443,7 @@ func mutationAnnotationValue(configuration, webhook string, mutated bool) (strin
Webhook: webhook,
Mutated: mutated,
}
bytes, err := encodingjson.Marshal(m)
bytes, err := utiljson.Marshal(m)
return string(bytes), err
}
@ -455,6 +454,6 @@ func jsonPatchAnnotationValue(configuration, webhook string, patch interface{})
Patch: patch,
PatchType: string(admissionv1.PatchTypeJSONPatch),
}
bytes, err := encodingjson.Marshal(p)
bytes, err := utiljson.Marshal(p)
return string(bytes), err
}

4
staging/src/k8s.io/cli-runtime/pkg/resource/builder_test.go
View File

@ -50,7 +50,7 @@ import (
utiltesting "k8s.io/client-go/util/testing"
// TODO we need to remove this linkage and create our own scheme
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes/scheme"
)
@ -1762,7 +1762,7 @@ func TestUnstructured(t *testing.T) {
{
name: "badpod",
file: "badpod.json",
expectedError: "v1.ObjectMeta.Annotations",
expectedError: "ObjectMeta.",
},
}

7
staging/src/k8s.io/cli-runtime/pkg/resource/metadata_decoder.go
View File

@ -20,12 +20,9 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer/json"
utiljson "k8s.io/apimachinery/pkg/util/json"
)
// hold a single instance of the case-sensitive decoder
var caseSensitiveJsonIterator = json.CaseSensitiveJSONIterator()
// metadataValidatingDecoder wraps a decoder and additionally ensures metadata schema fields decode before returning an unstructured object
type metadataValidatingDecoder struct {
decoder runtime.Decoder
@ -47,7 +44,7 @@ func (m *metadataValidatingDecoder) Decode(data []byte, defaults *schema.GroupVe
// make sure the data can decode into ObjectMeta before we return,
// so we don't silently truncate schema errors in metadata later with accesser get/set calls
v := &metadataOnlyObject{}
if typedErr := caseSensitiveJsonIterator.Unmarshal(data, v); typedErr != nil {
if typedErr := utiljson.Unmarshal(data, v); typedErr != nil {
return obj, gvk, typedErr
}
return obj, gvk, err

5
staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/downloader.go
View File

@ -22,8 +22,7 @@ import (
"net/http"
"strings"
jsoniter "github.com/json-iterator/go"
utiljson "k8s.io/apimachinery/pkg/util/json"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/kube-openapi/pkg/validation/spec"
@ -80,7 +79,7 @@ func (s *Downloader) Download(handler http.Handler, etag string) (returnSpec *sp
return nil, "", http.StatusNotFound, nil
case http.StatusOK:
openAPISpec := &spec.Swagger{}
if err := jsoniter.ConfigCompatibleWithStandardLibrary.Unmarshal(writer.data, openAPISpec); err != nil {
if err := utiljson.Unmarshal(writer.data, openAPISpec); err != nil {
return nil, "", 0, err
}
newEtag = writer.Header().Get("Etag")

2
staging/src/k8s.io/pod-security-admission/admission/api/load/load_test.go
View File

@ -277,7 +277,7 @@ exemptions:
"apiVersion":"pod-security.admission.config.k8s.io/v1alpha1",
"kind":"PodSecurityConfiguration",
"deflaults":{"enforce":"baseline"}}`),
expectErr: `unknown field: deflaults`,
expectErr: `unknown field "deflaults"`,
},
}