mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-25 12:43:23 +00:00
Switch from json-iterator to utiljson
This commit is contained in:
parent
d5de03f0d3
commit
bba877d3a6
@ -1133,7 +1133,7 @@ profiles:
|
||||
ConfigFile: unknownFieldConfig,
|
||||
Logs: logs.NewOptions(),
|
||||
},
|
||||
expectedError: "found unknown field: foo",
|
||||
expectedError: `unknown field "foo"`,
|
||||
checkErrFn: runtime.IsStrictDecodingError,
|
||||
},
|
||||
{
|
||||
|
@ -26,7 +26,6 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
@ -38,7 +37,6 @@ import (
|
||||
"k8s.io/apimachinery/pkg/conversion"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/runtime/serializer/json"
|
||||
"k8s.io/apimachinery/pkg/runtime/serializer/streaming"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
@ -589,59 +587,6 @@ func BenchmarkDecodeIntoJSON(b *testing.B) {
|
||||
b.StopTimer()
|
||||
}
|
||||
|
||||
// BenchmarkDecodeIntoJSONCodecGenConfigFast provides a baseline
|
||||
// for JSON decode performance with jsoniter.ConfigFast
|
||||
func BenchmarkDecodeIntoJSONCodecGenConfigFast(b *testing.B) {
|
||||
kcodec := legacyscheme.Codecs.LegacyCodec(v1.SchemeGroupVersion)
|
||||
items := benchmarkItems(b)
|
||||
width := len(items)
|
||||
encoded := make([][]byte, width)
|
||||
for i := range items {
|
||||
data, err := runtime.Encode(kcodec, &items[i])
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
encoded[i] = data
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
obj := v1.Pod{}
|
||||
if err := jsoniter.ConfigFastest.Unmarshal(encoded[i%width], &obj); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
b.StopTimer()
|
||||
}
|
||||
|
||||
// BenchmarkDecodeIntoJSONCodecGenConfigCompatibleWithStandardLibrary provides a
|
||||
// baseline for JSON decode performance with
|
||||
// jsoniter.ConfigCompatibleWithStandardLibrary, but with case sensitivity set
|
||||
// to true
|
||||
func BenchmarkDecodeIntoJSONCodecGenConfigCompatibleWithStandardLibrary(b *testing.B) {
|
||||
kcodec := legacyscheme.Codecs.LegacyCodec(v1.SchemeGroupVersion)
|
||||
items := benchmarkItems(b)
|
||||
width := len(items)
|
||||
encoded := make([][]byte, width)
|
||||
for i := range items {
|
||||
data, err := runtime.Encode(kcodec, &items[i])
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
encoded[i] = data
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
iter := json.CaseSensitiveJSONIterator()
|
||||
for i := 0; i < b.N; i++ {
|
||||
obj := v1.Pod{}
|
||||
if err := iter.Unmarshal(encoded[i%width], &obj); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
b.StopTimer()
|
||||
}
|
||||
|
||||
// BenchmarkEncodeYAMLMarshal provides a baseline for regular YAML encode performance
|
||||
func BenchmarkEncodeYAMLMarshal(b *testing.B) {
|
||||
items := benchmarkItems(b)
|
||||
|
@ -19,12 +19,10 @@ package testing
|
||||
import (
|
||||
"math/rand"
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
fuzz "github.com/google/gofuzz"
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/apitesting/fuzzer"
|
||||
@ -260,75 +258,3 @@ func BenchmarkFromUnstructuredViaJSON(b *testing.B) {
|
||||
}
|
||||
b.StopTimer()
|
||||
}
|
||||
|
||||
func BenchmarkToUnstructuredViaJSONIter(b *testing.B) {
|
||||
items := benchmarkItems(b)
|
||||
size := len(items)
|
||||
var keys []string
|
||||
for k := range jsonIterConfig {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, name := range keys {
|
||||
c := jsonIterConfig[name]
|
||||
b.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
data, err := c.Marshal(&items[i%size])
|
||||
if err != nil {
|
||||
b.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
unstr := map[string]interface{}{}
|
||||
if err := c.Unmarshal(data, &unstr); err != nil {
|
||||
b.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
b.StopTimer()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
var jsonIterConfig = map[string]jsoniter.API{
|
||||
"default": jsoniter.ConfigDefault,
|
||||
"fastest": jsoniter.ConfigFastest,
|
||||
"compat": jsoniter.ConfigCompatibleWithStandardLibrary,
|
||||
}
|
||||
|
||||
func BenchmarkFromUnstructuredViaJSONIter(b *testing.B) {
|
||||
items := benchmarkItems(b)
|
||||
var unstr []map[string]interface{}
|
||||
for i := range items {
|
||||
data, err := jsoniter.Marshal(&items[i])
|
||||
if err != nil {
|
||||
b.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
item := map[string]interface{}{}
|
||||
if err := json.Unmarshal(data, &item); err != nil {
|
||||
b.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
unstr = append(unstr, item)
|
||||
}
|
||||
size := len(items)
|
||||
var keys []string
|
||||
for k := range jsonIterConfig {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, name := range keys {
|
||||
c := jsonIterConfig[name]
|
||||
b.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
item, err := c.Marshal(unstr[i%size])
|
||||
if err != nil {
|
||||
b.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
obj := v1.Pod{}
|
||||
if err := c.Unmarshal(item, &obj); err != nil {
|
||||
b.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
b.StopTimer()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
@ -263,7 +263,7 @@ profiles:
|
||||
- Score: 2
|
||||
Utilization: 1
|
||||
`),
|
||||
wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoder error for {"scoringStrategy":{"requestedToCapacityRatio":{"shape":[{"Score":2,"Utilization":1}]}}}: v1beta2.NodeResourcesFitArgs.ScoringStrategy: v1beta2.ScoringStrategy.RequestedToCapacityRatio: v1beta2.RequestedToCapacityRatioParam.Shape: []v1beta2.UtilizationShapePoint: v1beta2.UtilizationShapePoint.ReadObject: found unknown field: Score, error found in #10 byte of ...|:[{"Score":2,"Utiliz|..., bigger context ...|gy":{"requestedToCapacityRatio":{"shape":[{"Score":2,"Utilization":1}]}}}|...`,
|
||||
wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoding error: unknown field "Score", unknown field "Utilization"`,
|
||||
},
|
||||
{
|
||||
name: "v1beta2 NodeResourcesFitArgs resources encoding is strict",
|
||||
@ -279,7 +279,7 @@ profiles:
|
||||
- Name: cpu
|
||||
Weight: 1
|
||||
`),
|
||||
wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoder error for {"scoringStrategy":{"resources":[{"Name":"cpu","Weight":1}]}}: v1beta2.NodeResourcesFitArgs.ScoringStrategy: v1beta2.ScoringStrategy.Resources: []v1beta2.ResourceSpec: v1beta2.ResourceSpec.ReadObject: found unknown field: Name, error found in #10 byte of ...|":[{"Name":"cpu","We|..., bigger context ...|{"scoringStrategy":{"resources":[{"Name":"cpu","Weight":1}]}}|...`,
|
||||
wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoding error: unknown field "Name", unknown field "Weight"`,
|
||||
},
|
||||
{
|
||||
name: "out-of-tree plugin args",
|
||||
@ -604,7 +604,7 @@ profiles:
|
||||
- Score: 2
|
||||
Utilization: 1
|
||||
`),
|
||||
wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoder error for {"scoringStrategy":{"requestedToCapacityRatio":{"shape":[{"Score":2,"Utilization":1}]}}}: v1beta3.NodeResourcesFitArgs.ScoringStrategy: v1beta3.ScoringStrategy.RequestedToCapacityRatio: v1beta3.RequestedToCapacityRatioParam.Shape: []v1beta3.UtilizationShapePoint: v1beta3.UtilizationShapePoint.ReadObject: found unknown field: Score, error found in #10 byte of ...|:[{"Score":2,"Utiliz|..., bigger context ...|gy":{"requestedToCapacityRatio":{"shape":[{"Score":2,"Utilization":1}]}}}|...`,
|
||||
wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoding error: unknown field "Score", unknown field "Utilization"`,
|
||||
},
|
||||
{
|
||||
name: "v1beta3 NodeResourcesFitArgs resources encoding is strict",
|
||||
@ -620,7 +620,7 @@ profiles:
|
||||
- Name: cpu
|
||||
Weight: 1
|
||||
`),
|
||||
wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoder error for {"scoringStrategy":{"resources":[{"Name":"cpu","Weight":1}]}}: v1beta3.NodeResourcesFitArgs.ScoringStrategy: v1beta3.ScoringStrategy.Resources: []v1beta3.ResourceSpec: v1beta3.ResourceSpec.ReadObject: found unknown field: Name, error found in #10 byte of ...|":[{"Name":"cpu","We|..., bigger context ...|{"scoringStrategy":{"resources":[{"Name":"cpu","Weight":1}]}}|...`,
|
||||
wantErr: `decoding .profiles[0].pluginConfig[0]: decoding args for plugin NodeResourcesFit: strict decoding error: unknown field "Name", unknown field "Weight"`,
|
||||
},
|
||||
{
|
||||
name: "out-of-tree plugin args",
|
||||
|
@ -22,11 +22,9 @@ import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/runtime/serializer/json"
|
||||
utiljson "k8s.io/apimachinery/pkg/util/json"
|
||||
)
|
||||
|
||||
var encodingjson = json.CaseSensitiveJSONIterator()
|
||||
|
||||
// GetObjectMeta does conversion of JSON to ObjectMeta. It first tries json.Unmarshal into a metav1.ObjectMeta
|
||||
// type. If that does not work and dropMalformedFields is true, it does field-by-field best-effort conversion
|
||||
// throwing away fields which lead to errors.
|
||||
@ -38,11 +36,11 @@ func GetObjectMeta(obj map[string]interface{}, dropMalformedFields bool) (*metav
|
||||
|
||||
// round-trip through JSON first, hoping that unmarshalling just works
|
||||
objectMeta := &metav1.ObjectMeta{}
|
||||
metadataBytes, err := encodingjson.Marshal(metadata)
|
||||
metadataBytes, err := utiljson.Marshal(metadata)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if err = encodingjson.Unmarshal(metadataBytes, objectMeta); err == nil {
|
||||
if err = utiljson.Unmarshal(metadataBytes, objectMeta); err == nil {
|
||||
// if successful, return
|
||||
return objectMeta, true, nil
|
||||
}
|
||||
@ -63,11 +61,11 @@ func GetObjectMeta(obj map[string]interface{}, dropMalformedFields bool) (*metav
|
||||
testObjectMeta := &metav1.ObjectMeta{}
|
||||
for k, v := range metadataMap {
|
||||
// serialize a single field
|
||||
if singleFieldBytes, err := encodingjson.Marshal(map[string]interface{}{k: v}); err == nil {
|
||||
if singleFieldBytes, err := utiljson.Marshal(map[string]interface{}{k: v}); err == nil {
|
||||
// do a test unmarshal
|
||||
if encodingjson.Unmarshal(singleFieldBytes, testObjectMeta) == nil {
|
||||
if utiljson.Unmarshal(singleFieldBytes, testObjectMeta) == nil {
|
||||
// if that succeeds, unmarshal for real
|
||||
encodingjson.Unmarshal(singleFieldBytes, accumulatedObjectMeta)
|
||||
utiljson.Unmarshal(singleFieldBytes, accumulatedObjectMeta)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -31,6 +31,7 @@ import (
|
||||
"k8s.io/apimachinery/pkg/runtime/serializer"
|
||||
"k8s.io/apimachinery/pkg/runtime/serializer/json"
|
||||
"k8s.io/apimachinery/pkg/util/diff"
|
||||
utiljson "k8s.io/apimachinery/pkg/util/json"
|
||||
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
|
||||
)
|
||||
|
||||
@ -114,12 +115,12 @@ func TestMalformedObjectMetaFields(t *testing.T) {
|
||||
}
|
||||
|
||||
// See if it can unmarshal to object meta
|
||||
spuriousJSON, err := encodingjson.Marshal(spuriousMetaMap)
|
||||
spuriousJSON, err := utiljson.Marshal(spuriousMetaMap)
|
||||
if err != nil {
|
||||
t.Fatalf("error on %v=%#v: %v", pth, v, err)
|
||||
}
|
||||
expectedObjectMeta := &metav1.ObjectMeta{}
|
||||
if err := encodingjson.Unmarshal(spuriousJSON, expectedObjectMeta); err != nil {
|
||||
if err := utiljson.Unmarshal(spuriousJSON, expectedObjectMeta); err != nil {
|
||||
// if standard json unmarshal would fail decoding this field, drop the field entirely
|
||||
truncatedMetaMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(fuzzedObjectMeta.DeepCopy())
|
||||
if err != nil {
|
||||
@ -133,12 +134,12 @@ func TestMalformedObjectMetaFields(t *testing.T) {
|
||||
DeleteJSONPath(truncatedMetaMap, pth[:1], 0)
|
||||
}
|
||||
|
||||
truncatedJSON, err := encodingjson.Marshal(truncatedMetaMap)
|
||||
truncatedJSON, err := utiljson.Marshal(truncatedMetaMap)
|
||||
if err != nil {
|
||||
t.Fatalf("error on %v=%#v: %v", pth, v, err)
|
||||
}
|
||||
expectedObjectMeta = &metav1.ObjectMeta{}
|
||||
if err := encodingjson.Unmarshal(truncatedJSON, expectedObjectMeta); err != nil {
|
||||
if err := utiljson.Unmarshal(truncatedJSON, expectedObjectMeta); err != nil {
|
||||
t.Fatalf("error on %v=%#v: %v", pth, v, err)
|
||||
}
|
||||
}
|
||||
@ -190,7 +191,7 @@ func TestGetObjectMetaNils(t *testing.T) {
|
||||
}
|
||||
|
||||
// double check this what the kube JSON decode is doing
|
||||
bs, _ := encodingjson.Marshal(u.UnstructuredContent())
|
||||
bs, _ := utiljson.Marshal(u.UnstructuredContent())
|
||||
kubeObj, _, err := clientgoscheme.Codecs.UniversalDecoder(corev1.SchemeGroupVersion).Decode(bs, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
@ -21,7 +21,7 @@ import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"k8s.io/apimachinery/pkg/runtime/serializer/json"
|
||||
utiljson "k8s.io/apimachinery/pkg/util/json"
|
||||
)
|
||||
|
||||
type GroupVersionHolder struct {
|
||||
@ -46,13 +46,12 @@ func TestGroupVersionUnmarshalJSON(t *testing.T) {
|
||||
if !reflect.DeepEqual(result.GV, c.expect) {
|
||||
t.Errorf("JSON codec failed to unmarshal input '%s': expected %+v, got %+v", c.input, c.expect, result.GV)
|
||||
}
|
||||
// test the json-iterator codec
|
||||
iter := json.CaseSensitiveJSONIterator()
|
||||
if err := iter.Unmarshal(c.input, &result); err != nil {
|
||||
t.Errorf("json-iterator codec failed to unmarshal input '%v': %v", c.input, err)
|
||||
// test the utiljson codec
|
||||
if err := utiljson.Unmarshal(c.input, &result); err != nil {
|
||||
t.Errorf("util/json codec failed to unmarshal input '%v': %v", c.input, err)
|
||||
}
|
||||
if !reflect.DeepEqual(result.GV, c.expect) {
|
||||
t.Errorf("json-iterator codec failed to unmarshal input '%s': expected %+v, got %+v", c.input, c.expect, result.GV)
|
||||
t.Errorf("util/json codec failed to unmarshal input '%s': expected %+v, got %+v", c.input, c.expect, result.GV)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -21,7 +21,7 @@ import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"k8s.io/apimachinery/pkg/runtime/serializer/json"
|
||||
utiljson "k8s.io/apimachinery/pkg/util/json"
|
||||
)
|
||||
|
||||
func TestVerbsMarshalJSON(t *testing.T) {
|
||||
@ -56,10 +56,9 @@ func TestVerbsJsonIterUnmarshalJSON(t *testing.T) {
|
||||
{`{"verbs":["delete"]}`, APIResource{Verbs: Verbs([]string{"delete"})}},
|
||||
}
|
||||
|
||||
iter := json.CaseSensitiveJSONIterator()
|
||||
for i, c := range cases {
|
||||
var result APIResource
|
||||
if err := iter.Unmarshal([]byte(c.input), &result); err != nil {
|
||||
if err := utiljson.Unmarshal([]byte(c.input), &result); err != nil {
|
||||
t.Errorf("[%d] Failed to unmarshal input '%v': %v", i, c.input, err)
|
||||
}
|
||||
if !reflect.DeepEqual(result, c.result) {
|
||||
|
@ -274,7 +274,7 @@ func TestRoundTrip(t *testing.T) {
|
||||
{
|
||||
// Test slice of interface{} with different values.
|
||||
obj: &D{
|
||||
A: []interface{}{3.0, "3.0", nil},
|
||||
A: []interface{}{float64(3.5), int64(4), "3.0", nil},
|
||||
},
|
||||
},
|
||||
}
|
||||
@ -322,11 +322,11 @@ func TestUnrecognized(t *testing.T) {
|
||||
err error
|
||||
}{
|
||||
{
|
||||
data: "{\"da\":[3.0,\"3.0\",null]}",
|
||||
data: "{\"da\":[3.5,4,\"3.0\",null]}",
|
||||
obj: &D{},
|
||||
},
|
||||
{
|
||||
data: "{\"ea\":[3.0,\"3.0\",null]}",
|
||||
data: "{\"ea\":[3.5,4,\"3.0\",null]}",
|
||||
obj: &E{},
|
||||
},
|
||||
{
|
||||
|
@ -19,6 +19,7 @@ package runtime
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
)
|
||||
@ -124,20 +125,26 @@ func IsMissingVersion(err error) bool {
|
||||
// strictDecodingError is a base error type that is returned by a strict Decoder such
|
||||
// as UniversalStrictDecoder.
|
||||
type strictDecodingError struct {
|
||||
message string
|
||||
data string
|
||||
errors []error
|
||||
}
|
||||
|
||||
// NewStrictDecodingError creates a new strictDecodingError object.
|
||||
func NewStrictDecodingError(message string, data string) error {
|
||||
func NewStrictDecodingError(errors []error) error {
|
||||
return &strictDecodingError{
|
||||
message: message,
|
||||
data: data,
|
||||
errors: errors,
|
||||
}
|
||||
}
|
||||
|
||||
func (e *strictDecodingError) Error() string {
|
||||
return fmt.Sprintf("strict decoder error for %s: %s", e.data, e.message)
|
||||
var s strings.Builder
|
||||
s.WriteString("strict decoding error: ")
|
||||
for i, err := range e.errors {
|
||||
if i != 0 {
|
||||
s.WriteString(", ")
|
||||
}
|
||||
s.WriteString(err.Error())
|
||||
}
|
||||
return s.String()
|
||||
}
|
||||
|
||||
// IsStrictDecodingError returns true if the error indicates that the provided object
|
||||
|
@ -20,10 +20,8 @@ import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"strconv"
|
||||
"unsafe"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
"github.com/modern-go/reflect2"
|
||||
kjson "sigs.k8s.io/json"
|
||||
"sigs.k8s.io/yaml"
|
||||
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
@ -110,79 +108,6 @@ type Serializer struct {
|
||||
var _ runtime.Serializer = &Serializer{}
|
||||
var _ recognizer.RecognizingDecoder = &Serializer{}
|
||||
|
||||
type customNumberExtension struct {
|
||||
jsoniter.DummyExtension
|
||||
}
|
||||
|
||||
func (cne *customNumberExtension) CreateDecoder(typ reflect2.Type) jsoniter.ValDecoder {
|
||||
if typ.String() == "interface {}" {
|
||||
return customNumberDecoder{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type customNumberDecoder struct {
|
||||
}
|
||||
|
||||
func (customNumberDecoder) Decode(ptr unsafe.Pointer, iter *jsoniter.Iterator) {
|
||||
switch iter.WhatIsNext() {
|
||||
case jsoniter.NumberValue:
|
||||
var number jsoniter.Number
|
||||
iter.ReadVal(&number)
|
||||
i64, err := strconv.ParseInt(string(number), 10, 64)
|
||||
if err == nil {
|
||||
*(*interface{})(ptr) = i64
|
||||
return
|
||||
}
|
||||
f64, err := strconv.ParseFloat(string(number), 64)
|
||||
if err == nil {
|
||||
*(*interface{})(ptr) = f64
|
||||
return
|
||||
}
|
||||
iter.ReportError("DecodeNumber", err.Error())
|
||||
default:
|
||||
*(*interface{})(ptr) = iter.Read()
|
||||
}
|
||||
}
|
||||
|
||||
// CaseSensitiveJSONIterator returns a jsoniterator API that's configured to be
|
||||
// case-sensitive when unmarshalling, and otherwise compatible with
|
||||
// the encoding/json standard library.
|
||||
func CaseSensitiveJSONIterator() jsoniter.API {
|
||||
config := jsoniter.Config{
|
||||
EscapeHTML: true,
|
||||
SortMapKeys: true,
|
||||
ValidateJsonRawMessage: true,
|
||||
CaseSensitive: true,
|
||||
}.Froze()
|
||||
// Force jsoniter to decode number to interface{} via int64/float64, if possible.
|
||||
config.RegisterExtension(&customNumberExtension{})
|
||||
return config
|
||||
}
|
||||
|
||||
// StrictCaseSensitiveJSONIterator returns a jsoniterator API that's configured to be
|
||||
// case-sensitive, but also disallows unknown fields when unmarshalling. It is compatible with
|
||||
// the encoding/json standard library.
|
||||
func StrictCaseSensitiveJSONIterator() jsoniter.API {
|
||||
config := jsoniter.Config{
|
||||
EscapeHTML: true,
|
||||
SortMapKeys: true,
|
||||
ValidateJsonRawMessage: true,
|
||||
CaseSensitive: true,
|
||||
DisallowUnknownFields: true,
|
||||
}.Froze()
|
||||
// Force jsoniter to decode number to interface{} via int64/float64, if possible.
|
||||
config.RegisterExtension(&customNumberExtension{})
|
||||
return config
|
||||
}
|
||||
|
||||
// Private copies of jsoniter to try to shield against possible mutations
|
||||
// from outside. Still does not protect from package level jsoniter.Register*() functions - someone calling them
|
||||
// in some other library will mess with every usage of the jsoniter library in the whole program.
|
||||
// See https://github.com/json-iterator/go/issues/265
|
||||
var caseSensitiveJSONIterator = CaseSensitiveJSONIterator()
|
||||
var strictCaseSensitiveJSONIterator = StrictCaseSensitiveJSONIterator()
|
||||
|
||||
// gvkWithDefaults returns group kind and version defaulting from provided default
|
||||
func gvkWithDefaults(actual, defaultGVK schema.GroupVersionKind) schema.GroupVersionKind {
|
||||
if len(actual.Kind) == 0 {
|
||||
@ -237,7 +162,7 @@ func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, i
|
||||
types, _, err := s.typer.ObjectKinds(into)
|
||||
switch {
|
||||
case runtime.IsNotRegisteredError(err), isUnstructured:
|
||||
if err := caseSensitiveJSONIterator.Unmarshal(data, into); err != nil {
|
||||
if err := kjson.UnmarshalCaseSensitivePreserveInts(data, into); err != nil {
|
||||
return nil, actual, err
|
||||
}
|
||||
return into, actual, nil
|
||||
@ -261,35 +186,35 @@ func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, i
|
||||
return nil, actual, err
|
||||
}
|
||||
|
||||
if err := caseSensitiveJSONIterator.Unmarshal(data, obj); err != nil {
|
||||
return nil, actual, err
|
||||
}
|
||||
|
||||
// If the deserializer is non-strict, return successfully here.
|
||||
// If the deserializer is non-strict, return here.
|
||||
if !s.options.Strict {
|
||||
if err := kjson.UnmarshalCaseSensitivePreserveInts(data, obj); err != nil {
|
||||
return nil, actual, err
|
||||
}
|
||||
return obj, actual, nil
|
||||
}
|
||||
|
||||
// In strict mode pass the data trough the YAMLToJSONStrict converter.
|
||||
// This is done to catch duplicate fields regardless of encoding (JSON or YAML). For JSON data,
|
||||
// the output would equal the input, unless there is a parsing error such as duplicate fields.
|
||||
// As we know this was successful in the non-strict case, the only error that may be returned here
|
||||
// is because of the newly-added strictness. hence we know we can return the typed strictDecoderError
|
||||
// the actual error is that the object contains duplicate fields.
|
||||
altered, err := yaml.YAMLToJSONStrict(originalData)
|
||||
var allStrictErrs []error
|
||||
if s.options.Yaml {
|
||||
// In strict mode pass the original data through the YAMLToJSONStrict converter.
|
||||
// This is done to catch duplicate fields in YAML that would have been dropped in the original YAMLToJSON conversion.
|
||||
// TODO: rework YAMLToJSONStrict to return warnings about duplicate fields without terminating so we don't have to do this twice.
|
||||
_, err := yaml.YAMLToJSONStrict(originalData)
|
||||
if err != nil {
|
||||
allStrictErrs = append(allStrictErrs, err)
|
||||
}
|
||||
}
|
||||
|
||||
strictJSONErrs, err := kjson.UnmarshalStrict(data, obj)
|
||||
if err != nil {
|
||||
return nil, actual, runtime.NewStrictDecodingError(err.Error(), string(originalData))
|
||||
// fatal decoding error, not due to strictness
|
||||
return nil, actual, err
|
||||
}
|
||||
// As performance is not an issue for now for the strict deserializer (one has regardless to do
|
||||
// the unmarshal twice), we take the sanitized, altered data that is guaranteed to have no duplicated
|
||||
// fields, and unmarshal this into a copy of the already-populated obj. Any error that occurs here is
|
||||
// due to that a matching field doesn't exist in the object. hence we can return a typed strictDecoderError,
|
||||
// the actual error is that the object contains unknown field.
|
||||
strictObj := obj.DeepCopyObject()
|
||||
if err := strictCaseSensitiveJSONIterator.Unmarshal(altered, strictObj); err != nil {
|
||||
return nil, actual, runtime.NewStrictDecodingError(err.Error(), string(originalData))
|
||||
allStrictErrs = append(allStrictErrs, strictJSONErrs...)
|
||||
if len(allStrictErrs) > 0 {
|
||||
// return the successfully decoded object along with the strict errors
|
||||
return obj, actual, runtime.NewStrictDecodingError(allStrictErrs)
|
||||
}
|
||||
// Always return the same object as the non-strict serializer to avoid any deviations.
|
||||
return obj, actual, nil
|
||||
}
|
||||
|
||||
|
@ -123,7 +123,6 @@ func testcases() []testcase {
|
||||
var decoders = map[string]func([]byte, interface{}) error{
|
||||
"gojson": gojson.Unmarshal,
|
||||
"utiljson": utiljson.Unmarshal,
|
||||
"jsoniter": CaseSensitiveJSONIterator().Unmarshal,
|
||||
}
|
||||
|
||||
func TestJSONLimits(t *testing.T) {
|
||||
|
@ -39,8 +39,7 @@ type testDecodable struct {
|
||||
Interface interface{} `json:"interface"`
|
||||
}
|
||||
|
||||
// DecodableSpec has 15 fields. json-iterator treats struct with more than 10
|
||||
// fields differently from struct that has less than 10 fields.
|
||||
// DecodableSpec has 15 fields.
|
||||
type DecodableSpec struct {
|
||||
A int `json:"A"`
|
||||
B int `json:"B"`
|
||||
@ -264,7 +263,7 @@ func TestDecode(t *testing.T) {
|
||||
creater: &mockCreater{obj: &testDecodable{}},
|
||||
expectedGVK: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"},
|
||||
errFn: func(err error) bool {
|
||||
return strings.Contains(err.Error(), `json_test.testDecodable.Interface: DecodeNumber: strconv.ParseFloat: parsing "1e1000": value out of range`)
|
||||
return strings.Contains(err.Error(), `json: cannot unmarshal number 1e1000 into Go struct field testDecodable.interface of type float64`)
|
||||
},
|
||||
},
|
||||
// Unmarshalling is case-sensitive
|
||||
@ -298,7 +297,7 @@ func TestDecode(t *testing.T) {
|
||||
typer: &mockTyper{gvk: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}},
|
||||
expectedGVK: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"},
|
||||
errFn: func(err error) bool {
|
||||
return strings.Contains(err.Error(), "found unknown field")
|
||||
return strings.Contains(err.Error(), `unknown field "unknown"`)
|
||||
},
|
||||
strict: true,
|
||||
},
|
||||
@ -309,7 +308,7 @@ func TestDecode(t *testing.T) {
|
||||
typer: &mockTyper{gvk: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}},
|
||||
expectedGVK: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"},
|
||||
errFn: func(err error) bool {
|
||||
return strings.Contains(err.Error(), "found unknown field: unknown")
|
||||
return strings.Contains(err.Error(), `unknown field "unknown"`)
|
||||
},
|
||||
yaml: true,
|
||||
strict: true,
|
||||
@ -321,7 +320,7 @@ func TestDecode(t *testing.T) {
|
||||
typer: &mockTyper{gvk: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"}},
|
||||
expectedGVK: &schema.GroupVersionKind{Kind: "Test", Group: "other", Version: "blah"},
|
||||
errFn: func(err error) bool {
|
||||
return strings.Contains(err.Error(), `"value" already set in map`)
|
||||
return strings.Contains(err.Error(), `duplicate field "value"`)
|
||||
},
|
||||
strict: true,
|
||||
},
|
||||
@ -526,7 +525,7 @@ func TestDecode(t *testing.T) {
|
||||
if !test.errFn(err) {
|
||||
t.Errorf("%d: failed: %v", i, err)
|
||||
}
|
||||
if obj != nil {
|
||||
if !runtime.IsStrictDecodingError(err) && obj != nil {
|
||||
t.Errorf("%d: should have returned nil object", i)
|
||||
}
|
||||
continue
|
||||
|
@ -17,10 +17,11 @@ limitations under the License.
|
||||
package json
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
kjson "sigs.k8s.io/json"
|
||||
)
|
||||
|
||||
// NewEncoder delegates to json.NewEncoder
|
||||
@ -38,50 +39,11 @@ func Marshal(v interface{}) ([]byte, error) {
|
||||
// limit recursive depth to prevent stack overflow errors
|
||||
const maxDepth = 10000
|
||||
|
||||
// Unmarshal unmarshals the given data
|
||||
// If v is a *map[string]interface{}, *[]interface{}, or *interface{} numbers
|
||||
// are converted to int64 or float64
|
||||
// Unmarshal unmarshals the given data.
|
||||
// Object keys are case-sensitive.
|
||||
// Numbers decoded into interface{} fields are converted to int64 or float64.
|
||||
func Unmarshal(data []byte, v interface{}) error {
|
||||
switch v := v.(type) {
|
||||
case *map[string]interface{}:
|
||||
// Build a decoder from the given data
|
||||
decoder := json.NewDecoder(bytes.NewBuffer(data))
|
||||
// Preserve numbers, rather than casting to float64 automatically
|
||||
decoder.UseNumber()
|
||||
// Run the decode
|
||||
if err := decoder.Decode(v); err != nil {
|
||||
return err
|
||||
}
|
||||
// If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64
|
||||
return ConvertMapNumbers(*v, 0)
|
||||
|
||||
case *[]interface{}:
|
||||
// Build a decoder from the given data
|
||||
decoder := json.NewDecoder(bytes.NewBuffer(data))
|
||||
// Preserve numbers, rather than casting to float64 automatically
|
||||
decoder.UseNumber()
|
||||
// Run the decode
|
||||
if err := decoder.Decode(v); err != nil {
|
||||
return err
|
||||
}
|
||||
// If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64
|
||||
return ConvertSliceNumbers(*v, 0)
|
||||
|
||||
case *interface{}:
|
||||
// Build a decoder from the given data
|
||||
decoder := json.NewDecoder(bytes.NewBuffer(data))
|
||||
// Preserve numbers, rather than casting to float64 automatically
|
||||
decoder.UseNumber()
|
||||
// Run the decode
|
||||
if err := decoder.Decode(v); err != nil {
|
||||
return err
|
||||
}
|
||||
// If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64
|
||||
return ConvertInterfaceNumbers(v, 0)
|
||||
|
||||
default:
|
||||
return json.Unmarshal(data, v)
|
||||
}
|
||||
return kjson.UnmarshalCaseSensitivePreserveInts(data, v)
|
||||
}
|
||||
|
||||
// ConvertInterfaceNumbers converts any json.Number values to int64 or float64.
|
||||
|
@ -34,6 +34,7 @@ import (
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/runtime/serializer/json"
|
||||
utiljson "k8s.io/apimachinery/pkg/util/json"
|
||||
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
||||
"k8s.io/apiserver/pkg/admission"
|
||||
admissionmetrics "k8s.io/apiserver/pkg/admission/metrics"
|
||||
@ -61,8 +62,6 @@ const (
|
||||
MutationAuditAnnotationFailedOpenKeyPrefix string = "failed-open." + MutationAuditAnnotationPrefix
|
||||
)
|
||||
|
||||
var encodingjson = json.CaseSensitiveJSONIterator()
|
||||
|
||||
type mutatingDispatcher struct {
|
||||
cm *webhookutil.ClientManager
|
||||
plugin *Plugin
|
||||
@ -444,7 +443,7 @@ func mutationAnnotationValue(configuration, webhook string, mutated bool) (strin
|
||||
Webhook: webhook,
|
||||
Mutated: mutated,
|
||||
}
|
||||
bytes, err := encodingjson.Marshal(m)
|
||||
bytes, err := utiljson.Marshal(m)
|
||||
return string(bytes), err
|
||||
}
|
||||
|
||||
@ -455,6 +454,6 @@ func jsonPatchAnnotationValue(configuration, webhook string, patch interface{})
|
||||
Patch: patch,
|
||||
PatchType: string(admissionv1.PatchTypeJSONPatch),
|
||||
}
|
||||
bytes, err := encodingjson.Marshal(p)
|
||||
bytes, err := utiljson.Marshal(p)
|
||||
return string(bytes), err
|
||||
}
|
||||
|
@ -50,7 +50,7 @@ import (
|
||||
utiltesting "k8s.io/client-go/util/testing"
|
||||
|
||||
// TODO we need to remove this linkage and create our own scheme
|
||||
"k8s.io/api/core/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/client-go/kubernetes/scheme"
|
||||
)
|
||||
|
||||
@ -1762,7 +1762,7 @@ func TestUnstructured(t *testing.T) {
|
||||
{
|
||||
name: "badpod",
|
||||
file: "badpod.json",
|
||||
expectedError: "v1.ObjectMeta.Annotations",
|
||||
expectedError: "ObjectMeta.",
|
||||
},
|
||||
}
|
||||
|
||||
|
@ -20,12 +20,9 @@ import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/runtime/serializer/json"
|
||||
utiljson "k8s.io/apimachinery/pkg/util/json"
|
||||
)
|
||||
|
||||
// hold a single instance of the case-sensitive decoder
|
||||
var caseSensitiveJsonIterator = json.CaseSensitiveJSONIterator()
|
||||
|
||||
// metadataValidatingDecoder wraps a decoder and additionally ensures metadata schema fields decode before returning an unstructured object
|
||||
type metadataValidatingDecoder struct {
|
||||
decoder runtime.Decoder
|
||||
@ -47,7 +44,7 @@ func (m *metadataValidatingDecoder) Decode(data []byte, defaults *schema.GroupVe
|
||||
// make sure the data can decode into ObjectMeta before we return,
|
||||
// so we don't silently truncate schema errors in metadata later with accesser get/set calls
|
||||
v := &metadataOnlyObject{}
|
||||
if typedErr := caseSensitiveJsonIterator.Unmarshal(data, v); typedErr != nil {
|
||||
if typedErr := utiljson.Unmarshal(data, v); typedErr != nil {
|
||||
return obj, gvk, typedErr
|
||||
}
|
||||
return obj, gvk, err
|
||||
|
@ -22,8 +22,7 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
|
||||
utiljson "k8s.io/apimachinery/pkg/util/json"
|
||||
"k8s.io/apiserver/pkg/authentication/user"
|
||||
"k8s.io/apiserver/pkg/endpoints/request"
|
||||
"k8s.io/kube-openapi/pkg/validation/spec"
|
||||
@ -80,7 +79,7 @@ func (s *Downloader) Download(handler http.Handler, etag string) (returnSpec *sp
|
||||
return nil, "", http.StatusNotFound, nil
|
||||
case http.StatusOK:
|
||||
openAPISpec := &spec.Swagger{}
|
||||
if err := jsoniter.ConfigCompatibleWithStandardLibrary.Unmarshal(writer.data, openAPISpec); err != nil {
|
||||
if err := utiljson.Unmarshal(writer.data, openAPISpec); err != nil {
|
||||
return nil, "", 0, err
|
||||
}
|
||||
newEtag = writer.Header().Get("Etag")
|
||||
|
@ -277,7 +277,7 @@ exemptions:
|
||||
"apiVersion":"pod-security.admission.config.k8s.io/v1alpha1",
|
||||
"kind":"PodSecurityConfiguration",
|
||||
"deflaults":{"enforce":"baseline"}}`),
|
||||
expectErr: `unknown field: deflaults`,
|
||||
expectErr: `unknown field "deflaults"`,
|
||||
},
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user