mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-22 03:11:40 +00:00
Use json-iterator for JSON, kill off codecgen
This commit is contained in:
Tim Hockin
parent
58aa139f99
commit
5728b1970a
@ -34,7 +34,6 @@ fi
|
||||
# Some things we want in godeps aren't code dependencies, so ./...
|
||||
# won't pick them up.
|
||||
REQUIRED_BINS=(
|
||||
"github.com/ugorji/go/codec/codecgen"
|
||||
"github.com/onsi/ginkgo/ginkgo"
|
||||
"github.com/jteeuwen/go-bindata/go-bindata"
|
||||
"./..."
|
||||
|
||||
@ -383,7 +383,7 @@ kube::golang::setup_env() {
|
||||
# Unset GOBIN in case it already exists in the current session.
|
||||
unset GOBIN
|
||||
|
||||
# This seems to matter to some tools (godep, ugorji, ginkgo...)
|
||||
# This seems to matter to some tools (godep, ginkgo...)
|
||||
export GO15VENDOREXPERIMENT=1
|
||||
}
|
||||
|
||||
|
||||
@ -63,7 +63,6 @@ fi
|
||||
BASH_TARGETS="
|
||||
update-generated-protobuf
|
||||
update-codegen
|
||||
update-codecgen
|
||||
update-generated-docs
|
||||
update-generated-swagger-docs
|
||||
update-swagger-spec
|
||||
|
||||
@ -1,176 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright 2015 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
|
||||
source "${KUBE_ROOT}/hack/lib/init.sh"
|
||||
|
||||
kube::golang::setup_env
|
||||
|
||||
# The sort at the end makes sure we feed the topological sort a deterministic
|
||||
# list (since there aren't many dependencies).
|
||||
|
||||
generated_files=($(
|
||||
find . -not \( \
|
||||
\( \
|
||||
-wholename './output' \
|
||||
-o -wholename './_output' \
|
||||
-o -wholename './staging' \
|
||||
-o -wholename './release' \
|
||||
-o -wholename './target' \
|
||||
-o -wholename '*/third_party/*' \
|
||||
-o -wholename '*/vendor/*' \
|
||||
-o -wholename '*/codecgen-*-1234.generated.go' \
|
||||
\) -prune \
|
||||
\) -name '*.generated.go' | LC_ALL=C sort -r
|
||||
|
||||
find ./staging/src -not \( \
|
||||
\( \
|
||||
-wholename './output' \
|
||||
-o -wholename './_output' \
|
||||
-o -wholename './release' \
|
||||
-o -wholename './target' \
|
||||
-o -wholename '*/third_party/*' \
|
||||
-o -wholename '*/codecgen-*-1234.generated.go' \
|
||||
\) -prune \
|
||||
\) -name '*.generated.go' | sed 's,staging/src,vendor,' | LC_ALL=C sort -r
|
||||
))
|
||||
number=${#generated_files[@]}
|
||||
###for f in $(echo "${generated_files[@]}" | LC_ALL=C sort); do
|
||||
### echo "DBG: generated_files: $f"
|
||||
###done
|
||||
|
||||
function package () {
|
||||
dirname "${1}" | sed 's,\./,k8s.io/kubernetes/,'
|
||||
}
|
||||
|
||||
# extract package list we care about.
|
||||
dep_packages=($(
|
||||
for f in "${generated_files[@]}"; do
|
||||
package "${f}"
|
||||
done | LC_ALL=C sort -u
|
||||
))
|
||||
###for f in $(echo "${dep_packages[@]}"); do
|
||||
### echo "DBG: dep_packages: $f"
|
||||
###done
|
||||
|
||||
# Register function to be called on EXIT to remove codecgen
|
||||
# binary and also to touch the files that should be regenerated
|
||||
# since they are first removed.
|
||||
# This is necessary to make the script work after previous failure.
|
||||
function cleanup {
|
||||
rm -f "${CODECGEN:-}"
|
||||
pushd "${KUBE_ROOT}" > /dev/null
|
||||
for f in "${generated_files[@]}"; do
|
||||
touch "${f}" || true
|
||||
done
|
||||
popd > /dev/null
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# Precompute dependencies for all directories.
|
||||
# Then sort all files in the dependency order.
|
||||
result=""
|
||||
dep_packages_grep_pattern=$(echo ${dep_packages[@]} | tr " " '|')
|
||||
###echo "DBG: dep_packages_grep_pattern: ${dep_packages_grep_pattern}"
|
||||
for (( i=0; i<number; i++ )); do
|
||||
visited[${i}]=false
|
||||
file="${generated_files[${i}]/\.generated\.go/.go}"
|
||||
deps[${i}]=$(go list -f '{{range .Deps}}{{.}}{{"\n"}}{{end}}' ${file} | grep -Eow "^\\(${dep_packages_grep_pattern}\\)\$" | tr $"\n" ' ' || true)
|
||||
###echo "DBG: deps[$file]: ${deps[$i]}"
|
||||
done
|
||||
|
||||
function depends {
|
||||
rhs="$(package ${generated_files[$2]})"
|
||||
####echo "DBG: does generated_files[$1] depend on ${rhs}?"
|
||||
for dep in ${deps[$1]}; do
|
||||
###echo "DBG: checking against $dep"
|
||||
if [[ "${dep}" == "${rhs}" ]]; then
|
||||
###echo "DBG: = yes"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
###echo "DBG: = no"
|
||||
return 1
|
||||
}
|
||||
|
||||
function tsort {
|
||||
visited[$1]=true
|
||||
local j=0
|
||||
for (( j=0; j<number; j++ )); do
|
||||
if ! ${visited[${j}]}; then
|
||||
if depends "$1" ${j}; then
|
||||
tsort $j
|
||||
fi
|
||||
fi
|
||||
done
|
||||
result="${result} $1"
|
||||
}
|
||||
echo "Building dependencies"
|
||||
for (( i=0; i<number; i++ )); do
|
||||
###echo "DBG: considering ${generated_files[${i}]}"
|
||||
if ! ${visited[${i}]}; then
|
||||
###echo "DBG: tsorting ${generated_files[${i}]}"
|
||||
tsort ${i}
|
||||
fi
|
||||
done
|
||||
index=(${result})
|
||||
|
||||
haveindex=${index:-}
|
||||
if [[ -z ${haveindex} ]]; then
|
||||
echo No files found for $0
|
||||
echo A previous run of $0 may have deleted all the files and then crashed.
|
||||
echo Use 'touch' to create files named 'types.generated.go' listed as deleted in 'git status'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Building codecgen"
|
||||
make generated_files
|
||||
CODECGEN="${PWD}/codecgen_binary"
|
||||
go build -o "${CODECGEN}" ./vendor/github.com/ugorji/go/codec/codecgen
|
||||
|
||||
# Running codecgen fails if some of the files doesn't compile.
|
||||
# Thus (since all the files are completely auto-generated and
|
||||
# not required for the code to be compilable, we first remove
|
||||
# them and the regenerate them.
|
||||
for (( i=0; i < number; i++ )); do
|
||||
rm -f "${generated_files[${i}]}"
|
||||
done
|
||||
|
||||
# Generate files in the dependency order.
|
||||
for current in "${index[@]}"; do
|
||||
generated_file=${generated_files[${current}]}
|
||||
initial_dir=${PWD}
|
||||
file=${generated_file/\.generated\.go/.go}
|
||||
echo "processing ${file}"
|
||||
# codecgen work only if invoked from directory where the file
|
||||
# is located.
|
||||
pushd "$(dirname ${file})" > /dev/null
|
||||
base_file=$(basename "${file}")
|
||||
base_generated_file=$(basename "${generated_file}")
|
||||
# We use '-d 1234' flag to have a deterministic output every time.
|
||||
# The constant was just randomly chosen.
|
||||
###echo "DBG: running ${CODECGEN} -d 1234 -o ${base_generated_file} ${base_file}"
|
||||
${CODECGEN} -d 1234 -o "${base_generated_file}" "${base_file}"
|
||||
# Add boilerplate at the beginning of the generated file.
|
||||
sed 's/YEAR/2016/' "${initial_dir}/hack/boilerplate/boilerplate.go.txt" > "${base_generated_file}.tmp"
|
||||
cat "${base_generated_file}" >> "${base_generated_file}.tmp"
|
||||
mv "${base_generated_file}.tmp" "${base_generated_file}"
|
||||
popd > /dev/null
|
||||
done
|
||||
@ -1,67 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright 2015 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
|
||||
source "${KUBE_ROOT}/hack/lib/init.sh"
|
||||
|
||||
kube::golang::setup_env
|
||||
|
||||
generated_files=($(
|
||||
find . -not \( \
|
||||
\( \
|
||||
-wholename './output' \
|
||||
-o -wholename './_output' \
|
||||
-o -wholename './release' \
|
||||
-o -wholename './target' \
|
||||
-o -wholename '*/third_party/*' \
|
||||
-o -wholename '*/vendor/*' \
|
||||
\) -prune \
|
||||
\) -name '*.generated.go'))
|
||||
|
||||
function cleanup {
|
||||
for generated_file in ${generated_files[@]}; do
|
||||
rm -f "${generated_file}.original"
|
||||
done
|
||||
}
|
||||
trap cleanup EXIT SIGINT
|
||||
|
||||
for generated_file in ${generated_files[@]}; do
|
||||
cat "${generated_file}" > "${generated_file}.original"
|
||||
done
|
||||
|
||||
${KUBE_ROOT}/hack/update-codecgen.sh
|
||||
|
||||
ret=0
|
||||
# Generate files in the dependency order.
|
||||
for generated_file in ${generated_files[@]}; do
|
||||
cur=0
|
||||
diff -Naupr -I 'Auto generated by' "${generated_file}" "${generated_file}.original" || cur=$?
|
||||
if [[ $cur -eq 0 ]]; then
|
||||
echo "${generated_file} up to date."
|
||||
else
|
||||
echo "${generated_file} was out of date. Please run hack/update-codecgen.sh. (If you're running locally, this was run for you already.)"
|
||||
ret=1
|
||||
fi
|
||||
|
||||
done
|
||||
|
||||
if [ "${ret}" -ne "0" ]; then
|
||||
exit $ret
|
||||
fi
|
||||
@ -27,7 +27,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/ugorji/go/codec"
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/api/extensions/v1beta1"
|
||||
@ -536,7 +536,7 @@ func BenchmarkDecodeIntoJSON(b *testing.B) {
|
||||
b.StopTimer()
|
||||
}
|
||||
|
||||
// BenchmarkDecodeJSON provides a baseline for codecgen JSON decode performance
|
||||
// BenchmarkDecodeJSON provides a baseline for JSON decode performance
|
||||
func BenchmarkDecodeIntoJSONCodecGen(b *testing.B) {
|
||||
kcodec := testapi.Default.Codec()
|
||||
items := benchmarkItems(b)
|
||||
@ -549,12 +549,11 @@ func BenchmarkDecodeIntoJSONCodecGen(b *testing.B) {
|
||||
}
|
||||
encoded[i] = data
|
||||
}
|
||||
handler := &codec.JsonHandle{}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
obj := v1.Pod{}
|
||||
if err := codec.NewDecoderBytes(encoded[i%width], handler).Decode(&obj); err != nil {
|
||||
if err := jsoniter.ConfigFastest.Unmarshal(encoded[i%width], &obj); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
@ -145,15 +145,11 @@ func v1alpha1FuzzerFuncs(codecs runtimeserializer.CodecFactory) []interface{} {
|
||||
case 2:
|
||||
r.Cells[i] = c.RandBool()
|
||||
case 3:
|
||||
// maps roundtrip as map[interface{}]interface{}, but the json codec cannot encode that
|
||||
// TODO: get maps to roundtrip properly
|
||||
/*
|
||||
x := map[string]interface{}{}
|
||||
for j := c.Intn(10) + 1; j >= 0; j-- {
|
||||
x[c.RandString()] = c.RandString()
|
||||
}
|
||||
r.Cells[i] = x
|
||||
*/
|
||||
x := map[string]interface{}{}
|
||||
for j := c.Intn(10) + 1; j >= 0; j-- {
|
||||
x[c.RandString()] = c.RandString()
|
||||
}
|
||||
r.Cells[i] = x
|
||||
case 4:
|
||||
x := make([]interface{}, c.Intn(10))
|
||||
for i := range x {
|
||||
|
||||
@ -21,7 +21,7 @@ import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/ugorji/go/codec"
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
)
|
||||
|
||||
type GroupVersionHolder struct {
|
||||
@ -46,12 +46,12 @@ func TestGroupVersionUnmarshalJSON(t *testing.T) {
|
||||
if !reflect.DeepEqual(result.GV, c.expect) {
|
||||
t.Errorf("JSON codec failed to unmarshal input '%s': expected %+v, got %+v", c.input, c.expect, result.GV)
|
||||
}
|
||||
// test the Ugorji codec
|
||||
if err := codec.NewDecoderBytes(c.input, new(codec.JsonHandle)).Decode(&result); err != nil {
|
||||
t.Errorf("Ugorji codec failed to unmarshal input '%v': %v", c.input, err)
|
||||
// test the json-iterator codec
|
||||
if err := jsoniter.ConfigFastest.Unmarshal(c.input, &result); err != nil {
|
||||
t.Errorf("json-iterator codec failed to unmarshal input '%v': %v", c.input, err)
|
||||
}
|
||||
if !reflect.DeepEqual(result.GV, c.expect) {
|
||||
t.Errorf("Ugorji codec failed to unmarshal input '%s': expected %+v, got %+v", c.input, c.expect, result.GV)
|
||||
t.Errorf("json-iterator codec failed to unmarshal input '%s': expected %+v, got %+v", c.input, c.expect, result.GV)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -21,7 +21,7 @@ import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/ugorji/go/codec"
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
)
|
||||
|
||||
func TestVerbsUgorjiMarshalJSON(t *testing.T) {
|
||||
@ -58,7 +58,7 @@ func TestVerbsUgorjiUnmarshalJSON(t *testing.T) {
|
||||
|
||||
for i, c := range cases {
|
||||
var result APIResource
|
||||
if err := codec.NewDecoderBytes([]byte(c.input), new(codec.JsonHandle)).Decode(&result); err != nil {
|
||||
if err := jsoniter.ConfigFastest.Unmarshal([]byte(c.input), &result); err != nil {
|
||||
t.Errorf("[%d] Failed to unmarshal input '%v': %v", i, c.input, err)
|
||||
}
|
||||
if !reflect.DeepEqual(result, c.result) {
|
||||
|
||||
@ -19,9 +19,11 @@ package json
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"strconv"
|
||||
"unsafe"
|
||||
|
||||
"github.com/ghodss/yaml"
|
||||
"github.com/ugorji/go/codec"
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
@ -66,6 +68,36 @@ type Serializer struct {
|
||||
var _ runtime.Serializer = &Serializer{}
|
||||
var _ recognizer.RecognizingDecoder = &Serializer{}
|
||||
|
||||
func init() {
|
||||
// Force jsoniter to decode number to interface{} via ints, if possible.
|
||||
decodeNumberAsInt64IfPossible := func(ptr unsafe.Pointer, iter *jsoniter.Iterator) {
|
||||
switch iter.WhatIsNext() {
|
||||
case jsoniter.NumberValue:
|
||||
var number json.Number
|
||||
iter.ReadVal(&number)
|
||||
u64, err := strconv.ParseUint(string(number), 10, 64)
|
||||
if err == nil {
|
||||
*(*interface{})(ptr) = u64
|
||||
return
|
||||
}
|
||||
i64, err := strconv.ParseInt(string(number), 10, 64)
|
||||
if err == nil {
|
||||
*(*interface{})(ptr) = i64
|
||||
return
|
||||
}
|
||||
f64, err := strconv.ParseFloat(string(number), 64)
|
||||
if err == nil {
|
||||
*(*interface{})(ptr) = f64
|
||||
return
|
||||
}
|
||||
// Not much we can do here.
|
||||
default:
|
||||
*(*interface{})(ptr) = iter.Read()
|
||||
}
|
||||
}
|
||||
jsoniter.RegisterTypeDecoderFunc("interface {}", decodeNumberAsInt64IfPossible)
|
||||
}
|
||||
|
||||
// Decode attempts to convert the provided data into YAML or JSON, extract the stored schema kind, apply the provided default gvk, and then
|
||||
// load that data into an object matching the desired schema kind or the provided into. If into is *runtime.Unknown, the raw data will be
|
||||
// extracted and no decoding will be performed. If into is not registered with the typer, then the object will be straight decoded using
|
||||
@ -121,7 +153,7 @@ func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, i
|
||||
types, _, err := s.typer.ObjectKinds(into)
|
||||
switch {
|
||||
case runtime.IsNotRegisteredError(err):
|
||||
if err := codec.NewDecoderBytes(data, new(codec.JsonHandle)).Decode(into); err != nil {
|
||||
if err := jsoniter.ConfigFastest.Unmarshal(data, into); err != nil {
|
||||
return nil, actual, err
|
||||
}
|
||||
return into, actual, nil
|
||||
@ -155,7 +187,7 @@ func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, i
|
||||
return nil, actual, err
|
||||
}
|
||||
|
||||
if err := codec.NewDecoderBytes(data, new(codec.JsonHandle)).Decode(obj); err != nil {
|
||||
if err := jsoniter.ConfigFastest.Unmarshal(data, obj); err != nil {
|
||||
return nil, actual, err
|
||||
}
|
||||
return obj, actual, nil
|
||||
@ -164,7 +196,7 @@ func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, i
|
||||
// Encode serializes the provided object to the given writer.
|
||||
func (s *Serializer) Encode(obj runtime.Object, w io.Writer) error {
|
||||
if s.yaml {
|
||||
json, err := json.Marshal(obj)
|
||||
json, err := jsoniter.ConfigFastest.Marshal(obj)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -177,7 +209,7 @@ func (s *Serializer) Encode(obj runtime.Object, w io.Writer) error {
|
||||
}
|
||||
|
||||
if s.pretty {
|
||||
data, err := json.MarshalIndent(obj, "", " ")
|
||||
data, err := jsoniter.ConfigFastest.MarshalIndent(obj, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user