Use json-iterator for JSON, kill off codecgen

hack/godep-save.sh

@@ -34,7 +34,6 @@ fi
 # Some things we want in godeps aren't code dependencies, so ./...
 # won't pick them up.
 REQUIRED_BINS=(
-  "github.com/ugorji/go/codec/codecgen"
   "github.com/onsi/ginkgo/ginkgo"
   "github.com/jteeuwen/go-bindata/go-bindata"
   "./..."

hack/lib/golang.sh

@@ -383,7 +383,7 @@ kube::golang::setup_env() {
   # Unset GOBIN in case it already exists in the current session.
   unset GOBIN
 
-  # This seems to matter to some tools (godep, ugorji, ginkgo...)
+  # This seems to matter to some tools (godep, ginkgo...)
   export GO15VENDOREXPERIMENT=1
 }
 

hack/update-all.sh

@@ -63,7 +63,6 @@ fi
 BASH_TARGETS="
 	update-generated-protobuf
 	update-codegen
-	update-codecgen
 	update-generated-docs
 	update-generated-swagger-docs
 	update-swagger-spec

hack/update-codecgen.sh

@@ -1,176 +0,0 @@
-#!/bin/bash
-
-# Copyright 2015 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
-source "${KUBE_ROOT}/hack/lib/init.sh"
-
-kube::golang::setup_env
-
-# The sort at the end makes sure we feed the topological sort a deterministic
-# list (since there aren't many dependencies).
-
-generated_files=($(
-  find . -not \( \
-      \( \
-        -wholename './output' \
-        -o -wholename './_output' \
-        -o -wholename './staging' \
-        -o -wholename './release' \
-        -o -wholename './target' \
-        -o -wholename '*/third_party/*' \
-        -o -wholename '*/vendor/*' \
-        -o -wholename '*/codecgen-*-1234.generated.go' \
-      \) -prune \
-    \) -name '*.generated.go' | LC_ALL=C sort -r
-
-  find ./staging/src -not \( \
-      \( \
-        -wholename './output' \
-        -o -wholename './_output' \
-        -o -wholename './release' \
-        -o -wholename './target' \
-        -o -wholename '*/third_party/*' \
-        -o -wholename '*/codecgen-*-1234.generated.go' \
-      \) -prune \
-    \) -name '*.generated.go' | sed 's,staging/src,vendor,' | LC_ALL=C sort -r
-))
-number=${#generated_files[@]}
-###for f in $(echo "${generated_files[@]}" | LC_ALL=C sort); do
-###    echo "DBG: generated_files:   $f"
-###done
-
-function package () {
-    dirname "${1}" | sed 's,\./,k8s.io/kubernetes/,'
-}
-
-# extract package list we care about.
-dep_packages=($(
-    for f in "${generated_files[@]}"; do
-        package "${f}"
-    done | LC_ALL=C sort -u
-))
-###for f in $(echo "${dep_packages[@]}"); do
-###    echo "DBG: dep_packages:   $f"
-###done
-
-# Register function to be called on EXIT to remove codecgen
-# binary and also to touch the files that should be regenerated
-# since they are first removed.
-# This is necessary to make the script work after previous failure.
-function cleanup {
-  rm -f "${CODECGEN:-}"
-  pushd "${KUBE_ROOT}" > /dev/null
-  for f in "${generated_files[@]}"; do
-    touch "${f}" || true
-  done
-  popd > /dev/null
-}
-trap cleanup EXIT
-
-# Precompute dependencies for all directories.
-# Then sort all files in the dependency order.
-result=""
-dep_packages_grep_pattern=$(echo ${dep_packages[@]} | tr " " '|')
-###echo "DBG: dep_packages_grep_pattern: ${dep_packages_grep_pattern}"
-for (( i=0; i<number; i++ )); do
-  visited[${i}]=false
-  file="${generated_files[${i}]/\.generated\.go/.go}"
-  deps[${i}]=$(go list -f '{{range .Deps}}{{.}}{{"\n"}}{{end}}' ${file} | grep -Eow "^\\(${dep_packages_grep_pattern}\\)\$" | tr $"\n" ' ' || true)
-  ###echo "DBG: deps[$file]:   ${deps[$i]}"
-done
-
-function depends {
-  rhs="$(package ${generated_files[$2]})"
-  ####echo "DBG: does generated_files[$1] depend on ${rhs}?"
-  for dep in ${deps[$1]}; do
-    ###echo "DBG:   checking against $dep"
-    if [[ "${dep}" == "${rhs}" ]]; then
-      ###echo "DBG: = yes"
-      return 0
-    fi
-  done
-  ###echo "DBG: = no"
-  return 1
-}
-
-function tsort {
-  visited[$1]=true
-  local j=0
-  for (( j=0; j<number; j++ )); do
-    if ! ${visited[${j}]}; then
-      if depends "$1" ${j}; then
-        tsort $j
-      fi
-    fi
-  done
-  result="${result} $1"
-}
-echo "Building dependencies"
-for (( i=0; i<number; i++ )); do
-  ###echo "DBG: considering ${generated_files[${i}]}"
-  if ! ${visited[${i}]}; then
-    ###echo "DBG: tsorting ${generated_files[${i}]}"
-    tsort ${i}
-  fi
-done
-index=(${result})
-
-haveindex=${index:-}
-if [[ -z ${haveindex} ]]; then
-  echo No files found for $0
-  echo A previous run of $0 may have deleted all the files and then crashed.
-  echo Use 'touch' to create files named 'types.generated.go' listed as deleted in 'git status'
-  exit 1
-fi
-
-echo "Building codecgen"
-make generated_files
-CODECGEN="${PWD}/codecgen_binary"
-go build -o "${CODECGEN}" ./vendor/github.com/ugorji/go/codec/codecgen
-
-# Running codecgen fails if some of the files doesn't compile.
-# Thus (since all the files are completely auto-generated and
-# not required for the code to be compilable, we first remove
-# them and the regenerate them.
-for (( i=0; i < number; i++ )); do
-  rm -f "${generated_files[${i}]}"
-done
-
-# Generate files in the dependency order.
-for current in "${index[@]}"; do
-  generated_file=${generated_files[${current}]}
-  initial_dir=${PWD}
-  file=${generated_file/\.generated\.go/.go}
-  echo "processing ${file}"
-  # codecgen work only if invoked from directory where the file
-  # is located.
-  pushd "$(dirname ${file})" > /dev/null
-  base_file=$(basename "${file}")
-  base_generated_file=$(basename "${generated_file}")
-  # We use '-d 1234' flag to have a deterministic output every time.
-  # The constant was just randomly chosen.
-  ###echo "DBG: running ${CODECGEN} -d 1234 -o ${base_generated_file} ${base_file}"
-  ${CODECGEN} -d 1234 -o "${base_generated_file}" "${base_file}"
-  # Add boilerplate at the beginning of the generated file.
-  sed 's/YEAR/2016/' "${initial_dir}/hack/boilerplate/boilerplate.go.txt" > "${base_generated_file}.tmp"
-  cat "${base_generated_file}" >> "${base_generated_file}.tmp"
-  mv "${base_generated_file}.tmp" "${base_generated_file}"
-  popd > /dev/null
-done

hack/verify-codecgen.sh

@@ -1,67 +0,0 @@
-#!/bin/bash
-
-# Copyright 2015 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
-source "${KUBE_ROOT}/hack/lib/init.sh"
-
-kube::golang::setup_env
-
-generated_files=($(
-  find . -not \( \
-      \( \
-        -wholename './output' \
-        -o -wholename './_output' \
-        -o -wholename './release' \
-        -o -wholename './target' \
-        -o -wholename '*/third_party/*' \
-        -o -wholename '*/vendor/*' \
-      \) -prune \
-    \) -name '*.generated.go'))
-
-function cleanup {
-  for generated_file in ${generated_files[@]}; do
-    rm -f "${generated_file}.original"
-  done
-}
-trap cleanup EXIT SIGINT
-
-for generated_file in ${generated_files[@]}; do
-  cat "${generated_file}" > "${generated_file}.original"
-done
-
-${KUBE_ROOT}/hack/update-codecgen.sh
-
-ret=0
-# Generate files in the dependency order.
-for generated_file in ${generated_files[@]}; do
-  cur=0
-  diff -Naupr -I 'Auto generated by' "${generated_file}" "${generated_file}.original" || cur=$?
-  if [[ $cur -eq 0 ]]; then
-    echo "${generated_file} up to date."
-  else
-    echo "${generated_file} was out of date. Please run hack/update-codecgen.sh. (If you're running locally, this was run for you already.)"
-    ret=1
-  fi
-
-done
-
-if [ "${ret}" -ne "0" ]; then
-  exit $ret
-fi

pkg/api/serialization_test.go

@@ -27,7 +27,7 @@ import (
 	"testing"
 
 	"github.com/golang/protobuf/proto"
-	"github.com/ugorji/go/codec"
+	jsoniter "github.com/json-iterator/go"
 
 	"k8s.io/api/core/v1"
 	"k8s.io/api/extensions/v1beta1"
@@ -536,7 +536,7 @@ func BenchmarkDecodeIntoJSON(b *testing.B) {
 	b.StopTimer()
 }
 
-// BenchmarkDecodeJSON provides a baseline for codecgen JSON decode performance
+// BenchmarkDecodeJSON provides a baseline for JSON decode performance
 func BenchmarkDecodeIntoJSONCodecGen(b *testing.B) {
 	kcodec := testapi.Default.Codec()
 	items := benchmarkItems(b)
@@ -549,12 +549,11 @@ func BenchmarkDecodeIntoJSONCodecGen(b *testing.B) {
 		}
 		encoded[i] = data
 	}
-	handler := &codec.JsonHandle{}
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		obj := v1.Pod{}
-		if err := codec.NewDecoderBytes(encoded[i%width], handler).Decode(&obj); err != nil {
+		if err := jsoniter.ConfigFastest.Unmarshal(encoded[i%width], &obj); err != nil {
 			b.Fatal(err)
 		}
 	}

staging/src/k8s.io/apimachinery/pkg/apis/meta/fuzzer/fuzzer.go

@@ -145,15 +145,11 @@ func v1alpha1FuzzerFuncs(codecs runtimeserializer.CodecFactory) []interface{} {
 				case 2:
 					r.Cells[i] = c.RandBool()
 				case 3:
-					// maps roundtrip as map[interface{}]interface{}, but the json codec cannot encode that
+					x := map[string]interface{}{}
-					// TODO: get maps to roundtrip properly
+					for j := c.Intn(10) + 1; j >= 0; j-- {
-					/*
+						x[c.RandString()] = c.RandString()
-						x := map[string]interface{}{}
+					}
-						for j := c.Intn(10) + 1; j >= 0; j-- {
+					r.Cells[i] = x
-							x[c.RandString()] = c.RandString()
-						}
-						r.Cells[i] = x
-					*/
 				case 4:
 					x := make([]interface{}, c.Intn(10))
 					for i := range x {

staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/group_version_test.go

@@ -21,7 +21,7 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/ugorji/go/codec"
+	jsoniter "github.com/json-iterator/go"
 )
 
 type GroupVersionHolder struct {
@@ -46,12 +46,12 @@ func TestGroupVersionUnmarshalJSON(t *testing.T) {
 		if !reflect.DeepEqual(result.GV, c.expect) {
 			t.Errorf("JSON codec failed to unmarshal input '%s': expected %+v, got %+v", c.input, c.expect, result.GV)
 		}
-		// test the Ugorji codec
+		// test the json-iterator codec
-		if err := codec.NewDecoderBytes(c.input, new(codec.JsonHandle)).Decode(&result); err != nil {
+		if err := jsoniter.ConfigFastest.Unmarshal(c.input, &result); err != nil {
-			t.Errorf("Ugorji codec failed to unmarshal input '%v': %v", c.input, err)
+			t.Errorf("json-iterator codec failed to unmarshal input '%v': %v", c.input, err)
 		}
 		if !reflect.DeepEqual(result.GV, c.expect) {
-			t.Errorf("Ugorji codec failed to unmarshal input '%s': expected %+v, got %+v", c.input, c.expect, result.GV)
+			t.Errorf("json-iterator codec failed to unmarshal input '%s': expected %+v, got %+v", c.input, c.expect, result.GV)
 		}
 	}
 }

staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types_test.go

@@ -21,7 +21,7 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/ugorji/go/codec"
+	jsoniter "github.com/json-iterator/go"
 )
 
 func TestVerbsUgorjiMarshalJSON(t *testing.T) {
@@ -58,7 +58,7 @@ func TestVerbsUgorjiUnmarshalJSON(t *testing.T) {
 
 	for i, c := range cases {
 		var result APIResource
-		if err := codec.NewDecoderBytes([]byte(c.input), new(codec.JsonHandle)).Decode(&result); err != nil {
+		if err := jsoniter.ConfigFastest.Unmarshal([]byte(c.input), &result); err != nil {
 			t.Errorf("[%d] Failed to unmarshal input '%v': %v", i, c.input, err)
 		}
 		if !reflect.DeepEqual(result, c.result) {

staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json.go

@@ -19,9 +19,11 @@ package json
 import (
 	"encoding/json"
 	"io"
+	"strconv"
+	"unsafe"
 
 	"github.com/ghodss/yaml"
-	"github.com/ugorji/go/codec"
+	jsoniter "github.com/json-iterator/go"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -66,6 +68,36 @@ type Serializer struct {
 var _ runtime.Serializer = &Serializer{}
 var _ recognizer.RecognizingDecoder = &Serializer{}
 
+func init() {
+	// Force jsoniter to decode number to interface{} via ints, if possible.
+	decodeNumberAsInt64IfPossible := func(ptr unsafe.Pointer, iter *jsoniter.Iterator) {
+		switch iter.WhatIsNext() {
+		case jsoniter.NumberValue:
+			var number json.Number
+			iter.ReadVal(&number)
+			u64, err := strconv.ParseUint(string(number), 10, 64)
+			if err == nil {
+				*(*interface{})(ptr) = u64
+				return
+			}
+			i64, err := strconv.ParseInt(string(number), 10, 64)
+			if err == nil {
+				*(*interface{})(ptr) = i64
+				return
+			}
+			f64, err := strconv.ParseFloat(string(number), 64)
+			if err == nil {
+				*(*interface{})(ptr) = f64
+				return
+			}
+			// Not much we can do here.
+		default:
+			*(*interface{})(ptr) = iter.Read()
+		}
+	}
+	jsoniter.RegisterTypeDecoderFunc("interface {}", decodeNumberAsInt64IfPossible)
+}
+
 // Decode attempts to convert the provided data into YAML or JSON, extract the stored schema kind, apply the provided default gvk, and then
 // load that data into an object matching the desired schema kind or the provided into. If into is *runtime.Unknown, the raw data will be
 // extracted and no decoding will be performed. If into is not registered with the typer, then the object will be straight decoded using
@@ -121,7 +153,7 @@ func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, i
 		types, _, err := s.typer.ObjectKinds(into)
 		switch {
 		case runtime.IsNotRegisteredError(err):
-			if err := codec.NewDecoderBytes(data, new(codec.JsonHandle)).Decode(into); err != nil {
+			if err := jsoniter.ConfigFastest.Unmarshal(data, into); err != nil {
 				return nil, actual, err
 			}
 			return into, actual, nil
@@ -155,7 +187,7 @@ func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, i
 		return nil, actual, err
 	}
 
-	if err := codec.NewDecoderBytes(data, new(codec.JsonHandle)).Decode(obj); err != nil {
+	if err := jsoniter.ConfigFastest.Unmarshal(data, obj); err != nil {
 		return nil, actual, err
 	}
 	return obj, actual, nil
@@ -164,7 +196,7 @@ func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, i
 // Encode serializes the provided object to the given writer.
 func (s *Serializer) Encode(obj runtime.Object, w io.Writer) error {
 	if s.yaml {
-		json, err := json.Marshal(obj)
+		json, err := jsoniter.ConfigFastest.Marshal(obj)
 		if err != nil {
 			return err
 		}
@@ -177,7 +209,7 @@ func (s *Serializer) Encode(obj runtime.Object, w io.Writer) error {
 	}
 
 	if s.pretty {
-		data, err := json.MarshalIndent(obj, "", "  ")
+		data, err := jsoniter.ConfigFastest.MarshalIndent(obj, "", "  ")
 		if err != nil {
 			return err
 		}