From 04d972cb4bab108b12f181f86b7c0f0854a1037a Mon Sep 17 00:00:00 2001 From: Andrea Nodari Date: Fri, 23 Oct 2020 17:26:10 +0200 Subject: [PATCH] Add yaml util to unmarshal numbers into int/float --- .../k8s.io/apimachinery/pkg/util/json/json.go | 33 ++++++++------- .../k8s.io/apimachinery/pkg/util/yaml/BUILD | 1 + .../apimachinery/pkg/util/yaml/decoder.go | 31 ++++++++++++++ .../pkg/util/yaml/decoder_test.go | 41 +++++++++++++++++++ .../apiserver/pkg/endpoints/handlers/BUILD | 2 +- .../apiserver/pkg/endpoints/handlers/patch.go | 2 +- 6 files changed, 93 insertions(+), 17 deletions(-) diff --git a/staging/src/k8s.io/apimachinery/pkg/util/json/json.go b/staging/src/k8s.io/apimachinery/pkg/util/json/json.go index 204834883fa..778e58f704b 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/json/json.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/json/json.go @@ -39,7 +39,8 @@ func Marshal(v interface{}) ([]byte, error) { const maxDepth = 10000 // Unmarshal unmarshals the given data -// If v is a *map[string]interface{}, numbers are converted to int64 or float64 +// If v is a *map[string]interface{}, *[]interface{}, or *interface{} numbers +// are converted to int64 or float64 func Unmarshal(data []byte, v interface{}) error { switch v := v.(type) { case *map[string]interface{}: @@ -52,7 +53,7 @@ func Unmarshal(data []byte, v interface{}) error { return err } // If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64 - return convertMapNumbers(*v, 0) + return ConvertMapNumbers(*v, 0) case *[]interface{}: // Build a decoder from the given data @@ -64,7 +65,7 @@ func Unmarshal(data []byte, v interface{}) error { return err } // If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64 - return convertSliceNumbers(*v, 0) + return ConvertSliceNumbers(*v, 0) case *interface{}: // Build a decoder from the given data @@ -76,29 +77,31 @@ func Unmarshal(data []byte, v interface{}) error { return err } // If the decode succeeds, post-process the map to convert json.Number objects to int64 or float64 - return convertInterfaceNumbers(v, 0) + return ConvertInterfaceNumbers(v, 0) default: return json.Unmarshal(data, v) } } -func convertInterfaceNumbers(v *interface{}, depth int) error { +// ConvertInterfaceNumbers converts any json.Number values to int64 or float64. +// Values which are map[string]interface{} or []interface{} are recursively visited +func ConvertInterfaceNumbers(v *interface{}, depth int) error { var err error switch v2 := (*v).(type) { case json.Number: *v, err = convertNumber(v2) case map[string]interface{}: - err = convertMapNumbers(v2, depth+1) + err = ConvertMapNumbers(v2, depth+1) case []interface{}: - err = convertSliceNumbers(v2, depth+1) + err = ConvertSliceNumbers(v2, depth+1) } return err } -// convertMapNumbers traverses the map, converting any json.Number values to int64 or float64. +// ConvertMapNumbers traverses the map, converting any json.Number values to int64 or float64. // values which are map[string]interface{} or []interface{} are recursively visited -func convertMapNumbers(m map[string]interface{}, depth int) error { +func ConvertMapNumbers(m map[string]interface{}, depth int) error { if depth > maxDepth { return fmt.Errorf("exceeded max depth of %d", maxDepth) } @@ -109,9 +112,9 @@ func convertMapNumbers(m map[string]interface{}, depth int) error { case json.Number: m[k], err = convertNumber(v) case map[string]interface{}: - err = convertMapNumbers(v, depth+1) + err = ConvertMapNumbers(v, depth+1) case []interface{}: - err = convertSliceNumbers(v, depth+1) + err = ConvertSliceNumbers(v, depth+1) } if err != nil { return err @@ -120,9 +123,9 @@ func convertMapNumbers(m map[string]interface{}, depth int) error { return nil } -// convertSliceNumbers traverses the slice, converting any json.Number values to int64 or float64. +// ConvertSliceNumbers traverses the slice, converting any json.Number values to int64 or float64. // values which are map[string]interface{} or []interface{} are recursively visited -func convertSliceNumbers(s []interface{}, depth int) error { +func ConvertSliceNumbers(s []interface{}, depth int) error { if depth > maxDepth { return fmt.Errorf("exceeded max depth of %d", maxDepth) } @@ -133,9 +136,9 @@ func convertSliceNumbers(s []interface{}, depth int) error { case json.Number: s[i], err = convertNumber(v) case map[string]interface{}: - err = convertMapNumbers(v, depth+1) + err = ConvertMapNumbers(v, depth+1) case []interface{}: - err = convertSliceNumbers(v, depth+1) + err = ConvertSliceNumbers(v, depth+1) } if err != nil { return err diff --git a/staging/src/k8s.io/apimachinery/pkg/util/yaml/BUILD b/staging/src/k8s.io/apimachinery/pkg/util/yaml/BUILD index a1787159279..cb3c98e6f21 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/yaml/BUILD +++ b/staging/src/k8s.io/apimachinery/pkg/util/yaml/BUILD @@ -18,6 +18,7 @@ go_library( importmap = "k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/yaml", importpath = "k8s.io/apimachinery/pkg/util/yaml", deps = [ + "//staging/src/k8s.io/apimachinery/pkg/util/json:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library", "//vendor/sigs.k8s.io/yaml:go_default_library", ], diff --git a/staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder.go b/staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder.go index 492171faf4b..7fe70646770 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder.go @@ -26,10 +26,41 @@ import ( "strings" "unicode" + jsonutil "k8s.io/apimachinery/pkg/util/json" + "k8s.io/klog/v2" "sigs.k8s.io/yaml" ) +// Unmarshal unmarshals the given data +// If v is a *map[string]interface{}, *[]interface{}, or *interface{} numbers +// are converted to int64 or float64 +func Unmarshal(data []byte, v interface{}) error { + preserveIntFloat := func(d *json.Decoder) *json.Decoder { + d.UseNumber() + return d + } + switch v := v.(type) { + case *map[string]interface{}: + if err := yaml.Unmarshal(data, v, preserveIntFloat); err != nil { + return err + } + return jsonutil.ConvertMapNumbers(*v, 0) + case *[]interface{}: + if err := yaml.Unmarshal(data, v, preserveIntFloat); err != nil { + return err + } + return jsonutil.ConvertSliceNumbers(*v, 0) + case *interface{}: + if err := yaml.Unmarshal(data, v, preserveIntFloat); err != nil { + return err + } + return jsonutil.ConvertInterfaceNumbers(v, 0) + default: + return yaml.Unmarshal(data, v) + } +} + // ToJSON converts a single YAML document into a JSON document // or returns an error. If the document appears to be JSON the // YAML decoding path is not used (so that error messages are diff --git a/staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder_test.go b/staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder_test.go index b4c5f0cd9c1..1f4b158f968 100644 --- a/staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder_test.go +++ b/staging/src/k8s.io/apimachinery/pkg/util/yaml/decoder_test.go @@ -430,3 +430,44 @@ stuff: 1 t.Fatalf("expected %q to be of type YAMLSyntaxError", err.Error()) } } + +func TestUnmarshal(t *testing.T) { + mapWithIntegerBytes := []byte(`replicas: 1`) + mapWithInteger := make(map[string]interface{}) + if err := Unmarshal(mapWithIntegerBytes, &mapWithInteger); err != nil { + t.Fatalf("unexpected error unmarshaling yaml: %v", err) + } + if _, ok := mapWithInteger["replicas"].(int64); !ok { + t.Fatalf(`Expected number in map to be int64 but got "%T"`, mapWithInteger["replicas"]) + } + + sliceWithIntegerBytes := []byte(`- 1`) + var sliceWithInteger []interface{} + if err := Unmarshal(sliceWithIntegerBytes, &sliceWithInteger); err != nil { + t.Fatalf("unexpected error unmarshaling yaml: %v", err) + } + if _, ok := sliceWithInteger[0].(int64); !ok { + t.Fatalf(`Expected number in slice to be int64 but got "%T"`, sliceWithInteger[0]) + } + + integerBytes := []byte(`1`) + var integer interface{} + if err := Unmarshal(integerBytes, &integer); err != nil { + t.Fatalf("unexpected error unmarshaling yaml: %v", err) + } + if _, ok := integer.(int64); !ok { + t.Fatalf(`Expected number to be int64 but got "%T"`, integer) + } + + otherTypeBytes := []byte(`123: 2`) + otherType := make(map[int]interface{}) + if err := Unmarshal(otherTypeBytes, &otherType); err != nil { + t.Fatalf("unexpected error unmarshaling yaml: %v", err) + } + if _, ok := otherType[123].(int64); ok { + t.Fatalf(`Expected number not to be converted to int64`) + } + if _, ok := otherType[123].(float64); !ok { + t.Fatalf(`Expected number to be float64 but got "%T"`, otherType[123]) + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/BUILD b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/BUILD index 6d6ef4a97be..17873a495a4 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/BUILD @@ -87,6 +87,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/strategicpatch:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", "//staging/src/k8s.io/apiserver/pkg/audit:go_default_library", @@ -108,7 +109,6 @@ go_library( "//vendor/google.golang.org/grpc/status:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library", "//vendor/k8s.io/utils/trace:go_default_library", - "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/patch.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/patch.go index a043bb3b251..4e85f8e1622 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/patch.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/patch.go @@ -38,6 +38,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/audit" "k8s.io/apiserver/pkg/authorization/authorizer" @@ -49,7 +50,6 @@ import ( "k8s.io/apiserver/pkg/util/dryrun" utilfeature "k8s.io/apiserver/pkg/util/feature" utiltrace "k8s.io/utils/trace" - "sigs.k8s.io/yaml" ) const (