From c0b7858946c253eab7a81f8d2ede9d7cddcaa16a Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Sat, 19 Feb 2022 09:47:46 -0500 Subject: [PATCH] Fill API compatibility data with identifying values rather than random data --- .../api/apitesting/roundtrip/compatibility.go | 126 +++--------- .../pkg/api/apitesting/roundtrip/construct.go | 179 ++++++++++++++++++ 2 files changed, 205 insertions(+), 100 deletions(-) create mode 100644 staging/src/k8s.io/apimachinery/pkg/api/apitesting/roundtrip/construct.go diff --git a/staging/src/k8s.io/apimachinery/pkg/api/apitesting/roundtrip/compatibility.go b/staging/src/k8s.io/apimachinery/pkg/api/apitesting/roundtrip/compatibility.go index 7b3c4dc73b7..8272f9fa358 100644 --- a/staging/src/k8s.io/apimachinery/pkg/api/apitesting/roundtrip/compatibility.go +++ b/staging/src/k8s.io/apimachinery/pkg/api/apitesting/roundtrip/compatibility.go @@ -23,25 +23,18 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "sort" - "strconv" "strings" "testing" - "time" "github.com/google/go-cmp/cmp" - fuzz "github.com/google/gofuzz" apiequality "k8s.io/apimachinery/pkg/api/equality" - apimeta "k8s.io/apimachinery/pkg/api/meta" - genericfuzzer "k8s.io/apimachinery/pkg/apis/meta/fuzzer" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/runtime/serializer/json" "k8s.io/apimachinery/pkg/runtime/serializer/protobuf" - "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" ) @@ -50,7 +43,7 @@ import ( // // Example use: `NewCompatibilityTestOptions(scheme).Complete(t).Run(t)` type CompatibilityTestOptions struct { - // Scheme is used to create new objects for fuzzing, decoding, and for constructing serializers. + // Scheme is used to create new objects for filling, decoding, and for constructing serializers. // Required. Scheme *runtime.Scheme @@ -61,7 +54,7 @@ type CompatibilityTestOptions struct { // TestDataDirCurrentVersion points to a directory containing compatibility test data for the current version. // Complete() populates this with "/HEAD" if unset. // Within this directory, `...[json|yaml|pb]` files are required to exist, and are: - // * verified to match serialized FuzzedObjects[GVK] + // * verified to match serialized FilledObjects[GVK] // * verified to decode without error // * verified to round-trip byte-for-byte when re-encoded // * verified to be semantically equal when decoded into memory @@ -79,20 +72,25 @@ type CompatibilityTestOptions struct { // Complete() populates this with Scheme.AllKnownTypes() if unset. Kinds []schema.GroupVersionKind - // FuzzedObjects is an optional set of fuzzed objects to use for verifying HEAD fixtures. - // Complete() populates this with the result of CompatibilityTestObject(Kinds[*], Scheme, FuzzFuncs) for any missing kinds. - // Objects must be deterministically fuzzed and identical on every invocation. - FuzzedObjects map[schema.GroupVersionKind]runtime.Object + // FilledObjects is an optional set of pre-filled objects to use for verifying HEAD fixtures. + // Complete() populates this with the result of CompatibilityTestObject(Kinds[*], Scheme, FillFuncs) for any missing kinds. + // Objects must deterministically populate every field and be identical on every invocation. + FilledObjects map[schema.GroupVersionKind]runtime.Object - // FuzzFuncs is an optional set of custom fuzzing functions to use to construct FuzzedObjects. - // They *must* not use any random source other than the passed-in fuzzer. - FuzzFuncs []interface{} + // FillFuncs is an optional map of custom functions to use to fill instances of particular types. + FillFuncs map[reflect.Type]FillFunc JSON runtime.Serializer YAML runtime.Serializer Proto runtime.Serializer } +// FillFunc is a function that populates all serializable fields in obj. +// s and i are string and integer values relevant to the object being populated +// (for example, the json key or protobuf tag containing the object) +// that can be used when filling the object to make the object content identifiable +type FillFunc func(s string, i int, obj interface{}) + func NewCompatibilityTestOptions(scheme *runtime.Scheme) *CompatibilityTestOptions { return &CompatibilityTestOptions{Scheme: scheme} } @@ -163,19 +161,23 @@ func (c *CompatibilityTestOptions) Complete(t *testing.T) *CompatibilityTestOpti return false }) - // Fuzz any missing objects - if c.FuzzedObjects == nil { - c.FuzzedObjects = map[schema.GroupVersionKind]runtime.Object{} + // Fill any missing objects + if c.FilledObjects == nil { + c.FilledObjects = map[schema.GroupVersionKind]runtime.Object{} + } + fillFuncs := defaultFillFuncs() + for k, v := range c.FillFuncs { + fillFuncs[k] = v } for _, gvk := range c.Kinds { - if _, ok := c.FuzzedObjects[gvk]; ok { + if _, ok := c.FilledObjects[gvk]; ok { continue } - obj, err := CompatibilityTestObject(c.Scheme, gvk, c.FuzzFuncs) + obj, err := CompatibilityTestObject(c.Scheme, gvk, fillFuncs) if err != nil { t.Fatal(err) } - c.FuzzedObjects[gvk] = obj + c.FilledObjects[gvk] = obj } if c.JSON == nil { @@ -191,82 +193,6 @@ func (c *CompatibilityTestOptions) Complete(t *testing.T) *CompatibilityTestOpti return c } -// CompatibilityTestObject returns a deterministically fuzzed object for the specified GVK -func CompatibilityTestObject(scheme *runtime.Scheme, gvk schema.GroupVersionKind, fuzzFuncs []interface{}) (runtime.Object, error) { - // Construct the object - obj, err := scheme.New(gvk) - if err != nil { - return nil, err - } - - // Fuzz it - CompatibilityTestFuzzer(scheme, fuzzFuncs).Fuzz(obj) - - // Set the kind and apiVersion - if typeAcc, err := apimeta.TypeAccessor(obj); err != nil { - return nil, err - } else { - typeAcc.SetKind(gvk.Kind) - typeAcc.SetAPIVersion(gvk.GroupVersion().String()) - } - - return obj, nil -} - -// CompatibilityTestFuzzer returns a fuzzer for the given scheme: -// - fixed seed (deterministic output that lets us generate the same fixtures on every run) -// - 0 nil chance (populate all fields) -// - 1 numelements (populate and bound all lists) -// - 20 max depth (don't recurse infinitely) -// - meta fuzzing functions added -// - custom fuzzing functions to make strings and managedFields more readable in fixtures -func CompatibilityTestFuzzer(scheme *runtime.Scheme, fuzzFuncs []interface{}) *fuzz.Fuzzer { - fuzzer := fuzz.NewWithSeed(0).NilChance(0).NumElements(1, 1).MaxDepth(20) - fuzzer = fuzzer.Funcs(genericfuzzer.Funcs(serializer.NewCodecFactory(scheme))...) - fuzzString := 1 - fuzzIntOrString := 1 - fuzzMicroTime := int64(1) - fuzzer.Funcs( - // avoid crazy strings - func(s *string, c fuzz.Continue) { - fuzzString++ - *s = strconv.Itoa(fuzzString) - }, - func(i **intstr.IntOrString, c fuzz.Continue) { - fuzzIntOrString++ - tmp := intstr.FromInt(fuzzIntOrString) - _ = tmp - *i = &tmp - }, - func(t **metav1.MicroTime, c fuzz.Continue) { - if t != nil && *t != nil { - // use type-defined fuzzing for non-nil objects - (*t).Fuzz(c) - return - } - fuzzMicroTime++ - tmp := metav1.NewMicroTime(time.Unix(fuzzMicroTime, 0)) - *t = &tmp - }, - // limit managed fields to two levels - func(f *[]metav1.ManagedFieldsEntry, c fuzz.Continue) { - field := metav1.ManagedFieldsEntry{} - c.Fuzz(&field) - if field.FieldsV1 != nil { - field.FieldsV1.Raw = []byte("{}") - } - *f = []metav1.ManagedFieldsEntry{field} - }, - func(r *runtime.RawExtension, c fuzz.Continue) { - // generate a raw object in normalized form - // TODO: test non-normalized round-tripping... YAMLToJSON normalizes and makes exact comparisons fail - r.Raw = []byte(`{"apiVersion":"example.com/v1","kind":"CustomType","spec":{"replicas":1},"status":{"available":1}}`) - }, - ) - fuzzer.Funcs(fuzzFuncs...) - return fuzzer -} - func (c *CompatibilityTestOptions) Run(t *testing.T) { usedHEADFixtures := sets.NewString() @@ -304,7 +230,7 @@ func (c *CompatibilityTestOptions) Run(t *testing.T) { } func (c *CompatibilityTestOptions) runCurrentVersionTest(t *testing.T, gvk schema.GroupVersionKind, usedFiles sets.String) { - expectedObject := c.FuzzedObjects[gvk] + expectedObject := c.FilledObjects[gvk] expectedJSON, expectedYAML, expectedProto := c.encode(t, expectedObject) actualJSON, actualYAML, actualProto, err := read(c.TestDataDirCurrentVersion, gvk, "", usedFiles) diff --git a/staging/src/k8s.io/apimachinery/pkg/api/apitesting/roundtrip/construct.go b/staging/src/k8s.io/apimachinery/pkg/api/apitesting/roundtrip/construct.go new file mode 100644 index 00000000000..6833db58493 --- /dev/null +++ b/staging/src/k8s.io/apimachinery/pkg/api/apitesting/roundtrip/construct.go @@ -0,0 +1,179 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package roundtrip + +import ( + "fmt" + "reflect" + "strconv" + "strings" + "time" + + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/intstr" +) + +func defaultFillFuncs() map[reflect.Type]FillFunc { + funcs := map[reflect.Type]FillFunc{} + funcs[reflect.TypeOf(&runtime.RawExtension{})] = func(s string, i int, obj interface{}) { + // generate a raw object in normalized form + // TODO: test non-normalized round-tripping... YAMLToJSON normalizes and makes exact comparisons fail + obj.(*runtime.RawExtension).Raw = []byte(`{"apiVersion":"example.com/v1","kind":"CustomType","spec":{"replicas":1},"status":{"available":1}}`) + } + funcs[reflect.TypeOf(&metav1.TypeMeta{})] = func(s string, i int, obj interface{}) { + // APIVersion and Kind are not serialized in all formats (notably protobuf), so clear by default for cross-format checking. + obj.(*metav1.TypeMeta).APIVersion = "" + obj.(*metav1.TypeMeta).Kind = "" + } + funcs[reflect.TypeOf(&metav1.FieldsV1{})] = func(s string, i int, obj interface{}) { + obj.(*metav1.FieldsV1).Raw = []byte(`{}`) + } + funcs[reflect.TypeOf(&metav1.Time{})] = func(s string, i int, obj interface{}) { + // use the integer as an offset from the year + obj.(*metav1.Time).Time = time.Date(2000+i, 1, 1, 1, 1, 1, 0, time.UTC) + } + funcs[reflect.TypeOf(&metav1.MicroTime{})] = func(s string, i int, obj interface{}) { + // use the integer as an offset from the year, and as a microsecond + obj.(*metav1.MicroTime).Time = time.Date(2000+i, 1, 1, 1, 1, 1, i*int(time.Microsecond), time.UTC) + } + funcs[reflect.TypeOf(&intstr.IntOrString{})] = func(s string, i int, obj interface{}) { + // use the string as a string value + obj.(*intstr.IntOrString).Type = intstr.String + obj.(*intstr.IntOrString).StrVal = s + "Value" + } + return funcs +} + +// CompatibilityTestObject returns a deterministically filled object for the specified GVK +func CompatibilityTestObject(scheme *runtime.Scheme, gvk schema.GroupVersionKind, fillFuncs map[reflect.Type]FillFunc) (runtime.Object, error) { + // Construct the object + obj, err := scheme.New(gvk) + if err != nil { + return nil, err + } + + fill("", 0, reflect.TypeOf(obj), reflect.ValueOf(obj), fillFuncs, map[reflect.Type]bool{}) + + // Set the kind and apiVersion + if typeAcc, err := apimeta.TypeAccessor(obj); err != nil { + return nil, err + } else { + typeAcc.SetKind(gvk.Kind) + typeAcc.SetAPIVersion(gvk.GroupVersion().String()) + } + + return obj, nil +} + +func fill(dataString string, dataInt int, t reflect.Type, v reflect.Value, fillFuncs map[reflect.Type]FillFunc, filledTypes map[reflect.Type]bool) { + if filledTypes[t] { + // we already filled this type, avoid recursing infinitely + return + } + filledTypes[t] = true + defer delete(filledTypes, t) + + // if nil, populate pointers with a zero-value instance of the underlying type + if t.Kind() == reflect.Ptr && v.IsNil() { + if v.CanSet() { + v.Set(reflect.New(t.Elem())) + } else if v.IsNil() { + panic(fmt.Errorf("unsettable nil pointer of type %v in field %s", t, dataString)) + } + } + + if f, ok := fillFuncs[t]; ok { + // use the custom fill function for this type + f(dataString, dataInt, v.Interface()) + return + } + + switch t.Kind() { + case reflect.Slice: + // populate with a single-item slice + v.Set(reflect.MakeSlice(t, 1, 1)) + // recurse to populate the item, preserving the data context + fill(dataString, dataInt, t.Elem(), v.Index(0), fillFuncs, filledTypes) + + case reflect.Map: + // construct the key, which must be a string type, possibly converted to a type alias of string + key := reflect.ValueOf(dataString + "Key").Convert(t.Key()) + // construct a zero-value item + item := reflect.New(t.Elem()) + // recurse to populate the item, preserving the data context + fill(dataString, dataInt, t.Elem(), item.Elem(), fillFuncs, filledTypes) + // store in the map + v.Set(reflect.MakeMap(t)) + v.SetMapIndex(key, item.Elem()) + + case reflect.Struct: + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + + if !field.IsExported() { + continue + } + + // use the json field name, which must be stable + dataString := strings.Split(field.Tag.Get("json"), ",")[0] + if len(dataString) == 0 { + // fall back to the struct field name if there is no json field name + dataString = " " + field.Name + } + + // use the protobuf tag, which must be stable + dataInt := 0 + if protobufTagParts := strings.Split(field.Tag.Get("protobuf"), ","); len(protobufTagParts) > 1 { + if tag, err := strconv.Atoi(protobufTagParts[1]); err != nil { + panic(err) + } else { + dataInt = tag + } + } + if dataInt == 0 { + // fall back to the length of dataString as a backup + dataInt = -len(dataString) + } + + fieldType := field.Type + fieldValue := v.Field(i) + + fill(dataString, dataInt, reflect.PtrTo(fieldType), fieldValue.Addr(), fillFuncs, filledTypes) + } + + case reflect.Ptr: + fill(dataString, dataInt, t.Elem(), v.Elem(), fillFuncs, filledTypes) + + case reflect.String: + // use Convert to set into string alias types correctly + v.Set(reflect.ValueOf(dataString + "Value").Convert(t)) + + case reflect.Bool: + // set to true to ensure we serialize omitempty fields + v.Set(reflect.ValueOf(true).Convert(t)) + + case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + // use Convert to set into int alias types and different int widths correctly + v.Set(reflect.ValueOf(dataInt).Convert(t)) + + default: + panic(fmt.Errorf("unhandled type %v in field %s", t, dataString)) + } +}