From 1d365762a5358e346b3133b369eadd9be6d4c85b Mon Sep 17 00:00:00 2001 From: Tim Hockin Date: Wed, 11 Dec 2024 14:28:15 -0800 Subject: [PATCH] Introduce versioned validation test utilitizes and add fuzz tester This makes a bold assumption: that the errors (count and basic content) will be the same across versions. If this turns out to be untrue, this may need to get more sophisticated. It should fail obviously when we hit that edge. --- pkg/api/testing/validation.go | 107 ++++++++++++++++ pkg/api/testing/validation_test.go | 58 +++++++++ .../pkg/runtime/testing/validation.go | 119 ++++++++++++++++++ 3 files changed, 284 insertions(+) create mode 100644 pkg/api/testing/validation.go create mode 100644 pkg/api/testing/validation_test.go create mode 100644 staging/src/k8s.io/apimachinery/pkg/runtime/testing/validation.go diff --git a/pkg/api/testing/validation.go b/pkg/api/testing/validation.go new file mode 100644 index 00000000000..0bd2930ee48 --- /dev/null +++ b/pkg/api/testing/validation.go @@ -0,0 +1,107 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "bytes" + "sort" + "strconv" + "testing" + + k8sruntime "k8s.io/apimachinery/pkg/runtime" + runtimetest "k8s.io/apimachinery/pkg/runtime/testing" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/kubernetes/pkg/api/legacyscheme" +) + +// VerifyVersionedValidationEquivalence tests that all versions of an API return equivalent validation errors. +func VerifyVersionedValidationEquivalence(t *testing.T, obj, old k8sruntime.Object) { + t.Helper() + + // Accumulate errors from all versioned validation, per version. + all := map[string]field.ErrorList{} + accumulate := func(t *testing.T, gv string, errs field.ErrorList) { + all[gv] = errs + } + if old == nil { + runtimetest.RunValidationForEachVersion(t, legacyscheme.Scheme, sets.Set[string]{}, obj, accumulate) + } else { + runtimetest.RunUpdateValidationForEachVersion(t, legacyscheme.Scheme, sets.Set[string]{}, obj, old, accumulate) + } + + // Make a copy so we can modify it. + other := map[string]field.ErrorList{} + // Index for nicer output. + keys := []string{} + for k, v := range all { + other[k] = v + keys = append(keys, k) + } + sort.Strings(keys) + + // Compare each lhs to each rhs. + for _, lk := range keys { + lv := all[lk] + // remove lk since to prevent comparison to itself and because this + // iteration will compare it to any version it has not yet been + // compared to. e.g. [1, 2, 3] vs. [1, 2, 3] yields: + // 1 vs. 2 + // 1 vs. 3 + // 2 vs. 3 + delete(other, lk) + // don't compare to ourself + for _, rk := range keys { + rv, found := other[rk] + if !found { + continue // done already + } + if len(lv) != len(rv) { + t.Errorf("different error count (%d vs. %d)\n%s: %v\n%s: %v", len(lv), len(rv), lk, fmtErrs(lv), rk, fmtErrs(rv)) + continue + } + next := false + for i := range lv { + if l, r := lv[i], rv[i]; l.Type != r.Type || l.Detail != r.Detail { + t.Errorf("different errors\n%s: %v\n%s: %v", lk, fmtErrs(lv), rk, fmtErrs(rv)) + next = true + break + } + } + if next { + continue + } + } + } +} + +// helper for nicer output +func fmtErrs(errs field.ErrorList) string { + if len(errs) == 0 { + return "" + } + if len(errs) == 1 { + return strconv.Quote(errs[0].Error()) + } + buf := bytes.Buffer{} + for _, e := range errs { + buf.WriteString("\n") + buf.WriteString(strconv.Quote(e.Error())) + } + + return buf.String() +} diff --git a/pkg/api/testing/validation_test.go b/pkg/api/testing/validation_test.go new file mode 100644 index 00000000000..868f1adf41e --- /dev/null +++ b/pkg/api/testing/validation_test.go @@ -0,0 +1,58 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "math/rand" + "testing" + + "k8s.io/apimachinery/pkg/api/apitesting/fuzzer" + "k8s.io/apimachinery/pkg/api/apitesting/roundtrip" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/kubernetes/pkg/api/legacyscheme" +) + +// FIXME: Automatically finds all group/versions supporting declarative validation, or add +// a reflexive test that verifies that they are all registered. +func TestVersionedValidationByFuzzing(t *testing.T) { + typesWithDeclarativeValidation := []schema.GroupVersion{ + // Registered group versions for versioned validation fuzz testing: + } + + for _, gv := range typesWithDeclarativeValidation { + t.Run(gv.String(), func(t *testing.T) { + for i := 0; i < *roundtrip.FuzzIters; i++ { + f := fuzzer.FuzzerFor(FuzzerFuncs, rand.NewSource(rand.Int63()), legacyscheme.Codecs) + for kind := range legacyscheme.Scheme.KnownTypes(gv) { + obj, err := legacyscheme.Scheme.New(gv.WithKind(kind)) + if err != nil { + t.Fatalf("could not create a %v: %s", kind, err) + } + f.Fill(obj) + VerifyVersionedValidationEquivalence(t, obj, nil) + + old, err := legacyscheme.Scheme.New(gv.WithKind(kind)) + if err != nil { + t.Fatalf("could not create a %v: %s", kind, err) + } + f.Fill(old) + VerifyVersionedValidationEquivalence(t, obj, old) + } + } + }) + } +} diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/testing/validation.go b/staging/src/k8s.io/apimachinery/pkg/runtime/testing/validation.go new file mode 100644 index 00000000000..cf7744f1096 --- /dev/null +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/testing/validation.go @@ -0,0 +1,119 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "context" + "testing" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +type VersionValidationRunner func(t *testing.T, gv string, versionValidationErrors field.ErrorList) + +// RunValidationForEachVersion runs f as a subtest of t for each version of the given unversioned object. +// Each subtest is named by GroupVersionKind. Each call to f is provided the field.ErrorList results +// of converting the unversioned object to a version and validating it. +// +// Only autogenerated validation is run. To test both handwritten and autogenerated validation: +// +// RunValidationForEachVersion(t, testCase.pod, func(t *testing.T, versionValidationErrors field.ErrorList) { +// errs := ValidatePod(testCase.obj) // hand written validation +// errs = append(errs, versionValidationErrors...) // generated declarative validation +// // Validate that the errors are what was expected for this test case. +// }) +func RunValidationForEachVersion(t *testing.T, scheme *runtime.Scheme, options sets.Set[string], unversioned runtime.Object, fn VersionValidationRunner) { + runValidation(t, scheme, options, unversioned, fn) +} + +// RunUpdateValidationForEachVersion is like RunValidationForEachVersion but for update validation. +func RunUpdateValidationForEachVersion(t *testing.T, scheme *runtime.Scheme, options sets.Set[string], unversioned, unversionedOld runtime.Object, fn VersionValidationRunner) { + runUpdateValidation(t, scheme, options, unversioned, unversionedOld, fn) +} + +// RunStatusValidationForEachVersion is like RunUpdateValidationForEachVersion but for status validation. +func RunStatusValidationForEachVersion(t *testing.T, scheme *runtime.Scheme, options sets.Set[string], unversioned, unversionedOld runtime.Object, fn VersionValidationRunner) { + runUpdateValidation(t, scheme, options, unversioned, unversionedOld, fn, "status") +} + +func runValidation(t *testing.T, scheme *runtime.Scheme, options sets.Set[string], unversioned runtime.Object, fn VersionValidationRunner, subresources ...string) { + unversionedGVKs, _, err := scheme.ObjectKinds(unversioned) + if err != nil { + t.Fatal(err) + } + for _, unversionedGVK := range unversionedGVKs { + gvs := scheme.VersionsForGroupKind(unversionedGVK.GroupKind()) + for _, gv := range gvs { + gvk := gv.WithKind(unversionedGVK.Kind) + t.Run(gvk.String(), func(t *testing.T) { + if gvk.Version != runtime.APIVersionInternal { // skip internal + versioned, err := scheme.New(gvk) + if err != nil { + t.Fatal(err) + } + err = scheme.Convert(unversioned, versioned, nil) + if err != nil { + t.Fatal(err) + } + fn(t, gv.String(), scheme.Validate(context.Background(), options, versioned, subresources...)) + } + }) + } + } +} + +func runUpdateValidation(t *testing.T, scheme *runtime.Scheme, options sets.Set[string], unversionedNew, unversionedOld runtime.Object, fn VersionValidationRunner, subresources ...string) { + unversionedGVKs, _, err := scheme.ObjectKinds(unversionedNew) + if err != nil { + t.Fatal(err) + } + for _, unversionedGVK := range unversionedGVKs { + gvs := scheme.VersionsForGroupKind(unversionedGVK.GroupKind()) + for _, gv := range gvs { + gvk := gv.WithKind(unversionedGVK.Kind) + t.Run(gvk.String(), func(t *testing.T) { + if gvk.Version != runtime.APIVersionInternal { // skip internal + versionedNew, err := scheme.New(gvk) + if err != nil { + t.Fatal(err) + } + err = scheme.Convert(unversionedNew, versionedNew, nil) + if err != nil { + t.Fatal(err) + } + + var versionedOld runtime.Object + if unversionedOld != nil { + versionedOld, err = scheme.New(gvk) + if err != nil { + t.Fatal(err) + } + + err = scheme.Convert(unversionedOld, versionedOld, nil) + if err != nil { + t.Fatal(err) + } + } + + fn(t, gv.String(), scheme.ValidateUpdate(context.Background(), options, versionedNew, versionedOld, subresources...)) + } + }) + } + } +}