Merge pull request #130739 from jpbetz/declarative-validation-test-infra

Introduce versioned validation test utilitizes and add fuzz tester

pkg/api/testing/validation.go

@@ -0,0 +1,107 @@
+/*
+Copyright 2025 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testing
+
+import (
+	"bytes"
+	"sort"
+	"strconv"
+	"testing"
+
+	k8sruntime "k8s.io/apimachinery/pkg/runtime"
+	runtimetest "k8s.io/apimachinery/pkg/runtime/testing"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
+)
+
+// VerifyVersionedValidationEquivalence tests that all versions of an API return equivalent validation errors.
+func VerifyVersionedValidationEquivalence(t *testing.T, obj, old k8sruntime.Object) {
+	t.Helper()
+
+	// Accumulate errors from all versioned validation, per version.
+	all := map[string]field.ErrorList{}
+	accumulate := func(t *testing.T, gv string, errs field.ErrorList) {
+		all[gv] = errs
+	}
+	if old == nil {
+		runtimetest.RunValidationForEachVersion(t, legacyscheme.Scheme, sets.Set[string]{}, obj, accumulate)
+	} else {
+		runtimetest.RunUpdateValidationForEachVersion(t, legacyscheme.Scheme, sets.Set[string]{}, obj, old, accumulate)
+	}
+
+	// Make a copy so we can modify it.
+	other := map[string]field.ErrorList{}
+	// Index for nicer output.
+	keys := []string{}
+	for k, v := range all {
+		other[k] = v
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	// Compare each lhs to each rhs.
+	for _, lk := range keys {
+		lv := all[lk]
+		// remove lk since to prevent comparison to itself and because this
+		// iteration will compare it to any version it has not yet been
+		// compared to. e.g. [1, 2, 3] vs. [1, 2, 3] yields:
+		//   1 vs. 2
+		//   1 vs. 3
+		//   2 vs. 3
+		delete(other, lk)
+		// don't compare to ourself
+		for _, rk := range keys {
+			rv, found := other[rk]
+			if !found {
+				continue // done already
+			}
+			if len(lv) != len(rv) {
+				t.Errorf("different error count (%d vs. %d)\n%s: %v\n%s: %v", len(lv), len(rv), lk, fmtErrs(lv), rk, fmtErrs(rv))
+				continue
+			}
+			next := false
+			for i := range lv {
+				if l, r := lv[i], rv[i]; l.Type != r.Type || l.Detail != r.Detail {
+					t.Errorf("different errors\n%s: %v\n%s: %v", lk, fmtErrs(lv), rk, fmtErrs(rv))
+					next = true
+					break
+				}
+			}
+			if next {
+				continue
+			}
+		}
+	}
+}
+
+// helper for nicer output
+func fmtErrs(errs field.ErrorList) string {
+	if len(errs) == 0 {
+		return "<no errors>"
+	}
+	if len(errs) == 1 {
+		return strconv.Quote(errs[0].Error())
+	}
+	buf := bytes.Buffer{}
+	for _, e := range errs {
+		buf.WriteString("\n")
+		buf.WriteString(strconv.Quote(e.Error()))
+	}
+
+	return buf.String()
+}

pkg/api/testing/validation_test.go

@@ -0,0 +1,58 @@
+/*
+Copyright 2025 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testing
+
+import (
+	"math/rand"
+	"testing"
+
+	"k8s.io/apimachinery/pkg/api/apitesting/fuzzer"
+	"k8s.io/apimachinery/pkg/api/apitesting/roundtrip"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
+)
+
+// FIXME: Automatically finds all group/versions supporting declarative validation, or add
+// a reflexive test that verifies that they are all registered.
+func TestVersionedValidationByFuzzing(t *testing.T) {
+	typesWithDeclarativeValidation := []schema.GroupVersion{
+		// Registered group versions for versioned validation fuzz testing:
+	}
+
+	for _, gv := range typesWithDeclarativeValidation {
+		t.Run(gv.String(), func(t *testing.T) {
+			for i := 0; i < *roundtrip.FuzzIters; i++ {
+				f := fuzzer.FuzzerFor(FuzzerFuncs, rand.NewSource(rand.Int63()), legacyscheme.Codecs)
+				for kind := range legacyscheme.Scheme.KnownTypes(gv) {
+					obj, err := legacyscheme.Scheme.New(gv.WithKind(kind))
+					if err != nil {
+						t.Fatalf("could not create a %v: %s", kind, err)
+					}
+					f.Fill(obj)
+					VerifyVersionedValidationEquivalence(t, obj, nil)
+
+					old, err := legacyscheme.Scheme.New(gv.WithKind(kind))
+					if err != nil {
+						t.Fatalf("could not create a %v: %s", kind, err)
+					}
+					f.Fill(old)
+					VerifyVersionedValidationEquivalence(t, obj, old)
+				}
+			}
+		})
+	}
+}

staging/src/k8s.io/apimachinery/pkg/runtime/testing/validation.go

@@ -0,0 +1,119 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testing
+
+import (
+	"context"
+	"testing"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+)
+
+type VersionValidationRunner func(t *testing.T, gv string, versionValidationErrors field.ErrorList)
+
+// RunValidationForEachVersion runs f as a subtest of t for each version of the given unversioned object.
+// Each subtest is named by GroupVersionKind.  Each call to f is provided the field.ErrorList results
+// of converting the unversioned object to a version and validating it.
+//
+// Only autogenerated validation is run. To test both handwritten and autogenerated validation:
+//
+//		RunValidationForEachVersion(t, testCase.pod, func(t *testing.T, versionValidationErrors field.ErrorList) {
+//		  errs := ValidatePod(testCase.obj) // hand written validation
+//	      errs = append(errs, versionValidationErrors...) // generated declarative validation
+//		  // Validate that the errors are what was expected for this test case.
+//		})
+func RunValidationForEachVersion(t *testing.T, scheme *runtime.Scheme, options sets.Set[string], unversioned runtime.Object, fn VersionValidationRunner) {
+	runValidation(t, scheme, options, unversioned, fn)
+}
+
+// RunUpdateValidationForEachVersion is like RunValidationForEachVersion but for update validation.
+func RunUpdateValidationForEachVersion(t *testing.T, scheme *runtime.Scheme, options sets.Set[string], unversioned, unversionedOld runtime.Object, fn VersionValidationRunner) {
+	runUpdateValidation(t, scheme, options, unversioned, unversionedOld, fn)
+}
+
+// RunStatusValidationForEachVersion is like RunUpdateValidationForEachVersion but for status validation.
+func RunStatusValidationForEachVersion(t *testing.T, scheme *runtime.Scheme, options sets.Set[string], unversioned, unversionedOld runtime.Object, fn VersionValidationRunner) {
+	runUpdateValidation(t, scheme, options, unversioned, unversionedOld, fn, "status")
+}
+
+func runValidation(t *testing.T, scheme *runtime.Scheme, options sets.Set[string], unversioned runtime.Object, fn VersionValidationRunner, subresources ...string) {
+	unversionedGVKs, _, err := scheme.ObjectKinds(unversioned)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, unversionedGVK := range unversionedGVKs {
+		gvs := scheme.VersionsForGroupKind(unversionedGVK.GroupKind())
+		for _, gv := range gvs {
+			gvk := gv.WithKind(unversionedGVK.Kind)
+			t.Run(gvk.String(), func(t *testing.T) {
+				if gvk.Version != runtime.APIVersionInternal { // skip internal
+					versioned, err := scheme.New(gvk)
+					if err != nil {
+						t.Fatal(err)
+					}
+					err = scheme.Convert(unversioned, versioned, nil)
+					if err != nil {
+						t.Fatal(err)
+					}
+					fn(t, gv.String(), scheme.Validate(context.Background(), options, versioned, subresources...))
+				}
+			})
+		}
+	}
+}
+
+func runUpdateValidation(t *testing.T, scheme *runtime.Scheme, options sets.Set[string], unversionedNew, unversionedOld runtime.Object, fn VersionValidationRunner, subresources ...string) {
+	unversionedGVKs, _, err := scheme.ObjectKinds(unversionedNew)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, unversionedGVK := range unversionedGVKs {
+		gvs := scheme.VersionsForGroupKind(unversionedGVK.GroupKind())
+		for _, gv := range gvs {
+			gvk := gv.WithKind(unversionedGVK.Kind)
+			t.Run(gvk.String(), func(t *testing.T) {
+				if gvk.Version != runtime.APIVersionInternal { // skip internal
+					versionedNew, err := scheme.New(gvk)
+					if err != nil {
+						t.Fatal(err)
+					}
+					err = scheme.Convert(unversionedNew, versionedNew, nil)
+					if err != nil {
+						t.Fatal(err)
+					}
+
+					var versionedOld runtime.Object
+					if unversionedOld != nil {
+						versionedOld, err = scheme.New(gvk)
+						if err != nil {
+							t.Fatal(err)
+						}
+
+						err = scheme.Convert(unversionedOld, versionedOld, nil)
+						if err != nil {
+							t.Fatal(err)
+						}
+					}
+
+					fn(t, gv.String(), scheme.ValidateUpdate(context.Background(), options, versionedNew, versionedOld, subresources...))
+				}
+			})
+		}
+	}
+}