diff --git a/pkg/api/testing/validation.go b/pkg/api/testing/validation.go new file mode 100644 index 00000000000..0bd2930ee48 --- /dev/null +++ b/pkg/api/testing/validation.go @@ -0,0 +1,107 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "bytes" + "sort" + "strconv" + "testing" + + k8sruntime "k8s.io/apimachinery/pkg/runtime" + runtimetest "k8s.io/apimachinery/pkg/runtime/testing" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/kubernetes/pkg/api/legacyscheme" +) + +// VerifyVersionedValidationEquivalence tests that all versions of an API return equivalent validation errors. +func VerifyVersionedValidationEquivalence(t *testing.T, obj, old k8sruntime.Object) { + t.Helper() + + // Accumulate errors from all versioned validation, per version. + all := map[string]field.ErrorList{} + accumulate := func(t *testing.T, gv string, errs field.ErrorList) { + all[gv] = errs + } + if old == nil { + runtimetest.RunValidationForEachVersion(t, legacyscheme.Scheme, sets.Set[string]{}, obj, accumulate) + } else { + runtimetest.RunUpdateValidationForEachVersion(t, legacyscheme.Scheme, sets.Set[string]{}, obj, old, accumulate) + } + + // Make a copy so we can modify it. + other := map[string]field.ErrorList{} + // Index for nicer output. + keys := []string{} + for k, v := range all { + other[k] = v + keys = append(keys, k) + } + sort.Strings(keys) + + // Compare each lhs to each rhs. + for _, lk := range keys { + lv := all[lk] + // remove lk since to prevent comparison to itself and because this + // iteration will compare it to any version it has not yet been + // compared to. e.g. [1, 2, 3] vs. [1, 2, 3] yields: + // 1 vs. 2 + // 1 vs. 3 + // 2 vs. 3 + delete(other, lk) + // don't compare to ourself + for _, rk := range keys { + rv, found := other[rk] + if !found { + continue // done already + } + if len(lv) != len(rv) { + t.Errorf("different error count (%d vs. %d)\n%s: %v\n%s: %v", len(lv), len(rv), lk, fmtErrs(lv), rk, fmtErrs(rv)) + continue + } + next := false + for i := range lv { + if l, r := lv[i], rv[i]; l.Type != r.Type || l.Detail != r.Detail { + t.Errorf("different errors\n%s: %v\n%s: %v", lk, fmtErrs(lv), rk, fmtErrs(rv)) + next = true + break + } + } + if next { + continue + } + } + } +} + +// helper for nicer output +func fmtErrs(errs field.ErrorList) string { + if len(errs) == 0 { + return "" + } + if len(errs) == 1 { + return strconv.Quote(errs[0].Error()) + } + buf := bytes.Buffer{} + for _, e := range errs { + buf.WriteString("\n") + buf.WriteString(strconv.Quote(e.Error())) + } + + return buf.String() +} diff --git a/pkg/api/testing/validation_test.go b/pkg/api/testing/validation_test.go new file mode 100644 index 00000000000..868f1adf41e --- /dev/null +++ b/pkg/api/testing/validation_test.go @@ -0,0 +1,58 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "math/rand" + "testing" + + "k8s.io/apimachinery/pkg/api/apitesting/fuzzer" + "k8s.io/apimachinery/pkg/api/apitesting/roundtrip" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/kubernetes/pkg/api/legacyscheme" +) + +// FIXME: Automatically finds all group/versions supporting declarative validation, or add +// a reflexive test that verifies that they are all registered. +func TestVersionedValidationByFuzzing(t *testing.T) { + typesWithDeclarativeValidation := []schema.GroupVersion{ + // Registered group versions for versioned validation fuzz testing: + } + + for _, gv := range typesWithDeclarativeValidation { + t.Run(gv.String(), func(t *testing.T) { + for i := 0; i < *roundtrip.FuzzIters; i++ { + f := fuzzer.FuzzerFor(FuzzerFuncs, rand.NewSource(rand.Int63()), legacyscheme.Codecs) + for kind := range legacyscheme.Scheme.KnownTypes(gv) { + obj, err := legacyscheme.Scheme.New(gv.WithKind(kind)) + if err != nil { + t.Fatalf("could not create a %v: %s", kind, err) + } + f.Fill(obj) + VerifyVersionedValidationEquivalence(t, obj, nil) + + old, err := legacyscheme.Scheme.New(gv.WithKind(kind)) + if err != nil { + t.Fatalf("could not create a %v: %s", kind, err) + } + f.Fill(old) + VerifyVersionedValidationEquivalence(t, obj, old) + } + } + }) + } +} diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/testing/validation.go b/staging/src/k8s.io/apimachinery/pkg/runtime/testing/validation.go new file mode 100644 index 00000000000..cf7744f1096 --- /dev/null +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/testing/validation.go @@ -0,0 +1,119 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "context" + "testing" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +type VersionValidationRunner func(t *testing.T, gv string, versionValidationErrors field.ErrorList) + +// RunValidationForEachVersion runs f as a subtest of t for each version of the given unversioned object. +// Each subtest is named by GroupVersionKind. Each call to f is provided the field.ErrorList results +// of converting the unversioned object to a version and validating it. +// +// Only autogenerated validation is run. To test both handwritten and autogenerated validation: +// +// RunValidationForEachVersion(t, testCase.pod, func(t *testing.T, versionValidationErrors field.ErrorList) { +// errs := ValidatePod(testCase.obj) // hand written validation +// errs = append(errs, versionValidationErrors...) // generated declarative validation +// // Validate that the errors are what was expected for this test case. +// }) +func RunValidationForEachVersion(t *testing.T, scheme *runtime.Scheme, options sets.Set[string], unversioned runtime.Object, fn VersionValidationRunner) { + runValidation(t, scheme, options, unversioned, fn) +} + +// RunUpdateValidationForEachVersion is like RunValidationForEachVersion but for update validation. +func RunUpdateValidationForEachVersion(t *testing.T, scheme *runtime.Scheme, options sets.Set[string], unversioned, unversionedOld runtime.Object, fn VersionValidationRunner) { + runUpdateValidation(t, scheme, options, unversioned, unversionedOld, fn) +} + +// RunStatusValidationForEachVersion is like RunUpdateValidationForEachVersion but for status validation. +func RunStatusValidationForEachVersion(t *testing.T, scheme *runtime.Scheme, options sets.Set[string], unversioned, unversionedOld runtime.Object, fn VersionValidationRunner) { + runUpdateValidation(t, scheme, options, unversioned, unversionedOld, fn, "status") +} + +func runValidation(t *testing.T, scheme *runtime.Scheme, options sets.Set[string], unversioned runtime.Object, fn VersionValidationRunner, subresources ...string) { + unversionedGVKs, _, err := scheme.ObjectKinds(unversioned) + if err != nil { + t.Fatal(err) + } + for _, unversionedGVK := range unversionedGVKs { + gvs := scheme.VersionsForGroupKind(unversionedGVK.GroupKind()) + for _, gv := range gvs { + gvk := gv.WithKind(unversionedGVK.Kind) + t.Run(gvk.String(), func(t *testing.T) { + if gvk.Version != runtime.APIVersionInternal { // skip internal + versioned, err := scheme.New(gvk) + if err != nil { + t.Fatal(err) + } + err = scheme.Convert(unversioned, versioned, nil) + if err != nil { + t.Fatal(err) + } + fn(t, gv.String(), scheme.Validate(context.Background(), options, versioned, subresources...)) + } + }) + } + } +} + +func runUpdateValidation(t *testing.T, scheme *runtime.Scheme, options sets.Set[string], unversionedNew, unversionedOld runtime.Object, fn VersionValidationRunner, subresources ...string) { + unversionedGVKs, _, err := scheme.ObjectKinds(unversionedNew) + if err != nil { + t.Fatal(err) + } + for _, unversionedGVK := range unversionedGVKs { + gvs := scheme.VersionsForGroupKind(unversionedGVK.GroupKind()) + for _, gv := range gvs { + gvk := gv.WithKind(unversionedGVK.Kind) + t.Run(gvk.String(), func(t *testing.T) { + if gvk.Version != runtime.APIVersionInternal { // skip internal + versionedNew, err := scheme.New(gvk) + if err != nil { + t.Fatal(err) + } + err = scheme.Convert(unversionedNew, versionedNew, nil) + if err != nil { + t.Fatal(err) + } + + var versionedOld runtime.Object + if unversionedOld != nil { + versionedOld, err = scheme.New(gvk) + if err != nil { + t.Fatal(err) + } + + err = scheme.Convert(unversionedOld, versionedOld, nil) + if err != nil { + t.Fatal(err) + } + } + + fn(t, gv.String(), scheme.ValidateUpdate(context.Background(), options, versionedNew, versionedOld, subresources...)) + } + }) + } + } +}