mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-31 15:25:57 +00:00
Introduce versioned validation test utilitizes and add fuzz tester
This makes a bold assumption: that the errors (count and basic content) will be the same across versions. If this turns out to be untrue, this may need to get more sophisticated. It should fail obviously when we hit that edge.
This commit is contained in:
Tim Hockin
Joe Betz
parent
31d16ff2c2
commit
1d365762a5
107
pkg/api/testing/validation.go
Normal file
107
pkg/api/testing/validation.go
Normal file
@ -0,0 +1,107 @@
|
||||
/*
|
||||
Copyright 2025 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package testing
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"sort"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
k8sruntime "k8s.io/apimachinery/pkg/runtime"
|
||||
runtimetest "k8s.io/apimachinery/pkg/runtime/testing"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/apimachinery/pkg/util/validation/field"
|
||||
"k8s.io/kubernetes/pkg/api/legacyscheme"
|
||||
)
|
||||
|
||||
// VerifyVersionedValidationEquivalence tests that all versions of an API return equivalent validation errors.
|
||||
func VerifyVersionedValidationEquivalence(t *testing.T, obj, old k8sruntime.Object) {
|
||||
t.Helper()
|
||||
|
||||
// Accumulate errors from all versioned validation, per version.
|
||||
all := map[string]field.ErrorList{}
|
||||
accumulate := func(t *testing.T, gv string, errs field.ErrorList) {
|
||||
all[gv] = errs
|
||||
}
|
||||
if old == nil {
|
||||
runtimetest.RunValidationForEachVersion(t, legacyscheme.Scheme, sets.Set[string]{}, obj, accumulate)
|
||||
} else {
|
||||
runtimetest.RunUpdateValidationForEachVersion(t, legacyscheme.Scheme, sets.Set[string]{}, obj, old, accumulate)
|
||||
}
|
||||
|
||||
// Make a copy so we can modify it.
|
||||
other := map[string]field.ErrorList{}
|
||||
// Index for nicer output.
|
||||
keys := []string{}
|
||||
for k, v := range all {
|
||||
other[k] = v
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
// Compare each lhs to each rhs.
|
||||
for _, lk := range keys {
|
||||
lv := all[lk]
|
||||
// remove lk since to prevent comparison to itself and because this
|
||||
// iteration will compare it to any version it has not yet been
|
||||
// compared to. e.g. [1, 2, 3] vs. [1, 2, 3] yields:
|
||||
// 1 vs. 2
|
||||
// 1 vs. 3
|
||||
// 2 vs. 3
|
||||
delete(other, lk)
|
||||
// don't compare to ourself
|
||||
for _, rk := range keys {
|
||||
rv, found := other[rk]
|
||||
if !found {
|
||||
continue // done already
|
||||
}
|
||||
if len(lv) != len(rv) {
|
||||
t.Errorf("different error count (%d vs. %d)\n%s: %v\n%s: %v", len(lv), len(rv), lk, fmtErrs(lv), rk, fmtErrs(rv))
|
||||
continue
|
||||
}
|
||||
next := false
|
||||
for i := range lv {
|
||||
if l, r := lv[i], rv[i]; l.Type != r.Type || l.Detail != r.Detail {
|
||||
t.Errorf("different errors\n%s: %v\n%s: %v", lk, fmtErrs(lv), rk, fmtErrs(rv))
|
||||
next = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if next {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// helper for nicer output
|
||||
func fmtErrs(errs field.ErrorList) string {
|
||||
if len(errs) == 0 {
|
||||
return "<no errors>"
|
||||
}
|
||||
if len(errs) == 1 {
|
||||
return strconv.Quote(errs[0].Error())
|
||||
}
|
||||
buf := bytes.Buffer{}
|
||||
for _, e := range errs {
|
||||
buf.WriteString("\n")
|
||||
buf.WriteString(strconv.Quote(e.Error()))
|
||||
}
|
||||
|
||||
return buf.String()
|
||||
}
|
||||
58
pkg/api/testing/validation_test.go
Normal file
58
pkg/api/testing/validation_test.go
Normal file
@ -0,0 +1,58 @@
|
||||
/*
|
||||
Copyright 2025 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package testing
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"testing"
|
||||
|
||||
"k8s.io/apimachinery/pkg/api/apitesting/fuzzer"
|
||||
"k8s.io/apimachinery/pkg/api/apitesting/roundtrip"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/kubernetes/pkg/api/legacyscheme"
|
||||
)
|
||||
|
||||
// FIXME: Automatically finds all group/versions supporting declarative validation, or add
|
||||
// a reflexive test that verifies that they are all registered.
|
||||
func TestVersionedValidationByFuzzing(t *testing.T) {
|
||||
typesWithDeclarativeValidation := []schema.GroupVersion{
|
||||
// Registered group versions for versioned validation fuzz testing:
|
||||
}
|
||||
|
||||
for _, gv := range typesWithDeclarativeValidation {
|
||||
t.Run(gv.String(), func(t *testing.T) {
|
||||
for i := 0; i < *roundtrip.FuzzIters; i++ {
|
||||
f := fuzzer.FuzzerFor(FuzzerFuncs, rand.NewSource(rand.Int63()), legacyscheme.Codecs)
|
||||
for kind := range legacyscheme.Scheme.KnownTypes(gv) {
|
||||
obj, err := legacyscheme.Scheme.New(gv.WithKind(kind))
|
||||
if err != nil {
|
||||
t.Fatalf("could not create a %v: %s", kind, err)
|
||||
}
|
||||
f.Fill(obj)
|
||||
VerifyVersionedValidationEquivalence(t, obj, nil)
|
||||
|
||||
old, err := legacyscheme.Scheme.New(gv.WithKind(kind))
|
||||
if err != nil {
|
||||
t.Fatalf("could not create a %v: %s", kind, err)
|
||||
}
|
||||
f.Fill(old)
|
||||
VerifyVersionedValidationEquivalence(t, obj, old)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,119 @@
|
||||
/*
|
||||
Copyright 2024 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package testing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/apimachinery/pkg/util/validation/field"
|
||||
)
|
||||
|
||||
type VersionValidationRunner func(t *testing.T, gv string, versionValidationErrors field.ErrorList)
|
||||
|
||||
// RunValidationForEachVersion runs f as a subtest of t for each version of the given unversioned object.
|
||||
// Each subtest is named by GroupVersionKind. Each call to f is provided the field.ErrorList results
|
||||
// of converting the unversioned object to a version and validating it.
|
||||
//
|
||||
// Only autogenerated validation is run. To test both handwritten and autogenerated validation:
|
||||
//
|
||||
// RunValidationForEachVersion(t, testCase.pod, func(t *testing.T, versionValidationErrors field.ErrorList) {
|
||||
// errs := ValidatePod(testCase.obj) // hand written validation
|
||||
// errs = append(errs, versionValidationErrors...) // generated declarative validation
|
||||
// // Validate that the errors are what was expected for this test case.
|
||||
// })
|
||||
func RunValidationForEachVersion(t *testing.T, scheme *runtime.Scheme, options sets.Set[string], unversioned runtime.Object, fn VersionValidationRunner) {
|
||||
runValidation(t, scheme, options, unversioned, fn)
|
||||
}
|
||||
|
||||
// RunUpdateValidationForEachVersion is like RunValidationForEachVersion but for update validation.
|
||||
func RunUpdateValidationForEachVersion(t *testing.T, scheme *runtime.Scheme, options sets.Set[string], unversioned, unversionedOld runtime.Object, fn VersionValidationRunner) {
|
||||
runUpdateValidation(t, scheme, options, unversioned, unversionedOld, fn)
|
||||
}
|
||||
|
||||
// RunStatusValidationForEachVersion is like RunUpdateValidationForEachVersion but for status validation.
|
||||
func RunStatusValidationForEachVersion(t *testing.T, scheme *runtime.Scheme, options sets.Set[string], unversioned, unversionedOld runtime.Object, fn VersionValidationRunner) {
|
||||
runUpdateValidation(t, scheme, options, unversioned, unversionedOld, fn, "status")
|
||||
}
|
||||
|
||||
func runValidation(t *testing.T, scheme *runtime.Scheme, options sets.Set[string], unversioned runtime.Object, fn VersionValidationRunner, subresources ...string) {
|
||||
unversionedGVKs, _, err := scheme.ObjectKinds(unversioned)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, unversionedGVK := range unversionedGVKs {
|
||||
gvs := scheme.VersionsForGroupKind(unversionedGVK.GroupKind())
|
||||
for _, gv := range gvs {
|
||||
gvk := gv.WithKind(unversionedGVK.Kind)
|
||||
t.Run(gvk.String(), func(t *testing.T) {
|
||||
if gvk.Version != runtime.APIVersionInternal { // skip internal
|
||||
versioned, err := scheme.New(gvk)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = scheme.Convert(unversioned, versioned, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fn(t, gv.String(), scheme.Validate(context.Background(), options, versioned, subresources...))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func runUpdateValidation(t *testing.T, scheme *runtime.Scheme, options sets.Set[string], unversionedNew, unversionedOld runtime.Object, fn VersionValidationRunner, subresources ...string) {
|
||||
unversionedGVKs, _, err := scheme.ObjectKinds(unversionedNew)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, unversionedGVK := range unversionedGVKs {
|
||||
gvs := scheme.VersionsForGroupKind(unversionedGVK.GroupKind())
|
||||
for _, gv := range gvs {
|
||||
gvk := gv.WithKind(unversionedGVK.Kind)
|
||||
t.Run(gvk.String(), func(t *testing.T) {
|
||||
if gvk.Version != runtime.APIVersionInternal { // skip internal
|
||||
versionedNew, err := scheme.New(gvk)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = scheme.Convert(unversionedNew, versionedNew, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var versionedOld runtime.Object
|
||||
if unversionedOld != nil {
|
||||
versionedOld, err = scheme.New(gvk)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = scheme.Convert(unversionedOld, versionedOld, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
fn(t, gv.String(), scheme.ValidateUpdate(context.Background(), options, versionedNew, versionedOld, subresources...))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user