diff --git a/go.mod b/go.mod index a5f0613cc8e..c4ea3e0682f 100644 --- a/go.mod +++ b/go.mod @@ -118,6 +118,7 @@ require ( k8s.io/sample-apiserver v0.0.0 k8s.io/system-validators v1.11.1 k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 sigs.k8s.io/knftables v0.0.17 sigs.k8s.io/randfill v1.0.0 sigs.k8s.io/structured-merge-diff/v6 v6.3.0 @@ -217,7 +218,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect - sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/kustomize/api v0.20.1 // indirect sigs.k8s.io/kustomize/kustomize/v5 v5.7.1 // indirect sigs.k8s.io/kustomize/kyaml v0.20.1 // indirect diff --git a/test/e2e/dra/dra.go b/test/e2e/dra/dra.go index 1cfb33efbe0..778d61f92a4 100644 --- a/test/e2e/dra/dra.go +++ b/test/e2e/dra/dra.go @@ -52,6 +52,7 @@ import ( drautils "k8s.io/kubernetes/test/e2e/dra/utils" "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" + e2econformance "k8s.io/kubernetes/test/e2e/framework/conformance" e2edaemonset "k8s.io/kubernetes/test/e2e/framework/daemonset" e2eevents "k8s.io/kubernetes/test/e2e/framework/events" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" @@ -77,6 +78,156 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { // modify /var/lib/kubelet/plugins. f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged + f.Context("CRUD Tests", func() { + /* + Release: v1.? + Testname: CRUD operations for deviceclasses + Description: kube-apiserver must support create/update/list/patch/delete operations for resource.k8s.io/v1 DeviceClass. + */ + framework.It("resource.k8s.io/v1 DeviceClass", func(ctx context.Context) { + e2econformance.TestResource(ctx, f, + &e2econformance.ResourceTestcase[*resourceapi.DeviceClass]{ + GVR: resourceapi.SchemeGroupVersion.WithResource("deviceclasses"), + Namespaced: ptr.To(false), + InitialSpec: &resourceapi.DeviceClass{ + Spec: resourceapi.DeviceClassSpec{ + Selectors: []resourceapi.DeviceSelector{{ + CEL: &resourceapi.CELDeviceSelector{ + Expression: "false", // Matches no devices + }, + }}, + }, + }, + UpdateSpec: func(obj *resourceapi.DeviceClass) *resourceapi.DeviceClass { + obj.Spec.Selectors[0].CEL.Expression = "1 == 0" // Still matches no devices. + return obj + }, + StrategicMergePatchSpec: `{"spec": {"selectors": [{"cel": {"expression": "1 == 0"}}]}}`, + }, + ) + }) + + /* + Release: v1.? + Testname: CRUD operations for resourceclaims + Description: kube-apiserver must support create/update/list/patch/delete operations for resource.k8s.io/v1 ResourceClaim. + */ + framework.It("resource.k8s.io/v1 ResourceClaim", func(ctx context.Context) { + e2econformance.TestResource(ctx, f, + &e2econformance.ResourceTestcase[*resourceapi.ResourceClaim]{ + GVR: resourceapi.SchemeGroupVersion.WithResource("resourceclaims"), + Namespaced: ptr.To(true), + InitialSpec: &resourceapi.ResourceClaim{ + Spec: resourceapi.ResourceClaimSpec{ + Devices: resourceapi.DeviceClaim{ + Requests: []resourceapi.DeviceRequest{{ + Name: "req-0", + Exactly: &resourceapi.ExactDeviceRequest{ + DeviceClassName: "dra.example.com", + }, + }}, + }, + }, + }, + UpdateSpec: func(obj *resourceapi.ResourceClaim) *resourceapi.ResourceClaim { + // The spec is immutable, so let's add a label instead. + if obj.Labels == nil { + obj.Labels = make(map[string]string) + } + obj.Labels["test.dra.example.com"] = "test" + return obj + }, + UpdateStatus: func(obj *resourceapi.ResourceClaim) *resourceapi.ResourceClaim { + // Nothing allocated" is a valid allocation result. + obj.Status.Allocation = &resourceapi.AllocationResult{} + return obj + }, + + // This test case uses all available patch types to demonstrate what this looks like + // and that TestResource supports them. Strategic merge patch, apply patch, and JSON + // merge patch are identical for these simple modifications. + // + // Testing with only one patch type would be sufficient for conformance. + ApplyPatchSpec: `{"metadata": {"labels": {"test.dra.example.com": "test"}}}`, + StrategicMergePatchSpec: `{"metadata": {"labels": {"test.dra.example.com": "test"}}}`, + JSONMergePatchSpec: `{"metadata": {"labels": {"test.dra.example.com": "test"}}}`, + JSONPatchSpec: `[{"op": "add", "path": "/metadata/labels/test.dra.example.com", "value": "test"}]`, + ApplyPatchStatus: `{"status": {"allocation": {}}}`, + StrategicMergePatchStatus: `{"status": {"allocation": {}}}`, + JSONMergePatchStatus: `{"status": {"allocation": {}}}`, + JSONPatchStatus: `[{"op": "add", "path": "/status/allocation", "value": {}}]`, + }) + }) + + /* + Release: v1.? + Testname: CRUD operations for resourceclaimtemplates + Description: kube-apiserver must support create/update/list/patch/delete operations for resource.k8s.io/v1 ResourceClaimTemplate. + */ + framework.It("resource.k8s.io/v1 ResourceClaimTemplate", func(ctx context.Context) { + e2econformance.TestResource(ctx, f, + &e2econformance.ResourceTestcase[*resourceapi.ResourceClaimTemplate]{ + GVR: resourceapi.SchemeGroupVersion.WithResource("resourceclaimtemplates"), + Namespaced: ptr.To(true), + InitialSpec: &resourceapi.ResourceClaimTemplate{ + Spec: resourceapi.ResourceClaimTemplateSpec{ + Spec: resourceapi.ResourceClaimSpec{ + Devices: resourceapi.DeviceClaim{ + Requests: []resourceapi.DeviceRequest{{ + Name: "req-0", + Exactly: &resourceapi.ExactDeviceRequest{ + DeviceClassName: "dra.example.com", + }, + }}, + }, + }, + }, + }, + UpdateSpec: func(obj *resourceapi.ResourceClaimTemplate) *resourceapi.ResourceClaimTemplate { + // The spec is immutable, so let's add a label instead. + if obj.Labels == nil { + obj.Labels = make(map[string]string) + } + obj.Labels["test.dra.example.com"] = "test" + return obj + }, + StrategicMergePatchSpec: `{"metadata": {"labels": {"test.dra.example.com": "test"}}}`, + }, + ) + }) + + /* + Release: v1.? + Testname: CRUD operations for resoureslices + Description: kube-apiserver must support create/update/list/patch/delete operations for resource.k8s.io/v1 ResourceSlice. + */ + framework.It("resource.k8s.io/v1 ResourceSlice", func(ctx context.Context) { + e2econformance.TestResource(ctx, f, + &e2econformance.ResourceTestcase[*resourceapi.ResourceSlice]{ + GVR: resourceapi.SchemeGroupVersion.WithResource("resourceslices"), + Namespaced: ptr.To(false), + InitialSpec: &resourceapi.ResourceSlice{ + Spec: resourceapi.ResourceSliceSpec{ + Driver: "dra.example.com", + Pool: resourceapi.ResourcePool{ + Name: "cluster", + Generation: 1, + ResourceSliceCount: 1, + }, + NodeName: ptr.To("no-such-node"), + // The pool is empty -> no devices. + }, + }, + UpdateSpec: func(obj *resourceapi.ResourceSlice) *resourceapi.ResourceSlice { + obj.Spec.Devices = []resourceapi.Device{{Name: "device-0"}} + return obj + }, + StrategicMergePatchSpec: `{"spec": {"devices": [{"name": "device-0"}]}}`, + }, + ) + }) + }) + f.Context("kubelet", feature.DynamicResourceAllocation, func() { nodes := drautils.NewNodes(f, 1, 1) driver := drautils.NewDriver(f, nodes, drautils.NetworkResources(10, false)) diff --git a/test/e2e/framework/conformance/.import-restrictions b/test/e2e/framework/conformance/.import-restrictions new file mode 100644 index 00000000000..03b5ee5ec2c --- /dev/null +++ b/test/e2e/framework/conformance/.import-restrictions @@ -0,0 +1,12 @@ +# This E2E framework sub-package is currently allowed to use arbitrary +# dependencies except of k/k/pkg, therefore we need to override the +# restrictions from the parent .import-restrictions file. +# +# At some point it may become useful to also check this package's +# dependencies more careful. +rules: + - selectorRegexp: "^k8s[.]io/kubernetes/pkg" + allowedPrefixes: [] + + - selectorRegexp: "" + allowedPrefixes: [ "" ] diff --git a/test/e2e/framework/conformance/compare.go b/test/e2e/framework/conformance/compare.go new file mode 100644 index 00000000000..29ab0c2ac2c --- /dev/null +++ b/test/e2e/framework/conformance/compare.go @@ -0,0 +1,99 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package architecture + +import ( + "github.com/google/go-cmp/cmp" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// compareObjects checks that all expected fields are set as expected. +// The actual object may have additional fields, their values are ignored. +func compareObjects(expected, actual *unstructured.Unstructured) string { + diff := cmp.Diff(expected.Object, actual.Object, + // Fields which are not in the expected object can be ignored. + // Only existing fields need to be compared. + // + // A maybe (?) simpler approach would be to trim the actual object, + // then compare with go-cmp. The advantage of telling go-cmp to + // ignore fields is that they show up as truncated ("...") in the diff, + // which is a bit more correct. + cmp.FilterPath(func(path cmp.Path) bool { + return fieldIsMissing(expected.Object, path) + }, cmp.Ignore()), + ) + return diff +} + +// fieldIsMissing returns true if the field identified by the path is not +// present in the object. It works by recursively descending along the +// path and checking the corresponding content of the object along the way. +func fieldIsMissing(obj map[string]any, path cmp.Path) bool { + // First entry is a NOP. + missing := fieldIsMissingStep(obj, path[1:]) + // Uncomment for debugging... + // fmt.Printf("fieldIsMissing: %s %v\n", path.GoString(), missing) + return missing +} + +func fieldIsMissingStep(value any, path []cmp.PathStep) bool { + if len(path) == 0 { + // Done, full path was checked. + return false + } + // We only need to descend for certain lookup steps, + // everything else is treated as "not missing" and thus + // gets compared. + switch pathElement := path[0].(type) { + case cmp.MapIndex: + key := pathElement.Key().String() + value, ok := value.(map[string]any) + if !ok { + // Type mismatch. + return false + } + entry, found := value[key] + if !found { + return true + } + return fieldIsMissingStep(entry, path[1:]) + case cmp.SliceIndex: + key := pathElement.Key() + value, ok := value.([]any) + if !ok { + // Type mismatch. + return false + } + if key < 0 { + // Not sure why go-cmp uses a negative index, so let's compare it. + return false + } + if key >= len(value) { + // Slice is smaller -> missing entry. + return true + } + entry := value[key] + return fieldIsMissingStep(entry, path[1:]) + case cmp.TypeAssertion: + // Actual value type will be checked when needed, + // skip the assertion here. + return fieldIsMissingStep(value, path[1:]) + default: + return false + } +} diff --git a/test/e2e/framework/conformance/compare_test.go b/test/e2e/framework/conformance/compare_test.go new file mode 100644 index 00000000000..dcfc054b315 --- /dev/null +++ b/test/e2e/framework/conformance/compare_test.go @@ -0,0 +1,113 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package architecture + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func TestCompareObjects(t *testing.T) { + for name, tc := range map[string]struct { + expected, actual string + expectedDiff string + }{ + "equal": { + expected: `{"kind":"test","hello":"world"}`, + actual: `{"kind":"test","hello":"world"}`, + }, + + "missing": { + expected: `{"kind":"test","hello":"world"}`, + actual: `{"kind":"test"}`, + expectedDiff: ` map[string]any{ +- "hello": string("world"), + "kind": string("test"), + } +`, + }, + + "added": { + expected: `{"kind":"test","hello":"world"}`, + actual: `{"kind":"test","hello":"world","foo":"bar"}`, + }, + + "replaced": { + expected: `{"kind":"test","hello":"world"}`, + actual: `{"kind":"test","hello":1}`, + expectedDiff: ` map[string]any{ +- "hello": string("world"), ++ "hello": int64(1), + "kind": string("test"), + } +`, + }, + + "recursive": { + expected: `{"kind":"test","spec":{"hello":"world","removed":42}}`, + actual: `{"kind":"test","spec":{"hello":1,"added":42}}`, + expectedDiff: ` map[string]any{ + "kind": string("test"), + "spec": map[string]any{ + ... // 1 ignored entry +- "hello": string("world"), ++ "hello": int64(1), +- "removed": int64(42), + }, + } +`, + }, + + "list": { + expected: `{"kind":"test","items":[{"index":0},{"hello":"world","removed":42}]}`, + actual: `{"kind":"test","items":[{"index":0,"added":true},{"hello":1,"added":42},{"new-entry": true},"new-non-object-entry"]}`, + expectedDiff: ` map[string]any{ + "items": []any{ + map[string]any{"index": int64(0), ...}, ++ map[string]any{"added": int64(42), "hello": int64(1)}, ++ map[string]any{"new-entry": bool(true)}, +- map[string]any{"hello": string("world"), "removed": int64(42)}, ++ string("new-non-object-entry"), + }, + "kind": string("test"), + }`, + }, + } { + t.Run(name, func(t *testing.T) { + var expected, actual unstructured.Unstructured + require.NoError(t, expected.UnmarshalJSON([]byte(tc.expected)), "unmarshal expected") + require.NoError(t, actual.UnmarshalJSON([]byte(tc.actual)), "unmarshal actual") + actualDiff := compareObjects(&expected, &actual) + t.Logf("Actual diff:\n%s", actualDiff) + // Upstream go-cmp does not want the diff output to be checked in + // tests because it is not stable. They intentionally randomly + // switch between space and non-break space to enforce that + // (https://github.com/google/go-cmp/issues/366). + // + // Therefore the expected diff is merely informative, we only check + // for empty vs. not empty. + if tc.expectedDiff == "" { + require.Empty(t, actualDiff) + } else { + require.NotEmpty(t, actualDiff) + } + }) + } +} diff --git a/test/e2e/framework/conformance/conformance.go b/test/e2e/framework/conformance/conformance.go new file mode 100644 index 00000000000..e1c6139abf6 --- /dev/null +++ b/test/e2e/framework/conformance/conformance.go @@ -0,0 +1,946 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package architecture + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + gtypes "github.com/onsi/gomega/types" + + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/resourceversion" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/dynamic/dynamicinformer" + "k8s.io/client-go/informers" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + apimachineryutils "k8s.io/kubernetes/test/e2e/common/apimachinery" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/utils/format" + "k8s.io/utils/ptr" + k8sjson "sigs.k8s.io/json" +) + +// ResourceTestcaseInterface describes how to test one particular API endpoint +// by executing different operations against it. +// +// The content of created or patched objects is verified by ensuring that +// all fields are set as in the sent object. Extra fields or map entries +// are ignored. +// +// Basic create/read/update/delete (CRUD) semantic is covered, which +// is the minimum that is required for conformance testing of a +// GA feature. Actual functional testing is desirable, but not +// required. +// +// See [ResourceTestcase] for an implementation of this interface +// where test data is provided as Go objects and patch strings. +type ResourceTestcaseInterface interface { + // GetGroupVersionResource returns the API group, version, and resource (plural form, lower case). + GetGroupVersionResource() schema.GroupVersionResource + + // IsNamespaced defines whether the object must be created inside a namespace. + IsNamespaced() bool + + // HasStatus defines whether the resource has a "status" sub-resource. + // + // Other sub-resources are not supported by this common test code. + HasStatus() bool + + // VerifyContent defines whether the content of objects returned by + // the apiserver gets compared against the content that was sent. + // + // If enabled, all field values that were sent must also be included + // in the returned object. Additional fields and list or map entries + // may get added (for example, because of defaulting or mutating + // admission). + // + // This should not be enabled in conformance tests because admission + // is allowed to modify what is being stored. + VerifyContent() bool + + // GetInitialObject returns the data which is going to be used in a Create call. + // + // For cluster-scoped resources the test namespace can be used + // to create a name which does not conflict with other objects + // because it is unique while the test runs. + // + // It does not need to be set for namespaced resources because the + // caller will ensure that. The caller cannot do that for the name because + // different resources have different rules for what names are valid + GetInitialObject(namespace string) *unstructured.Unstructured + + // GetUpdateSpec modifies an existing object. + // It gets called for the result of creating the initial object. + // + // Ideally it should change the spec (hence the name). + // If that is impossible, then adding some label is also okay. + // The goal is to add some fields that can be checked for + // after an Update. + // + // May modify and return the input object. + GetUpdateSpec(object *unstructured.Unstructured) *unstructured.Unstructured + + // GetUpdateStatus modifies the status of an existing object. + // It gets called for the result of creating the initial object + // and then updating its spec. + // + // May modify and return the input object. + GetUpdateStatus(object *unstructured.Unstructured) *unstructured.Unstructured + + // GetPatchSpec describes how to generate patches. + // + // Each patch is applied to the initial object by itself, without the other patches. + // An empty slice is valid and disables testing of patching. This may not be sufficient + // for full conformance testing of the resource. + // + // If content verification is enabled, then this must cause the same change as GetUpdateSpec + // because verification of the patch result uses the GetUpdateSpec result as reference. + GetPatchSpec() []Patch + + // GetPatchStatus is like GetPatchSpec for the status. + // + // The initial object with the updated spec gets patched, + // so the result must match the result of GetUpdateStatus + // applied to GetInitialObject if content verification is + // enabled. + GetPatchStatus(object *unstructured.Unstructured) []Patch +} + +// Patch contains the parameters for a Patch API call. +// +// The data must match an existing object. +// +// There's no retry loop because of conflicts, so the patch should not include +// a check of the ResourceVersion. Checking the UID in the patch is encouraged to prevent +// patching a replaced resource. +type Patch struct { + GetData func(object *unstructured.Unstructured) []byte + Type types.PatchType +} + +// ResourceTestcase provides test data for testing operations for a resource. +// Test data is based on the native Go type of the resource. +// The template parameter must be a pointer to the native Go type. +// +// The data is used like this: +// - create InitialSpec -> update with UpdateSpec -> update status with UpdateStatus +// - create InitialSpec -> apply StrategicMergePatchSpec and compare against UpdateSpec -> apply StrategicMergePatchStatus and compare against UpdateStatus +type ResourceTestcase[T runtime.Object] struct { + // GroupResourceVersion identifies the API group, version and resource (plural form, lower case) + // within that API which is to be tested. + GVR schema.GroupVersionResource + + // Namespaced must be true if the resource must be created in a + // namespace, false if it is cluster-scoped. Leaving it unset is + // an error. + // + // Namespaced resources get created in the test namespace. + // + // The name of cluster-scoped resources gets extended with + // `-` to make it unique. + Namespaced *bool + + // ContentVerificationEnabled defines whether the content of objects returned by + // the apiserver gets compared against the content that was sent. + // + // If enabled, all field values that were sent must also be included + // in the returned object. Additional fields and list or map entries + // may get added (for example, because of defaulting or mutating + // admission). + // + // This should not be enabled in conformance tests because admission + // is allowed to modify what is being stored. + ContentVerificationEnabled bool + + // InitialSpec must contain the initial state of a valid resource, without a status. + InitialSpec T + + // UpdateSpec gets called for the created initial object + // and must update something, ideally the spec (hence the name). + // If that is not possible, then adding some label also works + // for the sake of testing an update. + // + // May modify and return the input object. + UpdateSpec func(T) T + + // UpdateStatus gets called for the updated object + // and must add a status. + // + // May be nil if no status is supported. + // + // May modify and return the input object. + UpdateStatus func(T) T + + // StrategicMergePatchSpec may modify fields in InitialSpec + // with a strategic merge patch + // (https://github.com/kubernetes/community/blob/master/contributors/devel/sig-api-machinery/strategic-merge-patch.md). + // Muse use JSON encoding. + // + // If content verification is enabled, then this must contain the same change as UpdateSpec + // because verification of the patch result uses UpdateSpec as reference. + StrategicMergePatchSpec string + + // StrategicMergePatchStatus may add status fields + // with a strategic merge patch + // (https://github.com/kubernetes/community/blob/master/contributors/devel/sig-api-machinery/strategic-merge-patch.md) + // Must use JSON encoding. + // + // The initial object with the updated spec gets patched, + // so the result must match GetUpdateStatus + // applied to GetInitialObject if content verification is + // enabled. + // + // If empty, the status sub-resource is not getting tested. + // May contain the name, but that's not required. + StrategicMergePatchStatus string + + // ApplyPatchSpec corresponds to StrategicMergePatchSpec, + // using the JSON encoding of a server-side-apply (SSA) patch + // (https://kubernetes.io/docs/reference/using-api/server-side-apply). + ApplyPatchSpec string + + // ApplyPatchStatus corresponds to StrategicMergePatchStatus, + // using the JSON encoding of a server-side-apply (SSA) patch + // (https://kubernetes.io/docs/reference/using-api/server-side-apply). + ApplyPatchStatus string + + // JSONPatchSpec corresponds to StrategicMergePatchSpec, + // using a JSON patch (https://tools.ietf.org/html/rfc6902). + JSONPatchSpec string + + // JSONPatchStatus corresponds to StrategicMergePatchStatus, + // using a JSON patch (https://tools.ietf.org/html/rfc6902). + JSONPatchStatus string + + // JSONMergePatchSpec corresponds to StrategicMergePatchSpec, + // using a JSON merge patch (https://tools.ietf.org/html/rfc7386). + JSONMergePatchSpec string + + // JSONMergePatchStatus corresponds to StrategicMergePatchStatus, + // using a JSON merge patch (https://tools.ietf.org/html/rfc7386). + JSONMergePatchStatus string +} + +var _ ResourceTestcaseInterface = &ResourceTestcase[*v1.Pod]{} + +func (tc *ResourceTestcase[T]) GetGroupVersionResource() schema.GroupVersionResource { + return tc.GVR +} + +func (tc *ResourceTestcase[T]) IsNamespaced() bool { + if tc.Namespaced == nil { + framework.Fail("Test case error: Namespaced must be set") + } + + return *tc.Namespaced +} + +func (tc *ResourceTestcase[T]) HasStatus() bool { + return tc.UpdateStatus != nil +} + +func (tc *ResourceTestcase[T]) VerifyContent() bool { + return tc.ContentVerificationEnabled +} + +func (tc *ResourceTestcase[T]) GetInitialObject(namespace string) *unstructured.Unstructured { + object := tc.toUnstructured("InitialSpec", tc.InitialSpec) + if object.GetName() == "" { + object.SetName("test") + } + if !tc.IsNamespaced() { + object.SetName(object.GetName() + "-" + namespace) + } + + return object +} + +func (tc *ResourceTestcase[T]) GetUpdateSpec(in *unstructured.Unstructured) *unstructured.Unstructured { + out := tc.fromUnstructured("existing object", in) + out = tc.UpdateSpec(out) + return tc.toUnstructured("updated object", out) +} + +func (tc *ResourceTestcase[T]) GetUpdateStatus(in *unstructured.Unstructured) *unstructured.Unstructured { + out := tc.fromUnstructured("updated object", in) + out = tc.UpdateStatus(out) + return tc.toUnstructured("updated object with status", out) +} + +func (tc *ResourceTestcase[T]) GetPatchSpec() []Patch { + var patches []Patch + + if tc.StrategicMergePatchSpec != "" { + patches = append(patches, Patch{ + Type: types.StrategicMergePatchType, + GetData: func(existingObject *unstructured.Unstructured) []byte { + patch := tc.createPatchObject("StrategicMergePatchSpec", tc.StrategicMergePatchSpec, existingObject) + + jsonData, err := patch.MarshalJSON() + framework.ExpectNoError(err, "re-encode spec patch as JSON") + + return jsonData + }, + }) + } + + if tc.ApplyPatchSpec != "" { + patches = append(patches, Patch{ + Type: types.ApplyPatchType, + GetData: func(existingObject *unstructured.Unstructured) []byte { + patch := tc.createPatchObject("ApplyPatchSpec", tc.ApplyPatchSpec, existingObject) + + jsonData, err := patch.MarshalJSON() + framework.ExpectNoError(err, "re-encode spec patch as JSON") + + return jsonData + }, + }) + } + + if tc.JSONMergePatchSpec != "" { + patches = append(patches, Patch{ + Type: types.MergePatchType, + GetData: func(existingObject *unstructured.Unstructured) []byte { + patch := tc.createPatchObject("JSONMergePatchSpec", tc.JSONMergePatchSpec, existingObject) + + jsonData, err := patch.MarshalJSON() + framework.ExpectNoError(err, "re-encode spec patch as JSON") + + return jsonData + }, + }) + } + + if tc.JSONPatchSpec != "" { + patches = append(patches, Patch{ + Type: types.JSONPatchType, + GetData: func(existingObject *unstructured.Unstructured) []byte { + return []byte(tc.JSONPatchSpec) + }, + }) + } + + return patches +} + +func (tc *ResourceTestcase[T]) GetPatchStatus(object *unstructured.Unstructured) []Patch { + var patches []Patch + + if tc.StrategicMergePatchStatus != "" { + patches = append(patches, Patch{ + Type: types.StrategicMergePatchType, + GetData: func(existingObject *unstructured.Unstructured) []byte { + patch := tc.createPatchObject("StrategicMergePatchStatus", tc.StrategicMergePatchStatus, existingObject) + + jsonData, err := patch.MarshalJSON() + framework.ExpectNoError(err, "re-encode status patch as JSON") + + return jsonData + }, + }) + } + + if tc.ApplyPatchStatus != "" { + patches = append(patches, Patch{ + Type: types.ApplyPatchType, + GetData: func(existingObject *unstructured.Unstructured) []byte { + patch := tc.createPatchObject("ApplyPatchStatus", tc.ApplyPatchStatus, existingObject) + + jsonData, err := patch.MarshalJSON() + framework.ExpectNoError(err, "re-encode status patch as JSON") + + return jsonData + }, + }) + } + + if tc.JSONMergePatchStatus != "" { + patches = append(patches, Patch{ + Type: types.MergePatchType, + GetData: func(existingObject *unstructured.Unstructured) []byte { + patch := tc.createPatchObject("JSONMergePatchStatus", tc.JSONMergePatchStatus, existingObject) + + jsonData, err := patch.MarshalJSON() + framework.ExpectNoError(err, "re-encode status patch as JSON") + + return jsonData + }, + }) + } + + if tc.JSONPatchStatus != "" { + patches = append(patches, Patch{ + Type: types.JSONPatchType, + GetData: func(existingObject *unstructured.Unstructured) []byte { + return []byte(tc.JSONPatchStatus) + }, + }) + } + + return patches +} + +func (tc *ResourceTestcase[T]) toUnstructured(what string, in T) *unstructured.Unstructured { + data, err := json.Marshal(in) + framework.ExpectNoError(err, "encode %s as JSON", what) + + out := tc.toUnstructuredFromJSON(what, data) + + return out +} + +func (tc *ResourceTestcase[T]) toUnstructuredFromJSON(what string, in []byte) *unstructured.Unstructured { + // UnmarshalCaseSensitivePreserveInts does not need kind (in contrast to unstructured.Unstructured.UnmarshalJSON) + // and matches the behavior of preserving ints that we get when receiving from the apiserver (in contrast to plain json.Unmarshal). + var out unstructured.Unstructured + err := k8sjson.UnmarshalCaseSensitivePreserveInts(in, &out.Object) + framework.ExpectNoError(err, "decode %s from JSON", what) + + return &out + +} + +func (tc *ResourceTestcase[T]) fromUnstructured(what string, in *unstructured.Unstructured) T { + data, err := in.MarshalJSON() + framework.ExpectNoError(err, "encode %s as JSON", what) + + var out T + err = k8sjson.UnmarshalCaseSensitivePreserveInts(data, &out) + framework.ExpectNoError(err, "decode %s from JSON", what) + + return out + +} + +// createPatchObject parses JSON data and then copies namespace/name/uid/kind/apiVersion from the existing object +// to make the patch complete. This works for strategic merge patches, apply patches and JSON merge patches. +func (tc *ResourceTestcase[T]) createPatchObject(what string, data string, existingObject *unstructured.Unstructured) *unstructured.Unstructured { + object := tc.toUnstructuredFromJSON(what, []byte(data)) + object.SetNamespace(existingObject.GetNamespace()) + object.SetName(existingObject.GetName()) + object.SetUID(existingObject.GetUID()) + object.SetAPIVersion(existingObject.GetAPIVersion()) + object.SetKind(existingObject.GetKind()) + return object +} + +// TestResource covers all the typical endpoints for a resource through +// dynamic client calls. +func TestResource(ctx context.Context, f *framework.Framework, tc ResourceTestcaseInterface) { + // Set up clients. + gvr := tc.GetGroupVersionResource() + gv := gvr.GroupVersion() + resource := gvr.Resource + resourceClient := f.DynamicClient.Resource(gvr) + var client dynamic.ResourceInterface + var resourceType string + if tc.IsNamespaced() { + client = resourceClient.Namespace(f.Namespace.Name) + resourceType = "namespaced resource" + } else { + client = resourceClient + resourceType = "cluster-scoped resource" + } + // e.g. `cluster-scoped resource "deviceclasses"` + // gvr.String() is too long and includes a comma ("resource.k8s.io/v1, Resource=deviceclasses"). + resourceType = fmt.Sprintf("%s %q", resourceType, gvr.Resource) + config := dynamic.ConfigFor(f.ClientConfig()) + httpClient, err := rest.HTTPClientFor(config) + framework.ExpectNoError(err, "construct HTTP client") + restClient, err := rest.UnversionedRESTClientForConfigAndClient(config, httpClient) + framework.ExpectNoError(err, "construct REST client") + + // All objects get one label added by the test for List and DeleteCollection. + // The label must get added to all objects returned by ResourceTestcase + // because the implementation of that interface is unaware of the extra label. + labelName := "e2e-test.kubernetes.io" + labelValue := f.UniqueName + listOptions := metav1.ListOptions{LabelSelector: labelName + "=" + labelValue} + addLabel := func(obj *unstructured.Unstructured) *unstructured.Unstructured { + obj = obj.DeepCopy() + labels := obj.GetLabels() + if labels == nil { + labels = make(map[string]string) + } + labels[labelName] = labelValue + obj.SetLabels(labels) + return obj + } + + // Prepare for Create, Get and List. + desiredInitialObject := addLabel(tc.GetInitialObject(f.Namespace.Name)) + if tc.IsNamespaced() { + desiredInitialObject.SetNamespace(f.Namespace.Name) + } + + getResource := func(ctx context.Context) (*unstructured.Unstructured, error) { + return client.Get(ctx, desiredInitialObject.GetName(), metav1.GetOptions{}) + } + desiredUpdatedObject := tc.GetUpdateSpec(desiredInitialObject.DeepCopy()) + var desiredUpdatedObjectWithStatus *unstructured.Unstructured + if tc.HasStatus() { + desiredUpdatedObjectWithStatus = addLabel(tc.GetUpdateStatus(desiredUpdatedObject)) + } + + // Get all resources in the API. The resulting list of resources must include what we are testing. + ginkgo.By(fmt.Sprintf("Get %s", gv)) + path := "/apis/" + gv.String() + var api unstructured.Unstructured + err = restClient. + Get(). + AbsPath(path). + Do(ctx). + Into(&api) + framework.ExpectNoError(err, "get resource API") + resources := api.Object["resources"].([]any) + index := slices.IndexFunc(resources, func(entry any) bool { + return entry.(map[string]any)["name"].(string) == resource + }) + if index < 0 { + framework.Failf("API for %s does not include entry for %s, got:\n%s", gv, resource, format.Object(api, 1)) + } + + // Set up informers, optionally also in the namespace. + // After each step we check that the informers catch up + // and what events they received in the meantime. + // They get stopped through test context cancellation. + var resourceEvents, namespaceEvents eventRecorder + resourceInformer := dynamicinformer.NewFilteredDynamicInformer(f.DynamicClient, gvr, "", 0, nil, func(options *metav1.ListOptions) { + options.LabelSelector = listOptions.LabelSelector + }) + _, err = resourceInformer.Informer().AddEventHandler(&resourceEvents) + framework.ExpectNoError(err, "register resource event handler") + listResource := func(_ context.Context) ([]runtime.Object, error) { + return resourceInformer.Lister().List(labels.Everything()) + } + go resourceInformer.Informer().RunWithContext(ctx) + informersHaveSynced := []cache.InformerSynced{resourceInformer.Informer().HasSynced} + var namespaceInformer informers.GenericInformer + var listNamespace func(_ context.Context) ([]runtime.Object, error) + if tc.IsNamespaced() { + namespaceInformer = dynamicinformer.NewFilteredDynamicInformer(f.DynamicClient, gvr, f.Namespace.Name, 0, nil, func(options *metav1.ListOptions) { + options.LabelSelector = listOptions.LabelSelector + }) + _, err = namespaceInformer.Informer().AddEventHandler(&namespaceEvents) + framework.ExpectNoError(err, "register namespace event handler") + listNamespace = func(_ context.Context) ([]runtime.Object, error) { + return namespaceInformer.Lister().List(labels.Everything()) + } + go namespaceInformer.Informer().RunWithContext(ctx) + informersHaveSynced = append(informersHaveSynced, namespaceInformer.Informer().HasSynced) + } + if !cache.WaitForNamedCacheSyncWithContext(ctx, informersHaveSynced...) { + ginkgo.Fail("informers should have synced and didn't") + } + + // matchObject generates a matcher which checks the result of a list operation + // against the expected object. Content verification is optional. Without it, + // only the namespace and name are checked. + matchObject := func(expectedObject *unstructured.Unstructured) gtypes.GomegaMatcher { + return &matchObjectList{expectedObject: expectedObject, verifyContent: tc.VerifyContent()} + } + gomega.Expect(listResource(ctx)).Should(matchObject(nil), "initial list of resources from informer cache") + gomega.Expect(resourceEvents.list()).To(gomega.HaveField("Events", gomega.BeEmpty()), "no events from resource informer yet") + if listNamespace != nil { + gomega.Expect(listNamespace(ctx)).Should(matchObject(nil), "initial list of namespace from informer cache") + gomega.Expect(resourceEvents.list()).To(gomega.HaveField("Events", gomega.BeEmpty()), "no events from namespace informer yet") + } + + // matchEvents generates a matcher which checks the informer event list. + // + // The events are expected to involve only the given object. + // The ResourceVersion must not decrease. + // The sequence of valid events is given as a regular expression + // which is applied to the string returned by [EventList.Types]. + matchEvents := func(obj *unstructured.Unstructured, regexp string) gtypes.GomegaMatcher { + // Verify namespace/name/uid of ids event. + ids := gomega.HaveField("Events", gomega.Or(gomega.BeEmpty(), gomega.HaveEach(gomega.HaveField("ID()", gomega.Equal(fmt.Sprintf("%s, %s", klog.KObj(obj), obj.GetUID())))))) + + // Match the regexp against the result of Types(). + order := gomega.HaveField("Types()", gomega.MatchRegexp(regexp)) + + // Include full object dump, HaveField itself doesn't. + return framework.GomegaObject(gomega.And(ids, order)) + } + + // ResourceVersion must not decrease. Delete events reset the sequence. + var resourceRV, namespaceRV string + nextEvents := func(rv *string, events eventList) { + ginkgo.GinkgoHelper() + + checkNext := func(obj any) { + ginkgo.GinkgoHelper() + if obj == nil { + return + } + if tomb, ok := obj.(cache.DeletedFinalStateUnknown); ok { + obj = tomb.Obj + } + metaData, err := meta.Accessor(obj) + framework.ExpectNoError(err, "access meta data") + gomega.Expect(metaData).To(apimachineryutils.HaveValidResourceVersion()) + + nextRV := metaData.GetResourceVersion() + if *rv == "" { + // Nothing to compare yet, initial version. + *rv = nextRV + return + } + + cmpResult, err := resourceversion.CompareResourceVersion(nextRV, *rv) + framework.ExpectNoError(err, "compare ResourceVersions") + if cmpResult < 0 { + framework.Failf("ResourceVersion %s in %s with UID %s is smaller than previous %s, should be equal or larger", nextRV, klog.KObj(metaData), metaData.GetUID(), *rv) + } + *rv = nextRV + } + + for _, e := range events.Events { + checkNext(e.oldObj) + checkNext(e.newObj) + } + } + + // Verification of the result after each step (= what). + // + // Can check the content of an object or be limited to just name/namespace. + // + // Also checks the informers. This is a bit redundant after read-only steps, + // but then it's also fast and thus can be done more often than strictly necessary. + // Collected informer events get reset, so each verify call must match + // events since the previous one. + verify := func(what string, expected, actual *unstructured.Unstructured, haveExpectedEvents gtypes.GomegaMatcher) { + ginkgo.GinkgoHelper() + + // This captures several different failures before handing them to Ginkgo. + var failures gomegaFailures + + if expected != nil { + if tc.VerifyContent() { + diff := compareObjects(expected, actual) + failures.Add(fmt.Sprintf("%s: unexpected actual object (- expected, + actual):\n%s", what, diff)) + } else { + failures.G().Expect(actual.GetName()).Should(gomega.Equal(expected.GetName()), "%s: name in returned object", what) + failures.G().Expect(actual.GetNamespace()).Should(gomega.Equal(expected.GetNamespace()), "%s: namespace in returned object", what) + } + failures.G().Expect(actual).To(apimachineryutils.HaveValidResourceVersion()) + } + + // Abort checking now if there were failures, otherwise we just risk timining out slowly. + failures.Check() + + failures.G().Eventually(ctx, listResource).Should(matchObject(expected), "list of resources from informer cache after %s", what) + if haveExpectedEvents != nil { + // Even if the cache is up-to-date we still need to wait for event delivery. + failures.G().Eventually(resourceEvents.list). + WithTimeout(5*time.Second). + Should(haveExpectedEvents, "list of resource informer events after %s", what) + } + nextEvents(&resourceRV, resourceEvents.reset()) + if listNamespace != nil { + failures.G().Eventually(ctx, listNamespace).Should(matchObject(expected), "list of namespace from informer cache after %s", what) + if haveExpectedEvents != nil { + failures.G().Eventually(namespaceEvents.list). + WithTimeout(5*time.Second). + Should(haveExpectedEvents, "list of namespace informer events after %s", what) + } + nextEvents(&namespaceRV, namespaceEvents.reset()) + } + failures.Check() + } + + // Create the initial resource. + ginkgo.By(fmt.Sprintf("Creating:\n%s", format.Object(desiredInitialObject, 1))) + existingObject, err := client.Create(ctx, desiredInitialObject, metav1.CreateOptions{FieldValidation: "Strict"}) + framework.ExpectNoError(err, "create initial %s", resourceType) + ginkgo.DeferCleanup(func(ctx context.Context) { + // Always clean up. + err = client.Delete(ctx, desiredInitialObject.GetName(), metav1.DeleteOptions{}) + if apierrors.IsNotFound(err) { + return + } + framework.ExpectNoError(err, "delete %s", resourceType) + ensureNotFound(ctx, getResource) + }) + verify("create", desiredInitialObject, existingObject, + // Initial creation of the object followed by some optional updates by cluster components. + matchEvents(existingObject, "^add,(update,)*$"), + ) + createdResourceVersion := existingObject.GetResourceVersion() + + // Get to check for existence. + ginkgo.By(fmt.Sprintf("Getting %s", klog.KObj(desiredInitialObject))) + existingObject, err = client.Get(ctx, desiredInitialObject.GetName(), metav1.GetOptions{}) + framework.ExpectNoError(err, "get updated %s", resourceType) + verify("get", desiredInitialObject, existingObject, + // Optional updates by cluster components. + matchEvents(existingObject, "^(update,)*$"), + ) + + // Update the resource. Retry because the existing object might have been updated in the meantime. + mustGet := false + gomega.Eventually(ctx, func(ctx context.Context) error { + if mustGet { + ginkgo.By(fmt.Sprintf("Getting updated %s", klog.KObj(desiredInitialObject))) + existingObject, err = getResource(ctx) + if err != nil { + return fmt.Errorf("get existing %s: %w", resourceType, err) + } + } + object := tc.GetUpdateSpec(existingObject.DeepCopy()) + ginkgo.By(fmt.Sprintf("Updating:\n%s", format.Object(object, 1))) + existingObject, err = client.Update(ctx, object, metav1.UpdateOptions{}) + if err == nil { + return nil + } + mustGet = apierrors.IsConflict(err) + if mustGet { + // Retry immediately. + return fmt.Errorf("update %s: %w", resourceType, err) + } + if retry, retryAfter := framework.ShouldRetry(err); retry { + // Retry with a delay. + return gomega.TryAgainAfter(retryAfter) + } + // Give up, some other error occurred. + return gomega.StopTrying(fmt.Sprintf("update %s", resourceType)).Wrap(err) + }).Should(gomega.Succeed()) + verify("update", desiredUpdatedObject, existingObject, + // At least one update. + matchEvents(existingObject, "^(update,)+$"), + ) + updatedResourceVersion := existingObject.GetResourceVersion() + cmpResult, err := resourceversion.CompareResourceVersion(createdResourceVersion, updatedResourceVersion) + framework.ExpectNoError(err, "compare ResourceVersion after create against ResourceVersion after update") + if cmpResult >= 0 { + framework.Failf("ResourceVersion should have increased during update and didn't (before: %s, after: %s)", createdResourceVersion, updatedResourceVersion) + } + + // Same for the status. In addition, read the status (same result, but different endpoint+method). + if tc.HasStatus() { + mustGet := false + gomega.Eventually(ctx, func(ctx context.Context) error { + if mustGet { + ginkgo.By(fmt.Sprintf("Getting updated %s", klog.KObj(desiredInitialObject))) + existingObject, err = client.Get(ctx, desiredInitialObject.GetName(), metav1.GetOptions{}, "status") + if err != nil { + return fmt.Errorf("get existing %s: %w", resourceType, err) + } + } + object := tc.GetUpdateStatus(existingObject) + ginkgo.By(fmt.Sprintf("Updating status:\n%s", format.Object(object, 1))) + existingObject, err = client.Update(ctx, object, metav1.UpdateOptions{}, "status") + if err == nil { + return nil + } + mustGet = apierrors.IsConflict(err) + if mustGet { + // Retry immediately. + return fmt.Errorf("update %s status: %w", resourceType, err) + } + if retry, retryAfter := framework.ShouldRetry(err); retry { + // Retry with a delay. + return gomega.TryAgainAfter(retryAfter) + } + // Give up, some other error occurred. + return gomega.StopTrying(fmt.Sprintf("update %s status", resourceType)).Wrap(err) + }).Should(gomega.Succeed()) + verify("update status", desiredUpdatedObjectWithStatus, existingObject, + // At least one update. + matchEvents(existingObject, "^(update,)+$"), + ) + + ginkgo.By(fmt.Sprintf("Getting %s status", klog.KObj(desiredInitialObject))) + existingObject, err = client.Get(ctx, desiredInitialObject.GetName(), metav1.GetOptions{}, "status") + framework.ExpectNoError(err, "get updated %s", resourceType) + verify("get", desiredUpdatedObjectWithStatus, existingObject, + // Optional updates by cluster components. + matchEvents(existingObject, "^(update,)*$"), + ) + } + + // Patch the resource, potentially using multiple different patch types. + // The result must be the same each time if content verification is enabled. + for _, patch := range tc.GetPatchSpec() { + // Delete the resource to start anew. + ginkgo.By(fmt.Sprintf("Deleting %s", klog.KObj(desiredInitialObject))) + err = client.Delete(ctx, desiredInitialObject.GetName(), metav1.DeleteOptions{}) + framework.ExpectNoError(err, "delete updated %s", resourceType) + ensureNotFound(ctx, getResource) + verify("delete", nil, nil, + // Optional updates, deletion. + // + // We have to verify this here because + // otherwise we have no guarantee that we see the delete event. + matchEvents(existingObject, "^(update,)*delete,$"), + ) + + // Recreate for patching. + ginkgo.By(fmt.Sprintf("Creating again:\n%s", format.Object(desiredInitialObject, 1))) + existingObject, err = client.Create(ctx, desiredInitialObject, metav1.CreateOptions{}) + framework.ExpectNoError(err, "create %s again", resourceType) + patchData := patch.GetData(existingObject) + + ginkgo.By(fmt.Sprintf("Patching with %s:\n%s", patch.Type, string(patchData))) + options := metav1.PatchOptions{FieldValidation: "Strict"} + switch patch.Type { + case types.ApplyYAMLPatchType, types.ApplyCBORPatchType: + options.FieldManager = "test-apply" + options.Force = ptr.To(true) + } + existingObject, err = client.Patch(ctx, desiredInitialObject.GetName(), patch.Type, patchData, options) + framework.ExpectNoError(err, "patch %s %s", patch.Type, resourceType) + verify(fmt.Sprintf("patch %s", patch.Type), desiredUpdatedObject, existingObject, + // Recreation and then at least one update. + matchEvents(existingObject, "^add,(update,)+$"), + ) + } + + // Same for status. The patches apply on top of the updated object. + for _, patch := range tc.GetPatchStatus(existingObject) { + // Delete the resource to start anew. + ginkgo.By(fmt.Sprintf("Deleting %s", klog.KObj(desiredInitialObject))) + err = client.Delete(ctx, desiredInitialObject.GetName(), metav1.DeleteOptions{}) + framework.ExpectNoError(err, "delete updated %s", resourceType) + ensureNotFound(ctx, getResource) + verify("delete", nil, nil, + // Optional updates, deletion. + // + // We have to verify this here because + // otherwise we have no guarantee that we see the delete event. + matchEvents(existingObject, "^(update,)*delete,$"), + ) + + // Recreate for patching. + ginkgo.By(fmt.Sprintf("Creating again:\n%s", format.Object(desiredInitialObject, 1))) + existingObject, err = client.Create(ctx, desiredInitialObject, metav1.CreateOptions{}) + framework.ExpectNoError(err, "create %s again", resourceType) + + // Update again. + mustGet := false + gomega.Eventually(ctx, func(ctx context.Context) error { + if mustGet { + ginkgo.By(fmt.Sprintf("Getting updated %s", klog.KObj(desiredInitialObject))) + existingObject, err = getResource(ctx) + if err != nil { + return fmt.Errorf("get existing %s: %w", resourceType, err) + } + } + object := tc.GetUpdateSpec(existingObject.DeepCopy()) + ginkgo.By(fmt.Sprintf("Updating:\n%s", format.Object(object, 1))) + existingObject, err = client.Update(ctx, object, metav1.UpdateOptions{}) + if err == nil { + return nil + } + mustGet = apierrors.IsConflict(err) + if mustGet { + // Retry immediately. + return fmt.Errorf("update %s: %w", resourceType, err) + } + if retry, retryAfter := framework.ShouldRetry(err); retry { + // Retry with a delay. + return gomega.TryAgainAfter(retryAfter) + } + // Give up, some other error occurred. + return gomega.StopTrying(fmt.Sprintf("update %s", resourceType)).Wrap(err) + }).Should(gomega.Succeed()) + patchData := patch.GetData(existingObject) + + ginkgo.By(fmt.Sprintf("Patching status with %s:\n%s", patch.Type, string(patchData))) + options := metav1.PatchOptions{FieldValidation: "Strict"} + switch patch.Type { + case types.ApplyYAMLPatchType, types.ApplyCBORPatchType: + options.FieldManager = "test-apply" + options.Force = ptr.To(true) + } + existingObject, err = client.Patch(ctx, desiredInitialObject.GetName(), patch.Type, patchData, options, "status") + framework.ExpectNoError(err, "patch %s %s status", patch.Type, resourceType) + verify(fmt.Sprintf("patch %s status", patch.Type), desiredUpdatedObject, existingObject, + // Recreation and then at least one update. + matchEvents(existingObject, "^add,(update,)+$"), + ) + } + + // Use the label as selector in List and DeleteCollection calls. + ginkgo.By(fmt.Sprintf("Listing %s collection with label selector %s", gvr, listOptions.LabelSelector)) + items, err := client.List(ctx, listOptions) + framework.ExpectNoError(err, "list %s", resourceType) + gomega.Expect(items.Items).Should(gomega.HaveLen(1), "Should have listed exactly the test resource.") + verify("list", desiredUpdatedObject, &items.Items[0], + // Optional updates by cluster components. + matchEvents(existingObject, "^(update,)*$"), + ) + + if tc.IsNamespaced() { + ginkgo.By(fmt.Sprintf("Listing %s without namespace and with label selector %s", gvr, listOptions.LabelSelector)) + items, err := resourceClient.List(ctx, listOptions) + framework.ExpectNoError(err, "list %s in all namespaces", resourceType) + gomega.Expect(items.Items).Should(gomega.HaveLen(1), "Should have listed exactly the test resource in all namespaces.") + verify("list all namespaces", desiredUpdatedObject, &items.Items[0], + // Optional updates by cluster components. + matchEvents(existingObject, "^(update,)*$"), + ) + } + + ginkgo.By(fmt.Sprintf("Deleting %s collection with label selector %s", gvr, listOptions.LabelSelector)) + err = client.DeleteCollection(ctx, metav1.DeleteOptions{}, listOptions) + framework.ExpectNoError(err, "delete collection of %s", resourceType) + ensureNotFound(ctx, getResource) + verify("list", nil, nil, + // Optional updates by cluster components, then deletion. + matchEvents(existingObject, "^(update,)*delete,$"), + ) +} + +// ensureNotFound ensures that the error returned by the get function is NotFound. +// This can be called after deleting an object to ensure that it really got removed +// and not just marked for deletion with a DeletionTimestamp. Deletion is not +// necessarily instantaneous e.g. because some cluster component might add its +// own finalizer. +func ensureNotFound(ctx context.Context, get func(context.Context) (*unstructured.Unstructured, error)) { + ginkgo.GinkgoHelper() + ginkgo.By("Checking for existence") + gomega.Eventually(ctx, func(ctx context.Context) error { + obj, err := framework.HandleRetry(get)(ctx) + switch { + case apierrors.IsNotFound(err): + return nil + case err != nil: + return fmt.Errorf("unexpected error after GET: %w", err) + default: + return fmt.Errorf("resource not removed yet:\n%s", format.Object(obj, 1)) + } + }).WithTimeout(30 * time.Second /* From prior conformance tests, e.g. https://github.com/kubernetes/kubernetes/blame/be361a18dda0f2fab1f5e25f8067a9ed43fc3b89/test/e2e/storage/storageclass.go#L152 */). + Should(gomega.Succeed()) +} diff --git a/test/e2e/framework/conformance/failures.go b/test/e2e/framework/conformance/failures.go new file mode 100644 index 00000000000..4c0d0558a0c --- /dev/null +++ b/test/e2e/framework/conformance/failures.go @@ -0,0 +1,60 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package architecture + +import ( + "fmt" + "strings" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + gtypes "github.com/onsi/gomega/types" +) + +type gomegaFailures struct { + failures []string +} + +var _ gtypes.GomegaTestingT = &gomegaFailures{} + +// Helper implements [gtyppes.GomegaTestingT]. +func (g *gomegaFailures) Helper() {} + +// Fatalf implements [gtypes.GomegaTestingT]. +func (g *gomegaFailures) Fatalf(format string, args ...any) { + g.Add(fmt.Sprintf(format, args...)) +} + +// Adds one failure. +func (g *gomegaFailures) Add(failure string) { + if !strings.HasSuffix(failure, "\n") { + failure += "\n" + } + g.failures = append(g.failures, failure) +} + +// Check fails via [ginkgo.Fail] if there were any failures. +func (g *gomegaFailures) Check() { + if len(g.failures) > 0 { + ginkgo.GinkgoHelper() + ginkgo.Fail(strings.Join(g.failures, "\n\n")) + } +} + +func (g *gomegaFailures) G() *gomega.WithT { + return gomega.NewWithT(g) +} diff --git a/test/e2e/framework/conformance/informerevents.go b/test/e2e/framework/conformance/informerevents.go new file mode 100644 index 00000000000..e5bc886d164 --- /dev/null +++ b/test/e2e/framework/conformance/informerevents.go @@ -0,0 +1,167 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package architecture + +import ( + "fmt" + "slices" + "strings" + "sync" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" +) + +// eventRecorder implements [cache.ResourceEventHandler] by recording all events. +// It is thread-safe. +type eventRecorder struct { + mutex sync.Mutex + events []event +} + +// event describes one add/update/delete event. +// They can be distinguished based on which object(s) are set. +// Delete events may contain a tombstone instead of the actual +// deleted object. +type event struct { + oldObj, newObj any + isInitialList bool +} + +func (e event) Type() string { + switch { + case e.oldObj == nil: + return "add" + case e.newObj == nil: + return "delete" + case e.oldObj != nil && e.newObj != nil: + return "update" + default: + return "null" + } +} + +// ID returns "[namespace/]name, uid" for the object +// described in the event. If for whatever reason +// that is different for the old and new event (an error!), +// it returns both strings separated by semicolon. +// If meta data access is not possible, the error string is returned. +// +// This is meant to be used with gomega.HaveEach and a matcher +// which checks for the expected ID. +func (e event) ID() string { + oldID := id(e.oldObj) + newID := id(e.newObj) + if oldID == newID { + return newID + } + if oldID != "" && newID != "" { + return oldID + "; " + newID + } + if newID != "" { + return newID + } + return oldID +} + +func id(obj any) string { + if obj == nil { + return "" + } + if tomb, ok := obj.(cache.DeletedFinalStateUnknown); ok { + obj = tomb.Obj + } + metaData, err := meta.Accessor(obj) + if err != nil { + return err.Error() + } + return fmt.Sprintf("%s, %s", klog.KObj(metaData).String(), metaData.GetUID()) +} + +var _ cache.ResourceEventHandler = &eventRecorder{} + +func (er *eventRecorder) OnAdd(obj any, isInitialList bool) { + er.mutex.Lock() + defer er.mutex.Unlock() + + er.events = append(er.events, event{ + newObj: obj, + isInitialList: isInitialList, + }) +} + +func (er *eventRecorder) OnUpdate(oldObj, newObj any) { + er.mutex.Lock() + defer er.mutex.Unlock() + + er.events = append(er.events, event{ + oldObj: oldObj, + newObj: newObj, + }) +} + +func (er *eventRecorder) OnDelete(obj any) { + er.mutex.Lock() + defer er.mutex.Unlock() + + er.events = append(er.events, event{ + oldObj: obj, + }) +} + +// list returns a shallow copy of the current list of events. +func (er *eventRecorder) list() eventList { + er.mutex.Lock() + defer er.mutex.Unlock() + + return eventList{Events: slices.Clone(er.events)} +} + +// reset clears the current list of events. +// Should only be called during idle periods. +func (er *eventRecorder) reset() eventList { + er.mutex.Lock() + defer er.mutex.Unlock() + + events := eventList{Events: er.events} + er.events = nil + return events +} + +// eventList adds pretty-printing to a slice of events. +type eventList struct { + Events []event +} + +// Types returns a comma-separated list of the type of each event. +// For the sake of simplicity the last entry also ends with a comma. +// +// This can be used in Gomega assertions like this: +// +// gomega.Expect(events).To(gomega.HaveField("Types()", gomega.MatchRegexp("^add,(update,)*$")) +// gomega.Expect(events).To(gomega.HaveField("Types()", gomega.MatchRegexp("^(update,)*$")) +func (el eventList) Types() string { + var buffer strings.Builder + + for _, e := range el.Events { + buffer.WriteString(e.Type()) + buffer.WriteRune(',') + } + + return buffer.String() +} diff --git a/test/e2e/framework/conformance/objectmatcher.go b/test/e2e/framework/conformance/objectmatcher.go new file mode 100644 index 00000000000..137305cd10c --- /dev/null +++ b/test/e2e/framework/conformance/objectmatcher.go @@ -0,0 +1,114 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package architecture + +import ( + "fmt" + "strings" + + "github.com/onsi/gomega/format" + gtypes "github.com/onsi/gomega/types" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog/v2" +) + +// matchObjectList is a custom matcher for the result of a generic lister List result +// ([]runtime.Object where each entry is *unstructured.Unstructed). The list is +// expected to have exactly one element or none. +type matchObjectList struct { + expectedObject *unstructured.Unstructured + verifyContent bool + lastDiff string +} + +var _ gtypes.GomegaMatcher = &matchObjectList{} + +func (m *matchObjectList) Match(actual any) (bool, error) { + // Reset state. + m.lastDiff = "" + + actualObjects, ok := actual.([]runtime.Object) + if !ok { + return false, fmt.Errorf("must be passed a []runtime.Object, got %T", actual) + } + + if m.expectedObject == nil { + // Must be empty, + return len(actualObjects) == 0, nil + } + + // Must have exactly the expected object. + if len(actualObjects) != 1 { + return false, nil + } + actualObject, ok := actualObjects[0].(*unstructured.Unstructured) + if !ok { + // Shouldn't happen. + return false, fmt.Errorf("expected *unstructured.Unstructured, got %T", actualObjects[0]) + } + + if m.verifyContent { + // Remember diff for failure message. + m.lastDiff = compareObjects(m.expectedObject, actualObject) + if m.lastDiff != "" { + return false, nil + } + return true, nil + } + return m.expectedObject.GetName() == actualObject.GetName() && m.expectedObject.GetNamespace() == actualObject.GetNamespace(), nil +} + +func (m *matchObjectList) FailureMessage(actual any) string { + return m.message(actual, "to") +} + +func (m *matchObjectList) NegatedFailureMessage(actual any) string { + return m.message(actual, "not to") +} + +func (m *matchObjectList) message(actual any, to string) string { + // Gomega renders []runtime.Object as nested maps. + // YAML is more readable. + var buffer strings.Builder + buffer.WriteString("Expected\n") + if actualObjects, ok := actual.([]runtime.Object); ok { + buffer.WriteString(fmt.Sprintf(" %T len:%d:\n", actualObjects, len(actualObjects))) + for _, object := range actualObjects { + buffer.WriteString(" ---\n") + if o, ok := object.(*unstructured.Unstructured); ok { + buffer.WriteString(format.Object(o, 2)) + } else { + buffer.WriteString(format.Object(object, 2)) + } + } + } else { + buffer.WriteString(format.Object(actual, 1)) + } + buffer.WriteString(fmt.Sprintf("\n%s contain exactly the following element:\n", to)) + if m.verifyContent { + buffer.WriteString(format.Object(m.expectedObject, 1)) + } else { + buffer.WriteString(" " + klog.KObj(m.expectedObject).String()) + } + if m.lastDiff != "" { + buffer.WriteString("\nDiff of checked fields (- expected, + actual):\n") + buffer.WriteString(m.lastDiff) + } + return buffer.String() +} diff --git a/test/e2e/framework/gomega.go b/test/e2e/framework/gomega.go new file mode 100644 index 00000000000..a2b885da32b --- /dev/null +++ b/test/e2e/framework/gomega.go @@ -0,0 +1,65 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package framework contains provider-independent helper code for +// building and running E2E tests with Ginkgo. The actual Ginkgo test +// suites gets assembled by combining this framework, the optional +// provider support code and specific tests via a separate .go file +// like Kubernetes' test/e2e.go. +package framework + +import ( + "fmt" + "strings" + + "github.com/onsi/gomega/format" + gtypes "github.com/onsi/gomega/types" +) + +// GomegaObject returns a matcher which appends a full dump of the actual value +// to the failure of the matcher that it wraps. This is useful e.g. for +// gomega.HaveField which otherwise only generates a message containing +// the field that it is checking, but not the object in which that field occurs. +func GomegaObject(shouldMatch gtypes.GomegaMatcher) gtypes.GomegaMatcher { + return &gomegaObjectMatcher{ + shouldMatch: shouldMatch, + } +} + +type gomegaObjectMatcher struct { + shouldMatch gtypes.GomegaMatcher +} + +func (m *gomegaObjectMatcher) Match(actual interface{}) (success bool, err error) { + return m.shouldMatch.Match(actual) +} + +func (m *gomegaObjectMatcher) FailureMessage(actual interface{}) (message string) { + return m.withDump(actual, m.shouldMatch.FailureMessage(actual)) +} + +func (m *gomegaObjectMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return m.withDump(actual, m.shouldMatch.NegatedFailureMessage(actual)) +} + +func (m *gomegaObjectMatcher) withDump(actual any, message string) string { + dump := format.Object(actual, 1) + if !strings.HasSuffix(message, "\n") { + message += "\n" + } + message += fmt.Sprintf("\nFull object:\n%s", dump) + return message +}