dra e2e: watch claims and validate them

Logging claims helps with debugging test failures. Checking the finalizer
catches unexpected behavior.

test/e2e/dra/deploy.go

@@ -28,8 +28,10 @@ import (
 	"sync"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
+	"github.com/onsi/gomega/format"
 	"google.golang.org/grpc"
 
 	appsv1 "k8s.io/api/apps/v1"
@@ -39,6 +41,8 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/selection"
+	resourceapiinformer "k8s.io/client-go/informers/resource/v1alpha2"
+	"k8s.io/client-go/tools/cache"
 	"k8s.io/dynamic-resource-allocation/kubeletplugin"
 	"k8s.io/klog/v2"
 	"k8s.io/kubernetes/test/e2e/dra/test-driver/app"
@@ -84,10 +88,56 @@ func NewNodes(f *framework.Framework, minNodes, maxNodes int) *Nodes {
 			nodes.NodeNames = append(nodes.NodeNames, node.Name)
 		}
 		framework.Logf("testing on nodes %v", nodes.NodeNames)
+
+		// Watch claims in the namespace. This is useful for monitoring a test
+		// and enables additional sanity checks.
+		claimInformer := resourceapiinformer.NewResourceClaimInformer(f.ClientSet, f.Namespace.Name, 100*time.Hour /* resync */, nil)
+		cancelCtx, cancel := context.WithCancelCause(context.Background())
+		var wg sync.WaitGroup
+		ginkgo.DeferCleanup(func() {
+			cancel(errors.New("test has completed"))
+			wg.Wait()
+		})
+		_, err = claimInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+			AddFunc: func(obj any) {
+				defer ginkgo.GinkgoRecover()
+				claim := obj.(*resourcev1alpha2.ResourceClaim)
+				framework.Logf("New claim:\n%s", format.Object(claim, 1))
+				validateClaim(claim)
+			},
+			UpdateFunc: func(oldObj, newObj any) {
+				defer ginkgo.GinkgoRecover()
+				oldClaim := oldObj.(*resourcev1alpha2.ResourceClaim)
+				newClaim := newObj.(*resourcev1alpha2.ResourceClaim)
+				framework.Logf("Updated claim:\n%s\nDiff:\n%s", format.Object(newClaim, 1), cmp.Diff(oldClaim, newClaim))
+				validateClaim(newClaim)
+			},
+			DeleteFunc: func(obj any) {
+				defer ginkgo.GinkgoRecover()
+				claim := obj.(*resourcev1alpha2.ResourceClaim)
+				framework.Logf("Deleted claim:\n%s", format.Object(claim, 1))
+			},
+		})
+		framework.ExpectNoError(err, "AddEventHandler")
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			claimInformer.Run(cancelCtx.Done())
+		}()
 	})
 	return nodes
 }
 
+func validateClaim(claim *resourcev1alpha2.ResourceClaim) {
+	// The apiserver doesn't enforce that a claim always has a finalizer
+	// while being allocated. This is a convention that whoever allocates a
+	// claim has to follow to prevent using a claim that is at risk of
+	// being deleted.
+	if claim.Status.Allocation != nil && len(claim.Finalizers) == 0 {
+		framework.Failf("Invalid claim: allocated without any finalizer:\n%s", format.Object(claim, 1))
+	}
+}
+
 // NewDriver sets up controller (as client of the cluster) and
 // kubelet plugin (via proxy) before the test runs. It cleans
 // up after the test.

test/e2e/dra/dra.go

@@ -128,8 +128,25 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
 				// arbitrary types we can simply fake somthing here.
 				claim := b.externalClaim(resourcev1alpha2.AllocationModeWaitForFirstConsumer)
 				b.create(ctx, claim)
+
 				claim, err := f.ClientSet.ResourceV1alpha2().ResourceClaims(f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{})
 				framework.ExpectNoError(err, "get claim")
+
+				claim.Finalizers = append(claim.Finalizers, "e2e.test/delete-protection")
+				claim, err = f.ClientSet.ResourceV1alpha2().ResourceClaims(f.Namespace.Name).Update(ctx, claim, metav1.UpdateOptions{})
+				framework.ExpectNoError(err, "add claim finalizer")
+
+				ginkgo.DeferCleanup(func(ctx context.Context) {
+					claim.Status.Allocation = nil
+					claim.Status.ReservedFor = nil
+					claim, err = f.ClientSet.ResourceV1alpha2().ResourceClaims(f.Namespace.Name).UpdateStatus(ctx, claim, metav1.UpdateOptions{})
+					framework.ExpectNoError(err, "update claim")
+
+					claim.Finalizers = nil
+					_, err = f.ClientSet.ResourceV1alpha2().ResourceClaims(f.Namespace.Name).Update(ctx, claim, metav1.UpdateOptions{})
+					framework.ExpectNoError(err, "remove claim finalizer")
+				})
+
 				claim.Status.Allocation = &resourcev1alpha2.AllocationResult{}
 				claim.Status.DriverName = driver.Name
 				claim.Status.ReservedFor = append(claim.Status.ReservedFor, resourcev1alpha2.ResourceClaimConsumerReference{
@@ -138,7 +155,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
 					Name:     "thing",
 					UID:      "12345",
 				})
-				_, err = f.ClientSet.ResourceV1alpha2().ResourceClaims(f.Namespace.Name).UpdateStatus(ctx, claim, metav1.UpdateOptions{})
+				claim, err = f.ClientSet.ResourceV1alpha2().ResourceClaims(f.Namespace.Name).UpdateStatus(ctx, claim, metav1.UpdateOptions{})
 				framework.ExpectNoError(err, "update claim")
 
 				pod := b.podExternal()