From f149d6d8f95c335bd830a2029dab0b2fc02ef791 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 8 Mar 2024 12:51:28 +0100 Subject: [PATCH] dra e2e: watch claims and validate them Logging claims helps with debugging test failures. Checking the finalizer catches unexpected behavior. --- test/e2e/dra/deploy.go | 50 ++++++++++++++++++++++++++++++++++++++++++ test/e2e/dra/dra.go | 19 +++++++++++++++- 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/test/e2e/dra/deploy.go b/test/e2e/dra/deploy.go index 8ad46d83932..15229198791 100644 --- a/test/e2e/dra/deploy.go +++ b/test/e2e/dra/deploy.go @@ -28,8 +28,10 @@ import ( "sync" "time" + "github.com/google/go-cmp/cmp" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" + "github.com/onsi/gomega/format" "google.golang.org/grpc" appsv1 "k8s.io/api/apps/v1" @@ -39,6 +41,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" + resourceapiinformer "k8s.io/client-go/informers/resource/v1alpha2" + "k8s.io/client-go/tools/cache" "k8s.io/dynamic-resource-allocation/kubeletplugin" "k8s.io/klog/v2" "k8s.io/kubernetes/test/e2e/dra/test-driver/app" @@ -84,10 +88,56 @@ func NewNodes(f *framework.Framework, minNodes, maxNodes int) *Nodes { nodes.NodeNames = append(nodes.NodeNames, node.Name) } framework.Logf("testing on nodes %v", nodes.NodeNames) + + // Watch claims in the namespace. This is useful for monitoring a test + // and enables additional sanity checks. + claimInformer := resourceapiinformer.NewResourceClaimInformer(f.ClientSet, f.Namespace.Name, 100*time.Hour /* resync */, nil) + cancelCtx, cancel := context.WithCancelCause(context.Background()) + var wg sync.WaitGroup + ginkgo.DeferCleanup(func() { + cancel(errors.New("test has completed")) + wg.Wait() + }) + _, err = claimInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj any) { + defer ginkgo.GinkgoRecover() + claim := obj.(*resourcev1alpha2.ResourceClaim) + framework.Logf("New claim:\n%s", format.Object(claim, 1)) + validateClaim(claim) + }, + UpdateFunc: func(oldObj, newObj any) { + defer ginkgo.GinkgoRecover() + oldClaim := oldObj.(*resourcev1alpha2.ResourceClaim) + newClaim := newObj.(*resourcev1alpha2.ResourceClaim) + framework.Logf("Updated claim:\n%s\nDiff:\n%s", format.Object(newClaim, 1), cmp.Diff(oldClaim, newClaim)) + validateClaim(newClaim) + }, + DeleteFunc: func(obj any) { + defer ginkgo.GinkgoRecover() + claim := obj.(*resourcev1alpha2.ResourceClaim) + framework.Logf("Deleted claim:\n%s", format.Object(claim, 1)) + }, + }) + framework.ExpectNoError(err, "AddEventHandler") + wg.Add(1) + go func() { + defer wg.Done() + claimInformer.Run(cancelCtx.Done()) + }() }) return nodes } +func validateClaim(claim *resourcev1alpha2.ResourceClaim) { + // The apiserver doesn't enforce that a claim always has a finalizer + // while being allocated. This is a convention that whoever allocates a + // claim has to follow to prevent using a claim that is at risk of + // being deleted. + if claim.Status.Allocation != nil && len(claim.Finalizers) == 0 { + framework.Failf("Invalid claim: allocated without any finalizer:\n%s", format.Object(claim, 1)) + } +} + // NewDriver sets up controller (as client of the cluster) and // kubelet plugin (via proxy) before the test runs. It cleans // up after the test. diff --git a/test/e2e/dra/dra.go b/test/e2e/dra/dra.go index 322153536b8..58522795dd0 100644 --- a/test/e2e/dra/dra.go +++ b/test/e2e/dra/dra.go @@ -128,8 +128,25 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, // arbitrary types we can simply fake somthing here. claim := b.externalClaim(resourcev1alpha2.AllocationModeWaitForFirstConsumer) b.create(ctx, claim) + claim, err := f.ClientSet.ResourceV1alpha2().ResourceClaims(f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{}) framework.ExpectNoError(err, "get claim") + + claim.Finalizers = append(claim.Finalizers, "e2e.test/delete-protection") + claim, err = f.ClientSet.ResourceV1alpha2().ResourceClaims(f.Namespace.Name).Update(ctx, claim, metav1.UpdateOptions{}) + framework.ExpectNoError(err, "add claim finalizer") + + ginkgo.DeferCleanup(func(ctx context.Context) { + claim.Status.Allocation = nil + claim.Status.ReservedFor = nil + claim, err = f.ClientSet.ResourceV1alpha2().ResourceClaims(f.Namespace.Name).UpdateStatus(ctx, claim, metav1.UpdateOptions{}) + framework.ExpectNoError(err, "update claim") + + claim.Finalizers = nil + _, err = f.ClientSet.ResourceV1alpha2().ResourceClaims(f.Namespace.Name).Update(ctx, claim, metav1.UpdateOptions{}) + framework.ExpectNoError(err, "remove claim finalizer") + }) + claim.Status.Allocation = &resourcev1alpha2.AllocationResult{} claim.Status.DriverName = driver.Name claim.Status.ReservedFor = append(claim.Status.ReservedFor, resourcev1alpha2.ResourceClaimConsumerReference{ @@ -138,7 +155,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, Name: "thing", UID: "12345", }) - _, err = f.ClientSet.ResourceV1alpha2().ResourceClaims(f.Namespace.Name).UpdateStatus(ctx, claim, metav1.UpdateOptions{}) + claim, err = f.ClientSet.ResourceV1alpha2().ResourceClaims(f.Namespace.Name).UpdateStatus(ctx, claim, metav1.UpdateOptions{}) framework.ExpectNoError(err, "update claim") pod := b.podExternal()