From 8fed05c5b78ddc232819820c616d10f51d92b030 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 12 Sep 2025 18:43:35 +0200 Subject: [PATCH] DRA E2E node: fix test cleanup dd9917ddce82f4b2975e2d9423a14507f88610cb fixed one test which did not wait for pods to be deleted and then, depending on the timing, left ResourceClaims prepared because the driver stopped before kubelet could call NodeUnprepareResources. But this is a more systematic issue also with other tests, so now the any test which starts a DRA plugin automatically uses the same common cleanup code: - delete pods in the test names - wait for the driver to not have any active ResourceClaims - stop the driver --- test/e2e_node/dra_test.go | 136 ++++++++++++++++++++++---------------- 1 file changed, 79 insertions(+), 57 deletions(-) diff --git a/test/e2e_node/dra_test.go b/test/e2e_node/dra_test.go index 4a2c065b612..9234558ef31 100644 --- a/test/e2e_node/dra_test.go +++ b/test/e2e_node/dra_test.go @@ -106,7 +106,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami f.Context("Resource Kubelet Plugin", f.WithSerial(), func() { ginkgo.It("must register after Kubelet restart", func(ctx context.Context) { - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) oldCalls := kubeletPlugin.GetGRPCCalls() getNewCalls := func() []testdriver.GRPCCall { @@ -122,11 +122,11 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami }) ginkgo.It("must register after plugin restart", func(ctx context.Context) { - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) ginkgo.By("restart Kubelet Plugin") kubeletPlugin.Stop() - kubeletPlugin = newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin = newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) ginkgo.By("wait for Kubelet plugin re-registration") gomega.Eventually(kubeletPlugin.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdrivergomega.BeRegistered) @@ -135,7 +135,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami // Test that the kubelet plugin manager retries plugin registration // when the GetInfo call fails, and succeeds once the call passes. ginkgo.It("must recover and register after registration failure", func(ctx context.Context) { - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) ginkgo.By("set GetInfo failure mode") kubeletPlugin.SetGetInfoError(fmt.Errorf("simulated GetInfo failure")) @@ -156,7 +156,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami }) ginkgo.It("must process pod created when kubelet is not running", func(ctx context.Context) { - newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) // Stop Kubelet ginkgo.By("stop kubelet") @@ -175,7 +175,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami }) ginkgo.It("must keep pod in pending state if NodePrepareResources times out", func(ctx context.Context) { - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) unblockNodePrepareResources := kubeletPlugin.BlockNodePrepareResources() defer unblockNodePrepareResources() @@ -195,7 +195,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami }) ginkgo.It("must run pod if NodePrepareResources fails and then succeeds", func(ctx context.Context) { - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) unsetNodePrepareResourcesFailureMode := kubeletPlugin.SetNodePrepareResourcesFailureMode() pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", true, []string{driverName}) @@ -220,7 +220,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami }) ginkgo.It("must run pod if NodeUnprepareResources fails and then succeeds", func(ctx context.Context) { - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) unsetNodeUnprepareResourcesFailureMode := kubeletPlugin.SetNodeUnprepareResourcesFailureMode() pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", true, []string{driverName}) @@ -242,7 +242,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami }) ginkgo.It("must retry NodePrepareResources after Kubelet restart", func(ctx context.Context) { - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) unsetNodePrepareResourcesFailureMode := kubeletPlugin.SetNodePrepareResourcesFailureMode() pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", true, []string{driverName}) @@ -273,7 +273,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami }) ginkgo.It("must retry NodeUnprepareResources after Kubelet restart", func(ctx context.Context) { - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) unsetNodeUnprepareResourcesFailureMode := kubeletPlugin.SetNodeUnprepareResourcesFailureMode() pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", true, []string{driverName}) @@ -300,7 +300,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami }) ginkgo.It("must call NodeUnprepareResources for deleted pod", func(ctx context.Context) { - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) unsetNodeUnprepareResourcesFailureMode := kubeletPlugin.SetNodeUnprepareResourcesFailureMode() pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", false, []string{driverName}) @@ -324,7 +324,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami }) ginkgo.It("must call NodeUnprepareResources for deleted pod after Kubelet restart", func(ctx context.Context) { - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) unsetNodeUnprepareResourcesFailureMode := kubeletPlugin.SetNodeUnprepareResourcesFailureMode() pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", false, []string{driverName}) @@ -355,7 +355,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami }) ginkgo.It("must not call NodePrepareResources for deleted pod after Kubelet restart", func(ctx context.Context) { - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) unblockNodePrepareResources := kubeletPlugin.BlockNodePrepareResources() pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", false, []string{driverName}) @@ -394,7 +394,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami gomega.Eventually(registrar.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdrivergomega.BeRegistered) ginkgo.By("start DRA plugin service") - draService := newDRAService(ctx, f.ClientSet, nodeName, driverName, datadir, opts...) + draService := newDRAService(ctx, f.ClientSet, f.Namespace.Name, nodeName, driverName, datadir, opts...) pod := createTestObjects(ctx, f.ClientSet, nodeName, f.Namespace.Name, "draclass", "external-claim", "drapod", false, []string{driverName}) @@ -426,7 +426,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami gomega.Eventually(registrar.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdrivergomega.BeRegistered) ginkgo.By("start DRA plugin service") - draService := newDRAService(ctx, f.ClientSet, nodeName, driverName, datadir, opts...) + draService := newDRAService(ctx, f.ClientSet, f.Namespace.Name, nodeName, driverName, datadir, opts...) pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drasleeppod" /* enables sleeping */, false /* pod is deleted below */, []string{driverName}) @@ -444,12 +444,14 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami gomega.Eventually(ctx, listResources(f.ClientSet)).Should(gomega.BeEmpty(), "ResourceSlices without plugin") ginkgo.By("restarting plugin") - draService = newDRAService(ctx, f.ClientSet, nodeName, driverName, datadir, opts...) + draService = newDRAService(ctx, f.ClientSet, f.Namespace.Name, nodeName, driverName, datadir, opts...) - ginkgo.By("stopping pod") + ginkgo.By("stopping pod and waiting for it to get removed, indicating that NodeUnprepareResources must have been called") err = f.ClientSet.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{}) - framework.ExpectNoError(err) + framework.ExpectNoError(err, "delete pod") gomega.Eventually(draService.GetGRPCCalls).WithTimeout(retryTestTimeout).Should(testdrivergomega.NodeUnprepareResourcesSucceeded) + err = e2epod.WaitForPodNotFoundInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace, f.Timeouts.PodDelete) + framework.ExpectNoError(err, "pod + claim shutdown") } ginkgo.DescribeTable("must be functional after service reconnect", functionalAfterServiceReconnect, @@ -464,7 +466,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami failOnClosedListener := func( ctx context.Context, - service func(ctx context.Context, clientSet kubernetes.Interface, nodeName, driverName, datadir string, opts ...any) *testdriver.ExamplePlugin, + service func(ctx context.Context, clientSet kubernetes.Interface, testNamespace, nodeName, driverName, datadir string, opts ...any) *testdriver.ExamplePlugin, listenerOptionFun func(listen func(ctx context.Context, path string) (net.Listener, error)) kubeletplugin.Option, ) { ginkgo.By("create a custom listener") @@ -483,6 +485,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami service( ctx, f.ClientSet, + f.Namespace.Name, getNodeName(ctx, f), driverName, "", @@ -500,7 +503,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami // The wrappedNewRegistrar function is used to create a new registrar // with the same signature as the newDRAService function, so that it can be // used in the DescribeTable. - wrappedNewRegistrar := func(ctx context.Context, clientSet kubernetes.Interface, nodeName, driverName, datadir string, opts ...any) *testdriver.ExamplePlugin { + wrappedNewRegistrar := func(ctx context.Context, clientSet kubernetes.Interface, testNamespace, nodeName, driverName, datadir string, opts ...any) *testdriver.ExamplePlugin { return newRegistrar(ctx, clientSet, nodeName, driverName, opts...) } ginkgo.DescribeTable("must report gRPC serving error", @@ -513,8 +516,8 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami f.Context("Two resource Kubelet Plugins", f.WithSerial(), func() { // start creates plugins which will get stopped when the context gets canceled. start := func(ctx context.Context) (*testdriver.ExamplePlugin, *testdriver.ExamplePlugin) { - kubeletPlugin1 := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), kubeletPlugin1Name) - kubeletPlugin2 := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), kubeletPlugin2Name) + kubeletPlugin1 := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), kubeletPlugin1Name) + kubeletPlugin2 := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), kubeletPlugin2Name) ginkgo.By("wait for Kubelet plugin registration") gomega.Eventually(kubeletPlugin1.GetGRPCCalls()).WithTimeout(pluginRegistrationTimeout).Should(testdrivergomega.BeRegistered) @@ -551,13 +554,8 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami framework.ExpectNoError(err) gomega.Expect(kubeletPlugin1.GetGRPCCalls()).Should(testdrivergomega.NodePrepareResourcesSucceeded, "Plugin 1 should have prepared resources.") gomega.Expect(kubeletPlugin2.GetGRPCCalls()).Should(testdrivergomega.NodePrepareResourcesSucceeded, "Plugin 2 should have prepared resources.") - driverName := func(element any) string { - el := element.(*testutil.Sample) - return string(el.Metric[testutil.LabelName("driver_name")]) - } - gomega.Expect(getKubeletMetrics(ctx)).Should(gstruct.MatchKeys(gstruct.IgnoreExtras, gstruct.Keys{ - "dra_resource_claims_in_use": gstruct.MatchAllElements(driverName, gstruct.Elements{ + "dra_resource_claims_in_use": gstruct.MatchAllElements(driverNameLabelKey, gstruct.Elements{ "": timelessSample(1), kubeletPlugin1Name: timelessSample(1), kubeletPlugin2Name: timelessSample(1), @@ -572,10 +570,10 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami gomega.Expect(kubeletPlugin1.GetGRPCCalls()).Should(testdrivergomega.NodeUnprepareResourcesSucceeded, "Plugin 2 should have unprepared resources.") gomega.Expect(kubeletPlugin2.GetGRPCCalls()).Should(testdrivergomega.NodeUnprepareResourcesSucceeded, "Plugin 2 should have unprepared resources.") gomega.Expect(getKubeletMetrics(ctx)).Should(gstruct.MatchKeys(gstruct.IgnoreExtras, gstruct.Keys{ - "dra_resource_claims_in_use": gstruct.MatchAllElements(driverName, gstruct.Elements{ + "dra_resource_claims_in_use": gstruct.MatchAllElements(driverNameLabelKey, gstruct.Elements{ "": timelessSample(0), }), - }), "metrics while pod is running") + }), "metrics after deleting pod") }) ginkgo.It("must run pod if NodePrepareResources fails for one plugin and then succeeds", func(ctx context.Context) { @@ -714,7 +712,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami matchNode := gomega.ConsistOf(matchResourcesByNodeName(nodeName)) ginkgo.By("start plugin and wait for ResourceSlice") - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) gomega.Eventually(ctx, listResources(f.ClientSet)).Should(matchNode, "ResourceSlice from kubelet plugin") gomega.Consistently(ctx, listResources(f.ClientSet)).WithTimeout(5*time.Second).Should(matchNode, "ResourceSlice from kubelet plugin") @@ -734,7 +732,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami gomega.Eventually(registrar.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdrivergomega.BeRegistered) ginkgo.By("start DRA plugin service") - kubeletPlugin := newDRAService(ctx, f.ClientSet, nodeName, driverName, datadir, opts...) + kubeletPlugin := newDRAService(ctx, f.ClientSet, f.Namespace.Name, nodeName, driverName, datadir, opts...) ginkgo.By("wait for ResourceSlice to be created by plugin") matchNode := gomega.ConsistOf(matchResourcesByNodeName(nodeName)) @@ -786,7 +784,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami gomega.Eventually(registrar.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdrivergomega.BeRegistered) ginkgo.By("start DRA plugin service") - kubeletPlugin := newDRAService(ctx, f.ClientSet, nodeName, driverName, datadir, opts...) + kubeletPlugin := newDRAService(ctx, f.ClientSet, f.Namespace.Name, nodeName, driverName, datadir, opts...) ginkgo.By("wait for ResourceSlice to be created by plugin") matchNode := gomega.ConsistOf(matchResourcesByNodeName(nodeName)) @@ -804,7 +802,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami time.Sleep(5 * time.Second) ginkgo.By("restarting plugin") - newDRAService(ctx, f.ClientSet, nodeName, driverName, datadir, opts...) + newDRAService(ctx, f.ClientSet, f.Namespace.Name, nodeName, driverName, datadir, opts...) ginkgo.By("ensuring unchanged ResourceSlices") gomega.Consistently(ctx, listResources(f.ClientSet)).WithTimeout(time.Minute).Should(gomega.Equal(slices), "ResourceSlices") @@ -827,7 +825,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami // reported by a DRA plugin are correctly reflected in the Pod's status. ginkgo.It("should reflect device health changes in the Pod's status", func(ctx context.Context) { ginkgo.By("Starting the test driver with channel-based control") - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) className := "health-test-class" claimName := "health-test-claim" @@ -881,7 +879,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami // stops and recovers to "Healthy" upon plugin restart. ginkgo.It("should update health to Unknown when plugin stops and recover upon restart", func(ctx context.Context) { ginkgo.By("Starting the test driver") - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) className := "unknown-test-class" claimName := "unknown-test-claim" @@ -914,7 +912,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami ginkgo.By("Restarting the DRA plugin to simulate recovery") // Re-initialize the plugin, which will re-register with the Kubelet. - kubeletPlugin = newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin = newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) ginkgo.By("Forcing a 'Healthy' status update after restart") kubeletPlugin.HealthControlChan <- testdriver.DeviceHealthUpdate{ @@ -947,7 +945,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami ginkgo.It("should not add health status to Pod when feature gate is disabled", func(ctx context.Context) { ginkgo.By("Starting a test driver") - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName, withHealthService(false)) + newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName, withHealthService(false)) className := "gate-disabled-class" claimName := "gate-disabled-claim" @@ -976,10 +974,6 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami } return fmt.Errorf("could not find container 'testcontainer' in pod status") }).WithContext(ctx).WithTimeout(30*time.Second).WithPolling(2*time.Second).Should(gomega.Succeed(), "The allocatedResourcesStatus field should be absent when the feature gate is disabled") - // Clean up and wait for the system to settle before the test ends - ginkgo.By("Cleaning up pod and waiting for resources to be unprepared") - e2epod.DeletePodOrFail(ctx, f.ClientSet, pod.Namespace, pod.Name) - gomega.Eventually(kubeletPlugin.GetGRPCCalls).WithTimeout(retryTestTimeout).Should(testdrivergomega.NodeUnprepareResourcesSucceeded) }) }) }) @@ -995,7 +989,7 @@ func withHealthService(enabled bool) pluginOption { } // Run Kubelet plugin and wait until it's registered -func newKubeletPlugin(ctx context.Context, clientSet kubernetes.Interface, nodeName, driverName string, options ...pluginOption) *testdriver.ExamplePlugin { +func newKubeletPlugin(ctx context.Context, clientSet kubernetes.Interface, testNamespace string, nodeName, driverName string, options ...pluginOption) *testdriver.ExamplePlugin { ginkgo.By("start Kubelet plugin") logger := klog.LoggerWithValues(klog.LoggerWithName(klog.Background(), "DRA kubelet plugin "+driverName), "node", nodeName) ctx = klog.NewContext(ctx, logger) @@ -1017,7 +1011,7 @@ func newKubeletPlugin(ctx context.Context, clientSet kubernetes.Interface, nodeN } plugin, err := testdriver.StartPlugin( - ctx, + context.WithoutCancel(ctx), // Stopping is handled in draServiceCleanup. cdiDir, driverName, clientSet, @@ -1041,14 +1035,9 @@ func newKubeletPlugin(ctx context.Context, clientSet kubernetes.Interface, nodeN ) framework.ExpectNoError(err) - gomega.Eventually(plugin.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdrivergomega.BeRegistered) + ginkgo.DeferCleanup(func(ctx context.Context) { draServiceCleanup(ctx, clientSet, driverName, testNamespace, plugin) }) - ginkgo.DeferCleanup(func(ctx context.Context) { - // kubelet should do this eventually, but better make sure. - // A separate test checks this explicitly. - framework.ExpectNoError(clientSet.ResourceV1().ResourceSlices().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{FieldSelector: resourceapi.ResourceSliceSelectorDriver + "=" + driverName})) - }) - ginkgo.DeferCleanup(plugin.Stop) + gomega.Eventually(plugin.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdrivergomega.BeRegistered) return plugin } @@ -1094,7 +1083,7 @@ func newRegistrar(ctx context.Context, clientSet kubernetes.Interface, nodeName, // // Returns: // - A pointer to the started ExamplePlugin instance. -func newDRAService(ctx context.Context, clientSet kubernetes.Interface, nodeName, driverName, datadir string, opts ...any) *testdriver.ExamplePlugin { +func newDRAService(ctx context.Context, clientSet kubernetes.Interface, testNamespace, nodeName, driverName, datadir string, opts ...any) *testdriver.ExamplePlugin { ginkgo.By("start only Kubelet plugin") logger := klog.LoggerWithValues(klog.LoggerWithName(klog.Background(), "kubelet plugin "+driverName), "node", nodeName) ctx = klog.NewContext(ctx, logger) @@ -1120,7 +1109,7 @@ func newDRAService(ctx context.Context, clientSet kubernetes.Interface, nodeName framework.ExpectNoError(err, "create DRA socket directory") plugin, err := testdriver.StartPlugin( - ctx, + context.WithoutCancel(ctx), // Stopping is handled in draServiceCleanup. cdiDir, driverName, clientSet, @@ -1144,14 +1133,40 @@ func newDRAService(ctx context.Context, clientSet kubernetes.Interface, nodeName ) framework.ExpectNoError(err) - ginkgo.DeferCleanup(func(ctx context.Context) { + ginkgo.DeferCleanup(func(ctx context.Context) { draServiceCleanup(ctx, clientSet, driverName, testNamespace, plugin) }) + + return plugin +} + +func draServiceCleanup(ctx context.Context, clientSet kubernetes.Interface, driverName string, testNamespace string, plugin *testdriver.ExamplePlugin) { + // Always stop at the end, even in the case of failures. + defer func() { + ginkgo.By("Stopping the driver and deleting ResourceSlices...") + plugin.Stop() + // kubelet should do this eventually, but better make sure. // A separate test checks this explicitly. framework.ExpectNoError(clientSet.ResourceV1().ResourceSlices().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{FieldSelector: resourceapi.ResourceSliceSelectorDriver + "=" + driverName})) - }) - ginkgo.DeferCleanup(plugin.Stop) + }() - return plugin + // Some test was leaking an in-use ResourceClaim + // (https://github.com/kubernetes/kubernetes/issues/133304) because it + // didn't delete its pod before the driver shutdown. The namespace + // deletion usually takes care of this, but that happens later and + // without the driver the pods stay pending. Therefore we delete all of + // the pods in the test namespace *before* proceeding with the driver + // shutdown. + ginkgo.By("Deleting pods and waiting for ResourceClaims to be released...") + framework.ExpectNoError(clientSet.CoreV1().Pods(testNamespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{})) + + // We allow for a delay because it takes a while till the kubelet fully deals with pod deletion. + // Typically there is no dra_resource_claims_in_use entry for the driver (covered by gstruct.IgnoreMissing). + // If there is one, it must be zero. + gomega.Eventually(ctx, getKubeletMetrics).Should(gstruct.MatchKeys(gstruct.IgnoreExtras, gstruct.Keys{ + "dra_resource_claims_in_use": gstruct.MatchElements(driverNameLabelKey, gstruct.IgnoreExtras|gstruct.IgnoreMissing, gstruct.Elements{ + driverName: timelessSample(0), + }), + }), "The test should have deleted all pods.\nInstead some claims prepared by the driver %s are still in use at the time when the driver gets uninstalled.", driverName) } // createTestObjects creates objects required by the test @@ -1518,3 +1533,10 @@ func (l *errorOnCloseListener) Close() error { func (*errorOnCloseListener) Addr() net.Addr { return &net.UnixAddr{Name: "errorOnCloseListener", Net: "unix"} } + +// driverNameLabelKey maps metric samples to their driver_name label value. +// Used with gstruct.MatchAllElements. +func driverNameLabelKey(element any) string { + el := element.(*testutil.Sample) + return string(el.Metric[testutil.LabelName("driver_name")]) +}