diff --git a/test/e2e_node/dra_test.go b/test/e2e_node/dra_test.go index 4a2c065b612..9234558ef31 100644 --- a/test/e2e_node/dra_test.go +++ b/test/e2e_node/dra_test.go @@ -106,7 +106,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami f.Context("Resource Kubelet Plugin", f.WithSerial(), func() { ginkgo.It("must register after Kubelet restart", func(ctx context.Context) { - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) oldCalls := kubeletPlugin.GetGRPCCalls() getNewCalls := func() []testdriver.GRPCCall { @@ -122,11 +122,11 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami }) ginkgo.It("must register after plugin restart", func(ctx context.Context) { - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) ginkgo.By("restart Kubelet Plugin") kubeletPlugin.Stop() - kubeletPlugin = newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin = newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) ginkgo.By("wait for Kubelet plugin re-registration") gomega.Eventually(kubeletPlugin.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdrivergomega.BeRegistered) @@ -135,7 +135,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami // Test that the kubelet plugin manager retries plugin registration // when the GetInfo call fails, and succeeds once the call passes. ginkgo.It("must recover and register after registration failure", func(ctx context.Context) { - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) ginkgo.By("set GetInfo failure mode") kubeletPlugin.SetGetInfoError(fmt.Errorf("simulated GetInfo failure")) @@ -156,7 +156,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami }) ginkgo.It("must process pod created when kubelet is not running", func(ctx context.Context) { - newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) // Stop Kubelet ginkgo.By("stop kubelet") @@ -175,7 +175,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami }) ginkgo.It("must keep pod in pending state if NodePrepareResources times out", func(ctx context.Context) { - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) unblockNodePrepareResources := kubeletPlugin.BlockNodePrepareResources() defer unblockNodePrepareResources() @@ -195,7 +195,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami }) ginkgo.It("must run pod if NodePrepareResources fails and then succeeds", func(ctx context.Context) { - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) unsetNodePrepareResourcesFailureMode := kubeletPlugin.SetNodePrepareResourcesFailureMode() pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", true, []string{driverName}) @@ -220,7 +220,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami }) ginkgo.It("must run pod if NodeUnprepareResources fails and then succeeds", func(ctx context.Context) { - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) unsetNodeUnprepareResourcesFailureMode := kubeletPlugin.SetNodeUnprepareResourcesFailureMode() pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", true, []string{driverName}) @@ -242,7 +242,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami }) ginkgo.It("must retry NodePrepareResources after Kubelet restart", func(ctx context.Context) { - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) unsetNodePrepareResourcesFailureMode := kubeletPlugin.SetNodePrepareResourcesFailureMode() pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", true, []string{driverName}) @@ -273,7 +273,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami }) ginkgo.It("must retry NodeUnprepareResources after Kubelet restart", func(ctx context.Context) { - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) unsetNodeUnprepareResourcesFailureMode := kubeletPlugin.SetNodeUnprepareResourcesFailureMode() pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", true, []string{driverName}) @@ -300,7 +300,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami }) ginkgo.It("must call NodeUnprepareResources for deleted pod", func(ctx context.Context) { - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) unsetNodeUnprepareResourcesFailureMode := kubeletPlugin.SetNodeUnprepareResourcesFailureMode() pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", false, []string{driverName}) @@ -324,7 +324,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami }) ginkgo.It("must call NodeUnprepareResources for deleted pod after Kubelet restart", func(ctx context.Context) { - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) unsetNodeUnprepareResourcesFailureMode := kubeletPlugin.SetNodeUnprepareResourcesFailureMode() pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", false, []string{driverName}) @@ -355,7 +355,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami }) ginkgo.It("must not call NodePrepareResources for deleted pod after Kubelet restart", func(ctx context.Context) { - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) unblockNodePrepareResources := kubeletPlugin.BlockNodePrepareResources() pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drapod", false, []string{driverName}) @@ -394,7 +394,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami gomega.Eventually(registrar.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdrivergomega.BeRegistered) ginkgo.By("start DRA plugin service") - draService := newDRAService(ctx, f.ClientSet, nodeName, driverName, datadir, opts...) + draService := newDRAService(ctx, f.ClientSet, f.Namespace.Name, nodeName, driverName, datadir, opts...) pod := createTestObjects(ctx, f.ClientSet, nodeName, f.Namespace.Name, "draclass", "external-claim", "drapod", false, []string{driverName}) @@ -426,7 +426,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami gomega.Eventually(registrar.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdrivergomega.BeRegistered) ginkgo.By("start DRA plugin service") - draService := newDRAService(ctx, f.ClientSet, nodeName, driverName, datadir, opts...) + draService := newDRAService(ctx, f.ClientSet, f.Namespace.Name, nodeName, driverName, datadir, opts...) pod := createTestObjects(ctx, f.ClientSet, getNodeName(ctx, f), f.Namespace.Name, "draclass", "external-claim", "drasleeppod" /* enables sleeping */, false /* pod is deleted below */, []string{driverName}) @@ -444,12 +444,14 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami gomega.Eventually(ctx, listResources(f.ClientSet)).Should(gomega.BeEmpty(), "ResourceSlices without plugin") ginkgo.By("restarting plugin") - draService = newDRAService(ctx, f.ClientSet, nodeName, driverName, datadir, opts...) + draService = newDRAService(ctx, f.ClientSet, f.Namespace.Name, nodeName, driverName, datadir, opts...) - ginkgo.By("stopping pod") + ginkgo.By("stopping pod and waiting for it to get removed, indicating that NodeUnprepareResources must have been called") err = f.ClientSet.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{}) - framework.ExpectNoError(err) + framework.ExpectNoError(err, "delete pod") gomega.Eventually(draService.GetGRPCCalls).WithTimeout(retryTestTimeout).Should(testdrivergomega.NodeUnprepareResourcesSucceeded) + err = e2epod.WaitForPodNotFoundInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace, f.Timeouts.PodDelete) + framework.ExpectNoError(err, "pod + claim shutdown") } ginkgo.DescribeTable("must be functional after service reconnect", functionalAfterServiceReconnect, @@ -464,7 +466,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami failOnClosedListener := func( ctx context.Context, - service func(ctx context.Context, clientSet kubernetes.Interface, nodeName, driverName, datadir string, opts ...any) *testdriver.ExamplePlugin, + service func(ctx context.Context, clientSet kubernetes.Interface, testNamespace, nodeName, driverName, datadir string, opts ...any) *testdriver.ExamplePlugin, listenerOptionFun func(listen func(ctx context.Context, path string) (net.Listener, error)) kubeletplugin.Option, ) { ginkgo.By("create a custom listener") @@ -483,6 +485,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami service( ctx, f.ClientSet, + f.Namespace.Name, getNodeName(ctx, f), driverName, "", @@ -500,7 +503,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami // The wrappedNewRegistrar function is used to create a new registrar // with the same signature as the newDRAService function, so that it can be // used in the DescribeTable. - wrappedNewRegistrar := func(ctx context.Context, clientSet kubernetes.Interface, nodeName, driverName, datadir string, opts ...any) *testdriver.ExamplePlugin { + wrappedNewRegistrar := func(ctx context.Context, clientSet kubernetes.Interface, testNamespace, nodeName, driverName, datadir string, opts ...any) *testdriver.ExamplePlugin { return newRegistrar(ctx, clientSet, nodeName, driverName, opts...) } ginkgo.DescribeTable("must report gRPC serving error", @@ -513,8 +516,8 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami f.Context("Two resource Kubelet Plugins", f.WithSerial(), func() { // start creates plugins which will get stopped when the context gets canceled. start := func(ctx context.Context) (*testdriver.ExamplePlugin, *testdriver.ExamplePlugin) { - kubeletPlugin1 := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), kubeletPlugin1Name) - kubeletPlugin2 := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), kubeletPlugin2Name) + kubeletPlugin1 := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), kubeletPlugin1Name) + kubeletPlugin2 := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), kubeletPlugin2Name) ginkgo.By("wait for Kubelet plugin registration") gomega.Eventually(kubeletPlugin1.GetGRPCCalls()).WithTimeout(pluginRegistrationTimeout).Should(testdrivergomega.BeRegistered) @@ -551,13 +554,8 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami framework.ExpectNoError(err) gomega.Expect(kubeletPlugin1.GetGRPCCalls()).Should(testdrivergomega.NodePrepareResourcesSucceeded, "Plugin 1 should have prepared resources.") gomega.Expect(kubeletPlugin2.GetGRPCCalls()).Should(testdrivergomega.NodePrepareResourcesSucceeded, "Plugin 2 should have prepared resources.") - driverName := func(element any) string { - el := element.(*testutil.Sample) - return string(el.Metric[testutil.LabelName("driver_name")]) - } - gomega.Expect(getKubeletMetrics(ctx)).Should(gstruct.MatchKeys(gstruct.IgnoreExtras, gstruct.Keys{ - "dra_resource_claims_in_use": gstruct.MatchAllElements(driverName, gstruct.Elements{ + "dra_resource_claims_in_use": gstruct.MatchAllElements(driverNameLabelKey, gstruct.Elements{ "": timelessSample(1), kubeletPlugin1Name: timelessSample(1), kubeletPlugin2Name: timelessSample(1), @@ -572,10 +570,10 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami gomega.Expect(kubeletPlugin1.GetGRPCCalls()).Should(testdrivergomega.NodeUnprepareResourcesSucceeded, "Plugin 2 should have unprepared resources.") gomega.Expect(kubeletPlugin2.GetGRPCCalls()).Should(testdrivergomega.NodeUnprepareResourcesSucceeded, "Plugin 2 should have unprepared resources.") gomega.Expect(getKubeletMetrics(ctx)).Should(gstruct.MatchKeys(gstruct.IgnoreExtras, gstruct.Keys{ - "dra_resource_claims_in_use": gstruct.MatchAllElements(driverName, gstruct.Elements{ + "dra_resource_claims_in_use": gstruct.MatchAllElements(driverNameLabelKey, gstruct.Elements{ "": timelessSample(0), }), - }), "metrics while pod is running") + }), "metrics after deleting pod") }) ginkgo.It("must run pod if NodePrepareResources fails for one plugin and then succeeds", func(ctx context.Context) { @@ -714,7 +712,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami matchNode := gomega.ConsistOf(matchResourcesByNodeName(nodeName)) ginkgo.By("start plugin and wait for ResourceSlice") - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) gomega.Eventually(ctx, listResources(f.ClientSet)).Should(matchNode, "ResourceSlice from kubelet plugin") gomega.Consistently(ctx, listResources(f.ClientSet)).WithTimeout(5*time.Second).Should(matchNode, "ResourceSlice from kubelet plugin") @@ -734,7 +732,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami gomega.Eventually(registrar.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdrivergomega.BeRegistered) ginkgo.By("start DRA plugin service") - kubeletPlugin := newDRAService(ctx, f.ClientSet, nodeName, driverName, datadir, opts...) + kubeletPlugin := newDRAService(ctx, f.ClientSet, f.Namespace.Name, nodeName, driverName, datadir, opts...) ginkgo.By("wait for ResourceSlice to be created by plugin") matchNode := gomega.ConsistOf(matchResourcesByNodeName(nodeName)) @@ -786,7 +784,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami gomega.Eventually(registrar.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdrivergomega.BeRegistered) ginkgo.By("start DRA plugin service") - kubeletPlugin := newDRAService(ctx, f.ClientSet, nodeName, driverName, datadir, opts...) + kubeletPlugin := newDRAService(ctx, f.ClientSet, f.Namespace.Name, nodeName, driverName, datadir, opts...) ginkgo.By("wait for ResourceSlice to be created by plugin") matchNode := gomega.ConsistOf(matchResourcesByNodeName(nodeName)) @@ -804,7 +802,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami time.Sleep(5 * time.Second) ginkgo.By("restarting plugin") - newDRAService(ctx, f.ClientSet, nodeName, driverName, datadir, opts...) + newDRAService(ctx, f.ClientSet, f.Namespace.Name, nodeName, driverName, datadir, opts...) ginkgo.By("ensuring unchanged ResourceSlices") gomega.Consistently(ctx, listResources(f.ClientSet)).WithTimeout(time.Minute).Should(gomega.Equal(slices), "ResourceSlices") @@ -827,7 +825,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami // reported by a DRA plugin are correctly reflected in the Pod's status. ginkgo.It("should reflect device health changes in the Pod's status", func(ctx context.Context) { ginkgo.By("Starting the test driver with channel-based control") - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) className := "health-test-class" claimName := "health-test-claim" @@ -881,7 +879,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami // stops and recovers to "Healthy" upon plugin restart. ginkgo.It("should update health to Unknown when plugin stops and recover upon restart", func(ctx context.Context) { ginkgo.By("Starting the test driver") - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) className := "unknown-test-class" claimName := "unknown-test-claim" @@ -914,7 +912,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami ginkgo.By("Restarting the DRA plugin to simulate recovery") // Re-initialize the plugin, which will re-register with the Kubelet. - kubeletPlugin = newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + kubeletPlugin = newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName) ginkgo.By("Forcing a 'Healthy' status update after restart") kubeletPlugin.HealthControlChan <- testdriver.DeviceHealthUpdate{ @@ -947,7 +945,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami ginkgo.It("should not add health status to Pod when feature gate is disabled", func(ctx context.Context) { ginkgo.By("Starting a test driver") - kubeletPlugin := newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName, withHealthService(false)) + newKubeletPlugin(ctx, f.ClientSet, f.Namespace.Name, getNodeName(ctx, f), driverName, withHealthService(false)) className := "gate-disabled-class" claimName := "gate-disabled-claim" @@ -976,10 +974,6 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), feature.Dynami } return fmt.Errorf("could not find container 'testcontainer' in pod status") }).WithContext(ctx).WithTimeout(30*time.Second).WithPolling(2*time.Second).Should(gomega.Succeed(), "The allocatedResourcesStatus field should be absent when the feature gate is disabled") - // Clean up and wait for the system to settle before the test ends - ginkgo.By("Cleaning up pod and waiting for resources to be unprepared") - e2epod.DeletePodOrFail(ctx, f.ClientSet, pod.Namespace, pod.Name) - gomega.Eventually(kubeletPlugin.GetGRPCCalls).WithTimeout(retryTestTimeout).Should(testdrivergomega.NodeUnprepareResourcesSucceeded) }) }) }) @@ -995,7 +989,7 @@ func withHealthService(enabled bool) pluginOption { } // Run Kubelet plugin and wait until it's registered -func newKubeletPlugin(ctx context.Context, clientSet kubernetes.Interface, nodeName, driverName string, options ...pluginOption) *testdriver.ExamplePlugin { +func newKubeletPlugin(ctx context.Context, clientSet kubernetes.Interface, testNamespace string, nodeName, driverName string, options ...pluginOption) *testdriver.ExamplePlugin { ginkgo.By("start Kubelet plugin") logger := klog.LoggerWithValues(klog.LoggerWithName(klog.Background(), "DRA kubelet plugin "+driverName), "node", nodeName) ctx = klog.NewContext(ctx, logger) @@ -1017,7 +1011,7 @@ func newKubeletPlugin(ctx context.Context, clientSet kubernetes.Interface, nodeN } plugin, err := testdriver.StartPlugin( - ctx, + context.WithoutCancel(ctx), // Stopping is handled in draServiceCleanup. cdiDir, driverName, clientSet, @@ -1041,14 +1035,9 @@ func newKubeletPlugin(ctx context.Context, clientSet kubernetes.Interface, nodeN ) framework.ExpectNoError(err) - gomega.Eventually(plugin.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdrivergomega.BeRegistered) + ginkgo.DeferCleanup(func(ctx context.Context) { draServiceCleanup(ctx, clientSet, driverName, testNamespace, plugin) }) - ginkgo.DeferCleanup(func(ctx context.Context) { - // kubelet should do this eventually, but better make sure. - // A separate test checks this explicitly. - framework.ExpectNoError(clientSet.ResourceV1().ResourceSlices().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{FieldSelector: resourceapi.ResourceSliceSelectorDriver + "=" + driverName})) - }) - ginkgo.DeferCleanup(plugin.Stop) + gomega.Eventually(plugin.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdrivergomega.BeRegistered) return plugin } @@ -1094,7 +1083,7 @@ func newRegistrar(ctx context.Context, clientSet kubernetes.Interface, nodeName, // // Returns: // - A pointer to the started ExamplePlugin instance. -func newDRAService(ctx context.Context, clientSet kubernetes.Interface, nodeName, driverName, datadir string, opts ...any) *testdriver.ExamplePlugin { +func newDRAService(ctx context.Context, clientSet kubernetes.Interface, testNamespace, nodeName, driverName, datadir string, opts ...any) *testdriver.ExamplePlugin { ginkgo.By("start only Kubelet plugin") logger := klog.LoggerWithValues(klog.LoggerWithName(klog.Background(), "kubelet plugin "+driverName), "node", nodeName) ctx = klog.NewContext(ctx, logger) @@ -1120,7 +1109,7 @@ func newDRAService(ctx context.Context, clientSet kubernetes.Interface, nodeName framework.ExpectNoError(err, "create DRA socket directory") plugin, err := testdriver.StartPlugin( - ctx, + context.WithoutCancel(ctx), // Stopping is handled in draServiceCleanup. cdiDir, driverName, clientSet, @@ -1144,14 +1133,40 @@ func newDRAService(ctx context.Context, clientSet kubernetes.Interface, nodeName ) framework.ExpectNoError(err) - ginkgo.DeferCleanup(func(ctx context.Context) { + ginkgo.DeferCleanup(func(ctx context.Context) { draServiceCleanup(ctx, clientSet, driverName, testNamespace, plugin) }) + + return plugin +} + +func draServiceCleanup(ctx context.Context, clientSet kubernetes.Interface, driverName string, testNamespace string, plugin *testdriver.ExamplePlugin) { + // Always stop at the end, even in the case of failures. + defer func() { + ginkgo.By("Stopping the driver and deleting ResourceSlices...") + plugin.Stop() + // kubelet should do this eventually, but better make sure. // A separate test checks this explicitly. framework.ExpectNoError(clientSet.ResourceV1().ResourceSlices().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{FieldSelector: resourceapi.ResourceSliceSelectorDriver + "=" + driverName})) - }) - ginkgo.DeferCleanup(plugin.Stop) + }() - return plugin + // Some test was leaking an in-use ResourceClaim + // (https://github.com/kubernetes/kubernetes/issues/133304) because it + // didn't delete its pod before the driver shutdown. The namespace + // deletion usually takes care of this, but that happens later and + // without the driver the pods stay pending. Therefore we delete all of + // the pods in the test namespace *before* proceeding with the driver + // shutdown. + ginkgo.By("Deleting pods and waiting for ResourceClaims to be released...") + framework.ExpectNoError(clientSet.CoreV1().Pods(testNamespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{})) + + // We allow for a delay because it takes a while till the kubelet fully deals with pod deletion. + // Typically there is no dra_resource_claims_in_use entry for the driver (covered by gstruct.IgnoreMissing). + // If there is one, it must be zero. + gomega.Eventually(ctx, getKubeletMetrics).Should(gstruct.MatchKeys(gstruct.IgnoreExtras, gstruct.Keys{ + "dra_resource_claims_in_use": gstruct.MatchElements(driverNameLabelKey, gstruct.IgnoreExtras|gstruct.IgnoreMissing, gstruct.Elements{ + driverName: timelessSample(0), + }), + }), "The test should have deleted all pods.\nInstead some claims prepared by the driver %s are still in use at the time when the driver gets uninstalled.", driverName) } // createTestObjects creates objects required by the test @@ -1518,3 +1533,10 @@ func (l *errorOnCloseListener) Close() error { func (*errorOnCloseListener) Addr() net.Addr { return &net.UnixAddr{Name: "errorOnCloseListener", Net: "unix"} } + +// driverNameLabelKey maps metric samples to their driver_name label value. +// Used with gstruct.MatchAllElements. +func driverNameLabelKey(element any) string { + el := element.(*testutil.Sample) + return string(el.Metric[testutil.LabelName("driver_name")]) +}