diff --git a/test/e2e/dra/deploy_device_plugin.go b/test/e2e/dra/deploy_device_plugin.go index e039a24488b..21b8b5a213a 100644 --- a/test/e2e/dra/deploy_device_plugin.go +++ b/test/e2e/dra/deploy_device_plugin.go @@ -17,14 +17,15 @@ limitations under the License. package dra import ( - "context" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kubernetes/test/e2e/framework" e2enode "k8s.io/kubernetes/test/e2e/framework/node" "k8s.io/kubernetes/test/e2e/storage/utils" + "k8s.io/kubernetes/test/utils/ktesting" ) // deployDevicePlugin deploys https://github.com/kubernetes/kubernetes/tree/111a2a0d2dfe13639724506f674bc4f342ccfbab/test/images/sample-device-plugin @@ -33,9 +34,9 @@ import ( // A test using this must run serially because the example plugin uses the hard-coded "example.com/resource" // extended resource name (returned as result) and deploying it twice for the same nodes from different // tests would conflict. -func deployDevicePlugin(ctx context.Context, f *framework.Framework, nodeNames []string) v1.ResourceName { - ginkgo.By("Deploy Device Plugin") - err := utils.CreateFromManifests(ctx, f, f.Namespace, func(item interface{}) error { +func deployDevicePlugin(tCtx ktesting.TContext, f *framework.Framework, nodeNames []string, skipCleanup bool) v1.ResourceName { + ginkgo.By("Deploy Sample Device Plugin DaemonSet") + err := utils.CreateFromManifests(tCtx, f, f.Namespace, func(item interface{}) error { switch item := item.(type) { case *appsv1.DaemonSet: item.Spec.Template.Spec.Affinity = &v1.Affinity{ @@ -58,8 +59,43 @@ func deployDevicePlugin(ctx context.Context, f *framework.Framework, nodeNames [ } return nil }, e2enode.SampleDevicePluginDSYAML) - framework.ExpectNoError(err, "deploy example device plugin DaemonSet") + framework.ExpectNoError(err, "deploy Sample Device Plugin DaemonSet") + + if !skipCleanup { + tCtx.CleanupCtx(func(tCtx ktesting.TContext) { + undeployDevicePlugin(tCtx, f, nodeNames) + }) + } + + // Wait for the Kubelet to update resource Allocatable + // based on the device plugin resources reported by the sample device plugin. + for _, nodeName := range nodeNames { + gomega.Eventually(tCtx, func() int64 { + node, err := f.ClientSet.CoreV1().Nodes().Get(tCtx, nodeName, metav1.GetOptions{}) + framework.ExpectNoError(err) + return e2enode.CountSampleDeviceAllocatable(node) + }).WithTimeout(f.Timeouts.PodStart).Should(gomega.BeNumerically("==", e2enode.SampleDevsAmount), "expected %d %q to be allocatable on node %q", e2enode.SampleDevsAmount, e2enode.SampleDeviceResourceName, nodeName) + } - // Hard-coded in https://github.com/kubernetes/kubernetes/blob/111a2a0d2dfe13639724506f674bc4f342ccfbab/test/images/sample-device-plugin/sampledeviceplugin.go#L34C17-L34C39. return e2enode.SampleDeviceResourceName } + +func undeployDevicePlugin(tCtx ktesting.TContext, f *framework.Framework, nodeNames []string) { + ginkgo.By("Undeploy Sample Device Plugin DaemonSet") + err := f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).DeleteCollection( + tCtx, + metav1.DeleteOptions{}, + metav1.ListOptions{LabelSelector: "k8s-app=" + e2enode.SampleDevicePluginName}, + ) + framework.ExpectNoError(err, "undeploy Sample Device Plugin DaemonSet") + + // Wait for the Kubelet to update resource Allocatable to 0 after the device plugin is undeployed. + // This is needed to ensure that the device plugin resources are not allocatable for pods after the device plugin is undeployed. + for _, nodeName := range nodeNames { + gomega.Eventually(tCtx, func() int64 { + node, err := f.ClientSet.CoreV1().Nodes().Get(tCtx, nodeName, metav1.GetOptions{}) + framework.ExpectNoError(err) + return e2enode.CountSampleDeviceAllocatable(node) + }).WithTimeout(f.Timeouts.PodStart).Should(gomega.BeZero(), "expected 0 %q to be allocatable on node %q after device plugin undeploy", e2enode.SampleDeviceResourceName, nodeName) + } +} diff --git a/test/e2e/dra/dra.go b/test/e2e/dra/dra.go index 9fe9ee3fe86..c27e562e29a 100644 --- a/test/e2e/dra/dra.go +++ b/test/e2e/dra/dra.go @@ -1084,7 +1084,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { // Serial because the example device plugin can only be deployed with one instance at a time. f.It("supports extended resources together with ResourceClaim", f.WithSerial(), func(ctx context.Context) { tCtx := f.TContext(ctx) - extendedResourceName := deployDevicePlugin(ctx, f, nodes.NodeNames[0:1]) + extendedResourceName := deployDevicePlugin(tCtx, f, nodes.NodeNames[0:1], false) pod := b.PodExternal() resources := v1.ResourceList{extendedResourceName: resource.MustParse("1")} @@ -2484,31 +2484,11 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { f.It("process extended resources after device plugin uninstall", f.WithSerial(), func(ctx context.Context) { tCtx := f.TContext(ctx) resourceName := b.ExtendedResourceName(drautils.SingletonIndex) - extendedResourceName := deployDevicePlugin(ctx, f, nodes.NodeNames[0:1]) + extendedResourceName := deployDevicePlugin(tCtx, f, nodes.NodeNames[0:1], false) gomega.Expect(string(extendedResourceName)).To(gomega.Equal(resourceName)) - getAllocatable := func() int { - node, err := f.ClientSet.CoreV1().Nodes().Get(ctx, nodes.NodeNames[0], metav1.GetOptions{}) - framework.ExpectNoError(err) - for name, quantity := range node.Status.Allocatable { - if string(name) == resourceName { - return int(quantity.Value()) - } - } - return -1 - } - gomega.Eventually(ctx, getAllocatable).WithTimeout(f.Timeouts.PodStart).Should(gomega.Equal(2)) - ginkgo.By("Uninstall Device Plugin") - err := f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).DeleteCollection( - ctx, - metav1.DeleteOptions{}, - metav1.ListOptions{LabelSelector: "k8s-app=sample-device-plugin"}, - ) - framework.ExpectNoError(err, "uninstall device plugin") - - ginkgo.By("Wait for NodeStatus.Allocatable = 0") - gomega.Eventually(ctx, getAllocatable).WithTimeout(f.Timeouts.PodDelete).Should(gomega.BeZero()) + undeployDevicePlugin(tCtx, f, nodes.NodeNames[0:1]) ginkgo.By("Create test pod") pod := b.Pod() @@ -2521,7 +2501,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { b.Create(tCtx, b.Class(drautils.SingletonIndex), pod) - err = e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod) + err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod) framework.ExpectNoError(err, "start pod") ginkgo.By("Check that pod is processed by the DRA driver") @@ -2678,7 +2658,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { f.It("must run pods with extended resource on dra nodes and device plugin nodes", f.WithSerial(), func(ctx context.Context) { tCtx := f.TContext(ctx) var objects []klog.KMetadata - extendedResourceName := deployDevicePlugin(ctx, f, nodes.ExtraNodeNames) + extendedResourceName := deployDevicePlugin(tCtx, f, nodes.ExtraNodeNames, false) // b.ExtendedResourceName(SingletonIndex) must be the same as the returned extendedResourceName. // b.ExtendedResourceName(SingletonIndex) is used for DRA drivers whereas // extendedResourceName is used for device plugin.