From 3d4bc44a2f807cd3e7308e682adddfc835109d11 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 29 May 2024 08:50:06 +0200 Subject: [PATCH] dra e2e node: addd test case for ResourceSlice handling during kubelet startup Any redundant object must get deleted, but not the ones of other names. --- test/e2e_node/dra_test.go | 105 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 100 insertions(+), 5 deletions(-) diff --git a/test/e2e_node/dra_test.go b/test/e2e_node/dra_test.go index 871f11e4abe..a07a4022ea3 100644 --- a/test/e2e_node/dra_test.go +++ b/test/e2e_node/dra_test.go @@ -34,9 +34,12 @@ import ( "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" + "github.com/onsi/gomega/gstruct" + "github.com/onsi/gomega/types" v1 "k8s.io/api/core/v1" resourcev1alpha2 "k8s.io/api/resource/v1alpha2" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" @@ -70,7 +73,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, f.Context("Resource Kubelet Plugin", f.WithSerial(), func() { ginkgo.BeforeEach(func(ctx context.Context) { - kubeletPlugin = newKubeletPlugin(ctx, getNodeName(ctx, f), driverName) + kubeletPlugin = newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) }) ginkgo.It("must register after Kubelet restart", func(ctx context.Context) { @@ -90,7 +93,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, ginkgo.It("must register after plugin restart", func(ctx context.Context) { ginkgo.By("restart Kubelet Plugin") kubeletPlugin.Stop() - kubeletPlugin = newKubeletPlugin(ctx, getNodeName(ctx, f), driverName) + kubeletPlugin = newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) ginkgo.By("wait for Kubelet plugin re-registration") gomega.Eventually(kubeletPlugin.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdriver.BeRegistered) @@ -307,8 +310,8 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, f.Context("Two resource Kubelet Plugins", f.WithSerial(), func() { ginkgo.BeforeEach(func(ctx context.Context) { - kubeletPlugin1 = newKubeletPlugin(ctx, getNodeName(ctx, f), kubeletPlugin1Name) - kubeletPlugin2 = newKubeletPlugin(ctx, getNodeName(ctx, f), kubeletPlugin2Name) + kubeletPlugin1 = newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), kubeletPlugin1Name) + kubeletPlugin2 = newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), kubeletPlugin2Name) ginkgo.By("wait for Kubelet plugin registration") gomega.Eventually(kubeletPlugin1.GetGRPCCalls()).WithTimeout(pluginRegistrationTimeout).Should(testdriver.BeRegistered) @@ -423,10 +426,73 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation, framework.ExpectNoError(err) }) }) + + f.Context("ResourceSlice", f.WithSerial(), func() { + listResources := func(ctx context.Context) ([]resourcev1alpha2.ResourceSlice, error) { + slices, err := f.ClientSet.ResourceV1alpha2().ResourceSlices().List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + return slices.Items, nil + } + + matchResourcesByNodeName := func(nodeName string) types.GomegaMatcher { + return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "NodeName": gomega.Equal(nodeName), + }) + } + + f.It("must be removed on kubelet startup", f.WithDisruptive(), func(ctx context.Context) { + gomega.Expect(listResources(ctx)).To(gomega.BeEmpty(), "ResourceSlices should have been deleted after previous test") + + ginkgo.By("stop kubelet") + startKubelet := stopKubelet() + ginkgo.DeferCleanup(func() { + if startKubelet != nil { + startKubelet() + } + }) + + ginkgo.By("create some ResourceSlices") + nodeName := getNodeName(ctx, f) + otherNodeName := nodeName + "-other" + createTestResourceSlice(ctx, f.ClientSet, nodeName, driverName) + createTestResourceSlice(ctx, f.ClientSet, nodeName+"-other", driverName) + + matchAll := gomega.ConsistOf(matchResourcesByNodeName(nodeName), matchResourcesByNodeName(otherNodeName)) + matchOtherNode := gomega.ConsistOf(matchResourcesByNodeName(otherNodeName)) + + gomega.Consistently(ctx, listResources).WithTimeout(5*time.Second).Should(matchAll, "ResourceSlices without kubelet") + + ginkgo.By("start kubelet") + startKubelet() + startKubelet = nil + + ginkgo.By("wait for exactly the node's ResourceSlice to get deleted") + gomega.Eventually(ctx, listResources).Should(matchOtherNode, "ResourceSlices with kubelet") + gomega.Consistently(ctx, listResources).WithTimeout(5*time.Second).Should(matchOtherNode, "ResourceSlices with kubelet") + }) + + f.It("must be removed after plugin unregistration", func(ctx context.Context) { + gomega.Expect(listResources(ctx)).To(gomega.BeEmpty(), "ResourceSlices should have been deleted after previous test") + nodeName := getNodeName(ctx, f) + matchNode := gomega.ConsistOf(matchResourcesByNodeName(nodeName)) + + ginkgo.By("start plugin and wait for ResourceSlice") + kubeletPlugin = newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName) + gomega.Eventually(ctx, listResources).Should(matchNode, "ResourceSlice from kubelet plugin") + gomega.Consistently(ctx, listResources).WithTimeout(5*time.Second).Should(matchNode, "ResourceSlice from kubelet plugin") + + ginkgo.By("stop plugin and wait for ResourceSlice removal") + kubeletPlugin.Stop() + gomega.Eventually(ctx, listResources).Should(gomega.BeEmpty(), "ResourceSlices with no plugin") + gomega.Consistently(ctx, listResources).WithTimeout(5*time.Second).Should(gomega.BeEmpty(), "ResourceSlices with no plugin") + }) + }) }) // Run Kubelet plugin and wait until it's registered -func newKubeletPlugin(ctx context.Context, nodeName, pluginName string) *testdriver.ExamplePlugin { +func newKubeletPlugin(ctx context.Context, clientSet kubernetes.Interface, nodeName, pluginName string) *testdriver.ExamplePlugin { ginkgo.By("start Kubelet plugin") logger := klog.LoggerWithValues(klog.LoggerWithName(klog.Background(), "kubelet plugin "+pluginName), "node", nodeName) ctx = klog.NewContext(ctx, logger) @@ -454,6 +520,11 @@ func newKubeletPlugin(ctx context.Context, nodeName, pluginName string) *testdri gomega.Eventually(plugin.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdriver.BeRegistered) + ginkgo.DeferCleanup(func(ctx context.Context) { + // kubelet should do this eventually, but better make sure. + // A separate test checks this explicitly. + framework.ExpectNoError(clientSet.ResourceV1alpha2().ResourceSlices().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{FieldSelector: "driverName=" + driverName})) + }) ginkgo.DeferCleanup(plugin.Stop) return plugin @@ -549,3 +620,27 @@ func createTestObjects(ctx context.Context, clientSet kubernetes.Interface, node return pod } + +func createTestResourceSlice(ctx context.Context, clientSet kubernetes.Interface, nodeName, driverName string) { + slice := &resourcev1alpha2.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + }, + NodeName: nodeName, + DriverName: driverName, + ResourceModel: resourcev1alpha2.ResourceModel{ + NamedResources: &resourcev1alpha2.NamedResourcesResources{}, + }, + } + + ginkgo.By(fmt.Sprintf("Creating ResourceSlice %s", nodeName)) + slice, err := clientSet.ResourceV1alpha2().ResourceSlices().Create(ctx, slice, metav1.CreateOptions{}) + framework.ExpectNoError(err, "create ResourceSlice") + ginkgo.DeferCleanup(func(ctx context.Context) { + ginkgo.By(fmt.Sprintf("Deleting ResourceSlice %s", nodeName)) + err := clientSet.ResourceV1alpha2().ResourceSlices().Delete(ctx, slice.Name, metav1.DeleteOptions{}) + if !apierrors.IsNotFound(err) { + framework.ExpectNoError(err, "delete ResourceSlice") + } + }) +}