github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-01 07:47:56 +00:00
Code Issues Projects Releases Wiki Activity

dra e2e node: addd test case for ResourceSlice handling during kubelet startup

Browse Source 
Any redundant object must get deleted, but not the ones of other names.
This commit is contained in:
Patrick Ohly 2024-05-29 08:50:06 +02:00
parent 5a99930ab5
commit 3d4bc44a2f
1 changed files with 100 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
test/e2e_node/dra_test.go
View File

@ -34,9 +34,12 @@ import (
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
"github.com/onsi/gomega/gstruct"
"github.com/onsi/gomega/types"
v1 "k8s.io/api/core/v1"
resourcev1alpha2 "k8s.io/api/resource/v1alpha2"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
@ -70,7 +73,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
f.Context("Resource Kubelet Plugin", f.WithSerial(), func() {
ginkgo.BeforeEach(func(ctx context.Context) {
kubeletPlugin = newKubeletPlugin(ctx, getNodeName(ctx, f), driverName)
kubeletPlugin = newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
})
ginkgo.It("must register after Kubelet restart", func(ctx context.Context) {
@ -90,7 +93,7 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
ginkgo.It("must register after plugin restart", func(ctx context.Context) {
ginkgo.By("restart Kubelet Plugin")
kubeletPlugin.Stop()
kubeletPlugin = newKubeletPlugin(ctx, getNodeName(ctx, f), driverName)
kubeletPlugin = newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
ginkgo.By("wait for Kubelet plugin re-registration")
gomega.Eventually(kubeletPlugin.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdriver.BeRegistered)
@ -307,8 +310,8 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
f.Context("Two resource Kubelet Plugins", f.WithSerial(), func() {
ginkgo.BeforeEach(func(ctx context.Context) {
kubeletPlugin1 = newKubeletPlugin(ctx, getNodeName(ctx, f), kubeletPlugin1Name)
kubeletPlugin2 = newKubeletPlugin(ctx, getNodeName(ctx, f), kubeletPlugin2Name)
kubeletPlugin1 = newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), kubeletPlugin1Name)
kubeletPlugin2 = newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), kubeletPlugin2Name)
ginkgo.By("wait for Kubelet plugin registration")
gomega.Eventually(kubeletPlugin1.GetGRPCCalls()).WithTimeout(pluginRegistrationTimeout).Should(testdriver.BeRegistered)
@ -423,10 +426,73 @@ var _ = framework.SIGDescribe("node")("DRA", feature.DynamicResourceAllocation,
framework.ExpectNoError(err)
})
})
f.Context("ResourceSlice", f.WithSerial(), func() {
listResources := func(ctx context.Context) ([]resourcev1alpha2.ResourceSlice, error) {
slices, err := f.ClientSet.ResourceV1alpha2().ResourceSlices().List(ctx, metav1.ListOptions{})
if err != nil {
return nil, err
}
return slices.Items, nil
}
matchResourcesByNodeName := func(nodeName string) types.GomegaMatcher {
return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
"NodeName": gomega.Equal(nodeName),
})
}
f.It("must be removed on kubelet startup", f.WithDisruptive(), func(ctx context.Context) {
gomega.Expect(listResources(ctx)).To(gomega.BeEmpty(), "ResourceSlices should have been deleted after previous test")
ginkgo.By("stop kubelet")
startKubelet := stopKubelet()
ginkgo.DeferCleanup(func() {
if startKubelet != nil {
startKubelet()
}
})
ginkgo.By("create some ResourceSlices")
nodeName := getNodeName(ctx, f)
otherNodeName := nodeName + "-other"
createTestResourceSlice(ctx, f.ClientSet, nodeName, driverName)
createTestResourceSlice(ctx, f.ClientSet, nodeName+"-other", driverName)
matchAll := gomega.ConsistOf(matchResourcesByNodeName(nodeName), matchResourcesByNodeName(otherNodeName))
matchOtherNode := gomega.ConsistOf(matchResourcesByNodeName(otherNodeName))
gomega.Consistently(ctx, listResources).WithTimeout(5*time.Second).Should(matchAll, "ResourceSlices without kubelet")
ginkgo.By("start kubelet")
startKubelet()
startKubelet = nil
ginkgo.By("wait for exactly the node's ResourceSlice to get deleted")
gomega.Eventually(ctx, listResources).Should(matchOtherNode, "ResourceSlices with kubelet")
gomega.Consistently(ctx, listResources).WithTimeout(5*time.Second).Should(matchOtherNode, "ResourceSlices with kubelet")
})
f.It("must be removed after plugin unregistration", func(ctx context.Context) {
gomega.Expect(listResources(ctx)).To(gomega.BeEmpty(), "ResourceSlices should have been deleted after previous test")
nodeName := getNodeName(ctx, f)
matchNode := gomega.ConsistOf(matchResourcesByNodeName(nodeName))
ginkgo.By("start plugin and wait for ResourceSlice")
kubeletPlugin = newKubeletPlugin(ctx, f.ClientSet, getNodeName(ctx, f), driverName)
gomega.Eventually(ctx, listResources).Should(matchNode, "ResourceSlice from kubelet plugin")
gomega.Consistently(ctx, listResources).WithTimeout(5*time.Second).Should(matchNode, "ResourceSlice from kubelet plugin")
ginkgo.By("stop plugin and wait for ResourceSlice removal")
kubeletPlugin.Stop()
gomega.Eventually(ctx, listResources).Should(gomega.BeEmpty(), "ResourceSlices with no plugin")
gomega.Consistently(ctx, listResources).WithTimeout(5*time.Second).Should(gomega.BeEmpty(), "ResourceSlices with no plugin")
})
})
})
// Run Kubelet plugin and wait until it's registered
func newKubeletPlugin(ctx context.Context, nodeName, pluginName string) *testdriver.ExamplePlugin {
func newKubeletPlugin(ctx context.Context, clientSet kubernetes.Interface, nodeName, pluginName string) *testdriver.ExamplePlugin {
ginkgo.By("start Kubelet plugin")
logger := klog.LoggerWithValues(klog.LoggerWithName(klog.Background(), "kubelet plugin "+pluginName), "node", nodeName)
ctx = klog.NewContext(ctx, logger)
@ -454,6 +520,11 @@ func newKubeletPlugin(ctx context.Context, nodeName, pluginName string) *testdri
gomega.Eventually(plugin.GetGRPCCalls).WithTimeout(pluginRegistrationTimeout).Should(testdriver.BeRegistered)
ginkgo.DeferCleanup(func(ctx context.Context) {
// kubelet should do this eventually, but better make sure.
// A separate test checks this explicitly.
framework.ExpectNoError(clientSet.ResourceV1alpha2().ResourceSlices().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{FieldSelector: "driverName=" + driverName}))
})
ginkgo.DeferCleanup(plugin.Stop)
return plugin
@ -549,3 +620,27 @@ func createTestObjects(ctx context.Context, clientSet kubernetes.Interface, node
return pod
}
func createTestResourceSlice(ctx context.Context, clientSet kubernetes.Interface, nodeName, driverName string) {
slice := &resourcev1alpha2.ResourceSlice{
ObjectMeta: metav1.ObjectMeta{
Name: nodeName,
},
NodeName: nodeName,
DriverName: driverName,
ResourceModel: resourcev1alpha2.ResourceModel{
NamedResources: &resourcev1alpha2.NamedResourcesResources{},
},
}
ginkgo.By(fmt.Sprintf("Creating ResourceSlice %s", nodeName))
slice, err := clientSet.ResourceV1alpha2().ResourceSlices().Create(ctx, slice, metav1.CreateOptions{})
framework.ExpectNoError(err, "create ResourceSlice")
ginkgo.DeferCleanup(func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Deleting ResourceSlice %s", nodeName))
err := clientSet.ResourceV1alpha2().ResourceSlices().Delete(ctx, slice.Name, metav1.DeleteOptions{})
if !apierrors.IsNotFound(err) {
framework.ExpectNoError(err, "delete ResourceSlice")
}
})
}