github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2026-07-17 20:00:07 +00:00
Code Issues Projects Releases Wiki Activity

DRA: e2e: refactor device plugin deployment

Browse Source 
Refactored deployDevicePlugin, added undeployDevicePlugin

- Add skipCleanup parameter to control cleanup behavior
- Wait for device plugin resources to become allocatable on deploy
- Add undeployDevicePlugin function to properly cleanup and wait for
  resources to be removed from node allocatable capacity
- Update tests to use refactored functions and remove duplicate code

These changes reduce test flakiness by explicitly waiting for expected
node states before proceeding with test operations.
This commit is contained in:
Ed Bartosh
2026-02-10 11:38:02 +02:00
parent 02485d02ea
commit 0fc147ed02
2 changed files with 48 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
test/e2e/dra/deploy_device_plugin.go
View File

@@ -17,14 +17,15 @@ limitations under the License.
package dra
import (
"context"
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/test/e2e/framework"
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
"k8s.io/kubernetes/test/e2e/storage/utils"
"k8s.io/kubernetes/test/utils/ktesting"
)
// deployDevicePlugin deploys https://github.com/kubernetes/kubernetes/tree/111a2a0d2dfe13639724506f674bc4f342ccfbab/test/images/sample-device-plugin
@@ -33,9 +34,9 @@ import (
// A test using this must run serially because the example plugin uses the hard-coded "example.com/resource"
// extended resource name (returned as result) and deploying it twice for the same nodes from different
// tests would conflict.
func deployDevicePlugin(ctx context.Context, f *framework.Framework, nodeNames []string) v1.ResourceName {
ginkgo.By("Deploy Device Plugin")
err := utils.CreateFromManifests(ctx, f, f.Namespace, func(item interface{}) error {
func deployDevicePlugin(tCtx ktesting.TContext, f *framework.Framework, nodeNames []string, skipCleanup bool) v1.ResourceName {
ginkgo.By("Deploy Sample Device Plugin DaemonSet")
err := utils.CreateFromManifests(tCtx, f, f.Namespace, func(item interface{}) error {
switch item := item.(type) {
case *appsv1.DaemonSet:
item.Spec.Template.Spec.Affinity = &v1.Affinity{
@@ -58,8 +59,43 @@ func deployDevicePlugin(ctx context.Context, f *framework.Framework, nodeNames [
}
return nil
}, e2enode.SampleDevicePluginDSYAML)
framework.ExpectNoError(err, "deploy example device plugin DaemonSet")
framework.ExpectNoError(err, "deploy Sample Device Plugin DaemonSet")
if !skipCleanup {
tCtx.CleanupCtx(func(tCtx ktesting.TContext) {
undeployDevicePlugin(tCtx, f, nodeNames)
})
}
// Wait for the Kubelet to update resource Allocatable
// based on the device plugin resources reported by the sample device plugin.
for _, nodeName := range nodeNames {
gomega.Eventually(tCtx, func() int64 {
node, err := f.ClientSet.CoreV1().Nodes().Get(tCtx, nodeName, metav1.GetOptions{})
framework.ExpectNoError(err)
return e2enode.CountSampleDeviceAllocatable(node)
}).WithTimeout(f.Timeouts.PodStart).Should(gomega.BeNumerically("==", e2enode.SampleDevsAmount), "expected %d %q to be allocatable on node %q", e2enode.SampleDevsAmount, e2enode.SampleDeviceResourceName, nodeName)
}
// Hard-coded in https://github.com/kubernetes/kubernetes/blob/111a2a0d2dfe13639724506f674bc4f342ccfbab/test/images/sample-device-plugin/sampledeviceplugin.go#L34C17-L34C39.
return e2enode.SampleDeviceResourceName
}
func undeployDevicePlugin(tCtx ktesting.TContext, f *framework.Framework, nodeNames []string) {
ginkgo.By("Undeploy Sample Device Plugin DaemonSet")
err := f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).DeleteCollection(
tCtx,
metav1.DeleteOptions{},
metav1.ListOptions{LabelSelector: "k8s-app=" + e2enode.SampleDevicePluginName},
)
framework.ExpectNoError(err, "undeploy Sample Device Plugin DaemonSet")
// Wait for the Kubelet to update resource Allocatable to 0 after the device plugin is undeployed.
// This is needed to ensure that the device plugin resources are not allocatable for pods after the device plugin is undeployed.
for _, nodeName := range nodeNames {
gomega.Eventually(tCtx, func() int64 {
node, err := f.ClientSet.CoreV1().Nodes().Get(tCtx, nodeName, metav1.GetOptions{})
framework.ExpectNoError(err)
return e2enode.CountSampleDeviceAllocatable(node)
}).WithTimeout(f.Timeouts.PodStart).Should(gomega.BeZero(), "expected 0 %q to be allocatable on node %q after device plugin undeploy", e2enode.SampleDeviceResourceName, nodeName)
}
}

30
test/e2e/dra/dra.go
View File

@@ -1084,7 +1084,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
// Serial because the example device plugin can only be deployed with one instance at a time.
f.It("supports extended resources together with ResourceClaim", f.WithSerial(), func(ctx context.Context) {
tCtx := f.TContext(ctx)
extendedResourceName := deployDevicePlugin(ctx, f, nodes.NodeNames[0:1])
extendedResourceName := deployDevicePlugin(tCtx, f, nodes.NodeNames[0:1], false)
pod := b.PodExternal()
resources := v1.ResourceList{extendedResourceName: resource.MustParse("1")}
@@ -2484,31 +2484,11 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
f.It("process extended resources after device plugin uninstall", f.WithSerial(), func(ctx context.Context) {
tCtx := f.TContext(ctx)
resourceName := b.ExtendedResourceName(drautils.SingletonIndex)
extendedResourceName := deployDevicePlugin(ctx, f, nodes.NodeNames[0:1])
extendedResourceName := deployDevicePlugin(tCtx, f, nodes.NodeNames[0:1], false)
gomega.Expect(string(extendedResourceName)).To(gomega.Equal(resourceName))
getAllocatable := func() int {
node, err := f.ClientSet.CoreV1().Nodes().Get(ctx, nodes.NodeNames[0], metav1.GetOptions{})
framework.ExpectNoError(err)
for name, quantity := range node.Status.Allocatable {
if string(name) == resourceName {
return int(quantity.Value())
}
}
return -1
}
gomega.Eventually(ctx, getAllocatable).WithTimeout(f.Timeouts.PodStart).Should(gomega.Equal(2))
ginkgo.By("Uninstall Device Plugin")
err := f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).DeleteCollection(
ctx,
metav1.DeleteOptions{},
metav1.ListOptions{LabelSelector: "k8s-app=sample-device-plugin"},
)
framework.ExpectNoError(err, "uninstall device plugin")
ginkgo.By("Wait for NodeStatus.Allocatable = 0")
gomega.Eventually(ctx, getAllocatable).WithTimeout(f.Timeouts.PodDelete).Should(gomega.BeZero())
undeployDevicePlugin(tCtx, f, nodes.NodeNames[0:1])
ginkgo.By("Create test pod")
pod := b.Pod()
@@ -2521,7 +2501,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
b.Create(tCtx, b.Class(drautils.SingletonIndex), pod)
err = e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
framework.ExpectNoError(err, "start pod")
ginkgo.By("Check that pod is processed by the DRA driver")
@@ -2678,7 +2658,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
f.It("must run pods with extended resource on dra nodes and device plugin nodes", f.WithSerial(), func(ctx context.Context) {
tCtx := f.TContext(ctx)
var objects []klog.KMetadata
extendedResourceName := deployDevicePlugin(ctx, f, nodes.ExtraNodeNames)
extendedResourceName := deployDevicePlugin(tCtx, f, nodes.ExtraNodeNames, false)
// b.ExtendedResourceName(SingletonIndex) must be the same as the returned extendedResourceName.
// b.ExtendedResourceName(SingletonIndex) is used for DRA drivers whereas
// extendedResourceName is used for device plugin.