From 92e00203e027bf88ddc695fc17e80c913a6b178a Mon Sep 17 00:00:00 2001 From: Francesco Romani Date: Tue, 21 Feb 2023 13:50:33 +0100 Subject: [PATCH] e2e: node: unify sample device plugin utilities Start to consolidate the sample device plugin utility and constants in a central place, because we need to use it in different e2e tests. Having a central dependency is better than a maze of entangled e2e tests depending on each other helpers. Signed-off-by: Francesco Romani --- test/e2e_node/device_plugin_test.go | 92 +++++------------------------ test/e2e_node/image_list.go | 29 ++++++++- test/e2e_node/podresources_test.go | 48 ++------------- test/e2e_node/util_sampledevice.go | 52 ++++++++++++++++ 4 files changed, 100 insertions(+), 121 deletions(-) create mode 100644 test/e2e_node/util_sampledevice.go diff --git a/test/e2e_node/device_plugin_test.go b/test/e2e_node/device_plugin_test.go index 5cdf50e2ba8..f8c4420b581 100644 --- a/test/e2e_node/device_plugin_test.go +++ b/test/e2e_node/device_plugin_test.go @@ -31,7 +31,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" kubeletdevicepluginv1beta1 "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" - e2etestfiles "k8s.io/kubernetes/test/e2e/framework/testfiles" admissionapi "k8s.io/pod-security-admission/api" "k8s.io/apimachinery/pkg/api/resource" @@ -44,17 +43,6 @@ import ( e2epod "k8s.io/kubernetes/test/e2e/framework/pod" ) -const ( - // sampleResourceName is the name of the example resource which is used in the e2e test - sampleResourceName = "example.com/resource" - // sampleDevicePluginName is the name of the device plugin pod - sampleDevicePluginName = "sample-device-plugin" - - // fake resource name - resourceName = "example.com/resource" - envVarNamePluginSockDir = "PLUGIN_SOCK_DIR" -) - var ( appsScheme = runtime.NewScheme() appsCodecs = serializer.NewCodecFactory(appsScheme) @@ -67,17 +55,6 @@ var _ = SIGDescribe("Device Plugin [Feature:DevicePluginProbe][NodeFeature:Devic testDevicePlugin(f, kubeletdevicepluginv1beta1.DevicePluginPath) }) -// numberOfSampleResources returns the number of resources advertised by a node. -func numberOfSampleResources(node *v1.Node) int64 { - val, ok := node.Status.Capacity[sampleResourceName] - - if !ok { - return 0 - } - - return val.Value() -} - // readDaemonSetV1OrDie reads daemonset object from bytes. Panics on error. func readDaemonSetV1OrDie(objBytes []byte) *appsv1.DaemonSet { appsv1.AddToScheme(appsScheme) @@ -133,31 +110,14 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) { }, f.Timeouts.PodDelete, f.Timeouts.Poll).Should(gomega.Succeed()) ginkgo.By("Scheduling a sample device plugin pod") - data, err := e2etestfiles.Read(SampleDevicePluginDSYAML) - if err != nil { - framework.Fail(err.Error()) - } - ds := readDaemonSetV1OrDie(data) - - dp := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: sampleDevicePluginName, - }, - Spec: ds.Spec.Template.Spec, - } - - for i := range dp.Spec.Containers[0].Env { - if dp.Spec.Containers[0].Env[i].Name == envVarNamePluginSockDir { - dp.Spec.Containers[0].Env[i].Value = pluginSockDir - } - } + dp := getSampleDevicePluginPod(pluginSockDir) dptemplate = dp.DeepCopy() devicePluginPod = e2epod.NewPodClient(f).CreateSync(ctx, dp) ginkgo.By("Waiting for devices to become available on the local node") gomega.Eventually(ctx, func(ctx context.Context) bool { node, ready := getLocalTestNode(ctx, f) - return ready && numberOfSampleResources(node) > 0 + return ready && CountSampleDeviceCapacity(node) > 0 }, 5*time.Minute, framework.Poll).Should(gomega.BeTrue()) framework.Logf("Successfully created device plugin pod") @@ -165,8 +125,8 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) { gomega.Eventually(ctx, func(ctx context.Context) bool { node, ready := getLocalTestNode(ctx, f) return ready && - numberOfDevicesCapacity(node, resourceName) == devsLen && - numberOfDevicesAllocatable(node, resourceName) == devsLen + CountSampleDeviceCapacity(node) == devsLen && + CountSampleDeviceAllocatable(node) == devsLen }, 30*time.Second, framework.Poll).Should(gomega.BeTrue()) }) @@ -191,7 +151,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) { ginkgo.By("Waiting for devices to become unavailable on the local node") gomega.Eventually(ctx, func(ctx context.Context) bool { node, ready := getLocalTestNode(ctx, f) - return ready && numberOfSampleResources(node) <= 0 + return ready && CountSampleDeviceCapacity(node) <= 0 }, 5*time.Minute, framework.Poll).Should(gomega.BeTrue()) ginkgo.By("devices now unavailable on the local node") @@ -199,7 +159,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) { ginkgo.It("Can schedule a pod that requires a device", func(ctx context.Context) { podRECMD := "devs=$(ls /tmp/ | egrep '^Dev-[0-9]+$') && echo stub devices: $devs && sleep 60" - pod1 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(resourceName, podRECMD)) + pod1 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(SampleDeviceResourceName, podRECMD)) deviceIDRE := "stub devices: (Dev-[0-9]+)" devID1 := parseLog(ctx, f, pod1.Name, pod1.Name, deviceIDRE) gomega.Expect(devID1).To(gomega.Not(gomega.Equal(""))) @@ -250,8 +210,8 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) { framework.ExpectEqual(len(v1alphaResourcesForOurPod.Containers[0].Devices), 1) framework.ExpectEqual(len(v1ResourcesForOurPod.Containers[0].Devices), 1) - framework.ExpectEqual(v1alphaResourcesForOurPod.Containers[0].Devices[0].ResourceName, resourceName) - framework.ExpectEqual(v1ResourcesForOurPod.Containers[0].Devices[0].ResourceName, resourceName) + framework.ExpectEqual(v1alphaResourcesForOurPod.Containers[0].Devices[0].ResourceName, SampleDeviceResourceName) + framework.ExpectEqual(v1ResourcesForOurPod.Containers[0].Devices[0].ResourceName, SampleDeviceResourceName) framework.ExpectEqual(len(v1alphaResourcesForOurPod.Containers[0].Devices[0].DeviceIds), 1) framework.ExpectEqual(len(v1ResourcesForOurPod.Containers[0].Devices[0].DeviceIds), 1) @@ -259,7 +219,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) { ginkgo.It("Keeps device plugin assignments across pod and kubelet restarts", func(ctx context.Context) { podRECMD := "devs=$(ls /tmp/ | egrep '^Dev-[0-9]+$') && echo stub devices: $devs && sleep 60" - pod1 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(resourceName, podRECMD)) + pod1 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(SampleDeviceResourceName, podRECMD)) deviceIDRE := "stub devices: (Dev-[0-9]+)" devID1 := parseLog(ctx, f, pod1.Name, pod1.Name, deviceIDRE) gomega.Expect(devID1).To(gomega.Not(gomega.Equal(""))) @@ -288,7 +248,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) { ginkgo.It("Keeps device plugin assignments after the device plugin has been re-registered", func(ctx context.Context) { podRECMD := "devs=$(ls /tmp/ | egrep '^Dev-[0-9]+$') && echo stub devices: $devs && sleep 60" - pod1 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(resourceName, podRECMD)) + pod1 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(SampleDeviceResourceName, podRECMD)) deviceIDRE := "stub devices: (Dev-[0-9]+)" devID1 := parseLog(ctx, f, pod1.Name, pod1.Name, deviceIDRE) gomega.Expect(devID1).To(gomega.Not(gomega.Equal(""))) @@ -322,12 +282,12 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) { gomega.Eventually(ctx, func() bool { node, ready := getLocalTestNode(ctx, f) return ready && - numberOfDevicesCapacity(node, resourceName) == devsLen && - numberOfDevicesAllocatable(node, resourceName) == devsLen + CountSampleDeviceCapacity(node) == devsLen && + CountSampleDeviceAllocatable(node) == devsLen }, 30*time.Second, framework.Poll).Should(gomega.BeTrue()) ginkgo.By("Creating another pod") - pod2 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(resourceName, podRECMD)) + pod2 := e2epod.NewPodClient(f).CreateSync(ctx, makeBusyboxPod(SampleDeviceResourceName, podRECMD)) ginkgo.By("Checking that pod got a different fake device") devID2 := parseLog(ctx, f, pod2.Name, pod2.Name, deviceIDRE) @@ -338,10 +298,10 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) { } // makeBusyboxPod returns a simple Pod spec with a busybox container -// that requests resourceName and runs the specified command. -func makeBusyboxPod(resourceName, cmd string) *v1.Pod { +// that requests SampleDeviceResourceName and runs the specified command. +func makeBusyboxPod(SampleDeviceResourceName, cmd string) *v1.Pod { podName := "device-plugin-test-" + string(uuid.NewUUID()) - rl := v1.ResourceList{v1.ResourceName(resourceName): *resource.NewQuantity(1, resource.DecimalSI)} + rl := v1.ResourceList{v1.ResourceName(SampleDeviceResourceName): *resource.NewQuantity(1, resource.DecimalSI)} return &v1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: podName}, @@ -397,23 +357,3 @@ func parseLog(ctx context.Context, f *framework.Framework, podName string, contN return matches[1] } - -// numberOfDevicesCapacity returns the number of devices of resourceName advertised by a node capacity -func numberOfDevicesCapacity(node *v1.Node, resourceName string) int64 { - val, ok := node.Status.Capacity[v1.ResourceName(resourceName)] - if !ok { - return 0 - } - - return val.Value() -} - -// numberOfDevicesAllocatable returns the number of devices of resourceName advertised by a node allocatable -func numberOfDevicesAllocatable(node *v1.Node, resourceName string) int64 { - val, ok := node.Status.Allocatable[v1.ResourceName(resourceName)] - if !ok { - return 0 - } - - return val.Value() -} diff --git a/test/e2e_node/image_list.go b/test/e2e_node/image_list.go index 641117d6095..611590642bc 100644 --- a/test/e2e_node/image_list.go +++ b/test/e2e_node/image_list.go @@ -24,13 +24,16 @@ import ( "sync" "time" + v1 "k8s.io/api/core/v1" "k8s.io/klog/v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" internalapi "k8s.io/cri-api/pkg/apis" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" commontest "k8s.io/kubernetes/test/e2e/common" + "k8s.io/kubernetes/test/e2e/framework" e2egpu "k8s.io/kubernetes/test/e2e/framework/gpu" e2emanifest "k8s.io/kubernetes/test/e2e/framework/manifest" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" @@ -45,9 +48,6 @@ const ( imagePullRetryDelay = time.Second // Number of parallel count to pull images. maxParallelImagePullCount = 5 - - // SampleDevicePluginDSYAML is the path of the daemonset template of the sample device plugin. // TODO: Parametrize it by making it a feature in TestFramework. - SampleDevicePluginDSYAML = "test/e2e/testing-manifests/sample-device-plugin.yaml" ) // NodePrePullImageList is a list of images used in node e2e test. These images will be prepulled @@ -239,6 +239,29 @@ func getSampleDevicePluginImage() (string, error) { return ds.Spec.Template.Spec.Containers[0].Image, nil } +// getSampleDevicePluginPod returns the Sample Device Plugin pod to be used e2e tests. +func getSampleDevicePluginPod(pluginSockDir string) *v1.Pod { + data, err := e2etestfiles.Read(SampleDevicePluginDSYAML) + if err != nil { + framework.Fail(err.Error()) + } + + ds := readDaemonSetV1OrDie(data) + dp := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: SampleDevicePluginName, + }, + Spec: ds.Spec.Template.Spec, + } + for i := range dp.Spec.Containers[0].Env { + if dp.Spec.Containers[0].Env[i].Name == SampleDeviceEnvVarNamePluginSockDir { + dp.Spec.Containers[0].Env[i].Value = pluginSockDir + } + } + + return dp +} + // getSRIOVDevicePluginImage returns the image of SRIOV device plugin. func getSRIOVDevicePluginImage() (string, error) { data, err := e2etestfiles.Read(SRIOVDevicePluginDSYAML) diff --git a/test/e2e_node/podresources_test.go b/test/e2e_node/podresources_test.go index cf37e6c19bd..11fd14dc88e 100644 --- a/test/e2e_node/podresources_test.go +++ b/test/e2e_node/podresources_test.go @@ -26,6 +26,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeletdevicepluginv1beta1 "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" kubeletpodresourcesv1 "k8s.io/kubelet/pkg/apis/podresources/v1" kubefeatures "k8s.io/kubernetes/pkg/features" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" @@ -45,7 +46,6 @@ import ( e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" - e2etestfiles "k8s.io/kubernetes/test/e2e/framework/testfiles" ) const ( @@ -892,12 +892,12 @@ func getOnlineCPUs() (cpuset.CPUSet, error) { func setupSampleDevicePluginOrFail(ctx context.Context, f *framework.Framework) *v1.Pod { e2enode.WaitForNodeToBeReady(ctx, f.ClientSet, framework.TestContext.NodeName, 5*time.Minute) - dp := getSampleDevicePluginPod() + dp := getSampleDevicePluginPod(kubeletdevicepluginv1beta1.DevicePluginPath) dp.Spec.NodeName = framework.TestContext.NodeName ginkgo.By("Create the sample device plugin pod") - dpPod = e2epod.NewPodClient(f).CreateSync(ctx, dp) + dpPod := e2epod.NewPodClient(f).CreateSync(ctx, dp) err := e2epod.WaitForPodCondition(ctx, f.ClientSet, dpPod.Namespace, dpPod.Name, "Ready", 120*time.Second, testutils.PodRunningReady) if err != nil { @@ -920,52 +920,16 @@ func teardownSampleDevicePluginOrFail(ctx context.Context, f *framework.Framewor waitForAllContainerRemoval(ctx, pod.Name, pod.Namespace) } -func findTopologyUnawareResource(node *v1.Node) int64 { - framework.Logf("Node status allocatable: %v", node.Status.Allocatable) - for key, val := range node.Status.Allocatable { - if string(key) == defaultTopologyUnawareResourceName { - v := val.Value() - if v > 0 { - return v - } - } - } - return 0 -} - -func waitForTopologyUnawareResources(ctx context.Context, f *framework.Framework, pod *v1.Pod) { +func waitForTopologyUnawareResources(ctx context.Context, f *framework.Framework) { ginkgo.By(fmt.Sprintf("Waiting for %q resources to become available on the local node", defaultTopologyUnawareResourceName)) gomega.Eventually(ctx, func(ctx context.Context) bool { node := getLocalNode(ctx, f) - resourceAmount := findTopologyUnawareResource(node) - return resourceAmount != 0 + resourceAmount := CountSampleDeviceAllocatable(node) + return resourceAmount > 0 }, 2*time.Minute, framework.Poll).Should(gomega.BeTrue()) } -// getSampleDevicePluginPod returns the Sample Device Plugin pod to be used e2e tests. -func getSampleDevicePluginPod() *v1.Pod { - data, err := e2etestfiles.Read(SampleDevicePluginDSYAML) - if err != nil { - framework.Fail(err.Error()) - } - - ds := readDaemonSetV1OrDie(data) - dp := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: sampleDevicePluginName, - }, - Spec: ds.Spec.Template.Spec, - } - for i := range dp.Spec.Containers[0].Env { - if dp.Spec.Containers[0].Env[i].Name == envVarNamePluginSockDir { - dp.Spec.Containers[0].Env[i].Value = pluginSockDir - } - } - - return dp -} - func getPodResourcesMetrics(ctx context.Context) (e2emetrics.KubeletMetrics, error) { // we are running out of good names, so we need to be unnecessarily specific to avoid clashes ginkgo.By("getting Pod Resources metrics from the metrics API") diff --git a/test/e2e_node/util_sampledevice.go b/test/e2e_node/util_sampledevice.go new file mode 100644 index 00000000000..a1e22ee761e --- /dev/null +++ b/test/e2e_node/util_sampledevice.go @@ -0,0 +1,52 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2enode + +import ( + v1 "k8s.io/api/core/v1" +) + +const ( + // SampleDevicePluginDSYAML is the path of the daemonset template of the sample device plugin. // TODO: Parametrize it by making it a feature in TestFramework. + SampleDevicePluginDSYAML = "test/e2e/testing-manifests/sample-device-plugin.yaml" + + // SampleDevicePluginName is the name of the device plugin pod + SampleDevicePluginName = "sample-device-plugin" + + // SampleDeviceResourceName is the name of the resource provided by the sample device plugin + SampleDeviceResourceName = "example.com/resource" + + SampleDeviceEnvVarNamePluginSockDir = "PLUGIN_SOCK_DIR" +) + +// CountSampleDeviceCapacity returns the number of devices of SampleDeviceResourceName advertised by a node capacity +func CountSampleDeviceCapacity(node *v1.Node) int64 { + val, ok := node.Status.Capacity[v1.ResourceName(SampleDeviceResourceName)] + if !ok { + return 0 + } + return val.Value() +} + +// CountSampleDeviceAllocatable returns the number of devices of SampleDeviceResourceName advertised by a node allocatable +func CountSampleDeviceAllocatable(node *v1.Node) int64 { + val, ok := node.Status.Allocatable[v1.ResourceName(SampleDeviceResourceName)] + if !ok { + return 0 + } + return val.Value() +}