diff --git a/test/e2e/framework/BUILD b/test/e2e/framework/BUILD index 4c2e7d058dd..f8ed44f1b20 100644 --- a/test/e2e/framework/BUILD +++ b/test/e2e/framework/BUILD @@ -119,7 +119,6 @@ filegroup( "//test/e2e/framework/autoscaling:all-srcs", "//test/e2e/framework/config:all-srcs", "//test/e2e/framework/deployment:all-srcs", - "//test/e2e/framework/deviceplugin:all-srcs", "//test/e2e/framework/endpoints:all-srcs", "//test/e2e/framework/ginkgowrapper:all-srcs", "//test/e2e/framework/gpu:all-srcs", diff --git a/test/e2e/framework/deviceplugin/BUILD b/test/e2e/framework/deviceplugin/BUILD deleted file mode 100644 index 6df4381186e..00000000000 --- a/test/e2e/framework/deviceplugin/BUILD +++ /dev/null @@ -1,30 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "go_default_library", - srcs = ["device_plugin_util.go"], - importpath = "k8s.io/kubernetes/test/e2e/framework/deviceplugin", - visibility = ["//visibility:public"], - deps = [ - "//staging/src/k8s.io/api/apps/v1:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", - "//test/e2e/framework/testfiles:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/test/e2e/framework/deviceplugin/device_plugin_util.go b/test/e2e/framework/deviceplugin/device_plugin_util.go deleted file mode 100644 index aef67d0c34e..00000000000 --- a/test/e2e/framework/deviceplugin/device_plugin_util.go +++ /dev/null @@ -1,76 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package deviceplugin - -import ( - appsv1 "k8s.io/api/apps/v1" - "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/kubernetes/test/e2e/framework/testfiles" -) - -const ( - // sampleResourceName is the name of the example resource which is used in the e2e test - sampleResourceName = "example.com/resource" - // sampleDevicePluginDSYAML is the path of the daemonset template of the sample device plugin. // TODO: Parametrize it by making it a feature in TestFramework. - sampleDevicePluginDSYAML = "test/e2e/testing-manifests/sample-device-plugin.yaml" - // sampleDevicePluginName is the name of the device plugin pod - sampleDevicePluginName = "sample-device-plugin" -) - -var ( - appsScheme = runtime.NewScheme() - appsCodecs = serializer.NewCodecFactory(appsScheme) -) - -// NumberOfSampleResources returns the number of resources advertised by a node -func NumberOfSampleResources(node *v1.Node) int64 { - val, ok := node.Status.Capacity[sampleResourceName] - - if !ok { - return 0 - } - - return val.Value() -} - -// GetSampleDevicePluginPod returns the Device Plugin pod for sample resources in e2e tests -func GetSampleDevicePluginPod() *v1.Pod { - ds := ReadDaemonSetV1OrDie(testfiles.ReadOrDie(sampleDevicePluginDSYAML)) - p := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: sampleDevicePluginName, - Namespace: metav1.NamespaceSystem, - }, - - Spec: ds.Spec.Template.Spec, - } - - return p -} - -// ReadDaemonSetV1OrDie reads daemonset object from bytes. Panics on error. -func ReadDaemonSetV1OrDie(objBytes []byte) *appsv1.DaemonSet { - appsv1.AddToScheme(appsScheme) - requiredObj, err := runtime.Decode(appsCodecs.UniversalDecoder(appsv1.SchemeGroupVersion), objBytes) - if err != nil { - panic(err) - } - return requiredObj.(*appsv1.DaemonSet) -} diff --git a/test/e2e_node/BUILD b/test/e2e_node/BUILD index bab6ada0410..b78f8af9e85 100644 --- a/test/e2e_node/BUILD +++ b/test/e2e_node/BUILD @@ -167,6 +167,7 @@ go_test( "//pkg/kubelet/types:go_default_library", "//pkg/security/apparmor:go_default_library", "//pkg/volume/util/fsquota:go_default_library", + "//staging/src/k8s.io/api/apps/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/scheduling/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", @@ -174,7 +175,9 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", @@ -186,12 +189,12 @@ go_test( "//staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2:go_default_library", "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library", - "//test/e2e/framework/deviceplugin:go_default_library", "//test/e2e/framework/gpu:go_default_library", "//test/e2e/framework/kubectl:go_default_library", "//test/e2e/framework/metrics:go_default_library", "//test/e2e/framework/node:go_default_library", "//test/e2e/framework/pod:go_default_library", + "//test/e2e/framework/testfiles:go_default_library", "//test/e2e/framework/volume:go_default_library", "//test/e2e_node/perf/workloads:go_default_library", "//test/e2e_node/services:go_default_library", @@ -212,14 +215,12 @@ go_test( "@io_bazel_rules_go//go/platform:android": [ "//pkg/kubelet/stats/pidlimit:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/component-base/cli/flag:go_default_library", "//test/e2e/framework/config:go_default_library", "//test/e2e/framework/kubelet:go_default_library", "//test/e2e/framework/perf:go_default_library", - "//test/e2e/framework/testfiles:go_default_library", "//test/e2e/generated:go_default_library", "//vendor/github.com/onsi/ginkgo/config:go_default_library", "//vendor/github.com/onsi/ginkgo/reporters:go_default_library", @@ -229,14 +230,12 @@ go_test( "@io_bazel_rules_go//go/platform:linux": [ "//pkg/kubelet/stats/pidlimit:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/component-base/cli/flag:go_default_library", "//test/e2e/framework/config:go_default_library", "//test/e2e/framework/kubelet:go_default_library", "//test/e2e/framework/perf:go_default_library", - "//test/e2e/framework/testfiles:go_default_library", "//test/e2e/generated:go_default_library", "//vendor/github.com/onsi/ginkgo/config:go_default_library", "//vendor/github.com/onsi/ginkgo/reporters:go_default_library", diff --git a/test/e2e_node/device_plugin_test.go b/test/e2e_node/device_plugin_test.go index 03a87f27adb..41e23a8dcd6 100644 --- a/test/e2e_node/device_plugin_test.go +++ b/test/e2e_node/device_plugin_test.go @@ -20,9 +20,14 @@ import ( "path/filepath" "time" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/kubernetes/test/e2e/framework/testfiles" + "regexp" - "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" @@ -30,7 +35,6 @@ import ( kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" kubeletpodresourcesv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" "k8s.io/kubernetes/test/e2e/framework" - dputil "k8s.io/kubernetes/test/e2e/framework/deviceplugin" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" @@ -39,17 +43,65 @@ import ( ) const ( + // sampleResourceName is the name of the example resource which is used in the e2e test + sampleResourceName = "example.com/resource" + // sampleDevicePluginDSYAML is the path of the daemonset template of the sample device plugin. // TODO: Parametrize it by making it a feature in TestFramework. + sampleDevicePluginDSYAML = "test/e2e/testing-manifests/sample-device-plugin.yaml" + // sampleDevicePluginName is the name of the device plugin pod + sampleDevicePluginName = "sample-device-plugin" + // fake resource name resourceName = "example.com/resource" envVarNamePluginSockDir = "PLUGIN_SOCK_DIR" ) +var ( + appsScheme = runtime.NewScheme() + appsCodecs = serializer.NewCodecFactory(appsScheme) +) + // Serial because the test restarts Kubelet var _ = framework.KubeDescribe("Device Plugin [Feature:DevicePluginProbe][NodeFeature:DevicePluginProbe][Serial]", func() { f := framework.NewDefaultFramework("device-plugin-errors") testDevicePlugin(f, "/var/lib/kubelet/plugins_registry") }) +// numberOfSampleResources returns the number of resources advertised by a node. +func numberOfSampleResources(node *v1.Node) int64 { + val, ok := node.Status.Capacity[sampleResourceName] + + if !ok { + return 0 + } + + return val.Value() +} + +// getSampleDevicePluginPod returns the Device Plugin pod for sample resources in e2e tests. +func getSampleDevicePluginPod() *v1.Pod { + ds := readDaemonSetV1OrDie(testfiles.ReadOrDie(sampleDevicePluginDSYAML)) + p := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: sampleDevicePluginName, + Namespace: metav1.NamespaceSystem, + }, + + Spec: ds.Spec.Template.Spec, + } + + return p +} + +// readDaemonSetV1OrDie reads daemonset object from bytes. Panics on error. +func readDaemonSetV1OrDie(objBytes []byte) *appsv1.DaemonSet { + appsv1.AddToScheme(appsScheme) + requiredObj, err := runtime.Decode(appsCodecs.UniversalDecoder(appsv1.SchemeGroupVersion), objBytes) + if err != nil { + panic(err) + } + return requiredObj.(*appsv1.DaemonSet) +} + func testDevicePlugin(f *framework.Framework, pluginSockDir string) { pluginSockDir = filepath.Join(pluginSockDir) + "/" ginkgo.Context("DevicePlugin", func() { @@ -63,7 +115,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) { ginkgo.It("Verifies the Kubelet device plugin functionality.", func() { ginkgo.By("Wait for node is ready to start with") e2enode.WaitForNodeToBeReady(f.ClientSet, framework.TestContext.NodeName, 5*time.Minute) - dp := dputil.GetSampleDevicePluginPod() + dp := getSampleDevicePluginPod() for i := range dp.Spec.Containers[0].Env { if dp.Spec.Containers[0].Env[i].Name == envVarNamePluginSockDir { dp.Spec.Containers[0].Env[i].Value = pluginSockDir @@ -77,7 +129,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) { ginkgo.By("Waiting for devices to become available on the local node") gomega.Eventually(func() bool { - return dputil.NumberOfSampleResources(getLocalNode(f)) > 0 + return numberOfSampleResources(getLocalNode(f)) > 0 }, 5*time.Minute, framework.Poll).Should(gomega.BeTrue()) framework.Logf("Successfully created device plugin pod")