Merge pull request #86061 from haosdent/clean-e2e-framework-deployment

e2e: move funs of framework/deviceplugin to e2e_node

test/e2e/framework/BUILD

@@ -119,7 +119,6 @@ filegroup(
         "//test/e2e/framework/autoscaling:all-srcs",
         "//test/e2e/framework/config:all-srcs",
         "//test/e2e/framework/deployment:all-srcs",
-        "//test/e2e/framework/deviceplugin:all-srcs",
         "//test/e2e/framework/endpoints:all-srcs",
         "//test/e2e/framework/ginkgowrapper:all-srcs",
         "//test/e2e/framework/gpu:all-srcs",

test/e2e/framework/deviceplugin/BUILD

@@ -1,30 +0,0 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
-
-go_library(
-    name = "go_default_library",
-    srcs = ["device_plugin_util.go"],
-    importpath = "k8s.io/kubernetes/test/e2e/framework/deviceplugin",
-    visibility = ["//visibility:public"],
-    deps = [
-        "//staging/src/k8s.io/api/apps/v1:go_default_library",
-        "//staging/src/k8s.io/api/core/v1:go_default_library",
-        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
-        "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
-        "//test/e2e/framework/testfiles:go_default_library",
-    ],
-)
-
-filegroup(
-    name = "package-srcs",
-    srcs = glob(["**"]),
-    tags = ["automanaged"],
-    visibility = ["//visibility:private"],
-)
-
-filegroup(
-    name = "all-srcs",
-    srcs = [":package-srcs"],
-    tags = ["automanaged"],
-    visibility = ["//visibility:public"],
-)

test/e2e/framework/deviceplugin/device_plugin_util.go

@@ -1,76 +0,0 @@
-/*
-Copyright 2019 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package deviceplugin
-
-import (
-	appsv1 "k8s.io/api/apps/v1"
-	"k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/runtime/serializer"
-	"k8s.io/kubernetes/test/e2e/framework/testfiles"
-)
-
-const (
-	// sampleResourceName is the name of the example resource which is used in the e2e test
-	sampleResourceName = "example.com/resource"
-	// sampleDevicePluginDSYAML is the path of the daemonset template of the sample device plugin. // TODO: Parametrize it by making it a feature in TestFramework.
-	sampleDevicePluginDSYAML = "test/e2e/testing-manifests/sample-device-plugin.yaml"
-	// sampleDevicePluginName is the name of the device plugin pod
-	sampleDevicePluginName = "sample-device-plugin"
-)
-
-var (
-	appsScheme = runtime.NewScheme()
-	appsCodecs = serializer.NewCodecFactory(appsScheme)
-)
-
-// NumberOfSampleResources returns the number of resources advertised by a node
-func NumberOfSampleResources(node *v1.Node) int64 {
-	val, ok := node.Status.Capacity[sampleResourceName]
-
-	if !ok {
-		return 0
-	}
-
-	return val.Value()
-}
-
-// GetSampleDevicePluginPod returns the Device Plugin pod for sample resources in e2e tests
-func GetSampleDevicePluginPod() *v1.Pod {
-	ds := ReadDaemonSetV1OrDie(testfiles.ReadOrDie(sampleDevicePluginDSYAML))
-	p := &v1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      sampleDevicePluginName,
-			Namespace: metav1.NamespaceSystem,
-		},
-
-		Spec: ds.Spec.Template.Spec,
-	}
-
-	return p
-}
-
-// ReadDaemonSetV1OrDie reads daemonset object from bytes. Panics on error.
-func ReadDaemonSetV1OrDie(objBytes []byte) *appsv1.DaemonSet {
-	appsv1.AddToScheme(appsScheme)
-	requiredObj, err := runtime.Decode(appsCodecs.UniversalDecoder(appsv1.SchemeGroupVersion), objBytes)
-	if err != nil {
-		panic(err)
-	}
-	return requiredObj.(*appsv1.DaemonSet)
-}

test/e2e_node/BUILD

@@ -167,6 +167,7 @@ go_test(
         "//pkg/kubelet/types:go_default_library",
         "//pkg/security/apparmor:go_default_library",
         "//pkg/volume/util/fsquota:go_default_library",
+        "//staging/src/k8s.io/api/apps/v1:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/api/scheduling/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library",
@@ -174,7 +175,9 @@ go_test(
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
@@ -186,12 +189,12 @@ go_test(
         "//staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2:go_default_library",
         "//test/e2e/common:go_default_library",
         "//test/e2e/framework:go_default_library",
-        "//test/e2e/framework/deviceplugin:go_default_library",
         "//test/e2e/framework/gpu:go_default_library",
         "//test/e2e/framework/kubectl:go_default_library",
         "//test/e2e/framework/metrics:go_default_library",
         "//test/e2e/framework/node:go_default_library",
         "//test/e2e/framework/pod:go_default_library",
+        "//test/e2e/framework/testfiles:go_default_library",
         "//test/e2e/framework/volume:go_default_library",
         "//test/e2e_node/perf/workloads:go_default_library",
         "//test/e2e_node/services:go_default_library",
@@ -212,14 +215,12 @@ go_test(
         "@io_bazel_rules_go//go/platform:android": [
             "//pkg/kubelet/stats/pidlimit:go_default_library",
             "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
-            "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
             "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
             "//staging/src/k8s.io/client-go/tools/cache:go_default_library",
             "//staging/src/k8s.io/component-base/cli/flag:go_default_library",
             "//test/e2e/framework/config:go_default_library",
             "//test/e2e/framework/kubelet:go_default_library",
             "//test/e2e/framework/perf:go_default_library",
-            "//test/e2e/framework/testfiles:go_default_library",
             "//test/e2e/generated:go_default_library",
             "//vendor/github.com/onsi/ginkgo/config:go_default_library",
             "//vendor/github.com/onsi/ginkgo/reporters:go_default_library",
@@ -229,14 +230,12 @@ go_test(
         "@io_bazel_rules_go//go/platform:linux": [
             "//pkg/kubelet/stats/pidlimit:go_default_library",
             "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
-            "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
             "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
             "//staging/src/k8s.io/client-go/tools/cache:go_default_library",
             "//staging/src/k8s.io/component-base/cli/flag:go_default_library",
             "//test/e2e/framework/config:go_default_library",
             "//test/e2e/framework/kubelet:go_default_library",
             "//test/e2e/framework/perf:go_default_library",
-            "//test/e2e/framework/testfiles:go_default_library",
             "//test/e2e/generated:go_default_library",
             "//vendor/github.com/onsi/ginkgo/config:go_default_library",
             "//vendor/github.com/onsi/ginkgo/reporters:go_default_library",

test/e2e_node/device_plugin_test.go

@@ -20,9 +20,14 @@ import (
 	"path/filepath"
 	"time"
 
+	appsv1 "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/kubernetes/test/e2e/framework/testfiles"
+
 	"regexp"
 
-	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/uuid"
@@ -30,7 +35,6 @@ import (
 	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
 	kubeletpodresourcesv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1"
 	"k8s.io/kubernetes/test/e2e/framework"
-	dputil "k8s.io/kubernetes/test/e2e/framework/deviceplugin"
 	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 
@@ -39,17 +43,65 @@ import (
 )
 
 const (
+	// sampleResourceName is the name of the example resource which is used in the e2e test
+	sampleResourceName = "example.com/resource"
+	// sampleDevicePluginDSYAML is the path of the daemonset template of the sample device plugin. // TODO: Parametrize it by making it a feature in TestFramework.
+	sampleDevicePluginDSYAML = "test/e2e/testing-manifests/sample-device-plugin.yaml"
+	// sampleDevicePluginName is the name of the device plugin pod
+	sampleDevicePluginName = "sample-device-plugin"
+
 	// fake resource name
 	resourceName            = "example.com/resource"
 	envVarNamePluginSockDir = "PLUGIN_SOCK_DIR"
 )
 
+var (
+	appsScheme = runtime.NewScheme()
+	appsCodecs = serializer.NewCodecFactory(appsScheme)
+)
+
 // Serial because the test restarts Kubelet
 var _ = framework.KubeDescribe("Device Plugin [Feature:DevicePluginProbe][NodeFeature:DevicePluginProbe][Serial]", func() {
 	f := framework.NewDefaultFramework("device-plugin-errors")
 	testDevicePlugin(f, "/var/lib/kubelet/plugins_registry")
 })
 
+// numberOfSampleResources returns the number of resources advertised by a node.
+func numberOfSampleResources(node *v1.Node) int64 {
+	val, ok := node.Status.Capacity[sampleResourceName]
+
+	if !ok {
+		return 0
+	}
+
+	return val.Value()
+}
+
+// getSampleDevicePluginPod returns the Device Plugin pod for sample resources in e2e tests.
+func getSampleDevicePluginPod() *v1.Pod {
+	ds := readDaemonSetV1OrDie(testfiles.ReadOrDie(sampleDevicePluginDSYAML))
+	p := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      sampleDevicePluginName,
+			Namespace: metav1.NamespaceSystem,
+		},
+
+		Spec: ds.Spec.Template.Spec,
+	}
+
+	return p
+}
+
+// readDaemonSetV1OrDie reads daemonset object from bytes. Panics on error.
+func readDaemonSetV1OrDie(objBytes []byte) *appsv1.DaemonSet {
+	appsv1.AddToScheme(appsScheme)
+	requiredObj, err := runtime.Decode(appsCodecs.UniversalDecoder(appsv1.SchemeGroupVersion), objBytes)
+	if err != nil {
+		panic(err)
+	}
+	return requiredObj.(*appsv1.DaemonSet)
+}
+
 func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
 	pluginSockDir = filepath.Join(pluginSockDir) + "/"
 	ginkgo.Context("DevicePlugin", func() {
@@ -63,7 +115,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
 		ginkgo.It("Verifies the Kubelet device plugin functionality.", func() {
 			ginkgo.By("Wait for node is ready to start with")
 			e2enode.WaitForNodeToBeReady(f.ClientSet, framework.TestContext.NodeName, 5*time.Minute)
-			dp := dputil.GetSampleDevicePluginPod()
+			dp := getSampleDevicePluginPod()
 			for i := range dp.Spec.Containers[0].Env {
 				if dp.Spec.Containers[0].Env[i].Name == envVarNamePluginSockDir {
 					dp.Spec.Containers[0].Env[i].Value = pluginSockDir
@@ -77,7 +129,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
 
 			ginkgo.By("Waiting for devices to become available on the local node")
 			gomega.Eventually(func() bool {
-				return dputil.NumberOfSampleResources(getLocalNode(f)) > 0
+				return numberOfSampleResources(getLocalNode(f)) > 0
 			}, 5*time.Minute, framework.Poll).Should(gomega.BeTrue())
 			framework.Logf("Successfully created device plugin pod")