Merge pull request #67928 from vikaschoudhary16/e2e-dp

Add stub device plugin for e2e tests

test/e2e/framework/BUILD

@@ -138,6 +138,7 @@ filegroup(
         "//test/e2e/framework/auth:all-srcs",
         "//test/e2e/framework/config:all-srcs",
         "//test/e2e/framework/deployment:all-srcs",
+        "//test/e2e/framework/deviceplugin:all-srcs",
         "//test/e2e/framework/endpoints:all-srcs",
         "//test/e2e/framework/ginkgowrapper:all-srcs",
         "//test/e2e/framework/gpu:all-srcs",

test/e2e/framework/deviceplugin/BUILD

@@ -0,0 +1,31 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["device_plugin_util.go"],
+    importpath = "k8s.io/kubernetes/test/e2e/framework/deviceplugin",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//staging/src/k8s.io/api/apps/v1:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
+        "//test/e2e/framework/testfiles:go_default_library",
+        "//vendor/github.com/onsi/ginkgo:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

test/e2e/framework/deviceplugin/device_plugin_util.go

@@ -0,0 +1,78 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package deviceplugin
+
+import (
+	appsv1 "k8s.io/api/apps/v1"
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/kubernetes/test/e2e/framework/testfiles"
+
+	"github.com/onsi/ginkgo"
+)
+
+const (
+	// sampleResourceName is the name of the example resource which is used in the e2e test
+	sampleResourceName = "example.com/resource"
+	// sampleDevicePluginDSYAML is the path of the daemonset template of the sample device plugin. // TODO: Parametrize it by making it a feature in TestFramework.
+	sampleDevicePluginDSYAML = "test/e2e/testing-manifests/sample-device-plugin.yaml"
+	// sampleDevicePluginName is the name of the device plugin pod
+	sampleDevicePluginName = "sample-device-plugin"
+)
+
+var (
+	appsScheme = runtime.NewScheme()
+	appsCodecs = serializer.NewCodecFactory(appsScheme)
+)
+
+// NumberOfSampleResources returns the number of resources advertised by a node
+func NumberOfSampleResources(node *v1.Node) int64 {
+	val, ok := node.Status.Capacity[sampleResourceName]
+
+	if !ok {
+		return 0
+	}
+
+	return val.Value()
+}
+
+// GetSampleDevicePluginPod returns the Device Plugin pod for sample resources in e2e tests
+func GetSampleDevicePluginPod() *v1.Pod {
+	ds := ReadDaemonSetV1OrDie(testfiles.ReadOrDie(sampleDevicePluginDSYAML, ginkgo.Fail))
+	p := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      sampleDevicePluginName,
+			Namespace: metav1.NamespaceSystem,
+		},
+
+		Spec: ds.Spec.Template.Spec,
+	}
+
+	return p
+}
+
+// ReadDaemonSetV1OrDie reads daemonset object from bytes. Panics on error.
+func ReadDaemonSetV1OrDie(objBytes []byte) *appsv1.DaemonSet {
+	appsv1.AddToScheme(appsScheme)
+	requiredObj, err := runtime.Decode(appsCodecs.UniversalDecoder(appsv1.SchemeGroupVersion), objBytes)
+	if err != nil {
+		panic(err)
+	}
+	return requiredObj.(*appsv1.DaemonSet)
+}

test/e2e/testing-manifests/sample-device-plugin.yaml

@@ -0,0 +1,51 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: sample-device-plugin-beta
+  namespace: kube-system
+  labels:
+    k8s-app: sample-device-plugin
+spec:
+  selector:
+    matchLabels:
+      k8s-app: sample-device-plugin
+  template:
+    metadata:
+      labels:
+        k8s-app: sample-device-plugin
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+      priorityClassName: system-node-critical
+      tolerations:
+      - operator: "Exists"
+        effect: "NoExecute"
+      - operator: "Exists"
+        effect: "NoSchedule"
+      volumes:
+      - name: device-plugin
+        hostPath:
+          path: /var/lib/kubelet/device-plugins
+      - name: plugins-registry-probe-mode
+        hostPath:
+          path: /var/lib/kubelet/plugins_registry 
+      - name: dev
+        hostPath:
+          path: /dev
+      containers:
+      - image: gcr.io/kubernetes-e2e-test-images/sample-device-plugin:1.0 
+        name: sample-device-plugin
+        env:
+        - name: PLUGIN_SOCK_DIR
+          value: "/var/lib/kubelet/device-plugins"
+        securityContext:
+           privileged: true
+        volumeMounts:
+        - name: device-plugin
+          mountPath: /var/lib/kubelet/device-plugins
+        - name: plugins-registry-probe-mode
+          mountPath: /var/lib/kubelet/plugins_registry
+        - name: dev
+          mountPath: /dev
+  updateStrategy:
+    type: RollingUpdate

test/e2e_node/BUILD

@@ -27,12 +27,10 @@ go_library(
     deps = [
         "//pkg/features:go_default_library",
         "//pkg/kubelet/apis/config:go_default_library",
-        "//pkg/kubelet/apis/deviceplugin/v1beta1:go_default_library",
         "//pkg/kubelet/apis/podresources:go_default_library",
         "//pkg/kubelet/apis/podresources/v1alpha1:go_default_library",
         "//pkg/kubelet/apis/stats/v1alpha1:go_default_library",
         "//pkg/kubelet/cm:go_default_library",
-        "//pkg/kubelet/cm/devicemanager:go_default_library",
         "//pkg/kubelet/kubeletconfig/util/codec:go_default_library",
         "//pkg/kubelet/metrics:go_default_library",
         "//pkg/kubelet/remote:go_default_library",
@@ -51,9 +49,11 @@ go_library(
         "//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library",
         "//test/e2e/common:go_default_library",
         "//test/e2e/framework:go_default_library",
+        "//test/e2e/framework/deviceplugin:go_default_library",
         "//test/e2e/framework/gpu:go_default_library",
         "//test/e2e/framework/log:go_default_library",
         "//test/e2e/framework/metrics:go_default_library",
+        "//test/e2e/framework/node:go_default_library",
         "//test/e2e/framework/pod:go_default_library",
         "//test/utils/image:go_default_library",
         "//vendor/github.com/blang/semver:go_default_library",
@@ -192,6 +192,8 @@ go_test(
             "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
             "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
             "//staging/src/k8s.io/client-go/tools/cache:go_default_library",
+            "//test/e2e/framework/testfiles:go_default_library",
+            "//test/e2e/generated:go_default_library",
             "//vendor/github.com/kardianos/osext:go_default_library",
             "//vendor/github.com/onsi/ginkgo/config:go_default_library",
             "//vendor/github.com/onsi/ginkgo/reporters:go_default_library",

test/e2e_node/device_plugin.go

@@ -17,8 +17,6 @@ limitations under the License.
 package e2e_node
 
 import (
-	"fmt"
-	"os"
 	"path/filepath"
 	"time"
 
@@ -31,11 +29,12 @@ import (
 	"k8s.io/kubernetes/pkg/features"
 	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
 	"k8s.io/kubernetes/test/e2e/framework"
+	dputil "k8s.io/kubernetes/test/e2e/framework/deviceplugin"
 	e2elog "k8s.io/kubernetes/test/e2e/framework/log"
+	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 
-	kubeletdevicepluginv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
+	resapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1"
-	dm "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -43,7 +42,8 @@ import (
 
 const (
 	// fake resource name
-	resourceName = "fake.com/resource"
+	resourceName            = "example.com/resource"
+	envVarNamePluginSockDir = "PLUGIN_SOCK_DIR"
 )
 
 // Serial because the test restarts Kubelet
@@ -63,27 +63,30 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
 			initialConfig.FeatureGates[string(features.KubeletPodResources)] = true
 		})
 		It("Verifies the Kubelet device plugin functionality.", func() {
-			By("Start stub device plugin")
+			By("Wait for node is ready to start with")
-			// fake devices for e2e test
+			e2enode.WaitForNodeToBeReady(f.ClientSet, framework.TestContext.NodeName, 5*time.Minute)
-			devs := []*kubeletdevicepluginv1beta1.Device{
+			dp := dputil.GetSampleDevicePluginPod()
-				{ID: "Dev-1", Health: kubeletdevicepluginv1beta1.Healthy},
+			for i := range dp.Spec.Containers[0].Env {
-				{ID: "Dev-2", Health: kubeletdevicepluginv1beta1.Healthy},
+				if dp.Spec.Containers[0].Env[i].Name == envVarNamePluginSockDir {
+					dp.Spec.Containers[0].Env[i].Value = pluginSockDir
+				}
 			}
-
+			e2elog.Logf("env %v", dp.Spec.Containers[0].Env)
-			socketPath := pluginSockDir + "dp." + fmt.Sprintf("%d", time.Now().Unix())
+			dp.Spec.NodeName = framework.TestContext.NodeName
-			e2elog.Logf("socketPath %v", socketPath)
+			By("Create sample device plugin pod")
-
+			devicePluginPod, err := f.ClientSet.CoreV1().Pods(metav1.NamespaceSystem).Create(dp)
-			dp1 := dm.NewDevicePluginStub(devs, socketPath, resourceName, false)
-			dp1.SetAllocFunc(stubAllocFunc)
-			err := dp1.Start()
 			framework.ExpectNoError(err)
 
-			By("Register resources")
+			By("Waiting for devices to become available on the local node")
-			err = dp1.Register(kubeletdevicepluginv1beta1.KubeletSocket, resourceName, pluginSockDir)
+			Eventually(func() bool {
-			framework.ExpectNoError(err)
+				return dputil.NumberOfSampleResources(getLocalNode(f)) > 0
+			}, 5*time.Minute, framework.Poll).Should(BeTrue())
+			e2elog.Logf("Successfully created device plugin pod")
 
-			By("Waiting for the resource exported by the stub device plugin to become available on the local node")
+			By("Waiting for the resource exported by the sample device plugin to become available on the local node")
-			devsLen := int64(len(devs))
+			// TODO(vikasc): Instead of hard-coding number of devices, provide number of devices in the sample-device-plugin using configmap
+			// and then use the same here
+			devsLen := int64(2)
 			Eventually(func() bool {
 				node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{})
 				framework.ExpectNoError(err)
@@ -99,15 +102,24 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
 			Expect(devId1).To(Not(Equal("")))
 
 			podResources, err := getNodeDevices()
+			var resourcesForOurPod *resapi.PodResources
+			e2elog.Logf("pod resources %v", podResources)
 			Expect(err).To(BeNil())
-			Expect(len(podResources.PodResources)).To(Equal(1))
+			Expect(len(podResources.PodResources)).To(Equal(2))
-			Expect(podResources.PodResources[0].Name).To(Equal(pod1.Name))
+			for _, res := range podResources.GetPodResources() {
-			Expect(podResources.PodResources[0].Namespace).To(Equal(pod1.Namespace))
+				if res.Name == pod1.Name {
-			Expect(len(podResources.PodResources[0].Containers)).To(Equal(1))
+					resourcesForOurPod = res
-			Expect(podResources.PodResources[0].Containers[0].Name).To(Equal(pod1.Spec.Containers[0].Name))
+				}
-			Expect(len(podResources.PodResources[0].Containers[0].Devices)).To(Equal(1))
+			}
-			Expect(podResources.PodResources[0].Containers[0].Devices[0].ResourceName).To(Equal(resourceName))
+			e2elog.Logf("resourcesForOurPod %v", resourcesForOurPod)
-			Expect(len(podResources.PodResources[0].Containers[0].Devices[0].DeviceIds)).To(Equal(1))
+			Expect(resourcesForOurPod).NotTo(BeNil())
+			Expect(resourcesForOurPod.Name).To(Equal(pod1.Name))
+			Expect(resourcesForOurPod.Namespace).To(Equal(pod1.Namespace))
+			Expect(len(resourcesForOurPod.Containers)).To(Equal(1))
+			Expect(resourcesForOurPod.Containers[0].Name).To(Equal(pod1.Spec.Containers[0].Name))
+			Expect(len(resourcesForOurPod.Containers[0].Devices)).To(Equal(1))
+			Expect(resourcesForOurPod.Containers[0].Devices[0].ResourceName).To(Equal(resourceName))
+			Expect(len(resourcesForOurPod.Containers[0].Devices[0].DeviceIds)).To(Equal(1))
 
 			pod1, err = f.PodClient().Get(pod1.Name, metav1.GetOptions{})
 			framework.ExpectNoError(err)
@@ -136,13 +148,20 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
 				return false
 			}, 5*time.Minute, framework.Poll).Should(BeTrue())
 
-			By("Re-Register resources")
+			By("Re-Register resources and deleting the pods and waiting for container removal")
-			dp1 = dm.NewDevicePluginStub(devs, socketPath, resourceName, false)
+			getOptions := metav1.GetOptions{}
-			dp1.SetAllocFunc(stubAllocFunc)
+			gp := int64(0)
-			err = dp1.Start()
+			deleteOptions := metav1.DeleteOptions{
+				GracePeriodSeconds: &gp,
+			}
+			err = f.ClientSet.CoreV1().Pods(metav1.NamespaceSystem).Delete(dp.Name, &deleteOptions)
 			framework.ExpectNoError(err)
+			waitForContainerRemoval(devicePluginPod.Spec.Containers[0].Name, devicePluginPod.Name, devicePluginPod.Namespace)
+			_, err = f.ClientSet.CoreV1().Pods(metav1.NamespaceSystem).Get(dp.Name, getOptions)
+			e2elog.Logf("Trying to get dp pod after deletion. err must be non-nil. err: %v", err)
+			framework.ExpectError(err)
 
-			err = dp1.Register(kubeletdevicepluginv1beta1.KubeletSocket, resourceName, pluginSockDir)
+			devicePluginPod, err = f.ClientSet.CoreV1().Pods(metav1.NamespaceSystem).Create(dp)
 			framework.ExpectNoError(err)
 
 			ensurePodContainerRestart(f, pod1.Name, pod1.Name)
@@ -166,9 +185,10 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
 
 			Expect(devId1).To(Not(Equal(devId2)))
 
-			By("Deleting device plugin.")
+			By("By deleting the pods and waiting for container removal")
-			err = dp1.Stop()
+			err = f.ClientSet.CoreV1().Pods(metav1.NamespaceSystem).Delete(dp.Name, &deleteOptions)
 			framework.ExpectNoError(err)
+			waitForContainerRemoval(devicePluginPod.Spec.Containers[0].Name, devicePluginPod.Name, devicePluginPod.Namespace)
 
 			By("Waiting for stub device plugin to become unhealthy on the local node")
 			Eventually(func() int64 {
@@ -187,12 +207,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
 			Expect(devIdRestart2).To(Equal(devId2))
 
 			By("Re-register resources")
-			dp1 = dm.NewDevicePluginStub(devs, socketPath, resourceName, false)
+			devicePluginPod, err = f.ClientSet.CoreV1().Pods(metav1.NamespaceSystem).Create(dp)
-			dp1.SetAllocFunc(stubAllocFunc)
-			err = dp1.Start()
-			framework.ExpectNoError(err)
-
-			err = dp1.Register(kubeletdevicepluginv1beta1.KubeletSocket, resourceName, pluginSockDir)
 			framework.ExpectNoError(err)
 
 			By("Waiting for the resource exported by the stub device plugin to become healthy on the local node")
@@ -202,9 +217,10 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) {
 				return numberOfDevicesAllocatable(node, resourceName)
 			}, 30*time.Second, framework.Poll).Should(Equal(devsLen))
 
-			By("Deleting device plugin again.")
+			By("by deleting the pods and waiting for container removal")
-			err = dp1.Stop()
+			err = f.ClientSet.CoreV1().Pods(metav1.NamespaceSystem).Delete(dp.Name, &deleteOptions)
 			framework.ExpectNoError(err)
+			waitForContainerRemoval(devicePluginPod.Spec.Containers[0].Name, devicePluginPod.Name, devicePluginPod.Namespace)
 
 			By("Waiting for stub device plugin to become unavailable on the local node")
 			Eventually(func() bool {
@@ -300,41 +316,3 @@ func numberOfDevicesAllocatable(node *v1.Node, resourceName string) int64 {
 
 	return val.Value()
 }
-
-// stubAllocFunc will pass to stub device plugin
-func stubAllocFunc(r *kubeletdevicepluginv1beta1.AllocateRequest, devs map[string]kubeletdevicepluginv1beta1.Device) (*kubeletdevicepluginv1beta1.AllocateResponse, error) {
-	var responses kubeletdevicepluginv1beta1.AllocateResponse
-	for _, req := range r.ContainerRequests {
-		response := &kubeletdevicepluginv1beta1.ContainerAllocateResponse{}
-		for _, requestID := range req.DevicesIDs {
-			dev, ok := devs[requestID]
-			if !ok {
-				return nil, fmt.Errorf("invalid allocation request with non-existing device %s", requestID)
-			}
-
-			if dev.Health != kubeletdevicepluginv1beta1.Healthy {
-				return nil, fmt.Errorf("invalid allocation request with unhealthy device: %s", requestID)
-			}
-
-			// create fake device file
-			fpath := filepath.Join("/tmp", dev.ID)
-
-			// clean first
-			os.RemoveAll(fpath)
-			f, err := os.Create(fpath)
-			if err != nil && !os.IsExist(err) {
-				return nil, fmt.Errorf("failed to create fake device file: %s", err)
-			}
-
-			f.Close()
-
-			response.Mounts = append(response.Mounts, &kubeletdevicepluginv1beta1.Mount{
-				ContainerPath: fpath,
-				HostPath:      fpath,
-			})
-		}
-		responses.ContainerResponses = append(responses.ContainerResponses, response)
-	}
-
-	return &responses, nil
-}

test/e2e_node/e2e_node_suite_test.go

@@ -41,6 +41,8 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/system"
 	commontest "k8s.io/kubernetes/test/e2e/common"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/testfiles"
+	"k8s.io/kubernetes/test/e2e/generated"
 	"k8s.io/kubernetes/test/e2e_node/services"
 
 	"github.com/kardianos/osext"
@@ -71,6 +73,13 @@ func init() {
 	// TODO(random-liu): Find who is using flag.Parse() and cause errors and move the following logic
 	// into TestContext.
 	// TODO(pohly): remove RegisterNodeFlags from test_context.go enable Viper config support here?
+
+	// Enable bindata file lookup as fallback.
+	testfiles.AddFileSource(testfiles.BindataFileSource{
+		Asset:      generated.Asset,
+		AssetNames: generated.AssetNames,
+	})
+
 }
 
 func TestMain(m *testing.M) {