diff --git a/test/e2e/framework/BUILD b/test/e2e/framework/BUILD index daa7b15dd91..eb4bb4f8afa 100644 --- a/test/e2e/framework/BUILD +++ b/test/e2e/framework/BUILD @@ -138,6 +138,7 @@ filegroup( "//test/e2e/framework/auth:all-srcs", "//test/e2e/framework/config:all-srcs", "//test/e2e/framework/deployment:all-srcs", + "//test/e2e/framework/deviceplugin:all-srcs", "//test/e2e/framework/endpoints:all-srcs", "//test/e2e/framework/ginkgowrapper:all-srcs", "//test/e2e/framework/gpu:all-srcs", diff --git a/test/e2e/framework/deviceplugin/BUILD b/test/e2e/framework/deviceplugin/BUILD new file mode 100644 index 00000000000..285fb5ae3d5 --- /dev/null +++ b/test/e2e/framework/deviceplugin/BUILD @@ -0,0 +1,31 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["device_plugin_util.go"], + importpath = "k8s.io/kubernetes/test/e2e/framework/deviceplugin", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/api/apps/v1:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", + "//test/e2e/framework/testfiles:go_default_library", + "//vendor/github.com/onsi/ginkgo:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/test/e2e/framework/deviceplugin/device_plugin_util.go b/test/e2e/framework/deviceplugin/device_plugin_util.go new file mode 100644 index 00000000000..0771bd58d9f --- /dev/null +++ b/test/e2e/framework/deviceplugin/device_plugin_util.go @@ -0,0 +1,78 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package deviceplugin + +import ( + appsv1 "k8s.io/api/apps/v1" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/kubernetes/test/e2e/framework/testfiles" + + "github.com/onsi/ginkgo" +) + +const ( + // sampleResourceName is the name of the example resource which is used in the e2e test + sampleResourceName = "example.com/resource" + // sampleDevicePluginDSYAML is the path of the daemonset template of the sample device plugin. // TODO: Parametrize it by making it a feature in TestFramework. + sampleDevicePluginDSYAML = "test/e2e/testing-manifests/sample-device-plugin.yaml" + // sampleDevicePluginName is the name of the device plugin pod + sampleDevicePluginName = "sample-device-plugin" +) + +var ( + appsScheme = runtime.NewScheme() + appsCodecs = serializer.NewCodecFactory(appsScheme) +) + +// NumberOfSampleResources returns the number of resources advertised by a node +func NumberOfSampleResources(node *v1.Node) int64 { + val, ok := node.Status.Capacity[sampleResourceName] + + if !ok { + return 0 + } + + return val.Value() +} + +// GetSampleDevicePluginPod returns the Device Plugin pod for sample resources in e2e tests +func GetSampleDevicePluginPod() *v1.Pod { + ds := ReadDaemonSetV1OrDie(testfiles.ReadOrDie(sampleDevicePluginDSYAML, ginkgo.Fail)) + p := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: sampleDevicePluginName, + Namespace: metav1.NamespaceSystem, + }, + + Spec: ds.Spec.Template.Spec, + } + + return p +} + +// ReadDaemonSetV1OrDie reads daemonset object from bytes. Panics on error. +func ReadDaemonSetV1OrDie(objBytes []byte) *appsv1.DaemonSet { + appsv1.AddToScheme(appsScheme) + requiredObj, err := runtime.Decode(appsCodecs.UniversalDecoder(appsv1.SchemeGroupVersion), objBytes) + if err != nil { + panic(err) + } + return requiredObj.(*appsv1.DaemonSet) +} diff --git a/test/e2e/testing-manifests/sample-device-plugin.yaml b/test/e2e/testing-manifests/sample-device-plugin.yaml new file mode 100644 index 00000000000..1c7baff5eb0 --- /dev/null +++ b/test/e2e/testing-manifests/sample-device-plugin.yaml @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: sample-device-plugin-beta + namespace: kube-system + labels: + k8s-app: sample-device-plugin +spec: + selector: + matchLabels: + k8s-app: sample-device-plugin + template: + metadata: + labels: + k8s-app: sample-device-plugin + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + priorityClassName: system-node-critical + tolerations: + - operator: "Exists" + effect: "NoExecute" + - operator: "Exists" + effect: "NoSchedule" + volumes: + - name: device-plugin + hostPath: + path: /var/lib/kubelet/device-plugins + - name: plugins-registry-probe-mode + hostPath: + path: /var/lib/kubelet/plugins_registry + - name: dev + hostPath: + path: /dev + containers: + - image: gcr.io/kubernetes-e2e-test-images/sample-device-plugin:1.0 + name: sample-device-plugin + env: + - name: PLUGIN_SOCK_DIR + value: "/var/lib/kubelet/device-plugins" + securityContext: + privileged: true + volumeMounts: + - name: device-plugin + mountPath: /var/lib/kubelet/device-plugins + - name: plugins-registry-probe-mode + mountPath: /var/lib/kubelet/plugins_registry + - name: dev + mountPath: /dev + updateStrategy: + type: RollingUpdate diff --git a/test/e2e_node/BUILD b/test/e2e_node/BUILD index 5de952b2b3e..44509dce946 100644 --- a/test/e2e_node/BUILD +++ b/test/e2e_node/BUILD @@ -27,12 +27,10 @@ go_library( deps = [ "//pkg/features:go_default_library", "//pkg/kubelet/apis/config:go_default_library", - "//pkg/kubelet/apis/deviceplugin/v1beta1:go_default_library", "//pkg/kubelet/apis/podresources:go_default_library", "//pkg/kubelet/apis/podresources/v1alpha1:go_default_library", "//pkg/kubelet/apis/stats/v1alpha1:go_default_library", "//pkg/kubelet/cm:go_default_library", - "//pkg/kubelet/cm/devicemanager:go_default_library", "//pkg/kubelet/kubeletconfig/util/codec:go_default_library", "//pkg/kubelet/metrics:go_default_library", "//pkg/kubelet/remote:go_default_library", @@ -51,9 +49,11 @@ go_library( "//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library", "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library", + "//test/e2e/framework/deviceplugin:go_default_library", "//test/e2e/framework/gpu:go_default_library", "//test/e2e/framework/log:go_default_library", "//test/e2e/framework/metrics:go_default_library", + "//test/e2e/framework/node:go_default_library", "//test/e2e/framework/pod:go_default_library", "//test/utils/image:go_default_library", "//vendor/github.com/blang/semver:go_default_library", @@ -192,6 +192,8 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", + "//test/e2e/framework/testfiles:go_default_library", + "//test/e2e/generated:go_default_library", "//vendor/github.com/kardianos/osext:go_default_library", "//vendor/github.com/onsi/ginkgo/config:go_default_library", "//vendor/github.com/onsi/ginkgo/reporters:go_default_library", diff --git a/test/e2e_node/device_plugin.go b/test/e2e_node/device_plugin.go index 43a9eaa6e66..2fd6cba712a 100644 --- a/test/e2e_node/device_plugin.go +++ b/test/e2e_node/device_plugin.go @@ -17,8 +17,6 @@ limitations under the License. package e2e_node import ( - "fmt" - "os" "path/filepath" "time" @@ -31,11 +29,12 @@ import ( "k8s.io/kubernetes/pkg/features" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" "k8s.io/kubernetes/test/e2e/framework" + dputil "k8s.io/kubernetes/test/e2e/framework/deviceplugin" e2elog "k8s.io/kubernetes/test/e2e/framework/log" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" - kubeletdevicepluginv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1" - dm "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager" + resapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -43,7 +42,8 @@ import ( const ( // fake resource name - resourceName = "fake.com/resource" + resourceName = "example.com/resource" + envVarNamePluginSockDir = "PLUGIN_SOCK_DIR" ) // Serial because the test restarts Kubelet @@ -63,27 +63,30 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) { initialConfig.FeatureGates[string(features.KubeletPodResources)] = true }) It("Verifies the Kubelet device plugin functionality.", func() { - By("Start stub device plugin") - // fake devices for e2e test - devs := []*kubeletdevicepluginv1beta1.Device{ - {ID: "Dev-1", Health: kubeletdevicepluginv1beta1.Healthy}, - {ID: "Dev-2", Health: kubeletdevicepluginv1beta1.Healthy}, + By("Wait for node is ready to start with") + e2enode.WaitForNodeToBeReady(f.ClientSet, framework.TestContext.NodeName, 5*time.Minute) + dp := dputil.GetSampleDevicePluginPod() + for i := range dp.Spec.Containers[0].Env { + if dp.Spec.Containers[0].Env[i].Name == envVarNamePluginSockDir { + dp.Spec.Containers[0].Env[i].Value = pluginSockDir + } } - - socketPath := pluginSockDir + "dp." + fmt.Sprintf("%d", time.Now().Unix()) - e2elog.Logf("socketPath %v", socketPath) - - dp1 := dm.NewDevicePluginStub(devs, socketPath, resourceName, false) - dp1.SetAllocFunc(stubAllocFunc) - err := dp1.Start() + e2elog.Logf("env %v", dp.Spec.Containers[0].Env) + dp.Spec.NodeName = framework.TestContext.NodeName + By("Create sample device plugin pod") + devicePluginPod, err := f.ClientSet.CoreV1().Pods(metav1.NamespaceSystem).Create(dp) framework.ExpectNoError(err) - By("Register resources") - err = dp1.Register(kubeletdevicepluginv1beta1.KubeletSocket, resourceName, pluginSockDir) - framework.ExpectNoError(err) + By("Waiting for devices to become available on the local node") + Eventually(func() bool { + return dputil.NumberOfSampleResources(getLocalNode(f)) > 0 + }, 5*time.Minute, framework.Poll).Should(BeTrue()) + e2elog.Logf("Successfully created device plugin pod") - By("Waiting for the resource exported by the stub device plugin to become available on the local node") - devsLen := int64(len(devs)) + By("Waiting for the resource exported by the sample device plugin to become available on the local node") + // TODO(vikasc): Instead of hard-coding number of devices, provide number of devices in the sample-device-plugin using configmap + // and then use the same here + devsLen := int64(2) Eventually(func() bool { node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{}) framework.ExpectNoError(err) @@ -99,15 +102,24 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) { Expect(devId1).To(Not(Equal(""))) podResources, err := getNodeDevices() + var resourcesForOurPod *resapi.PodResources + e2elog.Logf("pod resources %v", podResources) Expect(err).To(BeNil()) - Expect(len(podResources.PodResources)).To(Equal(1)) - Expect(podResources.PodResources[0].Name).To(Equal(pod1.Name)) - Expect(podResources.PodResources[0].Namespace).To(Equal(pod1.Namespace)) - Expect(len(podResources.PodResources[0].Containers)).To(Equal(1)) - Expect(podResources.PodResources[0].Containers[0].Name).To(Equal(pod1.Spec.Containers[0].Name)) - Expect(len(podResources.PodResources[0].Containers[0].Devices)).To(Equal(1)) - Expect(podResources.PodResources[0].Containers[0].Devices[0].ResourceName).To(Equal(resourceName)) - Expect(len(podResources.PodResources[0].Containers[0].Devices[0].DeviceIds)).To(Equal(1)) + Expect(len(podResources.PodResources)).To(Equal(2)) + for _, res := range podResources.GetPodResources() { + if res.Name == pod1.Name { + resourcesForOurPod = res + } + } + e2elog.Logf("resourcesForOurPod %v", resourcesForOurPod) + Expect(resourcesForOurPod).NotTo(BeNil()) + Expect(resourcesForOurPod.Name).To(Equal(pod1.Name)) + Expect(resourcesForOurPod.Namespace).To(Equal(pod1.Namespace)) + Expect(len(resourcesForOurPod.Containers)).To(Equal(1)) + Expect(resourcesForOurPod.Containers[0].Name).To(Equal(pod1.Spec.Containers[0].Name)) + Expect(len(resourcesForOurPod.Containers[0].Devices)).To(Equal(1)) + Expect(resourcesForOurPod.Containers[0].Devices[0].ResourceName).To(Equal(resourceName)) + Expect(len(resourcesForOurPod.Containers[0].Devices[0].DeviceIds)).To(Equal(1)) pod1, err = f.PodClient().Get(pod1.Name, metav1.GetOptions{}) framework.ExpectNoError(err) @@ -136,13 +148,20 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) { return false }, 5*time.Minute, framework.Poll).Should(BeTrue()) - By("Re-Register resources") - dp1 = dm.NewDevicePluginStub(devs, socketPath, resourceName, false) - dp1.SetAllocFunc(stubAllocFunc) - err = dp1.Start() + By("Re-Register resources and deleting the pods and waiting for container removal") + getOptions := metav1.GetOptions{} + gp := int64(0) + deleteOptions := metav1.DeleteOptions{ + GracePeriodSeconds: &gp, + } + err = f.ClientSet.CoreV1().Pods(metav1.NamespaceSystem).Delete(dp.Name, &deleteOptions) framework.ExpectNoError(err) + waitForContainerRemoval(devicePluginPod.Spec.Containers[0].Name, devicePluginPod.Name, devicePluginPod.Namespace) + _, err = f.ClientSet.CoreV1().Pods(metav1.NamespaceSystem).Get(dp.Name, getOptions) + e2elog.Logf("Trying to get dp pod after deletion. err must be non-nil. err: %v", err) + framework.ExpectError(err) - err = dp1.Register(kubeletdevicepluginv1beta1.KubeletSocket, resourceName, pluginSockDir) + devicePluginPod, err = f.ClientSet.CoreV1().Pods(metav1.NamespaceSystem).Create(dp) framework.ExpectNoError(err) ensurePodContainerRestart(f, pod1.Name, pod1.Name) @@ -166,9 +185,10 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) { Expect(devId1).To(Not(Equal(devId2))) - By("Deleting device plugin.") - err = dp1.Stop() + By("By deleting the pods and waiting for container removal") + err = f.ClientSet.CoreV1().Pods(metav1.NamespaceSystem).Delete(dp.Name, &deleteOptions) framework.ExpectNoError(err) + waitForContainerRemoval(devicePluginPod.Spec.Containers[0].Name, devicePluginPod.Name, devicePluginPod.Namespace) By("Waiting for stub device plugin to become unhealthy on the local node") Eventually(func() int64 { @@ -187,12 +207,7 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) { Expect(devIdRestart2).To(Equal(devId2)) By("Re-register resources") - dp1 = dm.NewDevicePluginStub(devs, socketPath, resourceName, false) - dp1.SetAllocFunc(stubAllocFunc) - err = dp1.Start() - framework.ExpectNoError(err) - - err = dp1.Register(kubeletdevicepluginv1beta1.KubeletSocket, resourceName, pluginSockDir) + devicePluginPod, err = f.ClientSet.CoreV1().Pods(metav1.NamespaceSystem).Create(dp) framework.ExpectNoError(err) By("Waiting for the resource exported by the stub device plugin to become healthy on the local node") @@ -202,9 +217,10 @@ func testDevicePlugin(f *framework.Framework, pluginSockDir string) { return numberOfDevicesAllocatable(node, resourceName) }, 30*time.Second, framework.Poll).Should(Equal(devsLen)) - By("Deleting device plugin again.") - err = dp1.Stop() + By("by deleting the pods and waiting for container removal") + err = f.ClientSet.CoreV1().Pods(metav1.NamespaceSystem).Delete(dp.Name, &deleteOptions) framework.ExpectNoError(err) + waitForContainerRemoval(devicePluginPod.Spec.Containers[0].Name, devicePluginPod.Name, devicePluginPod.Namespace) By("Waiting for stub device plugin to become unavailable on the local node") Eventually(func() bool { @@ -300,41 +316,3 @@ func numberOfDevicesAllocatable(node *v1.Node, resourceName string) int64 { return val.Value() } - -// stubAllocFunc will pass to stub device plugin -func stubAllocFunc(r *kubeletdevicepluginv1beta1.AllocateRequest, devs map[string]kubeletdevicepluginv1beta1.Device) (*kubeletdevicepluginv1beta1.AllocateResponse, error) { - var responses kubeletdevicepluginv1beta1.AllocateResponse - for _, req := range r.ContainerRequests { - response := &kubeletdevicepluginv1beta1.ContainerAllocateResponse{} - for _, requestID := range req.DevicesIDs { - dev, ok := devs[requestID] - if !ok { - return nil, fmt.Errorf("invalid allocation request with non-existing device %s", requestID) - } - - if dev.Health != kubeletdevicepluginv1beta1.Healthy { - return nil, fmt.Errorf("invalid allocation request with unhealthy device: %s", requestID) - } - - // create fake device file - fpath := filepath.Join("/tmp", dev.ID) - - // clean first - os.RemoveAll(fpath) - f, err := os.Create(fpath) - if err != nil && !os.IsExist(err) { - return nil, fmt.Errorf("failed to create fake device file: %s", err) - } - - f.Close() - - response.Mounts = append(response.Mounts, &kubeletdevicepluginv1beta1.Mount{ - ContainerPath: fpath, - HostPath: fpath, - }) - } - responses.ContainerResponses = append(responses.ContainerResponses, response) - } - - return &responses, nil -} diff --git a/test/e2e_node/e2e_node_suite_test.go b/test/e2e_node/e2e_node_suite_test.go index 553e21e408d..bf920672ba2 100644 --- a/test/e2e_node/e2e_node_suite_test.go +++ b/test/e2e_node/e2e_node_suite_test.go @@ -41,6 +41,8 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/util/system" commontest "k8s.io/kubernetes/test/e2e/common" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/testfiles" + "k8s.io/kubernetes/test/e2e/generated" "k8s.io/kubernetes/test/e2e_node/services" "github.com/kardianos/osext" @@ -71,6 +73,13 @@ func init() { // TODO(random-liu): Find who is using flag.Parse() and cause errors and move the following logic // into TestContext. // TODO(pohly): remove RegisterNodeFlags from test_context.go enable Viper config support here? + + // Enable bindata file lookup as fallback. + testfiles.AddFileSource(testfiles.BindataFileSource{ + Asset: generated.Asset, + AssetNames: generated.AssetNames, + }) + } func TestMain(m *testing.M) {