From 702646df2441c03f435065dd20bb3d8c1c8cbcf2 Mon Sep 17 00:00:00 2001 From: draveness Date: Sun, 25 Aug 2019 15:20:50 +0800 Subject: [PATCH] test(runtimeclass): add e2e tests for runtimeclass scheduling --- test/e2e/common/runtimeclass.go | 16 +--- test/e2e/framework/framework.go | 17 +++- test/e2e/node/BUILD | 6 ++ test/e2e/node/runtimeclass.go | 143 ++++++++++++++++++++++++++++++++ 4 files changed, 167 insertions(+), 15 deletions(-) create mode 100644 test/e2e/node/runtimeclass.go diff --git a/test/e2e/common/runtimeclass.go b/test/e2e/common/runtimeclass.go index f94ee80391f..577233315c7 100644 --- a/test/e2e/common/runtimeclass.go +++ b/test/e2e/common/runtimeclass.go @@ -20,7 +20,7 @@ import ( "fmt" "time" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -37,15 +37,6 @@ import ( "github.com/onsi/gomega" ) -const ( - // PreconfiguredRuntimeHandler is the name of the runtime handler that is expected to be - // preconfigured in the test environment. - PreconfiguredRuntimeHandler = "test-handler" - // DockerRuntimeHandler is a hardcoded runtime handler that is accepted by dockershim, and - // treated equivalently to a nil runtime handler. - DockerRuntimeHandler = "docker" -) - var _ = ginkgo.Describe("[sig-node] RuntimeClass", func() { f := framework.NewDefaultFramework("runtimeclass") @@ -64,10 +55,7 @@ var _ = ginkgo.Describe("[sig-node] RuntimeClass", func() { // This test requires that the PreconfiguredRuntimeHandler has already been set up on nodes. ginkgo.It("should run a Pod requesting a RuntimeClass with a configured handler [NodeFeature:RuntimeHandler]", func() { // The built-in docker runtime does not support configuring runtime handlers. - handler := PreconfiguredRuntimeHandler - if framework.TestContext.ContainerRuntime == "docker" { - handler = DockerRuntimeHandler - } + handler := framework.PreconfiguredRuntimeClassHandler() rcName := createRuntimeClass(f, "preconfigured-handler", handler) pod := f.PodClient().Create(newRuntimeClassPod(rcName)) diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index c3c93f4f694..8cee066b82f 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -31,7 +31,7 @@ import ( "sync" "time" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -884,3 +884,18 @@ func GetLogToFileFunc(file *os.File) func(format string, args ...interface{}) { writer.Flush() } } + +const ( + // preconfiguredRuntimeHandler is the name of the runtime handler that is expected to be + // preconfigured in the test environment. + preconfiguredRuntimeHandler = "test-handler" +) + +// PreconfiguredRuntimeClassHandler returns configured runtime handler. +func PreconfiguredRuntimeClassHandler() string { + if TestContext.ContainerRuntime == "docker" { + return TestContext.ContainerRuntime + } + + return preconfiguredRuntimeHandler +} diff --git a/test/e2e/node/BUILD b/test/e2e/node/BUILD index 52b30c5deeb..6b8fb54ef3e 100644 --- a/test/e2e/node/BUILD +++ b/test/e2e/node/BUILD @@ -14,6 +14,7 @@ go_library( "pod_gc.go", "pods.go", "pre_stop.go", + "runtimeclass.go", "security_context.go", "ssh.go", "ttlafterfinished.go", @@ -22,10 +23,13 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/kubelet/apis/stats/v1alpha1:go_default_library", + "//pkg/kubelet/runtimeclass/testing:go_default_library", "//pkg/master/ports:go_default_library", "//pkg/util/slice:go_default_library", "//staging/src/k8s.io/api/batch/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/api/node/v1beta1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", @@ -46,10 +50,12 @@ go_library( "//test/e2e/framework/ssh:go_default_library", "//test/e2e/framework/volume:go_default_library", "//test/e2e/perftype:go_default_library", + "//test/e2e/scheduling:go_default_library", "//test/utils:go_default_library", "//test/utils/image:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", + "//vendor/k8s.io/utils/pointer:go_default_library", ], ) diff --git a/test/e2e/node/runtimeclass.go b/test/e2e/node/runtimeclass.go new file mode 100644 index 00000000000..cea044f0cf8 --- /dev/null +++ b/test/e2e/node/runtimeclass.go @@ -0,0 +1,143 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node + +import ( + "fmt" + + v1 "k8s.io/api/core/v1" + "k8s.io/api/node/v1beta1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtimeclasstest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing" + "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + "k8s.io/kubernetes/test/e2e/scheduling" + imageutils "k8s.io/kubernetes/test/utils/image" + utilpointer "k8s.io/utils/pointer" + + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" +) + +var _ = ginkgo.Describe("[sig-node] RuntimeClass", func() { + f := framework.NewDefaultFramework("runtimeclass") + + ginkgo.It("should reject a Pod requesting a RuntimeClass with conflicting node selector", func() { + scheduling := &v1beta1.Scheduling{ + NodeSelector: map[string]string{ + "foo": "conflict", + }, + } + + runtimeClass := newRuntimeClass(f.Namespace.Name, "conflict-runtimeclass") + runtimeClass.Scheduling = scheduling + rc, err := f.ClientSet.NodeV1beta1().RuntimeClasses().Create(runtimeClass) + framework.ExpectNoError(err, "failed to create RuntimeClass resource") + + pod := newRuntimeClassPod(rc.GetName()) + pod.Spec.NodeSelector = map[string]string{ + "foo": "bar", + } + _, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod) + framework.ExpectError(err, "should be forbidden") + gomega.Expect(apierrs.IsForbidden(err)).To(gomega.BeTrue(), "should be forbidden error") + }) + + ginkgo.It("should run a Pod requesting a RuntimeClass with scheduling [NodeFeature:RuntimeHandler] ", func() { + nodeName := scheduling.GetNodeThatCanRunPod(f) + nodeSelector := map[string]string{ + "foo": "bar", + "fizz": "buzz", + } + tolerations := []v1.Toleration{ + { + Key: "foo", + Operator: v1.TolerationOpEqual, + Value: "bar", + Effect: v1.TaintEffectNoSchedule, + }, + } + scheduling := &v1beta1.Scheduling{ + NodeSelector: nodeSelector, + Tolerations: tolerations, + } + + ginkgo.By("Trying to apply a label on the found node.") + for key, value := range nodeSelector { + framework.AddOrUpdateLabelOnNode(f.ClientSet, nodeName, key, value) + framework.ExpectNodeHasLabel(f.ClientSet, nodeName, key, value) + defer framework.RemoveLabelOffNode(f.ClientSet, nodeName, key) + } + + ginkgo.By("Trying to apply taint on the found node.") + taint := v1.Taint{ + Key: "foo", + Value: "bar", + Effect: v1.TaintEffectNoSchedule, + } + framework.AddOrUpdateTaintOnNode(f.ClientSet, nodeName, taint) + framework.ExpectNodeHasTaint(f.ClientSet, nodeName, &taint) + defer framework.RemoveTaintOffNode(f.ClientSet, nodeName, taint) + + ginkgo.By("Trying to create runtimeclass and pod") + runtimeClass := newRuntimeClass(f.Namespace.Name, "non-conflict-runtimeclass") + runtimeClass.Scheduling = scheduling + rc, err := f.ClientSet.NodeV1beta1().RuntimeClasses().Create(runtimeClass) + framework.ExpectNoError(err, "failed to create RuntimeClass resource") + + pod := newRuntimeClassPod(rc.GetName()) + pod.Spec.NodeSelector = map[string]string{ + "foo": "bar", + } + pod = f.PodClient().Create(pod) + + framework.ExpectNoError(e2epod.WaitForPodNotPending(f.ClientSet, f.Namespace.Name, pod.Name)) + + // check that pod got scheduled on specified node. + scheduledPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(pod.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + framework.ExpectEqual(nodeName, scheduledPod.Spec.NodeName) + framework.ExpectEqual(nodeSelector, pod.Spec.NodeSelector) + gomega.Expect(pod.Spec.Tolerations).To(gomega.ContainElement(tolerations[0])) + }) +}) + +// newRuntimeClass returns a test runtime class. +func newRuntimeClass(namespace, name string) *v1beta1.RuntimeClass { + uniqueName := fmt.Sprintf("%s-%s", namespace, name) + return runtimeclasstest.NewRuntimeClass(uniqueName, framework.PreconfiguredRuntimeClassHandler()) +} + +// newRuntimeClassPod returns a test pod with the given runtimeClassName. +func newRuntimeClassPod(runtimeClassName string) *v1.Pod { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: fmt.Sprintf("test-runtimeclass-%s-", runtimeClassName), + }, + Spec: v1.PodSpec{ + RuntimeClassName: &runtimeClassName, + Containers: []v1.Container{{ + Name: "test", + Image: imageutils.GetE2EImage(imageutils.BusyBox), + Command: []string{"true"}, + }}, + RestartPolicy: v1.RestartPolicyNever, + AutomountServiceAccountToken: utilpointer.BoolPtr(false), + }, + } +}