diff --git a/test/e2e/common/runtimeclass.go b/test/e2e/common/runtimeclass.go index f7b8c443d04..b4217955c4e 100644 --- a/test/e2e/common/runtimeclass.go +++ b/test/e2e/common/runtimeclass.go @@ -30,9 +30,8 @@ import ( runtimeclasstest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing" "k8s.io/kubernetes/test/e2e/framework" e2eevents "k8s.io/kubernetes/test/e2e/framework/events" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" - imageutils "k8s.io/kubernetes/test/utils/image" - utilpointer "k8s.io/utils/pointer" "github.com/onsi/ginkgo" ) @@ -42,23 +41,23 @@ var _ = ginkgo.Describe("[sig-node] RuntimeClass", func() { ginkgo.It("should reject a Pod requesting a non-existent RuntimeClass", func() { rcName := f.Namespace.Name + "-nonexistent" - expectPodRejection(f, newRuntimeClassPod(rcName)) + expectPodRejection(f, e2enode.NewRuntimeClassPod(rcName)) }) ginkgo.It("should reject a Pod requesting a RuntimeClass with an unconfigured handler", func() { handler := f.Namespace.Name + "-handler" rcName := createRuntimeClass(f, "unconfigured-handler", handler) - pod := f.PodClient().Create(newRuntimeClassPod(rcName)) + pod := f.PodClient().Create(e2enode.NewRuntimeClassPod(rcName)) expectSandboxFailureEvent(f, pod, handler) }) // This test requires that the PreconfiguredRuntimeHandler has already been set up on nodes. ginkgo.It("should run a Pod requesting a RuntimeClass with a configured handler [NodeFeature:RuntimeHandler]", func() { // The built-in docker runtime does not support configuring runtime handlers. - handler := framework.PreconfiguredRuntimeClassHandler() + handler := e2enode.PreconfiguredRuntimeClassHandler(framework.TestContext.ContainerRuntime) rcName := createRuntimeClass(f, "preconfigured-handler", handler) - pod := f.PodClient().Create(newRuntimeClassPod(rcName)) + pod := f.PodClient().Create(e2enode.NewRuntimeClassPod(rcName)) expectPodSuccess(f, pod) }) @@ -83,7 +82,7 @@ var _ = ginkgo.Describe("[sig-node] RuntimeClass", func() { })) }) - expectPodRejection(f, newRuntimeClassPod(rcName)) + expectPodRejection(f, e2enode.NewRuntimeClassPod(rcName)) }) }) @@ -97,25 +96,6 @@ func createRuntimeClass(f *framework.Framework, name, handler string) string { return rc.GetName() } -// newRuntimeClassPod generates a test pod with the given runtimeClassName. -func newRuntimeClassPod(runtimeClassName string) *v1.Pod { - return &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: fmt.Sprintf("test-runtimeclass-%s-", runtimeClassName), - }, - Spec: v1.PodSpec{ - RuntimeClassName: &runtimeClassName, - Containers: []v1.Container{{ - Name: "test", - Image: imageutils.GetE2EImage(imageutils.BusyBox), - Command: []string{"true"}, - }}, - RestartPolicy: v1.RestartPolicyNever, - AutomountServiceAccountToken: utilpointer.BoolPtr(false), - }, - } -} - func expectPodRejection(f *framework.Framework, pod *v1.Pod) { // The Node E2E doesn't run the RuntimeClass admission controller, so we expect the rejection to // happen by the Kubelet. diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index aafa4def823..0fbe3d129dd 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -877,18 +877,3 @@ func (cl *ClusterVerification) ForEach(podFunc func(v1.Pod)) error { return err } - -const ( - // preconfiguredRuntimeHandler is the name of the runtime handler that is expected to be - // preconfigured in the test environment. - preconfiguredRuntimeHandler = "test-handler" -) - -// PreconfiguredRuntimeClassHandler returns configured runtime handler. -func PreconfiguredRuntimeClassHandler() string { - if TestContext.ContainerRuntime == "docker" { - return TestContext.ContainerRuntime - } - - return preconfiguredRuntimeHandler -} diff --git a/test/e2e/framework/node/BUILD b/test/e2e/framework/node/BUILD index 95c56e2f1b2..e4d3dba0066 100644 --- a/test/e2e/framework/node/BUILD +++ b/test/e2e/framework/node/BUILD @@ -4,6 +4,7 @@ go_library( name = "go_default_library", srcs = [ "resource.go", + "runtimeclass.go", "wait.go", ], importpath = "k8s.io/kubernetes/test/e2e/framework/node", @@ -22,6 +23,8 @@ go_library( "//test/e2e/framework/log:go_default_library", "//test/e2e/system:go_default_library", "//test/utils:go_default_library", + "//test/utils/image:go_default_library", + "//vendor/k8s.io/utils/pointer:go_default_library", ], ) diff --git a/test/e2e/framework/node/runtimeclass.go b/test/e2e/framework/node/runtimeclass.go new file mode 100644 index 00000000000..a1381200373 --- /dev/null +++ b/test/e2e/framework/node/runtimeclass.go @@ -0,0 +1,56 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node + +import ( + "fmt" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + imageutils "k8s.io/kubernetes/test/utils/image" + utilpointer "k8s.io/utils/pointer" +) + +// PreconfiguredRuntimeClassHandler returns configured runtime handler. +func PreconfiguredRuntimeClassHandler(handler string) string { + if handler == "docker" { + return handler + } + + // test-handler is the name of the runtime handler that is expected to be + // preconfigured in the test environment. + return "test-handler" +} + +// NewRuntimeClassPod returns a test pod with the given runtimeClassName +func NewRuntimeClassPod(runtimeClassName string) *v1.Pod { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: fmt.Sprintf("test-runtimeclass-%s-", runtimeClassName), + }, + Spec: v1.PodSpec{ + RuntimeClassName: &runtimeClassName, + Containers: []v1.Container{{ + Name: "test", + Image: imageutils.GetE2EImage(imageutils.BusyBox), + Command: []string{"true"}, + }}, + RestartPolicy: v1.RestartPolicyNever, + AutomountServiceAccountToken: utilpointer.BoolPtr(false), + }, + } +} diff --git a/test/e2e/node/BUILD b/test/e2e/node/BUILD index b256ff6c503..6e833e1fd3e 100644 --- a/test/e2e/node/BUILD +++ b/test/e2e/node/BUILD @@ -57,7 +57,6 @@ go_library( "//test/utils/image:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", - "//vendor/k8s.io/utils/pointer:go_default_library", ], ) diff --git a/test/e2e/node/runtimeclass.go b/test/e2e/node/runtimeclass.go index a25f718f93f..f60aa5900db 100644 --- a/test/e2e/node/runtimeclass.go +++ b/test/e2e/node/runtimeclass.go @@ -26,10 +26,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtimeclasstest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing" "k8s.io/kubernetes/test/e2e/framework" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" "k8s.io/kubernetes/test/e2e/scheduling" - imageutils "k8s.io/kubernetes/test/utils/image" - utilpointer "k8s.io/utils/pointer" "github.com/onsi/ginkgo" "github.com/onsi/gomega" @@ -45,12 +44,12 @@ var _ = ginkgo.Describe("[sig-node] RuntimeClass", func() { }, } - runtimeClass := newRuntimeClass(f.Namespace.Name, "conflict-runtimeclass") + runtimeClass := newRuntimeClass(f.Namespace.Name, "conflict-runtimeclass", framework.TestContext.ContainerRuntime) runtimeClass.Scheduling = scheduling rc, err := f.ClientSet.NodeV1beta1().RuntimeClasses().Create(context.TODO(), runtimeClass, metav1.CreateOptions{}) framework.ExpectNoError(err, "failed to create RuntimeClass resource") - pod := newRuntimeClassPod(rc.GetName()) + pod := e2enode.NewRuntimeClassPod(rc.GetName()) pod.Spec.NodeSelector = map[string]string{ "foo": "bar", } @@ -96,12 +95,12 @@ var _ = ginkgo.Describe("[sig-node] RuntimeClass", func() { defer framework.RemoveTaintOffNode(f.ClientSet, nodeName, taint) ginkgo.By("Trying to create runtimeclass and pod") - runtimeClass := newRuntimeClass(f.Namespace.Name, "non-conflict-runtimeclass") + runtimeClass := newRuntimeClass(f.Namespace.Name, "non-conflict-runtimeclass", framework.TestContext.ContainerRuntime) runtimeClass.Scheduling = scheduling rc, err := f.ClientSet.NodeV1beta1().RuntimeClasses().Create(context.TODO(), runtimeClass, metav1.CreateOptions{}) framework.ExpectNoError(err, "failed to create RuntimeClass resource") - pod := newRuntimeClassPod(rc.GetName()) + pod := e2enode.NewRuntimeClassPod(rc.GetName()) pod.Spec.NodeSelector = map[string]string{ "foo": "bar", } @@ -119,26 +118,7 @@ var _ = ginkgo.Describe("[sig-node] RuntimeClass", func() { }) // newRuntimeClass returns a test runtime class. -func newRuntimeClass(namespace, name string) *nodev1beta1.RuntimeClass { +func newRuntimeClass(namespace, name, handler string) *nodev1beta1.RuntimeClass { uniqueName := fmt.Sprintf("%s-%s", namespace, name) - return runtimeclasstest.NewRuntimeClass(uniqueName, framework.PreconfiguredRuntimeClassHandler()) -} - -// newRuntimeClassPod returns a test pod with the given runtimeClassName. -func newRuntimeClassPod(runtimeClassName string) *v1.Pod { - return &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: fmt.Sprintf("test-runtimeclass-%s-", runtimeClassName), - }, - Spec: v1.PodSpec{ - RuntimeClassName: &runtimeClassName, - Containers: []v1.Container{{ - Name: "test", - Image: imageutils.GetE2EImage(imageutils.BusyBox), - Command: []string{"true"}, - }}, - RestartPolicy: v1.RestartPolicyNever, - AutomountServiceAccountToken: utilpointer.BoolPtr(false), - }, - } + return runtimeclasstest.NewRuntimeClass(uniqueName, e2enode.PreconfiguredRuntimeClassHandler(handler)) } diff --git a/test/e2e/scheduling/BUILD b/test/e2e/scheduling/BUILD index a7bec3e7646..f56619f835c 100644 --- a/test/e2e/scheduling/BUILD +++ b/test/e2e/scheduling/BUILD @@ -24,6 +24,7 @@ go_library( "//pkg/scheduler/util:go_default_library", "//staging/src/k8s.io/api/apps/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/api/node/v1beta1:go_default_library", "//staging/src/k8s.io/api/scheduling/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", diff --git a/test/e2e/scheduling/predicates.go b/test/e2e/scheduling/predicates.go index ce5cf72a11f..954e9e1666c 100644 --- a/test/e2e/scheduling/predicates.go +++ b/test/e2e/scheduling/predicates.go @@ -22,6 +22,7 @@ import ( "time" v1 "k8s.io/api/core/v1" + nodev1beta1 "k8s.io/api/node/v1beta1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" @@ -61,6 +62,7 @@ type pausePodConfig struct { Affinity *v1.Affinity Annotations, Labels, NodeSelector map[string]string Resources *v1.ResourceRequirements + RuntimeClassHandler *string Tolerations []v1.Toleration NodeName string Ports []v1.ContainerPort @@ -196,6 +198,116 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() { verifyResult(cs, podsNeededForSaturation, 1, ns) }) + // This test verifies we don't allow scheduling of pods in a way that sum of limits + + // associated overhead is greater than machine's capacity. + // It assumes that cluster add-on pods stay stable and cannot be run in parallel + // with any other test that touches Nodes or Pods. + // Because of this we need to have precise control on what's running in the cluster. + // Test scenario: + // 1. Find the first ready node on the system, and add a fake resource for test + // 2. Create one with affinity to the particular node that uses 70% of the fake resource. + // 3. Wait for the pod to be scheduled. + // 4. Create another pod with affinity to the particular node that needs 20% of the fake resource and + // an overhead set as 25% of the fake resource. + // 5. Make sure this additional pod is not scheduled. + + ginkgo.Context("validates pod overhead is considered along with resource limits of pods that are allowed to run", func() { + var testNodeName string + var handler string + var beardsecond v1.ResourceName = "example.com/beardsecond" + + ginkgo.BeforeEach(func() { + WaitForStableCluster(cs, masterNodes) + ginkgo.By("Add RuntimeClass and fake resource") + + // find a node which can run a pod: + testNodeName = GetNodeThatCanRunPod(f) + + // Get node object: + node, err := cs.CoreV1().Nodes().Get(context.TODO(), testNodeName, metav1.GetOptions{}) + framework.ExpectNoError(err, "unable to get node object for node %v", testNodeName) + + // update Node API object with a fake resource + nodeCopy := node.DeepCopy() + nodeCopy.ResourceVersion = "0" + + nodeCopy.Status.Capacity[beardsecond] = resource.MustParse("1000") + _, err = cs.CoreV1().Nodes().UpdateStatus(context.TODO(), nodeCopy, metav1.UpdateOptions{}) + framework.ExpectNoError(err, "unable to apply fake resource to %v", testNodeName) + + // Register a runtimeClass with overhead set as 25% of the available beard-seconds + handler = e2enode.PreconfiguredRuntimeClassHandler(framework.TestContext.ContainerRuntime) + + rc := &nodev1beta1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{Name: handler}, + Handler: handler, + Overhead: &nodev1beta1.Overhead{ + PodFixed: v1.ResourceList{ + beardsecond: resource.MustParse("250"), + }, + }, + } + _, err = cs.NodeV1beta1().RuntimeClasses().Create(context.TODO(), rc, metav1.CreateOptions{}) + framework.ExpectNoError(err, "failed to create RuntimeClass resource") + }) + + ginkgo.AfterEach(func() { + ginkgo.By("Remove fake resource and RuntimeClass") + // remove fake resource: + if testNodeName != "" { + // Get node object: + node, err := cs.CoreV1().Nodes().Get(context.TODO(), testNodeName, metav1.GetOptions{}) + framework.ExpectNoError(err, "unable to get node object for node %v", testNodeName) + + nodeCopy := node.DeepCopy() + // force it to update + nodeCopy.ResourceVersion = "0" + delete(nodeCopy.Status.Capacity, beardsecond) + _, err = cs.CoreV1().Nodes().UpdateStatus(context.TODO(), nodeCopy, metav1.UpdateOptions{}) + framework.ExpectNoError(err, "unable to update node %v", testNodeName) + } + + // remove RuntimeClass + cs.NodeV1beta1().RuntimeClasses().Delete(context.TODO(), e2enode.PreconfiguredRuntimeClassHandler(framework.TestContext.ContainerRuntime), nil) + }) + + ginkgo.It("verify pod overhead is accounted for", func() { + framework.ExpectEqual(testNodeName != "", true) + + ginkgo.By("Starting Pod to consume most of the node's resource.") + + // Create pod which requires 70% of the available beard-seconds. + fillerPod := createPausePod(f, pausePodConfig{ + Name: "filler-pod-" + string(uuid.NewUUID()), + Resources: &v1.ResourceRequirements{ + Requests: v1.ResourceList{beardsecond: resource.MustParse("700")}, + Limits: v1.ResourceList{beardsecond: resource.MustParse("700")}, + }, + }) + + // Wait for filler pod to schedule. + framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(cs, fillerPod)) + + ginkgo.By("Creating another pod that requires unavailable amount of resources.") + // Create another pod that requires 20% of available beard-seconds, but utilizes the RuntimeClass + // which defines a pod overhead that requires an additional 25%. + // This pod should remain pending as at least 70% of beard-second in + // the node are already consumed. + podName := "additional-pod" + string(uuid.NewUUID()) + conf := pausePodConfig{ + RuntimeClassHandler: &handler, + Name: podName, + Labels: map[string]string{"name": "additional"}, + Resources: &v1.ResourceRequirements{ + Limits: v1.ResourceList{beardsecond: resource.MustParse("200")}, + }, + } + + WaitForSchedulerAfterAction(f, createPausePodAction(f, conf), ns, podName, false) + verifyResult(cs, 1, 1, ns) + }) + }) + // This test verifies we don't allow scheduling of pods in a way that sum of // resource requests of pods is greater than machines capacity. // It assumes that cluster add-on pods stay stable and cannot be run in parallel @@ -715,6 +827,7 @@ func initPausePod(f *framework.Framework, conf pausePodConfig) *v1.Pod { NodeSelector: conf.NodeSelector, Affinity: conf.Affinity, TopologySpreadConstraints: conf.TopologySpreadConstraints, + RuntimeClassName: conf.RuntimeClassHandler, Containers: []v1.Container{ { Name: conf.Name,