Merge pull request #87215 from egernst/e2e-scheduler

E2e scheduler: introduce e2e test for PodOverhead

test/e2e/common/runtimeclass.go

@@ -30,9 +30,8 @@ import (
 	runtimeclasstest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2eevents "k8s.io/kubernetes/test/e2e/framework/events"
+	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
-	imageutils "k8s.io/kubernetes/test/utils/image"
-	utilpointer "k8s.io/utils/pointer"
 
 	"github.com/onsi/ginkgo"
 )
@@ -42,23 +41,23 @@ var _ = ginkgo.Describe("[sig-node] RuntimeClass", func() {
 
 	ginkgo.It("should reject a Pod requesting a non-existent RuntimeClass", func() {
 		rcName := f.Namespace.Name + "-nonexistent"
-		expectPodRejection(f, newRuntimeClassPod(rcName))
+		expectPodRejection(f, e2enode.NewRuntimeClassPod(rcName))
 	})
 
 	ginkgo.It("should reject a Pod requesting a RuntimeClass with an unconfigured handler", func() {
 		handler := f.Namespace.Name + "-handler"
 		rcName := createRuntimeClass(f, "unconfigured-handler", handler)
-		pod := f.PodClient().Create(newRuntimeClassPod(rcName))
+		pod := f.PodClient().Create(e2enode.NewRuntimeClassPod(rcName))
 		expectSandboxFailureEvent(f, pod, handler)
 	})
 
 	// This test requires that the PreconfiguredRuntimeHandler has already been set up on nodes.
 	ginkgo.It("should run a Pod requesting a RuntimeClass with a configured handler [NodeFeature:RuntimeHandler]", func() {
 		// The built-in docker runtime does not support configuring runtime handlers.
-		handler := framework.PreconfiguredRuntimeClassHandler()
+		handler := e2enode.PreconfiguredRuntimeClassHandler(framework.TestContext.ContainerRuntime)
 
 		rcName := createRuntimeClass(f, "preconfigured-handler", handler)
-		pod := f.PodClient().Create(newRuntimeClassPod(rcName))
+		pod := f.PodClient().Create(e2enode.NewRuntimeClassPod(rcName))
 		expectPodSuccess(f, pod)
 	})
 
@@ -83,7 +82,7 @@ var _ = ginkgo.Describe("[sig-node] RuntimeClass", func() {
 			}))
 		})
 
-		expectPodRejection(f, newRuntimeClassPod(rcName))
+		expectPodRejection(f, e2enode.NewRuntimeClassPod(rcName))
 	})
 })
 
@@ -97,25 +96,6 @@ func createRuntimeClass(f *framework.Framework, name, handler string) string {
 	return rc.GetName()
 }
 
-// newRuntimeClassPod generates a test pod with the given runtimeClassName.
-func newRuntimeClassPod(runtimeClassName string) *v1.Pod {
-	return &v1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			GenerateName: fmt.Sprintf("test-runtimeclass-%s-", runtimeClassName),
-		},
-		Spec: v1.PodSpec{
-			RuntimeClassName: &runtimeClassName,
-			Containers: []v1.Container{{
-				Name:    "test",
-				Image:   imageutils.GetE2EImage(imageutils.BusyBox),
-				Command: []string{"true"},
-			}},
-			RestartPolicy:                v1.RestartPolicyNever,
-			AutomountServiceAccountToken: utilpointer.BoolPtr(false),
-		},
-	}
-}
-
 func expectPodRejection(f *framework.Framework, pod *v1.Pod) {
 	// The Node E2E doesn't run the RuntimeClass admission controller, so we expect the rejection to
 	// happen by the Kubelet.

test/e2e/framework/framework.go

@@ -877,18 +877,3 @@ func (cl *ClusterVerification) ForEach(podFunc func(v1.Pod)) error {
 
 	return err
 }
-
-const (
-	// preconfiguredRuntimeHandler is the name of the runtime handler that is expected to be
-	// preconfigured in the test environment.
-	preconfiguredRuntimeHandler = "test-handler"
-)
-
-// PreconfiguredRuntimeClassHandler returns configured runtime handler.
-func PreconfiguredRuntimeClassHandler() string {
-	if TestContext.ContainerRuntime == "docker" {
-		return TestContext.ContainerRuntime
-	}
-
-	return preconfiguredRuntimeHandler
-}

test/e2e/framework/node/BUILD

@@ -4,6 +4,7 @@ go_library(
     name = "go_default_library",
     srcs = [
         "resource.go",
+        "runtimeclass.go",
         "wait.go",
     ],
     importpath = "k8s.io/kubernetes/test/e2e/framework/node",
@@ -22,6 +23,8 @@ go_library(
         "//test/e2e/framework/log:go_default_library",
         "//test/e2e/system:go_default_library",
         "//test/utils:go_default_library",
+        "//test/utils/image:go_default_library",
+        "//vendor/k8s.io/utils/pointer:go_default_library",
     ],
 )
 

test/e2e/framework/node/runtimeclass.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package node
+
+import (
+	"fmt"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	imageutils "k8s.io/kubernetes/test/utils/image"
+	utilpointer "k8s.io/utils/pointer"
+)
+
+// PreconfiguredRuntimeClassHandler returns configured runtime handler.
+func PreconfiguredRuntimeClassHandler(handler string) string {
+	if handler == "docker" {
+		return handler
+	}
+
+	// test-handler is the name of the runtime handler that is expected to be
+	// preconfigured in the test environment.
+	return "test-handler"
+}
+
+// NewRuntimeClassPod returns a test pod with the given runtimeClassName
+func NewRuntimeClassPod(runtimeClassName string) *v1.Pod {
+	return &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: fmt.Sprintf("test-runtimeclass-%s-", runtimeClassName),
+		},
+		Spec: v1.PodSpec{
+			RuntimeClassName: &runtimeClassName,
+			Containers: []v1.Container{{
+				Name:    "test",
+				Image:   imageutils.GetE2EImage(imageutils.BusyBox),
+				Command: []string{"true"},
+			}},
+			RestartPolicy:                v1.RestartPolicyNever,
+			AutomountServiceAccountToken: utilpointer.BoolPtr(false),
+		},
+	}
+}

test/e2e/node/BUILD

@@ -57,7 +57,6 @@ go_library(
         "//test/utils/image:go_default_library",
         "//vendor/github.com/onsi/ginkgo:go_default_library",
         "//vendor/github.com/onsi/gomega:go_default_library",
-        "//vendor/k8s.io/utils/pointer:go_default_library",
     ],
 )
 

test/e2e/node/runtimeclass.go

@@ -26,10 +26,9 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	runtimeclasstest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing"
 	"k8s.io/kubernetes/test/e2e/framework"
+	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 	"k8s.io/kubernetes/test/e2e/scheduling"
-	imageutils "k8s.io/kubernetes/test/utils/image"
-	utilpointer "k8s.io/utils/pointer"
 
 	"github.com/onsi/ginkgo"
 	"github.com/onsi/gomega"
@@ -45,12 +44,12 @@ var _ = ginkgo.Describe("[sig-node] RuntimeClass", func() {
 			},
 		}
 
-		runtimeClass := newRuntimeClass(f.Namespace.Name, "conflict-runtimeclass")
+		runtimeClass := newRuntimeClass(f.Namespace.Name, "conflict-runtimeclass", framework.TestContext.ContainerRuntime)
 		runtimeClass.Scheduling = scheduling
 		rc, err := f.ClientSet.NodeV1beta1().RuntimeClasses().Create(context.TODO(), runtimeClass, metav1.CreateOptions{})
 		framework.ExpectNoError(err, "failed to create RuntimeClass resource")
 
-		pod := newRuntimeClassPod(rc.GetName())
+		pod := e2enode.NewRuntimeClassPod(rc.GetName())
 		pod.Spec.NodeSelector = map[string]string{
 			"foo": "bar",
 		}
@@ -96,12 +95,12 @@ var _ = ginkgo.Describe("[sig-node] RuntimeClass", func() {
 		defer framework.RemoveTaintOffNode(f.ClientSet, nodeName, taint)
 
 		ginkgo.By("Trying to create runtimeclass and pod")
-		runtimeClass := newRuntimeClass(f.Namespace.Name, "non-conflict-runtimeclass")
+		runtimeClass := newRuntimeClass(f.Namespace.Name, "non-conflict-runtimeclass", framework.TestContext.ContainerRuntime)
 		runtimeClass.Scheduling = scheduling
 		rc, err := f.ClientSet.NodeV1beta1().RuntimeClasses().Create(context.TODO(), runtimeClass, metav1.CreateOptions{})
 		framework.ExpectNoError(err, "failed to create RuntimeClass resource")
 
-		pod := newRuntimeClassPod(rc.GetName())
+		pod := e2enode.NewRuntimeClassPod(rc.GetName())
 		pod.Spec.NodeSelector = map[string]string{
 			"foo": "bar",
 		}
@@ -119,26 +118,7 @@ var _ = ginkgo.Describe("[sig-node] RuntimeClass", func() {
 })
 
 // newRuntimeClass returns a test runtime class.
-func newRuntimeClass(namespace, name string) *nodev1beta1.RuntimeClass {
+func newRuntimeClass(namespace, name, handler string) *nodev1beta1.RuntimeClass {
 	uniqueName := fmt.Sprintf("%s-%s", namespace, name)
-	return runtimeclasstest.NewRuntimeClass(uniqueName, framework.PreconfiguredRuntimeClassHandler())
+	return runtimeclasstest.NewRuntimeClass(uniqueName, e2enode.PreconfiguredRuntimeClassHandler(handler))
-}
-
-// newRuntimeClassPod returns a test pod with the given runtimeClassName.
-func newRuntimeClassPod(runtimeClassName string) *v1.Pod {
-	return &v1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			GenerateName: fmt.Sprintf("test-runtimeclass-%s-", runtimeClassName),
-		},
-		Spec: v1.PodSpec{
-			RuntimeClassName: &runtimeClassName,
-			Containers: []v1.Container{{
-				Name:    "test",
-				Image:   imageutils.GetE2EImage(imageutils.BusyBox),
-				Command: []string{"true"},
-			}},
-			RestartPolicy:                v1.RestartPolicyNever,
-			AutomountServiceAccountToken: utilpointer.BoolPtr(false),
-		},
-	}
 }

test/e2e/scheduling/BUILD

@@ -24,6 +24,7 @@ go_library(
         "//pkg/scheduler/util:go_default_library",
         "//staging/src/k8s.io/api/apps/v1:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/api/node/v1beta1:go_default_library",
         "//staging/src/k8s.io/api/scheduling/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",

test/e2e/scheduling/predicates.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	v1 "k8s.io/api/core/v1"
+	nodev1beta1 "k8s.io/api/node/v1beta1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -61,6 +62,7 @@ type pausePodConfig struct {
 	Affinity                          *v1.Affinity
 	Annotations, Labels, NodeSelector map[string]string
 	Resources                         *v1.ResourceRequirements
+	RuntimeClassHandler               *string
 	Tolerations                       []v1.Toleration
 	NodeName                          string
 	Ports                             []v1.ContainerPort
@@ -196,6 +198,116 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() {
 		verifyResult(cs, podsNeededForSaturation, 1, ns)
 	})
 
+	// This test verifies we don't allow scheduling of pods in a way that sum of limits +
+	// associated overhead is greater than machine's capacity.
+	// It assumes that cluster add-on pods stay stable and cannot be run in parallel
+	// with any other test that touches Nodes or Pods.
+	// Because of this we need to have precise control on what's running in the cluster.
+	// Test scenario:
+	// 1. Find the first ready node on the system, and add a fake resource for test
+	// 2. Create one with affinity to the particular node that uses 70% of the fake resource.
+	// 3. Wait for the pod to be scheduled.
+	// 4. Create another pod with affinity to the particular node that needs 20% of the fake resource and
+	//    an overhead set as 25% of the fake resource.
+	// 5. Make sure this additional pod is not scheduled.
+
+	ginkgo.Context("validates pod overhead is considered along with resource limits of pods that are allowed to run", func() {
+		var testNodeName string
+		var handler string
+		var beardsecond v1.ResourceName = "example.com/beardsecond"
+
+		ginkgo.BeforeEach(func() {
+			WaitForStableCluster(cs, masterNodes)
+			ginkgo.By("Add RuntimeClass and fake resource")
+
+			// find a node which can run a pod:
+			testNodeName = GetNodeThatCanRunPod(f)
+
+			// Get node object:
+			node, err := cs.CoreV1().Nodes().Get(context.TODO(), testNodeName, metav1.GetOptions{})
+			framework.ExpectNoError(err, "unable to get node object for node %v", testNodeName)
+
+			// update Node API object with a fake resource
+			nodeCopy := node.DeepCopy()
+			nodeCopy.ResourceVersion = "0"
+
+			nodeCopy.Status.Capacity[beardsecond] = resource.MustParse("1000")
+			_, err = cs.CoreV1().Nodes().UpdateStatus(context.TODO(), nodeCopy, metav1.UpdateOptions{})
+			framework.ExpectNoError(err, "unable to apply fake resource to %v", testNodeName)
+
+			// Register a runtimeClass with overhead set as 25% of the available beard-seconds
+			handler = e2enode.PreconfiguredRuntimeClassHandler(framework.TestContext.ContainerRuntime)
+
+			rc := &nodev1beta1.RuntimeClass{
+				ObjectMeta: metav1.ObjectMeta{Name: handler},
+				Handler:    handler,
+				Overhead: &nodev1beta1.Overhead{
+					PodFixed: v1.ResourceList{
+						beardsecond: resource.MustParse("250"),
+					},
+				},
+			}
+			_, err = cs.NodeV1beta1().RuntimeClasses().Create(context.TODO(), rc, metav1.CreateOptions{})
+			framework.ExpectNoError(err, "failed to create RuntimeClass resource")
+		})
+
+		ginkgo.AfterEach(func() {
+			ginkgo.By("Remove fake resource and RuntimeClass")
+			// remove fake resource:
+			if testNodeName != "" {
+				// Get node object:
+				node, err := cs.CoreV1().Nodes().Get(context.TODO(), testNodeName, metav1.GetOptions{})
+				framework.ExpectNoError(err, "unable to get node object for node %v", testNodeName)
+
+				nodeCopy := node.DeepCopy()
+				// force it to update
+				nodeCopy.ResourceVersion = "0"
+				delete(nodeCopy.Status.Capacity, beardsecond)
+				_, err = cs.CoreV1().Nodes().UpdateStatus(context.TODO(), nodeCopy, metav1.UpdateOptions{})
+				framework.ExpectNoError(err, "unable to update node %v", testNodeName)
+			}
+
+			// remove RuntimeClass
+			cs.NodeV1beta1().RuntimeClasses().Delete(context.TODO(), e2enode.PreconfiguredRuntimeClassHandler(framework.TestContext.ContainerRuntime), nil)
+		})
+
+		ginkgo.It("verify pod overhead is accounted for", func() {
+			framework.ExpectEqual(testNodeName != "", true)
+
+			ginkgo.By("Starting Pod to consume most of the node's resource.")
+
+			// Create pod which requires 70% of the available beard-seconds.
+			fillerPod := createPausePod(f, pausePodConfig{
+				Name: "filler-pod-" + string(uuid.NewUUID()),
+				Resources: &v1.ResourceRequirements{
+					Requests: v1.ResourceList{beardsecond: resource.MustParse("700")},
+					Limits:   v1.ResourceList{beardsecond: resource.MustParse("700")},
+				},
+			})
+
+			// Wait for filler pod to schedule.
+			framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(cs, fillerPod))
+
+			ginkgo.By("Creating another pod that requires unavailable amount of resources.")
+			// Create another pod that requires 20% of available beard-seconds, but utilizes the RuntimeClass
+			// which defines a pod overhead that requires an additional 25%.
+			// This pod should remain pending as at least 70% of beard-second in
+			// the node are already consumed.
+			podName := "additional-pod" + string(uuid.NewUUID())
+			conf := pausePodConfig{
+				RuntimeClassHandler: &handler,
+				Name:                podName,
+				Labels:              map[string]string{"name": "additional"},
+				Resources: &v1.ResourceRequirements{
+					Limits: v1.ResourceList{beardsecond: resource.MustParse("200")},
+				},
+			}
+
+			WaitForSchedulerAfterAction(f, createPausePodAction(f, conf), ns, podName, false)
+			verifyResult(cs, 1, 1, ns)
+		})
+	})
+
 	// This test verifies we don't allow scheduling of pods in a way that sum of
 	// resource requests of pods is greater than machines capacity.
 	// It assumes that cluster add-on pods stay stable and cannot be run in parallel
@@ -715,6 +827,7 @@ func initPausePod(f *framework.Framework, conf pausePodConfig) *v1.Pod {
 			NodeSelector:              conf.NodeSelector,
 			Affinity:                  conf.Affinity,
 			TopologySpreadConstraints: conf.TopologySpreadConstraints,
+			RuntimeClassName:          conf.RuntimeClassHandler,
 			Containers: []v1.Container{
 				{
 					Name:  conf.Name,