mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-31 23:37:01 +00:00
Merge pull request #87215 from egernst/e2e-scheduler
E2e scheduler: introduce e2e test for PodOverhead
This commit is contained in:
Kubernetes Prow Robot
GitHub
commit
3631887a28
@ -30,9 +30,8 @@ import (
|
||||
runtimeclasstest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
e2eevents "k8s.io/kubernetes/test/e2e/framework/events"
|
||||
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
|
||||
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
|
||||
imageutils "k8s.io/kubernetes/test/utils/image"
|
||||
utilpointer "k8s.io/utils/pointer"
|
||||
|
||||
"github.com/onsi/ginkgo"
|
||||
)
|
||||
@ -42,23 +41,23 @@ var _ = ginkgo.Describe("[sig-node] RuntimeClass", func() {
|
||||
|
||||
ginkgo.It("should reject a Pod requesting a non-existent RuntimeClass", func() {
|
||||
rcName := f.Namespace.Name + "-nonexistent"
|
||||
expectPodRejection(f, newRuntimeClassPod(rcName))
|
||||
expectPodRejection(f, e2enode.NewRuntimeClassPod(rcName))
|
||||
})
|
||||
|
||||
ginkgo.It("should reject a Pod requesting a RuntimeClass with an unconfigured handler", func() {
|
||||
handler := f.Namespace.Name + "-handler"
|
||||
rcName := createRuntimeClass(f, "unconfigured-handler", handler)
|
||||
pod := f.PodClient().Create(newRuntimeClassPod(rcName))
|
||||
pod := f.PodClient().Create(e2enode.NewRuntimeClassPod(rcName))
|
||||
expectSandboxFailureEvent(f, pod, handler)
|
||||
})
|
||||
|
||||
// This test requires that the PreconfiguredRuntimeHandler has already been set up on nodes.
|
||||
ginkgo.It("should run a Pod requesting a RuntimeClass with a configured handler [NodeFeature:RuntimeHandler]", func() {
|
||||
// The built-in docker runtime does not support configuring runtime handlers.
|
||||
handler := framework.PreconfiguredRuntimeClassHandler()
|
||||
handler := e2enode.PreconfiguredRuntimeClassHandler(framework.TestContext.ContainerRuntime)
|
||||
|
||||
rcName := createRuntimeClass(f, "preconfigured-handler", handler)
|
||||
pod := f.PodClient().Create(newRuntimeClassPod(rcName))
|
||||
pod := f.PodClient().Create(e2enode.NewRuntimeClassPod(rcName))
|
||||
expectPodSuccess(f, pod)
|
||||
})
|
||||
|
||||
@ -83,7 +82,7 @@ var _ = ginkgo.Describe("[sig-node] RuntimeClass", func() {
|
||||
}))
|
||||
})
|
||||
|
||||
expectPodRejection(f, newRuntimeClassPod(rcName))
|
||||
expectPodRejection(f, e2enode.NewRuntimeClassPod(rcName))
|
||||
})
|
||||
})
|
||||
|
||||
@ -97,25 +96,6 @@ func createRuntimeClass(f *framework.Framework, name, handler string) string {
|
||||
return rc.GetName()
|
||||
}
|
||||
|
||||
// newRuntimeClassPod generates a test pod with the given runtimeClassName.
|
||||
func newRuntimeClassPod(runtimeClassName string) *v1.Pod {
|
||||
return &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
GenerateName: fmt.Sprintf("test-runtimeclass-%s-", runtimeClassName),
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
RuntimeClassName: &runtimeClassName,
|
||||
Containers: []v1.Container{{
|
||||
Name: "test",
|
||||
Image: imageutils.GetE2EImage(imageutils.BusyBox),
|
||||
Command: []string{"true"},
|
||||
}},
|
||||
RestartPolicy: v1.RestartPolicyNever,
|
||||
AutomountServiceAccountToken: utilpointer.BoolPtr(false),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func expectPodRejection(f *framework.Framework, pod *v1.Pod) {
|
||||
// The Node E2E doesn't run the RuntimeClass admission controller, so we expect the rejection to
|
||||
// happen by the Kubelet.
|
||||
|
||||
@ -877,18 +877,3 @@ func (cl *ClusterVerification) ForEach(podFunc func(v1.Pod)) error {
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
const (
|
||||
// preconfiguredRuntimeHandler is the name of the runtime handler that is expected to be
|
||||
// preconfigured in the test environment.
|
||||
preconfiguredRuntimeHandler = "test-handler"
|
||||
)
|
||||
|
||||
// PreconfiguredRuntimeClassHandler returns configured runtime handler.
|
||||
func PreconfiguredRuntimeClassHandler() string {
|
||||
if TestContext.ContainerRuntime == "docker" {
|
||||
return TestContext.ContainerRuntime
|
||||
}
|
||||
|
||||
return preconfiguredRuntimeHandler
|
||||
}
|
||||
|
||||
@ -4,6 +4,7 @@ go_library(
|
||||
name = "go_default_library",
|
||||
srcs = [
|
||||
"resource.go",
|
||||
"runtimeclass.go",
|
||||
"wait.go",
|
||||
],
|
||||
importpath = "k8s.io/kubernetes/test/e2e/framework/node",
|
||||
@ -22,6 +23,8 @@ go_library(
|
||||
"//test/e2e/framework/log:go_default_library",
|
||||
"//test/e2e/system:go_default_library",
|
||||
"//test/utils:go_default_library",
|
||||
"//test/utils/image:go_default_library",
|
||||
"//vendor/k8s.io/utils/pointer:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
56
test/e2e/framework/node/runtimeclass.go
Normal file
56
test/e2e/framework/node/runtimeclass.go
Normal file
@ -0,0 +1,56 @@
|
||||
/*
|
||||
Copyright 2020 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package node
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
imageutils "k8s.io/kubernetes/test/utils/image"
|
||||
utilpointer "k8s.io/utils/pointer"
|
||||
)
|
||||
|
||||
// PreconfiguredRuntimeClassHandler returns configured runtime handler.
|
||||
func PreconfiguredRuntimeClassHandler(handler string) string {
|
||||
if handler == "docker" {
|
||||
return handler
|
||||
}
|
||||
|
||||
// test-handler is the name of the runtime handler that is expected to be
|
||||
// preconfigured in the test environment.
|
||||
return "test-handler"
|
||||
}
|
||||
|
||||
// NewRuntimeClassPod returns a test pod with the given runtimeClassName
|
||||
func NewRuntimeClassPod(runtimeClassName string) *v1.Pod {
|
||||
return &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
GenerateName: fmt.Sprintf("test-runtimeclass-%s-", runtimeClassName),
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
RuntimeClassName: &runtimeClassName,
|
||||
Containers: []v1.Container{{
|
||||
Name: "test",
|
||||
Image: imageutils.GetE2EImage(imageutils.BusyBox),
|
||||
Command: []string{"true"},
|
||||
}},
|
||||
RestartPolicy: v1.RestartPolicyNever,
|
||||
AutomountServiceAccountToken: utilpointer.BoolPtr(false),
|
||||
},
|
||||
}
|
||||
}
|
||||
@ -57,7 +57,6 @@ go_library(
|
||||
"//test/utils/image:go_default_library",
|
||||
"//vendor/github.com/onsi/ginkgo:go_default_library",
|
||||
"//vendor/github.com/onsi/gomega:go_default_library",
|
||||
"//vendor/k8s.io/utils/pointer:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@ -26,10 +26,9 @@ import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
runtimeclasstest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
|
||||
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
|
||||
"k8s.io/kubernetes/test/e2e/scheduling"
|
||||
imageutils "k8s.io/kubernetes/test/utils/image"
|
||||
utilpointer "k8s.io/utils/pointer"
|
||||
|
||||
"github.com/onsi/ginkgo"
|
||||
"github.com/onsi/gomega"
|
||||
@ -45,12 +44,12 @@ var _ = ginkgo.Describe("[sig-node] RuntimeClass", func() {
|
||||
},
|
||||
}
|
||||
|
||||
runtimeClass := newRuntimeClass(f.Namespace.Name, "conflict-runtimeclass")
|
||||
runtimeClass := newRuntimeClass(f.Namespace.Name, "conflict-runtimeclass", framework.TestContext.ContainerRuntime)
|
||||
runtimeClass.Scheduling = scheduling
|
||||
rc, err := f.ClientSet.NodeV1beta1().RuntimeClasses().Create(context.TODO(), runtimeClass, metav1.CreateOptions{})
|
||||
framework.ExpectNoError(err, "failed to create RuntimeClass resource")
|
||||
|
||||
pod := newRuntimeClassPod(rc.GetName())
|
||||
pod := e2enode.NewRuntimeClassPod(rc.GetName())
|
||||
pod.Spec.NodeSelector = map[string]string{
|
||||
"foo": "bar",
|
||||
}
|
||||
@ -96,12 +95,12 @@ var _ = ginkgo.Describe("[sig-node] RuntimeClass", func() {
|
||||
defer framework.RemoveTaintOffNode(f.ClientSet, nodeName, taint)
|
||||
|
||||
ginkgo.By("Trying to create runtimeclass and pod")
|
||||
runtimeClass := newRuntimeClass(f.Namespace.Name, "non-conflict-runtimeclass")
|
||||
runtimeClass := newRuntimeClass(f.Namespace.Name, "non-conflict-runtimeclass", framework.TestContext.ContainerRuntime)
|
||||
runtimeClass.Scheduling = scheduling
|
||||
rc, err := f.ClientSet.NodeV1beta1().RuntimeClasses().Create(context.TODO(), runtimeClass, metav1.CreateOptions{})
|
||||
framework.ExpectNoError(err, "failed to create RuntimeClass resource")
|
||||
|
||||
pod := newRuntimeClassPod(rc.GetName())
|
||||
pod := e2enode.NewRuntimeClassPod(rc.GetName())
|
||||
pod.Spec.NodeSelector = map[string]string{
|
||||
"foo": "bar",
|
||||
}
|
||||
@ -119,26 +118,7 @@ var _ = ginkgo.Describe("[sig-node] RuntimeClass", func() {
|
||||
})
|
||||
|
||||
// newRuntimeClass returns a test runtime class.
|
||||
func newRuntimeClass(namespace, name string) *nodev1beta1.RuntimeClass {
|
||||
func newRuntimeClass(namespace, name, handler string) *nodev1beta1.RuntimeClass {
|
||||
uniqueName := fmt.Sprintf("%s-%s", namespace, name)
|
||||
return runtimeclasstest.NewRuntimeClass(uniqueName, framework.PreconfiguredRuntimeClassHandler())
|
||||
}
|
||||
|
||||
// newRuntimeClassPod returns a test pod with the given runtimeClassName.
|
||||
func newRuntimeClassPod(runtimeClassName string) *v1.Pod {
|
||||
return &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
GenerateName: fmt.Sprintf("test-runtimeclass-%s-", runtimeClassName),
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
RuntimeClassName: &runtimeClassName,
|
||||
Containers: []v1.Container{{
|
||||
Name: "test",
|
||||
Image: imageutils.GetE2EImage(imageutils.BusyBox),
|
||||
Command: []string{"true"},
|
||||
}},
|
||||
RestartPolicy: v1.RestartPolicyNever,
|
||||
AutomountServiceAccountToken: utilpointer.BoolPtr(false),
|
||||
},
|
||||
}
|
||||
return runtimeclasstest.NewRuntimeClass(uniqueName, e2enode.PreconfiguredRuntimeClassHandler(handler))
|
||||
}
|
||||
|
||||
@ -24,6 +24,7 @@ go_library(
|
||||
"//pkg/scheduler/util:go_default_library",
|
||||
"//staging/src/k8s.io/api/apps/v1:go_default_library",
|
||||
"//staging/src/k8s.io/api/core/v1:go_default_library",
|
||||
"//staging/src/k8s.io/api/node/v1beta1:go_default_library",
|
||||
"//staging/src/k8s.io/api/scheduling/v1:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
|
||||
|
||||
@ -22,6 +22,7 @@ import (
|
||||
"time"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
nodev1beta1 "k8s.io/api/node/v1beta1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
@ -61,6 +62,7 @@ type pausePodConfig struct {
|
||||
Affinity *v1.Affinity
|
||||
Annotations, Labels, NodeSelector map[string]string
|
||||
Resources *v1.ResourceRequirements
|
||||
RuntimeClassHandler *string
|
||||
Tolerations []v1.Toleration
|
||||
NodeName string
|
||||
Ports []v1.ContainerPort
|
||||
@ -196,6 +198,116 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() {
|
||||
verifyResult(cs, podsNeededForSaturation, 1, ns)
|
||||
})
|
||||
|
||||
// This test verifies we don't allow scheduling of pods in a way that sum of limits +
|
||||
// associated overhead is greater than machine's capacity.
|
||||
// It assumes that cluster add-on pods stay stable and cannot be run in parallel
|
||||
// with any other test that touches Nodes or Pods.
|
||||
// Because of this we need to have precise control on what's running in the cluster.
|
||||
// Test scenario:
|
||||
// 1. Find the first ready node on the system, and add a fake resource for test
|
||||
// 2. Create one with affinity to the particular node that uses 70% of the fake resource.
|
||||
// 3. Wait for the pod to be scheduled.
|
||||
// 4. Create another pod with affinity to the particular node that needs 20% of the fake resource and
|
||||
// an overhead set as 25% of the fake resource.
|
||||
// 5. Make sure this additional pod is not scheduled.
|
||||
|
||||
ginkgo.Context("validates pod overhead is considered along with resource limits of pods that are allowed to run", func() {
|
||||
var testNodeName string
|
||||
var handler string
|
||||
var beardsecond v1.ResourceName = "example.com/beardsecond"
|
||||
|
||||
ginkgo.BeforeEach(func() {
|
||||
WaitForStableCluster(cs, masterNodes)
|
||||
ginkgo.By("Add RuntimeClass and fake resource")
|
||||
|
||||
// find a node which can run a pod:
|
||||
testNodeName = GetNodeThatCanRunPod(f)
|
||||
|
||||
// Get node object:
|
||||
node, err := cs.CoreV1().Nodes().Get(context.TODO(), testNodeName, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err, "unable to get node object for node %v", testNodeName)
|
||||
|
||||
// update Node API object with a fake resource
|
||||
nodeCopy := node.DeepCopy()
|
||||
nodeCopy.ResourceVersion = "0"
|
||||
|
||||
nodeCopy.Status.Capacity[beardsecond] = resource.MustParse("1000")
|
||||
_, err = cs.CoreV1().Nodes().UpdateStatus(context.TODO(), nodeCopy, metav1.UpdateOptions{})
|
||||
framework.ExpectNoError(err, "unable to apply fake resource to %v", testNodeName)
|
||||
|
||||
// Register a runtimeClass with overhead set as 25% of the available beard-seconds
|
||||
handler = e2enode.PreconfiguredRuntimeClassHandler(framework.TestContext.ContainerRuntime)
|
||||
|
||||
rc := &nodev1beta1.RuntimeClass{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: handler},
|
||||
Handler: handler,
|
||||
Overhead: &nodev1beta1.Overhead{
|
||||
PodFixed: v1.ResourceList{
|
||||
beardsecond: resource.MustParse("250"),
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err = cs.NodeV1beta1().RuntimeClasses().Create(context.TODO(), rc, metav1.CreateOptions{})
|
||||
framework.ExpectNoError(err, "failed to create RuntimeClass resource")
|
||||
})
|
||||
|
||||
ginkgo.AfterEach(func() {
|
||||
ginkgo.By("Remove fake resource and RuntimeClass")
|
||||
// remove fake resource:
|
||||
if testNodeName != "" {
|
||||
// Get node object:
|
||||
node, err := cs.CoreV1().Nodes().Get(context.TODO(), testNodeName, metav1.GetOptions{})
|
||||
framework.ExpectNoError(err, "unable to get node object for node %v", testNodeName)
|
||||
|
||||
nodeCopy := node.DeepCopy()
|
||||
// force it to update
|
||||
nodeCopy.ResourceVersion = "0"
|
||||
delete(nodeCopy.Status.Capacity, beardsecond)
|
||||
_, err = cs.CoreV1().Nodes().UpdateStatus(context.TODO(), nodeCopy, metav1.UpdateOptions{})
|
||||
framework.ExpectNoError(err, "unable to update node %v", testNodeName)
|
||||
}
|
||||
|
||||
// remove RuntimeClass
|
||||
cs.NodeV1beta1().RuntimeClasses().Delete(context.TODO(), e2enode.PreconfiguredRuntimeClassHandler(framework.TestContext.ContainerRuntime), nil)
|
||||
})
|
||||
|
||||
ginkgo.It("verify pod overhead is accounted for", func() {
|
||||
framework.ExpectEqual(testNodeName != "", true)
|
||||
|
||||
ginkgo.By("Starting Pod to consume most of the node's resource.")
|
||||
|
||||
// Create pod which requires 70% of the available beard-seconds.
|
||||
fillerPod := createPausePod(f, pausePodConfig{
|
||||
Name: "filler-pod-" + string(uuid.NewUUID()),
|
||||
Resources: &v1.ResourceRequirements{
|
||||
Requests: v1.ResourceList{beardsecond: resource.MustParse("700")},
|
||||
Limits: v1.ResourceList{beardsecond: resource.MustParse("700")},
|
||||
},
|
||||
})
|
||||
|
||||
// Wait for filler pod to schedule.
|
||||
framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(cs, fillerPod))
|
||||
|
||||
ginkgo.By("Creating another pod that requires unavailable amount of resources.")
|
||||
// Create another pod that requires 20% of available beard-seconds, but utilizes the RuntimeClass
|
||||
// which defines a pod overhead that requires an additional 25%.
|
||||
// This pod should remain pending as at least 70% of beard-second in
|
||||
// the node are already consumed.
|
||||
podName := "additional-pod" + string(uuid.NewUUID())
|
||||
conf := pausePodConfig{
|
||||
RuntimeClassHandler: &handler,
|
||||
Name: podName,
|
||||
Labels: map[string]string{"name": "additional"},
|
||||
Resources: &v1.ResourceRequirements{
|
||||
Limits: v1.ResourceList{beardsecond: resource.MustParse("200")},
|
||||
},
|
||||
}
|
||||
|
||||
WaitForSchedulerAfterAction(f, createPausePodAction(f, conf), ns, podName, false)
|
||||
verifyResult(cs, 1, 1, ns)
|
||||
})
|
||||
})
|
||||
|
||||
// This test verifies we don't allow scheduling of pods in a way that sum of
|
||||
// resource requests of pods is greater than machines capacity.
|
||||
// It assumes that cluster add-on pods stay stable and cannot be run in parallel
|
||||
@ -715,6 +827,7 @@ func initPausePod(f *framework.Framework, conf pausePodConfig) *v1.Pod {
|
||||
NodeSelector: conf.NodeSelector,
|
||||
Affinity: conf.Affinity,
|
||||
TopologySpreadConstraints: conf.TopologySpreadConstraints,
|
||||
RuntimeClassName: conf.RuntimeClassHandler,
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Name: conf.Name,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user