From 5a77ebe28b75187fc06dcd5a807b4877d524b29a Mon Sep 17 00:00:00 2001 From: Mike Dame Date: Fri, 29 Jan 2021 16:35:38 -0500 Subject: [PATCH] Scheduler: remove pkg/features dependency from NodeResources plugins --- pkg/kubelet/lifecycle/predicate.go | 4 +- pkg/scheduler/core/generic_scheduler_test.go | 12 +++- pkg/scheduler/eventhandlers.go | 4 +- .../default_preemption_test.go | 69 ++++++++++--------- .../framework/plugins/feature/feature.go | 2 + .../noderesources/balanced_allocation.go | 16 +++-- .../noderesources/balanced_allocation_test.go | 3 +- .../framework/plugins/noderesources/fit.go | 17 ++--- .../plugins/noderesources/fit_test.go | 14 ++-- .../plugins/noderesources/least_allocated.go | 4 +- .../noderesources/least_allocated_test.go | 3 +- .../plugins/noderesources/most_allocated.go | 4 +- .../noderesources/most_allocated_test.go | 3 +- .../requested_to_capacity_ratio.go | 10 +-- .../requested_to_capacity_ratio_test.go | 7 +- .../noderesources/resource_allocation.go | 19 ++--- pkg/scheduler/framework/plugins/registry.go | 62 ++++++++++------- pkg/scheduler/scheduler_test.go | 6 +- 18 files changed, 154 insertions(+), 105 deletions(-) diff --git a/pkg/kubelet/lifecycle/predicate.go b/pkg/kubelet/lifecycle/predicate.go index 2a17b1c9384..6ca98aa8518 100644 --- a/pkg/kubelet/lifecycle/predicate.go +++ b/pkg/kubelet/lifecycle/predicate.go @@ -20,9 +20,11 @@ import ( "fmt" v1 "k8s.io/api/core/v1" + "k8s.io/apiserver/pkg/util/feature" v1affinityhelper "k8s.io/component-helpers/scheduling/corev1/nodeaffinity" "k8s.io/klog/v2" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" + "k8s.io/kubernetes/pkg/features" schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeaffinity" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodename" @@ -226,7 +228,7 @@ func GeneralPredicates(pod *v1.Pod, nodeInfo *schedulerframework.NodeInfo) ([]Pr } var reasons []PredicateFailureReason - for _, r := range noderesources.Fits(pod, nodeInfo) { + for _, r := range noderesources.Fits(pod, nodeInfo, feature.DefaultFeatureGate.Enabled(features.PodOverhead)) { reasons = append(reasons, &InsufficientResourceError{ ResourceName: r.ResourceName, Requested: r.Requested, diff --git a/pkg/scheduler/core/generic_scheduler_test.go b/pkg/scheduler/core/generic_scheduler_test.go index 2cc27b31432..e27261a9b3d 100644 --- a/pkg/scheduler/core/generic_scheduler_test.go +++ b/pkg/scheduler/core/generic_scheduler_test.go @@ -31,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + apiruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" @@ -43,6 +44,7 @@ import ( schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/podtopologyspread" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/queuesort" @@ -1310,8 +1312,14 @@ func TestZeroRequest(t *testing.T) { pluginRegistrations := []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), - st.RegisterScorePlugin(noderesources.LeastAllocatedName, noderesources.NewLeastAllocated, 1), - st.RegisterScorePlugin(noderesources.BalancedAllocationName, noderesources.NewBalancedAllocation, 1), + st.RegisterScorePlugin(noderesources.LeastAllocatedName, + func(plArgs apiruntime.Object, fh framework.Handle) (framework.Plugin, error) { + return noderesources.NewLeastAllocated(plArgs, fh, feature.Features{}) + }, + 1), + st.RegisterScorePlugin(noderesources.BalancedAllocationName, func(plArgs apiruntime.Object, fh framework.Handle) (framework.Plugin, error) { + return noderesources.NewBalancedAllocation(plArgs, fh, feature.Features{}) + }, 1), st.RegisterScorePlugin(selectorspread.Name, selectorspread.New, 1), st.RegisterPreScorePlugin(selectorspread.Name, selectorspread.New), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), diff --git a/pkg/scheduler/eventhandlers.go b/pkg/scheduler/eventhandlers.go index 6b224cd4051..15ba318adb5 100644 --- a/pkg/scheduler/eventhandlers.go +++ b/pkg/scheduler/eventhandlers.go @@ -25,10 +25,12 @@ import ( v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" "k8s.io/client-go/tools/cache" v1helper "k8s.io/component-helpers/scheduling/corev1" "k8s.io/component-helpers/scheduling/corev1/nodeaffinity" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/framework" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodename" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeports" @@ -469,7 +471,7 @@ func preCheckForNode(nodeInfo *framework.NodeInfo) queue.PreEnqueueCheck { // cases (e.g., node resizing), "pod" may still fail a check but preemption helps. We deliberately // chose to ignore those cases as unschedulable pods will be re-queued eventually. return func(pod *v1.Pod) bool { - if len(noderesources.Fits(pod, nodeInfo)) != 0 { + if len(noderesources.Fits(pod, nodeInfo, feature.DefaultFeatureGate.Enabled(features.PodOverhead))) != 0 { return false } diff --git a/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption_test.go b/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption_test.go index 194c053a050..b0f3713d6fc 100644 --- a/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption_test.go +++ b/pkg/scheduler/framework/plugins/defaultpreemption/default_preemption_test.go @@ -30,6 +30,7 @@ import ( policy "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + apiruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/informers" clientsetfake "k8s.io/client-go/kubernetes/fake" @@ -98,6 +99,10 @@ func getDefaultDefaultPreemptionArgs() *config.DefaultPreemptionArgs { return dpa } +var nodeResourcesFitFunc = func(plArgs apiruntime.Object, fh framework.Handle) (framework.Plugin, error) { + return noderesources.NewFit(plArgs, fh, feature.Features{}) +} + func TestPostFilter(t *testing.T) { onePodRes := map[v1.ResourceName]string{v1.ResourcePods: "1"} nodeRes := map[v1.ResourceName]string{v1.ResourceCPU: "200m", v1.ResourceMemory: "400"} @@ -270,7 +275,7 @@ func TestPostFilter(t *testing.T) { // Register NodeResourceFit as the Filter & PreFilter plugin. registeredPlugins := []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), - st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), } var extenders []framework.Extender @@ -382,7 +387,7 @@ func TestDryRunPreemption(t *testing.T) { { name: "a pod that fits on both nodes when lower priority pods are preempted", registerPlugins: []st.RegisterPluginFunc{ - st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), }, nodeNames: []string{"node1", "node2"}, testPods: []*v1.Pod{ @@ -413,7 +418,7 @@ func TestDryRunPreemption(t *testing.T) { { name: "a pod that would fit on the nodes, but other pods running are higher priority, no preemption would happen", registerPlugins: []st.RegisterPluginFunc{ - st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), }, nodeNames: []string{"node1", "node2"}, testPods: []*v1.Pod{ @@ -429,7 +434,7 @@ func TestDryRunPreemption(t *testing.T) { { name: "medium priority pod is preempted, but lower priority one stays as it is small", registerPlugins: []st.RegisterPluginFunc{ - st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), }, nodeNames: []string{"node1", "node2"}, testPods: []*v1.Pod{ @@ -461,7 +466,7 @@ func TestDryRunPreemption(t *testing.T) { { name: "mixed priority pods are preempted", registerPlugins: []st.RegisterPluginFunc{ - st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), }, nodeNames: []string{"node1", "node2"}, testPods: []*v1.Pod{ @@ -492,7 +497,7 @@ func TestDryRunPreemption(t *testing.T) { { name: "mixed priority pods are preempted, pick later StartTime one when priorities are equal", registerPlugins: []st.RegisterPluginFunc{ - st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), }, nodeNames: []string{"node1", "node2"}, testPods: []*v1.Pod{ @@ -523,7 +528,7 @@ func TestDryRunPreemption(t *testing.T) { { name: "pod with anti-affinity is preempted", registerPlugins: []st.RegisterPluginFunc{ - st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), st.RegisterPluginAsExtensions(interpodaffinity.Name, func(plArgs runtime.Object, fh framework.Handle) (framework.Plugin, error) { return interpodaffinity.New(plArgs, fh, feature.Features{}) }, "Filter", "PreFilter"), @@ -594,7 +599,7 @@ func TestDryRunPreemption(t *testing.T) { { name: "get Unschedulable in the preemption phase when the filter plugins filtering the nodes", registerPlugins: []st.RegisterPluginFunc{ - st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), }, nodeNames: []string{"node1", "node2"}, testPods: []*v1.Pod{ @@ -611,7 +616,7 @@ func TestDryRunPreemption(t *testing.T) { { name: "preemption with violation of same pdb", registerPlugins: []st.RegisterPluginFunc{ - st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), }, nodeNames: []string{"node1"}, testPods: []*v1.Pod{ @@ -646,7 +651,7 @@ func TestDryRunPreemption(t *testing.T) { { name: "preemption with violation of the pdb with pod whose eviction was processed, the victim doesn't belong to DisruptedPods", registerPlugins: []st.RegisterPluginFunc{ - st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), }, nodeNames: []string{"node1"}, testPods: []*v1.Pod{ @@ -681,7 +686,7 @@ func TestDryRunPreemption(t *testing.T) { { name: "preemption with violation of the pdb with pod whose eviction was processed, the victim belongs to DisruptedPods", registerPlugins: []st.RegisterPluginFunc{ - st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), }, nodeNames: []string{"node1"}, testPods: []*v1.Pod{ @@ -716,7 +721,7 @@ func TestDryRunPreemption(t *testing.T) { { name: "preemption with violation of the pdb with pod whose eviction was processed, the victim which belongs to DisruptedPods is treated as 'nonViolating'", registerPlugins: []st.RegisterPluginFunc{ - st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), }, nodeNames: []string{"node1"}, testPods: []*v1.Pod{ @@ -754,7 +759,7 @@ func TestDryRunPreemption(t *testing.T) { name: "all nodes are possible candidates, but DefaultPreemptionArgs limits to 2", args: &config.DefaultPreemptionArgs{MinCandidateNodesPercentage: 40, MinCandidateNodesAbsolute: 1}, registerPlugins: []st.RegisterPluginFunc{ - st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), }, nodeNames: []string{"node1", "node2", "node3", "node4", "node5"}, testPods: []*v1.Pod{ @@ -791,7 +796,7 @@ func TestDryRunPreemption(t *testing.T) { name: "some nodes are not possible candidates, DefaultPreemptionArgs limits to 2", args: &config.DefaultPreemptionArgs{MinCandidateNodesPercentage: 40, MinCandidateNodesAbsolute: 1}, registerPlugins: []st.RegisterPluginFunc{ - st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), }, nodeNames: []string{"node1", "node2", "node3", "node4", "node5"}, testPods: []*v1.Pod{ @@ -828,7 +833,7 @@ func TestDryRunPreemption(t *testing.T) { name: "preemption offset across multiple scheduling cycles and wrap around", args: &config.DefaultPreemptionArgs{MinCandidateNodesPercentage: 40, MinCandidateNodesAbsolute: 1}, registerPlugins: []st.RegisterPluginFunc{ - st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), }, nodeNames: []string{"node1", "node2", "node3", "node4", "node5"}, testPods: []*v1.Pod{ @@ -897,7 +902,7 @@ func TestDryRunPreemption(t *testing.T) { name: "preemption looks past numCandidates until a non-PDB violating node is found", args: &config.DefaultPreemptionArgs{MinCandidateNodesPercentage: 40, MinCandidateNodesAbsolute: 2}, registerPlugins: []st.RegisterPluginFunc{ - st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), }, nodeNames: []string{"node1", "node2", "node3", "node4", "node5"}, testPods: []*v1.Pod{ @@ -1067,7 +1072,7 @@ func TestSelectBestCandidate(t *testing.T) { }{ { name: "a pod that fits on both nodes when lower priority pods are preempted", - registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), nodeNames: []string{"node1", "node2"}, pod: st.MakePod().Name("p").UID("p").Priority(highPriority).Req(largeRes).Obj(), pods: []*v1.Pod{ @@ -1078,7 +1083,7 @@ func TestSelectBestCandidate(t *testing.T) { }, { name: "node with min highest priority pod is picked", - registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), nodeNames: []string{"node1", "node2", "node3"}, pod: st.MakePod().Name("p").UID("p").Priority(highPriority).Req(veryLargeRes).Obj(), pods: []*v1.Pod{ @@ -1093,7 +1098,7 @@ func TestSelectBestCandidate(t *testing.T) { }, { name: "when highest priorities are the same, minimum sum of priorities is picked", - registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), nodeNames: []string{"node1", "node2", "node3"}, pod: st.MakePod().Name("p").UID("p").Priority(highPriority).Req(veryLargeRes).Obj(), pods: []*v1.Pod{ @@ -1108,7 +1113,7 @@ func TestSelectBestCandidate(t *testing.T) { }, { name: "when highest priority and sum are the same, minimum number of pods is picked", - registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), nodeNames: []string{"node1", "node2", "node3"}, pod: st.MakePod().Name("p").UID("p").Priority(highPriority).Req(veryLargeRes).Obj(), pods: []*v1.Pod{ @@ -1128,7 +1133,7 @@ func TestSelectBestCandidate(t *testing.T) { // pickOneNodeForPreemption adjusts pod priorities when finding the sum of the victims. This // test ensures that the logic works correctly. name: "sum of adjusted priorities is considered", - registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), nodeNames: []string{"node1", "node2", "node3"}, pod: st.MakePod().Name("p").UID("p").Priority(highPriority).Req(veryLargeRes).Obj(), pods: []*v1.Pod{ @@ -1145,7 +1150,7 @@ func TestSelectBestCandidate(t *testing.T) { }, { name: "non-overlapping lowest high priority, sum priorities, and number of pods", - registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), nodeNames: []string{"node1", "node2", "node3", "node4"}, pod: st.MakePod().Name("p").UID("p").Priority(veryHighPriority).Req(veryLargeRes).Obj(), pods: []*v1.Pod{ @@ -1166,7 +1171,7 @@ func TestSelectBestCandidate(t *testing.T) { }, { name: "same priority, same number of victims, different start time for each node's pod", - registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), nodeNames: []string{"node1", "node2", "node3"}, pod: st.MakePod().Name("p").UID("p").Priority(highPriority).Req(veryLargeRes).Obj(), pods: []*v1.Pod{ @@ -1181,7 +1186,7 @@ func TestSelectBestCandidate(t *testing.T) { }, { name: "same priority, same number of victims, different start time for all pods", - registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), nodeNames: []string{"node1", "node2", "node3"}, pod: st.MakePod().Name("p").UID("p").Priority(highPriority).Req(veryLargeRes).Obj(), pods: []*v1.Pod{ @@ -1196,7 +1201,7 @@ func TestSelectBestCandidate(t *testing.T) { }, { name: "different priority, same number of victims, different start time for all pods", - registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), nodeNames: []string{"node1", "node2", "node3"}, pod: st.MakePod().Name("p").UID("p").Priority(highPriority).Req(veryLargeRes).Obj(), pods: []*v1.Pod{ @@ -1459,7 +1464,7 @@ func TestPreempt(t *testing.T) { st.MakePod().Name("p3.1").UID("p3.1").Node("node3").Priority(midPriority).Req(mediumRes).Obj(), }, nodeNames: []string{"node1", "node2", "node3"}, - registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), expectedNode: "node1", expectedPods: []string{"p1.1", "p1.2"}, }, @@ -1494,7 +1499,7 @@ func TestPreempt(t *testing.T) { {Predicates: []st.FitPredicate{st.TruePredicateExtender}}, {Predicates: []st.FitPredicate{st.Node1PredicateExtender}}, }, - registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), expectedNode: "node1", expectedPods: []string{"p1.1", "p1.2"}, }, @@ -1510,7 +1515,7 @@ func TestPreempt(t *testing.T) { extenders: []*st.FakeExtender{ {Predicates: []st.FitPredicate{st.FalsePredicateExtender}}, }, - registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), expectedNode: "", expectedPods: []string{}, }, @@ -1527,7 +1532,7 @@ func TestPreempt(t *testing.T) { {Predicates: []st.FitPredicate{st.ErrorPredicateExtender}, Ignorable: true}, {Predicates: []st.FitPredicate{st.Node1PredicateExtender}}, }, - registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), expectedNode: "node1", expectedPods: []string{"p1.1", "p1.2"}, }, @@ -1544,7 +1549,7 @@ func TestPreempt(t *testing.T) { {Predicates: []st.FitPredicate{st.Node1PredicateExtender}, UnInterested: true}, {Predicates: []st.FitPredicate{st.TruePredicateExtender}}, }, - registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), //sum of priorities of all victims on node1 is larger than node2, node2 is chosen. expectedNode: "node2", expectedPods: []string{"p2.1"}, @@ -1559,7 +1564,7 @@ func TestPreempt(t *testing.T) { st.MakePod().Name("p3.1").UID("p3.1").Namespace(v1.NamespaceDefault).Node("node3").Priority(midPriority).Req(mediumRes).Obj(), }, nodeNames: []string{"node1", "node2", "node3"}, - registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), expectedNode: "", expectedPods: nil, }, @@ -1573,7 +1578,7 @@ func TestPreempt(t *testing.T) { st.MakePod().Name("p3.1").UID("p3.1").Namespace(v1.NamespaceDefault).Node("node3").Priority(midPriority).Req(mediumRes).Obj(), }, nodeNames: []string{"node1", "node2", "node3"}, - registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + registerPlugin: st.RegisterPluginAsExtensions(noderesources.FitName, nodeResourcesFitFunc, "Filter", "PreFilter"), expectedNode: "node1", expectedPods: []string{"p1.1", "p1.2"}, }, diff --git a/pkg/scheduler/framework/plugins/feature/feature.go b/pkg/scheduler/framework/plugins/feature/feature.go index 60313ee2f96..5575b9ab134 100644 --- a/pkg/scheduler/framework/plugins/feature/feature.go +++ b/pkg/scheduler/framework/plugins/feature/feature.go @@ -22,4 +22,6 @@ package feature type Features struct { EnablePodAffinityNamespaceSelector bool EnablePodDisruptionBudget bool + EnablePodOverhead bool + EnableBalanceAttachedNodeVolumes bool } diff --git a/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go b/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go index 93faebe5b69..dc8fd625ae2 100644 --- a/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go +++ b/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go @@ -23,9 +23,8 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" - utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/framework" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" ) // BalancedAllocation is a score plugin that calculates the difference between the cpu and memory fraction @@ -67,13 +66,15 @@ func (ba *BalancedAllocation) ScoreExtensions() framework.ScoreExtensions { } // NewBalancedAllocation initializes a new plugin and returns it. -func NewBalancedAllocation(_ runtime.Object, h framework.Handle) (framework.Plugin, error) { +func NewBalancedAllocation(_ runtime.Object, h framework.Handle, fts feature.Features) (framework.Plugin, error) { return &BalancedAllocation{ handle: h, resourceAllocationScorer: resourceAllocationScorer{ - BalancedAllocationName, - balancedResourceScorer, - defaultRequestedRatioResources, + Name: BalancedAllocationName, + scorer: balancedResourceScorer, + resourceToWeightMap: defaultRequestedRatioResources, + enablePodOverhead: fts.EnablePodOverhead, + enableBalanceAttachedNodeVolumes: fts.EnableBalanceAttachedNodeVolumes, }, }, nil } @@ -88,7 +89,8 @@ func balancedResourceScorer(requested, allocable resourceToValueMap, includeVolu return 0 } - if includeVolumes && utilfeature.DefaultFeatureGate.Enabled(features.BalanceAttachedNodeVolumes) && allocatableVolumes > 0 { + // includeVolumes is only true when BalanceAttachedNodeVolumes feature gate is enabled (see resource_allocation.go#score()) + if includeVolumes && allocatableVolumes > 0 { volumeFraction := float64(requestedVolumes) / float64(allocatableVolumes) if volumeFraction >= 1 { // if requested >= capacity, the corresponding host should never be preferred. diff --git a/pkg/scheduler/framework/plugins/noderesources/balanced_allocation_test.go b/pkg/scheduler/framework/plugins/noderesources/balanced_allocation_test.go index ac4267d339e..063d9170f6c 100644 --- a/pkg/scheduler/framework/plugins/noderesources/balanced_allocation_test.go +++ b/pkg/scheduler/framework/plugins/noderesources/balanced_allocation_test.go @@ -28,6 +28,7 @@ import ( featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/framework" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" "k8s.io/kubernetes/pkg/scheduler/framework/runtime" "k8s.io/kubernetes/pkg/scheduler/internal/cache" ) @@ -390,7 +391,7 @@ func TestNodeResourcesBalancedAllocation(t *testing.T) { } } fh, _ := runtime.NewFramework(nil, nil, runtime.WithSnapshotSharedLister(snapshot)) - p, _ := NewBalancedAllocation(nil, fh) + p, _ := NewBalancedAllocation(nil, fh, feature.Features{EnablePodOverhead: true, EnableBalanceAttachedNodeVolumes: true}) for i := range test.nodes { hostResult, err := p.(framework.ScorePlugin).Score(context.Background(), nil, test.pod, test.nodes[i].Name) diff --git a/pkg/scheduler/framework/plugins/noderesources/fit.go b/pkg/scheduler/framework/plugins/noderesources/fit.go index 6f2d0f95714..4d47287dff3 100644 --- a/pkg/scheduler/framework/plugins/noderesources/fit.go +++ b/pkg/scheduler/framework/plugins/noderesources/fit.go @@ -24,12 +24,11 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" - utilfeature "k8s.io/apiserver/pkg/util/feature" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/apis/config/validation" "k8s.io/kubernetes/pkg/scheduler/framework" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" ) var _ framework.PreFilterPlugin = &Fit{} @@ -49,6 +48,7 @@ const ( type Fit struct { ignoredResources sets.String ignoredResourceGroups sets.String + enablePodOverhead bool } // preFilterState computed at PreFilter and used at Filter. @@ -67,7 +67,7 @@ func (f *Fit) Name() string { } // NewFit initializes a new plugin and returns it. -func NewFit(plArgs runtime.Object, _ framework.Handle) (framework.Plugin, error) { +func NewFit(plArgs runtime.Object, _ framework.Handle, fts feature.Features) (framework.Plugin, error) { args, ok := plArgs.(*config.NodeResourcesFitArgs) if !ok { return nil, fmt.Errorf("want args to be of type NodeResourcesFitArgs, got %T", plArgs) @@ -78,6 +78,7 @@ func NewFit(plArgs runtime.Object, _ framework.Handle) (framework.Plugin, error) return &Fit{ ignoredResources: sets.NewString(args.IgnoredResources...), ignoredResourceGroups: sets.NewString(args.IgnoredResourceGroups...), + enablePodOverhead: fts.EnablePodOverhead, }, nil } @@ -108,7 +109,7 @@ func NewFit(plArgs runtime.Object, _ framework.Handle) (framework.Plugin, error) // Memory: 1G // // Result: CPU: 3, Memory: 3G -func computePodResourceRequest(pod *v1.Pod) *preFilterState { +func computePodResourceRequest(pod *v1.Pod, enablePodOverhead bool) *preFilterState { result := &preFilterState{} for _, container := range pod.Spec.Containers { result.Add(container.Resources.Requests) @@ -120,7 +121,7 @@ func computePodResourceRequest(pod *v1.Pod) *preFilterState { } // If Overhead is being utilized, add to the total requests for the pod - if pod.Spec.Overhead != nil && utilfeature.DefaultFeatureGate.Enabled(features.PodOverhead) { + if pod.Spec.Overhead != nil && enablePodOverhead { result.Add(pod.Spec.Overhead) } @@ -129,7 +130,7 @@ func computePodResourceRequest(pod *v1.Pod) *preFilterState { // PreFilter invoked at the prefilter extension point. func (f *Fit) PreFilter(ctx context.Context, cycleState *framework.CycleState, pod *v1.Pod) *framework.Status { - cycleState.Write(preFilterStateKey, computePodResourceRequest(pod)) + cycleState.Write(preFilterStateKey, computePodResourceRequest(pod, f.enablePodOverhead)) return nil } @@ -198,8 +199,8 @@ type InsufficientResource struct { } // Fits checks if node have enough resources to host the pod. -func Fits(pod *v1.Pod, nodeInfo *framework.NodeInfo) []InsufficientResource { - return fitsRequest(computePodResourceRequest(pod), nodeInfo, nil, nil) +func Fits(pod *v1.Pod, nodeInfo *framework.NodeInfo, enablePodOverhead bool) []InsufficientResource { + return fitsRequest(computePodResourceRequest(pod, enablePodOverhead), nodeInfo, nil, nil) } func fitsRequest(podRequest *preFilterState, nodeInfo *framework.NodeInfo, ignoredExtendedResources, ignoredResourceGroups sets.String) []InsufficientResource { diff --git a/pkg/scheduler/framework/plugins/noderesources/fit_test.go b/pkg/scheduler/framework/plugins/noderesources/fit_test.go index bdd67a356c9..ec2ed8471f8 100644 --- a/pkg/scheduler/framework/plugins/noderesources/fit_test.go +++ b/pkg/scheduler/framework/plugins/noderesources/fit_test.go @@ -27,9 +27,9 @@ import ( "k8s.io/apiserver/pkg/util/feature" "k8s.io/component-base/featuregate" featuregatetesting "k8s.io/component-base/featuregate/testing" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework" + plfeature "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" ) var ( @@ -414,7 +414,7 @@ func TestEnoughRequests(t *testing.T) { node := v1.Node{Status: v1.NodeStatus{Capacity: makeResources(10, 20, 32, 5, 20, 5).Capacity, Allocatable: makeAllocatableResources(10, 20, 32, 5, 20, 5)}} test.nodeInfo.SetNode(&node) - p, err := NewFit(&test.args, nil) + p, err := NewFit(&test.args, nil, plfeature.Features{EnablePodOverhead: true}) if err != nil { t.Fatal(err) } @@ -429,7 +429,7 @@ func TestEnoughRequests(t *testing.T) { t.Errorf("status does not match: %v, want: %v", gotStatus, test.wantStatus) } - gotInsufficientResources := fitsRequest(computePodResourceRequest(test.pod), test.nodeInfo, p.(*Fit).ignoredResources, p.(*Fit).ignoredResourceGroups) + gotInsufficientResources := fitsRequest(computePodResourceRequest(test.pod, true), test.nodeInfo, p.(*Fit).ignoredResources, p.(*Fit).ignoredResourceGroups) if !reflect.DeepEqual(gotInsufficientResources, test.wantInsufficientResources) { t.Errorf("insufficient resources do not match: %+v, want: %v", gotInsufficientResources, test.wantInsufficientResources) } @@ -442,7 +442,7 @@ func TestPreFilterDisabled(t *testing.T) { nodeInfo := framework.NewNodeInfo() node := v1.Node{} nodeInfo.SetNode(&node) - p, err := NewFit(&config.NodeResourcesFitArgs{}, nil) + p, err := NewFit(&config.NodeResourcesFitArgs{}, nil, plfeature.Features{EnablePodOverhead: true}) if err != nil { t.Fatal(err) } @@ -492,7 +492,7 @@ func TestNotEnoughRequests(t *testing.T) { node := v1.Node{Status: v1.NodeStatus{Capacity: v1.ResourceList{}, Allocatable: makeAllocatableResources(10, 20, 1, 0, 0, 0)}} test.nodeInfo.SetNode(&node) - p, err := NewFit(&config.NodeResourcesFitArgs{}, nil) + p, err := NewFit(&config.NodeResourcesFitArgs{}, nil, plfeature.Features{EnablePodOverhead: true}) if err != nil { t.Fatal(err) } @@ -545,7 +545,7 @@ func TestStorageRequests(t *testing.T) { newResourcePod(framework.Resource{MilliCPU: 2, Memory: 2})), name: "ephemeral local storage request is ignored due to disabled feature gate", features: map[featuregate.Feature]bool{ - features.LocalStorageCapacityIsolation: false, + "LocalStorageCapacityIsolation": false, }, }, { @@ -564,7 +564,7 @@ func TestStorageRequests(t *testing.T) { node := v1.Node{Status: v1.NodeStatus{Capacity: makeResources(10, 20, 32, 5, 20, 5).Capacity, Allocatable: makeAllocatableResources(10, 20, 32, 5, 20, 5)}} test.nodeInfo.SetNode(&node) - p, err := NewFit(&config.NodeResourcesFitArgs{}, nil) + p, err := NewFit(&config.NodeResourcesFitArgs{}, nil, plfeature.Features{EnablePodOverhead: true}) if err != nil { t.Fatal(err) } diff --git a/pkg/scheduler/framework/plugins/noderesources/least_allocated.go b/pkg/scheduler/framework/plugins/noderesources/least_allocated.go index e9ce2c41100..b4419e61d78 100644 --- a/pkg/scheduler/framework/plugins/noderesources/least_allocated.go +++ b/pkg/scheduler/framework/plugins/noderesources/least_allocated.go @@ -25,6 +25,7 @@ import ( "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/apis/config/validation" "k8s.io/kubernetes/pkg/scheduler/framework" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" ) // LeastAllocated is a score plugin that favors nodes with fewer allocation requested resources based on requested resources. @@ -65,7 +66,7 @@ func (la *LeastAllocated) ScoreExtensions() framework.ScoreExtensions { } // NewLeastAllocated initializes a new plugin and returns it. -func NewLeastAllocated(laArgs runtime.Object, h framework.Handle) (framework.Plugin, error) { +func NewLeastAllocated(laArgs runtime.Object, h framework.Handle, fts feature.Features) (framework.Plugin, error) { args, ok := laArgs.(*config.NodeResourcesLeastAllocatedArgs) if !ok { return nil, fmt.Errorf("want args to be of type NodeResourcesLeastAllocatedArgs, got %T", laArgs) @@ -85,6 +86,7 @@ func NewLeastAllocated(laArgs runtime.Object, h framework.Handle) (framework.Plu Name: LeastAllocatedName, scorer: leastResourceScorer(resToWeightMap), resourceToWeightMap: resToWeightMap, + enablePodOverhead: fts.EnablePodOverhead, }, }, nil } diff --git a/pkg/scheduler/framework/plugins/noderesources/least_allocated_test.go b/pkg/scheduler/framework/plugins/noderesources/least_allocated_test.go index 0543d1c9bae..2fd1fd59765 100644 --- a/pkg/scheduler/framework/plugins/noderesources/least_allocated_test.go +++ b/pkg/scheduler/framework/plugins/noderesources/least_allocated_test.go @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" "k8s.io/kubernetes/pkg/scheduler/framework/runtime" "k8s.io/kubernetes/pkg/scheduler/internal/cache" ) @@ -313,7 +314,7 @@ func TestNodeResourcesLeastAllocated(t *testing.T) { t.Run(test.name, func(t *testing.T) { snapshot := cache.NewSnapshot(test.pods, test.nodes) fh, _ := runtime.NewFramework(nil, nil, runtime.WithSnapshotSharedLister(snapshot)) - p, err := NewLeastAllocated(&test.args, fh) + p, err := NewLeastAllocated(&test.args, fh, feature.Features{EnablePodOverhead: true}) if test.wantErr != nil { if err != nil { diff --git a/pkg/scheduler/framework/plugins/noderesources/most_allocated.go b/pkg/scheduler/framework/plugins/noderesources/most_allocated.go index 8e45b5d2118..3e77e2008f9 100644 --- a/pkg/scheduler/framework/plugins/noderesources/most_allocated.go +++ b/pkg/scheduler/framework/plugins/noderesources/most_allocated.go @@ -25,6 +25,7 @@ import ( "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/apis/config/validation" "k8s.io/kubernetes/pkg/scheduler/framework" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" ) // MostAllocated is a score plugin that favors nodes with high allocation based on requested resources. @@ -63,7 +64,7 @@ func (ma *MostAllocated) ScoreExtensions() framework.ScoreExtensions { } // NewMostAllocated initializes a new plugin and returns it. -func NewMostAllocated(maArgs runtime.Object, h framework.Handle) (framework.Plugin, error) { +func NewMostAllocated(maArgs runtime.Object, h framework.Handle, fts feature.Features) (framework.Plugin, error) { args, ok := maArgs.(*config.NodeResourcesMostAllocatedArgs) if !ok { return nil, fmt.Errorf("want args to be of type NodeResourcesMostAllocatedArgs, got %T", args) @@ -83,6 +84,7 @@ func NewMostAllocated(maArgs runtime.Object, h framework.Handle) (framework.Plug Name: MostAllocatedName, scorer: mostResourceScorer(resToWeightMap), resourceToWeightMap: resToWeightMap, + enablePodOverhead: fts.EnablePodOverhead, }, }, nil } diff --git a/pkg/scheduler/framework/plugins/noderesources/most_allocated_test.go b/pkg/scheduler/framework/plugins/noderesources/most_allocated_test.go index c341dfcede8..87213946830 100644 --- a/pkg/scheduler/framework/plugins/noderesources/most_allocated_test.go +++ b/pkg/scheduler/framework/plugins/noderesources/most_allocated_test.go @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" "k8s.io/kubernetes/pkg/scheduler/framework/runtime" "k8s.io/kubernetes/pkg/scheduler/internal/cache" ) @@ -273,7 +274,7 @@ func TestNodeResourcesMostAllocated(t *testing.T) { t.Run(test.name, func(t *testing.T) { snapshot := cache.NewSnapshot(test.pods, test.nodes) fh, _ := runtime.NewFramework(nil, nil, runtime.WithSnapshotSharedLister(snapshot)) - p, err := NewMostAllocated(&test.args, fh) + p, err := NewMostAllocated(&test.args, fh, feature.Features{EnablePodOverhead: true}) if test.wantErr != nil { if err != nil { diff --git a/pkg/scheduler/framework/plugins/noderesources/requested_to_capacity_ratio.go b/pkg/scheduler/framework/plugins/noderesources/requested_to_capacity_ratio.go index f9d7f9af65d..d9cf416674e 100644 --- a/pkg/scheduler/framework/plugins/noderesources/requested_to_capacity_ratio.go +++ b/pkg/scheduler/framework/plugins/noderesources/requested_to_capacity_ratio.go @@ -26,6 +26,7 @@ import ( "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/apis/config/validation" "k8s.io/kubernetes/pkg/scheduler/framework" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/helper" ) @@ -36,7 +37,7 @@ const ( ) // NewRequestedToCapacityRatio initializes a new plugin and returns it. -func NewRequestedToCapacityRatio(plArgs runtime.Object, handle framework.Handle) (framework.Plugin, error) { +func NewRequestedToCapacityRatio(plArgs runtime.Object, handle framework.Handle, fts feature.Features) (framework.Plugin, error) { args, err := getRequestedToCapacityRatioArgs(plArgs) if err != nil { return nil, err @@ -68,9 +69,10 @@ func NewRequestedToCapacityRatio(plArgs runtime.Object, handle framework.Handle) return &RequestedToCapacityRatio{ handle: handle, resourceAllocationScorer: resourceAllocationScorer{ - RequestedToCapacityRatioName, - buildRequestedToCapacityRatioScorerFunction(shape, resourceToWeightMap), - resourceToWeightMap, + Name: RequestedToCapacityRatioName, + scorer: buildRequestedToCapacityRatioScorerFunction(shape, resourceToWeightMap), + resourceToWeightMap: resourceToWeightMap, + enablePodOverhead: fts.EnablePodOverhead, }, }, nil } diff --git a/pkg/scheduler/framework/plugins/noderesources/requested_to_capacity_ratio_test.go b/pkg/scheduler/framework/plugins/noderesources/requested_to_capacity_ratio_test.go index 663d02e703f..62d5e7829da 100644 --- a/pkg/scheduler/framework/plugins/noderesources/requested_to_capacity_ratio_test.go +++ b/pkg/scheduler/framework/plugins/noderesources/requested_to_capacity_ratio_test.go @@ -27,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/helper" "k8s.io/kubernetes/pkg/scheduler/framework/runtime" "k8s.io/kubernetes/pkg/scheduler/internal/cache" @@ -80,7 +81,7 @@ func TestRequestedToCapacityRatio(t *testing.T) { {Name: "cpu", Weight: 1}, }, } - p, err := NewRequestedToCapacityRatio(&args, fh) + p, err := NewRequestedToCapacityRatio(&args, fh, feature.Features{EnablePodOverhead: true}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -333,7 +334,7 @@ func TestResourceBinPackingSingleExtended(t *testing.T) { {Name: "intel.com/foo", Weight: 1}, }, } - p, err := NewRequestedToCapacityRatio(&args, fh) + p, err := NewRequestedToCapacityRatio(&args, fh, feature.Features{EnablePodOverhead: true}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -577,7 +578,7 @@ func TestResourceBinPackingMultipleExtended(t *testing.T) { {Name: "intel.com/bar", Weight: 5}, }, } - p, err := NewRequestedToCapacityRatio(&args, fh) + p, err := NewRequestedToCapacityRatio(&args, fh, feature.Features{EnablePodOverhead: true}) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/pkg/scheduler/framework/plugins/noderesources/resource_allocation.go b/pkg/scheduler/framework/plugins/noderesources/resource_allocation.go index 33e36abd645..e4a6e3f3ae1 100644 --- a/pkg/scheduler/framework/plugins/noderesources/resource_allocation.go +++ b/pkg/scheduler/framework/plugins/noderesources/resource_allocation.go @@ -18,9 +18,7 @@ package noderesources import ( v1 "k8s.io/api/core/v1" - utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/klog/v2" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/framework" schedutil "k8s.io/kubernetes/pkg/scheduler/util" ) @@ -36,6 +34,9 @@ type resourceAllocationScorer struct { Name string scorer func(requested, allocable resourceToValueMap, includeVolumes bool, requestedVolumes int, allocatableVolumes int) int64 resourceToWeightMap resourceToWeightMap + + enablePodOverhead bool + enableBalanceAttachedNodeVolumes bool } // resourceToValueMap contains resource name and score. @@ -55,18 +56,18 @@ func (r *resourceAllocationScorer) score( requested := make(resourceToValueMap, len(r.resourceToWeightMap)) allocatable := make(resourceToValueMap, len(r.resourceToWeightMap)) for resource := range r.resourceToWeightMap { - allocatable[resource], requested[resource] = calculateResourceAllocatableRequest(nodeInfo, pod, resource) + allocatable[resource], requested[resource] = calculateResourceAllocatableRequest(nodeInfo, pod, resource, r.enablePodOverhead) } var score int64 // Check if the pod has volumes and this could be added to scorer function for balanced resource allocation. - if len(pod.Spec.Volumes) > 0 && utilfeature.DefaultFeatureGate.Enabled(features.BalanceAttachedNodeVolumes) && nodeInfo.TransientInfo != nil { + if len(pod.Spec.Volumes) > 0 && r.enableBalanceAttachedNodeVolumes && nodeInfo.TransientInfo != nil { score = r.scorer(requested, allocatable, true, nodeInfo.TransientInfo.TransNodeInfo.RequestedVolumes, nodeInfo.TransientInfo.TransNodeInfo.AllocatableVolumesCount) } else { score = r.scorer(requested, allocatable, false, 0, 0) } if klog.V(10).Enabled() { - if len(pod.Spec.Volumes) > 0 && utilfeature.DefaultFeatureGate.Enabled(features.BalanceAttachedNodeVolumes) && nodeInfo.TransientInfo != nil { + if len(pod.Spec.Volumes) > 0 && r.enableBalanceAttachedNodeVolumes && nodeInfo.TransientInfo != nil { klog.Infof( "%v -> %v: %v, map of allocatable resources %v, map of requested resources %v , allocatable volumes %d, requested volumes %d, score %d", pod.Name, node.Name, r.Name, @@ -88,8 +89,8 @@ func (r *resourceAllocationScorer) score( } // calculateResourceAllocatableRequest returns resources Allocatable and Requested values -func calculateResourceAllocatableRequest(nodeInfo *framework.NodeInfo, pod *v1.Pod, resource v1.ResourceName) (int64, int64) { - podRequest := calculatePodResourceRequest(pod, resource) +func calculateResourceAllocatableRequest(nodeInfo *framework.NodeInfo, pod *v1.Pod, resource v1.ResourceName, enablePodOverhead bool) (int64, int64) { + podRequest := calculatePodResourceRequest(pod, resource, enablePodOverhead) switch resource { case v1.ResourceCPU: return nodeInfo.Allocatable.MilliCPU, (nodeInfo.NonZeroRequested.MilliCPU + podRequest) @@ -114,7 +115,7 @@ func calculateResourceAllocatableRequest(nodeInfo *framework.NodeInfo, pod *v1.P // calculatePodResourceRequest returns the total non-zero requests. If Overhead is defined for the pod and the // PodOverhead feature is enabled, the Overhead is added to the result. // podResourceRequest = max(sum(podSpec.Containers), podSpec.InitContainers) + overHead -func calculatePodResourceRequest(pod *v1.Pod, resource v1.ResourceName) int64 { +func calculatePodResourceRequest(pod *v1.Pod, resource v1.ResourceName, enablePodOverhead bool) int64 { var podRequest int64 for i := range pod.Spec.Containers { container := &pod.Spec.Containers[i] @@ -131,7 +132,7 @@ func calculatePodResourceRequest(pod *v1.Pod, resource v1.ResourceName) int64 { } // If Overhead is being utilized, add to the total requests for the pod - if pod.Spec.Overhead != nil && utilfeature.DefaultFeatureGate.Enabled(features.PodOverhead) { + if pod.Spec.Overhead != nil && enablePodOverhead { if quantity, found := pod.Spec.Overhead[resource]; found { podRequest += quantity.Value() } diff --git a/pkg/scheduler/framework/plugins/registry.go b/pkg/scheduler/framework/plugins/registry.go index f4d370273a5..9436b3c79ab 100644 --- a/pkg/scheduler/framework/plugins/registry.go +++ b/pkg/scheduler/framework/plugins/registry.go @@ -18,7 +18,7 @@ package plugins import ( apiruntime "k8s.io/apimachinery/pkg/runtime" - utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/framework" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder" @@ -50,33 +50,45 @@ import ( // through the WithFrameworkOutOfTreeRegistry option. func NewInTreeRegistry() runtime.Registry { fts := plfeature.Features{ - EnablePodAffinityNamespaceSelector: utilfeature.DefaultFeatureGate.Enabled(features.PodAffinityNamespaceSelector), - EnablePodDisruptionBudget: utilfeature.DefaultFeatureGate.Enabled(features.PodDisruptionBudget), + EnablePodAffinityNamespaceSelector: feature.DefaultFeatureGate.Enabled(features.PodAffinityNamespaceSelector), + EnablePodDisruptionBudget: feature.DefaultFeatureGate.Enabled(features.PodDisruptionBudget), + EnablePodOverhead: feature.DefaultFeatureGate.Enabled(features.PodOverhead), + EnableBalanceAttachedNodeVolumes: feature.DefaultFeatureGate.Enabled(features.BalanceAttachedNodeVolumes), } return runtime.Registry{ - selectorspread.Name: selectorspread.New, - imagelocality.Name: imagelocality.New, - tainttoleration.Name: tainttoleration.New, - nodename.Name: nodename.New, - nodeports.Name: nodeports.New, - nodepreferavoidpods.Name: nodepreferavoidpods.New, - nodeaffinity.Name: nodeaffinity.New, - podtopologyspread.Name: podtopologyspread.New, - nodeunschedulable.Name: nodeunschedulable.New, - noderesources.FitName: noderesources.NewFit, - noderesources.BalancedAllocationName: noderesources.NewBalancedAllocation, - noderesources.MostAllocatedName: noderesources.NewMostAllocated, - noderesources.LeastAllocatedName: noderesources.NewLeastAllocated, - noderesources.RequestedToCapacityRatioName: noderesources.NewRequestedToCapacityRatio, - volumebinding.Name: volumebinding.New, - volumerestrictions.Name: volumerestrictions.New, - volumezone.Name: volumezone.New, - nodevolumelimits.CSIName: nodevolumelimits.NewCSI, - nodevolumelimits.EBSName: nodevolumelimits.NewEBS, - nodevolumelimits.GCEPDName: nodevolumelimits.NewGCEPD, - nodevolumelimits.AzureDiskName: nodevolumelimits.NewAzureDisk, - nodevolumelimits.CinderName: nodevolumelimits.NewCinder, + selectorspread.Name: selectorspread.New, + imagelocality.Name: imagelocality.New, + tainttoleration.Name: tainttoleration.New, + nodename.Name: nodename.New, + nodeports.Name: nodeports.New, + nodepreferavoidpods.Name: nodepreferavoidpods.New, + nodeaffinity.Name: nodeaffinity.New, + podtopologyspread.Name: podtopologyspread.New, + nodeunschedulable.Name: nodeunschedulable.New, + noderesources.FitName: func(plArgs apiruntime.Object, fh framework.Handle) (framework.Plugin, error) { + return noderesources.NewFit(plArgs, fh, fts) + }, + noderesources.BalancedAllocationName: func(plArgs apiruntime.Object, fh framework.Handle) (framework.Plugin, error) { + return noderesources.NewBalancedAllocation(plArgs, fh, fts) + }, + noderesources.MostAllocatedName: func(plArgs apiruntime.Object, fh framework.Handle) (framework.Plugin, error) { + return noderesources.NewMostAllocated(plArgs, fh, fts) + }, + noderesources.LeastAllocatedName: func(plArgs apiruntime.Object, fh framework.Handle) (framework.Plugin, error) { + return noderesources.NewLeastAllocated(plArgs, fh, fts) + }, + noderesources.RequestedToCapacityRatioName: func(plArgs apiruntime.Object, fh framework.Handle) (framework.Plugin, error) { + return noderesources.NewRequestedToCapacityRatio(plArgs, fh, fts) + }, + volumebinding.Name: volumebinding.New, + volumerestrictions.Name: volumerestrictions.New, + volumezone.Name: volumezone.New, + nodevolumelimits.CSIName: nodevolumelimits.NewCSI, + nodevolumelimits.EBSName: nodevolumelimits.NewEBS, + nodevolumelimits.GCEPDName: nodevolumelimits.NewGCEPD, + nodevolumelimits.AzureDiskName: nodevolumelimits.NewAzureDisk, + nodevolumelimits.CinderName: nodevolumelimits.NewCinder, interpodaffinity.Name: func(plArgs apiruntime.Object, fh framework.Handle) (framework.Plugin, error) { return interpodaffinity.New(plArgs, fh, fts) }, diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index 53519bdc916..83de1c3e64e 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -37,6 +37,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + apiruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" @@ -51,6 +52,7 @@ import ( "k8s.io/kubernetes/pkg/scheduler/core" "k8s.io/kubernetes/pkg/scheduler/framework" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeports" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/queuesort" @@ -770,7 +772,9 @@ func TestSchedulerFailedSchedulingReasons(t *testing.T) { fns := []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), - st.RegisterPluginAsExtensions(noderesources.FitName, noderesources.NewFit, "Filter", "PreFilter"), + st.RegisterPluginAsExtensions(noderesources.FitName, func(plArgs apiruntime.Object, fh framework.Handle) (framework.Plugin, error) { + return noderesources.NewFit(plArgs, fh, feature.Features{}) + }, "Filter", "PreFilter"), } scheduler, _, errChan := setupTestScheduler(queuedPodStore, scache, informerFactory, nil, fns...)