diff --git a/pkg/scheduler/BUILD b/pkg/scheduler/BUILD index 44473230159..cc092096171 100644 --- a/pkg/scheduler/BUILD +++ b/pkg/scheduler/BUILD @@ -77,13 +77,17 @@ go_test( "//pkg/scheduler/core:go_default_library", "//pkg/scheduler/framework/plugins:go_default_library", "//pkg/scheduler/framework/plugins/nodelabel:go_default_library", + "//pkg/scheduler/framework/plugins/nodeports:go_default_library", + "//pkg/scheduler/framework/plugins/noderesources:go_default_library", "//pkg/scheduler/framework/plugins/serviceaffinity:go_default_library", + "//pkg/scheduler/framework/plugins/volumebinding:go_default_library", "//pkg/scheduler/framework/v1alpha1:go_default_library", "//pkg/scheduler/internal/cache:go_default_library", "//pkg/scheduler/internal/cache/fake:go_default_library", "//pkg/scheduler/internal/queue:go_default_library", "//pkg/scheduler/nodeinfo:go_default_library", "//pkg/scheduler/nodeinfo/snapshot:go_default_library", + "//pkg/scheduler/testing:go_default_library", "//pkg/scheduler/volumebinder:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/events/v1beta1:go_default_library", diff --git a/pkg/scheduler/algorithm_factory.go b/pkg/scheduler/algorithm_factory.go index 2dba04055a5..c101897436a 100644 --- a/pkg/scheduler/algorithm_factory.go +++ b/pkg/scheduler/algorithm_factory.go @@ -71,6 +71,7 @@ var ( schedulerFactoryMutex sync.RWMutex // maps that hold registered algorithm types + // TODO(Huang-Wei): remove this. fitPredicateMap = make(map[string]FitPredicateFactory) mandatoryFitPredicates = sets.NewString() priorityFunctionMap = make(map[string]PriorityConfigFactory) @@ -143,6 +144,7 @@ func ApplyPredicatesAndPriorities(s *Snapshot) { // RegisterFitPredicate registers a fit predicate with the algorithm // registry. Returns the name with which the predicate was registered. +// TODO(Huang-Wei): remove this. func RegisterFitPredicate(name string, predicate predicates.FitPredicate) string { return RegisterFitPredicateFactory(name, func(AlgorithmFactoryArgs) predicates.FitPredicate { return predicate }) } diff --git a/pkg/scheduler/core/BUILD b/pkg/scheduler/core/BUILD index 0e573e4ef04..6a4bf8837bd 100644 --- a/pkg/scheduler/core/BUILD +++ b/pkg/scheduler/core/BUILD @@ -55,6 +55,9 @@ go_test( "//pkg/scheduler/algorithm/priorities/util:go_default_library", "//pkg/scheduler/apis/config:go_default_library", "//pkg/scheduler/apis/extender/v1:go_default_library", + "//pkg/scheduler/framework/plugins/interpodaffinity:go_default_library", + "//pkg/scheduler/framework/plugins/noderesources:go_default_library", + "//pkg/scheduler/framework/plugins/podtopologyspread:go_default_library", "//pkg/scheduler/framework/v1alpha1:go_default_library", "//pkg/scheduler/internal/cache:go_default_library", "//pkg/scheduler/internal/queue:go_default_library", @@ -62,6 +65,7 @@ go_test( "//pkg/scheduler/listers/fake:go_default_library", "//pkg/scheduler/nodeinfo:go_default_library", "//pkg/scheduler/nodeinfo/snapshot:go_default_library", + "//pkg/scheduler/testing:go_default_library", "//pkg/scheduler/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", diff --git a/pkg/scheduler/core/extender_test.go b/pkg/scheduler/core/extender_test.go index f22af0f2407..998fcc6fa4d 100644 --- a/pkg/scheduler/core/extender_test.go +++ b/pkg/scheduler/core/extender_test.go @@ -42,6 +42,7 @@ import ( internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" + st "k8s.io/kubernetes/pkg/scheduler/testing" "k8s.io/kubernetes/pkg/scheduler/util" ) @@ -348,16 +349,16 @@ var _ algorithm.SchedulerExtender = &FakeExtender{} func TestGenericSchedulerWithExtenders(t *testing.T) { tests := []struct { - name string - predicates map[string]predicates.FitPredicate - prioritizers []priorities.PriorityConfig - extenders []FakeExtender - nodes []string - expectedResult ScheduleResult - expectsErr bool + name string + registerFilterPlugin st.RegisterFilterPluginFunc + prioritizers []priorities.PriorityConfig + extenders []FakeExtender + nodes []string + expectedResult ScheduleResult + expectsErr bool }{ { - predicates: map[string]predicates.FitPredicate{"true": truePredicate}, + registerFilterPlugin: st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), extenders: []FakeExtender{ { predicates: []fitPredicate{truePredicateExtender}, @@ -371,7 +372,7 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { name: "test 1", }, { - predicates: map[string]predicates.FitPredicate{"true": truePredicate}, + registerFilterPlugin: st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), extenders: []FakeExtender{ { predicates: []fitPredicate{truePredicateExtender}, @@ -385,7 +386,7 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { name: "test 2", }, { - predicates: map[string]predicates.FitPredicate{"true": truePredicate}, + registerFilterPlugin: st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), extenders: []FakeExtender{ { predicates: []fitPredicate{truePredicateExtender}, @@ -403,7 +404,7 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { name: "test 3", }, { - predicates: map[string]predicates.FitPredicate{"true": truePredicate}, + registerFilterPlugin: st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), extenders: []FakeExtender{ { predicates: []fitPredicate{machine2PredicateExtender}, @@ -417,7 +418,7 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { name: "test 4", }, { - predicates: map[string]predicates.FitPredicate{"true": truePredicate}, + registerFilterPlugin: st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), extenders: []FakeExtender{ { predicates: []fitPredicate{truePredicateExtender}, @@ -434,7 +435,7 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { name: "test 5", }, { - predicates: map[string]predicates.FitPredicate{"true": truePredicate}, + registerFilterPlugin: st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), extenders: []FakeExtender{ { predicates: []fitPredicate{truePredicateExtender}, @@ -456,8 +457,8 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { name: "test 6", }, { - predicates: map[string]predicates.FitPredicate{"true": truePredicate}, - prioritizers: []priorities.PriorityConfig{{Map: machine2Prioritizer, Weight: 20}}, + registerFilterPlugin: st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + prioritizers: []priorities.PriorityConfig{{Map: machine2Prioritizer, Weight: 20}}, extenders: []FakeExtender{ { predicates: []fitPredicate{truePredicateExtender}, @@ -481,8 +482,8 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { // If scheduler sends the pod by mistake, the test would fail // because of the errors from errorPredicateExtender and/or // errorPrioritizerExtender. - predicates: map[string]predicates.FitPredicate{"true": truePredicate}, - prioritizers: []priorities.PriorityConfig{{Map: machine2Prioritizer, Weight: 1}}, + registerFilterPlugin: st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + prioritizers: []priorities.PriorityConfig{{Map: machine2Prioritizer, Weight: 1}}, extenders: []FakeExtender{ { predicates: []fitPredicate{errorPredicateExtender}, @@ -505,7 +506,7 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { // // If scheduler did not ignore the extender, the test would fail // because of the errors from errorPredicateExtender. - predicates: map[string]predicates.FitPredicate{"true": truePredicate}, + registerFilterPlugin: st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), extenders: []FakeExtender{ { predicates: []fitPredicate{errorPredicateExtender}, @@ -540,15 +541,24 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { cache.AddNode(createNode(name)) } queue := internalqueue.NewSchedulingQueue(nil) + + registry := framework.Registry{} + plugins := &schedulerapi.Plugins{ + Filter: &schedulerapi.PluginSet{}, + } + var pluginConfigs []schedulerapi.PluginConfig + test.registerFilterPlugin(®istry, plugins, pluginConfigs) + fwk, _ := framework.NewFramework(registry, plugins, pluginConfigs) + scheduler := NewGenericScheduler( cache, queue, - test.predicates, + nil, predicates.EmptyMetadataProducer, test.prioritizers, priorities.EmptyMetadataProducer, emptySnapshot, - emptyFramework, + fwk, extenders, nil, informerFactory.Core().V1().PersistentVolumeClaims().Lister(), diff --git a/pkg/scheduler/core/generic_scheduler.go b/pkg/scheduler/core/generic_scheduler.go index ac501c400a3..89b438808f4 100644 --- a/pkg/scheduler/core/generic_scheduler.go +++ b/pkg/scheduler/core/generic_scheduler.go @@ -72,8 +72,9 @@ type FailedPredicateMap map[string][]predicates.PredicateFailureReason // FitError describes a fit error of a pod. type FitError struct { - Pod *v1.Pod - NumAllNodes int + Pod *v1.Pod + NumAllNodes int + // TODO(Huang-Wei): remove 'FailedPredicates' FailedPredicates FailedPredicateMap FilteredNodesStatuses framework.NodeToStatusMap } @@ -476,12 +477,13 @@ func (g *genericScheduler) numFeasibleNodesToFind(numAllNodes int32) (numNodes i // Filters the nodes to find the ones that fit based on the given predicate functions // Each node is passed through the predicate functions to determine if it is a fit +// TODO(Huang-Wei): remove 'FailedPredicateMap' from the return parameters. func (g *genericScheduler) findNodesThatFit(ctx context.Context, state *framework.CycleState, pod *v1.Pod) ([]*v1.Node, FailedPredicateMap, framework.NodeToStatusMap, error) { var filtered []*v1.Node failedPredicateMap := FailedPredicateMap{} filteredNodesStatuses := framework.NodeToStatusMap{} - if len(g.predicates) == 0 && !g.framework.HasFilterPlugins() { + if !g.framework.HasFilterPlugins() { filtered = g.nodeInfoSnapshot.ListNodes() } else { allNodes := len(g.nodeInfoSnapshot.NodeInfoList) @@ -506,7 +508,7 @@ func (g *genericScheduler) findNodesThatFit(ctx context.Context, state *framewor // We check the nodes starting from where we left off in the previous scheduling cycle, // this is to make sure all nodes have the same chance of being examined across pods. nodeInfo := g.nodeInfoSnapshot.NodeInfoList[(g.nextStartNodeIndex+i)%allNodes] - fits, failedPredicates, status, err := g.podFitsOnNode( + fits, _, status, err := g.podFitsOnNode( ctx, state, pod, @@ -531,9 +533,6 @@ func (g *genericScheduler) findNodesThatFit(ctx context.Context, state *framewor if !status.IsSuccess() { filteredNodesStatuses[nodeInfo.Node().Name] = status } - if len(failedPredicates) != 0 { - failedPredicateMap[nodeInfo.Node().Name] = failedPredicates - } predicateResultLock.Unlock() } } @@ -566,6 +565,7 @@ func (g *genericScheduler) findNodesThatFit(ctx context.Context, state *framewor return []*v1.Node{}, FailedPredicateMap{}, framework.NodeToStatusMap{}, err } + // TODO(Huang-Wei): refactor this to fill 'filteredNodesStatuses' instead of 'failedPredicateMap'. for failedNodeName, failedMsg := range failedMap { if _, found := failedPredicateMap[failedNodeName]; !found { failedPredicateMap[failedNodeName] = []predicates.PredicateFailureReason{} @@ -584,6 +584,7 @@ func (g *genericScheduler) findNodesThatFit(ctx context.Context, state *framewor // addNominatedPods adds pods with equal or greater priority which are nominated // to run on the node given in nodeInfo to meta and nodeInfo. It returns 1) whether // any pod was added, 2) augmented metadata, 3) augmented CycleState 4) augmented nodeInfo. +// TODO(Huang-Wei): remove 'meta predicates.Metadata' from the signature. func (g *genericScheduler) addNominatedPods(ctx context.Context, pod *v1.Pod, meta predicates.Metadata, state *framework.CycleState, nodeInfo *schedulernodeinfo.NodeInfo) (bool, predicates.Metadata, *framework.CycleState, *schedulernodeinfo.NodeInfo, error) { @@ -662,12 +663,11 @@ func (g *genericScheduler) podFitsOnNode( // nominated pods are running because they are not running right now and in fact, // they may end up getting scheduled to a different node. for i := 0; i < 2; i++ { - metaToUse := meta stateToUse := state nodeInfoToUse := info if i == 0 { var err error - podsAdded, metaToUse, stateToUse, nodeInfoToUse, err = g.addNominatedPods(ctx, pod, meta, state, info) + podsAdded, _, stateToUse, nodeInfoToUse, err = g.addNominatedPods(ctx, pod, meta, state, info) if err != nil { return false, []predicates.PredicateFailureReason{}, nil, err } @@ -675,33 +675,6 @@ func (g *genericScheduler) podFitsOnNode( break } - for _, predicateKey := range predicates.Ordering() { - var ( - fit bool - reasons []predicates.PredicateFailureReason - err error - ) - - if predicate, exist := g.predicates[predicateKey]; exist { - fit, reasons, err = predicate(pod, metaToUse, nodeInfoToUse) - if err != nil { - return false, []predicates.PredicateFailureReason{}, nil, err - } - - if !fit { - // eCache is available and valid, and predicates result is unfit, record the fail reasons - failedPredicates = append(failedPredicates, reasons...) - // if alwaysCheckAllPredicates is false, short circuit all predicates when one predicate fails. - if !alwaysCheckAllPredicates { - klog.V(5).Infoln("since alwaysCheckAllPredicates has not been set, the predicate " + - "evaluation is short circuited and there are chances " + - "of other predicates failing as well.") - break - } - } - } - } - status = g.framework.RunFilterPlugins(ctx, stateToUse, pod, nodeInfoToUse) if !status.IsSuccess() && !status.IsUnschedulable() { return false, failedPredicates, status, status.AsError() @@ -1270,6 +1243,7 @@ func podPassesBasicChecks(pod *v1.Pod, pvcLister corelisters.PersistentVolumeCla } // NewGenericScheduler creates a genericScheduler object. +// TODO(Huang-Wei): remove 'predicates' and 'alwaysCheckAllPredicates'. func NewGenericScheduler( cache internalcache.Cache, podQueue internalqueue.SchedulingQueue, diff --git a/pkg/scheduler/core/generic_scheduler_test.go b/pkg/scheduler/core/generic_scheduler_test.go index c0779790a91..fcf6f3f1e3c 100644 --- a/pkg/scheduler/core/generic_scheduler_test.go +++ b/pkg/scheduler/core/generic_scheduler_test.go @@ -43,44 +43,134 @@ import ( priorityutil "k8s.io/kubernetes/pkg/scheduler/algorithm/priorities/util" schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" extenderv1 "k8s.io/kubernetes/pkg/scheduler/apis/extender/v1" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/podtopologyspread" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue" schedulerlisters "k8s.io/kubernetes/pkg/scheduler/listers" fakelisters "k8s.io/kubernetes/pkg/scheduler/listers/fake" + "k8s.io/kubernetes/pkg/scheduler/nodeinfo" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" nodeinfosnapshot "k8s.io/kubernetes/pkg/scheduler/nodeinfo/snapshot" + st "k8s.io/kubernetes/pkg/scheduler/testing" ) var ( errPrioritize = fmt.Errorf("priority map encounters an error") - order = []string{"false", "true", "matches", "nopods", algorithmpredicates.MatchInterPodAffinityPred} + // TODO(Huang-Wei): remove 'order' and 'defer SetPredicatesOrderingDuringTest(order)()'. + order = []string{"FakeFilter", "FalseFilter", "TrueFilter", "MatchFilter", "NoPodsFilter", interpodaffinity.Name} ) -func falsePredicate(pod *v1.Pod, meta algorithmpredicates.Metadata, nodeInfo *schedulernodeinfo.NodeInfo) (bool, []algorithmpredicates.PredicateFailureReason, error) { - return false, []algorithmpredicates.PredicateFailureReason{algorithmpredicates.ErrFakePredicate}, nil +type trueFilterPlugin struct{} + +// Name returns name of the plugin. +func (pl *trueFilterPlugin) Name() string { + return "TrueFilter" } -func truePredicate(pod *v1.Pod, meta algorithmpredicates.Metadata, nodeInfo *schedulernodeinfo.NodeInfo) (bool, []algorithmpredicates.PredicateFailureReason, error) { - return true, nil, nil +// Filter invoked at the filter extension point. +func (pl *trueFilterPlugin) Filter(_ context.Context, _ *framework.CycleState, pod *v1.Pod, nodeInfo *nodeinfo.NodeInfo) *framework.Status { + return nil } -func matchesPredicate(pod *v1.Pod, meta algorithmpredicates.Metadata, nodeInfo *schedulernodeinfo.NodeInfo) (bool, []algorithmpredicates.PredicateFailureReason, error) { +// NewTrueFilterPlugin initializes a trueFilterPlugin and returns it. +func NewTrueFilterPlugin(_ *runtime.Unknown, _ framework.FrameworkHandle) (framework.Plugin, error) { + return &trueFilterPlugin{}, nil +} + +type falseFilterPlugin struct{} + +// Name returns name of the plugin. +func (pl *falseFilterPlugin) Name() string { + return "FalseFilter" +} + +// Filter invoked at the filter extension point. +func (pl *falseFilterPlugin) Filter(_ context.Context, _ *framework.CycleState, pod *v1.Pod, nodeInfo *nodeinfo.NodeInfo) *framework.Status { + return framework.NewStatus(framework.Unschedulable, algorithmpredicates.ErrFakePredicate.GetReason()) +} + +// NewFalseFilterPlugin initializes a falseFilterPlugin and returns it. +func NewFalseFilterPlugin(_ *runtime.Unknown, _ framework.FrameworkHandle) (framework.Plugin, error) { + return &falseFilterPlugin{}, nil +} + +type matchFilterPlugin struct{} + +// Name returns name of the plugin. +func (pl *matchFilterPlugin) Name() string { + return "MatchFilter" +} + +// Filter invoked at the filter extension point. +func (pl *matchFilterPlugin) Filter(_ context.Context, _ *framework.CycleState, pod *v1.Pod, nodeInfo *nodeinfo.NodeInfo) *framework.Status { node := nodeInfo.Node() if node == nil { - return false, nil, fmt.Errorf("node not found") + return framework.NewStatus(framework.Error, "node not found") } if pod.Name == node.Name { - return true, nil, nil + return nil } - return false, []algorithmpredicates.PredicateFailureReason{algorithmpredicates.ErrFakePredicate}, nil + return framework.NewStatus(framework.Unschedulable, algorithmpredicates.ErrFakePredicate.GetReason()) } -func hasNoPodsPredicate(pod *v1.Pod, meta algorithmpredicates.Metadata, nodeInfo *schedulernodeinfo.NodeInfo) (bool, []algorithmpredicates.PredicateFailureReason, error) { +// NewMatchFilterPlugin initializes a matchFilterPlugin and returns it. +func NewMatchFilterPlugin(_ *runtime.Unknown, _ framework.FrameworkHandle) (framework.Plugin, error) { + return &matchFilterPlugin{}, nil +} + +type noPodsFilterPlugin struct{} + +// Name returns name of the plugin. +func (pl *noPodsFilterPlugin) Name() string { + return "NoPodsFilter" +} + +// Filter invoked at the filter extension point. +func (pl *noPodsFilterPlugin) Filter(_ context.Context, _ *framework.CycleState, pod *v1.Pod, nodeInfo *nodeinfo.NodeInfo) *framework.Status { if len(nodeInfo.Pods()) == 0 { - return true, nil, nil + return nil + } + return framework.NewStatus(framework.Unschedulable, algorithmpredicates.ErrFakePredicate.GetReason()) +} + +// NewNoPodsFilterPlugin initializes a noPodsFilterPlugin and returns it. +func NewNoPodsFilterPlugin(_ *runtime.Unknown, _ framework.FrameworkHandle) (framework.Plugin, error) { + return &noPodsFilterPlugin{}, nil +} + +// fakeFilterPlugin is a test filter plugin to record how many times its Filter() function have +// been called, and it returns different 'Code' depending on its internal 'failedNodeReturnCodeMap'. +type fakeFilterPlugin struct { + numFilterCalled int32 + failedNodeReturnCodeMap map[string]framework.Code +} + +// Name returns name of the plugin. +func (pl *fakeFilterPlugin) Name() string { + return "FakeFilter" +} + +// Filter invoked at the filter extension point. +func (pl *fakeFilterPlugin) Filter(_ context.Context, _ *framework.CycleState, pod *v1.Pod, nodeInfo *nodeinfo.NodeInfo) *framework.Status { + atomic.AddInt32(&pl.numFilterCalled, 1) + + if returnCode, ok := pl.failedNodeReturnCodeMap[nodeInfo.Node().Name]; ok { + return framework.NewStatus(returnCode, fmt.Sprintf("injecting failure for pod %v", pod.Name)) + } + + return nil +} + +// NewFakeFilterPlugin initializes a fakeFilterPlugin and returns it. +func NewFakeFilterPlugin(failedNodeReturnCodeMap map[string]framework.Code) framework.PluginFactory { + return func(_ *runtime.Unknown, _ framework.FrameworkHandle) (framework.Plugin, error) { + return &fakeFilterPlugin{ + failedNodeReturnCodeMap: failedNodeReturnCodeMap, + }, nil } - return false, []algorithmpredicates.PredicateFailureReason{algorithmpredicates.ErrFakePredicate}, nil } func numericMapPriority(pod *v1.Pod, meta interface{}, nodeInfo *schedulernodeinfo.NodeInfo) (framework.NodeScore, error) { @@ -139,42 +229,6 @@ var emptyPluginRegistry = framework.Registry{} var emptyFramework, _ = framework.NewFramework(emptyPluginRegistry, nil, []schedulerapi.PluginConfig{}) var emptySnapshot = nodeinfosnapshot.NewEmptySnapshot() -// FakeFilterPlugin is a test filter plugin used by default scheduler. -type FakeFilterPlugin struct { - numFilterCalled int32 - failedNodeReturnCodeMap map[string]framework.Code -} - -// Name returns name of the plugin. -func (fp *FakeFilterPlugin) Name() string { - return "fake-filter-plugin" -} - -// reset is used to reset filter plugin. -func (fp *FakeFilterPlugin) reset() { - fp.numFilterCalled = 0 - fp.failedNodeReturnCodeMap = map[string]framework.Code{} -} - -// Filter is a test function that returns an error or nil, depending on the -// value of "failedNodeReturnCodeMap". -func (fp *FakeFilterPlugin) Filter(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeInfo *schedulernodeinfo.NodeInfo) *framework.Status { - atomic.AddInt32(&fp.numFilterCalled, 1) - - if returnCode, ok := fp.failedNodeReturnCodeMap[nodeInfo.Node().Name]; ok { - return framework.NewStatus(returnCode, fmt.Sprintf("injecting failure for pod %v", pod.Name)) - } - - return nil -} - -// newPlugin returns a plugin factory with specified Plugin. -func newPlugin(plugin framework.Plugin) framework.PluginFactory { - return func(_ *runtime.Unknown, fh framework.FrameworkHandle) (framework.Plugin, error) { - return plugin, nil - } -} - func makeNodeList(nodeNames []string) []*v1.Node { result := make([]*v1.Node, 0, len(nodeNames)) for _, nodeName := range nodeNames { @@ -256,52 +310,40 @@ func TestSelectHost(t *testing.T) { func TestGenericScheduler(t *testing.T) { defer algorithmpredicates.SetPredicatesOrderingDuringTest(order)() - filterPlugin := &FakeFilterPlugin{} - filterPluginRegistry := framework.Registry{filterPlugin.Name(): newPlugin(filterPlugin)} - filterFramework, err := framework.NewFramework(filterPluginRegistry, &schedulerapi.Plugins{ - Filter: &schedulerapi.PluginSet{ - Enabled: []schedulerapi.Plugin{ - { - Name: filterPlugin.Name(), - }, - }, - }, - }, []schedulerapi.PluginConfig{}) - if err != nil { - t.Errorf("Unexpected error when initialize scheduling framework, err :%v", err.Error()) - } - tests := []struct { - name string - predicates map[string]algorithmpredicates.FitPredicate - prioritizers []priorities.PriorityConfig - alwaysCheckAllPredicates bool - nodes []string - pvcs []v1.PersistentVolumeClaim - pod *v1.Pod - pods []*v1.Pod - buildPredMeta bool // build predicates metadata or not - filterFailedNodeReturnCodeMap map[string]framework.Code - expectedHosts sets.String - wErr error + name string + registerFilterPlugins []st.RegisterFilterPluginFunc + prioritizers []priorities.PriorityConfig + alwaysCheckAllPredicates bool + nodes []string + pvcs []v1.PersistentVolumeClaim + pod *v1.Pod + pods []*v1.Pod + buildPredMeta bool // build predicates metadata or not + expectedHosts sets.String + wErr error }{ { - predicates: map[string]algorithmpredicates.FitPredicate{"false": falsePredicate}, - nodes: []string{"machine1", "machine2"}, - pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "2", UID: types.UID("2")}}, - name: "test 1", + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin("FalseFilter", NewFalseFilterPlugin), + }, + nodes: []string{"machine1", "machine2"}, + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "2", UID: types.UID("2")}}, + name: "test 1", wErr: &FitError{ - Pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "2", UID: types.UID("2")}}, - NumAllNodes: 2, - FailedPredicates: FailedPredicateMap{ - "machine1": []algorithmpredicates.PredicateFailureReason{algorithmpredicates.ErrFakePredicate}, - "machine2": []algorithmpredicates.PredicateFailureReason{algorithmpredicates.ErrFakePredicate}, + Pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "2", UID: types.UID("2")}}, + NumAllNodes: 2, + FailedPredicates: FailedPredicateMap{}, + FilteredNodesStatuses: framework.NodeToStatusMap{ + "machine1": framework.NewStatus(framework.Unschedulable, algorithmpredicates.ErrFakePredicate.GetReason()), + "machine2": framework.NewStatus(framework.Unschedulable, algorithmpredicates.ErrFakePredicate.GetReason()), }, - FilteredNodesStatuses: framework.NodeToStatusMap{}, }, }, { - predicates: map[string]algorithmpredicates.FitPredicate{"true": truePredicate}, + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + }, nodes: []string{"machine1", "machine2"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "ignore", UID: types.UID("ignore")}}, expectedHosts: sets.NewString("machine1", "machine2"), @@ -310,7 +352,9 @@ func TestGenericScheduler(t *testing.T) { }, { // Fits on a machine where the pod ID matches the machine name - predicates: map[string]algorithmpredicates.FitPredicate{"matches": matchesPredicate}, + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin("MatchFilter", NewMatchFilterPlugin), + }, nodes: []string{"machine1", "machine2"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine2", UID: types.UID("machine2")}}, expectedHosts: sets.NewString("machine2"), @@ -318,7 +362,9 @@ func TestGenericScheduler(t *testing.T) { wErr: nil, }, { - predicates: map[string]algorithmpredicates.FitPredicate{"true": truePredicate}, + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + }, prioritizers: []priorities.PriorityConfig{{Map: numericMapPriority, Weight: 1}}, nodes: []string{"3", "2", "1"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "ignore", UID: types.UID("ignore")}}, @@ -327,7 +373,9 @@ func TestGenericScheduler(t *testing.T) { wErr: nil, }, { - predicates: map[string]algorithmpredicates.FitPredicate{"matches": matchesPredicate}, + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin("MatchFilter", NewMatchFilterPlugin), + }, prioritizers: []priorities.PriorityConfig{{Map: numericMapPriority, Weight: 1}}, nodes: []string{"3", "2", "1"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "2", UID: types.UID("2")}}, @@ -336,7 +384,9 @@ func TestGenericScheduler(t *testing.T) { wErr: nil, }, { - predicates: map[string]algorithmpredicates.FitPredicate{"true": truePredicate}, + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + }, prioritizers: []priorities.PriorityConfig{ { Map: numericMapPriority, @@ -355,26 +405,29 @@ func TestGenericScheduler(t *testing.T) { wErr: nil, }, { - predicates: map[string]algorithmpredicates.FitPredicate{"true": truePredicate, "false": falsePredicate}, + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + st.RegisterFilterPlugin("FalseFilter", NewFalseFilterPlugin), + }, prioritizers: []priorities.PriorityConfig{{Map: numericMapPriority, Weight: 1}}, nodes: []string{"3", "2", "1"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "2", UID: types.UID("2")}}, name: "test 7", wErr: &FitError{ - Pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "2", UID: types.UID("2")}}, - NumAllNodes: 3, - FailedPredicates: FailedPredicateMap{ - "3": []algorithmpredicates.PredicateFailureReason{algorithmpredicates.ErrFakePredicate}, - "2": []algorithmpredicates.PredicateFailureReason{algorithmpredicates.ErrFakePredicate}, - "1": []algorithmpredicates.PredicateFailureReason{algorithmpredicates.ErrFakePredicate}, + Pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "2", UID: types.UID("2")}}, + NumAllNodes: 3, + FailedPredicates: FailedPredicateMap{}, + FilteredNodesStatuses: framework.NodeToStatusMap{ + "3": framework.NewStatus(framework.Unschedulable, algorithmpredicates.ErrFakePredicate.GetReason()), + "2": framework.NewStatus(framework.Unschedulable, algorithmpredicates.ErrFakePredicate.GetReason()), + "1": framework.NewStatus(framework.Unschedulable, algorithmpredicates.ErrFakePredicate.GetReason()), }, - FilteredNodesStatuses: framework.NodeToStatusMap{}, }, }, { - predicates: map[string]algorithmpredicates.FitPredicate{ - "nopods": hasNoPodsPredicate, - "matches": matchesPredicate, + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin("NoPodsFilter", NewNoPodsFilterPlugin), + st.RegisterFilterPlugin("MatchFilter", NewMatchFilterPlugin), }, pods: []*v1.Pod{ { @@ -392,20 +445,22 @@ func TestGenericScheduler(t *testing.T) { nodes: []string{"1", "2"}, name: "test 8", wErr: &FitError{ - Pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "2", UID: types.UID("2")}}, - NumAllNodes: 2, - FailedPredicates: FailedPredicateMap{ - "1": []algorithmpredicates.PredicateFailureReason{algorithmpredicates.ErrFakePredicate}, - "2": []algorithmpredicates.PredicateFailureReason{algorithmpredicates.ErrFakePredicate}, + Pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "2", UID: types.UID("2")}}, + NumAllNodes: 2, + FailedPredicates: FailedPredicateMap{}, + FilteredNodesStatuses: framework.NodeToStatusMap{ + "1": framework.NewStatus(framework.Unschedulable, algorithmpredicates.ErrFakePredicate.GetReason()), + "2": framework.NewStatus(framework.Unschedulable, algorithmpredicates.ErrFakePredicate.GetReason()), }, - FilteredNodesStatuses: framework.NodeToStatusMap{}, }, }, { // Pod with existing PVC - predicates: map[string]algorithmpredicates.FitPredicate{"true": truePredicate}, - nodes: []string{"machine1", "machine2"}, - pvcs: []v1.PersistentVolumeClaim{{ObjectMeta: metav1.ObjectMeta{Name: "existingPVC"}}}, + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + }, + nodes: []string{"machine1", "machine2"}, + pvcs: []v1.PersistentVolumeClaim{{ObjectMeta: metav1.ObjectMeta{Name: "existingPVC"}}}, pod: &v1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "ignore", UID: types.UID("ignore")}, Spec: v1.PodSpec{ @@ -426,8 +481,10 @@ func TestGenericScheduler(t *testing.T) { }, { // Pod with non existing PVC - predicates: map[string]algorithmpredicates.FitPredicate{"true": truePredicate}, - nodes: []string{"machine1", "machine2"}, + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + }, + nodes: []string{"machine1", "machine2"}, pod: &v1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "ignore", UID: types.UID("ignore")}, Spec: v1.PodSpec{ @@ -447,9 +504,11 @@ func TestGenericScheduler(t *testing.T) { }, { // Pod with deleting PVC - predicates: map[string]algorithmpredicates.FitPredicate{"true": truePredicate}, - nodes: []string{"machine1", "machine2"}, - pvcs: []v1.PersistentVolumeClaim{{ObjectMeta: metav1.ObjectMeta{Name: "existingPVC", DeletionTimestamp: &metav1.Time{}}}}, + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + }, + nodes: []string{"machine1", "machine2"}, + pvcs: []v1.PersistentVolumeClaim{{ObjectMeta: metav1.ObjectMeta{Name: "existingPVC", DeletionTimestamp: &metav1.Time{}}}}, pod: &v1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "ignore", UID: types.UID("ignore")}, Spec: v1.PodSpec{ @@ -468,23 +527,9 @@ func TestGenericScheduler(t *testing.T) { wErr: fmt.Errorf("persistentvolumeclaim \"existingPVC\" is being deleted"), }, { - // alwaysCheckAllPredicates is true - predicates: map[string]algorithmpredicates.FitPredicate{"true": truePredicate, "matches": matchesPredicate, "false": falsePredicate}, - alwaysCheckAllPredicates: true, - nodes: []string{"1"}, - pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "2", UID: types.UID("2")}}, - name: "test alwaysCheckAllPredicates is true", - wErr: &FitError{ - Pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "2", UID: types.UID("2")}}, - NumAllNodes: 1, - FailedPredicates: FailedPredicateMap{ - "1": []algorithmpredicates.PredicateFailureReason{algorithmpredicates.ErrFakePredicate, algorithmpredicates.ErrFakePredicate}, - }, - FilteredNodesStatuses: framework.NodeToStatusMap{}, + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), }, - }, - { - predicates: map[string]algorithmpredicates.FitPredicate{"true": truePredicate}, prioritizers: []priorities.PriorityConfig{{Map: falseMapPriority, Weight: 1}, {Map: trueMapPriority, Reduce: getNodeReducePriority, Weight: 2}}, nodes: []string{"2", "1"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "2"}}, @@ -493,8 +538,8 @@ func TestGenericScheduler(t *testing.T) { }, { name: "test even pods spread predicate - 2 nodes with maxskew=1", - predicates: map[string]algorithmpredicates.FitPredicate{ - "matches": algorithmpredicates.EvenPodsSpreadPredicate, + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin(podtopologyspread.Name, podtopologyspread.New), }, nodes: []string{"machine1", "machine2"}, pod: &v1.Pod{ @@ -534,8 +579,8 @@ func TestGenericScheduler(t *testing.T) { }, { name: "test even pods spread predicate - 3 nodes with maxskew=2", - predicates: map[string]algorithmpredicates.FitPredicate{ - "matches": algorithmpredicates.EvenPodsSpreadPredicate, + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin(podtopologyspread.Name, podtopologyspread.New), }, nodes: []string{"machine1", "machine2", "machine3"}, pod: &v1.Pod{ @@ -592,13 +637,17 @@ func TestGenericScheduler(t *testing.T) { wErr: nil, }, { - name: "test with filter plugin returning Unschedulable status", - predicates: map[string]algorithmpredicates.FitPredicate{"true": truePredicate}, - prioritizers: []priorities.PriorityConfig{{Map: numericMapPriority, Weight: 1}}, - nodes: []string{"3"}, - pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "test-filter", UID: types.UID("test-filter")}}, - expectedHosts: nil, - filterFailedNodeReturnCodeMap: map[string]framework.Code{"3": framework.Unschedulable}, + name: "test with filter plugin returning Unschedulable status", + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin( + "FakeFilter", + NewFakeFilterPlugin(map[string]framework.Code{"3": framework.Unschedulable}), + ), + }, + prioritizers: []priorities.PriorityConfig{{Map: numericMapPriority, Weight: 1}}, + nodes: []string{"3"}, + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "test-filter", UID: types.UID("test-filter")}}, + expectedHosts: nil, wErr: &FitError{ Pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "test-filter", UID: types.UID("test-filter")}}, NumAllNodes: 1, @@ -609,13 +658,17 @@ func TestGenericScheduler(t *testing.T) { }, }, { - name: "test with filter plugin returning UnschedulableAndUnresolvable status", - predicates: map[string]algorithmpredicates.FitPredicate{"true": truePredicate}, - prioritizers: []priorities.PriorityConfig{{Map: numericMapPriority, Weight: 1}}, - nodes: []string{"3"}, - pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "test-filter", UID: types.UID("test-filter")}}, - expectedHosts: nil, - filterFailedNodeReturnCodeMap: map[string]framework.Code{"3": framework.UnschedulableAndUnresolvable}, + name: "test with filter plugin returning UnschedulableAndUnresolvable status", + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin( + "FakeFilter", + NewFakeFilterPlugin(map[string]framework.Code{"3": framework.UnschedulableAndUnresolvable}), + ), + }, + prioritizers: []priorities.PriorityConfig{{Map: numericMapPriority, Weight: 1}}, + nodes: []string{"3"}, + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "test-filter", UID: types.UID("test-filter")}}, + expectedHosts: nil, wErr: &FitError{ Pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "test-filter", UID: types.UID("test-filter")}}, NumAllNodes: 1, @@ -626,14 +679,18 @@ func TestGenericScheduler(t *testing.T) { }, }, { - name: "test with partial failed filter plugin", - predicates: map[string]algorithmpredicates.FitPredicate{"true": truePredicate}, - prioritizers: []priorities.PriorityConfig{{Map: numericMapPriority, Weight: 1}}, - nodes: []string{"1", "2"}, - pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "test-filter", UID: types.UID("test-filter")}}, - expectedHosts: nil, - filterFailedNodeReturnCodeMap: map[string]framework.Code{"1": framework.Unschedulable}, - wErr: nil, + name: "test with partial failed filter plugin", + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin( + "FakeFilter", + NewFakeFilterPlugin(map[string]framework.Code{"1": framework.Unschedulable}), + ), + }, + prioritizers: []priorities.PriorityConfig{{Map: numericMapPriority, Weight: 1}}, + nodes: []string{"1", "2"}, + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "test-filter", UID: types.UID("test-filter")}}, + expectedHosts: nil, + wErr: nil, }, } for _, test := range tests { @@ -641,7 +698,15 @@ func TestGenericScheduler(t *testing.T) { client := clientsetfake.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(client, 0) - filterPlugin.failedNodeReturnCodeMap = test.filterFailedNodeReturnCodeMap + registry := framework.Registry{} + plugins := &schedulerapi.Plugins{ + Filter: &schedulerapi.PluginSet{}, + } + var pluginConfigs []schedulerapi.PluginConfig + for _, f := range test.registerFilterPlugins { + f(®istry, plugins, pluginConfigs) + } + fwk, _ := framework.NewFramework(registry, plugins, pluginConfigs) cache := internalcache.New(time.Duration(0), wait.NeverStop) for _, pod := range test.pods { @@ -663,12 +728,12 @@ func TestGenericScheduler(t *testing.T) { scheduler := NewGenericScheduler( cache, internalqueue.NewSchedulingQueue(nil), - test.predicates, + nil, predMetaProducer, test.prioritizers, priorities.EmptyMetadataProducer, emptySnapshot, - filterFramework, + fwk, []algorithm.SchedulerExtender{}, nil, pvcLister, @@ -687,28 +752,36 @@ func TestGenericScheduler(t *testing.T) { if test.wErr == nil && len(test.nodes) != result.EvaluatedNodes { t.Errorf("Expected EvaluatedNodes: %d, got: %d", len(test.nodes), result.EvaluatedNodes) } - - filterPlugin.reset() }) } } // makeScheduler makes a simple genericScheduler for testing. -func makeScheduler(predicates map[string]algorithmpredicates.FitPredicate, nodes []*v1.Node) *genericScheduler { +func makeScheduler(nodes []*v1.Node, fns ...st.RegisterFilterPluginFunc) *genericScheduler { cache := internalcache.New(time.Duration(0), wait.NeverStop) for _, n := range nodes { cache.AddNode(n) } + registry := framework.Registry{} + plugins := &schedulerapi.Plugins{ + Filter: &schedulerapi.PluginSet{}, + } + var pluginConfigs []schedulerapi.PluginConfig + for _, f := range fns { + f(®istry, plugins, pluginConfigs) + } + fwk, _ := framework.NewFramework(registry, plugins, pluginConfigs) + s := NewGenericScheduler( cache, internalqueue.NewSchedulingQueue(nil), - predicates, + nil, algorithmpredicates.EmptyMetadataProducer, nil, priorities.EmptyMetadataProducer, emptySnapshot, - emptyFramework, + fwk, nil, nil, nil, nil, false, false, schedulerapi.DefaultPercentageOfNodesToScore, false) cache.UpdateNodeInfoSnapshot(s.(*genericScheduler).nodeInfoSnapshot) @@ -718,28 +791,32 @@ func makeScheduler(predicates map[string]algorithmpredicates.FitPredicate, nodes func TestFindFitAllError(t *testing.T) { defer algorithmpredicates.SetPredicatesOrderingDuringTest(order)() - predicates := map[string]algorithmpredicates.FitPredicate{"true": truePredicate, "matches": matchesPredicate} nodes := makeNodeList([]string{"3", "2", "1"}) - scheduler := makeScheduler(predicates, nodes) + scheduler := makeScheduler( + nodes, + st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + st.RegisterFilterPlugin("MatchFilter", NewMatchFilterPlugin), + ) - _, predicateMap, _, err := scheduler.findNodesThatFit(context.Background(), framework.NewCycleState(), &v1.Pod{}) + _, _, nodeToStatusMap, err := scheduler.findNodesThatFit(context.Background(), framework.NewCycleState(), &v1.Pod{}) if err != nil { t.Errorf("unexpected error: %v", err) } - if len(predicateMap) != len(nodes) { - t.Errorf("unexpected failed predicate map: %v", predicateMap) + if len(nodeToStatusMap) != len(nodes) { + t.Errorf("unexpected failed status map: %v", nodeToStatusMap) } for _, node := range nodes { t.Run(node.Name, func(t *testing.T) { - failures, found := predicateMap[node.Name] + status, found := nodeToStatusMap[node.Name] if !found { - t.Errorf("failed to find node in %v", predicateMap) + t.Errorf("failed to find node %v in %v", node.Name, nodeToStatusMap) } - if len(failures) != 1 || failures[0] != algorithmpredicates.ErrFakePredicate { - t.Errorf("unexpected failures: %v", failures) + reasons := status.Reasons() + if len(reasons) != 1 || reasons[0] != algorithmpredicates.ErrFakePredicate.GetReason() { + t.Errorf("unexpected failure reasons: %v", reasons) } }) } @@ -747,19 +824,22 @@ func TestFindFitAllError(t *testing.T) { func TestFindFitSomeError(t *testing.T) { defer algorithmpredicates.SetPredicatesOrderingDuringTest(order)() - predicates := map[string]algorithmpredicates.FitPredicate{"true": truePredicate, "matches": matchesPredicate} nodes := makeNodeList([]string{"3", "2", "1"}) - scheduler := makeScheduler(predicates, nodes) + scheduler := makeScheduler( + nodes, + st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + st.RegisterFilterPlugin("MatchFilter", NewMatchFilterPlugin), + ) pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "1", UID: types.UID("1")}} - _, predicateMap, _, err := scheduler.findNodesThatFit(context.Background(), framework.NewCycleState(), pod) + _, _, nodeToStatusMap, err := scheduler.findNodesThatFit(context.Background(), framework.NewCycleState(), pod) if err != nil { t.Errorf("unexpected error: %v", err) } - if len(predicateMap) != (len(nodes) - 1) { - t.Errorf("unexpected failed predicate map: %v", predicateMap) + if len(nodeToStatusMap) != len(nodes)-1 { + t.Errorf("unexpected failed status map: %v", nodeToStatusMap) } for _, node := range nodes { @@ -767,35 +847,25 @@ func TestFindFitSomeError(t *testing.T) { continue } t.Run(node.Name, func(t *testing.T) { - failures, found := predicateMap[node.Name] + status, found := nodeToStatusMap[node.Name] if !found { - t.Errorf("failed to find node in %v", predicateMap) + t.Errorf("failed to find node %v in %v", node.Name, nodeToStatusMap) } - if len(failures) != 1 || failures[0] != algorithmpredicates.ErrFakePredicate { - t.Errorf("unexpected failures: %v", failures) + reasons := status.Reasons() + if len(reasons) != 1 || reasons[0] != algorithmpredicates.ErrFakePredicate.GetReason() { + t.Errorf("unexpected failures: %v", reasons) } }) } } -type predicateCallCounter struct { - count int -} - -func (c *predicateCallCounter) truePredicate() algorithmpredicates.FitPredicate { - return func(pod *v1.Pod, meta algorithmpredicates.Metadata, nodeInfo *schedulernodeinfo.NodeInfo) (bool, []algorithmpredicates.PredicateFailureReason, error) { - c.count++ - return truePredicate(pod, meta, nodeInfo) - } -} - func TestFindFitPredicateCallCounts(t *testing.T) { defer algorithmpredicates.SetPredicatesOrderingDuringTest(order)() tests := []struct { name string pod *v1.Pod - expectedCount int + expectedCount int32 }{ { name: "nominated pods have lower priority, predicate is called once", @@ -810,8 +880,6 @@ func TestFindFitPredicateCallCounts(t *testing.T) { } for _, test := range tests { - pc := predicateCallCounter{} - predicates := map[string]algorithmpredicates.FitPredicate{"true": pc.truePredicate()} nodes := makeNodeList([]string{"1"}) cache := internalcache.New(time.Duration(0), wait.NeverStop) @@ -819,16 +887,31 @@ func TestFindFitPredicateCallCounts(t *testing.T) { cache.AddNode(n) } + registry := framework.Registry{} + plugin := fakeFilterPlugin{} + registry.Register("FakeFilter", func(_ *runtime.Unknown, fh framework.FrameworkHandle) (framework.Plugin, error) { + return &plugin, nil + }) + plugins := &schedulerapi.Plugins{ + Filter: &schedulerapi.PluginSet{ + Enabled: []schedulerapi.Plugin{{Name: "FakeFilter"}}, + }, + } + pluginConfigs := []schedulerapi.PluginConfig{ + {Name: "FakeFilter"}, + } + fwk, _ := framework.NewFramework(registry, plugins, pluginConfigs) + queue := internalqueue.NewSchedulingQueue(nil) scheduler := NewGenericScheduler( cache, queue, - predicates, + nil, algorithmpredicates.EmptyMetadataProducer, nil, priorities.EmptyMetadataProducer, emptySnapshot, - emptyFramework, + fwk, nil, nil, nil, nil, false, false, schedulerapi.DefaultPercentageOfNodesToScore, false).(*genericScheduler) cache.UpdateNodeInfoSnapshot(scheduler.nodeInfoSnapshot) @@ -839,8 +922,8 @@ func TestFindFitPredicateCallCounts(t *testing.T) { if err != nil { t.Errorf("unexpected error: %v", err) } - if test.expectedCount != pc.count { - t.Errorf("predicate was called %d times, expected is %d", pc.count, test.expectedCount) + if test.expectedCount != plugin.numFilterCalled { + t.Errorf("predicate was called %d times, expected is %d", plugin.numFilterCalled, test.expectedCount) } } } @@ -1150,104 +1233,103 @@ var startTime20190107 = metav1.Date(2019, 1, 7, 1, 1, 1, 0, time.UTC) func TestSelectNodesForPreemption(t *testing.T) { defer algorithmpredicates.SetPredicatesOrderingDuringTest(order)() - filterPlugin := &FakeFilterPlugin{} - filterPluginRegistry := framework.Registry{filterPlugin.Name(): newPlugin(filterPlugin)} - filterFramework, err := framework.NewFramework(filterPluginRegistry, &schedulerapi.Plugins{ - Filter: &schedulerapi.PluginSet{ - Enabled: []schedulerapi.Plugin{ - { - Name: filterPlugin.Name(), - }, - }, - }, - }, []schedulerapi.PluginConfig{}) - if err != nil { - t.Errorf("Unexpected error when initialize scheduling framework, err :%v", err.Error()) - } - tests := []struct { name string - predicates map[string]algorithmpredicates.FitPredicate + registerFilterPlugins []st.RegisterFilterPluginFunc nodes []string pod *v1.Pod pods []*v1.Pod filterReturnCode framework.Code expected map[string]map[string]bool // Map from node name to a list of pods names which should be preempted. - expectednumFilterCalled int32 + expectedNumFilterCalled int32 addAffinityPredicate bool }{ { - name: "a pod that does not fit on any machine", - predicates: map[string]algorithmpredicates.FitPredicate{"matches": falsePredicate}, - nodes: []string{"machine1", "machine2"}, - pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "new", UID: types.UID("new")}, Spec: v1.PodSpec{Priority: &highPriority}}, + name: "a pod that does not fit on any machine", + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin("FalseFilter", NewFalseFilterPlugin), + }, + nodes: []string{"machine1", "machine2"}, + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "new", UID: types.UID("new")}, Spec: v1.PodSpec{Priority: &highPriority}}, pods: []*v1.Pod{ {ObjectMeta: metav1.ObjectMeta{Name: "a", UID: types.UID("a")}, Spec: v1.PodSpec{Priority: &midPriority, NodeName: "machine1"}}, {ObjectMeta: metav1.ObjectMeta{Name: "b", UID: types.UID("b")}, Spec: v1.PodSpec{Priority: &midPriority, NodeName: "machine2"}}}, expected: map[string]map[string]bool{}, - expectednumFilterCalled: 2, + expectedNumFilterCalled: 2, }, { - name: "a pod that fits with no preemption", - predicates: map[string]algorithmpredicates.FitPredicate{"matches": truePredicate}, - nodes: []string{"machine1", "machine2"}, - pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "new", UID: types.UID("new")}, Spec: v1.PodSpec{Priority: &highPriority}}, + name: "a pod that fits with no preemption", + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + }, + nodes: []string{"machine1", "machine2"}, + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "new", UID: types.UID("new")}, Spec: v1.PodSpec{Priority: &highPriority}}, pods: []*v1.Pod{ {ObjectMeta: metav1.ObjectMeta{Name: "a", UID: types.UID("a")}, Spec: v1.PodSpec{Priority: &midPriority, NodeName: "machine1"}}, {ObjectMeta: metav1.ObjectMeta{Name: "b", UID: types.UID("b")}, Spec: v1.PodSpec{Priority: &midPriority, NodeName: "machine2"}}}, expected: map[string]map[string]bool{"machine1": {}, "machine2": {}}, - expectednumFilterCalled: 4, + expectedNumFilterCalled: 4, }, { - name: "a pod that fits on one machine with no preemption", - predicates: map[string]algorithmpredicates.FitPredicate{"matches": matchesPredicate}, - nodes: []string{"machine1", "machine2"}, - pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Priority: &highPriority}}, + name: "a pod that fits on one machine with no preemption", + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin("MatchFilter", NewMatchFilterPlugin), + }, + nodes: []string{"machine1", "machine2"}, + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Priority: &highPriority}}, pods: []*v1.Pod{ {ObjectMeta: metav1.ObjectMeta{Name: "a", UID: types.UID("a")}, Spec: v1.PodSpec{Priority: &midPriority, NodeName: "machine1"}}, {ObjectMeta: metav1.ObjectMeta{Name: "b", UID: types.UID("b")}, Spec: v1.PodSpec{Priority: &midPriority, NodeName: "machine2"}}}, expected: map[string]map[string]bool{"machine1": {}}, - expectednumFilterCalled: 3, + expectedNumFilterCalled: 3, }, { - name: "a pod that fits on both machines when lower priority pods are preempted", - predicates: map[string]algorithmpredicates.FitPredicate{"matches": algorithmpredicates.PodFitsResources}, - nodes: []string{"machine1", "machine2"}, - pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority}}, + name: "a pod that fits on both machines when lower priority pods are preempted", + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + }, + nodes: []string{"machine1", "machine2"}, + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority}}, pods: []*v1.Pod{ {ObjectMeta: metav1.ObjectMeta{Name: "a", UID: types.UID("a")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &midPriority, NodeName: "machine1"}}, {ObjectMeta: metav1.ObjectMeta{Name: "b", UID: types.UID("b")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &midPriority, NodeName: "machine2"}}}, expected: map[string]map[string]bool{"machine1": {"a": true}, "machine2": {"b": true}}, - expectednumFilterCalled: 4, + expectedNumFilterCalled: 4, }, { - name: "a pod that would fit on the machines, but other pods running are higher priority", - predicates: map[string]algorithmpredicates.FitPredicate{"matches": algorithmpredicates.PodFitsResources}, - nodes: []string{"machine1", "machine2"}, - pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &lowPriority}}, + name: "a pod that would fit on the machines, but other pods running are higher priority", + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + }, + nodes: []string{"machine1", "machine2"}, + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &lowPriority}}, pods: []*v1.Pod{ {ObjectMeta: metav1.ObjectMeta{Name: "a", UID: types.UID("a")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &midPriority, NodeName: "machine1"}}, {ObjectMeta: metav1.ObjectMeta{Name: "b", UID: types.UID("b")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &midPriority, NodeName: "machine2"}}}, expected: map[string]map[string]bool{}, - expectednumFilterCalled: 2, + expectedNumFilterCalled: 2, }, { - name: "medium priority pod is preempted, but lower priority one stays as it is small", - predicates: map[string]algorithmpredicates.FitPredicate{"matches": algorithmpredicates.PodFitsResources}, - nodes: []string{"machine1", "machine2"}, - pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority}}, + name: "medium priority pod is preempted, but lower priority one stays as it is small", + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + }, + nodes: []string{"machine1", "machine2"}, + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority}}, pods: []*v1.Pod{ {ObjectMeta: metav1.ObjectMeta{Name: "a", UID: types.UID("a")}, Spec: v1.PodSpec{Containers: smallContainers, Priority: &lowPriority, NodeName: "machine1"}}, {ObjectMeta: metav1.ObjectMeta{Name: "b", UID: types.UID("b")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &midPriority, NodeName: "machine1"}}, {ObjectMeta: metav1.ObjectMeta{Name: "c", UID: types.UID("c")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &midPriority, NodeName: "machine2"}}}, expected: map[string]map[string]bool{"machine1": {"b": true}, "machine2": {"c": true}}, - expectednumFilterCalled: 5, + expectedNumFilterCalled: 5, }, { - name: "mixed priority pods are preempted", - predicates: map[string]algorithmpredicates.FitPredicate{"matches": algorithmpredicates.PodFitsResources}, - nodes: []string{"machine1", "machine2"}, - pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority}}, + name: "mixed priority pods are preempted", + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + }, + nodes: []string{"machine1", "machine2"}, + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority}}, pods: []*v1.Pod{ {ObjectMeta: metav1.ObjectMeta{Name: "a", UID: types.UID("a")}, Spec: v1.PodSpec{Containers: smallContainers, Priority: &midPriority, NodeName: "machine1"}}, {ObjectMeta: metav1.ObjectMeta{Name: "b", UID: types.UID("b")}, Spec: v1.PodSpec{Containers: smallContainers, Priority: &lowPriority, NodeName: "machine1"}}, @@ -1255,13 +1337,15 @@ func TestSelectNodesForPreemption(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "d", UID: types.UID("d")}, Spec: v1.PodSpec{Containers: smallContainers, Priority: &highPriority, NodeName: "machine1"}}, {ObjectMeta: metav1.ObjectMeta{Name: "e", UID: types.UID("e")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority, NodeName: "machine2"}}}, expected: map[string]map[string]bool{"machine1": {"b": true, "c": true}}, - expectednumFilterCalled: 5, + expectedNumFilterCalled: 5, }, { - name: "mixed priority pods are preempted, pick later StartTime one when priorities are equal", - predicates: map[string]algorithmpredicates.FitPredicate{"matches": algorithmpredicates.PodFitsResources}, - nodes: []string{"machine1", "machine2"}, - pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority}}, + name: "mixed priority pods are preempted, pick later StartTime one when priorities are equal", + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + }, + nodes: []string{"machine1", "machine2"}, + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority}}, pods: []*v1.Pod{ {ObjectMeta: metav1.ObjectMeta{Name: "a", UID: types.UID("a")}, Spec: v1.PodSpec{Containers: smallContainers, Priority: &lowPriority, NodeName: "machine1"}, Status: v1.PodStatus{StartTime: &startTime20190107}}, {ObjectMeta: metav1.ObjectMeta{Name: "b", UID: types.UID("b")}, Spec: v1.PodSpec{Containers: smallContainers, Priority: &lowPriority, NodeName: "machine1"}, Status: v1.PodStatus{StartTime: &startTime20190106}}, @@ -1269,12 +1353,15 @@ func TestSelectNodesForPreemption(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "d", UID: types.UID("d")}, Spec: v1.PodSpec{Containers: smallContainers, Priority: &highPriority, NodeName: "machine1"}, Status: v1.PodStatus{StartTime: &startTime20190104}}, {ObjectMeta: metav1.ObjectMeta{Name: "e", UID: types.UID("e")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority, NodeName: "machine2"}, Status: v1.PodStatus{StartTime: &startTime20190103}}}, expected: map[string]map[string]bool{"machine1": {"a": true, "c": true}}, - expectednumFilterCalled: 5, + expectedNumFilterCalled: 5, }, { - name: "pod with anti-affinity is preempted", - predicates: map[string]algorithmpredicates.FitPredicate{"matches": algorithmpredicates.PodFitsResources}, - nodes: []string{"machine1", "machine2"}, + name: "pod with anti-affinity is preempted", + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterFilterPlugin(interpodaffinity.Name, interpodaffinity.New), + }, + nodes: []string{"machine1", "machine2"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ Name: "machine1", Labels: map[string]string{"pod": "preemptor"}}, Spec: v1.PodSpec{Containers: smallContainers, Priority: &highPriority}}, @@ -1300,13 +1387,13 @@ func TestSelectNodesForPreemption(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "d", UID: types.UID("d")}, Spec: v1.PodSpec{Containers: smallContainers, Priority: &highPriority, NodeName: "machine1"}}, {ObjectMeta: metav1.ObjectMeta{Name: "e", UID: types.UID("e")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority, NodeName: "machine2"}}}, expected: map[string]map[string]bool{"machine1": {"a": true}, "machine2": {}}, - expectednumFilterCalled: 4, + expectedNumFilterCalled: 4, addAffinityPredicate: true, }, { name: "preemption to resolve even pods spread FitError", - predicates: map[string]algorithmpredicates.FitPredicate{ - "matches": algorithmpredicates.EvenPodsSpreadPredicate, + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin(podtopologyspread.Name, podtopologyspread.New), }, nodes: []string{"node-a/zone1", "node-b/zone1", "node-x/zone2"}, pod: &v1.Pod{ @@ -1377,19 +1464,21 @@ func TestSelectNodesForPreemption(t *testing.T) { "node-a": {"pod-a2": true}, "node-b": {"pod-b1": true}, }, - expectednumFilterCalled: 6, + expectedNumFilterCalled: 6, }, { - name: "get Unschedulable in the preemption phase when the filter plugins filtering the nodes", - predicates: map[string]algorithmpredicates.FitPredicate{"matches": algorithmpredicates.PodFitsResources}, - nodes: []string{"machine1", "machine2"}, - pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority}}, + name: "get Unschedulable in the preemption phase when the filter plugins filtering the nodes", + registerFilterPlugins: []st.RegisterFilterPluginFunc{ + st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + }, + nodes: []string{"machine1", "machine2"}, + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority}}, pods: []*v1.Pod{ {ObjectMeta: metav1.ObjectMeta{Name: "a", UID: types.UID("a")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &midPriority, NodeName: "machine1"}}, {ObjectMeta: metav1.ObjectMeta{Name: "b", UID: types.UID("b")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &midPriority, NodeName: "machine2"}}}, filterReturnCode: framework.Unschedulable, expected: map[string]map[string]bool{}, - expectednumFilterCalled: 2, + expectedNumFilterCalled: 2, }, } labelKeys := []string{"hostname", "zone", "region"} @@ -1408,17 +1497,52 @@ func TestSelectNodesForPreemption(t *testing.T) { cache.AddNode(&v1.Node{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{"hostname": name}}}) } + var nodes []*v1.Node + for _, n := range test.nodes { + node := makeNode(n, 1000*5, priorityutil.DefaultMemoryRequest*5) + // if possible, split node name by '/' to form labels in a format of + // {"hostname": node.Name[0], "zone": node.Name[1], "region": node.Name[2]} + node.ObjectMeta.Labels = make(map[string]string) + for i, label := range strings.Split(node.Name, "/") { + node.ObjectMeta.Labels[labelKeys[i]] = label + } + node.Name = node.ObjectMeta.Labels["hostname"] + nodes = append(nodes, node) + } + + registry := framework.Registry{} + plugins := &schedulerapi.Plugins{ + Filter: &schedulerapi.PluginSet{}, + } + var pluginConfigs []schedulerapi.PluginConfig + // For each test, prepend a FakeFilterPlugin. + fakePlugin := fakeFilterPlugin{} + fakePlugin.failedNodeReturnCodeMap = filterFailedNodeReturnCodeMap + registerFakeFilterFunc := st.RegisterFilterPlugin( + "FakeFilter", + func(_ *runtime.Unknown, fh framework.FrameworkHandle) (framework.Plugin, error) { + return &fakePlugin, nil + }, + ) + registerFakeFilterFunc(®istry, plugins, pluginConfigs) + // Next, register other filter plugins defined in test struct. + for _, f := range test.registerFilterPlugins { + f(®istry, plugins, pluginConfigs) + } + // Use a real snapshot since it's needed in some Filter Plugin (e.g., PodAffinity) + snapshot := nodeinfosnapshot.NewSnapshot(nodeinfosnapshot.CreateNodeInfoMap(test.pods, nodes)) + fwk, _ := framework.NewFramework(registry, plugins, pluginConfigs, framework.WithSnapshotSharedLister(snapshot)) + factory := &algorithmpredicates.MetadataProducerFactory{} - filterPlugin.failedNodeReturnCodeMap = filterFailedNodeReturnCodeMap scheduler := NewGenericScheduler( nil, internalqueue.NewSchedulingQueue(nil), - test.predicates, + nil, factory.GetPredicateMetadata, nil, priorities.EmptyMetadataProducer, - emptySnapshot, - filterFramework, + snapshot, + fwk, []algorithm.SchedulerExtender{}, nil, nil, @@ -1431,25 +1555,6 @@ func TestSelectNodesForPreemption(t *testing.T) { assignDefaultStartTime(test.pods) - nodes := []*v1.Node{} - for _, n := range test.nodes { - node := makeNode(n, 1000*5, priorityutil.DefaultMemoryRequest*5) - // if possible, split node name by '/' to form labels in a format of - // {"hostname": node.Name[0], "zone": node.Name[1], "region": node.Name[2]} - node.ObjectMeta.Labels = make(map[string]string) - for i, label := range strings.Split(node.Name, "/") { - node.ObjectMeta.Labels[labelKeys[i]] = label - } - node.Name = node.ObjectMeta.Labels["hostname"] - nodes = append(nodes, node) - } - if test.addAffinityPredicate { - n := fakelisters.NewNodeInfoLister([]*v1.Node{nodes[0]}) - p := fakelisters.PodLister(test.pods) - test.predicates[algorithmpredicates.MatchInterPodAffinityPred] = algorithmpredicates.NewPodAffinityPredicate(n, p) - } - - g.nodeInfoSnapshot = nodeinfosnapshot.NewSnapshot(nodeinfosnapshot.CreateNodeInfoMap(test.pods, nodes)) // newnode simulate a case that a new node is added to the cluster, but nodeNameToInfo // doesn't have it yet. newnode := makeNode("newnode", 1000*5, priorityutil.DefaultMemoryRequest*5) @@ -1461,15 +1566,13 @@ func TestSelectNodesForPreemption(t *testing.T) { t.Error(err) } - if test.expectednumFilterCalled != filterPlugin.numFilterCalled { - t.Errorf("expected filterPlugin.numFilterCalled is %d,nut got %d", test.expectednumFilterCalled, filterPlugin.numFilterCalled) + if test.expectedNumFilterCalled != fakePlugin.numFilterCalled { + t.Errorf("expected fakePlugin.numFilterCalled is %d, but got %d", test.expectedNumFilterCalled, fakePlugin.numFilterCalled) } if err := checkPreemptionVictims(test.expected, nodeToPods); err != nil { t.Error(err) } - - filterPlugin.reset() }) } } @@ -1478,27 +1581,27 @@ func TestSelectNodesForPreemption(t *testing.T) { func TestPickOneNodeForPreemption(t *testing.T) { defer algorithmpredicates.SetPredicatesOrderingDuringTest(order)() tests := []struct { - name string - predicates map[string]algorithmpredicates.FitPredicate - nodes []string - pod *v1.Pod - pods []*v1.Pod - expected []string // any of the items is valid + name string + registerFilterPlugin st.RegisterFilterPluginFunc + nodes []string + pod *v1.Pod + pods []*v1.Pod + expected []string // any of the items is valid }{ { - name: "No node needs preemption", - predicates: map[string]algorithmpredicates.FitPredicate{"matches": algorithmpredicates.PodFitsResources}, - nodes: []string{"machine1"}, - pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority}}, + name: "No node needs preemption", + registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + nodes: []string{"machine1"}, + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority}}, pods: []*v1.Pod{ {ObjectMeta: metav1.ObjectMeta{Name: "m1.1", UID: types.UID("m1.1")}, Spec: v1.PodSpec{Containers: smallContainers, Priority: &midPriority, NodeName: "machine1"}, Status: v1.PodStatus{StartTime: &startTime}}}, expected: []string{"machine1"}, }, { - name: "a pod that fits on both machines when lower priority pods are preempted", - predicates: map[string]algorithmpredicates.FitPredicate{"matches": algorithmpredicates.PodFitsResources}, - nodes: []string{"machine1", "machine2"}, - pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority}}, + name: "a pod that fits on both machines when lower priority pods are preempted", + registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + nodes: []string{"machine1", "machine2"}, + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority}}, pods: []*v1.Pod{ {ObjectMeta: metav1.ObjectMeta{Name: "m1.1", UID: types.UID("m1.1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &midPriority, NodeName: "machine1"}, Status: v1.PodStatus{StartTime: &startTime}}, @@ -1506,10 +1609,10 @@ func TestPickOneNodeForPreemption(t *testing.T) { expected: []string{"machine1", "machine2"}, }, { - name: "a pod that fits on a machine with no preemption", - predicates: map[string]algorithmpredicates.FitPredicate{"matches": algorithmpredicates.PodFitsResources}, - nodes: []string{"machine1", "machine2", "machine3"}, - pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority}}, + name: "a pod that fits on a machine with no preemption", + registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + nodes: []string{"machine1", "machine2", "machine3"}, + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority}}, pods: []*v1.Pod{ {ObjectMeta: metav1.ObjectMeta{Name: "m1.1", UID: types.UID("m1.1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &midPriority, NodeName: "machine1"}, Status: v1.PodStatus{StartTime: &startTime}}, @@ -1517,10 +1620,10 @@ func TestPickOneNodeForPreemption(t *testing.T) { expected: []string{"machine3"}, }, { - name: "machine with min highest priority pod is picked", - predicates: map[string]algorithmpredicates.FitPredicate{"matches": algorithmpredicates.PodFitsResources}, - nodes: []string{"machine1", "machine2", "machine3"}, - pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: veryLargeContainers, Priority: &highPriority}}, + name: "machine with min highest priority pod is picked", + registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + nodes: []string{"machine1", "machine2", "machine3"}, + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: veryLargeContainers, Priority: &highPriority}}, pods: []*v1.Pod{ {ObjectMeta: metav1.ObjectMeta{Name: "m1.1", UID: types.UID("m1.1")}, Spec: v1.PodSpec{Containers: mediumContainers, Priority: &midPriority, NodeName: "machine1"}, Status: v1.PodStatus{StartTime: &startTime}}, {ObjectMeta: metav1.ObjectMeta{Name: "m1.2", UID: types.UID("m1.2")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &midPriority, NodeName: "machine1"}, Status: v1.PodStatus{StartTime: &startTime}}, @@ -1534,10 +1637,10 @@ func TestPickOneNodeForPreemption(t *testing.T) { expected: []string{"machine3"}, }, { - name: "when highest priorities are the same, minimum sum of priorities is picked", - predicates: map[string]algorithmpredicates.FitPredicate{"matches": algorithmpredicates.PodFitsResources}, - nodes: []string{"machine1", "machine2", "machine3"}, - pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: veryLargeContainers, Priority: &highPriority}}, + name: "when highest priorities are the same, minimum sum of priorities is picked", + registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + nodes: []string{"machine1", "machine2", "machine3"}, + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: veryLargeContainers, Priority: &highPriority}}, pods: []*v1.Pod{ {ObjectMeta: metav1.ObjectMeta{Name: "m1.1", UID: types.UID("m1.1")}, Spec: v1.PodSpec{Containers: mediumContainers, Priority: &midPriority, NodeName: "machine1"}, Status: v1.PodStatus{StartTime: &startTime}}, {ObjectMeta: metav1.ObjectMeta{Name: "m1.2", UID: types.UID("m1.2")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &midPriority, NodeName: "machine1"}, Status: v1.PodStatus{StartTime: &startTime}}, @@ -1551,10 +1654,10 @@ func TestPickOneNodeForPreemption(t *testing.T) { expected: []string{"machine2"}, }, { - name: "when highest priority and sum are the same, minimum number of pods is picked", - predicates: map[string]algorithmpredicates.FitPredicate{"matches": algorithmpredicates.PodFitsResources}, - nodes: []string{"machine1", "machine2", "machine3"}, - pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: veryLargeContainers, Priority: &highPriority}}, + name: "when highest priority and sum are the same, minimum number of pods is picked", + registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + nodes: []string{"machine1", "machine2", "machine3"}, + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: veryLargeContainers, Priority: &highPriority}}, pods: []*v1.Pod{ {ObjectMeta: metav1.ObjectMeta{Name: "m1.1", UID: types.UID("m1.1")}, Spec: v1.PodSpec{Containers: smallContainers, Priority: &midPriority, NodeName: "machine1"}, Status: v1.PodStatus{StartTime: &startTime}}, {ObjectMeta: metav1.ObjectMeta{Name: "m1.2", UID: types.UID("m1.2")}, Spec: v1.PodSpec{Containers: smallContainers, Priority: &negPriority, NodeName: "machine1"}, Status: v1.PodStatus{StartTime: &startTime}}, @@ -1573,10 +1676,10 @@ func TestPickOneNodeForPreemption(t *testing.T) { { // pickOneNodeForPreemption adjusts pod priorities when finding the sum of the victims. This // test ensures that the logic works correctly. - name: "sum of adjusted priorities is considered", - predicates: map[string]algorithmpredicates.FitPredicate{"matches": algorithmpredicates.PodFitsResources}, - nodes: []string{"machine1", "machine2", "machine3"}, - pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: veryLargeContainers, Priority: &highPriority}}, + name: "sum of adjusted priorities is considered", + registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + nodes: []string{"machine1", "machine2", "machine3"}, + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: veryLargeContainers, Priority: &highPriority}}, pods: []*v1.Pod{ {ObjectMeta: metav1.ObjectMeta{Name: "m1.1", UID: types.UID("m1.1")}, Spec: v1.PodSpec{Containers: smallContainers, Priority: &midPriority, NodeName: "machine1"}, Status: v1.PodStatus{StartTime: &startTime}}, {ObjectMeta: metav1.ObjectMeta{Name: "m1.2", UID: types.UID("m1.2")}, Spec: v1.PodSpec{Containers: smallContainers, Priority: &negPriority, NodeName: "machine1"}, Status: v1.PodStatus{StartTime: &startTime}}, @@ -1592,10 +1695,10 @@ func TestPickOneNodeForPreemption(t *testing.T) { expected: []string{"machine2"}, }, { - name: "non-overlapping lowest high priority, sum priorities, and number of pods", - predicates: map[string]algorithmpredicates.FitPredicate{"matches": algorithmpredicates.PodFitsResources}, - nodes: []string{"machine1", "machine2", "machine3", "machine4"}, - pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "pod1", UID: types.UID("pod1")}, Spec: v1.PodSpec{Containers: veryLargeContainers, Priority: &veryHighPriority}}, + name: "non-overlapping lowest high priority, sum priorities, and number of pods", + registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + nodes: []string{"machine1", "machine2", "machine3", "machine4"}, + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "pod1", UID: types.UID("pod1")}, Spec: v1.PodSpec{Containers: veryLargeContainers, Priority: &veryHighPriority}}, pods: []*v1.Pod{ {ObjectMeta: metav1.ObjectMeta{Name: "m1.1", UID: types.UID("m1.1")}, Spec: v1.PodSpec{Containers: smallContainers, Priority: &midPriority, NodeName: "machine1"}, Status: v1.PodStatus{StartTime: &startTime}}, {ObjectMeta: metav1.ObjectMeta{Name: "m1.2", UID: types.UID("m1.2")}, Spec: v1.PodSpec{Containers: smallContainers, Priority: &lowPriority, NodeName: "machine1"}, Status: v1.PodStatus{StartTime: &startTime}}, @@ -1616,10 +1719,10 @@ func TestPickOneNodeForPreemption(t *testing.T) { expected: []string{"machine1"}, }, { - name: "same priority, same number of victims, different start time for each machine's pod", - predicates: map[string]algorithmpredicates.FitPredicate{"matches": algorithmpredicates.PodFitsResources}, - nodes: []string{"machine1", "machine2", "machine3"}, - pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: veryLargeContainers, Priority: &highPriority}}, + name: "same priority, same number of victims, different start time for each machine's pod", + registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + nodes: []string{"machine1", "machine2", "machine3"}, + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: veryLargeContainers, Priority: &highPriority}}, pods: []*v1.Pod{ {ObjectMeta: metav1.ObjectMeta{Name: "m1.1", UID: types.UID("m1.1")}, Spec: v1.PodSpec{Containers: mediumContainers, Priority: &midPriority, NodeName: "machine1"}, Status: v1.PodStatus{StartTime: &startTime20190103}}, {ObjectMeta: metav1.ObjectMeta{Name: "m1.2", UID: types.UID("m1.2")}, Spec: v1.PodSpec{Containers: mediumContainers, Priority: &midPriority, NodeName: "machine1"}, Status: v1.PodStatus{StartTime: &startTime20190103}}, @@ -1633,10 +1736,10 @@ func TestPickOneNodeForPreemption(t *testing.T) { expected: []string{"machine2"}, }, { - name: "same priority, same number of victims, different start time for all pods", - predicates: map[string]algorithmpredicates.FitPredicate{"matches": algorithmpredicates.PodFitsResources}, - nodes: []string{"machine1", "machine2", "machine3"}, - pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: veryLargeContainers, Priority: &highPriority}}, + name: "same priority, same number of victims, different start time for all pods", + registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + nodes: []string{"machine1", "machine2", "machine3"}, + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: veryLargeContainers, Priority: &highPriority}}, pods: []*v1.Pod{ {ObjectMeta: metav1.ObjectMeta{Name: "m1.1", UID: types.UID("m1.1")}, Spec: v1.PodSpec{Containers: mediumContainers, Priority: &midPriority, NodeName: "machine1"}, Status: v1.PodStatus{StartTime: &startTime20190105}}, {ObjectMeta: metav1.ObjectMeta{Name: "m1.2", UID: types.UID("m1.2")}, Spec: v1.PodSpec{Containers: mediumContainers, Priority: &midPriority, NodeName: "machine1"}, Status: v1.PodStatus{StartTime: &startTime20190103}}, @@ -1650,10 +1753,10 @@ func TestPickOneNodeForPreemption(t *testing.T) { expected: []string{"machine3"}, }, { - name: "different priority, same number of victims, different start time for all pods", - predicates: map[string]algorithmpredicates.FitPredicate{"matches": algorithmpredicates.PodFitsResources}, - nodes: []string{"machine1", "machine2", "machine3"}, - pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: veryLargeContainers, Priority: &highPriority}}, + name: "different priority, same number of victims, different start time for all pods", + registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + nodes: []string{"machine1", "machine2", "machine3"}, + pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: veryLargeContainers, Priority: &highPriority}}, pods: []*v1.Pod{ {ObjectMeta: metav1.ObjectMeta{Name: "m1.1", UID: types.UID("m1.1")}, Spec: v1.PodSpec{Containers: mediumContainers, Priority: &lowPriority, NodeName: "machine1"}, Status: v1.PodStatus{StartTime: &startTime20190105}}, {ObjectMeta: metav1.ObjectMeta{Name: "m1.2", UID: types.UID("m1.2")}, Spec: v1.PodSpec{Containers: mediumContainers, Priority: &midPriority, NodeName: "machine1"}, Status: v1.PodStatus{StartTime: &startTime20190103}}, @@ -1674,13 +1777,18 @@ func TestPickOneNodeForPreemption(t *testing.T) { nodes = append(nodes, makeNode(n, priorityutil.DefaultMilliCPURequest*5, priorityutil.DefaultMemoryRequest*5)) } snapshot := nodeinfosnapshot.NewSnapshot(nodeinfosnapshot.CreateNodeInfoMap(test.pods, nodes)) - fwk, _ := framework.NewFramework(emptyPluginRegistry, nil, []schedulerapi.PluginConfig{}, framework.WithSnapshotSharedLister(snapshot)) + registry := framework.Registry{} + plugins := &schedulerapi.Plugins{ + Filter: &schedulerapi.PluginSet{}, + } + var pluginConfigs []schedulerapi.PluginConfig + test.registerFilterPlugin(®istry, plugins, pluginConfigs) + fwk, _ := framework.NewFramework(registry, plugins, pluginConfigs, framework.WithSnapshotSharedLister(snapshot)) factory := algorithmpredicates.MetadataProducerFactory{} g := &genericScheduler{ framework: fwk, nodeInfoSnapshot: snapshot, - predicates: test.predicates, predicateMetaProducer: factory.GetPredicateMetadata, } assignDefaultStartTime(test.pods) @@ -1845,13 +1953,13 @@ func TestNodesWherePreemptionMightHelp(t *testing.T) { func TestPreempt(t *testing.T) { defer algorithmpredicates.SetPredicatesOrderingDuringTest(order)() - defaultFailedPredMap := FailedPredicateMap{ - "machine1": []algorithmpredicates.PredicateFailureReason{algorithmpredicates.NewInsufficientResourceError(v1.ResourceMemory, 1000, 500, 300)}, - "machine2": []algorithmpredicates.PredicateFailureReason{algorithmpredicates.ErrDiskConflict}, - "machine3": []algorithmpredicates.PredicateFailureReason{algorithmpredicates.NewInsufficientResourceError(v1.ResourceMemory, 1000, 600, 400)}, + defaultFailedNodeToStatusMap := framework.NodeToStatusMap{ + "machine1": framework.NewStatus(framework.Unschedulable, algorithmpredicates.NewInsufficientResourceError(v1.ResourceMemory, 1000, 500, 300).GetReason()), + "machine2": framework.NewStatus(framework.Unschedulable, algorithmpredicates.ErrDiskConflict.GetReason()), + "machine3": framework.NewStatus(framework.Unschedulable, algorithmpredicates.NewInsufficientResourceError(v1.ResourceMemory, 1000, 600, 400).GetReason()), } // Prepare 3 node names. - defaultNodeNames := []string{} + var defaultNodeNames []string for i := 1; i < 4; i++ { defaultNodeNames = append(defaultNodeNames, fmt.Sprintf("machine%d", i)) } @@ -1860,16 +1968,16 @@ func TestPreempt(t *testing.T) { preemptNever = v1.PreemptNever ) tests := []struct { - name string - pod *v1.Pod - pods []*v1.Pod - extenders []*FakeExtender - failedPredMap FailedPredicateMap - nodeNames []string - predicate algorithmpredicates.FitPredicate - buildPredMeta bool - expectedNode string - expectedPods []string // list of preempted pods + name string + pod *v1.Pod + pods []*v1.Pod + extenders []*FakeExtender + failedNodeToStatusMap framework.NodeToStatusMap + nodeNames []string + registerFilterPlugin st.RegisterFilterPluginFunc + buildPredMeta bool + expectedNode string + expectedPods []string // list of preempted pods }{ { name: "basic preemption logic", @@ -1884,8 +1992,9 @@ func TestPreempt(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "m2.1", UID: types.UID("m2.1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority, NodeName: "machine2"}, Status: v1.PodStatus{Phase: v1.PodRunning}}, {ObjectMeta: metav1.ObjectMeta{Name: "m3.1", UID: types.UID("m3.1")}, Spec: v1.PodSpec{Containers: mediumContainers, Priority: &midPriority, NodeName: "machine3"}, Status: v1.PodStatus{Phase: v1.PodRunning}}, }, - expectedNode: "machine1", - expectedPods: []string{"m1.1", "m1.2"}, + registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + expectedNode: "machine1", + expectedPods: []string{"m1.1", "m1.2"}, }, { name: "One node doesn't need any preemption", @@ -1900,8 +2009,9 @@ func TestPreempt(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "m2.1", UID: types.UID("m2.1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority, NodeName: "machine2"}, Status: v1.PodStatus{Phase: v1.PodRunning}}, }, - expectedNode: "machine3", - expectedPods: []string{}, + registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + expectedNode: "machine3", + expectedPods: []string{}, }, { name: "preemption for topology spread constraints", @@ -1969,16 +2079,16 @@ func TestPreempt(t *testing.T) { Status: v1.PodStatus{Phase: v1.PodRunning}, }, }, - failedPredMap: FailedPredicateMap{ - "node-a": []algorithmpredicates.PredicateFailureReason{algorithmpredicates.ErrTopologySpreadConstraintsNotMatch}, - "node-b": []algorithmpredicates.PredicateFailureReason{algorithmpredicates.ErrTopologySpreadConstraintsNotMatch}, - "node-x": []algorithmpredicates.PredicateFailureReason{algorithmpredicates.ErrTopologySpreadConstraintsNotMatch}, + failedNodeToStatusMap: framework.NodeToStatusMap{ + "node-a": framework.NewStatus(framework.Unschedulable, algorithmpredicates.ErrTopologySpreadConstraintsNotMatch.GetReason()), + "node-b": framework.NewStatus(framework.Unschedulable, algorithmpredicates.ErrTopologySpreadConstraintsNotMatch.GetReason()), + "node-x": framework.NewStatus(framework.Unschedulable, algorithmpredicates.ErrTopologySpreadConstraintsNotMatch.GetReason()), }, - predicate: algorithmpredicates.EvenPodsSpreadPredicate, - buildPredMeta: true, - nodeNames: []string{"node-a/zone1", "node-b/zone1", "node-x/zone2"}, - expectedNode: "node-b", - expectedPods: []string{"pod-b1"}, + buildPredMeta: true, + nodeNames: []string{"node-a/zone1", "node-b/zone1", "node-x/zone2"}, + registerFilterPlugin: st.RegisterFilterPlugin(podtopologyspread.Name, podtopologyspread.New), + expectedNode: "node-b", + expectedPods: []string{"pod-b1"}, }, { name: "Scheduler extenders allow only machine1, otherwise machine3 would have been chosen", @@ -2001,8 +2111,9 @@ func TestPreempt(t *testing.T) { predicates: []fitPredicate{machine1PredicateExtender}, }, }, - expectedNode: "machine1", - expectedPods: []string{"m1.1", "m1.2"}, + registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + expectedNode: "machine1", + expectedPods: []string{"m1.1", "m1.2"}, }, { name: "Scheduler extenders do not allow any preemption", @@ -2022,8 +2133,9 @@ func TestPreempt(t *testing.T) { predicates: []fitPredicate{falsePredicateExtender}, }, }, - expectedNode: "", - expectedPods: []string{}, + registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + expectedNode: "", + expectedPods: []string{}, }, { name: "One scheduler extender allows only machine1, the other returns error but ignorable. Only machine1 would be chosen", @@ -2047,8 +2159,9 @@ func TestPreempt(t *testing.T) { predicates: []fitPredicate{machine1PredicateExtender}, }, }, - expectedNode: "machine1", - expectedPods: []string{"m1.1", "m1.2"}, + registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + expectedNode: "machine1", + expectedPods: []string{"m1.1", "m1.2"}, }, { name: "One scheduler extender allows only machine1, but it is not interested in given pod, otherwise machine1 would have been chosen", @@ -2072,8 +2185,9 @@ func TestPreempt(t *testing.T) { predicates: []fitPredicate{truePredicateExtender}, }, }, - expectedNode: "machine3", - expectedPods: []string{}, + registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + expectedNode: "machine3", + expectedPods: []string{}, }, { name: "no preempting in pod", @@ -2088,8 +2202,9 @@ func TestPreempt(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "m2.1", UID: types.UID("m2.1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority, NodeName: "machine2"}, Status: v1.PodStatus{Phase: v1.PodRunning}}, {ObjectMeta: metav1.ObjectMeta{Name: "m3.1", UID: types.UID("m3.1")}, Spec: v1.PodSpec{Containers: mediumContainers, Priority: &midPriority, NodeName: "machine3"}, Status: v1.PodStatus{Phase: v1.PodRunning}}, }, - expectedNode: "", - expectedPods: nil, + registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + expectedNode: "", + expectedPods: nil, }, { name: "PreemptionPolicy is nil", @@ -2104,8 +2219,9 @@ func TestPreempt(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "m2.1", UID: types.UID("m2.1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority, NodeName: "machine2"}, Status: v1.PodStatus{Phase: v1.PodRunning}}, {ObjectMeta: metav1.ObjectMeta{Name: "m3.1", UID: types.UID("m3.1")}, Spec: v1.PodSpec{Containers: mediumContainers, Priority: &midPriority, NodeName: "machine3"}, Status: v1.PodStatus{Phase: v1.PodRunning}}, }, - expectedNode: "machine1", - expectedPods: []string{"m1.1", "m1.2"}, + registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + expectedNode: "machine1", + expectedPods: []string{"m1.1", "m1.2"}, }, } @@ -2115,7 +2231,6 @@ func TestPreempt(t *testing.T) { client := clientsetfake.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(client, 0) - t.Logf("===== Running test %v", t.Name()) stop := make(chan struct{}) cache := internalcache.New(time.Duration(0), stop) for _, pod := range test.pods { @@ -2149,24 +2264,29 @@ func TestPreempt(t *testing.T) { extender.cachedNodeNameToInfo = cachedNodeInfoMap extenders = append(extenders, extender) } - predicate := algorithmpredicates.PodFitsResources - if test.predicate != nil { - predicate = test.predicate - } predMetaProducer := algorithmpredicates.EmptyMetadataProducer if test.buildPredMeta { f := &algorithmpredicates.MetadataProducerFactory{} predMetaProducer = f.GetPredicateMetadata } + + registry := framework.Registry{} + plugins := &schedulerapi.Plugins{ + Filter: &schedulerapi.PluginSet{}, + } + var pluginConfigs []schedulerapi.PluginConfig + test.registerFilterPlugin(®istry, plugins, pluginConfigs) + fwk, _ := framework.NewFramework(registry, plugins, pluginConfigs) + scheduler := NewGenericScheduler( cache, internalqueue.NewSchedulingQueue(nil), - map[string]algorithmpredicates.FitPredicate{"matches": predicate}, + nil, predMetaProducer, []priorities.PriorityConfig{{Map: numericMapPriority, Weight: 1}}, priorities.EmptyMetadataProducer, emptySnapshot, - emptyFramework, + fwk, extenders, nil, informerFactory.Core().V1().PersistentVolumeClaims().Lister(), @@ -2178,11 +2298,11 @@ func TestPreempt(t *testing.T) { state := framework.NewCycleState() scheduler.Snapshot() // Call Preempt and check the expected results. - failedPredMap := defaultFailedPredMap - if test.failedPredMap != nil { - failedPredMap = test.failedPredMap + failedNodeToStatusMap := defaultFailedNodeToStatusMap + if test.failedNodeToStatusMap != nil { + failedNodeToStatusMap = test.failedNodeToStatusMap } - node, victims, _, err := scheduler.Preempt(context.Background(), state, test.pod, error(&FitError{Pod: test.pod, FailedPredicates: failedPredMap})) + node, victims, _, err := scheduler.Preempt(context.Background(), state, test.pod, error(&FitError{Pod: test.pod, FilteredNodesStatuses: failedNodeToStatusMap})) if err != nil { t.Errorf("unexpected error in preemption: %v", err) } @@ -2212,7 +2332,7 @@ func TestPreempt(t *testing.T) { test.pod.Status.NominatedNodeName = node.Name } // Call preempt again and make sure it doesn't preempt any more pods. - node, victims, _, err = scheduler.Preempt(context.Background(), state, test.pod, error(&FitError{Pod: test.pod, FailedPredicates: failedPredMap})) + node, victims, _, err = scheduler.Preempt(context.Background(), state, test.pod, error(&FitError{Pod: test.pod, FilteredNodesStatuses: failedNodeToStatusMap})) if err != nil { t.Errorf("unexpected error in preemption: %v", err) } @@ -2289,14 +2409,16 @@ func assignDefaultStartTime(pods []*v1.Pod) { func TestFairEvaluationForNodes(t *testing.T) { defer algorithmpredicates.SetPredicatesOrderingDuringTest(order)() - predicates := map[string]algorithmpredicates.FitPredicate{"true": truePredicate} numAllNodes := 500 nodeNames := make([]string, 0, numAllNodes) for i := 0; i < numAllNodes; i++ { nodeNames = append(nodeNames, strconv.Itoa(i)) } nodes := makeNodeList(nodeNames) - g := makeScheduler(predicates, nodes) + g := makeScheduler( + nodes, + st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + ) // To make numAllNodes % nodesToFind != 0 g.percentageOfNodesToScore = 30 nodesToFind := int(g.numFeasibleNodesToFind(int32(numAllNodes))) diff --git a/pkg/scheduler/factory_test.go b/pkg/scheduler/factory_test.go index 881bd8411c9..d805a4d092d 100644 --- a/pkg/scheduler/factory_test.go +++ b/pkg/scheduler/factory_test.go @@ -231,16 +231,28 @@ func TestCreateFromEmptyConfig(t *testing.T) { // Test configures a scheduler from a policy that does not specify any // predicate/priority. // The predicate/priority from DefaultProvider will be used. +// TODO(Huang-Wei): refactor (or remove) this test along with eliminating 'RegisterFitPredicate()'. func TestCreateFromConfigWithUnspecifiedPredicatesOrPriorities(t *testing.T) { + predicateOne := "PredicateOne" client := fake.NewSimpleClientset() stopCh := make(chan struct{}) defer close(stopCh) factory := newConfigFactory(client, v1.DefaultHardPodAffinitySymmetricWeight, stopCh) + factory.registry.Register(predicateOne, func(_ *runtime.Unknown, fh framework.FrameworkHandle) (framework.Plugin, error) { + return &TestPlugin{name: predicateOne}, nil + }) + factory.pluginConfigProducerRegistry.RegisterPredicate(predicateOne, func(_ frameworkplugins.ConfigProducerArgs) (schedulerapi.Plugins, []schedulerapi.PluginConfig) { + return schedulerapi.Plugins{ + Filter: &schedulerapi.PluginSet{ + Enabled: []schedulerapi.Plugin{{Name: predicateOne}}, + }, + }, nil + }) - RegisterFitPredicate("PredicateOne", PredicateFunc) + RegisterFitPredicate(predicateOne, PredicateFunc) RegisterPriorityMapReduceFunction("PriorityOne", PriorityFunc, nil, 1) - RegisterAlgorithmProvider(schedulerapi.SchedulerDefaultProviderName, sets.NewString("PredicateOne"), sets.NewString("PriorityOne")) + RegisterAlgorithmProvider(schedulerapi.SchedulerDefaultProviderName, sets.NewString(predicateOne), sets.NewString("PriorityOne")) configData := []byte(`{ "kind" : "Policy", @@ -255,7 +267,7 @@ func TestCreateFromConfigWithUnspecifiedPredicatesOrPriorities(t *testing.T) { if err != nil { t.Fatalf("Failed to create scheduler from configuration: %v", err) } - if _, found := c.Algorithm.Predicates()["PredicateOne"]; !found { + if !foundPlugin(c.Plugins.Filter.Enabled, predicateOne) { t.Errorf("Expected predicate PredicateOne from %q", schedulerapi.SchedulerDefaultProviderName) } if len(c.Algorithm.Prioritizers()) != 1 || c.Algorithm.Prioritizers()[0].Name != "PriorityOne" { @@ -263,6 +275,15 @@ func TestCreateFromConfigWithUnspecifiedPredicatesOrPriorities(t *testing.T) { } } +func foundPlugin(plugins []schedulerapi.Plugin, name string) bool { + for _, plugin := range plugins { + if plugin.Name == name { + return true + } + } + return false +} + // Test configures a scheduler from a policy that contains empty // predicate/priority. // Empty predicate/priority sets will be used. diff --git a/pkg/scheduler/framework/plugins/interpodaffinity/interpod_affinity.go b/pkg/scheduler/framework/plugins/interpodaffinity/interpod_affinity.go index 3f562f55275..d6b5bfe610d 100644 --- a/pkg/scheduler/framework/plugins/interpodaffinity/interpod_affinity.go +++ b/pkg/scheduler/framework/plugins/interpodaffinity/interpod_affinity.go @@ -48,7 +48,7 @@ func (pl *InterPodAffinity) Name() string { // Filter invoked at the filter extension point. func (pl *InterPodAffinity) Filter(ctx context.Context, cycleState *framework.CycleState, pod *v1.Pod, nodeInfo *nodeinfo.NodeInfo) *framework.Status { - meta, ok := migration.PredicateMetadata(cycleState).(predicates.Metadata) + meta, ok := migration.CovertStateRefToPredMeta(migration.PredicateMetadata(cycleState)) if !ok { return migration.ErrorToFrameworkStatus(fmt.Errorf("%+v convert to predicates.Metadata error", cycleState)) } diff --git a/pkg/scheduler/framework/plugins/migration/utils.go b/pkg/scheduler/framework/plugins/migration/utils.go index a0474bdde43..0896067405e 100644 --- a/pkg/scheduler/framework/plugins/migration/utils.go +++ b/pkg/scheduler/framework/plugins/migration/utils.go @@ -124,3 +124,13 @@ func PredicateMetadata(state *framework.CycleState) interface{} { } return meta } + +// CovertStateRefToPredMeta checks if 'stateRef' is nil, if it is, return nil; +// otherwise covert it to predicates metadata and return. +func CovertStateRefToPredMeta(stateRef interface{}) (predicates.Metadata, bool) { + if stateRef == nil { + return nil, true + } + meta, ok := stateRef.(predicates.Metadata) + return meta, ok +} diff --git a/pkg/scheduler/framework/plugins/noderesources/fit.go b/pkg/scheduler/framework/plugins/noderesources/fit.go index 67327c8c744..87a5724e691 100644 --- a/pkg/scheduler/framework/plugins/noderesources/fit.go +++ b/pkg/scheduler/framework/plugins/noderesources/fit.go @@ -43,7 +43,7 @@ func (f *Fit) Name() string { // Filter invoked at the filter extension point. func (f *Fit) Filter(ctx context.Context, cycleState *framework.CycleState, pod *v1.Pod, nodeInfo *nodeinfo.NodeInfo) *framework.Status { - meta, ok := migration.PredicateMetadata(cycleState).(predicates.Metadata) + meta, ok := migration.CovertStateRefToPredMeta(migration.PredicateMetadata(cycleState)) if !ok { return migration.ErrorToFrameworkStatus(fmt.Errorf("%+v convert to predicates.Metadata error", cycleState)) } diff --git a/pkg/scheduler/framework/plugins/podtopologyspread/pod_topology_spread.go b/pkg/scheduler/framework/plugins/podtopologyspread/pod_topology_spread.go index 2e7f3616173..3207aed0c76 100644 --- a/pkg/scheduler/framework/plugins/podtopologyspread/pod_topology_spread.go +++ b/pkg/scheduler/framework/plugins/podtopologyspread/pod_topology_spread.go @@ -47,7 +47,7 @@ func (pl *PodTopologySpread) Name() string { // Filter invoked at the filter extension point. func (pl *PodTopologySpread) Filter(ctx context.Context, cycleState *framework.CycleState, pod *v1.Pod, nodeInfo *nodeinfo.NodeInfo) *framework.Status { - meta, ok := migration.PredicateMetadata(cycleState).(predicates.Metadata) + meta, ok := migration.CovertStateRefToPredMeta(migration.PredicateMetadata(cycleState)) if !ok { return migration.ErrorToFrameworkStatus(fmt.Errorf("%+v convert to predicates.Metadata error", cycleState)) } diff --git a/pkg/scheduler/framework/plugins/serviceaffinity/service_affinity.go b/pkg/scheduler/framework/plugins/serviceaffinity/service_affinity.go index cbe06beed92..496451c6ca8 100644 --- a/pkg/scheduler/framework/plugins/serviceaffinity/service_affinity.go +++ b/pkg/scheduler/framework/plugins/serviceaffinity/service_affinity.go @@ -85,7 +85,7 @@ func (pl *ServiceAffinity) Name() string { // Filter invoked at the filter extension point. func (pl *ServiceAffinity) Filter(ctx context.Context, cycleState *framework.CycleState, pod *v1.Pod, nodeInfo *nodeinfo.NodeInfo) *framework.Status { - meta, ok := migration.PredicateMetadata(cycleState).(predicates.Metadata) + meta, ok := migration.CovertStateRefToPredMeta(migration.PredicateMetadata(cycleState)) if !ok { return framework.NewStatus(framework.Error, "looking up Metadata") } diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index 72d177986e8..1bb26fe59e8 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -49,12 +49,16 @@ import ( schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/core" frameworkplugins "k8s.io/kubernetes/pkg/scheduler/framework/plugins" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeports" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumebinding" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" fakecache "k8s.io/kubernetes/pkg/scheduler/internal/cache/fake" internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" nodeinfosnapshot "k8s.io/kubernetes/pkg/scheduler/nodeinfo/snapshot" + st "k8s.io/kubernetes/pkg/scheduler/testing" "k8s.io/kubernetes/pkg/scheduler/volumebinder" ) @@ -368,8 +372,9 @@ func TestSchedulerNoPhantomPodAfterExpire(t *testing.T) { scache.AddNode(&node) client := clientsetfake.NewSimpleClientset(&node) informerFactory := informers.NewSharedInformerFactory(client, 0) - predicateMap := map[string]predicates.FitPredicate{"PodFitsHostPorts": predicates.PodFitsHostPorts} - scheduler, bindingChan, _ := setupTestSchedulerWithOnePodOnNode(t, queuedPodStore, scache, informerFactory, stop, predicateMap, pod, &node) + + f := st.RegisterFilterPlugin("PodFitsHostPorts", nodeports.New) + scheduler, bindingChan, _ := setupTestSchedulerWithOnePodOnNode(t, queuedPodStore, scache, informerFactory, stop, f, pod, &node) waitPodExpireChan := make(chan struct{}) timeout := make(chan struct{}) @@ -431,8 +436,8 @@ func TestSchedulerNoPhantomPodAfterDelete(t *testing.T) { scache.AddNode(&node) client := clientsetfake.NewSimpleClientset(&node) informerFactory := informers.NewSharedInformerFactory(client, 0) - predicateMap := map[string]predicates.FitPredicate{"PodFitsHostPorts": predicates.PodFitsHostPorts} - scheduler, bindingChan, errChan := setupTestSchedulerWithOnePodOnNode(t, queuedPodStore, scache, informerFactory, stop, predicateMap, firstPod, &node) + f := st.RegisterFilterPlugin(nodeports.Name, nodeports.New) + scheduler, bindingChan, errChan := setupTestSchedulerWithOnePodOnNode(t, queuedPodStore, scache, informerFactory, stop, f, firstPod, &node) // We use conflicted pod ports to incur fit predicate failure. secondPod := podWithPort("bar", "", 8080) @@ -444,12 +449,15 @@ func TestSchedulerNoPhantomPodAfterDelete(t *testing.T) { select { case err := <-errChan: expectErr := &core.FitError{ - Pod: secondPod, - NumAllNodes: 1, - FailedPredicates: core.FailedPredicateMap{ - node.Name: []predicates.PredicateFailureReason{predicates.ErrPodNotFitsHostPorts}, + Pod: secondPod, + NumAllNodes: 1, + FailedPredicates: core.FailedPredicateMap{}, + FilteredNodesStatuses: framework.NodeToStatusMap{ + node.Name: framework.NewStatus( + framework.Unschedulable, + predicates.ErrPodNotFitsHostPorts.GetReason(), + ), }, - FilteredNodesStatuses: framework.NodeToStatusMap{}, } if !reflect.DeepEqual(expectErr, err) { t.Errorf("err want=%v, get=%v", expectErr, err) @@ -523,10 +531,10 @@ func TestSchedulerErrorWithLongBinding(t *testing.T) { client := clientsetfake.NewSimpleClientset(&node) informerFactory := informers.NewSharedInformerFactory(client, 0) - predicateMap := map[string]predicates.FitPredicate{"PodFitsHostPorts": predicates.PodFitsHostPorts} + f := st.RegisterFilterPlugin(nodeports.Name, nodeports.New) scheduler, bindingChan := setupTestSchedulerLongBindingWithRetry( - queuedPodStore, scache, informerFactory, predicateMap, stop, test.BindingDuration) + queuedPodStore, scache, informerFactory, f, stop, test.BindingDuration) informerFactory.Start(stop) informerFactory.WaitForCacheSync(stop) @@ -558,9 +566,9 @@ func TestSchedulerErrorWithLongBinding(t *testing.T) { // queuedPodStore: pods queued before processing. // cache: scheduler cache that might contain assumed pods. func setupTestSchedulerWithOnePodOnNode(t *testing.T, queuedPodStore *clientcache.FIFO, scache internalcache.Cache, - informerFactory informers.SharedInformerFactory, stop chan struct{}, predicateMap map[string]predicates.FitPredicate, pod *v1.Pod, node *v1.Node) (*Scheduler, chan *v1.Binding, chan error) { + informerFactory informers.SharedInformerFactory, stop chan struct{}, f st.RegisterFilterPluginFunc, pod *v1.Pod, node *v1.Node) (*Scheduler, chan *v1.Binding, chan error) { - scheduler, bindingChan, errChan := setupTestScheduler(queuedPodStore, scache, informerFactory, predicateMap, nil) + scheduler, bindingChan, errChan := setupTestScheduler(queuedPodStore, scache, informerFactory, f, nil) informerFactory.Start(stop) informerFactory.WaitForCacheSync(stop) @@ -630,19 +638,18 @@ func TestSchedulerFailedSchedulingReasons(t *testing.T) { } client := clientsetfake.NewSimpleClientset(objects...) informerFactory := informers.NewSharedInformerFactory(client, 0) - predicateMap := map[string]predicates.FitPredicate{ - "PodFitsResources": predicates.PodFitsResources, - } - // Create expected failure reasons for all the nodes. Hopefully they will get rolled up into a non-spammy summary. - failedPredicatesMap := core.FailedPredicateMap{} + // Create expected failure reasons for all the nodes. Hopefully they will get rolled up into a non-spammy summary. + failedNodeStatues := framework.NodeToStatusMap{} for _, node := range nodes { - failedPredicatesMap[node.Name] = []predicates.PredicateFailureReason{ - predicates.NewInsufficientResourceError(v1.ResourceCPU, 4000, 0, 2000), - predicates.NewInsufficientResourceError(v1.ResourceMemory, 500, 0, 100), - } + failedNodeStatues[node.Name] = framework.NewStatus( + framework.Unschedulable, + predicates.NewInsufficientResourceError(v1.ResourceCPU, 4000, 0, 2000).GetReason(), + predicates.NewInsufficientResourceError(v1.ResourceMemory, 500, 0, 100).GetReason(), + ) } - scheduler, _, errChan := setupTestScheduler(queuedPodStore, scache, informerFactory, predicateMap, nil) + f := st.RegisterFilterPlugin("PodFitsResources", noderesources.NewFit) + scheduler, _, errChan := setupTestScheduler(queuedPodStore, scache, informerFactory, f, nil) informerFactory.Start(stop) informerFactory.WaitForCacheSync(stop) @@ -654,8 +661,8 @@ func TestSchedulerFailedSchedulingReasons(t *testing.T) { expectErr := &core.FitError{ Pod: podWithTooBigResourceRequests, NumAllNodes: len(nodes), - FailedPredicates: failedPredicatesMap, - FilteredNodesStatuses: framework.NodeToStatusMap{}, + FailedPredicates: core.FailedPredicateMap{}, + FilteredNodesStatuses: failedNodeStatues, } if len(fmt.Sprint(expectErr)) > 150 { t.Errorf("message is too spammy ! %v ", len(fmt.Sprint(expectErr))) @@ -670,16 +677,23 @@ func TestSchedulerFailedSchedulingReasons(t *testing.T) { // queuedPodStore: pods queued before processing. // scache: scheduler cache that might contain assumed pods. -func setupTestScheduler(queuedPodStore *clientcache.FIFO, scache internalcache.Cache, informerFactory informers.SharedInformerFactory, predicateMap map[string]predicates.FitPredicate, recorder events.EventRecorder) (*Scheduler, chan *v1.Binding, chan error) { +func setupTestScheduler(queuedPodStore *clientcache.FIFO, scache internalcache.Cache, informerFactory informers.SharedInformerFactory, f st.RegisterFilterPluginFunc, recorder events.EventRecorder) (*Scheduler, chan *v1.Binding, chan error) { + registry := framework.Registry{} + plugins := &schedulerapi.Plugins{ + Filter: &schedulerapi.PluginSet{}, + } + var pluginConfigs []schedulerapi.PluginConfig + f(®istry, plugins, pluginConfigs) + fwk, _ := framework.NewFramework(registry, plugins, pluginConfigs) algo := core.NewGenericScheduler( scache, internalqueue.NewSchedulingQueue(nil), - predicateMap, + nil, predicates.EmptyMetadataProducer, []priorities.PriorityConfig{}, priorities.EmptyMetadataProducer, nodeinfosnapshot.NewEmptySnapshot(), - emptyFramework, + fwk, []algorithm.SchedulerExtender{}, nil, informerFactory.Core().V1().PersistentVolumeClaims().Lister(), @@ -710,7 +724,7 @@ func setupTestScheduler(queuedPodStore *clientcache.FIFO, scache internalcache.C Recorder: &events.FakeRecorder{}, podConditionUpdater: fakePodConditionUpdater{}, podPreemptor: fakePodPreemptor{}, - Framework: emptyFramework, + Framework: fwk, VolumeBinder: volumebinder.NewFakeVolumeBinder(&volumescheduling.FakeVolumeBinderConfig{AllBound: true}), } @@ -721,17 +735,24 @@ func setupTestScheduler(queuedPodStore *clientcache.FIFO, scache internalcache.C return sched, bindingChan, errChan } -func setupTestSchedulerLongBindingWithRetry(queuedPodStore *clientcache.FIFO, scache internalcache.Cache, informerFactory informers.SharedInformerFactory, predicateMap map[string]predicates.FitPredicate, stop chan struct{}, bindingTime time.Duration) (*Scheduler, chan *v1.Binding) { +func setupTestSchedulerLongBindingWithRetry(queuedPodStore *clientcache.FIFO, scache internalcache.Cache, informerFactory informers.SharedInformerFactory, f st.RegisterFilterPluginFunc, stop chan struct{}, bindingTime time.Duration) (*Scheduler, chan *v1.Binding) { + registry := framework.Registry{} + plugins := &schedulerapi.Plugins{ + Filter: &schedulerapi.PluginSet{}, + } + var pluginConfigs []schedulerapi.PluginConfig + f(®istry, plugins, pluginConfigs) + fwk, _ := framework.NewFramework(registry, plugins, pluginConfigs) queue := internalqueue.NewSchedulingQueue(nil) algo := core.NewGenericScheduler( scache, queue, - predicateMap, + nil, predicates.EmptyMetadataProducer, []priorities.PriorityConfig{}, priorities.EmptyMetadataProducer, nodeinfosnapshot.NewEmptySnapshot(), - emptyFramework, + fwk, []algorithm.SchedulerExtender{}, nil, informerFactory.Core().V1().PersistentVolumeClaims().Lister(), @@ -766,7 +787,7 @@ func setupTestSchedulerLongBindingWithRetry(queuedPodStore *clientcache.FIFO, sc podConditionUpdater: fakePodConditionUpdater{}, podPreemptor: fakePodPreemptor{}, StopEverything: stop, - Framework: emptyFramework, + Framework: fwk, VolumeBinder: volumebinder.NewFakeVolumeBinder(&volumescheduling.FakeVolumeBinderConfig{AllBound: true}), SchedulingQueue: queue, } @@ -788,12 +809,12 @@ func setupTestSchedulerWithVolumeBinding(fakeVolumeBinder *volumebinder.VolumeBi client := clientsetfake.NewSimpleClientset(&testNode, &testPVC) informerFactory := informers.NewSharedInformerFactory(client, 0) - predicateMap := map[string]predicates.FitPredicate{ - predicates.CheckVolumeBindingPred: predicates.NewVolumeBindingPredicate(fakeVolumeBinder), - } - recorder := broadcaster.NewRecorder(scheme.Scheme, "scheduler") - s, bindingChan, errChan := setupTestScheduler(queuedPodStore, scache, informerFactory, predicateMap, recorder) + volumeBindingNewFunc := func(_ *runtime.Unknown, _ framework.FrameworkHandle) (framework.Plugin, error) { + return volumebinding.NewFromVolumeBinder(fakeVolumeBinder), nil + } + f := st.RegisterFilterPlugin(volumebinding.Name, volumeBindingNewFunc) + s, bindingChan, errChan := setupTestScheduler(queuedPodStore, scache, informerFactory, f, recorder) informerFactory.Start(stop) informerFactory.WaitForCacheSync(stop) s.VolumeBinder = fakeVolumeBinder @@ -884,7 +905,7 @@ func TestSchedulerWithVolumeBinding(t *testing.T) { FindErr: findErr, }, eventReason: "FailedScheduling", - expectError: findErr, + expectError: fmt.Errorf("error while running %q filter plugin for pod %q: %v", volumebinding.Name, "foo", findErr), }, { name: "assume error", diff --git a/pkg/scheduler/testing/BUILD b/pkg/scheduler/testing/BUILD index 6688e88311c..4684d7629af 100644 --- a/pkg/scheduler/testing/BUILD +++ b/pkg/scheduler/testing/BUILD @@ -5,11 +5,14 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", srcs = [ + "framework_helpers.go", "workload_prep.go", "wrappers.go", ], importpath = "k8s.io/kubernetes/pkg/scheduler/testing", deps = [ + "//pkg/scheduler/apis/config:go_default_library", + "//pkg/scheduler/framework/v1alpha1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", ], diff --git a/pkg/scheduler/testing/framework_helpers.go b/pkg/scheduler/testing/framework_helpers.go new file mode 100644 index 00000000000..a9dbd00b455 --- /dev/null +++ b/pkg/scheduler/testing/framework_helpers.go @@ -0,0 +1,37 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" + framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" +) + +// RegisterFilterPluginFunc is a function signature used in method RegisterFilterPlugin() +// to register a Filter Plugin to a given registry. +type RegisterFilterPluginFunc func(reg *framework.Registry, plugins *schedulerapi.Plugins, pluginConfigs []schedulerapi.PluginConfig) + +// RegisterFilterPlugin returns a function to register a Filter Plugin to a given registry. +func RegisterFilterPlugin(pluginName string, pluginNewFunc framework.PluginFactory) RegisterFilterPluginFunc { + return func(reg *framework.Registry, plugins *schedulerapi.Plugins, pluginConfigs []schedulerapi.PluginConfig) { + reg.Register(pluginName, pluginNewFunc) + plugins.Filter.Enabled = append(plugins.Filter.Enabled, schedulerapi.Plugin{Name: pluginName}) + //lint:ignore SA4006 this value of pluginConfigs is never used. + //lint:ignore SA4010 this result of append is never used. + pluginConfigs = append(pluginConfigs, schedulerapi.PluginConfig{Name: pluginName}) + } +} diff --git a/test/integration/scheduler/BUILD b/test/integration/scheduler/BUILD index 7398f7fc731..74ed3df6e24 100644 --- a/test/integration/scheduler/BUILD +++ b/test/integration/scheduler/BUILD @@ -32,6 +32,7 @@ go_test( "//pkg/scheduler/algorithmprovider:go_default_library", "//pkg/scheduler/apis/config:go_default_library", "//pkg/scheduler/apis/extender/v1:go_default_library", + "//pkg/scheduler/framework/plugins/noderesources:go_default_library", "//pkg/scheduler/framework/v1alpha1:go_default_library", "//pkg/scheduler/nodeinfo:go_default_library", "//pkg/scheduler/testing:go_default_library", diff --git a/test/integration/scheduler/framework_test.go b/test/integration/scheduler/framework_test.go index 58c68b1e19e..7559444d1b8 100644 --- a/test/integration/scheduler/framework_test.go +++ b/test/integration/scheduler/framework_test.go @@ -30,6 +30,7 @@ import ( clientset "k8s.io/client-go/kubernetes" scheduler "k8s.io/kubernetes/pkg/scheduler" schedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" ) @@ -1450,6 +1451,11 @@ func TestPreemptWithPermitPlugin(t *testing.T) { // Create a plugin registry for testing. Register only a permit plugin. permitPlugin := &PermitPlugin{} registry, plugins := initRegistryAndConfig(permitPlugin) + // Fit filter plugin must be registered. + registry.Register(noderesources.FitName, noderesources.NewFit) + plugins.Filter = &schedulerconfig.PluginSet{ + Enabled: []schedulerconfig.Plugin{{Name: noderesources.FitName}}, + } // Create the master and the scheduler with the test plugin set. context := initTestSchedulerForFrameworkTest(t, initTestMaster(t, "preempt-with-permit-plugin", nil), 0,