diff --git a/pkg/scheduler/core/BUILD b/pkg/scheduler/core/BUILD index b28de9d99b3..3f81b698620 100644 --- a/pkg/scheduler/core/BUILD +++ b/pkg/scheduler/core/BUILD @@ -40,7 +40,6 @@ go_test( ], embed = [":go_default_library"], deps = [ - "//pkg/api/v1/pod:go_default_library", "//pkg/controller/volume/scheduling:go_default_library", "//pkg/scheduler/apis/config:go_default_library", "//pkg/scheduler/framework/plugins/defaultbinder:go_default_library", diff --git a/pkg/scheduler/core/extender_test.go b/pkg/scheduler/core/extender_test.go index 3df352f79f5..b726b1f1583 100644 --- a/pkg/scheduler/core/extender_test.go +++ b/pkg/scheduler/core/extender_test.go @@ -18,22 +18,17 @@ package core import ( "context" - "fmt" "reflect" - "sort" "testing" "time" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/informers" clientsetfake "k8s.io/client-go/kubernetes/fake" - extenderv1 "k8s.io/kube-scheduler/extender/v1" - podutil "k8s.io/kubernetes/pkg/api/v1/pod" schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/queuesort" @@ -42,340 +37,29 @@ import ( internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue" "k8s.io/kubernetes/pkg/scheduler/profile" st "k8s.io/kubernetes/pkg/scheduler/testing" - "k8s.io/kubernetes/pkg/scheduler/util" ) -type fitPredicate func(pod *v1.Pod, node *v1.Node) (bool, error) -type priorityFunc func(pod *v1.Pod, nodes []*v1.Node) (*framework.NodeScoreList, error) - -type priorityConfig struct { - function priorityFunc - weight int64 -} - -func errorPredicateExtender(pod *v1.Pod, node *v1.Node) (bool, error) { - return false, fmt.Errorf("Some error") -} - -func falsePredicateExtender(pod *v1.Pod, node *v1.Node) (bool, error) { - return false, nil -} - -func truePredicateExtender(pod *v1.Pod, node *v1.Node) (bool, error) { - return true, nil -} - -func machine1PredicateExtender(pod *v1.Pod, node *v1.Node) (bool, error) { - if node.Name == "machine1" { - return true, nil - } - return false, nil -} - -func machine2PredicateExtender(pod *v1.Pod, node *v1.Node) (bool, error) { - if node.Name == "machine2" { - return true, nil - } - return false, nil -} - -func errorPrioritizerExtender(pod *v1.Pod, nodes []*v1.Node) (*framework.NodeScoreList, error) { - return &framework.NodeScoreList{}, fmt.Errorf("Some error") -} - -func machine1PrioritizerExtender(pod *v1.Pod, nodes []*v1.Node) (*framework.NodeScoreList, error) { - result := framework.NodeScoreList{} - for _, node := range nodes { - score := 1 - if node.Name == "machine1" { - score = 10 - } - result = append(result, framework.NodeScore{Name: node.Name, Score: int64(score)}) - } - return &result, nil -} - -func machine2PrioritizerExtender(pod *v1.Pod, nodes []*v1.Node) (*framework.NodeScoreList, error) { - result := framework.NodeScoreList{} - for _, node := range nodes { - score := 1 - if node.Name == "machine2" { - score = 10 - } - result = append(result, framework.NodeScore{Name: node.Name, Score: int64(score)}) - } - return &result, nil -} - -type machine2PrioritizerPlugin struct{} - -func newMachine2PrioritizerPlugin() framework.PluginFactory { - return func(_ runtime.Object, _ framework.FrameworkHandle) (framework.Plugin, error) { - return &machine2PrioritizerPlugin{}, nil - } -} - -func (pl *machine2PrioritizerPlugin) Name() string { - return "Machine2Prioritizer" -} - -func (pl *machine2PrioritizerPlugin) Score(_ context.Context, _ *framework.CycleState, _ *v1.Pod, nodeName string) (int64, *framework.Status) { - score := 10 - if nodeName == "machine2" { - score = 100 - } - return int64(score), nil -} - -func (pl *machine2PrioritizerPlugin) ScoreExtensions() framework.ScoreExtensions { - return nil -} - -type FakeExtender struct { - predicates []fitPredicate - prioritizers []priorityConfig - weight int64 - nodeCacheCapable bool - filteredNodes []*v1.Node - unInterested bool - ignorable bool - - // Cached node information for fake extender - cachedNodeNameToInfo map[string]*framework.NodeInfo -} - -func (f *FakeExtender) Name() string { - return "FakeExtender" -} - -func (f *FakeExtender) IsIgnorable() bool { - return f.ignorable -} - -func (f *FakeExtender) SupportsPreemption() bool { - // Assume preempt verb is always defined. - return true -} - -func (f *FakeExtender) ProcessPreemption( - pod *v1.Pod, - nodeNameToVictims map[string]*extenderv1.Victims, - nodeInfos framework.NodeInfoLister, -) (map[string]*extenderv1.Victims, error) { - nodeNameToVictimsCopy := map[string]*extenderv1.Victims{} - // We don't want to change the original nodeNameToVictims - for k, v := range nodeNameToVictims { - // In real world implementation, extender's user should have their own way to get node object - // by name if needed (e.g. query kube-apiserver etc). - // - // For test purpose, we just use node from parameters directly. - nodeNameToVictimsCopy[k] = v - } - - for nodeName, victims := range nodeNameToVictimsCopy { - // Try to do preemption on extender side. - nodeInfo, _ := nodeInfos.Get(nodeName) - extenderVictimPods, extenderPDBViolations, fits, err := f.selectVictimsOnNodeByExtender(pod, nodeInfo.Node()) - if err != nil { - return nil, err - } - // If it's unfit after extender's preemption, this node is unresolvable by preemption overall, - // let's remove it from potential preemption nodes. - if !fits { - delete(nodeNameToVictimsCopy, nodeName) - } else { - // Append new victims to original victims - nodeNameToVictimsCopy[nodeName].Pods = append(victims.Pods, extenderVictimPods...) - nodeNameToVictimsCopy[nodeName].NumPDBViolations = victims.NumPDBViolations + int64(extenderPDBViolations) - } - } - return nodeNameToVictimsCopy, nil -} - -// selectVictimsOnNodeByExtender checks the given nodes->pods map with predicates on extender's side. -// Returns: -// 1. More victim pods (if any) amended by preemption phase of extender. -// 2. Number of violating victim (used to calculate PDB). -// 3. Fits or not after preemption phase on extender's side. -func (f *FakeExtender) selectVictimsOnNodeByExtender(pod *v1.Pod, node *v1.Node) ([]*v1.Pod, int, bool, error) { - // If a extender support preemption but have no cached node info, let's run filter to make sure - // default scheduler's decision still stand with given pod and node. - if !f.nodeCacheCapable { - fits, err := f.runPredicate(pod, node) - if err != nil { - return nil, 0, false, err - } - if !fits { - return nil, 0, false, nil - } - return []*v1.Pod{}, 0, true, nil - } - - // Otherwise, as a extender support preemption and have cached node info, we will assume cachedNodeNameToInfo is available - // and get cached node info by given node name. - nodeInfoCopy := f.cachedNodeNameToInfo[node.GetName()].Clone() - - var potentialVictims []*v1.Pod - - removePod := func(rp *v1.Pod) { - nodeInfoCopy.RemovePod(rp) - } - addPod := func(ap *v1.Pod) { - nodeInfoCopy.AddPod(ap) - } - // As the first step, remove all the lower priority pods from the node and - // check if the given pod can be scheduled. - podPriority := podutil.GetPodPriority(pod) - for _, p := range nodeInfoCopy.Pods { - if podutil.GetPodPriority(p.Pod) < podPriority { - potentialVictims = append(potentialVictims, p.Pod) - removePod(p.Pod) - } - } - sort.Slice(potentialVictims, func(i, j int) bool { return util.MoreImportantPod(potentialVictims[i], potentialVictims[j]) }) - - // If the new pod does not fit after removing all the lower priority pods, - // we are almost done and this node is not suitable for preemption. - fits, err := f.runPredicate(pod, nodeInfoCopy.Node()) - if err != nil { - return nil, 0, false, err - } - if !fits { - return nil, 0, false, nil - } - - var victims []*v1.Pod - - // TODO(harry): handle PDBs in the future. - numViolatingVictim := 0 - - reprievePod := func(p *v1.Pod) bool { - addPod(p) - fits, _ := f.runPredicate(pod, nodeInfoCopy.Node()) - if !fits { - removePod(p) - victims = append(victims, p) - } - return fits - } - - // For now, assume all potential victims to be non-violating. - // Now we try to reprieve non-violating victims. - for _, p := range potentialVictims { - reprievePod(p) - } - - return victims, numViolatingVictim, true, nil -} - -// runPredicate run predicates of extender one by one for given pod and node. -// Returns: fits or not. -func (f *FakeExtender) runPredicate(pod *v1.Pod, node *v1.Node) (bool, error) { - fits := true - var err error - for _, predicate := range f.predicates { - fits, err = predicate(pod, node) - if err != nil { - return false, err - } - if !fits { - break - } - } - return fits, nil -} - -func (f *FakeExtender) Filter(pod *v1.Pod, nodes []*v1.Node) ([]*v1.Node, extenderv1.FailedNodesMap, error) { - filtered := []*v1.Node{} - failedNodesMap := extenderv1.FailedNodesMap{} - for _, node := range nodes { - fits, err := f.runPredicate(pod, node) - if err != nil { - return []*v1.Node{}, extenderv1.FailedNodesMap{}, err - } - if fits { - filtered = append(filtered, node) - } else { - failedNodesMap[node.Name] = "FakeExtender failed" - } - } - - f.filteredNodes = filtered - if f.nodeCacheCapable { - return filtered, failedNodesMap, nil - } - return filtered, failedNodesMap, nil -} - -func (f *FakeExtender) Prioritize(pod *v1.Pod, nodes []*v1.Node) (*extenderv1.HostPriorityList, int64, error) { - result := extenderv1.HostPriorityList{} - combinedScores := map[string]int64{} - for _, prioritizer := range f.prioritizers { - weight := prioritizer.weight - if weight == 0 { - continue - } - priorityFunc := prioritizer.function - prioritizedList, err := priorityFunc(pod, nodes) - if err != nil { - return &extenderv1.HostPriorityList{}, 0, err - } - for _, hostEntry := range *prioritizedList { - combinedScores[hostEntry.Name] += hostEntry.Score * weight - } - } - for host, score := range combinedScores { - result = append(result, extenderv1.HostPriority{Host: host, Score: score}) - } - return &result, f.weight, nil -} - -func (f *FakeExtender) Bind(binding *v1.Binding) error { - if len(f.filteredNodes) != 0 { - for _, node := range f.filteredNodes { - if node.Name == binding.Target.Name { - f.filteredNodes = nil - return nil - } - } - err := fmt.Errorf("Node %v not in filtered nodes %v", binding.Target.Name, f.filteredNodes) - f.filteredNodes = nil - return err - } - return nil -} - -func (f *FakeExtender) IsBinder() bool { - return true -} - -func (f *FakeExtender) IsInterested(pod *v1.Pod) bool { - return !f.unInterested -} - -var _ framework.Extender = &FakeExtender{} - func TestGenericSchedulerWithExtenders(t *testing.T) { tests := []struct { name string registerPlugins []st.RegisterPluginFunc - extenders []FakeExtender + extenders []st.FakeExtender nodes []string expectedResult ScheduleResult expectsErr bool }{ { registerPlugins: []st.RegisterPluginFunc{ - st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + st.RegisterFilterPlugin("TrueFilter", st.NewTrueFilterPlugin), st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, - extenders: []FakeExtender{ + extenders: []st.FakeExtender{ { - predicates: []fitPredicate{truePredicateExtender}, + Predicates: []st.FitPredicate{st.TruePredicateExtender}, }, { - predicates: []fitPredicate{errorPredicateExtender}, + Predicates: []st.FitPredicate{st.ErrorPredicateExtender}, }, }, nodes: []string{"machine1", "machine2"}, @@ -384,16 +68,16 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { }, { registerPlugins: []st.RegisterPluginFunc{ - st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + st.RegisterFilterPlugin("TrueFilter", st.NewTrueFilterPlugin), st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, - extenders: []FakeExtender{ + extenders: []st.FakeExtender{ { - predicates: []fitPredicate{truePredicateExtender}, + Predicates: []st.FitPredicate{st.TruePredicateExtender}, }, { - predicates: []fitPredicate{falsePredicateExtender}, + Predicates: []st.FitPredicate{st.FalsePredicateExtender}, }, }, nodes: []string{"machine1", "machine2"}, @@ -402,16 +86,16 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { }, { registerPlugins: []st.RegisterPluginFunc{ - st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + st.RegisterFilterPlugin("TrueFilter", st.NewTrueFilterPlugin), st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, - extenders: []FakeExtender{ + extenders: []st.FakeExtender{ { - predicates: []fitPredicate{truePredicateExtender}, + Predicates: []st.FitPredicate{st.TruePredicateExtender}, }, { - predicates: []fitPredicate{machine1PredicateExtender}, + Predicates: []st.FitPredicate{st.Machine1PredicateExtender}, }, }, nodes: []string{"machine1", "machine2"}, @@ -424,16 +108,16 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { }, { registerPlugins: []st.RegisterPluginFunc{ - st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + st.RegisterFilterPlugin("TrueFilter", st.NewTrueFilterPlugin), st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, - extenders: []FakeExtender{ + extenders: []st.FakeExtender{ { - predicates: []fitPredicate{machine2PredicateExtender}, + Predicates: []st.FitPredicate{st.Machine2PredicateExtender}, }, { - predicates: []fitPredicate{machine1PredicateExtender}, + Predicates: []st.FitPredicate{st.Machine1PredicateExtender}, }, }, nodes: []string{"machine1", "machine2"}, @@ -442,15 +126,15 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { }, { registerPlugins: []st.RegisterPluginFunc{ - st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + st.RegisterFilterPlugin("TrueFilter", st.NewTrueFilterPlugin), st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, - extenders: []FakeExtender{ + extenders: []st.FakeExtender{ { - predicates: []fitPredicate{truePredicateExtender}, - prioritizers: []priorityConfig{{errorPrioritizerExtender, 10}}, - weight: 1, + Predicates: []st.FitPredicate{st.TruePredicateExtender}, + Prioritizers: []st.PriorityConfig{{Function: st.ErrorPrioritizerExtender, Weight: 10}}, + Weight: 1, }, }, nodes: []string{"machine1"}, @@ -463,20 +147,20 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { }, { registerPlugins: []st.RegisterPluginFunc{ - st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + st.RegisterFilterPlugin("TrueFilter", st.NewTrueFilterPlugin), st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, - extenders: []FakeExtender{ + extenders: []st.FakeExtender{ { - predicates: []fitPredicate{truePredicateExtender}, - prioritizers: []priorityConfig{{machine1PrioritizerExtender, 10}}, - weight: 1, + Predicates: []st.FitPredicate{st.TruePredicateExtender}, + Prioritizers: []st.PriorityConfig{{Function: st.Machine1PrioritizerExtender, Weight: 10}}, + Weight: 1, }, { - predicates: []fitPredicate{truePredicateExtender}, - prioritizers: []priorityConfig{{machine2PrioritizerExtender, 10}}, - weight: 5, + Predicates: []st.FitPredicate{st.TruePredicateExtender}, + Prioritizers: []st.PriorityConfig{{Function: st.Machine2PrioritizerExtender, Weight: 10}}, + Weight: 5, }, }, nodes: []string{"machine1", "machine2"}, @@ -489,16 +173,16 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { }, { registerPlugins: []st.RegisterPluginFunc{ - st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), - st.RegisterScorePlugin("Machine2Prioritizer", newMachine2PrioritizerPlugin(), 20), + st.RegisterFilterPlugin("TrueFilter", st.NewTrueFilterPlugin), + st.RegisterScorePlugin("Machine2Prioritizer", st.NewMachine2PrioritizerPlugin(), 20), st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, - extenders: []FakeExtender{ + extenders: []st.FakeExtender{ { - predicates: []fitPredicate{truePredicateExtender}, - prioritizers: []priorityConfig{{machine1PrioritizerExtender, 10}}, - weight: 1, + Predicates: []st.FitPredicate{st.TruePredicateExtender}, + Prioritizers: []st.PriorityConfig{{Function: st.Machine1PrioritizerExtender, Weight: 10}}, + Weight: 1, }, }, nodes: []string{"machine1", "machine2"}, @@ -518,16 +202,16 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { // because of the errors from errorPredicateExtender and/or // errorPrioritizerExtender. registerPlugins: []st.RegisterPluginFunc{ - st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), - st.RegisterScorePlugin("Machine2Prioritizer", newMachine2PrioritizerPlugin(), 1), + st.RegisterFilterPlugin("TrueFilter", st.NewTrueFilterPlugin), + st.RegisterScorePlugin("Machine2Prioritizer", st.NewMachine2PrioritizerPlugin(), 1), st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, - extenders: []FakeExtender{ + extenders: []st.FakeExtender{ { - predicates: []fitPredicate{errorPredicateExtender}, - prioritizers: []priorityConfig{{errorPrioritizerExtender, 10}}, - unInterested: true, + Predicates: []st.FitPredicate{st.ErrorPredicateExtender}, + Prioritizers: []st.PriorityConfig{{Function: st.ErrorPrioritizerExtender, Weight: 10}}, + UnInterested: true, }, }, nodes: []string{"machine1", "machine2"}, @@ -546,17 +230,17 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { // If scheduler did not ignore the extender, the test would fail // because of the errors from errorPredicateExtender. registerPlugins: []st.RegisterPluginFunc{ - st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + st.RegisterFilterPlugin("TrueFilter", st.NewTrueFilterPlugin), st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, - extenders: []FakeExtender{ + extenders: []st.FakeExtender{ { - predicates: []fitPredicate{errorPredicateExtender}, - ignorable: true, + Predicates: []st.FitPredicate{st.ErrorPredicateExtender}, + Ignorable: true, }, { - predicates: []fitPredicate{machine1PredicateExtender}, + Predicates: []st.FitPredicate{st.Machine1PredicateExtender}, }, }, nodes: []string{"machine1", "machine2"}, diff --git a/pkg/scheduler/core/generic_scheduler_test.go b/pkg/scheduler/core/generic_scheduler_test.go index ecd9950a7cb..cf7e6d88156 100644 --- a/pkg/scheduler/core/generic_scheduler_test.go +++ b/pkg/scheduler/core/generic_scheduler_test.go @@ -23,7 +23,6 @@ import ( "reflect" "strconv" "strings" - "sync/atomic" "testing" "time" @@ -66,66 +65,6 @@ var ( errPrioritize = fmt.Errorf("priority map encounters an error") ) -const ErrReasonFake = "Nodes failed the fake predicate" - -type trueFilterPlugin struct{} - -// Name returns name of the plugin. -func (pl *trueFilterPlugin) Name() string { - return "TrueFilter" -} - -// Filter invoked at the filter extension point. -func (pl *trueFilterPlugin) Filter(_ context.Context, _ *framework.CycleState, pod *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status { - return nil -} - -// NewTrueFilterPlugin initializes a trueFilterPlugin and returns it. -func NewTrueFilterPlugin(_ runtime.Object, _ framework.FrameworkHandle) (framework.Plugin, error) { - return &trueFilterPlugin{}, nil -} - -type falseFilterPlugin struct{} - -// Name returns name of the plugin. -func (pl *falseFilterPlugin) Name() string { - return "FalseFilter" -} - -// Filter invoked at the filter extension point. -func (pl *falseFilterPlugin) Filter(_ context.Context, _ *framework.CycleState, pod *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status { - return framework.NewStatus(framework.Unschedulable, ErrReasonFake) -} - -// NewFalseFilterPlugin initializes a falseFilterPlugin and returns it. -func NewFalseFilterPlugin(_ runtime.Object, _ framework.FrameworkHandle) (framework.Plugin, error) { - return &falseFilterPlugin{}, nil -} - -type matchFilterPlugin struct{} - -// Name returns name of the plugin. -func (pl *matchFilterPlugin) Name() string { - return "MatchFilter" -} - -// Filter invoked at the filter extension point. -func (pl *matchFilterPlugin) Filter(_ context.Context, _ *framework.CycleState, pod *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status { - node := nodeInfo.Node() - if node == nil { - return framework.NewStatus(framework.Error, "node not found") - } - if pod.Name == node.Name { - return nil - } - return framework.NewStatus(framework.Unschedulable, ErrReasonFake) -} - -// NewMatchFilterPlugin initializes a matchFilterPlugin and returns it. -func NewMatchFilterPlugin(_ runtime.Object, _ framework.FrameworkHandle) (framework.Plugin, error) { - return &matchFilterPlugin{}, nil -} - type noPodsFilterPlugin struct{} // Name returns name of the plugin. @@ -138,7 +77,7 @@ func (pl *noPodsFilterPlugin) Filter(_ context.Context, _ *framework.CycleState, if len(nodeInfo.Pods) == 0 { return nil } - return framework.NewStatus(framework.Unschedulable, ErrReasonFake) + return framework.NewStatus(framework.Unschedulable, st.ErrReasonFake) } // NewNoPodsFilterPlugin initializes a noPodsFilterPlugin and returns it. @@ -146,38 +85,6 @@ func NewNoPodsFilterPlugin(_ runtime.Object, _ framework.FrameworkHandle) (frame return &noPodsFilterPlugin{}, nil } -// fakeFilterPlugin is a test filter plugin to record how many times its Filter() function have -// been called, and it returns different 'Code' depending on its internal 'failedNodeReturnCodeMap'. -type fakeFilterPlugin struct { - numFilterCalled int32 - failedNodeReturnCodeMap map[string]framework.Code -} - -// Name returns name of the plugin. -func (pl *fakeFilterPlugin) Name() string { - return "FakeFilter" -} - -// Filter invoked at the filter extension point. -func (pl *fakeFilterPlugin) Filter(_ context.Context, _ *framework.CycleState, pod *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status { - atomic.AddInt32(&pl.numFilterCalled, 1) - - if returnCode, ok := pl.failedNodeReturnCodeMap[nodeInfo.Node().Name]; ok { - return framework.NewStatus(returnCode, fmt.Sprintf("injecting failure for pod %v", pod.Name)) - } - - return nil -} - -// NewFakeFilterPlugin initializes a fakeFilterPlugin and returns it. -func NewFakeFilterPlugin(failedNodeReturnCodeMap map[string]framework.Code) framework.PluginFactory { - return func(_ runtime.Object, _ framework.FrameworkHandle) (framework.Plugin, error) { - return &fakeFilterPlugin{ - failedNodeReturnCodeMap: failedNodeReturnCodeMap, - }, nil - } -} - type numericMapPlugin struct{} func newNumericMapPlugin() framework.PluginFactory { @@ -386,7 +293,7 @@ func TestGenericScheduler(t *testing.T) { { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), - st.RegisterFilterPlugin("FalseFilter", NewFalseFilterPlugin), + st.RegisterFilterPlugin("FalseFilter", st.NewFalseFilterPlugin), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, @@ -396,15 +303,15 @@ func TestGenericScheduler(t *testing.T) { Pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "2", UID: types.UID("2")}}, NumAllNodes: 2, FilteredNodesStatuses: framework.NodeToStatusMap{ - "machine1": framework.NewStatus(framework.Unschedulable, ErrReasonFake), - "machine2": framework.NewStatus(framework.Unschedulable, ErrReasonFake), + "machine1": framework.NewStatus(framework.Unschedulable, st.ErrReasonFake), + "machine2": framework.NewStatus(framework.Unschedulable, st.ErrReasonFake), }, }, }, { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), - st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + st.RegisterFilterPlugin("TrueFilter", st.NewTrueFilterPlugin), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, @@ -417,7 +324,7 @@ func TestGenericScheduler(t *testing.T) { // Fits on a machine where the pod ID matches the machine name registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), - st.RegisterFilterPlugin("MatchFilter", NewMatchFilterPlugin), + st.RegisterFilterPlugin("MatchFilter", st.NewMatchFilterPlugin), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, @@ -429,7 +336,7 @@ func TestGenericScheduler(t *testing.T) { { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), - st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + st.RegisterFilterPlugin("TrueFilter", st.NewTrueFilterPlugin), st.RegisterScorePlugin("NumericMap", newNumericMapPlugin(), 1), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, @@ -442,7 +349,7 @@ func TestGenericScheduler(t *testing.T) { { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), - st.RegisterFilterPlugin("MatchFilter", NewMatchFilterPlugin), + st.RegisterFilterPlugin("MatchFilter", st.NewMatchFilterPlugin), st.RegisterScorePlugin("NumericMap", newNumericMapPlugin(), 1), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, @@ -455,7 +362,7 @@ func TestGenericScheduler(t *testing.T) { { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), - st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + st.RegisterFilterPlugin("TrueFilter", st.NewTrueFilterPlugin), st.RegisterScorePlugin("NumericMap", newNumericMapPlugin(), 1), st.RegisterScorePlugin("ReverseNumericMap", newReverseNumericMapPlugin(), 2), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), @@ -469,8 +376,8 @@ func TestGenericScheduler(t *testing.T) { { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), - st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), - st.RegisterFilterPlugin("FalseFilter", NewFalseFilterPlugin), + st.RegisterFilterPlugin("TrueFilter", st.NewTrueFilterPlugin), + st.RegisterFilterPlugin("FalseFilter", st.NewFalseFilterPlugin), st.RegisterScorePlugin("NumericMap", newNumericMapPlugin(), 1), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, @@ -481,9 +388,9 @@ func TestGenericScheduler(t *testing.T) { Pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "2", UID: types.UID("2")}}, NumAllNodes: 3, FilteredNodesStatuses: framework.NodeToStatusMap{ - "3": framework.NewStatus(framework.Unschedulable, ErrReasonFake), - "2": framework.NewStatus(framework.Unschedulable, ErrReasonFake), - "1": framework.NewStatus(framework.Unschedulable, ErrReasonFake), + "3": framework.NewStatus(framework.Unschedulable, st.ErrReasonFake), + "2": framework.NewStatus(framework.Unschedulable, st.ErrReasonFake), + "1": framework.NewStatus(framework.Unschedulable, st.ErrReasonFake), }, }, }, @@ -491,7 +398,7 @@ func TestGenericScheduler(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin("NoPodsFilter", NewNoPodsFilterPlugin), - st.RegisterFilterPlugin("MatchFilter", NewMatchFilterPlugin), + st.RegisterFilterPlugin("MatchFilter", st.NewMatchFilterPlugin), st.RegisterScorePlugin("NumericMap", newNumericMapPlugin(), 1), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, @@ -513,8 +420,8 @@ func TestGenericScheduler(t *testing.T) { Pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "2", UID: types.UID("2")}}, NumAllNodes: 2, FilteredNodesStatuses: framework.NodeToStatusMap{ - "1": framework.NewStatus(framework.Unschedulable, ErrReasonFake), - "2": framework.NewStatus(framework.Unschedulable, ErrReasonFake), + "1": framework.NewStatus(framework.Unschedulable, st.ErrReasonFake), + "2": framework.NewStatus(framework.Unschedulable, st.ErrReasonFake), }, }, }, @@ -522,7 +429,7 @@ func TestGenericScheduler(t *testing.T) { // Pod with existing PVC registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), - st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + st.RegisterFilterPlugin("TrueFilter", st.NewTrueFilterPlugin), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, @@ -549,7 +456,7 @@ func TestGenericScheduler(t *testing.T) { // Pod with non existing PVC registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), - st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + st.RegisterFilterPlugin("TrueFilter", st.NewTrueFilterPlugin), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, @@ -574,7 +481,7 @@ func TestGenericScheduler(t *testing.T) { // Pod with deleting PVC registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), - st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + st.RegisterFilterPlugin("TrueFilter", st.NewTrueFilterPlugin), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, @@ -599,7 +506,7 @@ func TestGenericScheduler(t *testing.T) { { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), - st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + st.RegisterFilterPlugin("TrueFilter", st.NewTrueFilterPlugin), st.RegisterScorePlugin("FalseMap", newFalseMapPlugin(), 1), st.RegisterScorePlugin("TrueMap", newTrueMapPlugin(), 2), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), @@ -727,7 +634,7 @@ func TestGenericScheduler(t *testing.T) { st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin( "FakeFilter", - NewFakeFilterPlugin(map[string]framework.Code{"3": framework.Unschedulable}), + st.NewFakeFilterPlugin(map[string]framework.Code{"3": framework.Unschedulable}), ), st.RegisterScorePlugin("NumericMap", newNumericMapPlugin(), 1), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), @@ -749,7 +656,7 @@ func TestGenericScheduler(t *testing.T) { st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin( "FakeFilter", - NewFakeFilterPlugin(map[string]framework.Code{"3": framework.UnschedulableAndUnresolvable}), + st.NewFakeFilterPlugin(map[string]framework.Code{"3": framework.UnschedulableAndUnresolvable}), ), st.RegisterScorePlugin("NumericMap", newNumericMapPlugin(), 1), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), @@ -771,7 +678,7 @@ func TestGenericScheduler(t *testing.T) { st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin( "FakeFilter", - NewFakeFilterPlugin(map[string]framework.Code{"1": framework.Unschedulable}), + st.NewFakeFilterPlugin(map[string]framework.Code{"1": framework.Unschedulable}), ), st.RegisterScorePlugin("NumericMap", newNumericMapPlugin(), 1), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), @@ -864,8 +771,8 @@ func TestFindFitAllError(t *testing.T) { scheduler := makeScheduler(nodes) prof, err := makeProfile( st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), - st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), - st.RegisterFilterPlugin("MatchFilter", NewMatchFilterPlugin), + st.RegisterFilterPlugin("TrueFilter", st.NewTrueFilterPlugin), + st.RegisterFilterPlugin("MatchFilter", st.NewMatchFilterPlugin), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), ) if err != nil { @@ -889,7 +796,7 @@ func TestFindFitAllError(t *testing.T) { t.Errorf("failed to find node %v in %v", node.Name, nodeToStatusMap) } reasons := status.Reasons() - if len(reasons) != 1 || reasons[0] != ErrReasonFake { + if len(reasons) != 1 || reasons[0] != st.ErrReasonFake { t.Errorf("unexpected failure reasons: %v", reasons) } }) @@ -901,8 +808,8 @@ func TestFindFitSomeError(t *testing.T) { scheduler := makeScheduler(nodes) prof, err := makeProfile( st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), - st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), - st.RegisterFilterPlugin("MatchFilter", NewMatchFilterPlugin), + st.RegisterFilterPlugin("TrueFilter", st.NewTrueFilterPlugin), + st.RegisterFilterPlugin("MatchFilter", st.NewMatchFilterPlugin), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), ) if err != nil { @@ -930,7 +837,7 @@ func TestFindFitSomeError(t *testing.T) { t.Errorf("failed to find node %v in %v", node.Name, nodeToStatusMap) } reasons := status.Reasons() - if len(reasons) != 1 || reasons[0] != ErrReasonFake { + if len(reasons) != 1 || reasons[0] != st.ErrReasonFake { t.Errorf("unexpected failures: %v", reasons) } }) @@ -958,7 +865,7 @@ func TestFindFitPredicateCallCounts(t *testing.T) { for _, test := range tests { nodes := makeNodeList([]string{"1"}) - plugin := fakeFilterPlugin{} + plugin := st.FakeFilterPlugin{} registerFakeFilterFunc := st.RegisterFilterPlugin( "FakeFilter", func(_ runtime.Object, fh framework.FrameworkHandle) (framework.Plugin, error) { @@ -986,8 +893,8 @@ func TestFindFitPredicateCallCounts(t *testing.T) { if err != nil { t.Errorf("unexpected error: %v", err) } - if test.expectedCount != plugin.numFilterCalled { - t.Errorf("predicate was called %d times, expected is %d", plugin.numFilterCalled, test.expectedCount) + if test.expectedCount != plugin.NumFilterCalled { + t.Errorf("predicate was called %d times, expected is %d", plugin.NumFilterCalled, test.expectedCount) } } } @@ -1286,7 +1193,7 @@ func TestSelectNodesForPreemption(t *testing.T) { name: "a pod that does not fit on any machine", registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), - st.RegisterFilterPlugin("FalseFilter", NewFalseFilterPlugin), + st.RegisterFilterPlugin("FalseFilter", st.NewFalseFilterPlugin), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, @@ -1301,7 +1208,7 @@ func TestSelectNodesForPreemption(t *testing.T) { name: "a pod that fits with no preemption", registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), - st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + st.RegisterFilterPlugin("TrueFilter", st.NewTrueFilterPlugin), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, @@ -1316,7 +1223,7 @@ func TestSelectNodesForPreemption(t *testing.T) { name: "a pod that fits on one machine with no preemption", registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), - st.RegisterFilterPlugin("MatchFilter", NewMatchFilterPlugin), + st.RegisterFilterPlugin("MatchFilter", st.NewMatchFilterPlugin), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, @@ -1592,8 +1499,8 @@ func TestSelectNodesForPreemption(t *testing.T) { } // For each test, prepend a FakeFilterPlugin. - fakePlugin := fakeFilterPlugin{} - fakePlugin.failedNodeReturnCodeMap = filterFailedNodeReturnCodeMap + fakePlugin := st.FakeFilterPlugin{} + fakePlugin.FailedNodeReturnCodeMap = filterFailedNodeReturnCodeMap registerFakeFilterFunc := st.RegisterFilterPlugin( "FakeFilter", func(_ runtime.Object, fh framework.FrameworkHandle) (framework.Plugin, error) { @@ -1637,8 +1544,8 @@ func TestSelectNodesForPreemption(t *testing.T) { t.Error(err) } - if test.expectedNumFilterCalled != fakePlugin.numFilterCalled { - t.Errorf("expected fakePlugin.numFilterCalled is %d, but got %d", test.expectedNumFilterCalled, fakePlugin.numFilterCalled) + if test.expectedNumFilterCalled != fakePlugin.NumFilterCalled { + t.Errorf("expected fakePlugin.numFilterCalled is %d, but got %d", test.expectedNumFilterCalled, fakePlugin.NumFilterCalled) } if err := checkPreemptionVictims(test.expected, nodeToPods); err != nil { @@ -2063,7 +1970,7 @@ func TestPreempt(t *testing.T) { name string pod *v1.Pod pods []*v1.Pod - extenders []*FakeExtender + extenders []*st.FakeExtender failedNodeToStatusMap framework.NodeToStatusMap nodeNames []string registerPlugins []st.RegisterPluginFunc @@ -2210,12 +2117,12 @@ func TestPreempt(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "m2.1", UID: types.UID("m2.1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &midPriority, NodeName: "machine2"}, Status: v1.PodStatus{Phase: v1.PodRunning}}, }, - extenders: []*FakeExtender{ + extenders: []*st.FakeExtender{ { - predicates: []fitPredicate{truePredicateExtender}, + Predicates: []st.FitPredicate{st.TruePredicateExtender}, }, { - predicates: []fitPredicate{machine1PredicateExtender}, + Predicates: []st.FitPredicate{st.Machine1PredicateExtender}, }, }, registerPlugins: []st.RegisterPluginFunc{ @@ -2239,9 +2146,9 @@ func TestPreempt(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "m2.1", UID: types.UID("m2.1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &midPriority, NodeName: "machine2"}, Status: v1.PodStatus{Phase: v1.PodRunning}}, }, - extenders: []*FakeExtender{ + extenders: []*st.FakeExtender{ { - predicates: []fitPredicate{falsePredicateExtender}, + Predicates: []st.FitPredicate{st.FalsePredicateExtender}, }, }, registerPlugins: []st.RegisterPluginFunc{ @@ -2265,13 +2172,13 @@ func TestPreempt(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "m2.1", UID: types.UID("m2.1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &midPriority, NodeName: "machine2"}, Status: v1.PodStatus{Phase: v1.PodRunning}}, }, - extenders: []*FakeExtender{ + extenders: []*st.FakeExtender{ { - predicates: []fitPredicate{errorPredicateExtender}, - ignorable: true, + Predicates: []st.FitPredicate{st.ErrorPredicateExtender}, + Ignorable: true, }, { - predicates: []fitPredicate{machine1PredicateExtender}, + Predicates: []st.FitPredicate{st.Machine1PredicateExtender}, }, }, registerPlugins: []st.RegisterPluginFunc{ @@ -2295,13 +2202,13 @@ func TestPreempt(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: "m2.1", UID: types.UID("m2.1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &midPriority, NodeName: "machine2"}, Status: v1.PodStatus{Phase: v1.PodRunning}}, }, - extenders: []*FakeExtender{ + extenders: []*st.FakeExtender{ { - predicates: []fitPredicate{machine1PredicateExtender}, - unInterested: true, + Predicates: []st.FitPredicate{st.Machine1PredicateExtender}, + UnInterested: true, }, { - predicates: []fitPredicate{truePredicateExtender}, + Predicates: []st.FitPredicate{st.TruePredicateExtender}, }, }, registerPlugins: []st.RegisterPluginFunc{ @@ -2394,7 +2301,7 @@ func TestPreempt(t *testing.T) { var extenders []framework.Extender for _, extender := range test.extenders { // Set nodeInfoMap as extenders cached node information. - extender.cachedNodeNameToInfo = cachedNodeInfoMap + extender.CachedNodeNameToInfo = cachedNodeInfoMap extenders = append(extenders, extender) } @@ -2540,7 +2447,7 @@ func TestFairEvaluationForNodes(t *testing.T) { g := makeScheduler(nodes) prof, err := makeProfile( st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), - st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + st.RegisterFilterPlugin("TrueFilter", st.NewTrueFilterPlugin), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), ) if err != nil { diff --git a/pkg/scheduler/testing/BUILD b/pkg/scheduler/testing/BUILD index 849000e8e11..87fd5868e69 100644 --- a/pkg/scheduler/testing/BUILD +++ b/pkg/scheduler/testing/BUILD @@ -5,17 +5,23 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", srcs = [ + "fake_extender.go", + "fake_plugins.go", "framework_helpers.go", "workload_prep.go", "wrappers.go", ], importpath = "k8s.io/kubernetes/pkg/scheduler/testing", deps = [ + "//pkg/api/v1/pod:go_default_library", "//pkg/scheduler/apis/config:go_default_library", "//pkg/scheduler/framework/v1alpha1:go_default_library", + "//pkg/scheduler/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/kube-scheduler/extender/v1:go_default_library", ], ) diff --git a/pkg/scheduler/testing/fake_extender.go b/pkg/scheduler/testing/fake_extender.go new file mode 100644 index 00000000000..b4fc16a6c1c --- /dev/null +++ b/pkg/scheduler/testing/fake_extender.go @@ -0,0 +1,370 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "context" + "fmt" + "sort" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + extenderv1 "k8s.io/kube-scheduler/extender/v1" + podutil "k8s.io/kubernetes/pkg/api/v1/pod" + framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" + "k8s.io/kubernetes/pkg/scheduler/util" +) + +// FitPredicate is a function type which is used in fake extender. +type FitPredicate func(pod *v1.Pod, node *v1.Node) (bool, error) + +// PriorityFunc is a function type which is used in fake extender. +type PriorityFunc func(pod *v1.Pod, nodes []*v1.Node) (*framework.NodeScoreList, error) + +// PriorityConfig is used in fake extender to perform Prioritize function. +type PriorityConfig struct { + Function PriorityFunc + Weight int64 +} + +// ErrorPredicateExtender implements FitPredicate function to always return error. +func ErrorPredicateExtender(pod *v1.Pod, node *v1.Node) (bool, error) { + return false, fmt.Errorf("some error") +} + +// FalsePredicateExtender implements FitPredicate function to always return false. +func FalsePredicateExtender(pod *v1.Pod, node *v1.Node) (bool, error) { + return false, nil +} + +// TruePredicateExtender implements FitPredicate function to always return true. +func TruePredicateExtender(pod *v1.Pod, node *v1.Node) (bool, error) { + return true, nil +} + +// Machine1PredicateExtender implements FitPredicate function to return true +// when the given node's name is "machine1"; otherwise return false. +func Machine1PredicateExtender(pod *v1.Pod, node *v1.Node) (bool, error) { + if node.Name == "machine1" { + return true, nil + } + return false, nil +} + +// Machine2PredicateExtender implements FitPredicate function to return true +// when the given node's name is "machine2"; otherwise return false. +func Machine2PredicateExtender(pod *v1.Pod, node *v1.Node) (bool, error) { + if node.Name == "machine2" { + return true, nil + } + return false, nil +} + +// ErrorPrioritizerExtender implements PriorityFunc function to always return error. +func ErrorPrioritizerExtender(pod *v1.Pod, nodes []*v1.Node) (*framework.NodeScoreList, error) { + return &framework.NodeScoreList{}, fmt.Errorf("some error") +} + +// Machine1PrioritizerExtender implements PriorityFunc function to give score 10 +// if the given node's name is "machine1"; otherwise score 1. +func Machine1PrioritizerExtender(pod *v1.Pod, nodes []*v1.Node) (*framework.NodeScoreList, error) { + result := framework.NodeScoreList{} + for _, node := range nodes { + score := 1 + if node.Name == "machine1" { + score = 10 + } + result = append(result, framework.NodeScore{Name: node.Name, Score: int64(score)}) + } + return &result, nil +} + +// Machine2PrioritizerExtender implements PriorityFunc function to give score 10 +// if the given node's name is "machine2"; otherwise score 1. +func Machine2PrioritizerExtender(pod *v1.Pod, nodes []*v1.Node) (*framework.NodeScoreList, error) { + result := framework.NodeScoreList{} + for _, node := range nodes { + score := 1 + if node.Name == "machine2" { + score = 10 + } + result = append(result, framework.NodeScore{Name: node.Name, Score: int64(score)}) + } + return &result, nil +} + +type machine2PrioritizerPlugin struct{} + +// NewMachine2PrioritizerPlugin returns a factory function to build machine2PrioritizerPlugin. +func NewMachine2PrioritizerPlugin() framework.PluginFactory { + return func(_ runtime.Object, _ framework.FrameworkHandle) (framework.Plugin, error) { + return &machine2PrioritizerPlugin{}, nil + } +} + +// Name returns name of the plugin. +func (pl *machine2PrioritizerPlugin) Name() string { + return "Machine2Prioritizer" +} + +// Score return score 100 if the given nodeName is "machine2"; otherwise return score 10. +func (pl *machine2PrioritizerPlugin) Score(_ context.Context, _ *framework.CycleState, _ *v1.Pod, nodeName string) (int64, *framework.Status) { + score := 10 + if nodeName == "machine2" { + score = 100 + } + return int64(score), nil +} + +// ScoreExtensions returns nil. +func (pl *machine2PrioritizerPlugin) ScoreExtensions() framework.ScoreExtensions { + return nil +} + +// FakeExtender is a data struct which implements the Extender interface. +type FakeExtender struct { + Predicates []FitPredicate + Prioritizers []PriorityConfig + Weight int64 + NodeCacheCapable bool + FilteredNodes []*v1.Node + UnInterested bool + Ignorable bool + + // Cached node information for fake extender + CachedNodeNameToInfo map[string]*framework.NodeInfo +} + +// Name returns name of the extender. +func (f *FakeExtender) Name() string { + return "FakeExtender" +} + +// IsIgnorable returns a bool value indicating whether internal errors can be ignored. +func (f *FakeExtender) IsIgnorable() bool { + return f.Ignorable +} + +// SupportsPreemption returns true indicating the extender supports preemption. +func (f *FakeExtender) SupportsPreemption() bool { + // Assume preempt verb is always defined. + return true +} + +// ProcessPreemption implements the extender preempt function. +func (f *FakeExtender) ProcessPreemption( + pod *v1.Pod, + nodeNameToVictims map[string]*extenderv1.Victims, + nodeInfos framework.NodeInfoLister, +) (map[string]*extenderv1.Victims, error) { + nodeNameToVictimsCopy := map[string]*extenderv1.Victims{} + // We don't want to change the original nodeNameToVictims + for k, v := range nodeNameToVictims { + // In real world implementation, extender's user should have their own way to get node object + // by name if needed (e.g. query kube-apiserver etc). + // + // For test purpose, we just use node from parameters directly. + nodeNameToVictimsCopy[k] = v + } + + for nodeName, victims := range nodeNameToVictimsCopy { + // Try to do preemption on extender side. + nodeInfo, _ := nodeInfos.Get(nodeName) + extenderVictimPods, extenderPDBViolations, fits, err := f.selectVictimsOnNodeByExtender(pod, nodeInfo.Node()) + if err != nil { + return nil, err + } + // If it's unfit after extender's preemption, this node is unresolvable by preemption overall, + // let's remove it from potential preemption nodes. + if !fits { + delete(nodeNameToVictimsCopy, nodeName) + } else { + // Append new victims to original victims + nodeNameToVictimsCopy[nodeName].Pods = append(victims.Pods, extenderVictimPods...) + nodeNameToVictimsCopy[nodeName].NumPDBViolations = victims.NumPDBViolations + int64(extenderPDBViolations) + } + } + return nodeNameToVictimsCopy, nil +} + +// selectVictimsOnNodeByExtender checks the given nodes->pods map with predicates on extender's side. +// Returns: +// 1. More victim pods (if any) amended by preemption phase of extender. +// 2. Number of violating victim (used to calculate PDB). +// 3. Fits or not after preemption phase on extender's side. +func (f *FakeExtender) selectVictimsOnNodeByExtender(pod *v1.Pod, node *v1.Node) ([]*v1.Pod, int, bool, error) { + // If a extender support preemption but have no cached node info, let's run filter to make sure + // default scheduler's decision still stand with given pod and node. + if !f.NodeCacheCapable { + fits, err := f.runPredicate(pod, node) + if err != nil { + return nil, 0, false, err + } + if !fits { + return nil, 0, false, nil + } + return []*v1.Pod{}, 0, true, nil + } + + // Otherwise, as a extender support preemption and have cached node info, we will assume cachedNodeNameToInfo is available + // and get cached node info by given node name. + nodeInfoCopy := f.CachedNodeNameToInfo[node.GetName()].Clone() + + var potentialVictims []*v1.Pod + + removePod := func(rp *v1.Pod) { + nodeInfoCopy.RemovePod(rp) + } + addPod := func(ap *v1.Pod) { + nodeInfoCopy.AddPod(ap) + } + // As the first step, remove all the lower priority pods from the node and + // check if the given pod can be scheduled. + podPriority := podutil.GetPodPriority(pod) + for _, p := range nodeInfoCopy.Pods { + if podutil.GetPodPriority(p.Pod) < podPriority { + potentialVictims = append(potentialVictims, p.Pod) + removePod(p.Pod) + } + } + sort.Slice(potentialVictims, func(i, j int) bool { return util.MoreImportantPod(potentialVictims[i], potentialVictims[j]) }) + + // If the new pod does not fit after removing all the lower priority pods, + // we are almost done and this node is not suitable for preemption. + fits, err := f.runPredicate(pod, nodeInfoCopy.Node()) + if err != nil { + return nil, 0, false, err + } + if !fits { + return nil, 0, false, nil + } + + var victims []*v1.Pod + + // TODO(harry): handle PDBs in the future. + numViolatingVictim := 0 + + reprievePod := func(p *v1.Pod) bool { + addPod(p) + fits, _ := f.runPredicate(pod, nodeInfoCopy.Node()) + if !fits { + removePod(p) + victims = append(victims, p) + } + return fits + } + + // For now, assume all potential victims to be non-violating. + // Now we try to reprieve non-violating victims. + for _, p := range potentialVictims { + reprievePod(p) + } + + return victims, numViolatingVictim, true, nil +} + +// runPredicate run predicates of extender one by one for given pod and node. +// Returns: fits or not. +func (f *FakeExtender) runPredicate(pod *v1.Pod, node *v1.Node) (bool, error) { + fits := true + var err error + for _, predicate := range f.Predicates { + fits, err = predicate(pod, node) + if err != nil { + return false, err + } + if !fits { + break + } + } + return fits, nil +} + +// Filter implements the extender Filter function. +func (f *FakeExtender) Filter(pod *v1.Pod, nodes []*v1.Node) ([]*v1.Node, extenderv1.FailedNodesMap, error) { + var filtered []*v1.Node + failedNodesMap := extenderv1.FailedNodesMap{} + for _, node := range nodes { + fits, err := f.runPredicate(pod, node) + if err != nil { + return []*v1.Node{}, extenderv1.FailedNodesMap{}, err + } + if fits { + filtered = append(filtered, node) + } else { + failedNodesMap[node.Name] = "FakeExtender failed" + } + } + + f.FilteredNodes = filtered + if f.NodeCacheCapable { + return filtered, failedNodesMap, nil + } + return filtered, failedNodesMap, nil +} + +// Prioritize implements the extender Prioritize function. +func (f *FakeExtender) Prioritize(pod *v1.Pod, nodes []*v1.Node) (*extenderv1.HostPriorityList, int64, error) { + result := extenderv1.HostPriorityList{} + combinedScores := map[string]int64{} + for _, prioritizer := range f.Prioritizers { + weight := prioritizer.Weight + if weight == 0 { + continue + } + priorityFunc := prioritizer.Function + prioritizedList, err := priorityFunc(pod, nodes) + if err != nil { + return &extenderv1.HostPriorityList{}, 0, err + } + for _, hostEntry := range *prioritizedList { + combinedScores[hostEntry.Name] += hostEntry.Score * weight + } + } + for host, score := range combinedScores { + result = append(result, extenderv1.HostPriority{Host: host, Score: score}) + } + return &result, f.Weight, nil +} + +// Bind implements the extender Bind function. +func (f *FakeExtender) Bind(binding *v1.Binding) error { + if len(f.FilteredNodes) != 0 { + for _, node := range f.FilteredNodes { + if node.Name == binding.Target.Name { + f.FilteredNodes = nil + return nil + } + } + err := fmt.Errorf("Node %v not in filtered nodes %v", binding.Target.Name, f.FilteredNodes) + f.FilteredNodes = nil + return err + } + return nil +} + +// IsBinder returns true indicating the extender implements the Binder function. +func (f *FakeExtender) IsBinder() bool { + return true +} + +// IsInterested returns a bool true indicating whether extender +func (f *FakeExtender) IsInterested(pod *v1.Pod) bool { + return !f.UnInterested +} + +var _ framework.Extender = &FakeExtender{} diff --git a/pkg/scheduler/testing/fake_plugins.go b/pkg/scheduler/testing/fake_plugins.go new file mode 100644 index 00000000000..bd7ac3f52dd --- /dev/null +++ b/pkg/scheduler/testing/fake_plugins.go @@ -0,0 +1,124 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "context" + "fmt" + "sync/atomic" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" +) + +// ErrReasonFake is a fake error message denotes the filter function errored. +const ErrReasonFake = "Nodes failed the fake plugin" + +// FalseFilterPlugin is a filter plugin which always return Unschedulable when Filter function is called. +type FalseFilterPlugin struct{} + +// Name returns name of the plugin. +func (pl *FalseFilterPlugin) Name() string { + return "FalseFilter" +} + +// Filter invoked at the filter extension point. +func (pl *FalseFilterPlugin) Filter(_ context.Context, _ *framework.CycleState, pod *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status { + return framework.NewStatus(framework.Unschedulable, ErrReasonFake) +} + +// NewFalseFilterPlugin initializes a FalseFilterPlugin and returns it. +func NewFalseFilterPlugin(_ runtime.Object, _ framework.FrameworkHandle) (framework.Plugin, error) { + return &FalseFilterPlugin{}, nil +} + +// TrueFilterPlugin is a filter plugin which always return Success when Filter function is called. +type TrueFilterPlugin struct{} + +// Name returns name of the plugin. +func (pl *TrueFilterPlugin) Name() string { + return "TrueFilter" +} + +// Filter invoked at the filter extension point. +func (pl *TrueFilterPlugin) Filter(_ context.Context, _ *framework.CycleState, pod *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status { + return nil +} + +// NewTrueFilterPlugin initializes a TrueFilterPlugin and returns it. +func NewTrueFilterPlugin(_ runtime.Object, _ framework.FrameworkHandle) (framework.Plugin, error) { + return &TrueFilterPlugin{}, nil +} + +// FakeFilterPlugin is a test filter plugin to record how many times its Filter() function have +// been called, and it returns different 'Code' depending on its internal 'failedNodeReturnCodeMap'. +type FakeFilterPlugin struct { + NumFilterCalled int32 + FailedNodeReturnCodeMap map[string]framework.Code +} + +// Name returns name of the plugin. +func (pl *FakeFilterPlugin) Name() string { + return "FakeFilter" +} + +// Filter invoked at the filter extension point. +func (pl *FakeFilterPlugin) Filter(_ context.Context, _ *framework.CycleState, pod *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status { + atomic.AddInt32(&pl.NumFilterCalled, 1) + + if returnCode, ok := pl.FailedNodeReturnCodeMap[nodeInfo.Node().Name]; ok { + return framework.NewStatus(returnCode, fmt.Sprintf("injecting failure for pod %v", pod.Name)) + } + + return nil +} + +// NewFakeFilterPlugin initializes a fakeFilterPlugin and returns it. +func NewFakeFilterPlugin(failedNodeReturnCodeMap map[string]framework.Code) framework.PluginFactory { + return func(_ runtime.Object, _ framework.FrameworkHandle) (framework.Plugin, error) { + return &FakeFilterPlugin{ + FailedNodeReturnCodeMap: failedNodeReturnCodeMap, + }, nil + } +} + +// MatchFilterPlugin is a filter plugin which return Success when the evaluated pod and node +// have the same name; otherwise return Unschedulable. +type MatchFilterPlugin struct{} + +// Name returns name of the plugin. +func (pl *MatchFilterPlugin) Name() string { + return "MatchFilter" +} + +// Filter invoked at the filter extension point. +func (pl *MatchFilterPlugin) Filter(_ context.Context, _ *framework.CycleState, pod *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status { + node := nodeInfo.Node() + if node == nil { + return framework.NewStatus(framework.Error, "node not found") + } + if pod.Name == node.Name { + return nil + } + return framework.NewStatus(framework.Unschedulable, ErrReasonFake) +} + +// NewMatchFilterPlugin initializes a MatchFilterPlugin and returns it. +func NewMatchFilterPlugin(_ runtime.Object, _ framework.FrameworkHandle) (framework.Plugin, error) { + return &MatchFilterPlugin{}, nil +}