diff --git a/pkg/scheduler/BUILD b/pkg/scheduler/BUILD index 23a5de16e96..f2da5fb62c4 100644 --- a/pkg/scheduler/BUILD +++ b/pkg/scheduler/BUILD @@ -18,6 +18,7 @@ go_library( "//pkg/scheduler/apis/config/validation:go_default_library", "//pkg/scheduler/core:go_default_library", "//pkg/scheduler/framework/plugins:go_default_library", + "//pkg/scheduler/framework/plugins/defaultbinder:go_default_library", "//pkg/scheduler/framework/plugins/interpodaffinity:go_default_library", "//pkg/scheduler/framework/plugins/noderesources:go_default_library", "//pkg/scheduler/framework/plugins/queuesort:go_default_library", @@ -66,6 +67,7 @@ go_test( "//pkg/scheduler/apis/extender/v1:go_default_library", "//pkg/scheduler/core:go_default_library", "//pkg/scheduler/framework/plugins:go_default_library", + "//pkg/scheduler/framework/plugins/defaultbinder:go_default_library", "//pkg/scheduler/framework/plugins/nodelabel:go_default_library", "//pkg/scheduler/framework/plugins/nodeports:go_default_library", "//pkg/scheduler/framework/plugins/noderesources:go_default_library", @@ -95,7 +97,6 @@ go_test( "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1/fake:go_default_library", "//staging/src/k8s.io/client-go/testing:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/events:go_default_library", diff --git a/pkg/scheduler/algorithmprovider/BUILD b/pkg/scheduler/algorithmprovider/BUILD index bb6cdf6141e..9c82fa07ddc 100644 --- a/pkg/scheduler/algorithmprovider/BUILD +++ b/pkg/scheduler/algorithmprovider/BUILD @@ -13,6 +13,7 @@ go_library( deps = [ "//pkg/features:go_default_library", "//pkg/scheduler/apis/config:go_default_library", + "//pkg/scheduler/framework/plugins/defaultbinder:go_default_library", "//pkg/scheduler/framework/plugins/defaultpodtopologyspread:go_default_library", "//pkg/scheduler/framework/plugins/imagelocality:go_default_library", "//pkg/scheduler/framework/plugins/interpodaffinity:go_default_library", @@ -42,6 +43,7 @@ go_test( deps = [ "//pkg/features:go_default_library", "//pkg/scheduler/apis/config:go_default_library", + "//pkg/scheduler/framework/plugins/defaultbinder:go_default_library", "//pkg/scheduler/framework/plugins/defaultpodtopologyspread:go_default_library", "//pkg/scheduler/framework/plugins/imagelocality:go_default_library", "//pkg/scheduler/framework/plugins/interpodaffinity:go_default_library", diff --git a/pkg/scheduler/algorithmprovider/registry.go b/pkg/scheduler/algorithmprovider/registry.go index 8aba8dd09dc..093a60c519f 100644 --- a/pkg/scheduler/algorithmprovider/registry.go +++ b/pkg/scheduler/algorithmprovider/registry.go @@ -26,6 +26,7 @@ import ( "k8s.io/klog" "k8s.io/kubernetes/pkg/features" schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultpodtopologyspread" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/imagelocality" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity" @@ -133,6 +134,11 @@ func getDefaultConfig(hardPodAffinityWeight int64) *Config { {Name: tainttoleration.Name, Weight: 1}, }, }, + Bind: &schedulerapi.PluginSet{ + Enabled: []schedulerapi.Plugin{ + {Name: defaultbinder.Name}, + }, + }, }, FrameworkPluginConfig: []schedulerapi.PluginConfig{ { diff --git a/pkg/scheduler/algorithmprovider/registry_test.go b/pkg/scheduler/algorithmprovider/registry_test.go index c397dacf584..e9174d73a0a 100644 --- a/pkg/scheduler/algorithmprovider/registry_test.go +++ b/pkg/scheduler/algorithmprovider/registry_test.go @@ -21,13 +21,13 @@ import ( "testing" "github.com/google/go-cmp/cmp" - "k8s.io/kubernetes/pkg/scheduler/framework/plugins/queuesort" "k8s.io/apimachinery/pkg/runtime" utilfeature "k8s.io/apiserver/pkg/util/feature" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/features" schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultpodtopologyspread" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/imagelocality" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity" @@ -39,6 +39,7 @@ import ( "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeunschedulable" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodevolumelimits" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/podtopologyspread" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/queuesort" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/tainttoleration" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumebinding" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumerestrictions" @@ -98,6 +99,11 @@ func TestClusterAutoscalerProvider(t *testing.T) { {Name: tainttoleration.Name, Weight: 1}, }, }, + Bind: &schedulerapi.PluginSet{ + Enabled: []schedulerapi.Plugin{ + {Name: defaultbinder.Name}, + }, + }, }, FrameworkPluginConfig: []schedulerapi.PluginConfig{ { @@ -175,6 +181,11 @@ func TestApplyFeatureGates(t *testing.T) { {Name: tainttoleration.Name, Weight: 1}, }, }, + Bind: &schedulerapi.PluginSet{ + Enabled: []schedulerapi.Plugin{ + {Name: defaultbinder.Name}, + }, + }, }, FrameworkPluginConfig: []schedulerapi.PluginConfig{ { @@ -244,6 +255,11 @@ func TestApplyFeatureGates(t *testing.T) { {Name: noderesources.ResourceLimitsName, Weight: 1}, }, }, + Bind: &schedulerapi.PluginSet{ + Enabled: []schedulerapi.Plugin{ + {Name: defaultbinder.Name}, + }, + }, }, FrameworkPluginConfig: []schedulerapi.PluginConfig{ { diff --git a/pkg/scheduler/apis/config/testing/compatibility_test.go b/pkg/scheduler/apis/config/testing/compatibility_test.go index cfb10635403..bc540d46d0a 100644 --- a/pkg/scheduler/apis/config/testing/compatibility_test.go +++ b/pkg/scheduler/apis/config/testing/compatibility_test.go @@ -74,6 +74,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {Name: "NodeAffinity"}, {Name: "TaintToleration"}, }, + "BindPlugin": {{Name: "DefaultBinder"}}, }, }, // This is a special test for the case where a policy is specified without specifying any filters. @@ -93,6 +94,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {Name: "NodeUnschedulable"}, {Name: "TaintToleration"}, }, + "BindPlugin": {{Name: "DefaultBinder"}}, }, }, // Do not change this JSON after the corresponding release has been tagged. @@ -140,6 +142,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {Name: "DefaultPodTopologySpread", Weight: 2}, {Name: "ServiceAffinity", Weight: 3}, }, + "BindPlugin": {{Name: "DefaultBinder"}}, }, }, @@ -195,6 +198,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {Name: "DefaultPodTopologySpread", Weight: 2}, {Name: "ServiceAffinity", Weight: 6}, // Weight is the 3 * number of custom ServiceAntiAffinity priorities }, + "BindPlugin": {{Name: "DefaultBinder"}}, }, }, // Do not change this JSON after the corresponding release has been tagged. @@ -259,6 +263,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {Name: "DefaultPodTopologySpread", Weight: 2}, {Name: "ServiceAffinity", Weight: 3}, }, + "BindPlugin": {{Name: "DefaultBinder"}}, }, }, @@ -332,6 +337,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {Name: "DefaultPodTopologySpread", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, + "BindPlugin": {{Name: "DefaultBinder"}}, }, }, @@ -409,6 +415,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {Name: "DefaultPodTopologySpread", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, + "BindPlugin": {{Name: "DefaultBinder"}}, }, }, // Do not change this JSON after the corresponding release has been tagged. @@ -495,6 +502,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {Name: "DefaultPodTopologySpread", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, + "BindPlugin": {{Name: "DefaultBinder"}}, }, wantExtenders: []config.Extender{{ URLPrefix: "/prefix", @@ -592,6 +600,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {Name: "DefaultPodTopologySpread", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, + "BindPlugin": {{Name: "DefaultBinder"}}, }, wantExtenders: []config.Extender{{ URLPrefix: "/prefix", @@ -691,6 +700,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {Name: "DefaultPodTopologySpread", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, + "BindPlugin": {{Name: "DefaultBinder"}}, }, wantExtenders: []config.Extender{{ URLPrefix: "/prefix", @@ -793,6 +803,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {Name: "DefaultPodTopologySpread", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, + "BindPlugin": {{Name: "DefaultBinder"}}, }, wantExtenders: []config.Extender{{ URLPrefix: "/prefix", @@ -908,6 +919,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {Name: "DefaultPodTopologySpread", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, + "BindPlugin": {{Name: "DefaultBinder"}}, }, wantExtenders: []config.Extender{{ URLPrefix: "/prefix", @@ -1025,6 +1037,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {Name: "DefaultPodTopologySpread", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, + "BindPlugin": {{Name: "DefaultBinder"}}, }, wantExtenders: []config.Extender{{ URLPrefix: "/prefix", @@ -1142,6 +1155,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {Name: "DefaultPodTopologySpread", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, + "BindPlugin": {{Name: "DefaultBinder"}}, }, wantExtenders: []config.Extender{{ URLPrefix: "/prefix", @@ -1263,6 +1277,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {Name: "DefaultPodTopologySpread", Weight: 2}, {Name: "TaintToleration", Weight: 2}, }, + "BindPlugin": {{Name: "DefaultBinder"}}, }, wantExtenders: []config.Extender{{ URLPrefix: "/prefix", @@ -1309,6 +1324,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "ScorePlugin": { {Name: "PodTopologySpread", Weight: 2}, }, + "BindPlugin": {{Name: "DefaultBinder"}}, }, }, { @@ -1336,6 +1352,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "ScorePlugin": { {Name: "NodeResourceLimits", Weight: 2}, }, + "BindPlugin": {{Name: "DefaultBinder"}}, }, }, } @@ -1438,6 +1455,7 @@ func TestAlgorithmProviderCompatibility(t *testing.T) { {Name: "DefaultPodTopologySpread", Weight: 1}, {Name: "TaintToleration", Weight: 1}, }, + "BindPlugin": {{Name: "DefaultBinder"}}, } testcases := []struct { @@ -1497,6 +1515,7 @@ func TestAlgorithmProviderCompatibility(t *testing.T) { {Name: "DefaultPodTopologySpread", Weight: 1}, {Name: "TaintToleration", Weight: 1}, }, + "BindPlugin": {{Name: "DefaultBinder"}}, }, }, } diff --git a/pkg/scheduler/core/BUILD b/pkg/scheduler/core/BUILD index f0c94c858d5..aef9861c5ff 100644 --- a/pkg/scheduler/core/BUILD +++ b/pkg/scheduler/core/BUILD @@ -46,6 +46,7 @@ go_test( "//pkg/api/v1/pod:go_default_library", "//pkg/scheduler/apis/config:go_default_library", "//pkg/scheduler/apis/extender/v1:go_default_library", + "//pkg/scheduler/framework/plugins/defaultbinder:go_default_library", "//pkg/scheduler/framework/plugins/defaultpodtopologyspread:go_default_library", "//pkg/scheduler/framework/plugins/interpodaffinity:go_default_library", "//pkg/scheduler/framework/plugins/nodeaffinity:go_default_library", diff --git a/pkg/scheduler/core/extender_test.go b/pkg/scheduler/core/extender_test.go index eec65a2ae42..0de01e32c57 100644 --- a/pkg/scheduler/core/extender_test.go +++ b/pkg/scheduler/core/extender_test.go @@ -35,6 +35,7 @@ import ( podutil "k8s.io/kubernetes/pkg/api/v1/pod" schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" extenderv1 "k8s.io/kubernetes/pkg/scheduler/apis/extender/v1" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/queuesort" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" @@ -367,6 +368,7 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, extenders: []FakeExtender{ { @@ -384,6 +386,7 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, extenders: []FakeExtender{ { @@ -401,6 +404,7 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, extenders: []FakeExtender{ { @@ -422,6 +426,7 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, extenders: []FakeExtender{ { @@ -439,6 +444,7 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, extenders: []FakeExtender{ { @@ -459,6 +465,7 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, extenders: []FakeExtender{ { @@ -485,6 +492,7 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), st.RegisterScorePlugin("Machine2Prioritizer", newMachine2PrioritizerPlugin(), 20), st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, extenders: []FakeExtender{ { @@ -513,6 +521,7 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), st.RegisterScorePlugin("Machine2Prioritizer", newMachine2PrioritizerPlugin(), 1), st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, extenders: []FakeExtender{ { @@ -539,6 +548,7 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, extenders: []FakeExtender{ { @@ -575,17 +585,10 @@ func TestGenericSchedulerWithExtenders(t *testing.T) { } queue := internalqueue.NewSchedulingQueue(nil) - registry := framework.Registry{} - plugins := &schedulerapi.Plugins{ - QueueSort: &schedulerapi.PluginSet{}, - Filter: &schedulerapi.PluginSet{}, - Score: &schedulerapi.PluginSet{}, + fwk, err := st.NewFramework(test.registerPlugins, framework.WithClientSet(client)) + if err != nil { + t.Fatal(err) } - var pluginConfigs []schedulerapi.PluginConfig - for _, f := range test.registerPlugins { - f(®istry, plugins, pluginConfigs) - } - fwk, _ := framework.NewFramework(registry, plugins, pluginConfigs) scheduler := NewGenericScheduler( cache, diff --git a/pkg/scheduler/core/generic_scheduler_test.go b/pkg/scheduler/core/generic_scheduler_test.go index c8ea4fcf7be..f71196387c8 100644 --- a/pkg/scheduler/core/generic_scheduler_test.go +++ b/pkg/scheduler/core/generic_scheduler_test.go @@ -38,6 +38,7 @@ import ( clientsetfake "k8s.io/client-go/kubernetes/fake" schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" extenderv1 "k8s.io/kubernetes/pkg/scheduler/apis/extender/v1" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultpodtopologyspread" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeaffinity" @@ -386,6 +387,7 @@ func TestGenericScheduler(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin("FalseFilter", NewFalseFilterPlugin), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "2", UID: types.UID("2")}}, @@ -403,6 +405,7 @@ func TestGenericScheduler(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "ignore", UID: types.UID("ignore")}}, @@ -415,6 +418,7 @@ func TestGenericScheduler(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin("MatchFilter", NewMatchFilterPlugin), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine2", UID: types.UID("machine2")}}, @@ -427,6 +431,7 @@ func TestGenericScheduler(t *testing.T) { st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), st.RegisterScorePlugin("NumericMap", newNumericMapPlugin(), 1), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"3", "2", "1"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "ignore", UID: types.UID("ignore")}}, @@ -439,6 +444,7 @@ func TestGenericScheduler(t *testing.T) { st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin("MatchFilter", NewMatchFilterPlugin), st.RegisterScorePlugin("NumericMap", newNumericMapPlugin(), 1), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"3", "2", "1"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "2", UID: types.UID("2")}}, @@ -452,6 +458,7 @@ func TestGenericScheduler(t *testing.T) { st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), st.RegisterScorePlugin("NumericMap", newNumericMapPlugin(), 1), st.RegisterScorePlugin("ReverseNumericMap", newReverseNumericMapPlugin(), 2), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"3", "2", "1"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "2", UID: types.UID("2")}}, @@ -465,6 +472,7 @@ func TestGenericScheduler(t *testing.T) { st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), st.RegisterFilterPlugin("FalseFilter", NewFalseFilterPlugin), st.RegisterScorePlugin("NumericMap", newNumericMapPlugin(), 1), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"3", "2", "1"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "2", UID: types.UID("2")}}, @@ -485,6 +493,7 @@ func TestGenericScheduler(t *testing.T) { st.RegisterFilterPlugin("NoPodsFilter", NewNoPodsFilterPlugin), st.RegisterFilterPlugin("MatchFilter", NewMatchFilterPlugin), st.RegisterScorePlugin("NumericMap", newNumericMapPlugin(), 1), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, pods: []*v1.Pod{ { @@ -514,6 +523,7 @@ func TestGenericScheduler(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, pvcs: []v1.PersistentVolumeClaim{{ObjectMeta: metav1.ObjectMeta{Name: "existingPVC"}}}, @@ -540,6 +550,7 @@ func TestGenericScheduler(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, pod: &v1.Pod{ @@ -564,6 +575,7 @@ func TestGenericScheduler(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, pvcs: []v1.PersistentVolumeClaim{{ObjectMeta: metav1.ObjectMeta{Name: "existingPVC", DeletionTimestamp: &metav1.Time{}}}}, @@ -590,6 +602,7 @@ func TestGenericScheduler(t *testing.T) { st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), st.RegisterScorePlugin("FalseMap", newFalseMapPlugin(), 1), st.RegisterScorePlugin("TrueMap", newTrueMapPlugin(), 2), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"2", "1"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "2"}}, @@ -607,6 +620,7 @@ func TestGenericScheduler(t *testing.T) { "PreFilter", "Filter", ), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, pod: &v1.Pod{ @@ -654,6 +668,7 @@ func TestGenericScheduler(t *testing.T) { "PreFilter", "Filter", ), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2", "machine3"}, pod: &v1.Pod{ @@ -717,6 +732,7 @@ func TestGenericScheduler(t *testing.T) { NewFakeFilterPlugin(map[string]framework.Code{"3": framework.Unschedulable}), ), st.RegisterScorePlugin("NumericMap", newNumericMapPlugin(), 1), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"3"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "test-filter", UID: types.UID("test-filter")}}, @@ -738,6 +754,7 @@ func TestGenericScheduler(t *testing.T) { NewFakeFilterPlugin(map[string]framework.Code{"3": framework.UnschedulableAndUnresolvable}), ), st.RegisterScorePlugin("NumericMap", newNumericMapPlugin(), 1), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"3"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "test-filter", UID: types.UID("test-filter")}}, @@ -759,6 +776,7 @@ func TestGenericScheduler(t *testing.T) { NewFakeFilterPlugin(map[string]framework.Code{"1": framework.Unschedulable}), ), st.RegisterScorePlugin("NumericMap", newNumericMapPlugin(), 1), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"1", "2"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "test-filter", UID: types.UID("test-filter")}}, @@ -782,20 +800,11 @@ func TestGenericScheduler(t *testing.T) { cache.AddNode(node) } - registry := framework.Registry{} - // TODO: instantiate the plugins dynamically. - plugins := &schedulerapi.Plugins{ - QueueSort: &schedulerapi.PluginSet{}, - PreFilter: &schedulerapi.PluginSet{}, - Filter: &schedulerapi.PluginSet{}, - Score: &schedulerapi.PluginSet{}, - } - var pluginConfigs []schedulerapi.PluginConfig - for _, f := range test.registerPlugins { - f(®istry, plugins, pluginConfigs) - } snapshot := internalcache.NewSnapshot(test.pods, nodes) - fwk, _ := framework.NewFramework(registry, plugins, pluginConfigs, framework.WithSnapshotSharedLister(snapshot)) + fwk, err := st.NewFramework(test.registerPlugins, framework.WithSnapshotSharedLister(snapshot)) + if err != nil { + t.Fatal(err) + } var pvcs []v1.PersistentVolumeClaim pvcs = append(pvcs, test.pvcs...) @@ -834,18 +843,7 @@ func makeScheduler(nodes []*v1.Node, fns ...st.RegisterPluginFunc) *genericSched cache.AddNode(n) } - registry := framework.Registry{} - // TODO: instantiate the plugins dynamically. - plugins := &schedulerapi.Plugins{ - QueueSort: &schedulerapi.PluginSet{}, - Filter: &schedulerapi.PluginSet{}, - } - var pluginConfigs []schedulerapi.PluginConfig - for _, f := range fns { - f(®istry, plugins, pluginConfigs) - } - fwk, _ := framework.NewFramework(registry, plugins, pluginConfigs) - + fwk, _ := st.NewFramework(fns) s := NewGenericScheduler( cache, internalqueue.NewSchedulingQueue(nil), @@ -865,6 +863,7 @@ func TestFindFitAllError(t *testing.T) { st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), st.RegisterFilterPlugin("MatchFilter", NewMatchFilterPlugin), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), ) _, nodeToStatusMap, err := scheduler.findNodesThatFitPod(context.Background(), framework.NewCycleState(), &v1.Pod{}) @@ -898,6 +897,7 @@ func TestFindFitSomeError(t *testing.T) { st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), st.RegisterFilterPlugin("MatchFilter", NewMatchFilterPlugin), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), ) pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "1", UID: types.UID("1")}} @@ -954,25 +954,22 @@ func TestFindFitPredicateCallCounts(t *testing.T) { cache.AddNode(n) } - registry := framework.Registry{} plugin := fakeFilterPlugin{} - registry.Register(queuesort.Name, queuesort.New) - registry.Register("FakeFilter", func(_ *runtime.Unknown, fh framework.FrameworkHandle) (framework.Plugin, error) { - return &plugin, nil - }) - plugins := &schedulerapi.Plugins{ - QueueSort: &schedulerapi.PluginSet{ - Enabled: []schedulerapi.Plugin{{Name: queuesort.Name}}, - }, - Filter: &schedulerapi.PluginSet{ - Enabled: []schedulerapi.Plugin{{Name: "FakeFilter"}}, + registerFakeFilterFunc := st.RegisterFilterPlugin( + "FakeFilter", + func(_ *runtime.Unknown, fh framework.FrameworkHandle) (framework.Plugin, error) { + return &plugin, nil }, + ) + registerPlugins := []st.RegisterPluginFunc{ + st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), + registerFakeFilterFunc, + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), } - pluginConfigs := []schedulerapi.PluginConfig{ - {Name: queuesort.Name}, - {Name: "FakeFilter"}, + fwk, err := st.NewFramework(registerPlugins) + if err != nil { + t.Fatal(err) } - fwk, _ := framework.NewFramework(registry, plugins, pluginConfigs) queue := internalqueue.NewSchedulingQueue(nil) scheduler := NewGenericScheduler( @@ -985,7 +982,7 @@ func TestFindFitPredicateCallCounts(t *testing.T) { cache.UpdateSnapshot(scheduler.nodeInfoSnapshot) queue.UpdateNominatedPodForNode(&v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: types.UID("nominated")}, Spec: v1.PodSpec{Priority: &midPriority}}, "1") - _, _, err := scheduler.findNodesThatFitPod(context.Background(), framework.NewCycleState(), test.pod) + _, _, err = scheduler.findNodesThatFitPod(context.Background(), framework.NewCycleState(), test.pod) if err != nil { t.Errorf("unexpected error: %v", err) @@ -1115,29 +1112,16 @@ func TestZeroRequest(t *testing.T) { snapshot := internalcache.NewSnapshot(test.pods, test.nodes) - registry := framework.Registry{} - // TODO: instantiate the plugins dynamically. - plugins := &schedulerapi.Plugins{ - QueueSort: &schedulerapi.PluginSet{}, - Filter: &schedulerapi.PluginSet{}, - PostFilter: &schedulerapi.PluginSet{}, - Score: &schedulerapi.PluginSet{}, - } - var pluginConfigs []schedulerapi.PluginConfig pluginRegistrations := []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterScorePlugin(noderesources.LeastAllocatedName, noderesources.NewLeastAllocated, 1), st.RegisterScorePlugin(noderesources.BalancedAllocationName, noderesources.NewBalancedAllocation, 1), st.RegisterScorePlugin(defaultpodtopologyspread.Name, defaultpodtopologyspread.New, 1), st.RegisterPostFilterPlugin(defaultpodtopologyspread.Name, defaultpodtopologyspread.New), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), } - for _, f := range pluginRegistrations { - f(®istry, plugins, pluginConfigs) - } - fwk, err := framework.NewFramework( - registry, - plugins, - pluginConfigs, + fwk, err := st.NewFramework( + pluginRegistrations, framework.WithInformerFactory(informerFactory), framework.WithSnapshotSharedLister(snapshot), framework.WithClientSet(client), @@ -1302,6 +1286,7 @@ func TestSelectNodesForPreemption(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin("FalseFilter", NewFalseFilterPlugin), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "new", UID: types.UID("new")}, Spec: v1.PodSpec{Priority: &highPriority}}, @@ -1316,6 +1301,7 @@ func TestSelectNodesForPreemption(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "new", UID: types.UID("new")}, Spec: v1.PodSpec{Priority: &highPriority}}, @@ -1330,6 +1316,7 @@ func TestSelectNodesForPreemption(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin("MatchFilter", NewMatchFilterPlugin), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Priority: &highPriority}}, @@ -1344,6 +1331,7 @@ func TestSelectNodesForPreemption(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority}}, @@ -1358,6 +1346,7 @@ func TestSelectNodesForPreemption(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &lowPriority}}, @@ -1372,6 +1361,7 @@ func TestSelectNodesForPreemption(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority}}, @@ -1387,6 +1377,7 @@ func TestSelectNodesForPreemption(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority}}, @@ -1404,6 +1395,7 @@ func TestSelectNodesForPreemption(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority}}, @@ -1422,6 +1414,7 @@ func TestSelectNodesForPreemption(t *testing.T) { st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), st.RegisterFilterPlugin(interpodaffinity.Name, interpodaffinity.New), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{ @@ -1462,6 +1455,7 @@ func TestSelectNodesForPreemption(t *testing.T) { "PreFilter", "Filter", ), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"node-a/zone1", "node-b/zone1", "node-x/zone2"}, pod: &v1.Pod{ @@ -1539,6 +1533,7 @@ func TestSelectNodesForPreemption(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority}}, @@ -1579,14 +1574,6 @@ func TestSelectNodesForPreemption(t *testing.T) { nodes = append(nodes, node) } - registry := framework.Registry{} - // TODO: instantiate the plugins dynamically. - plugins := &schedulerapi.Plugins{ - QueueSort: &schedulerapi.PluginSet{}, - PreFilter: &schedulerapi.PluginSet{}, - Filter: &schedulerapi.PluginSet{}, - } - var pluginConfigs []schedulerapi.PluginConfig // For each test, prepend a FakeFilterPlugin. fakePlugin := fakeFilterPlugin{} fakePlugin.failedNodeReturnCodeMap = filterFailedNodeReturnCodeMap @@ -1596,14 +1583,13 @@ func TestSelectNodesForPreemption(t *testing.T) { return &fakePlugin, nil }, ) - registerFakeFilterFunc(®istry, plugins, pluginConfigs) - // Next, register other filter plugins defined in test struct. - for _, f := range test.registerPlugins { - f(®istry, plugins, pluginConfigs) - } + registerPlugins := append([]st.RegisterPluginFunc{registerFakeFilterFunc}, test.registerPlugins...) // Use a real snapshot since it's needed in some Filter Plugin (e.g., PodAffinity) snapshot := internalcache.NewSnapshot(test.pods, nodes) - fwk, _ := framework.NewFramework(registry, plugins, pluginConfigs, framework.WithSnapshotSharedLister(snapshot)) + fwk, err := st.NewFramework(registerPlugins, framework.WithSnapshotSharedLister(snapshot)) + if err != nil { + t.Fatal(err) + } scheduler := NewGenericScheduler( nil, @@ -1662,6 +1648,7 @@ func TestPickOneNodeForPreemption(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority}}, @@ -1674,6 +1661,7 @@ func TestPickOneNodeForPreemption(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority}}, @@ -1688,6 +1676,7 @@ func TestPickOneNodeForPreemption(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2", "machine3"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority}}, @@ -1702,6 +1691,7 @@ func TestPickOneNodeForPreemption(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2", "machine3"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: veryLargeContainers, Priority: &highPriority}}, @@ -1722,6 +1712,7 @@ func TestPickOneNodeForPreemption(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2", "machine3"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: veryLargeContainers, Priority: &highPriority}}, @@ -1742,6 +1733,7 @@ func TestPickOneNodeForPreemption(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2", "machine3"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: veryLargeContainers, Priority: &highPriority}}, @@ -1767,6 +1759,7 @@ func TestPickOneNodeForPreemption(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2", "machine3"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: veryLargeContainers, Priority: &highPriority}}, @@ -1789,6 +1782,7 @@ func TestPickOneNodeForPreemption(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2", "machine3", "machine4"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "pod1", UID: types.UID("pod1")}, Spec: v1.PodSpec{Containers: veryLargeContainers, Priority: &veryHighPriority}}, @@ -1816,6 +1810,7 @@ func TestPickOneNodeForPreemption(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2", "machine3"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: veryLargeContainers, Priority: &highPriority}}, @@ -1836,6 +1831,7 @@ func TestPickOneNodeForPreemption(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2", "machine3"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: veryLargeContainers, Priority: &highPriority}}, @@ -1856,6 +1852,7 @@ func TestPickOneNodeForPreemption(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, nodes: []string{"machine1", "machine2", "machine3"}, pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}, Spec: v1.PodSpec{Containers: veryLargeContainers, Priority: &highPriority}}, @@ -1879,17 +1876,10 @@ func TestPickOneNodeForPreemption(t *testing.T) { nodes = append(nodes, makeNode(n, schedutil.DefaultMilliCPURequest*5, schedutil.DefaultMemoryRequest*5)) } snapshot := internalcache.NewSnapshot(test.pods, nodes) - registry := framework.Registry{} - // TODO: instantiate the plugins dynamically. - plugins := &schedulerapi.Plugins{ - QueueSort: &schedulerapi.PluginSet{}, - Filter: &schedulerapi.PluginSet{}, + fwk, err := st.NewFramework(test.registerPlugins, framework.WithSnapshotSharedLister(snapshot)) + if err != nil { + t.Fatal(err) } - var pluginConfigs []schedulerapi.PluginConfig - for _, f := range test.registerPlugins { - f(®istry, plugins, pluginConfigs) - } - fwk, _ := framework.NewFramework(registry, plugins, pluginConfigs, framework.WithSnapshotSharedLister(snapshot)) g := &genericScheduler{ framework: fwk, @@ -2076,6 +2066,7 @@ func TestPreempt(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, expectedNode: "machine1", expectedPods: []string{"m1.1", "m1.2"}, @@ -2096,6 +2087,7 @@ func TestPreempt(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, expectedNode: "machine3", expectedPods: []string{}, @@ -2181,6 +2173,7 @@ func TestPreempt(t *testing.T) { "PreFilter", "Filter", ), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, expectedNode: "node-b", expectedPods: []string{"pod-b1"}, @@ -2209,6 +2202,7 @@ func TestPreempt(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, expectedNode: "machine1", expectedPods: []string{"m1.1", "m1.2"}, @@ -2234,6 +2228,7 @@ func TestPreempt(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, expectedNode: "", expectedPods: []string{}, @@ -2263,6 +2258,7 @@ func TestPreempt(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, expectedNode: "machine1", expectedPods: []string{"m1.1", "m1.2"}, @@ -2292,6 +2288,7 @@ func TestPreempt(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, expectedNode: "machine3", expectedPods: []string{}, @@ -2312,6 +2309,7 @@ func TestPreempt(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, expectedNode: "", expectedPods: nil, @@ -2332,6 +2330,7 @@ func TestPreempt(t *testing.T) { registerPlugins: []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), }, expectedNode: "machine1", expectedPods: []string{"m1.1", "m1.2"}, @@ -2380,19 +2379,11 @@ func TestPreempt(t *testing.T) { extenders = append(extenders, extender) } - registry := framework.Registry{} - // TODO: instantiate the plugins dynamically. - plugins := &schedulerapi.Plugins{ - QueueSort: &schedulerapi.PluginSet{}, - PreFilter: &schedulerapi.PluginSet{}, - Filter: &schedulerapi.PluginSet{}, - } - var pluginConfigs []schedulerapi.PluginConfig - for _, f := range test.registerPlugins { - f(®istry, plugins, pluginConfigs) - } snapshot := internalcache.NewSnapshot(test.pods, nodes) - fwk, _ := framework.NewFramework(registry, plugins, pluginConfigs, framework.WithSnapshotSharedLister(snapshot)) + fwk, err := st.NewFramework(test.registerPlugins, framework.WithSnapshotSharedLister(snapshot)) + if err != nil { + t.Fatal(err) + } scheduler := NewGenericScheduler( cache, @@ -2533,6 +2524,7 @@ func TestFairEvaluationForNodes(t *testing.T) { nodes, st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), ) // To make numAllNodes % nodesToFind != 0 g.percentageOfNodesToScore = 30 diff --git a/pkg/scheduler/factory.go b/pkg/scheduler/factory.go index 8df07f97826..32e3ac8f118 100644 --- a/pkg/scheduler/factory.go +++ b/pkg/scheduler/factory.go @@ -42,6 +42,7 @@ import ( "k8s.io/kubernetes/pkg/scheduler/apis/config/validation" "k8s.io/kubernetes/pkg/scheduler/core" frameworkplugins "k8s.io/kubernetes/pkg/scheduler/framework/plugins" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/queuesort" @@ -121,7 +122,7 @@ func (c *Configurator) create(extenders []core.SchedulerExtender) (*Scheduler, e framework.WithVolumeBinder(c.volumeBinder), ) if err != nil { - klog.Fatalf("error initializing the scheduling framework: %v", err) + return nil, fmt.Errorf("initializing the scheduling framework: %v", err) } podQueue := internalqueue.NewSchedulingQueue( @@ -287,11 +288,15 @@ func (c *Configurator) createFromConfig(policy schedulerapi.Policy) (*Scheduler, // Combine all framework configurations. If this results in any duplication, framework // instantiation should fail. var defaultPlugins schedulerapi.Plugins - // "PrioritySort" is neither a predicate nor priority before. We make it as the default QueueSort plugin. + // "PrioritySort" and "DefaultBinder" were neither predicates nor priorities + // before. We add them by default. defaultPlugins.Append(&schedulerapi.Plugins{ QueueSort: &schedulerapi.PluginSet{ Enabled: []schedulerapi.Plugin{{Name: queuesort.Name}}, }, + Bind: &schedulerapi.PluginSet{ + Enabled: []schedulerapi.Plugin{{Name: defaultbinder.Name}}, + }, }) defaultPlugins.Append(pluginsForPredicates) defaultPlugins.Append(pluginsForPriorities) diff --git a/pkg/scheduler/factory_test.go b/pkg/scheduler/factory_test.go index e34ce027ae9..c87732e16d4 100644 --- a/pkg/scheduler/factory_test.go +++ b/pkg/scheduler/factory_test.go @@ -24,6 +24,7 @@ import ( "testing" "time" + "github.com/google/go-cmp/cmp" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -32,7 +33,6 @@ import ( "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" - fakeV1 "k8s.io/client-go/kubernetes/typed/core/v1/fake" clienttesting "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" apitesting "k8s.io/kubernetes/pkg/api/testing" @@ -41,6 +41,7 @@ import ( "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme" extenderv1 "k8s.io/kubernetes/pkg/scheduler/apis/extender/v1" frameworkplugins "k8s.io/kubernetes/pkg/scheduler/framework/plugins" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodelabel" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/queuesort" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/serviceaffinity" @@ -109,8 +110,14 @@ func TestCreateFromConfig(t *testing.T) { t.Errorf("Wrong hardPodAffinitySymmetricWeight, ecpected: %d, got: %d", v1.DefaultHardPodAffinitySymmetricWeight, hpa) } queueSortPls := sched.Framework.ListPlugins()["QueueSortPlugin"] - if len(queueSortPls) == 0 || queueSortPls[0].Name != queuesort.Name { - t.Errorf("QueueSort plugin %q not registered.", queuesort.Name) + wantQueuePls := []schedulerapi.Plugin{{Name: queuesort.Name}} + if diff := cmp.Diff(wantQueuePls, queueSortPls); diff != "" { + t.Errorf("Unexpected QueueSort plugins (-want, +got): %s", diff) + } + bindPls := sched.Framework.ListPlugins()["BindPlugin"] + wantBindPls := []schedulerapi.Plugin{{Name: defaultbinder.Name}} + if diff := cmp.Diff(wantBindPls, bindPls); diff != "" { + t.Errorf("Unexpected Bind plugins (-want, +got): %s", diff) } // Verify that node label predicate/priority are converted to framework plugins. @@ -371,43 +378,6 @@ func testClientGetPodRequest(client *fake.Clientset, t *testing.T, podNs string, } } -// TODO(#87157): Move to DefaultBinding Plugin tests when it is introduced. -func TestDefaultBinding(t *testing.T) { - binding := &v1.Binding{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: metav1.NamespaceDefault, - Name: "foo", - }, - Target: v1.ObjectReference{ - Name: "foohost.kubernetes.mydomain.com", - }, - } - - testPod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: binding.GetName(), Namespace: metav1.NamespaceDefault}, - Spec: apitesting.V1DeepEqualSafePodSpec(), - } - client := fake.NewSimpleClientset(&v1.PodList{Items: []v1.Pod{*testPod}}) - - sched := Scheduler{client: client} - if err := sched.defaultBinding(binding); err != nil { - t.Errorf("Unexpected error: %v", err) - return - } - - pods := client.CoreV1().Pods(metav1.NamespaceDefault).(*fakeV1.FakePods) - actualBinding, err := pods.GetBinding(binding.GetName()) - if err != nil { - t.Fatalf("Unexpected error: %v", err) - return - } - if !reflect.DeepEqual(binding, actualBinding) { - t.Errorf("Binding did not match expectation") - t.Logf("Expected: %v", binding) - t.Logf("Actual: %v", actualBinding) - } -} - func newConfigFactoryWithFrameworkRegistry( client clientset.Interface, hardPodAffinitySymmetricWeight int32, stopCh <-chan struct{}, registry framework.Registry) *Configurator { diff --git a/pkg/scheduler/framework/plugins/BUILD b/pkg/scheduler/framework/plugins/BUILD index 6081076ae07..03dd18f3875 100644 --- a/pkg/scheduler/framework/plugins/BUILD +++ b/pkg/scheduler/framework/plugins/BUILD @@ -11,6 +11,7 @@ go_library( deps = [ "//pkg/features:go_default_library", "//pkg/scheduler/apis/config:go_default_library", + "//pkg/scheduler/framework/plugins/defaultbinder:go_default_library", "//pkg/scheduler/framework/plugins/defaultpodtopologyspread:go_default_library", "//pkg/scheduler/framework/plugins/imagelocality:go_default_library", "//pkg/scheduler/framework/plugins/interpodaffinity:go_default_library", @@ -48,6 +49,7 @@ filegroup( name = "all-srcs", srcs = [ ":package-srcs", + "//pkg/scheduler/framework/plugins/defaultbinder:all-srcs", "//pkg/scheduler/framework/plugins/defaultpodtopologyspread:all-srcs", "//pkg/scheduler/framework/plugins/examples:all-srcs", "//pkg/scheduler/framework/plugins/helper:all-srcs", diff --git a/pkg/scheduler/framework/plugins/defaultbinder/BUILD b/pkg/scheduler/framework/plugins/defaultbinder/BUILD new file mode 100644 index 00000000000..e39de91ed18 --- /dev/null +++ b/pkg/scheduler/framework/plugins/defaultbinder/BUILD @@ -0,0 +1,44 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["default_binder.go"], + importpath = "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder", + visibility = ["//visibility:public"], + deps = [ + "//pkg/scheduler/framework/v1alpha1:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["default_binder_test.go"], + embed = [":go_default_library"], + deps = [ + "//pkg/scheduler/framework/v1alpha1:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", + "//staging/src/k8s.io/client-go/testing:go_default_library", + "//vendor/github.com/google/go-cmp/cmp:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/scheduler/framework/plugins/defaultbinder/default_binder.go b/pkg/scheduler/framework/plugins/defaultbinder/default_binder.go new file mode 100644 index 00000000000..7ce88d296f5 --- /dev/null +++ b/pkg/scheduler/framework/plugins/defaultbinder/default_binder.go @@ -0,0 +1,61 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package defaultbinder + +import ( + "context" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog" + framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" +) + +// Name of the plugin used in the plugin registry and configurations. +const Name = "DefaultBinder" + +// DefaultBinder binds pods to nodes using a k8s client. +type DefaultBinder struct { + handle framework.FrameworkHandle +} + +var _ framework.BindPlugin = &DefaultBinder{} + +// New creates a DefaultBinder. +func New(_ *runtime.Unknown, handle framework.FrameworkHandle) (framework.Plugin, error) { + return &DefaultBinder{handle: handle}, nil +} + +// Name returns the name of the plugin. +func (b DefaultBinder) Name() string { + return Name +} + +// Bind binds pods to nodes using the k8s client. +func (b DefaultBinder) Bind(ctx context.Context, state *framework.CycleState, p *v1.Pod, nodeName string) *framework.Status { + klog.V(3).Infof("Attempting to bind %v/%v to %v", p.Namespace, p.Name, nodeName) + binding := &v1.Binding{ + ObjectMeta: metav1.ObjectMeta{Namespace: p.Namespace, Name: p.Name, UID: p.UID}, + Target: v1.ObjectReference{Kind: "Node", Name: nodeName}, + } + err := b.handle.ClientSet().CoreV1().Pods(binding.Namespace).Bind(binding) + if err != nil { + return framework.NewStatus(framework.Error, err.Error()) + } + return nil +} diff --git a/pkg/scheduler/framework/plugins/defaultbinder/default_binder_test.go b/pkg/scheduler/framework/plugins/defaultbinder/default_binder_test.go new file mode 100644 index 00000000000..ae765ab13fd --- /dev/null +++ b/pkg/scheduler/framework/plugins/defaultbinder/default_binder_test.go @@ -0,0 +1,83 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package defaultbinder + +import ( + "context" + "errors" + "testing" + + "github.com/google/go-cmp/cmp" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + clienttesting "k8s.io/client-go/testing" + framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" +) + +func TestDefaultBinder(t *testing.T) { + testPod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "ns"}, + } + testNode := "foohost.kubernetes.mydomain.com" + tests := []struct { + name string + injectErr error + wantBinding *v1.Binding + }{ + { + name: "successful", + wantBinding: &v1.Binding{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: "foo"}, + Target: v1.ObjectReference{Kind: "Node", Name: testNode}, + }, + }, { + name: "binding error", + injectErr: errors.New("binding error"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var gotBinding *v1.Binding + client := fake.NewSimpleClientset(testPod) + client.PrependReactor("create", "pods", func(action clienttesting.Action) (bool, runtime.Object, error) { + if action.GetSubresource() != "binding" { + return false, nil, nil + } + if tt.injectErr != nil { + return true, nil, tt.injectErr + } + gotBinding = action.(clienttesting.CreateAction).GetObject().(*v1.Binding) + return true, gotBinding, nil + }) + + fh, err := framework.NewFramework(nil, nil, nil, framework.WithClientSet(client)) + if err != nil { + t.Fatal(err) + } + binder := &DefaultBinder{handle: fh} + status := binder.Bind(context.Background(), nil, testPod, "foohost.kubernetes.mydomain.com") + if got := status.AsError(); (tt.injectErr != nil) != (got != nil) { + t.Errorf("got error %q, want %q", got, tt.injectErr) + } + if diff := cmp.Diff(tt.wantBinding, gotBinding); diff != "" { + t.Errorf("got different binding (-want, +got): %s", diff) + } + }) + } +} diff --git a/pkg/scheduler/framework/plugins/registry.go b/pkg/scheduler/framework/plugins/registry.go index cd4ad049711..d3dd8543c02 100644 --- a/pkg/scheduler/framework/plugins/registry.go +++ b/pkg/scheduler/framework/plugins/registry.go @@ -17,6 +17,7 @@ limitations under the License. package plugins import ( + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultpodtopologyspread" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/imagelocality" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity" @@ -70,5 +71,6 @@ func NewInTreeRegistry() framework.Registry { nodelabel.Name: nodelabel.New, serviceaffinity.Name: serviceaffinity.New, queuesort.Name: queuesort.New, + defaultbinder.Name: defaultbinder.New, } } diff --git a/pkg/scheduler/framework/v1alpha1/framework.go b/pkg/scheduler/framework/v1alpha1/framework.go index f7a182d8896..620cefa7556 100644 --- a/pkg/scheduler/framework/v1alpha1/framework.go +++ b/pkg/scheduler/framework/v1alpha1/framework.go @@ -251,10 +251,12 @@ func NewFramework(r Registry, plugins *config.Plugins, args []config.PluginConfi if len(f.queueSortPlugins) == 0 { return nil, fmt.Errorf("no queue sort plugin is enabled") } - if len(f.queueSortPlugins) > 1 { return nil, fmt.Errorf("only one queue sort plugin can be enabled") } + if len(f.bindPlugins) == 0 { + return nil, fmt.Errorf("at least one bind plugin is needed") + } return f, nil } @@ -653,7 +655,7 @@ func (f *framework) RunBindPlugins(ctx context.Context, state *CycleState, pod * continue } if !status.IsSuccess() { - msg := fmt.Sprintf("bind plugin %q failed to bind pod \"%v/%v\": %v", bp.Name(), pod.Namespace, pod.Name, status.Message()) + msg := fmt.Sprintf("plugin %q failed to bind pod \"%v/%v\": %v", bp.Name(), pod.Namespace, pod.Name, status.Message()) klog.Error(msg) return NewStatus(Error, msg) } diff --git a/pkg/scheduler/framework/v1alpha1/framework_test.go b/pkg/scheduler/framework/v1alpha1/framework_test.go index 7f508cdde4f..595ed2fec4a 100644 --- a/pkg/scheduler/framework/v1alpha1/framework_test.go +++ b/pkg/scheduler/framework/v1alpha1/framework_test.go @@ -46,6 +46,7 @@ const ( duplicatePluginName = "duplicate-plugin" testPlugin = "test-plugin" permitPlugin = "permit-plugin" + bindPlugin = "bind-plugin" ) // TestScoreWithNormalizePlugin implements ScoreWithNormalizePlugin interface. @@ -292,6 +293,23 @@ func (pl *TestQueueSortPlugin) Less(_, _ *PodInfo) bool { return false } +var _ BindPlugin = &TestBindPlugin{} + +func newBindPlugin(_ *runtime.Unknown, _ FrameworkHandle) (Plugin, error) { + return &TestBindPlugin{}, nil +} + +// TestBindPlugin is a no-op implementation for Bind extension point. +type TestBindPlugin struct{} + +func (t TestBindPlugin) Name() string { + return bindPlugin +} + +func (t TestBindPlugin) Bind(ctx context.Context, state *CycleState, p *v1.Pod, nodeName string) *Status { + return nil +} + var registry = func() Registry { r := make(Registry) r.Register(scoreWithNormalizePlugin1, newScoreWithNormalizePlugin1) @@ -318,19 +336,29 @@ var nodes = []*v1.Node{ {ObjectMeta: metav1.ObjectMeta{Name: "node2"}}, } -func newFrameworkWithQueueSortEnabled(r Registry, pl *config.Plugins, plc []config.PluginConfig, opts ...Option) (Framework, error) { +func newFrameworkWithQueueSortAndBind(r Registry, pl *config.Plugins, plc []config.PluginConfig, opts ...Option) (Framework, error) { if _, ok := r[queueSortPlugin]; !ok { r[queueSortPlugin] = newQueueSortPlugin } + if _, ok := r[bindPlugin]; !ok { + r[bindPlugin] = newBindPlugin + } plugins := &config.Plugins{} - plugins.Append(&config.Plugins{ - QueueSort: &config.PluginSet{ - Enabled: []config.Plugin{ - {Name: queueSortPlugin}, - }, - }, - }) plugins.Append(pl) + if plugins.QueueSort == nil || len(plugins.QueueSort.Enabled) == 0 { + plugins.Append(&config.Plugins{ + QueueSort: &config.PluginSet{ + Enabled: []config.Plugin{{Name: queueSortPlugin}}, + }, + }) + } + if plugins.Bind == nil || len(plugins.Bind.Enabled) == 0 { + plugins.Append(&config.Plugins{ + Bind: &config.PluginSet{ + Enabled: []config.Plugin{{Name: bindPlugin}}, + }, + }) + } return NewFramework(r, plugins, plc, opts...) } @@ -371,7 +399,7 @@ func TestInitFrameworkWithScorePlugins(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, err := newFrameworkWithQueueSortEnabled(registry, tt.plugins, emptyArgs) + _, err := newFrameworkWithQueueSortAndBind(registry, tt.plugins, emptyArgs) if tt.initErr && err == nil { t.Fatal("Framework initialization should fail") } @@ -567,7 +595,7 @@ func TestRunScorePlugins(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Inject the results via Args in PluginConfig. - f, err := newFrameworkWithQueueSortEnabled(registry, tt.plugins, tt.pluginConfigs) + f, err := newFrameworkWithQueueSortAndBind(registry, tt.plugins, tt.pluginConfigs) if err != nil { t.Fatalf("Failed to create framework for testing: %v", err) } @@ -605,7 +633,7 @@ func TestPreFilterPlugins(t *testing.T) { }) plugins := &config.Plugins{PreFilter: &config.PluginSet{Enabled: []config.Plugin{{Name: preFilterWithExtensionsPluginName}, {Name: preFilterPluginName}}}} t.Run("TestPreFilterPlugin", func(t *testing.T) { - f, err := newFrameworkWithQueueSortEnabled(r, plugins, emptyArgs) + f, err := newFrameworkWithQueueSortAndBind(r, plugins, emptyArgs) if err != nil { t.Fatalf("Failed to create framework for testing: %v", err) } @@ -831,7 +859,7 @@ func TestFilterPlugins(t *testing.T) { config.Plugin{Name: pl.name}) } - f, err := newFrameworkWithQueueSortEnabled(registry, cfgPls, emptyArgs, WithRunAllFilters(tt.runAllFilters)) + f, err := newFrameworkWithQueueSortAndBind(registry, cfgPls, emptyArgs, WithRunAllFilters(tt.runAllFilters)) if err != nil { t.Fatalf("fail to create framework: %s", err) } @@ -1036,7 +1064,7 @@ func TestRecordingMetrics(t *testing.T) { Unreserve: pluginSet, } recorder := newMetricsRecorder(100, time.Nanosecond) - f, err := newFrameworkWithQueueSortEnabled(r, plugins, emptyArgs, withMetricsRecorder(recorder)) + f, err := newFrameworkWithQueueSortAndBind(r, plugins, emptyArgs, withMetricsRecorder(recorder)) if err != nil { t.Fatalf("Failed to create framework for testing: %v", err) } @@ -1055,6 +1083,72 @@ func TestRecordingMetrics(t *testing.T) { } } +func TestRunBindPlugins(t *testing.T) { + tests := []struct { + name string + injects []Code + wantStatus Code + }{ + { + name: "simple success", + injects: []Code{Success}, + wantStatus: Success, + }, + { + name: "error on second", + injects: []Code{Skip, Error, Success}, + wantStatus: Error, + }, + { + name: "all skip", + injects: []Code{Skip, Skip, Skip}, + wantStatus: Skip, + }, + { + name: "error on third, but not reached", + injects: []Code{Skip, Success, Error}, + wantStatus: Success, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + metrics.Register() + metrics.FrameworkExtensionPointDuration.Reset() + metrics.PluginExecutionDuration.Reset() + + pluginSet := &config.PluginSet{} + r := make(Registry) + for i, inj := range tt.injects { + name := fmt.Sprintf("bind-%d", i) + plugin := &TestPlugin{name: name, inj: injectedResult{BindStatus: int(inj)}} + r.Register(name, + func(_ *runtime.Unknown, fh FrameworkHandle) (Plugin, error) { + return plugin, nil + }) + pluginSet.Enabled = append(pluginSet.Enabled, config.Plugin{Name: name}) + } + plugins := &config.Plugins{Bind: pluginSet} + recorder := newMetricsRecorder(100, time.Nanosecond) + fwk, err := newFrameworkWithQueueSortAndBind(r, plugins, emptyArgs, withMetricsRecorder(recorder)) + if err != nil { + t.Fatal(err) + } + + st := fwk.RunBindPlugins(context.Background(), state, pod, "") + if st.Code() != tt.wantStatus { + t.Errorf("got status code %s, want %s", st.Code(), tt.wantStatus) + } + + // Stop the goroutine which records metrics and ensure it's stopped. + close(recorder.stopCh) + <-recorder.isStoppedCh + // Try to clean up the metrics buffer again in case it's not empty. + recorder.flushMetrics() + collectAndCompareFrameworkMetrics(t, "Bind", tt.wantStatus) + }) + } +} + func TestPermitWaitingMetric(t *testing.T) { tests := []struct { name string @@ -1078,14 +1172,17 @@ func TestPermitWaitingMetric(t *testing.T) { plugin := &TestPlugin{name: testPlugin, inj: tt.inject} r := make(Registry) - r.Register(testPlugin, + err := r.Register(testPlugin, func(_ *runtime.Unknown, fh FrameworkHandle) (Plugin, error) { return plugin, nil }) + if err != nil { + t.Fatal(err) + } plugins := &config.Plugins{ Permit: &config.PluginSet{Enabled: []config.Plugin{{Name: testPlugin, Weight: 1}}}, } - f, err := newFrameworkWithQueueSortEnabled(r, plugins, emptyArgs) + f, err := newFrameworkWithQueueSortAndBind(r, plugins, emptyArgs) if err != nil { t.Fatalf("Failed to create framework for testing: %v", err) } @@ -1115,7 +1212,7 @@ func TestRejectWaitingPod(t *testing.T) { Permit: &config.PluginSet{Enabled: []config.Plugin{{Name: permitPlugin, Weight: 1}}}, } - f, err := newFrameworkWithQueueSortEnabled(r, plugins, emptyArgs) + f, err := newFrameworkWithQueueSortAndBind(r, plugins, emptyArgs) if err != nil { t.Fatalf("Failed to create framework for testing: %v", err) } @@ -1181,6 +1278,7 @@ func injectNormalizeRes(inj injectedResult, scores NodeScoreList) *Status { } func collectAndComparePluginMetrics(t *testing.T, wantExtensionPoint, wantPlugin string, wantStatus Code) { + t.Helper() m := collectHistogramMetric(metrics.PluginExecutionDuration) if len(m.Label) != 3 { t.Fatalf("Unexpected number of label pairs, got: %v, want: 2", len(m.Label)) @@ -1208,6 +1306,7 @@ func collectAndComparePluginMetrics(t *testing.T, wantExtensionPoint, wantPlugin } func collectAndCompareFrameworkMetrics(t *testing.T, wantExtensionPoint string, wantStatus Code) { + t.Helper() m := collectHistogramMetric(metrics.FrameworkExtensionPointDuration) if len(m.Label) != 2 { diff --git a/pkg/scheduler/framework/v1alpha1/interface.go b/pkg/scheduler/framework/v1alpha1/interface.go index 12e1626efc3..1c7fb523434 100644 --- a/pkg/scheduler/framework/v1alpha1/interface.go +++ b/pkg/scheduler/framework/v1alpha1/interface.go @@ -480,9 +480,9 @@ type Framework interface { // RunBindPlugins runs the set of configured bind plugins. A bind plugin may choose // whether or not to handle the given Pod. If a bind plugin chooses to skip the - // binding, it should return code=4("skip") status. Otherwise, it should return "Error" + // binding, it should return code=5("skip") status. Otherwise, it should return "Error" // or "Success". If none of the plugins handled binding, RunBindPlugins returns - // code=4("skip") status. + // code=5("skip") status. RunBindPlugins(ctx context.Context, state *CycleState, pod *v1.Pod, nodeName string) *Status // HasFilterPlugins returns true if at least one filter plugin is defined. diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index da83daa833b..d1045c92dda 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -118,9 +118,6 @@ type Scheduler struct { SchedulingQueue internalqueue.SchedulingQueue scheduledPodsHasSynced func() bool - - // TODO(#87157): Remove this when the DefaultBinding Plugin is introduced. - client clientset.Interface } // Cache returns the cache in scheduler for test to check the data in scheduler. @@ -333,7 +330,6 @@ func New(client clientset.Interface, sched.podConditionUpdater = &podConditionUpdaterImpl{client} sched.podPreemptor = &podPreemptorImpl{client} sched.scheduledPodsHasSynced = podInformer.Informer().HasSynced - sched.client = client AddAllEventHandlers(sched, options.schedulerName, informerFactory, podInformer) return sched, nil @@ -507,7 +503,7 @@ func (sched *Scheduler) assume(assumed *v1.Pod, host string) error { } // bind binds a pod to a given node defined in a binding object. -// The precedence for binding is: (1) extenders, (2) plugins and (3) default binding. +// The precedence for binding is: (1) extenders and (2) framework plugins. // We expect this to run asynchronously, so we handle binding metrics internally. func (sched *Scheduler) bind(ctx context.Context, assumed *v1.Pod, targetNode string, state *framework.CycleState) (err error) { start := time.Now() @@ -515,18 +511,7 @@ func (sched *Scheduler) bind(ctx context.Context, assumed *v1.Pod, targetNode st sched.finishBinding(assumed, targetNode, start, err) }() - binding := &v1.Binding{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: assumed.Namespace, - Name: assumed.Name, - UID: assumed.UID, - }, - Target: v1.ObjectReference{ - Kind: "Node", - Name: targetNode, - }, - } - bound, err := sched.extendersBinding(assumed, binding) + bound, err := sched.extendersBinding(assumed, targetNode) if bound { return err } @@ -534,32 +519,26 @@ func (sched *Scheduler) bind(ctx context.Context, assumed *v1.Pod, targetNode st if bindStatus.IsSuccess() { return nil } - if bindStatus.Code() != framework.Skip { - return fmt.Errorf("bind failure, code: %d: %v", bindStatus.Code(), bindStatus.Message()) + if bindStatus.Code() == framework.Error { + return bindStatus.AsError() } - // All bind plugins chose to skip binding of this pod, call original binding - // function. If binding succeeds then PodScheduled condition will be updated - // in apiserver so that it's atomic with setting host. - return sched.defaultBinding(binding) + return fmt.Errorf("bind status: %s, %v", bindStatus.Code().String(), bindStatus.Message()) } // TODO(#87159): Move this to a Plugin. -func (sched *Scheduler) extendersBinding(assumed *v1.Pod, binding *v1.Binding) (bool, error) { +func (sched *Scheduler) extendersBinding(pod *v1.Pod, node string) (bool, error) { for _, extender := range sched.Algorithm.Extenders() { - if !extender.IsBinder() || !extender.IsInterested(assumed) { + if !extender.IsBinder() || !extender.IsInterested(pod) { continue } - return true, extender.Bind(binding) + return true, extender.Bind(&v1.Binding{ + ObjectMeta: metav1.ObjectMeta{Namespace: pod.Namespace, Name: pod.Name, UID: pod.UID}, + Target: v1.ObjectReference{Kind: "Node", Name: node}, + }) } return false, nil } -// TODO(#87157): Move this to a Plugin. -func (sched *Scheduler) defaultBinding(binding *v1.Binding) error { - klog.V(3).Infof("Attempting to bind %v/%v to %v", binding.Namespace, binding.Name, binding.Target.Name) - return sched.client.CoreV1().Pods(binding.Namespace).Bind(binding) -} - func (sched *Scheduler) finishBinding(assumed *v1.Pod, targetNode string, start time.Time, err error) { if finErr := sched.SchedulerCache.FinishBinding(assumed); finErr != nil { klog.Errorf("scheduler cache FinishBinding failed: %v", finErr) diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index 904fc0732b4..c57d9f80394 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -47,6 +47,7 @@ import ( schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config" "k8s.io/kubernetes/pkg/scheduler/core" frameworkplugins "k8s.io/kubernetes/pkg/scheduler/framework/plugins" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeports" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/queuesort" @@ -59,13 +60,6 @@ import ( "k8s.io/kubernetes/pkg/scheduler/volumebinder" ) -var ( - emptyPluginRegistry = framework.Registry{} - // emptyFramework is an empty framework used in tests. - // Note: If the test runs in goroutine, please don't use this variable to avoid a race condition. - emptyFramework, _ = framework.NewFramework(emptyPluginRegistry, nil, nil) -) - type fakePodConditionUpdater struct{} func (fc fakePodConditionUpdater) update(pod *v1.Pod, podCondition *v1.PodCondition) error { @@ -205,7 +199,7 @@ func TestSchedulerCreation(t *testing.T) { } } -func TestScheduler(t *testing.T) { +func TestSchedulerScheduleOne(t *testing.T) { testNode := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "machine1", UID: types.UID("machine1")}} client := clientsetfake.NewSimpleClientset(&testNode) eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: client.EventsV1beta1().Events("")}) @@ -247,7 +241,7 @@ func TestScheduler(t *testing.T) { expectBind: &v1.Binding{ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: types.UID("foo")}, Target: v1.ObjectReference{Kind: "Node", Name: testNode.Name}}, expectAssumedPod: podWithID("foo", testNode.Name), injectBindError: errB, - expectError: errB, + expectError: errors.New("plugin \"DefaultBinder\" failed to bind pod \"/foo\": binder"), expectErrorPod: podWithID("foo", testNode.Name), expectForgetPod: podWithID("foo", testNode.Name), eventReason: "FailedScheduling", @@ -295,6 +289,13 @@ func TestScheduler(t *testing.T) { gotBinding = action.(clienttesting.CreateAction).GetObject().(*v1.Binding) return true, gotBinding, item.injectBindError }) + fwk, err := st.NewFramework([]st.RegisterPluginFunc{ + st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), + }, framework.WithClientSet(client)) + if err != nil { + t.Fatal(err) + } s := &Scheduler{ SchedulerCache: sCache, @@ -307,10 +308,9 @@ func TestScheduler(t *testing.T) { NextPod: func() *framework.PodInfo { return &framework.PodInfo{Pod: item.sendPod} }, - Framework: emptyFramework, + Framework: fwk, Recorder: eventBroadcaster.NewRecorder(scheme.Scheme, "scheduler"), VolumeBinder: volumebinder.NewFakeVolumeBinder(&volumescheduling.FakeVolumeBinderConfig{AllBound: true}), - client: client, } called := make(chan struct{}) stopFunc := eventBroadcaster.StartEventWatcher(func(obj runtime.Object) { @@ -355,6 +355,7 @@ func TestSchedulerNoPhantomPodAfterExpire(t *testing.T) { fns := []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), st.RegisterPluginAsExtensions(nodeports.Name, 1, nodeports.New, "Filter", "PreFilter"), } scheduler, bindingChan, _ := setupTestSchedulerWithOnePodOnNode(t, queuedPodStore, scache, informerFactory, stop, pod, &node, fns...) @@ -421,6 +422,7 @@ func TestSchedulerNoPhantomPodAfterDelete(t *testing.T) { informerFactory := informers.NewSharedInformerFactory(client, 0) fns := []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), st.RegisterPluginAsExtensions(nodeports.Name, 1, nodeports.New, "Filter", "PreFilter"), } scheduler, bindingChan, errChan := setupTestSchedulerWithOnePodOnNode(t, queuedPodStore, scache, informerFactory, stop, firstPod, &node, fns...) @@ -566,6 +568,7 @@ func TestSchedulerFailedSchedulingReasons(t *testing.T) { } fns := []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), st.RegisterFilterPlugin("PodFitsResources", noderesources.NewFit), } scheduler, _, errChan := setupTestScheduler(queuedPodStore, scache, informerFactory, nil, nil, fns...) @@ -596,22 +599,22 @@ func TestSchedulerFailedSchedulingReasons(t *testing.T) { // queuedPodStore: pods queued before processing. // scache: scheduler cache that might contain assumed pods. func setupTestScheduler(queuedPodStore *clientcache.FIFO, scache internalcache.Cache, informerFactory informers.SharedInformerFactory, recorder events.EventRecorder, fakeVolumeBinder *volumebinder.VolumeBinder, fns ...st.RegisterPluginFunc) (*Scheduler, chan *v1.Binding, chan error) { - registry := framework.Registry{} if fakeVolumeBinder == nil { // Create default volume binder if it didn't set. fakeVolumeBinder = volumebinder.NewFakeVolumeBinder(&volumescheduling.FakeVolumeBinderConfig{AllBound: true}) } - // TODO: instantiate the plugins dynamically. - plugins := &schedulerapi.Plugins{ - QueueSort: &schedulerapi.PluginSet{}, - PreFilter: &schedulerapi.PluginSet{}, - Filter: &schedulerapi.PluginSet{}, - } - var pluginConfigs []schedulerapi.PluginConfig - for _, f := range fns { - f(®istry, plugins, pluginConfigs) - } - fwk, _ := framework.NewFramework(registry, plugins, pluginConfigs, framework.WithVolumeBinder(fakeVolumeBinder)) + bindingChan := make(chan *v1.Binding, 1) + client := clientsetfake.NewSimpleClientset() + client.PrependReactor("create", "pods", func(action clienttesting.Action) (bool, runtime.Object, error) { + var b *v1.Binding + if action.GetSubresource() == "binding" { + b := action.(clienttesting.CreateAction).GetObject().(*v1.Binding) + bindingChan <- b + } + return true, b, nil + }) + + fwk, _ := st.NewFramework(fns, framework.WithClientSet(client), framework.WithVolumeBinder(fakeVolumeBinder)) algo := core.NewGenericScheduler( scache, internalqueue.NewSchedulingQueue(nil), @@ -625,19 +628,8 @@ func setupTestScheduler(queuedPodStore *clientcache.FIFO, scache internalcache.C schedulerapi.DefaultPercentageOfNodesToScore, false, ) - bindingChan := make(chan *v1.Binding, 1) + errChan := make(chan error, 1) - - client := clientsetfake.NewSimpleClientset() - client.PrependReactor("create", "pods", func(action clienttesting.Action) (bool, runtime.Object, error) { - var b *v1.Binding - if action.GetSubresource() == "binding" { - b := action.(clienttesting.CreateAction).GetObject().(*v1.Binding) - bindingChan <- b - } - return true, b, nil - }) - sched := &Scheduler{ SchedulerCache: scache, Algorithm: algo, @@ -652,7 +644,6 @@ func setupTestScheduler(queuedPodStore *clientcache.FIFO, scache internalcache.C podPreemptor: fakePodPreemptor{}, Framework: fwk, VolumeBinder: fakeVolumeBinder, - client: client, } if recorder != nil { @@ -679,6 +670,7 @@ func setupTestSchedulerWithVolumeBinding(fakeVolumeBinder *volumebinder.VolumeBi recorder := broadcaster.NewRecorder(scheme.Scheme, "scheduler") fns := []st.RegisterPluginFunc{ st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), st.RegisterFilterPlugin(volumebinding.Name, volumebinding.New), } s, bindingChan, errChan := setupTestScheduler(queuedPodStore, scache, informerFactory, recorder, fakeVolumeBinder, fns...) @@ -988,7 +980,10 @@ func TestSchedulerBinding(t *testing.T) { } return false, nil, nil }) - fwk, err := framework.NewFramework(framework.Registry{}, nil, nil) + fwk, err := st.NewFramework([]st.RegisterPluginFunc{ + st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), + st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), + }, framework.WithClientSet(client)) if err != nil { t.Fatal(err) } @@ -1013,7 +1008,6 @@ func TestSchedulerBinding(t *testing.T) { Framework: fwk, Recorder: &events.FakeRecorder{}, SchedulerCache: scache, - client: client, } err = sched.bind(context.Background(), pod, "node", nil) if err != nil { diff --git a/pkg/scheduler/testing/framework_helpers.go b/pkg/scheduler/testing/framework_helpers.go index 3c08ddf5906..f91ac483883 100644 --- a/pkg/scheduler/testing/framework_helpers.go +++ b/pkg/scheduler/testing/framework_helpers.go @@ -21,6 +21,17 @@ import ( framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" ) +// NewFramework creates a Framework from the register functions and options. +func NewFramework(fns []RegisterPluginFunc, opts ...framework.Option) (framework.Framework, error) { + registry := framework.Registry{} + plugins := &schedulerapi.Plugins{} + var pluginConfigs []schedulerapi.PluginConfig + for _, f := range fns { + f(®istry, plugins, pluginConfigs) + } + return framework.NewFramework(registry, plugins, pluginConfigs, opts...) +} + // RegisterPluginFunc is a function signature used in method RegisterFilterPlugin() // to register a Filter Plugin to a given registry. type RegisterPluginFunc func(reg *framework.Registry, plugins *schedulerapi.Plugins, pluginConfigs []schedulerapi.PluginConfig) @@ -45,16 +56,21 @@ func RegisterPostFilterPlugin(pluginName string, pluginNewFunc framework.PluginF return RegisterPluginAsExtensions(pluginName, 1, pluginNewFunc, "PostFilter") } +// RegisterBindPlugin returns a function to register a Bind Plugin to a given registry. +func RegisterBindPlugin(pluginName string, pluginNewFunc framework.PluginFactory) RegisterPluginFunc { + return RegisterPluginAsExtensions(pluginName, 1, pluginNewFunc, "Bind") +} + // RegisterPluginAsExtensions returns a function to register a Plugin as given extensionPoints to a given registry. func RegisterPluginAsExtensions(pluginName string, weight int32, pluginNewFunc framework.PluginFactory, extensions ...string) RegisterPluginFunc { return func(reg *framework.Registry, plugins *schedulerapi.Plugins, pluginConfigs []schedulerapi.PluginConfig) { reg.Register(pluginName, pluginNewFunc) for _, extension := range extensions { - pluginSet := getPluginSetByExtension(plugins, extension) - if pluginSet == nil { + ps := getPluginSetByExtension(plugins, extension) + if ps == nil { continue } - pluginSet.Enabled = append(pluginSet.Enabled, schedulerapi.Plugin{Name: pluginName, Weight: weight}) + ps.Enabled = append(ps.Enabled, schedulerapi.Plugin{Name: pluginName, Weight: weight}) } //lint:ignore SA4006 this value of pluginConfigs is never used. //lint:ignore SA4010 this result of append is never used. @@ -65,22 +81,29 @@ func RegisterPluginAsExtensions(pluginName string, weight int32, pluginNewFunc f func getPluginSetByExtension(plugins *schedulerapi.Plugins, extension string) *schedulerapi.PluginSet { switch extension { case "QueueSort": - return plugins.QueueSort + return initializeIfNeeded(&plugins.QueueSort) case "Filter": - return plugins.Filter + return initializeIfNeeded(&plugins.Filter) case "PreFilter": - return plugins.PreFilter + return initializeIfNeeded(&plugins.PreFilter) case "PostFilter": - return plugins.PostFilter + return initializeIfNeeded(&plugins.PostFilter) case "Score": - return plugins.Score + return initializeIfNeeded(&plugins.Score) case "Bind": - return plugins.Bind + return initializeIfNeeded(&plugins.Bind) case "Reserve": - return plugins.Reserve + return initializeIfNeeded(&plugins.Reserve) case "Permit": - return plugins.Permit + return initializeIfNeeded(&plugins.Permit) default: return nil } } + +func initializeIfNeeded(s **schedulerapi.PluginSet) *schedulerapi.PluginSet { + if *s == nil { + *s = &schedulerapi.PluginSet{} + } + return *s +} diff --git a/test/integration/scheduler/BUILD b/test/integration/scheduler/BUILD index c98e062f462..8a13dd3a743 100644 --- a/test/integration/scheduler/BUILD +++ b/test/integration/scheduler/BUILD @@ -31,6 +31,7 @@ go_test( "//pkg/scheduler:go_default_library", "//pkg/scheduler/apis/config:go_default_library", "//pkg/scheduler/apis/extender/v1:go_default_library", + "//pkg/scheduler/framework/plugins/defaultbinder:go_default_library", "//pkg/scheduler/framework/v1alpha1:go_default_library", "//pkg/scheduler/nodeinfo:go_default_library", "//pkg/scheduler/testing:go_default_library", diff --git a/test/integration/scheduler/framework_test.go b/test/integration/scheduler/framework_test.go index 25e4c299428..c3562c576d6 100644 --- a/test/integration/scheduler/framework_test.go +++ b/test/integration/scheduler/framework_test.go @@ -30,6 +30,7 @@ import ( clientset "k8s.io/client-go/kubernetes" scheduler "k8s.io/kubernetes/pkg/scheduler" schedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" ) @@ -871,7 +872,9 @@ func TestBindPlugin(t *testing.T) { Enabled: []schedulerconfig.Plugin{{Name: unreservePlugin.Name()}}, }, Bind: &schedulerconfig.PluginSet{ - Enabled: []schedulerconfig.Plugin{{Name: bindPlugin1.Name()}, {Name: bindPlugin2.Name()}}, + // Put DefaultBinder last. + Enabled: []schedulerconfig.Plugin{{Name: bindPlugin1.Name()}, {Name: bindPlugin2.Name()}, {Name: defaultbinder.Name}}, + Disabled: []schedulerconfig.Plugin{{Name: defaultbinder.Name}}, }, PostBind: &schedulerconfig.PluginSet{ Enabled: []schedulerconfig.Plugin{{Name: postBindPlugin.Name()}}, diff --git a/test/integration/scheduler/scheduler_test.go b/test/integration/scheduler/scheduler_test.go index 567404e796e..0eace3381c2 100644 --- a/test/integration/scheduler/scheduler_test.go +++ b/test/integration/scheduler/scheduler_test.go @@ -91,6 +91,7 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) { "ScorePlugin": { {Name: "ImageLocality", Weight: 1}, }, + "BindPlugin": {{Name: "DefaultBinder"}}, }, }, { @@ -136,6 +137,7 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) { {Name: "DefaultPodTopologySpread", Weight: 1}, {Name: "TaintToleration", Weight: 1}, }, + "BindPlugin": {{Name: "DefaultBinder"}}, }, }, { @@ -151,6 +153,7 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) { {Name: "NodeUnschedulable"}, {Name: "TaintToleration"}, }, + "BindPlugin": {{Name: "DefaultBinder"}}, }, }, { @@ -175,6 +178,7 @@ priorities: "ScorePlugin": { {Name: "ImageLocality", Weight: 1}, }, + "BindPlugin": {{Name: "DefaultBinder"}}, }, }, { @@ -219,6 +223,7 @@ kind: Policy {Name: "DefaultPodTopologySpread", Weight: 1}, {Name: "TaintToleration", Weight: 1}, }, + "BindPlugin": {{Name: "DefaultBinder"}}, }, }, { @@ -233,6 +238,7 @@ priorities: [] {Name: "NodeUnschedulable"}, {Name: "TaintToleration"}, }, + "BindPlugin": {{Name: "DefaultBinder"}}, }, }, } {