Merge pull request #89338 from skilxn-go/AddCompatibilityTestForPluginArgs

Add compatibility tests for plugin arguments

pkg/scheduler/apis/config/testing/compatibility_test.go

@@ -25,6 +25,7 @@ import (
 
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes/fake"
@@ -1534,62 +1535,159 @@ func TestAlgorithmProviderCompatibility(t *testing.T) {
 }
 
 func TestPluginsConfigurationCompatibility(t *testing.T) {
+	defaultPlugins := map[string][]config.Plugin{
+		"QueueSortPlugin": {
+			{Name: "PrioritySort"},
+		},
+		"PreFilterPlugin": {
+			{Name: "NodeResourcesFit"},
+			{Name: "NodePorts"},
+			{Name: "InterPodAffinity"},
+			{Name: "PodTopologySpread"},
+		},
+		"FilterPlugin": {
+			{Name: "NodeUnschedulable"},
+			{Name: "NodeResourcesFit"},
+			{Name: "NodeName"},
+			{Name: "NodePorts"},
+			{Name: "NodeAffinity"},
+			{Name: "VolumeRestrictions"},
+			{Name: "TaintToleration"},
+			{Name: "EBSLimits"},
+			{Name: "GCEPDLimits"},
+			{Name: "NodeVolumeLimits"},
+			{Name: "AzureDiskLimits"},
+			{Name: "VolumeBinding"},
+			{Name: "VolumeZone"},
+			{Name: "InterPodAffinity"},
+			{Name: "PodTopologySpread"},
+		},
+		"PreScorePlugin": {
+			{Name: "InterPodAffinity"},
+			{Name: "DefaultPodTopologySpread"},
+			{Name: "TaintToleration"},
+			{Name: "PodTopologySpread"},
+		},
+		"ScorePlugin": {
+			{Name: "NodeResourcesBalancedAllocation", Weight: 1},
+			{Name: "ImageLocality", Weight: 1},
+			{Name: "InterPodAffinity", Weight: 1},
+			{Name: "NodeResourcesLeastAllocated", Weight: 1},
+			{Name: "NodeAffinity", Weight: 1},
+			{Name: "NodePreferAvoidPods", Weight: 10000},
+			{Name: "DefaultPodTopologySpread", Weight: 1},
+			{Name: "TaintToleration", Weight: 1},
+			{Name: "PodTopologySpread", Weight: 1},
+		},
+		"BindPlugin": {{Name: "DefaultBinder"}},
+	}
+
 	testcases := []struct {
-		name        string
+		name         string
-		plugins     config.Plugins
+		plugins      config.Plugins
-		wantPlugins map[string][]config.Plugin
+		wantPlugins  map[string][]config.Plugin
+		pluginConfig []config.PluginConfig
 	}{
 		{
-			name: "No plugins specified",
+			name:        "default plugins",
-			wantPlugins: map[string][]config.Plugin{
+			wantPlugins: defaultPlugins,
-				"QueueSortPlugin": {
+		},
-					{Name: "PrioritySort"},
+		{
+			name:        "default plugins with customized plugin config",
+			wantPlugins: defaultPlugins,
+			pluginConfig: []config.PluginConfig{
+				{
+					Name: "NodeResourcesFit",
+					Args: runtime.Unknown{
+						Raw: []byte(`{
+							"ignoredResources": [
+								"foo",
+								"bar"
+							]
+						}`),
+					},
 				},
-				"PreFilterPlugin": {
+				{
-					{Name: "NodeResourcesFit"},
+					Name: "PodTopologySpread",
-					{Name: "NodePorts"},
+					Args: runtime.Unknown{
-					{Name: "InterPodAffinity"},
+						Raw: []byte(`{
-					{Name: "PodTopologySpread"},
+							"defaultConstraints": [
+								{
+									"maxSkew": 1,
+									"topologyKey": "foo",
+									"whenUnsatisfiable": "DoNotSchedule"
+								},
+								{
+									"maxSkew": 10,
+									"topologyKey": "bar",
+									"whenUnsatisfiable": "ScheduleAnyway"
+								}
+							]
+						}`),
+					},
 				},
-				"FilterPlugin": {
+				{
-					{Name: "NodeUnschedulable"},
+					Name: "RequestedToCapacityRatio",
-					{Name: "NodeResourcesFit"},
+					Args: runtime.Unknown{
-					{Name: "NodeName"},
+						Raw: []byte(`{
-					{Name: "NodePorts"},
+							"shape":[
-					{Name: "NodeAffinity"},
+								"Utilization": 5,
-					{Name: "VolumeRestrictions"},
+								"Score": 5
-					{Name: "TaintToleration"},
+							],
-					{Name: "EBSLimits"},
+							"resources":[
-					{Name: "GCEPDLimits"},
+								"Name": "cpu",
-					{Name: "NodeVolumeLimits"},
+								"Weight": 10
-					{Name: "AzureDiskLimits"},
+							]
-					{Name: "VolumeBinding"},
+						}`),
-					{Name: "VolumeZone"},
+					},
-					{Name: "InterPodAffinity"},
-					{Name: "PodTopologySpread"},
 				},
-				"PreScorePlugin": {
+				{
-					{Name: "InterPodAffinity"},
+					Name: "InterPodAffinity",
-					{Name: "DefaultPodTopologySpread"},
+					Args: runtime.Unknown{
-					{Name: "TaintToleration"},
+						Raw: []byte(`{
-					{Name: "PodTopologySpread"},
+							"HardPodAffinityWeight": 100
+						}`),
+					},
 				},
-				"ScorePlugin": {
+				{
-					{Name: "NodeResourcesBalancedAllocation", Weight: 1},
+					Name: "NodeLabel",
-					{Name: "ImageLocality", Weight: 1},
+					Args: runtime.Unknown{
-					{Name: "InterPodAffinity", Weight: 1},
+						Raw: []byte(`{
-					{Name: "NodeResourcesLeastAllocated", Weight: 1},
+							"presentLabels": [
-					{Name: "NodeAffinity", Weight: 1},
+								"foo",
-					{Name: "NodePreferAvoidPods", Weight: 10000},
+								"bar"
-					{Name: "DefaultPodTopologySpread", Weight: 1},
+							],
-					{Name: "TaintToleration", Weight: 1},
+							"absentLabels": [
-					{Name: "PodTopologySpread", Weight: 1},
+								"apple"
+							],
+							"presentLabelsPreference": [
+								"dog"
+							],
+							"absentLabelsPreference": [
+								"cat"
+							]
+						}`),
+					},
+				},
+				{
+					Name: "ServiceAffinity",
+					Args: runtime.Unknown{
+						Raw: []byte(`{
+							affinityLabels: [
+								"foo",
+								"bar"
+							],
+							antiAffinityLabelsPreference: [
+								"disk",
+								"flash"
+							]
+						}`),
+					},
 				},
-				"BindPlugin": {{Name: "DefaultBinder"}},
 			},
 		},
 		{
-			name: "Disable some default plugins",
+			name: "disable some default plugins",
 			plugins: config.Plugins{
 				PreFilter: &config.PluginSet{
 					Disabled: []config.Plugin{
@@ -1648,7 +1746,7 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
 			},
 		},
 		{
-			name: "Reverse default plugins order with changing score weight",
+			name: "reverse default plugins order with changing score weight",
 			plugins: config.Plugins{
 				QueueSort: &config.PluginSet{
 					Enabled: []config.Plugin{
@@ -1779,6 +1877,15 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
 				scheduler.WithProfiles(config.KubeSchedulerProfile{
 					SchedulerName: v1.DefaultSchedulerName,
 					Plugins:       &tc.plugins,
+					PluginConfig:  tc.pluginConfig,
+				}),
+				scheduler.WithBuildFrameworkCapturer(func(p config.KubeSchedulerProfile) {
+					if p.SchedulerName != v1.DefaultSchedulerName {
+						t.Errorf("unexpected scheduler name (want %q, got %q)", v1.DefaultSchedulerName, p.SchedulerName)
+					}
+					if diff := cmp.Diff(tc.pluginConfig, p.PluginConfig); diff != "" {
+						t.Errorf("unexpected plugins diff (-want, +got): %s", diff)
+					}
 				}),
 			)
 
@@ -1786,7 +1893,7 @@ func TestPluginsConfigurationCompatibility(t *testing.T) {
 				t.Fatalf("Error constructing: %v", err)
 			}
 
-			defProf := sched.Profiles["default-scheduler"]
+			defProf := sched.Profiles[v1.DefaultSchedulerName]
 			gotPlugins := defProf.ListPlugins()
 			if diff := cmp.Diff(tc.wantPlugins, gotPlugins); diff != "" {
 				t.Errorf("unexpected plugins diff (-want, +got): %s", diff)

pkg/scheduler/factory.go

@@ -104,13 +104,17 @@ type Configurator struct {
 
 	enableNonPreempting bool
 
-	profiles         []schedulerapi.KubeSchedulerProfile
+	profiles          []schedulerapi.KubeSchedulerProfile
-	registry         framework.Registry
+	registry          framework.Registry
-	nodeInfoSnapshot *internalcache.Snapshot
+	nodeInfoSnapshot  *internalcache.Snapshot
-	extenders        []schedulerapi.Extender
+	extenders         []schedulerapi.Extender
+	frameworkCapturer FrameworkCapturer
 }
 
 func (c *Configurator) buildFramework(p schedulerapi.KubeSchedulerProfile) (framework.Framework, error) {
+	if c.frameworkCapturer != nil {
+		c.frameworkCapturer(p)
+	}
 	return framework.NewFramework(
 		c.registry,
 		p.Plugins,

pkg/scheduler/framework/plugins/queuesort/priority_sort.go

@@ -45,6 +45,6 @@ func (pl *PrioritySort) Less(pInfo1, pInfo2 *framework.PodInfo) bool {
 }
 
 // New initializes a new plugin and returns it.
-func New(plArgs *runtime.Unknown, handle framework.FrameworkHandle) (framework.Plugin, error) {
+func New(_ *runtime.Unknown, handle framework.FrameworkHandle) (framework.Plugin, error) {
 	return &PrioritySort{}, nil
 }

pkg/scheduler/scheduler.go

@@ -134,6 +134,7 @@ type schedulerOptions struct {
 	frameworkOutOfTreeRegistry framework.Registry
 	profiles                   []schedulerapi.KubeSchedulerProfile
 	extenders                  []schedulerapi.Extender
+	frameworkCapturer          FrameworkCapturer
 }
 
 // Option configures a Scheduler
@@ -204,6 +205,16 @@ func WithExtenders(e ...schedulerapi.Extender) Option {
 	}
 }
 
+// FrameworkCapturer is used for registering a notify function in building framework.
+type FrameworkCapturer func(schedulerapi.KubeSchedulerProfile)
+
+// WithBuildFrameworkCapturer sets a notify function for getting buildFramework details.
+func WithBuildFrameworkCapturer(fc FrameworkCapturer) Option {
+	return func(o *schedulerOptions) {
+		o.frameworkCapturer = fc
+	}
+}
+
 var defaultSchedulerOptions = schedulerOptions{
 	profiles: []schedulerapi.KubeSchedulerProfile{
 		// Profiles' default plugins are set from the algorithm provider.
@@ -273,6 +284,7 @@ func New(client clientset.Interface,
 		registry:                 registry,
 		nodeInfoSnapshot:         snapshot,
 		extenders:                options.extenders,
+		frameworkCapturer:        options.frameworkCapturer,
 	}
 
 	metrics.Register()