From 715068312a163d342d36f9a5b851a78f4fb76fa4 Mon Sep 17 00:00:00 2001 From: draveness Date: Fri, 8 Nov 2019 14:40:21 +0800 Subject: [PATCH] feat(scheduler): convert InterPodAffinity to score plugin --- .../apis/config/testing/compatibility_test.go | 71 +- .../framework/plugins/default_registry.go | 5 + .../framework/plugins/interpodaffinity/BUILD | 4 + .../interpodaffinity/interpod_affinity.go | 32 +- .../interpod_affinity_test.go | 649 +++++++++++++++++- test/integration/scheduler/scheduler_test.go | 10 +- 6 files changed, 723 insertions(+), 48 deletions(-) diff --git a/pkg/scheduler/apis/config/testing/compatibility_test.go b/pkg/scheduler/apis/config/testing/compatibility_test.go index e6eff445530..1508e3bdaea 100644 --- a/pkg/scheduler/apis/config/testing/compatibility_test.go +++ b/pkg/scheduler/apis/config/testing/compatibility_test.go @@ -261,10 +261,8 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {"name": "InterPodAffinityPriority", "weight": 2} ] }`, - wantPredicates: sets.NewString(), - wantPrioritizers: sets.NewString( - "InterPodAffinityPriority", - ), + wantPredicates: sets.NewString(), + wantPrioritizers: sets.NewString(), wantPlugins: map[string][]config.Plugin{ "FilterPlugin": { {Name: "NodeUnschedulable"}, @@ -285,6 +283,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "ScorePlugin": { {Name: "NodeResourcesBalancedAllocation", Weight: 2}, {Name: "ImageLocality", Weight: 2}, + {Name: "InterPodAffinity", Weight: 2}, {Name: "NodeResourcesLeastAllocated", Weight: 2}, {Name: "NodeAffinity", Weight: 2}, {Name: "DefaultPodTopologySpread", Weight: 2}, @@ -327,10 +326,8 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { {"name": "MostRequestedPriority", "weight": 2} ] }`, - wantPredicates: sets.NewString(), - wantPrioritizers: sets.NewString( - "InterPodAffinityPriority", - ), + wantPredicates: sets.NewString(), + wantPrioritizers: sets.NewString(), wantPlugins: map[string][]config.Plugin{ "FilterPlugin": { {Name: "NodeUnschedulable"}, @@ -351,6 +348,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "ScorePlugin": { {Name: "NodeResourcesBalancedAllocation", Weight: 2}, {Name: "ImageLocality", Weight: 2}, + {Name: "InterPodAffinity", Weight: 2}, {Name: "NodeResourcesLeastAllocated", Weight: 2}, {Name: "NodeResourcesMostAllocated", Weight: 2}, {Name: "NodeAffinity", Weight: 2}, @@ -404,10 +402,8 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "nodeCacheCapable": true }] }`, - wantPredicates: sets.NewString(), - wantPrioritizers: sets.NewString( - "InterPodAffinityPriority", - ), + wantPredicates: sets.NewString(), + wantPrioritizers: sets.NewString(), wantPlugins: map[string][]config.Plugin{ "FilterPlugin": { {Name: "NodeUnschedulable"}, @@ -428,6 +424,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "ScorePlugin": { {Name: "NodeResourcesBalancedAllocation", Weight: 2}, {Name: "ImageLocality", Weight: 2}, + {Name: "InterPodAffinity", Weight: 2}, {Name: "NodeResourcesLeastAllocated", Weight: 2}, {Name: "NodeResourcesMostAllocated", Weight: 2}, {Name: "NodeAffinity", Weight: 2}, @@ -492,10 +489,8 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "nodeCacheCapable": true }] }`, - wantPredicates: sets.NewString(), - wantPrioritizers: sets.NewString( - "InterPodAffinityPriority", - ), + wantPredicates: sets.NewString(), + wantPrioritizers: sets.NewString(), wantPlugins: map[string][]config.Plugin{ "FilterPlugin": { {Name: "NodeUnschedulable"}, @@ -516,6 +511,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "ScorePlugin": { {Name: "NodeResourcesBalancedAllocation", Weight: 2}, {Name: "ImageLocality", Weight: 2}, + {Name: "InterPodAffinity", Weight: 2}, {Name: "NodeResourcesLeastAllocated", Weight: 2}, {Name: "NodeResourcesMostAllocated", Weight: 2}, {Name: "NodeAffinity", Weight: 2}, @@ -581,10 +577,8 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "nodeCacheCapable": true }] }`, - wantPredicates: sets.NewString(), - wantPrioritizers: sets.NewString( - "InterPodAffinityPriority", - ), + wantPredicates: sets.NewString(), + wantPrioritizers: sets.NewString(), wantPlugins: map[string][]config.Plugin{ "FilterPlugin": { {Name: "NodeUnschedulable"}, @@ -606,6 +600,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "ScorePlugin": { {Name: "NodeResourcesBalancedAllocation", Weight: 2}, {Name: "ImageLocality", Weight: 2}, + {Name: "InterPodAffinity", Weight: 2}, {Name: "NodeResourcesLeastAllocated", Weight: 2}, {Name: "NodeResourcesMostAllocated", Weight: 2}, {Name: "NodeAffinity", Weight: 2}, @@ -674,10 +669,8 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "ignorable":true }] }`, - wantPredicates: sets.NewString(), - wantPrioritizers: sets.NewString( - "InterPodAffinityPriority", - ), + wantPredicates: sets.NewString(), + wantPrioritizers: sets.NewString(), wantPlugins: map[string][]config.Plugin{ "FilterPlugin": { {Name: "NodeUnschedulable"}, @@ -699,6 +692,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "ScorePlugin": { {Name: "NodeResourcesBalancedAllocation", Weight: 2}, {Name: "ImageLocality", Weight: 2}, + {Name: "InterPodAffinity", Weight: 2}, {Name: "NodeResourcesLeastAllocated", Weight: 2}, {Name: "NodeResourcesMostAllocated", Weight: 2}, {Name: "NodeAffinity", Weight: 2}, @@ -779,10 +773,8 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "ignorable":true }] }`, - wantPredicates: sets.NewString(), - wantPrioritizers: sets.NewString( - "InterPodAffinityPriority", - ), + wantPredicates: sets.NewString(), + wantPrioritizers: sets.NewString(), wantPlugins: map[string][]config.Plugin{ "FilterPlugin": { {Name: "NodeUnschedulable"}, @@ -804,6 +796,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "ScorePlugin": { {Name: "NodeResourcesBalancedAllocation", Weight: 2}, {Name: "ImageLocality", Weight: 2}, + {Name: "InterPodAffinity", Weight: 2}, {Name: "NodeResourcesLeastAllocated", Weight: 2}, {Name: "NodeResourcesMostAllocated", Weight: 2}, {Name: "NodeAffinity", Weight: 2}, @@ -886,10 +879,8 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "ignorable":true }] }`, - wantPredicates: sets.NewString(), - wantPrioritizers: sets.NewString( - "InterPodAffinityPriority", - ), + wantPredicates: sets.NewString(), + wantPrioritizers: sets.NewString(), wantPlugins: map[string][]config.Plugin{ "FilterPlugin": { {Name: "NodeUnschedulable"}, @@ -912,6 +903,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "ScorePlugin": { {Name: "NodeResourcesBalancedAllocation", Weight: 2}, {Name: "ImageLocality", Weight: 2}, + {Name: "InterPodAffinity", Weight: 2}, {Name: "NodeResourcesLeastAllocated", Weight: 2}, {Name: "NodeResourcesMostAllocated", Weight: 2}, {Name: "NodeAffinity", Weight: 2}, @@ -993,10 +985,8 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "ignorable":true }] }`, - wantPredicates: sets.NewString(), - wantPrioritizers: sets.NewString( - "InterPodAffinityPriority", - ), + wantPredicates: sets.NewString(), + wantPrioritizers: sets.NewString(), wantPlugins: map[string][]config.Plugin{ "FilterPlugin": { {Name: "NodeUnschedulable"}, @@ -1020,6 +1010,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "ScorePlugin": { {Name: "NodeResourcesBalancedAllocation", Weight: 2}, {Name: "ImageLocality", Weight: 2}, + {Name: "InterPodAffinity", Weight: 2}, {Name: "NodeResourcesLeastAllocated", Weight: 2}, {Name: "NodeResourcesMostAllocated", Weight: 2}, {Name: "NodeAffinity", Weight: 2}, @@ -1105,10 +1096,8 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "ignorable":true }] }`, - wantPredicates: sets.NewString(), - wantPrioritizers: sets.NewString( - "InterPodAffinityPriority", - ), + wantPredicates: sets.NewString(), + wantPrioritizers: sets.NewString(), wantPlugins: map[string][]config.Plugin{ "FilterPlugin": { {Name: "NodeUnschedulable"}, @@ -1132,6 +1121,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { "ScorePlugin": { {Name: "NodeResourcesBalancedAllocation", Weight: 2}, {Name: "ImageLocality", Weight: 2}, + {Name: "InterPodAffinity", Weight: 2}, {Name: "NodeResourcesLeastAllocated", Weight: 2}, {Name: "NodeResourcesMostAllocated", Weight: 2}, {Name: "NodeAffinity", Weight: 2}, @@ -1233,6 +1223,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { scoreToPriorityMap := map[string]string{ "DefaultPodTopologySpread": "SelectorSpreadPriority", "ImageLocality": "ImageLocalityPriority", + "InterPodAffinity": "InterPodAffinityPriority", "NodeAffinity": "NodeAffinityPriority", "NodePreferAvoidPods": "NodePreferAvoidPodsPriority", "TaintToleration": "TaintTolerationPriority", diff --git a/pkg/scheduler/framework/plugins/default_registry.go b/pkg/scheduler/framework/plugins/default_registry.go index 09dc752828e..ee058f040c2 100644 --- a/pkg/scheduler/framework/plugins/default_registry.go +++ b/pkg/scheduler/framework/plugins/default_registry.go @@ -241,6 +241,11 @@ func NewDefaultConfigProducerRegistry() *ConfigProducerRegistry { plugins.Score = appendToPluginSet(plugins.Score, imagelocality.Name, &args.Weight) return }) + registry.RegisterPriority(priorities.InterPodAffinityPriority, + func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { + plugins.Score = appendToPluginSet(plugins.Score, interpodaffinity.Name, &args.Weight) + return + }) registry.RegisterPriority(priorities.NodePreferAvoidPodsPriority, func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) { plugins.Score = appendToPluginSet(plugins.Score, nodepreferavoidpods.Name, &args.Weight) diff --git a/pkg/scheduler/framework/plugins/interpodaffinity/BUILD b/pkg/scheduler/framework/plugins/interpodaffinity/BUILD index 5ed715284b0..d9861323b94 100644 --- a/pkg/scheduler/framework/plugins/interpodaffinity/BUILD +++ b/pkg/scheduler/framework/plugins/interpodaffinity/BUILD @@ -7,6 +7,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/scheduler/algorithm/predicates:go_default_library", + "//pkg/scheduler/algorithm/priorities:go_default_library", "//pkg/scheduler/framework/plugins/migration:go_default_library", "//pkg/scheduler/framework/v1alpha1:go_default_library", "//pkg/scheduler/nodeinfo:go_default_library", @@ -21,11 +22,14 @@ go_test( embed = [":go_default_library"], deps = [ "//pkg/scheduler/algorithm/predicates:go_default_library", + "//pkg/scheduler/algorithm/priorities:go_default_library", "//pkg/scheduler/framework/plugins/migration:go_default_library", "//pkg/scheduler/framework/v1alpha1:go_default_library", "//pkg/scheduler/nodeinfo/snapshot:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/client-go/informers:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", ], ) diff --git a/pkg/scheduler/framework/plugins/interpodaffinity/interpod_affinity.go b/pkg/scheduler/framework/plugins/interpodaffinity/interpod_affinity.go index e8353aa41f3..08991e9e1b4 100644 --- a/pkg/scheduler/framework/plugins/interpodaffinity/interpod_affinity.go +++ b/pkg/scheduler/framework/plugins/interpodaffinity/interpod_affinity.go @@ -20,9 +20,10 @@ import ( "context" "fmt" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" + "k8s.io/kubernetes/pkg/scheduler/algorithm/priorities" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/migration" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" "k8s.io/kubernetes/pkg/scheduler/nodeinfo" @@ -30,10 +31,12 @@ import ( // InterPodAffinity is a plugin that checks inter pod affinity type InterPodAffinity struct { + handle framework.FrameworkHandle predicate predicates.FitPredicate } var _ framework.FilterPlugin = &InterPodAffinity{} +var _ framework.ScorePlugin = &InterPodAffinity{} // Name is the name of the plugin used in the plugin registry and configurations. const Name = "InterPodAffinity" @@ -53,9 +56,36 @@ func (pl *InterPodAffinity) Filter(ctx context.Context, cycleState *framework.Cy return migration.PredicateResultToFrameworkStatus(reasons, err) } +// Score invoked at the Score extension point. +// The "score" returned in this function is the matching number of pods on the `nodeName`, +// it is normalized later. +func (pl *InterPodAffinity) Score(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeName string) (int64, *framework.Status) { + nodeInfo, err := pl.handle.SnapshotSharedLister().NodeInfos().Get(nodeName) + if err != nil { + return 0, framework.NewStatus(framework.Error, fmt.Sprintf("getting node %q from Snapshot: %v", nodeName, err)) + } + + meta := migration.PriorityMetadata(state) + s, err := priorities.CalculateInterPodAffinityPriorityMap(pod, meta, nodeInfo) + return s.Score, migration.ErrorToFrameworkStatus(err) +} + +// NormalizeScore invoked after scoring all nodes. +func (pl *InterPodAffinity) NormalizeScore(ctx context.Context, state *framework.CycleState, pod *v1.Pod, scores framework.NodeScoreList) *framework.Status { + meta := migration.PriorityMetadata(state) + err := priorities.CalculateInterPodAffinityPriorityReduce(pod, meta, pl.handle.SnapshotSharedLister(), scores) + return migration.ErrorToFrameworkStatus(err) +} + +// ScoreExtensions of the Score plugin. +func (pl *InterPodAffinity) ScoreExtensions() framework.ScoreExtensions { + return pl +} + // New initializes a new plugin and returns it. func New(_ *runtime.Unknown, h framework.FrameworkHandle) (framework.Plugin, error) { return &InterPodAffinity{ + handle: h, predicate: predicates.NewPodAffinityPredicate(h.SnapshotSharedLister().NodeInfos(), h.SnapshotSharedLister().Pods()), }, nil } diff --git a/pkg/scheduler/framework/plugins/interpodaffinity/interpod_affinity_test.go b/pkg/scheduler/framework/plugins/interpodaffinity/interpod_affinity_test.go index 778a6d067a5..02a31650b10 100644 --- a/pkg/scheduler/framework/plugins/interpodaffinity/interpod_affinity_test.go +++ b/pkg/scheduler/framework/plugins/interpodaffinity/interpod_affinity_test.go @@ -21,9 +21,12 @@ import ( "reflect" "testing" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + clientsetfake "k8s.io/client-go/kubernetes/fake" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" + "k8s.io/kubernetes/pkg/scheduler/algorithm/priorities" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/migration" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" nodeinfosnapshot "k8s.io/kubernetes/pkg/scheduler/nodeinfo/snapshot" @@ -1449,3 +1452,647 @@ func TestMultipleNodes(t *testing.T) { }) } } + +func TestInterPodAffinityPriority(t *testing.T) { + labelRgChina := map[string]string{ + "region": "China", + } + labelRgIndia := map[string]string{ + "region": "India", + } + labelAzAz1 := map[string]string{ + "az": "az1", + } + labelAzAz2 := map[string]string{ + "az": "az2", + } + labelRgChinaAzAz1 := map[string]string{ + "region": "China", + "az": "az1", + } + podLabelSecurityS1 := map[string]string{ + "security": "S1", + } + podLabelSecurityS2 := map[string]string{ + "security": "S2", + } + // considered only preferredDuringSchedulingIgnoredDuringExecution in pod affinity + stayWithS1InRegion := &v1.Affinity{ + PodAffinity: &v1.PodAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{ + { + Weight: 5, + PodAffinityTerm: v1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "security", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"S1"}, + }, + }, + }, + TopologyKey: "region", + }, + }, + }, + }, + } + stayWithS2InRegion := &v1.Affinity{ + PodAffinity: &v1.PodAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{ + { + Weight: 6, + PodAffinityTerm: v1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "security", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"S2"}, + }, + }, + }, + TopologyKey: "region", + }, + }, + }, + }, + } + affinity3 := &v1.Affinity{ + PodAffinity: &v1.PodAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{ + { + Weight: 8, + PodAffinityTerm: v1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "security", + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{"S1"}, + }, { + Key: "security", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"S2"}, + }, + }, + }, + TopologyKey: "region", + }, + }, { + Weight: 2, + PodAffinityTerm: v1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "security", + Operator: metav1.LabelSelectorOpExists, + }, { + Key: "wrongkey", + Operator: metav1.LabelSelectorOpDoesNotExist, + }, + }, + }, + TopologyKey: "region", + }, + }, + }, + }, + } + hardAffinity := &v1.Affinity{ + PodAffinity: &v1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "security", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"S1", "value2"}, + }, + }, + }, + TopologyKey: "region", + }, { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "security", + Operator: metav1.LabelSelectorOpExists, + }, { + Key: "wrongkey", + Operator: metav1.LabelSelectorOpDoesNotExist, + }, + }, + }, + TopologyKey: "region", + }, + }, + }, + } + awayFromS1InAz := &v1.Affinity{ + PodAntiAffinity: &v1.PodAntiAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{ + { + Weight: 5, + PodAffinityTerm: v1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "security", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"S1"}, + }, + }, + }, + TopologyKey: "az", + }, + }, + }, + }, + } + // to stay away from security S2 in any az. + awayFromS2InAz := &v1.Affinity{ + PodAntiAffinity: &v1.PodAntiAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{ + { + Weight: 5, + PodAffinityTerm: v1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "security", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"S2"}, + }, + }, + }, + TopologyKey: "az", + }, + }, + }, + }, + } + // to stay with security S1 in same region, stay away from security S2 in any az. + stayWithS1InRegionAwayFromS2InAz := &v1.Affinity{ + PodAffinity: &v1.PodAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{ + { + Weight: 8, + PodAffinityTerm: v1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "security", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"S1"}, + }, + }, + }, + TopologyKey: "region", + }, + }, + }, + }, + PodAntiAffinity: &v1.PodAntiAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{ + { + Weight: 5, + PodAffinityTerm: v1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "security", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"S2"}, + }, + }, + }, + TopologyKey: "az", + }, + }, + }, + }, + } + + tests := []struct { + pod *v1.Pod + pods []*v1.Pod + nodes []*v1.Node + expectedList framework.NodeScoreList + name string + }{ + { + pod: &v1.Pod{Spec: v1.PodSpec{NodeName: ""}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + nodes: []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: labelRgChina}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: labelRgIndia}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine3", Labels: labelAzAz1}}, + }, + expectedList: []framework.NodeScore{{Name: "machine1", Score: 0}, {Name: "machine2", Score: 0}, {Name: "machine3", Score: 0}}, + name: "all machines are same priority as Affinity is nil", + }, + // the node(machine1) that have the label {"region": "China"} (match the topology key) and that have existing pods that match the labelSelector get high score + // the node(machine3) that don't have the label {"region": "whatever the value is"} (mismatch the topology key) but that have existing pods that match the labelSelector get low score + // the node(machine2) that have the label {"region": "China"} (match the topology key) but that have existing pods that mismatch the labelSelector get low score + { + pod: &v1.Pod{Spec: v1.PodSpec{NodeName: "", Affinity: stayWithS1InRegion}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + pods: []*v1.Pod{ + {Spec: v1.PodSpec{NodeName: "machine1"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + {Spec: v1.PodSpec{NodeName: "machine2"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS2}}, + {Spec: v1.PodSpec{NodeName: "machine3"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + }, + nodes: []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: labelRgChina}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: labelRgIndia}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine3", Labels: labelAzAz1}}, + }, + expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: 0}, {Name: "machine3", Score: 0}}, + name: "Affinity: pod that matches topology key & pods in nodes will get high score comparing to others" + + "which doesn't match either pods in nodes or in topology key", + }, + // the node1(machine1) that have the label {"region": "China"} (match the topology key) and that have existing pods that match the labelSelector get high score + // the node2(machine2) that have the label {"region": "China"}, match the topology key and have the same label value with node1, get the same high score with node1 + // the node3(machine3) that have the label {"region": "India"}, match the topology key but have a different label value, don't have existing pods that match the labelSelector, + // get a low score. + { + pod: &v1.Pod{Spec: v1.PodSpec{NodeName: "", Affinity: stayWithS1InRegion}}, + pods: []*v1.Pod{ + {Spec: v1.PodSpec{NodeName: "machine1"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + }, + nodes: []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: labelRgChina}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: labelRgChinaAzAz1}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine3", Labels: labelRgIndia}}, + }, + expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: framework.MaxNodeScore}, {Name: "machine3", Score: 0}}, + name: "All the nodes that have the same topology key & label value with one of them has an existing pod that match the affinity rules, have the same score", + }, + // there are 2 regions, say regionChina(machine1,machine3,machine4) and regionIndia(machine2,machine5), both regions have nodes that match the preference. + // But there are more nodes(actually more existing pods) in regionChina that match the preference than regionIndia. + // Then, nodes in regionChina get higher score than nodes in regionIndia, and all the nodes in regionChina should get a same score(high score), + // while all the nodes in regionIndia should get another same score(low score). + { + pod: &v1.Pod{Spec: v1.PodSpec{NodeName: "", Affinity: stayWithS2InRegion}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + pods: []*v1.Pod{ + {Spec: v1.PodSpec{NodeName: "machine1"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS2}}, + {Spec: v1.PodSpec{NodeName: "machine1"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS2}}, + {Spec: v1.PodSpec{NodeName: "machine2"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS2}}, + {Spec: v1.PodSpec{NodeName: "machine3"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS2}}, + {Spec: v1.PodSpec{NodeName: "machine4"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS2}}, + {Spec: v1.PodSpec{NodeName: "machine5"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS2}}, + }, + nodes: []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: labelRgChina}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: labelRgIndia}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine3", Labels: labelRgChina}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine4", Labels: labelRgChina}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine5", Labels: labelRgIndia}}, + }, + expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: 50}, {Name: "machine3", Score: framework.MaxNodeScore}, {Name: "machine4", Score: framework.MaxNodeScore}, {Name: "machine5", Score: 50}}, + name: "Affinity: nodes in one region has more matching pods comparing to other reqion, so the region which has more macthes will get high score", + }, + // Test with the different operators and values for pod affinity scheduling preference, including some match failures. + { + pod: &v1.Pod{Spec: v1.PodSpec{NodeName: "", Affinity: affinity3}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + pods: []*v1.Pod{ + {Spec: v1.PodSpec{NodeName: "machine1"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + {Spec: v1.PodSpec{NodeName: "machine2"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS2}}, + {Spec: v1.PodSpec{NodeName: "machine3"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + }, + nodes: []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: labelRgChina}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: labelRgIndia}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine3", Labels: labelAzAz1}}, + }, + expectedList: []framework.NodeScore{{Name: "machine1", Score: 20}, {Name: "machine2", Score: framework.MaxNodeScore}, {Name: "machine3", Score: 0}}, + name: "Affinity: different Label operators and values for pod affinity scheduling preference, including some match failures ", + }, + // Test the symmetry cases for affinity, the difference between affinity and symmetry is not the pod wants to run together with some existing pods, + // but the existing pods have the inter pod affinity preference while the pod to schedule satisfy the preference. + { + pod: &v1.Pod{Spec: v1.PodSpec{NodeName: ""}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS2}}, + pods: []*v1.Pod{ + {Spec: v1.PodSpec{NodeName: "machine1", Affinity: stayWithS1InRegion}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + {Spec: v1.PodSpec{NodeName: "machine2", Affinity: stayWithS2InRegion}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS2}}, + }, + nodes: []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: labelRgChina}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: labelRgIndia}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine3", Labels: labelAzAz1}}, + }, + expectedList: []framework.NodeScore{{Name: "machine1", Score: 0}, {Name: "machine2", Score: framework.MaxNodeScore}, {Name: "machine3", Score: 0}}, + name: "Affinity symmetry: considered only the preferredDuringSchedulingIgnoredDuringExecution in pod affinity symmetry", + }, + { + pod: &v1.Pod{Spec: v1.PodSpec{NodeName: ""}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + pods: []*v1.Pod{ + {Spec: v1.PodSpec{NodeName: "machine1", Affinity: hardAffinity}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + {Spec: v1.PodSpec{NodeName: "machine2", Affinity: hardAffinity}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS2}}, + }, + nodes: []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: labelRgChina}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: labelRgIndia}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine3", Labels: labelAzAz1}}, + }, + expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: framework.MaxNodeScore}, {Name: "machine3", Score: 0}}, + name: "Affinity symmetry: considered RequiredDuringSchedulingIgnoredDuringExecution in pod affinity symmetry", + }, + + // The pod to schedule prefer to stay away from some existing pods at node level using the pod anti affinity. + // the nodes that have the label {"node": "bar"} (match the topology key) and that have existing pods that match the labelSelector get low score + // the nodes that don't have the label {"node": "whatever the value is"} (mismatch the topology key) but that have existing pods that match the labelSelector get high score + // the nodes that have the label {"node": "bar"} (match the topology key) but that have existing pods that mismatch the labelSelector get high score + // there are 2 nodes, say node1 and node2, both nodes have pods that match the labelSelector and have topology-key in node.Labels. + // But there are more pods on node1 that match the preference than node2. Then, node1 get a lower score than node2. + { + pod: &v1.Pod{Spec: v1.PodSpec{NodeName: "", Affinity: awayFromS1InAz}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + pods: []*v1.Pod{ + {Spec: v1.PodSpec{NodeName: "machine1"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + {Spec: v1.PodSpec{NodeName: "machine2"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS2}}, + }, + nodes: []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: labelAzAz1}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: labelRgChina}}, + }, + expectedList: []framework.NodeScore{{Name: "machine1", Score: 0}, {Name: "machine2", Score: framework.MaxNodeScore}}, + name: "Anti Affinity: pod that doesnot match existing pods in node will get high score ", + }, + { + pod: &v1.Pod{Spec: v1.PodSpec{NodeName: "", Affinity: awayFromS1InAz}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + pods: []*v1.Pod{ + {Spec: v1.PodSpec{NodeName: "machine1"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + {Spec: v1.PodSpec{NodeName: "machine2"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + }, + nodes: []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: labelAzAz1}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: labelRgChina}}, + }, + expectedList: []framework.NodeScore{{Name: "machine1", Score: 0}, {Name: "machine2", Score: framework.MaxNodeScore}}, + name: "Anti Affinity: pod that does not matches topology key & matches the pods in nodes will get higher score comparing to others ", + }, + { + pod: &v1.Pod{Spec: v1.PodSpec{NodeName: "", Affinity: awayFromS1InAz}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + pods: []*v1.Pod{ + {Spec: v1.PodSpec{NodeName: "machine1"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + {Spec: v1.PodSpec{NodeName: "machine1"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + {Spec: v1.PodSpec{NodeName: "machine2"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS2}}, + }, + nodes: []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: labelAzAz1}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: labelRgIndia}}, + }, + expectedList: []framework.NodeScore{{Name: "machine1", Score: 0}, {Name: "machine2", Score: framework.MaxNodeScore}}, + name: "Anti Affinity: one node has more matching pods comparing to other node, so the node which has more unmacthes will get high score", + }, + // Test the symmetry cases for anti affinity + { + pod: &v1.Pod{Spec: v1.PodSpec{NodeName: ""}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS2}}, + pods: []*v1.Pod{ + {Spec: v1.PodSpec{NodeName: "machine1", Affinity: awayFromS2InAz}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + {Spec: v1.PodSpec{NodeName: "machine2", Affinity: awayFromS1InAz}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS2}}, + }, + nodes: []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: labelAzAz1}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: labelAzAz2}}, + }, + expectedList: []framework.NodeScore{{Name: "machine1", Score: 0}, {Name: "machine2", Score: framework.MaxNodeScore}}, + name: "Anti Affinity symmetry: the existing pods in node which has anti affinity match will get high score", + }, + // Test both affinity and anti-affinity + { + pod: &v1.Pod{Spec: v1.PodSpec{NodeName: "", Affinity: stayWithS1InRegionAwayFromS2InAz}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + pods: []*v1.Pod{ + {Spec: v1.PodSpec{NodeName: "machine1"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + {Spec: v1.PodSpec{NodeName: "machine2"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + }, + nodes: []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: labelRgChina}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: labelAzAz1}}, + }, + expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: 0}}, + name: "Affinity and Anti Affinity: considered only preferredDuringSchedulingIgnoredDuringExecution in both pod affinity & anti affinity", + }, + // Combined cases considering both affinity and anti-affinity, the pod to schedule and existing pods have the same labels (they are in the same RC/service), + // the pod prefer to run together with its brother pods in the same region, but wants to stay away from them at node level, + // so that all the pods of a RC/service can stay in a same region but trying to separate with each other + // machine-1,machine-3,machine-4 are in ChinaRegion others machin-2,machine-5 are in IndiaRegion + { + pod: &v1.Pod{Spec: v1.PodSpec{NodeName: "", Affinity: stayWithS1InRegionAwayFromS2InAz}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + pods: []*v1.Pod{ + {Spec: v1.PodSpec{NodeName: "machine1"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + {Spec: v1.PodSpec{NodeName: "machine1"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + {Spec: v1.PodSpec{NodeName: "machine2"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + {Spec: v1.PodSpec{NodeName: "machine3"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + {Spec: v1.PodSpec{NodeName: "machine3"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + {Spec: v1.PodSpec{NodeName: "machine4"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + {Spec: v1.PodSpec{NodeName: "machine5"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + }, + nodes: []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: labelRgChinaAzAz1}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: labelRgIndia}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine3", Labels: labelRgChina}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine4", Labels: labelRgChina}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine5", Labels: labelRgIndia}}, + }, + expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: 40}, {Name: "machine3", Score: framework.MaxNodeScore}, {Name: "machine4", Score: framework.MaxNodeScore}, {Name: "machine5", Score: 40}}, + name: "Affinity and Anti Affinity: considering both affinity and anti-affinity, the pod to schedule and existing pods have the same labels", + }, + // Consider Affinity, Anti Affinity and symmetry together. + // for Affinity, the weights are: 8, 0, 0, 0 + // for Anti Affinity, the weights are: 0, -5, 0, 0 + // for Affinity symmetry, the weights are: 0, 0, 8, 0 + // for Anti Affinity symmetry, the weights are: 0, 0, 0, -5 + { + pod: &v1.Pod{Spec: v1.PodSpec{NodeName: "", Affinity: stayWithS1InRegionAwayFromS2InAz}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + pods: []*v1.Pod{ + {Spec: v1.PodSpec{NodeName: "machine1"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + {Spec: v1.PodSpec{NodeName: "machine2"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS2}}, + {Spec: v1.PodSpec{NodeName: "machine3", Affinity: stayWithS1InRegionAwayFromS2InAz}}, + {Spec: v1.PodSpec{NodeName: "machine4", Affinity: awayFromS1InAz}}, + }, + nodes: []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: labelRgChina}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: labelAzAz1}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine3", Labels: labelRgIndia}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine4", Labels: labelAzAz2}}, + }, + expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: 0}, {Name: "machine3", Score: framework.MaxNodeScore}, {Name: "machine4", Score: 0}}, + name: "Affinity and Anti Affinity and symmetry: considered only preferredDuringSchedulingIgnoredDuringExecution in both pod affinity & anti affinity & symmetry", + }, + // Cover https://github.com/kubernetes/kubernetes/issues/82796 which panics upon: + // 1. Some nodes in a topology don't have pods with affinity, but other nodes in the same topology have. + // 2. The incoming pod doesn't have affinity. + { + pod: &v1.Pod{Spec: v1.PodSpec{NodeName: ""}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + pods: []*v1.Pod{ + {Spec: v1.PodSpec{NodeName: "machine1"}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}}, + {Spec: v1.PodSpec{NodeName: "machine2", Affinity: stayWithS1InRegionAwayFromS2InAz}}, + }, + nodes: []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: labelRgChina}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: labelRgChina}}, + }, + expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: framework.MaxNodeScore}}, + name: "Avoid panic when partial nodes in a topology don't have pods with affinity", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + state := framework.NewCycleState() + snapshot := nodeinfosnapshot.NewSnapshot(test.pods, test.nodes) + fh, _ := framework.NewFramework(nil, nil, nil, framework.WithSnapshotSharedLister(snapshot)) + + client := clientsetfake.NewSimpleClientset() + informerFactory := informers.NewSharedInformerFactory(client, 0) + + metaDataProducer := priorities.NewPriorityMetadataFactory( + informerFactory.Core().V1().Services().Lister(), + informerFactory.Core().V1().ReplicationControllers().Lister(), + informerFactory.Apps().V1().ReplicaSets().Lister(), + informerFactory.Apps().V1().StatefulSets().Lister(), + 1, + ) + + metaData := metaDataProducer(test.pod, test.nodes, snapshot) + + state.Write(migration.PrioritiesStateKey, &migration.PrioritiesStateData{Reference: metaData}) + + p, _ := New(nil, fh) + var gotList framework.NodeScoreList + for _, n := range test.nodes { + nodeName := n.ObjectMeta.Name + score, status := p.(framework.ScorePlugin).Score(context.Background(), state, test.pod, nodeName) + if !status.IsSuccess() { + t.Errorf("unexpected error: %v", status) + } + gotList = append(gotList, framework.NodeScore{Name: nodeName, Score: score}) + } + + status := p.(framework.ScorePlugin).ScoreExtensions().NormalizeScore(context.Background(), state, test.pod, gotList) + if !status.IsSuccess() { + t.Errorf("unexpected error: %v", status) + } + + if !reflect.DeepEqual(test.expectedList, gotList) { + t.Errorf("expected:\n\t%+v,\ngot:\n\t%+v", test.expectedList, gotList) + } + + }) + } +} + +func TestHardPodAffinitySymmetricWeight(t *testing.T) { + podLabelServiceS1 := map[string]string{ + "service": "S1", + } + labelRgChina := map[string]string{ + "region": "China", + } + labelRgIndia := map[string]string{ + "region": "India", + } + labelAzAz1 := map[string]string{ + "az": "az1", + } + hardPodAffinity := &v1.Affinity{ + PodAffinity: &v1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "service", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"S1"}, + }, + }, + }, + TopologyKey: "region", + }, + }, + }, + } + tests := []struct { + pod *v1.Pod + pods []*v1.Pod + nodes []*v1.Node + hardPodAffinityWeight int32 + expectedList framework.NodeScoreList + name string + }{ + { + pod: &v1.Pod{Spec: v1.PodSpec{NodeName: ""}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelServiceS1}}, + pods: []*v1.Pod{ + {Spec: v1.PodSpec{NodeName: "machine1", Affinity: hardPodAffinity}}, + {Spec: v1.PodSpec{NodeName: "machine2", Affinity: hardPodAffinity}}, + }, + nodes: []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: labelRgChina}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: labelRgIndia}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine3", Labels: labelAzAz1}}, + }, + hardPodAffinityWeight: v1.DefaultHardPodAffinitySymmetricWeight, + expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: framework.MaxNodeScore}, {Name: "machine3", Score: 0}}, + name: "Hard Pod Affinity symmetry: hard pod affinity symmetry weights 1 by default, then nodes that match the hard pod affinity symmetry rules, get a high score", + }, + { + pod: &v1.Pod{Spec: v1.PodSpec{NodeName: ""}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelServiceS1}}, + pods: []*v1.Pod{ + {Spec: v1.PodSpec{NodeName: "machine1", Affinity: hardPodAffinity}}, + {Spec: v1.PodSpec{NodeName: "machine2", Affinity: hardPodAffinity}}, + }, + nodes: []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: labelRgChina}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: labelRgIndia}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine3", Labels: labelAzAz1}}, + }, + hardPodAffinityWeight: 0, + expectedList: []framework.NodeScore{{Name: "machine1", Score: 0}, {Name: "machine2", Score: 0}, {Name: "machine3", Score: 0}}, + name: "Hard Pod Affinity symmetry: hard pod affinity symmetry is closed(weights 0), then nodes that match the hard pod affinity symmetry rules, get same score with those not match", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + state := framework.NewCycleState() + snapshot := nodeinfosnapshot.NewSnapshot(test.pods, test.nodes) + fh, _ := framework.NewFramework(nil, nil, nil, framework.WithSnapshotSharedLister(snapshot)) + + client := clientsetfake.NewSimpleClientset() + informerFactory := informers.NewSharedInformerFactory(client, 0) + + metaDataProducer := priorities.NewPriorityMetadataFactory( + informerFactory.Core().V1().Services().Lister(), + informerFactory.Core().V1().ReplicationControllers().Lister(), + informerFactory.Apps().V1().ReplicaSets().Lister(), + informerFactory.Apps().V1().StatefulSets().Lister(), + test.hardPodAffinityWeight, + ) + + metaData := metaDataProducer(test.pod, test.nodes, snapshot) + + state.Write(migration.PrioritiesStateKey, &migration.PrioritiesStateData{Reference: metaData}) + + p, _ := New(nil, fh) + var gotList framework.NodeScoreList + for _, n := range test.nodes { + nodeName := n.ObjectMeta.Name + score, status := p.(framework.ScorePlugin).Score(context.Background(), state, test.pod, nodeName) + if !status.IsSuccess() { + t.Errorf("unexpected error: %v", status) + } + gotList = append(gotList, framework.NodeScore{Name: nodeName, Score: score}) + } + + status := p.(framework.ScorePlugin).ScoreExtensions().NormalizeScore(context.Background(), state, test.pod, gotList) + if !status.IsSuccess() { + t.Errorf("unexpected error: %v", status) + } + + if !reflect.DeepEqual(test.expectedList, gotList) { + t.Errorf("expected:\n\t%+v,\ngot:\n\t%+v", test.expectedList, gotList) + } + }) + } +} diff --git a/test/integration/scheduler/scheduler_test.go b/test/integration/scheduler/scheduler_test.go index 9ca5c062f11..a0992ef897a 100644 --- a/test/integration/scheduler/scheduler_test.go +++ b/test/integration/scheduler/scheduler_test.go @@ -129,9 +129,7 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) { "kind" : "Policy", "apiVersion" : "v1" }`, - expectedPrioritizers: sets.NewString( - "InterPodAffinityPriority", - ), + expectedPrioritizers: sets.NewString(), expectedPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { {Name: "NodeUnschedulable"}, @@ -152,6 +150,7 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) { "ScorePlugin": { {Name: "NodeResourcesBalancedAllocation", Weight: 1}, {Name: "ImageLocality", Weight: 1}, + {Name: "InterPodAffinity", Weight: 1}, {Name: "NodeResourcesLeastAllocated", Weight: 1}, {Name: "NodeAffinity", Weight: 1}, {Name: "NodePreferAvoidPods", Weight: 10000}, @@ -207,9 +206,7 @@ priorities: policy: `apiVersion: v1 kind: Policy `, - expectedPrioritizers: sets.NewString( - "InterPodAffinityPriority", - ), + expectedPrioritizers: sets.NewString(), expectedPlugins: map[string][]kubeschedulerconfig.Plugin{ "FilterPlugin": { {Name: "NodeUnschedulable"}, @@ -230,6 +227,7 @@ kind: Policy "ScorePlugin": { {Name: "NodeResourcesBalancedAllocation", Weight: 1}, {Name: "ImageLocality", Weight: 1}, + {Name: "InterPodAffinity", Weight: 1}, {Name: "NodeResourcesLeastAllocated", Weight: 1}, {Name: "NodeAffinity", Weight: 1}, {Name: "NodePreferAvoidPods", Weight: 10000},