github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-20 10:20:51 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #108884 from kerthcet/feature/implement-nodeInclusionPolicy

Browse Source 
feat: implement node inclusion policy in scheduler
This commit is contained in:
Kubernetes Prow Robot 2022-05-11 08:47:02 -07:00 committed by GitHub
commit 3bcbf3de11
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 2039 additions and 447 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
pkg/scheduler/framework/plugins/defaultpreemption/default_preemption_test.go
View File

@ -644,8 +644,8 @@ func TestDryRunPreemption(t *testing.T) {
nodeNames: []string{"node-a/zone1", "node-b/zone1", "node-x/zone2"},
testPods: []*v1.Pod{
st.MakePod().Name("p").UID("p").Label("foo", "").Priority(highPriority).
SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, "hostname", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil).
SpreadConstraint(1, "hostname", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil).
Obj(),
},
initPods: []*v1.Pod{
@ -1486,8 +1486,8 @@ func TestPreempt(t *testing.T) {
{
name: "preemption for topology spread constraints",
pod: st.MakePod().Name("p").UID("p").Namespace(v1.NamespaceDefault).Label("foo", "").Priority(highPriority).
SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, "hostname", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil).
SpreadConstraint(1, "hostname", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil).
Obj(),
pods: []*v1.Pod{
st.MakePod().Name("p-a1").UID("p-a1").Namespace(v1.NamespaceDefault).Node("node-a").Label("foo", "").Priority(highPriority).Obj(),

1
pkg/scheduler/framework/plugins/feature/feature.go
View File

@ -25,4 +25,5 @@ type Features struct {
EnableReadWriteOncePod bool
EnableVolumeCapacityPriority bool
EnableMinDomainsInPodTopologySpread bool
EnableNodeInclusionPolicyInPodTopologySpread bool
}

36
pkg/scheduler/framework/plugins/podtopologyspread/common.go
View File

@ -20,6 +20,8 @@ import (
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
v1helper "k8s.io/component-helpers/scheduling/corev1"
"k8s.io/component-helpers/scheduling/corev1/nodeaffinity"
"k8s.io/kubernetes/pkg/scheduler/framework"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/helper"
)
@ -37,13 +39,31 @@ type topologySpreadConstraint struct {
TopologyKey string
Selector labels.Selector
MinDomains int32
NodeAffinityPolicy v1.NodeInclusionPolicy
NodeTaintsPolicy v1.NodeInclusionPolicy
}
func (tsc *topologySpreadConstraint) matchNodeInclusionPolicies(pod *v1.Pod, node *v1.Node, require nodeaffinity.RequiredNodeAffinity) bool {
if tsc.NodeAffinityPolicy == v1.NodeInclusionPolicyHonor {
// We ignore parsing errors here for backwards compatibility.
if match, _ := require.Match(node); !match {
return false
}
}
if tsc.NodeTaintsPolicy == v1.NodeInclusionPolicyHonor {
if _, untolerated := v1helper.FindMatchingUntoleratedTaint(node.Spec.Taints, pod.Spec.Tolerations, nil); untolerated {
return false
}
}
return true
}
// buildDefaultConstraints builds the constraints for a pod using
// .DefaultConstraints and the selectors from the services, replication
// controllers, replica sets and stateful sets that match the pod.
func (pl *PodTopologySpread) buildDefaultConstraints(p *v1.Pod, action v1.UnsatisfiableConstraintAction) ([]topologySpreadConstraint, error) {
constraints, err := filterTopologySpreadConstraints(pl.defaultConstraints, action, pl.enableMinDomainsInPodTopologySpread)
constraints, err := filterTopologySpreadConstraints(pl.defaultConstraints, action, pl.enableMinDomainsInPodTopologySpread, pl.enableNodeInclusionPolicyInPodTopologySpread)
if err != nil || len(constraints) == 0 {
return nil, err
}
@ -67,7 +87,7 @@ func nodeLabelsMatchSpreadConstraints(nodeLabels map[string]string, constraints
return true
}
func filterTopologySpreadConstraints(constraints []v1.TopologySpreadConstraint, action v1.UnsatisfiableConstraintAction, enableMinDomainsInPodTopologySpread bool) ([]topologySpreadConstraint, error) {
func filterTopologySpreadConstraints(constraints []v1.TopologySpreadConstraint, action v1.UnsatisfiableConstraintAction, enableMinDomainsInPodTopologySpread, enableNodeInclusionPolicyInPodTopologySpread bool) ([]topologySpreadConstraint, error) {
var result []topologySpreadConstraint
for _, c := range constraints {
if c.WhenUnsatisfiable == action {
@ -79,11 +99,21 @@ func filterTopologySpreadConstraints(constraints []v1.TopologySpreadConstraint,
MaxSkew: c.MaxSkew,
TopologyKey: c.TopologyKey,
Selector: selector,
MinDomains: 1, // if MinDomains is nil, we treat MinDomains as 1.
MinDomains: 1, // If MinDomains is nil, we treat MinDomains as 1.
NodeAffinityPolicy: v1.NodeInclusionPolicyHonor, // If NodeAffinityPolicy is nil, we treat NodeAffinityPolicy as "Honor".
NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore, // If NodeTaintsPolicy is nil, we treat NodeTaintsPolicy as "Ignore".
}
if enableMinDomainsInPodTopologySpread && c.MinDomains != nil {
tsc.MinDomains = *c.MinDomains
}
if enableNodeInclusionPolicyInPodTopologySpread {
if c.NodeAffinityPolicy != nil {
tsc.NodeAffinityPolicy = *c.NodeAffinityPolicy
}
if c.NodeTaintsPolicy != nil {
tsc.NodeTaintsPolicy = *c.NodeTaintsPolicy
}
}
result = append(result, tsc)
}
}

22
pkg/scheduler/framework/plugins/podtopologyspread/filtering.go
View File

@ -231,7 +231,12 @@ func (pl *PodTopologySpread) calPreFilterState(ctx context.Context, pod *v1.Pod)
if len(pod.Spec.TopologySpreadConstraints) > 0 {
// We have feature gating in APIServer to strip the spec
// so don't need to re-check feature gate, just check length of Constraints.
constraints, err = filterTopologySpreadConstraints(pod.Spec.TopologySpreadConstraints, v1.DoNotSchedule, pl.enableMinDomainsInPodTopologySpread)
constraints, err = filterTopologySpreadConstraints(
pod.Spec.TopologySpreadConstraints,
v1.DoNotSchedule,
pl.enableMinDomainsInPodTopologySpread,
pl.enableNodeInclusionPolicyInPodTopologySpread,
)
if err != nil {
return nil, fmt.Errorf("obtaining pod's hard topology spread constraints: %w", err)
}
@ -251,8 +256,8 @@ func (pl *PodTopologySpread) calPreFilterState(ctx context.Context, pod *v1.Pod)
TpPairToMatchNum: make(map[topologyPair]int, sizeHeuristic(len(allNodes), constraints)),
}
requiredSchedulingTerm := nodeaffinity.GetRequiredNodeAffinity(pod)
tpCountsByNode := make([]map[topologyPair]int, len(allNodes))
requiredNodeAffinity := nodeaffinity.GetRequiredNodeAffinity(pod)
processNode := func(i int) {
nodeInfo := allNodes[i]
node := nodeInfo.Node()
@ -260,13 +265,15 @@ func (pl *PodTopologySpread) calPreFilterState(ctx context.Context, pod *v1.Pod)
klog.ErrorS(nil, "Node not found")
return
}
// In accordance to design, if NodeAffinity or NodeSelector is defined,
if !pl.enableNodeInclusionPolicyInPodTopologySpread {
// spreading is applied to nodes that pass those filters.
// Ignore parsing errors for backwards compatibility.
match, _ := requiredSchedulingTerm.Match(node)
if !match {
if match, _ := requiredNodeAffinity.Match(node); !match {
return
}
}
// Ensure current node's labels contains all topologyKeys in 'Constraints'.
if !nodeLabelsMatchSpreadConstraints(node.Labels, constraints) {
return
@ -274,6 +281,11 @@ func (pl *PodTopologySpread) calPreFilterState(ctx context.Context, pod *v1.Pod)
tpCounts := make(map[topologyPair]int, len(constraints))
for _, c := range constraints {
if pl.enableNodeInclusionPolicyInPodTopologySpread &&
!c.matchNodeInclusionPolicies(pod, node, requiredNodeAffinity) {
continue
}
pair := topologyPair{key: c.TopologyKey, value: node.Labels[c.TopologyKey]}
count := countPodsMatchSelector(nodeInfo.Pods, c.Selector, pod.Namespace)
tpCounts[pair] = count

1114
pkg/scheduler/framework/plugins/podtopologyspread/filtering_test.go
View File

File diff suppressed because it is too large Load Diff

2
pkg/scheduler/framework/plugins/podtopologyspread/plugin.go
View File

@ -63,6 +63,7 @@ type PodTopologySpread struct {
replicaSets appslisters.ReplicaSetLister
statefulSets appslisters.StatefulSetLister
enableMinDomainsInPodTopologySpread bool
enableNodeInclusionPolicyInPodTopologySpread bool
}
var _ framework.PreFilterPlugin = &PodTopologySpread{}
@ -98,6 +99,7 @@ func New(plArgs runtime.Object, h framework.Handle, fts feature.Features) (frame
sharedLister: h.SnapshotSharedLister(),
defaultConstraints: args.DefaultConstraints,
enableMinDomainsInPodTopologySpread: fts.EnableMinDomainsInPodTopologySpread,
enableNodeInclusionPolicyInPodTopologySpread: fts.EnableNodeInclusionPolicyInPodTopologySpread,
}
if args.DefaultingType == config.SystemDefaulting {
pl.defaultConstraints = systemDefaultConstraints

26
pkg/scheduler/framework/plugins/podtopologyspread/scoring.go
View File

@ -59,7 +59,12 @@ func (s *preScoreState) Clone() framework.StateData {
func (pl *PodTopologySpread) initPreScoreState(s *preScoreState, pod *v1.Pod, filteredNodes []*v1.Node, requireAllTopologies bool) error {
var err error
if len(pod.Spec.TopologySpreadConstraints) > 0 {
s.Constraints, err = filterTopologySpreadConstraints(pod.Spec.TopologySpreadConstraints, v1.ScheduleAnyway, pl.enableMinDomainsInPodTopologySpread)
s.Constraints, err = filterTopologySpreadConstraints(
pod.Spec.TopologySpreadConstraints,
v1.ScheduleAnyway,
pl.enableMinDomainsInPodTopologySpread,
pl.enableNodeInclusionPolicyInPodTopologySpread,
)
if err != nil {
return fmt.Errorf("obtaining pod's soft topology spread constraints: %w", err)
}
@ -148,14 +153,25 @@ func (pl *PodTopologySpread) PreScore(
if node == nil {
return
}
// (1) `node` should satisfy incoming pod's NodeSelector/NodeAffinity
// (2) All topologyKeys need to be present in `node`
match, _ := requiredNodeAffinity.Match(node)
if !match || (requireAllTopologies && !nodeLabelsMatchSpreadConstraints(node.Labels, state.Constraints)) {
if !pl.enableNodeInclusionPolicyInPodTopologySpread {
// `node` should satisfy incoming pod's NodeSelector/NodeAffinity
if match, _ := requiredNodeAffinity.Match(node); !match {
return
}
}
// All topologyKeys need to be present in `node`
if requireAllTopologies && !nodeLabelsMatchSpreadConstraints(node.Labels, state.Constraints) {
return
}
for _, c := range state.Constraints {
if pl.enableNodeInclusionPolicyInPodTopologySpread &&
!c.matchNodeInclusionPolicies(pod, node, requiredNodeAffinity) {
continue
}
pair := topologyPair{key: c.TopologyKey, value: node.Labels[c.TopologyKey]}
// If current topology pair is not associated with any candidate node,
// continue to avoid unnecessary calculation.

485
pkg/scheduler/framework/plugins/podtopologyspread/scoring_test.go
View File

@ -18,6 +18,7 @@ package podtopologyspread
import (
"context"
"fmt"
"testing"
"github.com/google/go-cmp/cmp"
@ -48,12 +49,13 @@ func TestPreScoreStateEmptyNodes(t *testing.T) {
objs []runtime.Object
config config.PodTopologySpreadArgs
want *preScoreState
enableNodeInclustionPolicy bool
}{
{
name: "normal case",
pod: st.MakePod().Name("p").Label("foo", "").
SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, "zone", v1.ScheduleAnyway, fooSelector, nil, nil, nil).
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil).
Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("zone", "zone1").Label(v1.LabelHostname, "node-a").Obj(),
@ -68,14 +70,18 @@ func TestPreScoreStateEmptyNodes(t *testing.T) {
{
MaxSkew: 1,
TopologyKey: "zone",
Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()),
Selector: mustConvertLabelSelectorAsSelector(t, fooSelector),
MinDomains: 1,
NodeAffinityPolicy: v1.NodeInclusionPolicyHonor,
NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore,
},
{
MaxSkew: 1,
TopologyKey: v1.LabelHostname,
Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()),
Selector: mustConvertLabelSelectorAsSelector(t, fooSelector),
MinDomains: 1,
NodeAffinityPolicy: v1.NodeInclusionPolicyHonor,
NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore,
},
},
IgnoredNodes: sets.NewString(),
@ -89,8 +95,8 @@ func TestPreScoreStateEmptyNodes(t *testing.T) {
{
name: "node-x doesn't have label zone",
pod: st.MakePod().Name("p").Label("foo", "").
SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("bar").Obj(), nil).
SpreadConstraint(1, "zone", v1.ScheduleAnyway, fooSelector, nil, nil, nil).
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, barSelector, nil, nil, nil).
Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("zone", "zone1").Label(v1.LabelHostname, "node-a").Obj(),
@ -105,14 +111,18 @@ func TestPreScoreStateEmptyNodes(t *testing.T) {
{
MaxSkew: 1,
TopologyKey: "zone",
Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()),
Selector: mustConvertLabelSelectorAsSelector(t, fooSelector),
MinDomains: 1,
NodeAffinityPolicy: v1.NodeInclusionPolicyHonor,
NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore,
},
{
MaxSkew: 1,
TopologyKey: v1.LabelHostname,
Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("bar").Obj()),
Selector: mustConvertLabelSelectorAsSelector(t, barSelector),
MinDomains: 1,
NodeAffinityPolicy: v1.NodeInclusionPolicyHonor,
NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore,
},
},
IgnoredNodes: sets.NewString("node-x"),
@ -136,21 +146,25 @@ func TestPreScoreStateEmptyNodes(t *testing.T) {
st.MakeNode().Name("node-d").Label(v1.LabelHostname, "node-d").Obj(),
},
objs: []runtime.Object{
&appsv1.ReplicaSet{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "rs1"}, Spec: appsv1.ReplicaSetSpec{Selector: st.MakeLabelSelector().Exists("foo").Obj()}},
&appsv1.ReplicaSet{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "rs1"}, Spec: appsv1.ReplicaSetSpec{Selector: fooSelector}},
},
want: &preScoreState{
Constraints: []topologySpreadConstraint{
{
MaxSkew: 3,
TopologyKey: v1.LabelHostname,
Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()),
Selector: mustConvertLabelSelectorAsSelector(t, fooSelector),
MinDomains: 1,
NodeAffinityPolicy: v1.NodeInclusionPolicyHonor,
NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore,
},
{
MaxSkew: 5,
TopologyKey: v1.LabelTopologyZone,
Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()),
Selector: mustConvertLabelSelectorAsSelector(t, fooSelector),
MinDomains: 1,
NodeAffinityPolicy: v1.NodeInclusionPolicyHonor,
NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore,
},
},
IgnoredNodes: sets.NewString(),
@ -166,8 +180,15 @@ func TestPreScoreStateEmptyNodes(t *testing.T) {
pod: st.MakePod().Name("p").Namespace("default").Label("foo", "tar").Label("baz", "sup").OwnerReference("rs1", appsv1.SchemeGroupVersion.WithKind("ReplicaSet")).Obj(),
config: config.PodTopologySpreadArgs{
DefaultConstraints: []v1.TopologySpreadConstraint{
{MaxSkew: 1, TopologyKey: v1.LabelHostname, WhenUnsatisfiable: v1.ScheduleAnyway},
{MaxSkew: 2, TopologyKey: "rack", WhenUnsatisfiable: v1.DoNotSchedule},
{
MaxSkew: 1,
TopologyKey: v1.LabelHostname,
WhenUnsatisfiable: v1.ScheduleAnyway,
},
{MaxSkew: 2,
TopologyKey: "rack",
WhenUnsatisfiable: v1.DoNotSchedule,
},
{MaxSkew: 2, TopologyKey: "planet", WhenUnsatisfiable: v1.ScheduleAnyway},
},
DefaultingType: config.ListDefaulting,
@ -176,21 +197,25 @@ func TestPreScoreStateEmptyNodes(t *testing.T) {
st.MakeNode().Name("node-a").Label("rack", "rack1").Label(v1.LabelHostname, "node-a").Label("planet", "mars").Obj(),
},
objs: []runtime.Object{
&appsv1.ReplicaSet{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "rs1"}, Spec: appsv1.ReplicaSetSpec{Selector: st.MakeLabelSelector().Exists("foo").Obj()}},
&appsv1.ReplicaSet{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "rs1"}, Spec: appsv1.ReplicaSetSpec{Selector: fooSelector}},
},
want: &preScoreState{
Constraints: []topologySpreadConstraint{
{
MaxSkew: 1,
TopologyKey: v1.LabelHostname,
Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()),
Selector: mustConvertLabelSelectorAsSelector(t, fooSelector),
MinDomains: 1,
NodeAffinityPolicy: v1.NodeInclusionPolicyHonor,
NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore,
},
{
MaxSkew: 2,
TopologyKey: "planet",
Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()),
Selector: mustConvertLabelSelectorAsSelector(t, fooSelector),
MinDomains: 1,
NodeAffinityPolicy: v1.NodeInclusionPolicyHonor,
NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore,
},
},
IgnoredNodes: sets.NewString(),
@ -205,7 +230,11 @@ func TestPreScoreStateEmptyNodes(t *testing.T) {
pod: st.MakePod().Name("p").Namespace("default").Label("foo", "bar").Label("baz", "sup").OwnerReference("rs2", appsv1.SchemeGroupVersion.WithKind("ReplicaSet")).Obj(),
config: config.PodTopologySpreadArgs{
DefaultConstraints: []v1.TopologySpreadConstraint{
{MaxSkew: 2, TopologyKey: "planet", WhenUnsatisfiable: v1.ScheduleAnyway},
{
MaxSkew: 2,
TopologyKey: "planet",
WhenUnsatisfiable: v1.ScheduleAnyway,
},
},
DefaultingType: config.ListDefaulting,
},
@ -223,11 +252,16 @@ func TestPreScoreStateEmptyNodes(t *testing.T) {
name: "default constraints and a replicaset, but pod has constraints",
pod: st.MakePod().Name("p").Namespace("default").Label("foo", "bar").Label("baz", "sup").
OwnerReference("rs1", appsv1.SchemeGroupVersion.WithKind("ReplicaSet")).
SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Label("foo", "bar").Obj(), nil).
SpreadConstraint(2, "planet", v1.ScheduleAnyway, st.MakeLabelSelector().Label("baz", "sup").Obj(), nil).Obj(),
SpreadConstraint(1, "zone", v1.DoNotSchedule, barSelector, nil, nil, nil).
SpreadConstraint(2, "planet", v1.ScheduleAnyway, st.MakeLabelSelector().Label("baz", "sup").Obj(), nil, nil, nil).
Obj(),
config: config.PodTopologySpreadArgs{
DefaultConstraints: []v1.TopologySpreadConstraint{
{MaxSkew: 2, TopologyKey: "galaxy", WhenUnsatisfiable: v1.ScheduleAnyway},
{
MaxSkew: 2,
TopologyKey: "galaxy",
WhenUnsatisfiable: v1.ScheduleAnyway,
},
},
DefaultingType: config.ListDefaulting,
},
@ -235,7 +269,7 @@ func TestPreScoreStateEmptyNodes(t *testing.T) {
st.MakeNode().Name("node-a").Label("planet", "mars").Label("galaxy", "andromeda").Obj(),
},
objs: []runtime.Object{
&appsv1.ReplicaSet{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "rs1"}, Spec: appsv1.ReplicaSetSpec{Selector: st.MakeLabelSelector().Exists("foo").Obj()}},
&appsv1.ReplicaSet{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "rs1"}, Spec: appsv1.ReplicaSetSpec{Selector: fooSelector}},
},
want: &preScoreState{
Constraints: []topologySpreadConstraint{
@ -244,6 +278,8 @@ func TestPreScoreStateEmptyNodes(t *testing.T) {
TopologyKey: "planet",
Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Label("baz", "sup").Obj()),
MinDomains: 1,
NodeAffinityPolicy: v1.NodeInclusionPolicyHonor,
NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore,
},
},
IgnoredNodes: sets.NewString(),
@ -253,6 +289,208 @@ func TestPreScoreStateEmptyNodes(t *testing.T) {
TopologyNormalizingWeight: []float64{topologyNormalizingWeight(1)},
},
},
{
name: "NodeAffinityPolicy honored with labelSelectors",
pod: st.MakePod().Name("p").Label("foo", "").
NodeSelector(map[string]string{"foo": ""}).
SpreadConstraint(1, "zone", v1.ScheduleAnyway, barSelector, nil, nil, nil).
Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("foo", "").Label(v1.LabelHostname, "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("foo", "").Label(v1.LabelHostname, "node-b").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label(v1.LabelHostname, "node-x").Obj(),
},
config: config.PodTopologySpreadArgs{
DefaultingType: config.ListDefaulting,
},
want: &preScoreState{
Constraints: []topologySpreadConstraint{
{
MaxSkew: 1,
TopologyKey: "zone",
Selector: mustConvertLabelSelectorAsSelector(t, barSelector),
MinDomains: 1,
NodeAffinityPolicy: v1.NodeInclusionPolicyHonor,
NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore,
},
},
IgnoredNodes: sets.NewString(),
TopologyPairToPodCounts: map[topologyPair]*int64{
{key: "zone", value: "zone1"}: pointer.Int64Ptr(0),
{key: "zone", value: "zone2"}: pointer.Int64Ptr(0),
},
TopologyNormalizingWeight: []float64{topologyNormalizingWeight(2)},
},
enableNodeInclustionPolicy: true,
},
{
name: "NodeAffinityPolicy ignored with labelSelectors",
pod: st.MakePod().Name("p").Label("foo", "").
NodeSelector(map[string]string{"foo": ""}).
SpreadConstraint(1, "zone", v1.ScheduleAnyway, barSelector, nil, &ignorePolicy, nil).
Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("foo", "").Label(v1.LabelHostname, "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("foo", "").Label(v1.LabelHostname, "node-b").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label(v1.LabelHostname, "node-x").Obj(),
},
config: config.PodTopologySpreadArgs{
DefaultingType: config.ListDefaulting,
},
want: &preScoreState{
Constraints: []topologySpreadConstraint{
{
MaxSkew: 1,
TopologyKey: "zone",
Selector: mustConvertLabelSelectorAsSelector(t, barSelector),
MinDomains: 1,
NodeAffinityPolicy: v1.NodeInclusionPolicyIgnore,
NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore,
},
},
IgnoredNodes: sets.NewString(),
TopologyPairToPodCounts: map[topologyPair]*int64{
{key: "zone", value: "zone1"}: pointer.Int64Ptr(0),
{key: "zone", value: "zone2"}: pointer.Int64Ptr(0),
},
TopologyNormalizingWeight: []float64{topologyNormalizingWeight(2)},
},
enableNodeInclustionPolicy: true,
},
{
name: "NodeAffinityPolicy honored with nodeAffinity",
pod: st.MakePod().Name("p").Label("foo", "").
NodeAffinityIn("foo", []string{""}).
SpreadConstraint(1, "zone", v1.ScheduleAnyway, barSelector, nil, nil, nil).
Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("foo", "").Label(v1.LabelHostname, "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("foo", "").Label(v1.LabelHostname, "node-b").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label(v1.LabelHostname, "node-x").Obj(),
},
config: config.PodTopologySpreadArgs{
DefaultingType: config.ListDefaulting,
},
want: &preScoreState{
Constraints: []topologySpreadConstraint{
{
MaxSkew: 1,
TopologyKey: "zone",
Selector: mustConvertLabelSelectorAsSelector(t, barSelector),
MinDomains: 1,
NodeAffinityPolicy: v1.NodeInclusionPolicyHonor,
NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore,
},
},
IgnoredNodes: sets.NewString(),
TopologyPairToPodCounts: map[topologyPair]*int64{
{key: "zone", value: "zone1"}: pointer.Int64Ptr(0),
{key: "zone", value: "zone2"}: pointer.Int64Ptr(0),
},
TopologyNormalizingWeight: []float64{topologyNormalizingWeight(2)},
},
enableNodeInclustionPolicy: true,
},
{
name: "NodeAffinityPolicy ignored with nodeAffinity",
pod: st.MakePod().Name("p").Label("foo", "").
NodeAffinityIn("foo", []string{""}).
SpreadConstraint(1, "zone", v1.ScheduleAnyway, barSelector, nil, &ignorePolicy, nil).
Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("foo", "").Label(v1.LabelHostname, "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("foo", "").Label(v1.LabelHostname, "node-b").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label(v1.LabelHostname, "node-x").Obj(),
},
config: config.PodTopologySpreadArgs{
DefaultingType: config.ListDefaulting,
},
want: &preScoreState{
Constraints: []topologySpreadConstraint{
{
MaxSkew: 1,
TopologyKey: "zone",
Selector: mustConvertLabelSelectorAsSelector(t, barSelector),
MinDomains: 1,
NodeAffinityPolicy: v1.NodeInclusionPolicyIgnore,
NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore,
},
},
IgnoredNodes: sets.NewString(),
TopologyPairToPodCounts: map[topologyPair]*int64{
{key: "zone", value: "zone1"}: pointer.Int64Ptr(0),
{key: "zone", value: "zone2"}: pointer.Int64Ptr(0),
},
TopologyNormalizingWeight: []float64{topologyNormalizingWeight(2)},
},
enableNodeInclustionPolicy: true,
},
{
name: "NodeTaintsPolicy honored",
pod: st.MakePod().Name("p").Label("foo", "").
SpreadConstraint(1, "zone", v1.ScheduleAnyway, barSelector, nil, nil, &honorPolicy).
Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("foo", "").Label(v1.LabelHostname, "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("foo", "").Label(v1.LabelHostname, "node-b").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label(v1.LabelHostname, "node-x").Taints(taints).Obj(),
},
config: config.PodTopologySpreadArgs{
DefaultingType: config.ListDefaulting,
},
want: &preScoreState{
Constraints: []topologySpreadConstraint{
{
MaxSkew: 1,
TopologyKey: "zone",
Selector: mustConvertLabelSelectorAsSelector(t, barSelector),
MinDomains: 1,
NodeAffinityPolicy: v1.NodeInclusionPolicyHonor,
NodeTaintsPolicy: v1.NodeInclusionPolicyHonor,
},
},
IgnoredNodes: sets.NewString(),
TopologyPairToPodCounts: map[topologyPair]*int64{
{key: "zone", value: "zone1"}: pointer.Int64Ptr(0),
{key: "zone", value: "zone2"}: pointer.Int64Ptr(0),
},
TopologyNormalizingWeight: []float64{topologyNormalizingWeight(2)},
},
enableNodeInclustionPolicy: true,
},
{
name: "NodeTaintsPolicy ignored",
pod: st.MakePod().Name("p").Label("foo", "").
SpreadConstraint(1, "zone", v1.ScheduleAnyway, barSelector, nil, nil, nil).
Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("foo", "").Label(v1.LabelHostname, "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("foo", "").Label(v1.LabelHostname, "node-b").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label(v1.LabelHostname, "node-x").Taints(taints).Obj(),
},
config: config.PodTopologySpreadArgs{
DefaultingType: config.ListDefaulting,
},
want: &preScoreState{
Constraints: []topologySpreadConstraint{
{
MaxSkew: 1,
TopologyKey: "zone",
Selector: mustConvertLabelSelectorAsSelector(t, barSelector),
MinDomains: 1,
NodeAffinityPolicy: v1.NodeInclusionPolicyHonor,
NodeTaintsPolicy: v1.NodeInclusionPolicyIgnore,
},
},
IgnoredNodes: sets.NewString(),
TopologyPairToPodCounts: map[topologyPair]*int64{
{key: "zone", value: "zone1"}: pointer.Int64Ptr(0),
{key: "zone", value: "zone2"}: pointer.Int64Ptr(0),
},
TopologyNormalizingWeight: []float64{topologyNormalizingWeight(2)},
},
enableNodeInclustionPolicy: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@ -265,7 +503,7 @@ func TestPreScoreStateEmptyNodes(t *testing.T) {
if err != nil {
t.Fatalf("Failed creating framework runtime: %v", err)
}
pl, err := New(&tt.config, f, feature.Features{})
pl, err := New(&tt.config, f, feature.Features{EnableNodeInclusionPolicyInPodTopologySpread: tt.enableNodeInclustionPolicy})
if err != nil {
t.Fatalf("Failed creating plugin: %v", err)
}
@ -297,18 +535,18 @@ func TestPodTopologySpreadScore(t *testing.T) {
failedNodes []*v1.Node // nodes + failedNodes = all nodes
objs []runtime.Object
want framework.NodeScoreList
enableNodeInclustionPolicy bool
}{
// Explanation on the Legend:
// a) X/Y means there are X matching pods on node1 and Y on node2, both nodes are candidates
// (i.e. they have passed all predicates)
// b) X/~Y~ means there are X matching pods on node1 and Y on node2, but node Y is NOT a candidate
// c) X/?Y? means there are X matching pods on node1 and Y on node2, both nodes are candidates
// but node2 either i) doesn't have all required topologyKeys present, or ii) doesn't match
// incoming pod's nodeSelector/nodeAffinity
// but node2 doesn't have all required topologyKeys present.
{
name: "one constraint on node, no existing pods",
pod: st.MakePod().Name("p").Label("foo", "").
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil).
Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label(v1.LabelHostname, "node-a").Obj(),
@ -323,7 +561,7 @@ func TestPodTopologySpreadScore(t *testing.T) {
// if there is only one candidate node, it should be scored to 100
name: "one constraint on node, only one node is candidate",
pod: st.MakePod().Name("p").Label("foo", "").
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
@ -343,7 +581,7 @@ func TestPodTopologySpreadScore(t *testing.T) {
{
name: "one constraint on node, all nodes have the same number of matching pods",
pod: st.MakePod().Name("p").Label("foo", "").
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
@ -362,7 +600,7 @@ func TestPodTopologySpreadScore(t *testing.T) {
// matching pods spread as 2/1/0/3.
name: "one constraint on node, all 4 nodes are candidates",
pod: st.MakePod().Name("p").Label("foo", "").
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
@ -389,7 +627,7 @@ func TestPodTopologySpreadScore(t *testing.T) {
{
name: "one constraint on node, all 4 nodes are candidates, maxSkew=2",
pod: st.MakePod().Name("p").Label("foo", "").
SpreadConstraint(2, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(2, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil).
Obj(),
// matching pods spread as 2/1/0/3.
existingPods: []*v1.Pod{
@ -417,7 +655,7 @@ func TestPodTopologySpreadScore(t *testing.T) {
{
name: "one constraint on node, all 4 nodes are candidates, maxSkew=3",
pod: st.MakePod().Name("p").Label("foo", "").
SpreadConstraint(3, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(3, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil).
Obj(),
existingPods: []*v1.Pod{
// matching pods spread as 4/3/2/1.
@ -484,7 +722,7 @@ func TestPodTopologySpreadScore(t *testing.T) {
// matching pods spread as 4/2/1/~3~ (node4 is not a candidate)
name: "one constraint on node, 3 out of 4 nodes are candidates",
pod: st.MakePod().Name("p").Label("foo", "").
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
@ -516,7 +754,7 @@ func TestPodTopologySpreadScore(t *testing.T) {
// matching pods spread as 4/?2?/1/~3~, total = 4+?+1 = 5 (as node2 is problematic)
name: "one constraint on node, 3 out of 4 nodes are candidates, one node doesn't match topology key",
pod: st.MakePod().Name("p").Label("foo", "").
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
@ -548,7 +786,7 @@ func TestPodTopologySpreadScore(t *testing.T) {
// matching pods spread as 4/2/1/~3~
name: "one constraint on zone, 3 out of 4 nodes are candidates",
pod: st.MakePod().Name("p").Label("foo", "").
SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, "zone", v1.ScheduleAnyway, fooSelector, nil, nil, nil).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
@ -580,8 +818,8 @@ func TestPodTopologySpreadScore(t *testing.T) {
// matching pods spread as 2/~1~/2/~4~.
name: "two Constraints on zone and node, 2 out of 4 nodes are candidates",
pod: st.MakePod().Name("p").Label("foo", "").
SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, "zone", v1.ScheduleAnyway, fooSelector, nil, nil, nil).
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
@ -620,8 +858,8 @@ func TestPodTopologySpreadScore(t *testing.T) {
// For the second constraint (node): the matching pods spread as 0/1/0/1
name: "two Constraints on zone and node, with different labelSelectors",
pod: st.MakePod().Name("p").Label("foo", "").Label("bar", "").
SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("bar").Obj(), nil).
SpreadConstraint(1, "zone", v1.ScheduleAnyway, fooSelector, nil, nil, nil).
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, barSelector, nil, nil, nil).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
@ -648,8 +886,8 @@ func TestPodTopologySpreadScore(t *testing.T) {
// For the second constraint (node): the matching pods spread as 0/1/0/1
name: "two Constraints on zone and node, with different labelSelectors, some nodes have 0 pods",
pod: st.MakePod().Name("p").Label("foo", "").Label("bar", "").
SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("bar").Obj(), nil).
SpreadConstraint(1, "zone", v1.ScheduleAnyway, fooSelector, nil, nil, nil).
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, barSelector, nil, nil, nil).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p-b1").Node("node-b").Label("bar", "").Obj(),
@ -675,8 +913,8 @@ func TestPodTopologySpreadScore(t *testing.T) {
// For the second constraint (node): the matching pods spread as 0/1/0/~1~
name: "two Constraints on zone and node, with different labelSelectors, 3 out of 4 nodes are candidates",
pod: st.MakePod().Name("p").Label("foo", "").Label("bar", "").
SpreadConstraint(1, "zone", v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("bar").Obj(), nil).
SpreadConstraint(1, "zone", v1.ScheduleAnyway, fooSelector, nil, nil, nil).
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, barSelector, nil, nil, nil).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
@ -701,7 +939,7 @@ func TestPodTopologySpreadScore(t *testing.T) {
{
name: "existing pods in a different namespace do not count",
pod: st.MakePod().Name("p").Label("foo", "").
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Namespace("ns1").Node("node-a").Label("foo", "").Obj(),
@ -720,9 +958,9 @@ func TestPodTopologySpreadScore(t *testing.T) {
},
{
name: "terminating Pods should be excluded",
pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint(
1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil,
).Obj(),
pod: st.MakePod().Name("p").Label("foo", "").
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil).
Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label(v1.LabelHostname, "node-a").Obj(),
st.MakeNode().Name("node-b").Label(v1.LabelHostname, "node-b").Obj(),
@ -744,8 +982,10 @@ func TestPodTopologySpreadScore(t *testing.T) {
2,
"node",
v1.ScheduleAnyway,
st.MakeLabelSelector().Exists("foo").Obj(),
fooSelector,
pointer.Int32(10), // larger than the number of domains(3)
nil,
nil,
).Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("node", "node-a").Obj(),
@ -765,6 +1005,148 @@ func TestPodTopologySpreadScore(t *testing.T) {
{Name: "node-c", Score: 100},
},
},
{
name: "NodeAffinityPolicy honoed with labelSelectors",
pod: st.MakePod().Name("p").Label("foo", "").
NodeSelector(map[string]string{"foo": ""}).
SpreadConstraint(1, "node", v1.ScheduleAnyway, fooSelector, nil, nil, nil).
Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("node", "node-a").Label("foo", "").Obj(),
st.MakeNode().Name("node-b").Label("node", "node-b").Label("foo", "").Obj(),
st.MakeNode().Name("node-c").Label("node", "node-c").Obj(),
},
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(),
st.MakePod().Name("p-c1").Node("node-c").Label("foo", "").Obj(),
},
want: []framework.NodeScore{
{Name: "node-a", Score: 0},
{Name: "node-b", Score: 33},
{Name: "node-c", Score: 100},
},
enableNodeInclustionPolicy: true,
},
{
name: "NodeAffinityPolicy ignored with labelSelectors",
pod: st.MakePod().Name("p").Label("foo", "").
NodeSelector(map[string]string{"foo": ""}).
SpreadConstraint(1, "node", v1.ScheduleAnyway, fooSelector, nil, &ignorePolicy, nil).
Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("node", "node-a").Label("foo", "").Obj(),
st.MakeNode().Name("node-b").Label("node", "node-b").Label("foo", "").Obj(),
st.MakeNode().Name("node-c").Label("node", "node-c").Obj(),
},
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(),
st.MakePod().Name("p-c1").Node("node-c").Label("foo", "").Obj(),
},
want: []framework.NodeScore{
{Name: "node-a", Score: 66},
{Name: "node-b", Score: 100},
{Name: "node-c", Score: 100},
},
enableNodeInclustionPolicy: true,
},
{
name: "NodeAffinityPolicy honoed with nodeAffinity",
pod: st.MakePod().Name("p").Label("foo", "").
NodeAffinityIn("foo", []string{""}).
SpreadConstraint(1, "node", v1.ScheduleAnyway, fooSelector, nil, nil, nil).
Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("node", "node-a").Label("foo", "").Obj(),
st.MakeNode().Name("node-b").Label("node", "node-b").Label("foo", "").Obj(),
st.MakeNode().Name("node-c").Label("node", "node-c").Obj(),
},
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(),
st.MakePod().Name("p-c1").Node("node-c").Label("foo", "").Obj(),
},
want: []framework.NodeScore{
{Name: "node-a", Score: 0},
{Name: "node-b", Score: 33},
{Name: "node-c", Score: 100},
},
enableNodeInclustionPolicy: true,
},
{
name: "NodeAffinityPolicy ignored with nodeAffinity",
pod: st.MakePod().Name("p").Label("foo", "").
NodeAffinityIn("foo", []string{""}).
SpreadConstraint(1, "node", v1.ScheduleAnyway, fooSelector, nil, &ignorePolicy, nil).
Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("node", "node-a").Label("foo", "").Obj(),
st.MakeNode().Name("node-b").Label("node", "node-b").Label("foo", "").Obj(),
st.MakeNode().Name("node-c").Label("node", "node-c").Obj(),
},
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(),
st.MakePod().Name("p-c1").Node("node-c").Label("foo", "").Obj(),
},
want: []framework.NodeScore{
{Name: "node-a", Score: 66},
{Name: "node-b", Score: 100},
{Name: "node-c", Score: 100},
},
enableNodeInclustionPolicy: true,
},
{
name: "NodeTaintsPolicy honored",
pod: st.MakePod().Name("p").Label("foo", "").
SpreadConstraint(1, "node", v1.ScheduleAnyway, fooSelector, nil, nil, &honorPolicy).
Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("node", "node-a").Obj(),
st.MakeNode().Name("node-b").Label("node", "node-b").Obj(),
st.MakeNode().Name("node-c").Label("node", "node-c").Taints(taints).Obj(),
},
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(),
st.MakePod().Name("p-c1").Node("node-c").Label("foo", "").Obj(),
},
want: []framework.NodeScore{
{Name: "node-a", Score: 0},
{Name: "node-b", Score: 33},
{Name: "node-c", Score: 100},
},
enableNodeInclustionPolicy: true,
},
{
name: "NodeTaintsPolicy ignored",
pod: st.MakePod().Name("p").Label("foo", "").
SpreadConstraint(1, "node", v1.ScheduleAnyway, fooSelector, nil, nil, nil).
Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("node", "node-a").Obj(),
st.MakeNode().Name("node-b").Label("node", "node-b").Obj(),
st.MakeNode().Name("node-c").Label("node", "node-c").Taints(taints).Obj(),
},
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(),
st.MakePod().Name("p-c1").Node("node-c").Label("foo", "").Obj(),
},
want: []framework.NodeScore{
{Name: "node-a", Score: 66},
{Name: "node-b", Score: 100},
{Name: "node-c", Score: 100},
},
enableNodeInclustionPolicy: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@ -775,6 +1157,7 @@ func TestPodTopologySpreadScore(t *testing.T) {
state := framework.NewCycleState()
pl := plugintesting.SetupPluginWithInformers(ctx, t, podTopologySpreadFunc, &config.PodTopologySpreadArgs{DefaultingType: config.SystemDefaulting}, cache.NewSnapshot(tt.existingPods, allNodes), tt.objs)
p := pl.(*PodTopologySpread)
p.enableNodeInclusionPolicyInPodTopologySpread = tt.enableNodeInclustionPolicy
status := p.PreScore(context.Background(), state, tt.pod, tt.nodes)
if !status.IsSuccess() {
@ -785,12 +1168,14 @@ func TestPodTopologySpreadScore(t *testing.T) {
for _, n := range tt.nodes {
nodeName := n.Name
score, status := p.Score(ctx, state, tt.pod, nodeName)
fmt.Println("get score", score)
if !status.IsSuccess() {
t.Errorf("unexpected error: %v", status)
}
gotList = append(gotList, framework.NodeScore{Name: nodeName, Score: score})
}
fmt.Println(gotList)
status = p.NormalizeScore(ctx, state, tt.pod, gotList)
if !status.IsSuccess() {
t.Errorf("unexpected error: %v", status)
@ -813,7 +1198,7 @@ func BenchmarkTestPodTopologySpreadScore(b *testing.B) {
{
name: "1000nodes/single-constraint-zone",
pod: st.MakePod().Name("p").Label("foo", "").
SpreadConstraint(1, v1.LabelTopologyZone, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, v1.LabelTopologyZone, v1.ScheduleAnyway, fooSelector, nil, nil, nil).
Obj(),
existingPodsNum: 10000,
allNodesNum: 1000,
@ -822,7 +1207,7 @@ func BenchmarkTestPodTopologySpreadScore(b *testing.B) {
{
name: "1000nodes/single-constraint-node",
pod: st.MakePod().Name("p").Label("foo", "").
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, fooSelector, nil, nil, nil).
Obj(),
existingPodsNum: 10000,
allNodesNum: 1000,
@ -831,8 +1216,8 @@ func BenchmarkTestPodTopologySpreadScore(b *testing.B) {
{
name: "1000nodes/two-Constraints-zone-node",
pod: st.MakePod().Name("p").Label("foo", "").Label("bar", "").
SpreadConstraint(1, v1.LabelTopologyZone, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, st.MakeLabelSelector().Exists("bar").Obj(), nil).
SpreadConstraint(1, v1.LabelTopologyZone, v1.ScheduleAnyway, fooSelector, nil, nil, nil).
SpreadConstraint(1, v1.LabelHostname, v1.ScheduleAnyway, barSelector, nil, nil, nil).
Obj(),
existingPodsNum: 10000,
allNodesNum: 1000,

1
pkg/scheduler/framework/plugins/registry.go
View File

@ -49,6 +49,7 @@ func NewInTreeRegistry() runtime.Registry {
EnableReadWriteOncePod: feature.DefaultFeatureGate.Enabled(features.ReadWriteOncePod),
EnableVolumeCapacityPriority: feature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority),
EnableMinDomainsInPodTopologySpread: feature.DefaultFeatureGate.Enabled(features.MinDomainsInPodTopologySpread),
EnableNodeInclusionPolicyInPodTopologySpread: feature.DefaultFeatureGate.Enabled(features.NodeInclusionPolicyInPodTopologySpread),
}
return runtime.Registry{

4
pkg/scheduler/schedule_one_test.go
View File

@ -1754,7 +1754,7 @@ func TestSchedulerSchedulePod(t *testing.T) {
Operator: metav1.LabelSelectorOpExists,
},
},
}, nil).Obj(),
}, nil, nil, nil).Obj(),
pods: []*v1.Pod{
st.MakePod().Name("pod1").UID("pod1").Label("foo", "").Node("node1").Phase(v1.PodRunning).Obj(),
},
@ -1781,7 +1781,7 @@ func TestSchedulerSchedulePod(t *testing.T) {
Operator: metav1.LabelSelectorOpExists,
},
},
}, nil).Obj(),
}, nil, nil, nil).Obj(),
pods: []*v1.Pod{
st.MakePod().Name("pod1a").UID("pod1a").Label("foo", "").Node("node1").Phase(v1.PodRunning).Obj(),
st.MakePod().Name("pod1b").UID("pod1b").Label("foo", "").Node("node1").Phase(v1.PodRunning).Obj(),

4
pkg/scheduler/testing/wrappers.go
View File

@ -547,13 +547,15 @@ func (p *PodWrapper) PodAntiAffinityNotIn(labelKey, topologyKey string, vals []s
// SpreadConstraint constructs a TopologySpreadConstraint object and injects
// into the inner pod.
func (p *PodWrapper) SpreadConstraint(maxSkew int, tpKey string, mode v1.UnsatisfiableConstraintAction, selector *metav1.LabelSelector, minDomains *int32) *PodWrapper {
func (p *PodWrapper) SpreadConstraint(maxSkew int, tpKey string, mode v1.UnsatisfiableConstraintAction, selector *metav1.LabelSelector, minDomains *int32, nodeAffinityPolicy, nodeTaintsPolicy *v1.NodeInclusionPolicy) *PodWrapper {
c := v1.TopologySpreadConstraint{
MaxSkew: int32(maxSkew),
TopologyKey: tpKey,
WhenUnsatisfiable: mode,
LabelSelector: selector,
MinDomains: minDomains,
NodeAffinityPolicy: nodeAffinityPolicy,
NodeTaintsPolicy: nodeTaintsPolicy,
}
p.Spec.TopologySpreadConstraints = append(p.Spec.TopologySpreadConstraints, c)
return p

225
test/integration/scheduler/filters/filters_test.go
View File

@ -54,6 +54,12 @@ var (
const pollInterval = 100 * time.Millisecond
var (
ignorePolicy = v1.NodeInclusionPolicyIgnore
honorPolicy = v1.NodeInclusionPolicyHonor
taints = []v1.Taint{{Key: v1.TaintNodeUnschedulable, Value: "", Effect: v1.TaintEffectPreferNoSchedule}}
)
// TestInterPodAffinity verifies that scheduler's inter pod affinity and
// anti-affinity predicate functions works correctly.
func TestInterPodAffinity(t *testing.T) {
@ -1053,19 +1059,29 @@ func TestInterPodAffinityWithNamespaceSelector(t *testing.T) {
// TestPodTopologySpreadFilter verifies that EvenPodsSpread predicate functions well.
func TestPodTopologySpreadFilter(t *testing.T) {
pause := imageutils.GetPauseImageName()
// default nodes with labels "zone: zone-{0,1}" and "node: <node name>".
defaultNodes := []*v1.Node{
st.MakeNode().Name("node-0").Label("node", "node-0").Label("zone", "zone-0").Obj(),
st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-0").Obj(),
st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Obj(),
st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-1").Obj(),
}
tests := []struct {
name string
incomingPod *v1.Pod
existingPods []*v1.Pod
fits bool
nodes []*v1.Node
candidateNodes []string // nodes expected to schedule onto
enableMinDomains bool
enableNodeInclustionPolicy bool
}{
// note: naming starts at index 0
{
name: "place pod on a 1/1/0/1 cluster with MaxSkew=1, node-2 is the only fit",
incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause).
SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p0").Node("node-0").Label("foo", "").Container(pause).Obj(),
@ -1073,12 +1089,13 @@ func TestPodTopologySpreadFilter(t *testing.T) {
st.MakePod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(),
},
fits: true,
nodes: defaultNodes,
candidateNodes: []string{"node-2"},
},
{
name: "place pod on a 2/0/0/1 cluster with MaxSkew=2, node-{1,2,3} are good fits",
incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause).
SpreadConstraint(2, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(2, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p0a").Node("node-0").Label("foo", "").Container(pause).Obj(),
@ -1086,26 +1103,28 @@ func TestPodTopologySpreadFilter(t *testing.T) {
st.MakePod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(),
},
fits: true,
nodes: defaultNodes,
candidateNodes: []string{"node-1", "node-2", "node-3"},
},
{
name: "pod is required to be placed on zone0, so only node-1 fits",
incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause).
NodeAffinityIn("zone", []string{"zone-0"}).
SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p0").Node("node-0").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(),
},
fits: true,
nodes: defaultNodes,
candidateNodes: []string{"node-1"},
},
{
name: "two constraints: pod can only be placed to zone-1/node-2",
incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause).
SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil).
SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p0").Node("node-0").Label("foo", "").Container(pause).Obj(),
@ -1114,14 +1133,15 @@ func TestPodTopologySpreadFilter(t *testing.T) {
st.MakePod().Name("p3b").Node("node-3").Label("foo", "").Container(pause).Obj(),
},
fits: true,
nodes: defaultNodes,
candidateNodes: []string{"node-2"},
},
{
name: "pod cannot be placed onto any node",
incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause).
NodeAffinityNotIn("node", []string{"node-0"}). // mock a 3-node cluster
SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil).
SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(),
@ -1131,13 +1151,14 @@ func TestPodTopologySpreadFilter(t *testing.T) {
st.MakePod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(),
},
fits: false,
nodes: defaultNodes,
},
{
name: "high priority pod can preempt others",
incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause).Priority(100).
NodeAffinityNotIn("node", []string{"node-0"}). // mock a 3-node cluster
SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil).
SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().ZeroTerminationGracePeriod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(),
@ -1147,6 +1168,7 @@ func TestPodTopologySpreadFilter(t *testing.T) {
st.MakePod().ZeroTerminationGracePeriod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(),
},
fits: true,
nodes: defaultNodes,
candidateNodes: []string{"node-1", "node-2", "node-3"},
},
{
@ -1159,6 +1181,8 @@ func TestPodTopologySpreadFilter(t *testing.T) {
hardSpread,
st.MakeLabelSelector().Exists("foo").Obj(),
pointer.Int32(4), // larger than the number of domains (= 3)
nil,
nil,
).
Obj(),
existingPods: []*v1.Pod{
@ -1169,6 +1193,7 @@ func TestPodTopologySpreadFilter(t *testing.T) {
st.MakePod().ZeroTerminationGracePeriod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(),
},
fits: true,
nodes: defaultNodes,
candidateNodes: []string{"node-3"},
enableMinDomains: true,
},
@ -1182,6 +1207,8 @@ func TestPodTopologySpreadFilter(t *testing.T) {
hardSpread,
st.MakeLabelSelector().Exists("foo").Obj(),
pointer.Int32(2), // smaller than the number of domains (= 3)
nil,
nil,
).
Obj(),
existingPods: []*v1.Pod{
@ -1192,6 +1219,7 @@ func TestPodTopologySpreadFilter(t *testing.T) {
st.MakePod().ZeroTerminationGracePeriod().Name("p3").Node("node-3").Label("foo", "").Container(pause).Obj(),
},
fits: true,
nodes: defaultNodes,
candidateNodes: []string{"node-1", "node-2", "node-3"},
enableMinDomains: true,
},
@ -1204,6 +1232,8 @@ func TestPodTopologySpreadFilter(t *testing.T) {
v1.DoNotSchedule,
st.MakeLabelSelector().Exists("foo").Obj(),
pointer.Int32(3), // larger than the number of domains(2)
nil,
nil,
).Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p1a").Node("node-0").Label("foo", "").Container(pause).Obj(),
@ -1211,6 +1241,7 @@ func TestPodTopologySpreadFilter(t *testing.T) {
st.MakePod().Name("p3a").Node("node-2").Label("foo", "").Container(pause).Obj(),
},
fits: true,
nodes: defaultNodes,
candidateNodes: []string{"node-2", "node-3"},
enableMinDomains: true,
},
@ -1223,6 +1254,8 @@ func TestPodTopologySpreadFilter(t *testing.T) {
v1.DoNotSchedule,
st.MakeLabelSelector().Exists("foo").Obj(),
pointer.Int32(1), // smaller than the number of domains(2)
nil,
nil,
).Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(),
@ -1230,24 +1263,186 @@ func TestPodTopologySpreadFilter(t *testing.T) {
st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(),
},
fits: true,
nodes: defaultNodes,
candidateNodes: []string{"node-0", "node-1", "node-2", "node-3"},
enableMinDomains: true,
},
{
name: "NodeAffinityPolicy honored with labelSelectors, pods spread across zone as 2/1",
incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause).
NodeSelector(map[string]string{"foo": ""}).
SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(),
},
fits: true,
nodes: []*v1.Node{
st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(),
st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Obj(),
st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Obj(),
st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Label("foo", "").Obj(),
},
candidateNodes: []string{"node-4"}, // node-3 is filtered out by NodeAffinity plugin
enableNodeInclustionPolicy: true,
},
{
name: "NodeAffinityPolicy ignored with nodeAffinity, pods spread across zone as 1/~2~",
incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause).
NodeAffinityIn("foo", []string{""}).
SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, &ignorePolicy, nil).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(),
},
fits: true,
nodes: []*v1.Node{
st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(),
st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Obj(),
st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Obj(),
st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Label("foo", "").Obj(),
},
candidateNodes: []string{"node-1", "node-2"},
enableNodeInclustionPolicy: true,
},
{
name: "NodeTaintsPolicy honored, pods spread across zone as 2/1",
incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause).
SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, &honorPolicy).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(),
},
fits: true,
nodes: []*v1.Node{
st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(),
st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Obj(),
st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Taints(taints).Obj(),
st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Label("foo", "").Obj(),
},
candidateNodes: []string{"node-4"}, // node-3 is filtered out by TaintToleration plugin
enableNodeInclustionPolicy: true,
},
{
name: "NodeTaintsPolicy ignored, pods spread across zone as 2/2",
incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause).
SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(),
},
fits: true,
nodes: []*v1.Node{
st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(),
st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Obj(),
st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Taints(taints).Obj(),
st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Label("foo", "").Obj(),
},
candidateNodes: []string{"node-1", "node-2", "node-4"}, // node-3 is filtered out by TaintToleration plugin
enableNodeInclustionPolicy: true,
},
{
// 1. to fulfil "zone" constraint, pods spread across zones as 2/1
// 2. to fulfil "node" constraint, pods spread across zones as 1/1/~0~/1
// intersection of (1) and (2) returns node-4 as node-3 is filtered out by NodeAffinity plugin.
name: "two node inclusion Constraints, zone: honor/ignore, node: honor/ignore",
incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause).
NodeSelector(map[string]string{"foo": ""}).
SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil).
SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(),
},
fits: true,
nodes: []*v1.Node{
st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(),
st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Taints(taints).Obj(),
st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Obj(),
st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Label("foo", "").Obj(),
},
candidateNodes: []string{"node-4"},
enableNodeInclustionPolicy: true,
},
{
// 1. to fulfil "zone" constraint, pods spread across zones as 2/1
// 2. to fulfil "node" constraint, pods spread across zones as 1/1/~0~/1
// intersection of (1) and (2) returns node-4 as node-3 is filtered out by NodeAffinity plugin
name: "feature gate disabled, two node inclusion Constraints, zone: honor/ignore, node: honor/ignore",
incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause).
NodeSelector(map[string]string{"foo": ""}).
SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil).
SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(),
},
fits: true,
nodes: []*v1.Node{
st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(),
st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Taints(taints).Obj(),
st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Obj(),
st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Label("foo", "").Obj(),
},
candidateNodes: []string{"node-4"},
enableNodeInclustionPolicy: false,
},
{
// 1. to fulfil "zone" constraint, pods spread across zones as 2/2
// 2. to fulfil "node" constraint, pods spread across zones as 1/~0~/~0~/1
// intersection of (1) and (2) returns node-1 and node-4 as node-2, node-3 are filtered out by plugins
name: "two node inclusion Constraints, zone: ignore/ignore, node: honor/honor",
incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause).
NodeSelector(map[string]string{"foo": ""}).
SpreadConstraint(1, "zone", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, &ignorePolicy, nil).
SpreadConstraint(1, "node", v1.DoNotSchedule, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, &honorPolicy).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(),
},
fits: true,
nodes: []*v1.Node{
st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(),
st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Taints(taints).Obj(),
st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Obj(),
st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Label("foo", "").Obj(),
},
candidateNodes: []string{"node-1", "node-4"},
enableNodeInclustionPolicy: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.MinDomainsInPodTopologySpread, tt.enableMinDomains)()
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.NodeInclusionPolicyInPodTopologySpread, tt.enableNodeInclustionPolicy)()
testCtx := initTest(t, "pts-predicate")
cs := testCtx.ClientSet
ns := testCtx.NS.Name
defer testutils.CleanupTest(t, testCtx)
for i := 0; i < 4; i++ {
// Create nodes with labels "zone: zone-{0,1}" and "node: <node name>" to each node.
nodeName := fmt.Sprintf("node-%d", i)
zone := fmt.Sprintf("zone-%d", i/2)
_, err := createNode(cs, st.MakeNode().Name(nodeName).Label("node", nodeName).Label("zone", zone).Obj())
if err != nil {
for i := range tt.nodes {
if _, err := createNode(cs, tt.nodes[i]); err != nil {
t.Fatalf("Cannot create node: %v", err)
}
}

150
test/integration/scheduler/scoring/priorities_test.go
View File

@ -29,7 +29,10 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/util/wait"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kube-scheduler/config/v1beta3"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/scheduler"
configtesting "k8s.io/kubernetes/pkg/scheduler/apis/config/testing"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/imagelocality"
@ -59,6 +62,9 @@ var (
var (
hardSpread = v1.DoNotSchedule
softSpread = v1.ScheduleAnyway
ignorePolicy = v1.NodeInclusionPolicyIgnore
honorPolicy = v1.NodeInclusionPolicyHonor
taints = []v1.Taint{{Key: v1.TaintNodeUnschedulable, Value: "", Effect: v1.TaintEffectPreferNoSchedule}}
)
const (
@ -418,83 +424,142 @@ func makeContainersWithImages(images []string) []v1.Container {
// TestPodTopologySpreadScoring verifies that the PodTopologySpread Score plugin works.
func TestPodTopologySpreadScoring(t *testing.T) {
testCtx := initTestSchedulerForPriorityTest(t, podtopologyspread.Name)
defer testutils.CleanupTest(t, testCtx)
cs := testCtx.ClientSet
ns := testCtx.NS.Name
var nodes []*v1.Node
for i := 0; i < 4; i++ {
// Create nodes with labels "zone: zone-{0,1}" and "node: <node name>" to each node.
nodeName := fmt.Sprintf("node-%d", i)
zone := fmt.Sprintf("zone-%d", i/2)
node, err := createNode(cs, st.MakeNode().Name(nodeName).Label("node", nodeName).Label("zone", zone).Obj())
if err != nil {
t.Fatalf("Cannot create node: %v", err)
}
nodes = append(nodes, node)
}
// Taint the 0th node
pause := imageutils.GetPauseImageName()
taint := v1.Taint{
Key: "k1",
Value: "v1",
Effect: v1.TaintEffectNoSchedule,
}
if err := testutils.AddTaintToNode(cs, nodes[0].Name, taint); err != nil {
t.Fatalf("Adding taint to node failed: %v", err)
}
if err := testutils.WaitForNodeTaints(cs, nodes[0], []v1.Taint{taint}); err != nil {
t.Fatalf("Taint not seen on node: %v", err)
// default nodes with labels "zone: zone-{0,1}" and "node: <node name>".
defaultNodes := []*v1.Node{
st.MakeNode().Name("node-0").Label("node", "node-0").Label("zone", "zone-0").Taints([]v1.Taint{taint}).Obj(),
st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-0").Obj(),
st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Obj(),
st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-1").Obj(),
}
pause := imageutils.GetPauseImageName()
tests := []struct {
name string
incomingPod *v1.Pod
existingPods []*v1.Pod
fits bool
nodes []*v1.Node
want []string // nodes expected to schedule onto
enableNodeInclustionPolicy bool
}{
// note: naming starts at index 0
// the symbol ~X~ means that node is infeasible
{
name: "place pod on a ~0~/1/2/3 cluster with MaxSkew=1, node-1 is the preferred fit",
incomingPod: st.MakePod().Namespace(ns).Name("p").Label("foo", "").Container(pause).
SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil).
incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause).
SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Namespace(ns).Name("p1").Node("node-1").Label("foo", "").Container(pause).Obj(),
st.MakePod().Namespace(ns).Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(),
st.MakePod().Namespace(ns).Name("p2b").Node("node-2").Label("foo", "").Container(pause).Obj(),
st.MakePod().Namespace(ns).Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(),
st.MakePod().Namespace(ns).Name("p3b").Node("node-3").Label("foo", "").Container(pause).Obj(),
st.MakePod().Namespace(ns).Name("p3c").Node("node-3").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p1").Node("node-1").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p2b").Node("node-2").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p3b").Node("node-3").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p3c").Node("node-3").Label("foo", "").Container(pause).Obj(),
},
fits: true,
nodes: defaultNodes,
want: []string{"node-1"},
},
{
name: "combined with hardSpread constraint on a ~4~/0/1/2 cluster",
incomingPod: st.MakePod().Namespace(ns).Name("p").Label("foo", "").Container(pause).
SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil).
SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil).
incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause).
SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil).
SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Namespace(ns).Name("p0a").Node("node-0").Label("foo", "").Container(pause).Obj(),
st.MakePod().Namespace(ns).Name("p0b").Node("node-0").Label("foo", "").Container(pause).Obj(),
st.MakePod().Namespace(ns).Name("p0c").Node("node-0").Label("foo", "").Container(pause).Obj(),
st.MakePod().Namespace(ns).Name("p0d").Node("node-0").Label("foo", "").Container(pause).Obj(),
st.MakePod().Namespace(ns).Name("p2").Node("node-2").Label("foo", "").Container(pause).Obj(),
st.MakePod().Namespace(ns).Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(),
st.MakePod().Namespace(ns).Name("p3b").Node("node-3").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p0a").Node("node-0").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p0b").Node("node-0").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p0c").Node("node-0").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p0d").Node("node-0").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p2").Node("node-2").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p3a").Node("node-3").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p3b").Node("node-3").Label("foo", "").Container(pause).Obj(),
},
fits: true,
nodes: defaultNodes,
want: []string{"node-2"},
},
{
// 1. to fulfil "zone" constraint, pods spread across zones as ~3~/0
// 2. to fulfil "node" constraint, pods spread across zones as 1/~2~/0/~0~
// node-2 and node 4 are filtered out by plugins
name: "soft constraint with two node inclusion Constraints, zone: honor/ignore, node: honor/ignore",
incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause).
NodeSelector(map[string]string{"foo": ""}).
SpreadConstraint(1, "zone", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil).
SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, nil).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p2b").Node("node-2").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(),
},
fits: true,
nodes: []*v1.Node{
st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(),
st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Taints(taints).Obj(),
st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Label("foo", "").Obj(),
st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Obj(),
},
want: []string{"node-3"},
enableNodeInclustionPolicy: true,
},
{
// 1. to fulfil "zone" constraint, pods spread across zones as ~3~/~1~
// 2. to fulfil "node" constraint, pods spread across zones as 1/~0~/0/~0~
// node-2 and node 4 are filtered out by plugins
name: "soft constraint with two node inclusion Constraints, zone: ignore/ignore, node: honor/honor",
incomingPod: st.MakePod().Name("p").Label("foo", "").Container(pause).
NodeSelector(map[string]string{"foo": ""}).
SpreadConstraint(1, "zone", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, &ignorePolicy, nil).
SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("foo").Obj(), nil, nil, &honorPolicy).
Obj(),
existingPods: []*v1.Pod{
st.MakePod().Name("p1a").Node("node-1").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p2a").Node("node-2").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p2b").Node("node-2").Label("foo", "").Container(pause).Obj(),
st.MakePod().Name("p4a").Node("node-4").Label("foo", "").Container(pause).Obj(),
},
fits: true,
nodes: []*v1.Node{
st.MakeNode().Name("node-1").Label("node", "node-1").Label("zone", "zone-1").Label("foo", "").Obj(),
st.MakeNode().Name("node-2").Label("node", "node-2").Label("zone", "zone-1").Label("foo", "").Taints(taints).Obj(),
st.MakeNode().Name("node-3").Label("node", "node-3").Label("zone", "zone-2").Label("foo", "").Obj(),
st.MakeNode().Name("node-4").Label("node", "node-4").Label("zone", "zone-2").Obj(),
},
want: []string{"node-3"},
enableNodeInclustionPolicy: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.NodeInclusionPolicyInPodTopologySpread, tt.enableNodeInclustionPolicy)()
testCtx := initTestSchedulerForPriorityTest(t, podtopologyspread.Name)
defer testutils.CleanupTest(t, testCtx)
cs := testCtx.ClientSet
ns := testCtx.NS.Name
for i := range tt.nodes {
if _, err := createNode(cs, tt.nodes[i]); err != nil {
t.Fatalf("Cannot create node: %v", err)
}
}
// set namespace to pods
for i := range tt.existingPods {
tt.existingPods[i].SetNamespace(ns)
}
tt.incomingPod.SetNamespace(ns)
allPods := append(tt.existingPods, tt.incomingPod)
defer testutils.CleanupPods(cs, t, allPods)
for _, pod := range tt.existingPods {
@ -507,6 +572,7 @@ func TestPodTopologySpreadScoring(t *testing.T) {
t.Errorf("Test Failed: error while waiting for pod during test: %v", err)
}
}
testPod, err := cs.CoreV1().Pods(tt.incomingPod.Namespace).Create(context.TODO(), tt.incomingPod, metav1.CreateOptions{})
if err != nil && !apierrors.IsInvalid(err) {
t.Fatalf("Test Failed: error while creating pod during test: %v", err)