github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-28 14:07:14 +00:00
Code Issues Projects Releases Wiki Activity

Test PodTopologySpread.PreScore instead of internal pre-processing.

Browse Source 
Signed-off-by: Aldo Culquicondor <acondor@google.com>
This commit is contained in:
Aldo Culquicondor 2020-02-26 16:17:34 -05:00
parent 0e46803e77
commit fd5b298c50
3 changed files with 112 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/scheduler/framework/plugins/podtopologyspread/BUILD
View File

@ -41,6 +41,8 @@ go_test(
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/github.com/google/go-cmp/cmp:go_default_library",
"//vendor/k8s.io/utils/pointer:go_default_library",
], ],
) )

67
pkg/scheduler/framework/plugins/podtopologyspread/scoring.go
View File

@ -34,12 +34,13 @@ import (
const preScoreStateKey = "PreScore" + Name const preScoreStateKey = "PreScore" + Name
// preScoreState computed at PreScore and used at Score. // preScoreState computed at PreScore and used at Score.
// Fields are exported for comparison during testing.
type preScoreState struct { type preScoreState struct {
constraints []topologySpreadConstraint Constraints []topologySpreadConstraint
// nodeNameSet is a string set holding all node names which have all constraints[*].topologyKey present. // NodeNameSet is a string set holding all node names which have all Constraints[*].topologyKey present.
nodeNameSet sets.String NodeNameSet sets.String
// topologyPairToPodCounts is keyed with topologyPair, and valued with the number of matching pods. // TopologyPairToPodCounts is keyed with topologyPair, and valued with the number of matching pods.
topologyPairToPodCounts map[topologyPair]*int64 TopologyPairToPodCounts map[topologyPair]*int64
} }
// Clone implements the mandatory Clone interface. We don't really copy the data since // Clone implements the mandatory Clone interface. We don't really copy the data since
@ -50,8 +51,8 @@ func (s *preScoreState) Clone() framework.StateData {
// initialize iterates "filteredNodes" to filter out the nodes which don't have required topologyKey(s), // initialize iterates "filteredNodes" to filter out the nodes which don't have required topologyKey(s),
// and initialize two maps: // and initialize two maps:
// 1) s.topologyPairToPodCounts: keyed with both eligible topology pair and node names. // 1) s.TopologyPairToPodCounts: keyed with both eligible topology pair and node names.
// 2) s.nodeNameSet: keyed with node name, and valued with a *int64 pointer for eligible node only. // 2) s.NodeNameSet: keyed with node name, and valued with a *int64 pointer for eligible node only.
func (s *preScoreState) initialize(pod *v1.Pod, filteredNodes []*v1.Node) error { func (s *preScoreState) initialize(pod *v1.Pod, filteredNodes []*v1.Node) error {
constraints, err := filterTopologySpreadConstraints(pod.Spec.TopologySpreadConstraints, v1.ScheduleAnyway) constraints, err := filterTopologySpreadConstraints(pod.Spec.TopologySpreadConstraints, v1.ScheduleAnyway)
if err != nil { if err != nil {
@ -60,20 +61,20 @@ func (s *preScoreState) initialize(pod *v1.Pod, filteredNodes []*v1.Node) error
if constraints == nil { if constraints == nil {
return nil return nil
} }
s.constraints = constraints s.Constraints = constraints
for _, node := range filteredNodes { for _, node := range filteredNodes {
if !nodeLabelsMatchSpreadConstraints(node.Labels, s.constraints) { if !nodeLabelsMatchSpreadConstraints(node.Labels, s.Constraints) {
continue continue
} }
for _, constraint := range s.constraints { for _, constraint := range s.Constraints {
pair := topologyPair{key: constraint.topologyKey, value: node.Labels[constraint.topologyKey]} pair := topologyPair{key: constraint.TopologyKey, value: node.Labels[constraint.TopologyKey]}
if s.topologyPairToPodCounts[pair] == nil { if s.TopologyPairToPodCounts[pair] == nil {
s.topologyPairToPodCounts[pair] = new(int64) s.TopologyPairToPodCounts[pair] = new(int64)
} }
} }
s.nodeNameSet.Insert(node.Name) s.NodeNameSet.Insert(node.Name)
// For those nodes which don't have all required topologyKeys present, it's intentional to leave // For those nodes which don't have all required topologyKeys present, it's intentional to leave
// their entries absent in nodeNameSet, so that we're able to score them to 0 afterwards. // their entries absent in NodeNameSet, so that we're able to score them to 0 afterwards.
} }
return nil return nil
} }
@ -96,16 +97,16 @@ func (pl *PodTopologySpread) PreScore(
} }
state := &preScoreState{ state := &preScoreState{
nodeNameSet: sets.String{}, NodeNameSet: sets.String{},
topologyPairToPodCounts: make(map[topologyPair]*int64), TopologyPairToPodCounts: make(map[topologyPair]*int64),
} }
err = state.initialize(pod, filteredNodes) err = state.initialize(pod, filteredNodes)
if err != nil { if err != nil {
return framework.NewStatus(framework.Error, fmt.Sprintf("error when calculating preScoreState: %v", err)) return framework.NewStatus(framework.Error, fmt.Sprintf("error when calculating preScoreState: %v", err))
} }
// return if incoming pod doesn't have soft topology spread constraints. // return if incoming pod doesn't have soft topology spread Constraints.
if state.constraints == nil { if state.Constraints == nil {
cycleState.Write(preScoreStateKey, state) cycleState.Write(preScoreStateKey, state)
return nil return nil
} }
@ -119,15 +120,15 @@ func (pl *PodTopologySpread) PreScore(
// (1) `node` should satisfy incoming pod's NodeSelector/NodeAffinity // (1) `node` should satisfy incoming pod's NodeSelector/NodeAffinity
// (2) All topologyKeys need to be present in `node` // (2) All topologyKeys need to be present in `node`
if !pluginhelper.PodMatchesNodeSelectorAndAffinityTerms(pod, node) || if !pluginhelper.PodMatchesNodeSelectorAndAffinityTerms(pod, node) ||
!nodeLabelsMatchSpreadConstraints(node.Labels, state.constraints) { !nodeLabelsMatchSpreadConstraints(node.Labels, state.Constraints) {
return return
} }
for _, c := range state.constraints { for _, c := range state.Constraints {
pair := topologyPair{key: c.topologyKey, value: node.Labels[c.topologyKey]} pair := topologyPair{key: c.TopologyKey, value: node.Labels[c.TopologyKey]}
// If current topology pair is not associated with any candidate node, // If current topology pair is not associated with any candidate node,
// continue to avoid unnecessary calculation. // continue to avoid unnecessary calculation.
if state.topologyPairToPodCounts[pair] == nil { if state.TopologyPairToPodCounts[pair] == nil {
continue continue
} }
@ -138,11 +139,11 @@ func (pl *PodTopologySpread) PreScore(
if existingPod.DeletionTimestamp != nil || existingPod.Namespace != pod.Namespace { if existingPod.DeletionTimestamp != nil || existingPod.Namespace != pod.Namespace {
continue continue
} }
if c.selector.Matches(labels.Set(existingPod.Labels)) { if c.Selector.Matches(labels.Set(existingPod.Labels)) {
matchSum++ matchSum++
} }
} }
atomic.AddInt64(state.topologyPairToPodCounts[pair], matchSum) atomic.AddInt64(state.TopologyPairToPodCounts[pair], matchSum)
} }
} }
workqueue.ParallelizeUntil(ctx, 16, len(allNodes), processAllNode) workqueue.ParallelizeUntil(ctx, 16, len(allNodes), processAllNode)
@ -167,17 +168,17 @@ func (pl *PodTopologySpread) Score(ctx context.Context, cycleState *framework.Cy
} }
// Return if the node is not qualified. // Return if the node is not qualified.
if _, ok := s.nodeNameSet[node.Name]; !ok { if _, ok := s.NodeNameSet[node.Name]; !ok {
return 0, nil return 0, nil
} }
// For each present <pair>, current node gets a credit of <matchSum>. // For each present <pair>, current node gets a credit of <matchSum>.
// And we sum up <matchSum> and return it as this node's score. // And we sum up <matchSum> and return it as this node's score.
var score int64 var score int64
for _, c := range s.constraints { for _, c := range s.Constraints {
if tpVal, ok := node.Labels[c.topologyKey]; ok { if tpVal, ok := node.Labels[c.TopologyKey]; ok {
pair := topologyPair{key: c.topologyKey, value: tpVal} pair := topologyPair{key: c.TopologyKey, value: tpVal}
matchSum := *s.topologyPairToPodCounts[pair] matchSum := *s.TopologyPairToPodCounts[pair]
score += matchSum score += matchSum
} }
} }
@ -198,8 +199,8 @@ func (pl *PodTopologySpread) NormalizeScore(ctx context.Context, cycleState *fra
var minScore int64 = math.MaxInt64 var minScore int64 = math.MaxInt64
var total int64 var total int64
for _, score := range scores { for _, score := range scores {
// it's mandatory to check if <score.Name> is present in m.nodeNameSet // it's mandatory to check if <score.Name> is present in m.NodeNameSet
if _, ok := s.nodeNameSet[score.Name]; !ok { if _, ok := s.NodeNameSet[score.Name]; !ok {
continue continue
} }
total += score.Score total += score.Score
@ -228,7 +229,7 @@ func (pl *PodTopologySpread) NormalizeScore(ctx context.Context, cycleState *fra
continue continue
} }
if _, ok := s.nodeNameSet[node.Name]; !ok { if _, ok := s.NodeNameSet[node.Name]; !ok {
scores[i].Score = 0 scores[i].Score = 0
continue continue
} }

111
pkg/scheduler/framework/plugins/podtopologyspread/scoring_test.go
View File

@ -21,20 +21,31 @@ import (
"reflect" "reflect"
"testing" "testing"
"github.com/google/go-cmp/cmp"
v1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/sets"
framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
"k8s.io/kubernetes/pkg/scheduler/internal/cache" "k8s.io/kubernetes/pkg/scheduler/internal/cache"
st "k8s.io/kubernetes/pkg/scheduler/testing" st "k8s.io/kubernetes/pkg/scheduler/testing"
"k8s.io/utils/pointer"
) )
func TestPreScoreStateInitialize(t *testing.T) { func (s preScoreState) Equal(o preScoreState) bool {
type internal preScoreState
return cmp.Equal(internal(s), internal(o),
cmp.Comparer(func(s1 labels.Selector, s2 labels.Selector) bool {
return reflect.DeepEqual(s1, s2)
}),
)
}
func TestPreScoreStateEmptyNodes(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
pod *v1.Pod pod *v1.Pod
nodes []*v1.Node nodes []*v1.Node
wantNodeNameSet sets.String want *preScoreState
wantTopologyPairMap map[topologyPair]*int64
}{ }{
{ {
name: "normal case", name: "normal case",
@ -47,13 +58,27 @@ func TestPreScoreStateInitialize(t *testing.T) {
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(), st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(), st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
}, },
wantNodeNameSet: sets.NewString("node-a", "node-b", "node-x"), want: &preScoreState{
wantTopologyPairMap: map[topologyPair]*int64{ Constraints: []topologySpreadConstraint{
{key: "zone", value: "zone1"}: new(int64), {
{key: "zone", value: "zone2"}: new(int64), MaxSkew: 1,
{key: "node", value: "node-a"}: new(int64), TopologyKey: "zone",
{key: "node", value: "node-b"}: new(int64), Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()),
{key: "node", value: "node-x"}: new(int64), },
{
MaxSkew: 1,
TopologyKey: "node",
Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()),
},
},
NodeNameSet: sets.NewString("node-a", "node-b", "node-x"),
TopologyPairToPodCounts: map[topologyPair]*int64{
{key: "zone", value: "zone1"}: pointer.Int64Ptr(0),
{key: "zone", value: "zone2"}: pointer.Int64Ptr(0),
{key: "node", value: "node-a"}: pointer.Int64Ptr(0),
{key: "node", value: "node-b"}: pointer.Int64Ptr(0),
{key: "node", value: "node-x"}: pointer.Int64Ptr(0),
},
}, },
}, },
{ {
@ -67,28 +92,44 @@ func TestPreScoreStateInitialize(t *testing.T) {
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(), st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
st.MakeNode().Name("node-x").Label("node", "node-x").Obj(), st.MakeNode().Name("node-x").Label("node", "node-x").Obj(),
}, },
wantNodeNameSet: sets.NewString("node-a", "node-b"), want: &preScoreState{
wantTopologyPairMap: map[topologyPair]*int64{ Constraints: []topologySpreadConstraint{
{key: "zone", value: "zone1"}: new(int64), {
{key: "node", value: "node-a"}: new(int64), MaxSkew: 1,
{key: "node", value: "node-b"}: new(int64), TopologyKey: "zone",
Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()),
},
{
MaxSkew: 1,
TopologyKey: "node",
Selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("bar").Obj()),
},
},
NodeNameSet: sets.NewString("node-a", "node-b"),
TopologyPairToPodCounts: map[topologyPair]*int64{
{key: "zone", value: "zone1"}: pointer.Int64Ptr(0),
{key: "node", value: "node-a"}: pointer.Int64Ptr(0),
{key: "node", value: "node-b"}: pointer.Int64Ptr(0),
},
}, },
}, },
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
s := &preScoreState{ pl := PodTopologySpread{
nodeNameSet: sets.String{}, sharedLister: cache.NewSnapshot(nil, tt.nodes),
topologyPairToPodCounts: make(map[topologyPair]*int64),
} }
if err := s.initialize(tt.pod, tt.nodes); err != nil { cs := framework.NewCycleState()
if s := pl.PreScore(context.Background(), cs, tt.pod, tt.nodes); !s.IsSuccess() {
t.Fatal(s.AsError())
}
got, err := getPreScoreState(cs)
if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if !reflect.DeepEqual(s.nodeNameSet, tt.wantNodeNameSet) { if diff := cmp.Diff(tt.want, got); diff != "" {
t.Errorf("initilize().nodeNameSet = %#v, want %#v", s.nodeNameSet, tt.wantNodeNameSet) t.Errorf("PodTopologySpread#PreScore() returned (-want, +got):\n%s", diff)
}
if !reflect.DeepEqual(s.topologyPairToPodCounts, tt.wantTopologyPairMap) {
t.Errorf("initilize().topologyPairToPodCounts = %#v, want %#v", s.topologyPairToPodCounts, tt.wantTopologyPairMap)
} }
}) })
} }
@ -300,7 +341,7 @@ func TestPodTopologySpreadScore(t *testing.T) {
// matching pods spread as 2/~1~/2/~4~, total = 2+3 + 2+6 = 13 (zone and node should be both summed up) // matching pods spread as 2/~1~/2/~4~, total = 2+3 + 2+6 = 13 (zone and node should be both summed up)
// after reversing, it's 8/5 // after reversing, it's 8/5
// so scores = 800/8, 500/8 // so scores = 800/8, 500/8
name: "two constraints on zone and node, 2 out of 4 nodes are candidates", name: "two Constraints on zone and node, 2 out of 4 nodes are candidates",
pod: st.MakePod().Name("p").Label("foo", ""). pod: st.MakePod().Name("p").Label("foo", "").
SpreadConstraint(1, "zone", softSpread, st.MakeLabelSelector().Exists("foo").Obj()). SpreadConstraint(1, "zone", softSpread, st.MakeLabelSelector().Exists("foo").Obj()).
SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("foo").Obj()). SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("foo").Obj()).
@ -330,7 +371,7 @@ func TestPodTopologySpreadScore(t *testing.T) {
}, },
}, },
{ {
// If constraints hold different labelSelectors, it's a little complex. // If Constraints hold different labelSelectors, it's a little complex.
// +----------------------+------------------------+ // +----------------------+------------------------+
// | zone1 | zone2 | // | zone1 | zone2 |
// +----------------------+------------------------+ // +----------------------+------------------------+
@ -343,7 +384,7 @@ func TestPodTopologySpreadScore(t *testing.T) {
// sum them up gets: 2/3/1/2, and total number is 8. // sum them up gets: 2/3/1/2, and total number is 8.
// after reversing, it's 6/5/7/6 // after reversing, it's 6/5/7/6
// so scores = 600/7, 500/7, 700/7, 600/7 // so scores = 600/7, 500/7, 700/7, 600/7
name: "two constraints on zone and node, with different labelSelectors", name: "two Constraints on zone and node, with different labelSelectors",
pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). pod: st.MakePod().Name("p").Label("foo", "").Label("bar", "").
SpreadConstraint(1, "zone", softSpread, st.MakeLabelSelector().Exists("foo").Obj()). SpreadConstraint(1, "zone", softSpread, st.MakeLabelSelector().Exists("foo").Obj()).
SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("bar").Obj()). SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("bar").Obj()).
@ -374,7 +415,7 @@ func TestPodTopologySpreadScore(t *testing.T) {
// sum them up gets: 0/1/2/3, and total number is 6. // sum them up gets: 0/1/2/3, and total number is 6.
// after reversing, it's 6/5/4/3. // after reversing, it's 6/5/4/3.
// so scores = 600/6, 500/6, 400/6, 300/6 // so scores = 600/6, 500/6, 400/6, 300/6
name: "two constraints on zone and node, with different labelSelectors, some nodes have 0 pods", name: "two Constraints on zone and node, with different labelSelectors, some nodes have 0 pods",
pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). pod: st.MakePod().Name("p").Label("foo", "").Label("bar", "").
SpreadConstraint(1, "zone", softSpread, st.MakeLabelSelector().Exists("foo").Obj()). SpreadConstraint(1, "zone", softSpread, st.MakeLabelSelector().Exists("foo").Obj()).
SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("bar").Obj()). SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("bar").Obj()).
@ -404,7 +445,7 @@ func TestPodTopologySpreadScore(t *testing.T) {
// sum them up gets: 2/3/1, and total number is 6. // sum them up gets: 2/3/1, and total number is 6.
// after reversing, it's 4/3/5 // after reversing, it's 4/3/5
// so scores = 400/5, 300/5, 500/5 // so scores = 400/5, 300/5, 500/5
name: "two constraints on zone and node, with different labelSelectors, 3 out of 4 nodes are candidates", name: "two Constraints on zone and node, with different labelSelectors, 3 out of 4 nodes are candidates",
pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). pod: st.MakePod().Name("p").Label("foo", "").Label("bar", "").
SpreadConstraint(1, "zone", softSpread, st.MakeLabelSelector().Exists("foo").Obj()). SpreadConstraint(1, "zone", softSpread, st.MakeLabelSelector().Exists("foo").Obj()).
SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("bar").Obj()). SpreadConstraint(1, "node", softSpread, st.MakeLabelSelector().Exists("bar").Obj()).
@ -495,8 +536,8 @@ func TestPodTopologySpreadScore(t *testing.T) {
if !status.IsSuccess() { if !status.IsSuccess() {
t.Errorf("unexpected error: %v", status) t.Errorf("unexpected error: %v", status)
} }
if !reflect.DeepEqual(tt.want, gotList) { if diff := cmp.Diff(tt.want, gotList); diff != "" {
t.Errorf("expected:\n\t%+v,\ngot:\n\t%+v", tt.want, gotList) t.Errorf("unexpected scores (-want,+got):\n%s", diff)
} }
}) })
} }
@ -529,7 +570,7 @@ func BenchmarkTestPodTopologySpreadScore(b *testing.B) {
filteredNodesNum: 500, filteredNodesNum: 500,
}, },
{ {
name: "1000nodes/two-constraints-zone-node", name: "1000nodes/two-Constraints-zone-node",
pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). pod: st.MakePod().Name("p").Label("foo", "").Label("bar", "").
SpreadConstraint(1, v1.LabelZoneFailureDomain, softSpread, st.MakeLabelSelector().Exists("foo").Obj()). SpreadConstraint(1, v1.LabelZoneFailureDomain, softSpread, st.MakeLabelSelector().Exists("foo").Obj()).
SpreadConstraint(1, v1.LabelHostname, softSpread, st.MakeLabelSelector().Exists("bar").Obj()). SpreadConstraint(1, v1.LabelHostname, softSpread, st.MakeLabelSelector().Exists("bar").Obj()).