From e85473c2df1065e8fa8901f8b3dba64833a61b26 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Tue, 17 Sep 2019 20:23:20 -0700 Subject: [PATCH] Benchmark test for PodAffinity --- .../algorithm/predicates/metadata_test.go | 2 +- .../priorities/even_pods_spread_test.go | 2 +- .../priorities/interpod_affinity_test.go | 55 ++++++++++ pkg/scheduler/testing/workload_prep.go | 81 +++++++++++++- pkg/scheduler/testing/wrappers.go | 102 +++++++++++++++++- 5 files changed, 235 insertions(+), 7 deletions(-) diff --git a/pkg/scheduler/algorithm/predicates/metadata_test.go b/pkg/scheduler/algorithm/predicates/metadata_test.go index 7c036ab3fe3..b31dcec0a4a 100644 --- a/pkg/scheduler/algorithm/predicates/metadata_test.go +++ b/pkg/scheduler/algorithm/predicates/metadata_test.go @@ -1682,7 +1682,7 @@ func BenchmarkTestGetTPMapMatchingSpreadConstraints(b *testing.B) { } for _, tt := range tests { b.Run(tt.name, func(b *testing.B) { - existingPods, allNodes, _ := st.MakeNodesAndPods(tt.pod, tt.existingPodsNum, tt.allNodesNum, tt.filteredNodesNum) + existingPods, allNodes, _ := st.MakeNodesAndPodsForEvenPodsSpread(tt.pod, tt.existingPodsNum, tt.allNodesNum, tt.filteredNodesNum) nodeNameToInfo := schedulernodeinfo.CreateNodeNameToInfoMap(existingPods, allNodes) b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/pkg/scheduler/algorithm/priorities/even_pods_spread_test.go b/pkg/scheduler/algorithm/priorities/even_pods_spread_test.go index af421fd254f..f330e341353 100644 --- a/pkg/scheduler/algorithm/priorities/even_pods_spread_test.go +++ b/pkg/scheduler/algorithm/priorities/even_pods_spread_test.go @@ -483,7 +483,7 @@ func BenchmarkTestCalculateEvenPodsSpreadPriority(b *testing.B) { } for _, tt := range tests { b.Run(tt.name, func(b *testing.B) { - existingPods, allNodes, filteredNodes := st.MakeNodesAndPods(tt.pod, tt.existingPodsNum, tt.allNodesNum, tt.filteredNodesNum) + existingPods, allNodes, filteredNodes := st.MakeNodesAndPodsForEvenPodsSpread(tt.pod, tt.existingPodsNum, tt.allNodesNum, tt.filteredNodesNum) nodeNameToInfo := schedulernodeinfo.CreateNodeNameToInfoMap(existingPods, allNodes) b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/pkg/scheduler/algorithm/priorities/interpod_affinity_test.go b/pkg/scheduler/algorithm/priorities/interpod_affinity_test.go index efbebf37f75..b7cafd35dd6 100644 --- a/pkg/scheduler/algorithm/priorities/interpod_affinity_test.go +++ b/pkg/scheduler/algorithm/priorities/interpod_affinity_test.go @@ -25,6 +25,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" + st "k8s.io/kubernetes/pkg/scheduler/testing" ) type FakeNodeListInfo []*v1.Node @@ -612,3 +613,57 @@ func TestHardPodAffinitySymmetricWeight(t *testing.T) { }) } } + +func BenchmarkInterPodAffinityPriority(b *testing.B) { + tests := []struct { + name string + pod *v1.Pod + existingPodsNum int + allNodesNum int + prepFunc func(existingPodsNum, allNodesNum int) (existingPods []*v1.Pod, allNodes []*v1.Node) + }{ + { + name: "1000nodes/incoming pod without PodAffinity and existing pods without PodAffinity", + pod: st.MakePod().Name("p").Label("foo", "").Obj(), + existingPodsNum: 10000, + allNodesNum: 1000, + prepFunc: st.MakeNodesAndPods, + }, + { + name: "1000nodes/incoming pod with PodAffinity and existing pods without PodAffinity", + pod: st.MakePod().Name("p").Label("foo", "").PodAffinityExists("foo", "zone", st.PodAffinityWithPreferredReq).Obj(), + existingPodsNum: 10000, + allNodesNum: 1000, + prepFunc: st.MakeNodesAndPods, + }, + { + name: "1000nodes/incoming pod without PodAffinity and existing pods with PodAffinity", + pod: st.MakePod().Name("p").Label("foo", "").Obj(), + existingPodsNum: 10000, + allNodesNum: 1000, + prepFunc: st.MakeNodesAndPodsForPodAffinity, + }, + { + name: "1000nodes/incoming pod with PodAffinity and existing pods with PodAffinity", + pod: st.MakePod().Name("p").Label("foo", "").PodAffinityExists("foo", "zone", st.PodAffinityWithPreferredReq).Obj(), + existingPodsNum: 10000, + allNodesNum: 1000, + prepFunc: st.MakeNodesAndPodsForPodAffinity, + }, + } + + for _, tt := range tests { + b.Run(tt.name, func(b *testing.B) { + existingPods, allNodes := tt.prepFunc(tt.existingPodsNum, tt.allNodesNum) + nodeNameToInfo := schedulernodeinfo.CreateNodeNameToInfoMap(existingPods, allNodes) + interPodAffinity := InterPodAffinity{ + info: FakeNodeListInfo(allNodes), + hardPodAffinityWeight: v1.DefaultHardPodAffinitySymmetricWeight, + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + interPodAffinity.CalculateInterPodAffinityPriority(tt.pod, nodeNameToInfo, allNodes) + } + }) + } +} diff --git a/pkg/scheduler/testing/workload_prep.go b/pkg/scheduler/testing/workload_prep.go index d70868aae5e..3280543b225 100644 --- a/pkg/scheduler/testing/workload_prep.go +++ b/pkg/scheduler/testing/workload_prep.go @@ -22,15 +22,15 @@ import ( "k8s.io/api/core/v1" ) -// MakeNodesAndPods serves as a testing helper for EvenPodsSpread feature. +// MakeNodesAndPodsForEvenPodsSpread serves as a testing helper for EvenPodsSpread feature. // It builds a fake cluster containing running Pods and Nodes. // The size of Pods and Nodes are determined by input arguments. // The specs of Pods and Nodes are generated with the following rules: // - If `pod` has "node" as a topologyKey, each generated node is applied with a unique label: "node: node". // - If `pod` has "zone" as a topologyKey, each generated node is applied with a rotating label: "zone: zone[0-9]". -// - Depending on "lableSelector.MatchExpressions[0].Key" the `pod` has in each topologySpreadConstraint, +// - Depending on "labelSelector.MatchExpressions[0].Key" the `pod` has in each topologySpreadConstraint, // each generated pod will be applied with label "key1", "key1,key2", ..., "key1,key2,...,keyN" in a rotating manner. -func MakeNodesAndPods(pod *v1.Pod, existingPodsNum, allNodesNum, filteredNodesNum int) (existingPods []*v1.Pod, allNodes []*v1.Node, filteredNodes []*v1.Node) { +func MakeNodesAndPodsForEvenPodsSpread(pod *v1.Pod, existingPodsNum, allNodesNum, filteredNodesNum int) (existingPods []*v1.Pod, allNodes []*v1.Node, filteredNodes []*v1.Node) { var topologyKeys []string var labels []string zones := 10 @@ -65,3 +65,78 @@ func MakeNodesAndPods(pod *v1.Pod, existingPodsNum, allNodesNum, filteredNodesNu } return } + +// MakeNodesAndPodsForPodAffinity serves as a testing helper for Pod(Anti)Affinity feature. +// It builds a fake cluster containing running Pods and Nodes. +// For simplicity, the Nodes will be labelled with "region", "zone" and "node". Nodes[i] will be applied with: +// - "region": "region" + i%3 +// - "zone": "zone" + i%10 +// - "node": "node" + i +// The Pods will be applied with various combinations of PodAffinity and PodAntiAffinity terms. +func MakeNodesAndPodsForPodAffinity(existingPodsNum, allNodesNum int) (existingPods []*v1.Pod, allNodes []*v1.Node) { + tpKeyToSizeMap := map[string]int{ + "region": 3, + "zone": 10, + "node": allNodesNum, + } + // build nodes to spread across all topology domains + for i := 0; i < allNodesNum; i++ { + nodeName := fmt.Sprintf("node%d", i) + nodeWrapper := MakeNode().Name(nodeName) + for tpKey, size := range tpKeyToSizeMap { + nodeWrapper = nodeWrapper.Label(tpKey, fmt.Sprintf("%s%d", tpKey, i%size)) + } + allNodes = append(allNodes, nodeWrapper.Obj()) + } + + labels := []string{"foo", "bar", "baz"} + tpKeys := []string{"region", "zone", "node"} + + // Build pods. + // Each pod will be created with one affinity and one anti-affinity terms using all combinations of + // affinity and anti-affinity kinds listed below + // e.g., the first pod will have {affinity, anti-affinity} terms of kinds {NilPodAffinity, NilPodAffinity}; + // the second will be {NilPodAffinity, PodAntiAffinityWithRequiredReq}, etc. + affinityKinds := []PodAffinityKind{ + NilPodAffinity, + PodAffinityWithRequiredReq, + PodAffinityWithPreferredReq, + PodAffinityWithRequiredPreferredReq, + } + antiAffinityKinds := []PodAffinityKind{ + NilPodAffinity, + PodAntiAffinityWithRequiredReq, + PodAntiAffinityWithPreferredReq, + PodAntiAffinityWithRequiredPreferredReq, + } + + totalSize := len(affinityKinds) * len(antiAffinityKinds) + for i := 0; i < existingPodsNum; i++ { + podWrapper := MakePod().Name(fmt.Sprintf("pod%d", i)).Node(fmt.Sprintf("node%d", i%allNodesNum)) + label, tpKey := labels[i%len(labels)], tpKeys[i%len(tpKeys)] + + affinityIdx := i % totalSize + // len(affinityKinds) is equal to len(antiAffinityKinds) + leftIdx, rightIdx := affinityIdx/len(affinityKinds), affinityIdx%len(affinityKinds) + podWrapper = podWrapper.PodAffinityExists(label, tpKey, affinityKinds[leftIdx]) + podWrapper = podWrapper.PodAntiAffinityExists(label, tpKey, antiAffinityKinds[rightIdx]) + existingPods = append(existingPods, podWrapper.Obj()) + } + + return +} + +// MakeNodesAndPods serves as a testing helper to generate regular Nodes and Pods +// that don't use any advanced scheduling features. +func MakeNodesAndPods(existingPodsNum, allNodesNum int) (existingPods []*v1.Pod, allNodes []*v1.Node) { + // build nodes + for i := 0; i < allNodesNum; i++ { + allNodes = append(allNodes, MakeNode().Name(fmt.Sprintf("node%d", i)).Obj()) + } + // build pods + for i := 0; i < existingPodsNum; i++ { + podWrapper := MakePod().Name(fmt.Sprintf("pod%d", i)).Node(fmt.Sprintf("node%d", i%allNodesNum)) + existingPods = append(existingPods, podWrapper.Obj()) + } + return +} diff --git a/pkg/scheduler/testing/wrappers.go b/pkg/scheduler/testing/wrappers.go index afe98372abf..48d4b6dcb9c 100644 --- a/pkg/scheduler/testing/wrappers.go +++ b/pkg/scheduler/testing/wrappers.go @@ -190,7 +190,7 @@ func (p *PodWrapper) NodeSelector(m map[string]string) *PodWrapper { } // NodeAffinityIn creates a HARD node affinity (with the operator In) -// and injects into the innner pod. +// and injects into the inner pod. func (p *PodWrapper) NodeAffinityIn(key string, vals []string) *PodWrapper { if p.Spec.Affinity == nil { p.Spec.Affinity = &v1.Affinity{} @@ -204,7 +204,7 @@ func (p *PodWrapper) NodeAffinityIn(key string, vals []string) *PodWrapper { } // NodeAffinityNotIn creates a HARD node affinity (with the operator NotIn) -// and injects into the innner pod. +// and injects into the inner pod. func (p *PodWrapper) NodeAffinityNotIn(key string, vals []string) *PodWrapper { if p.Spec.Affinity == nil { p.Spec.Affinity = &v1.Affinity{} @@ -217,6 +217,104 @@ func (p *PodWrapper) NodeAffinityNotIn(key string, vals []string) *PodWrapper { return p } +// PodAffinityKind represents different kinds of PodAffinity. +type PodAffinityKind int + +const ( + // NilPodAffinity is a no-op which doesn't apply any PodAffinity. + NilPodAffinity PodAffinityKind = iota + // PodAffinityWithRequiredReq applies a HARD requirement to pod.spec.affinity.PodAffinity. + PodAffinityWithRequiredReq + // PodAffinityWithPreferredReq applies a SOFT requirement to pod.spec.affinity.PodAffinity. + PodAffinityWithPreferredReq + // PodAffinityWithRequiredPreferredReq applies HARD and SOFT requirements to pod.spec.affinity.PodAffinity. + PodAffinityWithRequiredPreferredReq + // PodAntiAffinityWithRequiredReq applies a HARD requirement to pod.spec.affinity.PodAntiAffinity. + PodAntiAffinityWithRequiredReq + // PodAntiAffinityWithPreferredReq applies a SOFT requirement to pod.spec.affinity.PodAntiAffinity. + PodAntiAffinityWithPreferredReq + // PodAntiAffinityWithRequiredPreferredReq applies HARD and SOFT requirements to pod.spec.affinity.PodAntiAffinity. + PodAntiAffinityWithRequiredPreferredReq +) + +// PodAffinityExists creates an PodAffinity with the operator "Exists" +// and injects into the inner pod. +func (p *PodWrapper) PodAffinityExists(labelKey, topologyKey string, kind PodAffinityKind) *PodWrapper { + if kind == NilPodAffinity { + return p + } + + if p.Spec.Affinity == nil { + p.Spec.Affinity = &v1.Affinity{} + } + if p.Spec.Affinity.PodAffinity == nil { + p.Spec.Affinity.PodAffinity = &v1.PodAffinity{} + } + labelSelector := MakeLabelSelector().Exists(labelKey).Obj() + term := v1.PodAffinityTerm{LabelSelector: labelSelector, TopologyKey: topologyKey} + switch kind { + case PodAffinityWithRequiredReq: + p.Spec.Affinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution = append( + p.Spec.Affinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution, + term, + ) + case PodAffinityWithPreferredReq: + p.Spec.Affinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution = append( + p.Spec.Affinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution, + v1.WeightedPodAffinityTerm{Weight: 1, PodAffinityTerm: term}, + ) + case PodAffinityWithRequiredPreferredReq: + p.Spec.Affinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution = append( + p.Spec.Affinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution, + term, + ) + p.Spec.Affinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution = append( + p.Spec.Affinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution, + v1.WeightedPodAffinityTerm{Weight: 1, PodAffinityTerm: term}, + ) + } + return p +} + +// PodAntiAffinityExists creates an PodAntiAffinity with the operator "Exists" +// and injects into the inner pod. +func (p *PodWrapper) PodAntiAffinityExists(labelKey, topologyKey string, kind PodAffinityKind) *PodWrapper { + if kind == NilPodAffinity { + return p + } + + if p.Spec.Affinity == nil { + p.Spec.Affinity = &v1.Affinity{} + } + if p.Spec.Affinity.PodAntiAffinity == nil { + p.Spec.Affinity.PodAntiAffinity = &v1.PodAntiAffinity{} + } + labelSelector := MakeLabelSelector().Exists(labelKey).Obj() + term := v1.PodAffinityTerm{LabelSelector: labelSelector, TopologyKey: topologyKey} + switch kind { + case PodAntiAffinityWithRequiredReq: + p.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution = append( + p.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, + term, + ) + case PodAntiAffinityWithPreferredReq: + p.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution = append( + p.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution, + v1.WeightedPodAffinityTerm{Weight: 1, PodAffinityTerm: term}, + ) + case PodAntiAffinityWithRequiredPreferredReq: + p.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution = append( + p.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, + term, + ) + p.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution = append( + p.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution, + v1.WeightedPodAffinityTerm{Weight: 1, PodAffinityTerm: term}, + ) + } + return p +} + // SpreadConstraint constructs a TopologySpreadConstraint object and injects // into the inner pod. func (p *PodWrapper) SpreadConstraint(maxSkew int, tpKey string, mode v1.UnsatisfiableConstraintAction, selector *metav1.LabelSelector) *PodWrapper {