github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-05 10:19:50 +00:00
Code Issues Projects Releases Wiki Activity

EvenPodsSpread: match selector of each constraint independently

Browse Source 
- see more discussion at https://github.com/kubernetes/kubernetes/pull/77760#discussion_r305107973
This commit is contained in:
Wei Huang 2019-07-22 17:31:36 -07:00
parent c23aef40f2
commit f822487f05
No known key found for this signature in database
GPG Key ID: BE5E9752F8B6E005
2 changed files with 103 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
pkg/scheduler/algorithm/predicates/metadata.go
View File

@ -104,7 +104,7 @@ type predicateMetadata struct {
// from scheduler extender configuration and does not change per pod. // from scheduler extender configuration and does not change per pod.
ignoredExtendedResources sets.String ignoredExtendedResources sets.String
// Similar to the map for pod (anti-)affinity, but imposes additional min matches info // Similar to the map for pod (anti-)affinity, but imposes additional min matches info
// to describe mininum match number on each topology spread constraint // to describe minimum match number on each topology spread constraint.
topologyPairsPodSpreadMap *topologyPairsPodSpreadMap topologyPairsPodSpreadMap *topologyPairsPodSpreadMap
} }
@ -190,8 +190,8 @@ func (pfactory *PredicateMetadataFactory) GetMetadata(pod *v1.Pod, nodeNameToInf
} }
func getTPMapMatchingSpreadConstraints(pod *v1.Pod, nodeInfoMap map[string]*schedulernodeinfo.NodeInfo) (*topologyPairsPodSpreadMap, error) { func getTPMapMatchingSpreadConstraints(pod *v1.Pod, nodeInfoMap map[string]*schedulernodeinfo.NodeInfo) (*topologyPairsPodSpreadMap, error) {
// we have feature gating in APIserver to strip the spec // We have feature gating in APIServer to strip the spec
// so don't need to re-check feature gate, just check length of constraints // so don't need to re-check feature gate, just check length of constraints.
constraints := getHardTopologySpreadConstraints(pod) constraints := getHardTopologySpreadConstraints(pod)
if len(constraints) == 0 { if len(constraints) == 0 {
return nil, nil return nil, nil
@ -206,7 +206,7 @@ func getTPMapMatchingSpreadConstraints(pod *v1.Pod, nodeInfoMap map[string]*sche
var lock sync.Mutex var lock sync.Mutex
topologyPairsPodSpreadMap := &topologyPairsPodSpreadMap{ topologyPairsPodSpreadMap := &topologyPairsPodSpreadMap{
// topologyKeyToMinPodsMap will be initilized with proper size later. // topologyKeyToMinPodsMap will be initialized with proper size later.
topologyPairsMaps: newTopologyPairsMaps(), topologyPairsMaps: newTopologyPairsMaps(),
} }
@ -230,7 +230,7 @@ func getTPMapMatchingSpreadConstraints(pod *v1.Pod, nodeInfoMap map[string]*sche
if !podMatchesNodeSelectorAndAffinityTerms(pod, node) { if !podMatchesNodeSelectorAndAffinityTerms(pod, node) {
return return
} }
// ensure current node's labels contains all topologyKeys in 'constraints' // Ensure current node's labels contains all topologyKeys in 'constraints'.
for _, constraint := range constraints { for _, constraint := range constraints {
if _, ok := node.Labels[constraint.TopologyKey]; !ok { if _, ok := node.Labels[constraint.TopologyKey]; !ok {
return return
@ -240,13 +240,17 @@ func getTPMapMatchingSpreadConstraints(pod *v1.Pod, nodeInfoMap map[string]*sche
nodeTopologyMaps := newTopologyPairsMaps() nodeTopologyMaps := newTopologyPairsMaps()
// nodeInfo.Pods() can be empty; or all pods don't fit // nodeInfo.Pods() can be empty; or all pods don't fit
for _, existingPod := range nodeInfo.Pods() { for _, existingPod := range nodeInfo.Pods() {
ok, err := podMatchesAllSpreadConstraints(existingPod, pod.Namespace, constraints) if existingPod.Namespace != pod.Namespace {
continue
}
podLabelSet := labels.Set(existingPod.Labels)
for _, constraint := range constraints {
ok, err := podMatchesSpreadConstraint(podLabelSet, constraint)
if err != nil { if err != nil {
errCh.SendErrorWithCancel(err, cancel) errCh.SendErrorWithCancel(err, cancel)
return return
} }
if ok { if ok {
for _, constraint := range constraints {
// constraint.TopologyKey is already guaranteed to be present // constraint.TopologyKey is already guaranteed to be present
pair := topologyPair{key: constraint.TopologyKey, value: node.Labels[constraint.TopologyKey]} pair := topologyPair{key: constraint.TopologyKey, value: node.Labels[constraint.TopologyKey]}
nodeTopologyMaps.addTopologyPair(pair, existingPod) nodeTopologyMaps.addTopologyPair(pair, existingPod)
@ -254,7 +258,7 @@ func getTPMapMatchingSpreadConstraints(pod *v1.Pod, nodeInfoMap map[string]*sche
} }
} }
// If needed, append topology pair without entry of pods. // If needed, append topology pair without entry of pods.
// For example, on node-x, there is no pod matching spread constraints // For example, on node-x, there is no pod matching spread constraints,
// but node-x should be also considered as a match (with match number 0) // but node-x should be also considered as a match (with match number 0)
// i.e. <node: node-x>: {} // i.e. <node: node-x>: {}
for _, constraint := range constraints { for _, constraint := range constraints {
@ -281,8 +285,8 @@ func getTPMapMatchingSpreadConstraints(pod *v1.Pod, nodeInfoMap map[string]*sche
topologyPairsPodSpreadMap.topologyKeyToMinPodsMap[constraint.TopologyKey] = math.MaxInt32 topologyPairsPodSpreadMap.topologyKeyToMinPodsMap[constraint.TopologyKey] = math.MaxInt32
} }
for pair, podSet := range topologyPairsPodSpreadMap.topologyPairToPods { for pair, podSet := range topologyPairsPodSpreadMap.topologyPairToPods {
// TODO(Huang-Wei): short circuit all portions of <topologyKey: any value> // TODO(Huang-Wei): short circuit unvisited portions of <topologyKey: any value>
// if we see 0 as min match of the topologyKey // if we already see 0 as min match of that topologyKey.
if l := int32(len(podSet)); l < topologyPairsPodSpreadMap.topologyKeyToMinPodsMap[pair.key] { if l := int32(len(podSet)); l < topologyPairsPodSpreadMap.topologyKeyToMinPodsMap[pair.key] {
topologyPairsPodSpreadMap.topologyKeyToMinPodsMap[pair.key] = l topologyPairsPodSpreadMap.topologyKeyToMinPodsMap[pair.key] = l
} }
@ -301,23 +305,10 @@ func getHardTopologySpreadConstraints(pod *v1.Pod) (constraints []v1.TopologySpr
return return
} }
func podMatchesAllSpreadConstraints(pod *v1.Pod, ns string, constraints []v1.TopologySpreadConstraint) (bool, error) {
if len(constraints) == 0 || pod.Namespace != ns {
return false, nil
}
return podLabelsMatchesSpreadConstraints(pod.Labels, constraints)
}
// some corner cases: // some corner cases:
// 1. podLabels = nil, constraint.LabelSelector = nil => returns false // 1. podLabelSet = nil => returns (false, nil)
// 2. podLabels = nil => returns false // 2. constraint.LabelSelector = nil => returns (false, nil)
// 3. constraint.LabelSelector = nil => returns false func podMatchesSpreadConstraint(podLabelSet labels.Set, constraint v1.TopologySpreadConstraint) (bool, error) {
func podLabelsMatchesSpreadConstraints(podLabels map[string]string, constraints []v1.TopologySpreadConstraint) (bool, error) {
if len(constraints) == 0 {
return false, nil
}
podLabelSet := labels.Set(podLabels)
for _, constraint := range constraints {
selector, err := metav1.LabelSelectorAsSelector(constraint.LabelSelector) selector, err := metav1.LabelSelectorAsSelector(constraint.LabelSelector)
if err != nil { if err != nil {
return false, err return false, err
@ -325,7 +316,6 @@ func podLabelsMatchesSpreadConstraints(podLabels map[string]string, constraints
if !selector.Matches(podLabelSet) { if !selector.Matches(podLabelSet) {
return false, nil return false, nil
} }
}
return true, nil return true, nil
} }
@ -337,9 +327,7 @@ func newTopologyPairsMaps() *topologyPairsMaps {
func (m *topologyPairsMaps) addTopologyPair(pair topologyPair, pod *v1.Pod) { func (m *topologyPairsMaps) addTopologyPair(pair topologyPair, pod *v1.Pod) {
podFullName := schedutil.GetPodFullName(pod) podFullName := schedutil.GetPodFullName(pod)
if m.topologyPairToPods[pair] == nil { m.addTopologyPairWithoutPods(pair)
m.topologyPairToPods[pair] = make(map[*v1.Pod]struct{})
}
m.topologyPairToPods[pair][pod] = struct{}{} m.topologyPairToPods[pair][pod] = struct{}{}
if m.podToTopologyPairs[podFullName] == nil { if m.podToTopologyPairs[podFullName] == nil {
m.podToTopologyPairs[podFullName] = make(map[topologyPair]struct{}) m.podToTopologyPairs[podFullName] = make(map[topologyPair]struct{})

76
pkg/scheduler/algorithm/predicates/metadata_test.go
View File

@ -24,6 +24,7 @@ import (
"k8s.io/api/core/v1" "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo"
st "k8s.io/kubernetes/pkg/scheduler/testing" st "k8s.io/kubernetes/pkg/scheduler/testing"
) )
@ -825,19 +826,18 @@ func TestGetTPMapMatchingIncomingAffinityAntiAffinity(t *testing.T) {
} }
} }
func TestPodLabelsMatchesSpreadConstraints(t *testing.T) { func TestPodMatchesSpreadConstraint(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
podLabels map[string]string podLabels map[string]string
constraints []v1.TopologySpreadConstraint constraint v1.TopologySpreadConstraint
want bool want bool
wantErr bool wantErr bool
}{ }{
{ {
name: "normal match", name: "normal match",
podLabels: map[string]string{"foo": "", "bar": ""}, podLabels: map[string]string{"foo": "", "bar": ""},
constraints: []v1.TopologySpreadConstraint{ constraint: v1.TopologySpreadConstraint{
{
LabelSelector: &metav1.LabelSelector{ LabelSelector: &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{ MatchExpressions: []metav1.LabelSelectorRequirement{
{ {
@ -847,14 +847,12 @@ func TestPodLabelsMatchesSpreadConstraints(t *testing.T) {
}, },
}, },
}, },
},
want: true, want: true,
}, },
{ {
name: "normal mismatch", name: "normal mismatch",
podLabels: map[string]string{"foo": "", "baz": ""}, podLabels: map[string]string{"foo": "", "baz": ""},
constraints: []v1.TopologySpreadConstraint{ constraint: v1.TopologySpreadConstraint{
{
LabelSelector: &metav1.LabelSelector{ LabelSelector: &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{ MatchExpressions: []metav1.LabelSelectorRequirement{
{ {
@ -868,13 +866,11 @@ func TestPodLabelsMatchesSpreadConstraints(t *testing.T) {
}, },
}, },
}, },
},
want: false, want: false,
}, },
{ {
name: "podLabels is nil", name: "podLabels is nil",
constraints: []v1.TopologySpreadConstraint{ constraint: v1.TopologySpreadConstraint{
{
LabelSelector: &metav1.LabelSelector{ LabelSelector: &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{ MatchExpressions: []metav1.LabelSelectorRequirement{
{ {
@ -884,7 +880,6 @@ func TestPodLabelsMatchesSpreadConstraints(t *testing.T) {
}, },
}, },
}, },
},
want: false, want: false,
}, },
{ {
@ -893,31 +888,28 @@ func TestPodLabelsMatchesSpreadConstraints(t *testing.T) {
"foo": "", "foo": "",
"bar": "", "bar": "",
}, },
constraints: []v1.TopologySpreadConstraint{ constraint: v1.TopologySpreadConstraint{
{
MaxSkew: 1, MaxSkew: 1,
}, },
},
want: false, want: false,
}, },
{ {
name: "both podLabels and constraint.LabelSelector are nil", name: "both podLabels and constraint.LabelSelector are nil",
constraints: []v1.TopologySpreadConstraint{ constraint: v1.TopologySpreadConstraint{
{
MaxSkew: 1, MaxSkew: 1,
}, },
},
want: false, want: false,
}, },
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
got, err := podLabelsMatchesSpreadConstraints(tt.podLabels, tt.constraints) podLabelSet := labels.Set(tt.podLabels)
got, err := podMatchesSpreadConstraint(podLabelSet, tt.constraint)
if (err != nil) != tt.wantErr { if (err != nil) != tt.wantErr {
t.Errorf("podLabelsMatchesSpreadConstraints() error = %v, wantErr %v", err, tt.wantErr) t.Errorf("podMatchesSpreadConstraint() error = %v, wantErr %v", err, tt.wantErr)
} }
if got != tt.want { if got != tt.want {
t.Errorf("podLabelsMatchesSpreadConstraints() = %v, want %v", got, tt.want) t.Errorf("podMatchesSpreadConstraint() = %v, want %v", got, tt.want)
} }
}) })
} }
@ -996,7 +988,7 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
}, },
}, },
{ {
name: "namespace mis-match doesn't count", name: "namespace mismatch doesn't count",
pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint( pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint(
1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(), 1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(),
).Obj(), ).Obj(),
@ -1117,7 +1109,37 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
}, },
}, },
{ {
name: "different labelSelectors", name: "different labelSelectors - simple version",
pod: st.MakePod().Name("p").Label("foo", "").Label("bar", "").
SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()).
SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("bar").Obj()).
Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Obj(),
},
existingPods: []*v1.Pod{
st.MakePod().Name("p-a").Node("node-a").Label("foo", "").Obj(),
},
injectPodPointers: map[topologyPair][]int{
{key: "zone", value: "zone1"}: {0},
{key: "zone", value: "zone2"}: {},
{key: "node", value: "node-a"}: {},
{key: "node", value: "node-b"}: {},
{key: "node", value: "node-y"}: {},
},
want: &topologyPairsPodSpreadMap{
topologyKeyToMinPodsMap: map[string]int32{"zone": 0, "node": 0},
topologyPairsMaps: &topologyPairsMaps{
podToTopologyPairs: map[string]topologyPairSet{
"p-a_": newPairSet("zone", "zone1"),
},
},
},
},
{
name: "different labelSelectors - complex version",
pod: st.MakePod().Name("p").Label("foo", "").Label("bar", ""). pod: st.MakePod().Name("p").Label("foo", "").Label("bar", "").
SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()). SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()).
SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("bar").Obj()). SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("bar").Obj()).
@ -1137,18 +1159,22 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
st.MakePod().Name("p-y4").Node("node-y").Label("foo", "").Label("bar", "").Obj(), st.MakePod().Name("p-y4").Node("node-y").Label("foo", "").Label("bar", "").Obj(),
}, },
injectPodPointers: map[topologyPair][]int{ injectPodPointers: map[topologyPair][]int{
{key: "zone", value: "zone1"}: {1}, {key: "zone", value: "zone1"}: {0, 1, 2},
{key: "zone", value: "zone2"}: {4, 6}, {key: "zone", value: "zone2"}: {3, 4, 5, 6},
{key: "node", value: "node-a"}: {1}, {key: "node", value: "node-a"}: {1},
{key: "node", value: "node-b"}: {}, {key: "node", value: "node-b"}: {},
{key: "node", value: "node-y"}: {4, 6}, {key: "node", value: "node-y"}: {4, 6},
}, },
want: &topologyPairsPodSpreadMap{ want: &topologyPairsPodSpreadMap{
topologyKeyToMinPodsMap: map[string]int32{"zone": 1, "node": 0}, topologyKeyToMinPodsMap: map[string]int32{"zone": 3, "node": 0},
topologyPairsMaps: &topologyPairsMaps{ topologyPairsMaps: &topologyPairsMaps{
podToTopologyPairs: map[string]topologyPairSet{ podToTopologyPairs: map[string]topologyPairSet{
"p-a1_": newPairSet("zone", "zone1"),
"p-a2_": newPairSet("zone", "zone1", "node", "node-a"), "p-a2_": newPairSet("zone", "zone1", "node", "node-a"),
"p-b1_": newPairSet("zone", "zone1"),
"p-y1_": newPairSet("zone", "zone2"),
"p-y2_": newPairSet("zone", "zone2", "node", "node-y"), "p-y2_": newPairSet("zone", "zone2", "node", "node-y"),
"p-y3_": newPairSet("zone", "zone2"),
"p-y4_": newPairSet("zone", "zone2", "node", "node-y"), "p-y4_": newPairSet("zone", "zone2", "node", "node-y"),
}, },
}, },