mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-08-05 02:09:56 +00:00
Merge pull request #85157 from alculquicondor/refactor/selector
Store topology spread constraints in metadata with labels.Selector
This commit is contained in:
commit
209c025144
@ -116,6 +116,7 @@ func (paths *criticalPaths) update(tpVal string, num int32) {
|
|||||||
// (1) critical paths where the least pods are matched on each spread constraint.
|
// (1) critical paths where the least pods are matched on each spread constraint.
|
||||||
// (2) number of pods matched on each spread constraint.
|
// (2) number of pods matched on each spread constraint.
|
||||||
type evenPodsSpreadMetadata struct {
|
type evenPodsSpreadMetadata struct {
|
||||||
|
constraints []topologySpreadConstraint
|
||||||
// We record 2 critical paths instead of all critical paths here.
|
// We record 2 critical paths instead of all critical paths here.
|
||||||
// criticalPaths[0].matchNum always holds the minimum matching number.
|
// criticalPaths[0].matchNum always holds the minimum matching number.
|
||||||
// criticalPaths[1].matchNum is always greater or equal to criticalPaths[0].matchNum, but
|
// criticalPaths[1].matchNum is always greater or equal to criticalPaths[0].matchNum, but
|
||||||
@ -125,6 +126,15 @@ type evenPodsSpreadMetadata struct {
|
|||||||
tpPairToMatchNum map[topologyPair]int32
|
tpPairToMatchNum map[topologyPair]int32
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// topologySpreadConstraint is an internal version for a hard (DoNotSchedule
|
||||||
|
// unsatisfiable constraint action) v1.TopologySpreadConstraint and where the
|
||||||
|
// selector is parsed.
|
||||||
|
type topologySpreadConstraint struct {
|
||||||
|
maxSkew int32
|
||||||
|
topologyKey string
|
||||||
|
selector labels.Selector
|
||||||
|
}
|
||||||
|
|
||||||
type serviceAffinityMetadata struct {
|
type serviceAffinityMetadata struct {
|
||||||
matchingPodList []*v1.Pod
|
matchingPodList []*v1.Pod
|
||||||
matchingPodServices []*v1.Service
|
matchingPodServices []*v1.Service
|
||||||
@ -420,17 +430,20 @@ func getPodAffinityMetadata(pod *v1.Pod, allNodes []*schedulernodeinfo.NodeInfo,
|
|||||||
func getEvenPodsSpreadMetadata(pod *v1.Pod, allNodes []*schedulernodeinfo.NodeInfo) (*evenPodsSpreadMetadata, error) {
|
func getEvenPodsSpreadMetadata(pod *v1.Pod, allNodes []*schedulernodeinfo.NodeInfo) (*evenPodsSpreadMetadata, error) {
|
||||||
// We have feature gating in APIServer to strip the spec
|
// We have feature gating in APIServer to strip the spec
|
||||||
// so don't need to re-check feature gate, just check length of constraints.
|
// so don't need to re-check feature gate, just check length of constraints.
|
||||||
constraints := getHardTopologySpreadConstraints(pod)
|
constraints, err := filterHardTopologySpreadConstraints(pod.Spec.TopologySpreadConstraints)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
if len(constraints) == 0 {
|
if len(constraints) == 0 {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
errCh := schedutil.NewErrorChannel()
|
|
||||||
var lock sync.Mutex
|
var lock sync.Mutex
|
||||||
|
|
||||||
// TODO(Huang-Wei): It might be possible to use "make(map[topologyPair]*int32)".
|
// TODO(Huang-Wei): It might be possible to use "make(map[topologyPair]*int32)".
|
||||||
// In that case, need to consider how to init each tpPairToCount[pair] in an atomic fashion.
|
// In that case, need to consider how to init each tpPairToCount[pair] in an atomic fashion.
|
||||||
m := evenPodsSpreadMetadata{
|
m := evenPodsSpreadMetadata{
|
||||||
|
constraints: constraints,
|
||||||
tpKeyToCriticalPaths: make(map[string]*criticalPaths, len(constraints)),
|
tpKeyToCriticalPaths: make(map[string]*criticalPaths, len(constraints)),
|
||||||
tpPairToMatchNum: make(map[topologyPair]int32),
|
tpPairToMatchNum: make(map[topologyPair]int32),
|
||||||
}
|
}
|
||||||
@ -440,8 +453,6 @@ func getEvenPodsSpreadMetadata(pod *v1.Pod, allNodes []*schedulernodeinfo.NodeIn
|
|||||||
lock.Unlock()
|
lock.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
|
|
||||||
processNode := func(i int) {
|
processNode := func(i int) {
|
||||||
nodeInfo := allNodes[i]
|
nodeInfo := allNodes[i]
|
||||||
node := nodeInfo.Node()
|
node := nodeInfo.Node()
|
||||||
@ -466,28 +477,19 @@ func getEvenPodsSpreadMetadata(pod *v1.Pod, allNodes []*schedulernodeinfo.NodeIn
|
|||||||
if existingPod.Namespace != pod.Namespace {
|
if existingPod.Namespace != pod.Namespace {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
ok, err := PodMatchesSpreadConstraint(existingPod.Labels, constraint)
|
if constraint.selector.Matches(labels.Set(existingPod.Labels)) {
|
||||||
if err != nil {
|
|
||||||
errCh.SendErrorWithCancel(err, cancel)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if ok {
|
|
||||||
matchTotal++
|
matchTotal++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pair := topologyPair{key: constraint.TopologyKey, value: node.Labels[constraint.TopologyKey]}
|
pair := topologyPair{key: constraint.topologyKey, value: node.Labels[constraint.topologyKey]}
|
||||||
addTopologyPairMatchNum(pair, matchTotal)
|
addTopologyPairMatchNum(pair, matchTotal)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
workqueue.ParallelizeUntil(ctx, 16, len(allNodes), processNode)
|
workqueue.ParallelizeUntil(context.Background(), 16, len(allNodes), processNode)
|
||||||
|
|
||||||
if err := errCh.ReceiveError(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// calculate min match for each topology pair
|
// calculate min match for each topology pair
|
||||||
for i := 0; i < len(constraints); i++ {
|
for i := 0; i < len(constraints); i++ {
|
||||||
key := constraints[i].TopologyKey
|
key := constraints[i].topologyKey
|
||||||
m.tpKeyToCriticalPaths[key] = newCriticalPaths()
|
m.tpKeyToCriticalPaths[key] = newCriticalPaths()
|
||||||
}
|
}
|
||||||
for pair, num := range m.tpPairToMatchNum {
|
for pair, num := range m.tpPairToMatchNum {
|
||||||
@ -497,36 +499,28 @@ func getEvenPodsSpreadMetadata(pod *v1.Pod, allNodes []*schedulernodeinfo.NodeIn
|
|||||||
return &m, nil
|
return &m, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func getHardTopologySpreadConstraints(pod *v1.Pod) (constraints []v1.TopologySpreadConstraint) {
|
func filterHardTopologySpreadConstraints(constraints []v1.TopologySpreadConstraint) ([]topologySpreadConstraint, error) {
|
||||||
if pod != nil {
|
var result []topologySpreadConstraint
|
||||||
for _, constraint := range pod.Spec.TopologySpreadConstraints {
|
for _, c := range constraints {
|
||||||
if constraint.WhenUnsatisfiable == v1.DoNotSchedule {
|
if c.WhenUnsatisfiable == v1.DoNotSchedule {
|
||||||
constraints = append(constraints, constraint)
|
selector, err := metav1.LabelSelectorAsSelector(c.LabelSelector)
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// PodMatchesSpreadConstraint verifies if <constraint.LabelSelector> matches <podLabelSet>.
|
|
||||||
// Some corner cases:
|
|
||||||
// 1. podLabelSet = nil => returns (false, nil)
|
|
||||||
// 2. constraint.LabelSelector = nil => returns (false, nil)
|
|
||||||
func PodMatchesSpreadConstraint(podLabelSet labels.Set, constraint v1.TopologySpreadConstraint) (bool, error) {
|
|
||||||
selector, err := metav1.LabelSelectorAsSelector(constraint.LabelSelector)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if !selector.Matches(podLabelSet) {
|
result = append(result, topologySpreadConstraint{
|
||||||
return false, nil
|
maxSkew: c.MaxSkew,
|
||||||
|
topologyKey: c.TopologyKey,
|
||||||
|
selector: selector,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
return true, nil
|
}
|
||||||
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// NodeLabelsMatchSpreadConstraints checks if ALL topology keys in spread constraints are present in node labels.
|
// NodeLabelsMatchSpreadConstraints checks if ALL topology keys in spread constraints are present in node labels.
|
||||||
func NodeLabelsMatchSpreadConstraints(nodeLabels map[string]string, constraints []v1.TopologySpreadConstraint) bool {
|
func NodeLabelsMatchSpreadConstraints(nodeLabels map[string]string, constraints []topologySpreadConstraint) bool {
|
||||||
for _, constraint := range constraints {
|
for _, c := range constraints {
|
||||||
if _, ok := nodeLabels[constraint.TopologyKey]; !ok {
|
if _, ok := nodeLabels[c.topologyKey]; !ok {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -581,57 +575,55 @@ func (m *topologyPairsMaps) clone() *topologyPairsMaps {
|
|||||||
return copy
|
return copy
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *evenPodsSpreadMetadata) addPod(addedPod, preemptorPod *v1.Pod, node *v1.Node) error {
|
func (m *evenPodsSpreadMetadata) addPod(addedPod, preemptorPod *v1.Pod, node *v1.Node) {
|
||||||
return c.updatePod(addedPod, preemptorPod, node, 1)
|
m.updatePod(addedPod, preemptorPod, node, 1)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *evenPodsSpreadMetadata) removePod(deletedPod, preemptorPod *v1.Pod, node *v1.Node) error {
|
func (m *evenPodsSpreadMetadata) removePod(deletedPod, preemptorPod *v1.Pod, node *v1.Node) {
|
||||||
return c.updatePod(deletedPod, preemptorPod, node, -1)
|
m.updatePod(deletedPod, preemptorPod, node, -1)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *evenPodsSpreadMetadata) updatePod(updatedPod, preemptorPod *v1.Pod, node *v1.Node, delta int32) error {
|
func (m *evenPodsSpreadMetadata) updatePod(updatedPod, preemptorPod *v1.Pod, node *v1.Node, delta int32) {
|
||||||
if updatedPod.Namespace != preemptorPod.Namespace || node == nil {
|
if m == nil || updatedPod.Namespace != preemptorPod.Namespace || node == nil {
|
||||||
return nil
|
return
|
||||||
}
|
}
|
||||||
constraints := getHardTopologySpreadConstraints(preemptorPod)
|
if !NodeLabelsMatchSpreadConstraints(node.Labels, m.constraints) {
|
||||||
if !NodeLabelsMatchSpreadConstraints(node.Labels, constraints) {
|
return
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
podLabelSet := labels.Set(updatedPod.Labels)
|
podLabelSet := labels.Set(updatedPod.Labels)
|
||||||
for _, constraint := range constraints {
|
for _, constraint := range m.constraints {
|
||||||
if match, err := PodMatchesSpreadConstraint(podLabelSet, constraint); err != nil {
|
if !constraint.selector.Matches(podLabelSet) {
|
||||||
return err
|
|
||||||
} else if !match {
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
k, v := constraint.TopologyKey, node.Labels[constraint.TopologyKey]
|
k, v := constraint.topologyKey, node.Labels[constraint.topologyKey]
|
||||||
pair := topologyPair{key: k, value: v}
|
pair := topologyPair{key: k, value: v}
|
||||||
c.tpPairToMatchNum[pair] = c.tpPairToMatchNum[pair] + delta
|
m.tpPairToMatchNum[pair] = m.tpPairToMatchNum[pair] + delta
|
||||||
|
|
||||||
c.tpKeyToCriticalPaths[k].update(v, c.tpPairToMatchNum[pair])
|
m.tpKeyToCriticalPaths[k].update(v, m.tpPairToMatchNum[pair])
|
||||||
}
|
}
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *evenPodsSpreadMetadata) clone() *evenPodsSpreadMetadata {
|
func (m *evenPodsSpreadMetadata) clone() *evenPodsSpreadMetadata {
|
||||||
// c could be nil when EvenPodsSpread feature is disabled
|
// c could be nil when EvenPodsSpread feature is disabled
|
||||||
if c == nil {
|
if m == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
copy := evenPodsSpreadMetadata{
|
cp := evenPodsSpreadMetadata{
|
||||||
tpKeyToCriticalPaths: make(map[string]*criticalPaths),
|
// constraints are shared because they don't change.
|
||||||
tpPairToMatchNum: make(map[topologyPair]int32),
|
constraints: m.constraints,
|
||||||
|
tpKeyToCriticalPaths: make(map[string]*criticalPaths, len(m.tpKeyToCriticalPaths)),
|
||||||
|
tpPairToMatchNum: make(map[topologyPair]int32, len(m.tpPairToMatchNum)),
|
||||||
}
|
}
|
||||||
for tpKey, paths := range c.tpKeyToCriticalPaths {
|
for tpKey, paths := range m.tpKeyToCriticalPaths {
|
||||||
copy.tpKeyToCriticalPaths[tpKey] = &criticalPaths{paths[0], paths[1]}
|
cp.tpKeyToCriticalPaths[tpKey] = &criticalPaths{paths[0], paths[1]}
|
||||||
}
|
}
|
||||||
for tpPair, matchNum := range c.tpPairToMatchNum {
|
for tpPair, matchNum := range m.tpPairToMatchNum {
|
||||||
copyPair := topologyPair{key: tpPair.key, value: tpPair.value}
|
copyPair := topologyPair{key: tpPair.key, value: tpPair.value}
|
||||||
copy.tpPairToMatchNum[copyPair] = matchNum
|
cp.tpPairToMatchNum[copyPair] = matchNum
|
||||||
}
|
}
|
||||||
return ©
|
return &cp
|
||||||
}
|
}
|
||||||
|
|
||||||
// RemovePod changes predicateMetadata assuming that the given `deletedPod` is
|
// RemovePod changes predicateMetadata assuming that the given `deletedPod` is
|
||||||
@ -642,10 +634,7 @@ func (meta *predicateMetadata) RemovePod(deletedPod *v1.Pod, node *v1.Node) erro
|
|||||||
return fmt.Errorf("deletedPod and meta.pod must not be the same")
|
return fmt.Errorf("deletedPod and meta.pod must not be the same")
|
||||||
}
|
}
|
||||||
meta.podAffinityMetadata.removePod(deletedPod)
|
meta.podAffinityMetadata.removePod(deletedPod)
|
||||||
// Delete pod from the pod spread topology maps.
|
meta.evenPodsSpreadMetadata.removePod(deletedPod, meta.pod, node)
|
||||||
if err := meta.evenPodsSpreadMetadata.removePod(deletedPod, meta.pod, node); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
meta.serviceAffinityMetadata.removePod(deletedPod, node)
|
meta.serviceAffinityMetadata.removePod(deletedPod, node)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@ -667,9 +656,7 @@ func (meta *predicateMetadata) AddPod(addedPod *v1.Pod, node *v1.Node) error {
|
|||||||
}
|
}
|
||||||
// Update meta.evenPodsSpreadMetadata if meta.pod has hard spread constraints
|
// Update meta.evenPodsSpreadMetadata if meta.pod has hard spread constraints
|
||||||
// and addedPod matches that
|
// and addedPod matches that
|
||||||
if err := meta.evenPodsSpreadMetadata.addPod(addedPod, meta.pod, node); err != nil {
|
meta.evenPodsSpreadMetadata.addPod(addedPod, meta.pod, node)
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
meta.serviceAffinityMetadata.addPod(addedPod, meta.pod, node)
|
meta.serviceAffinityMetadata.addPod(addedPod, meta.pod, node)
|
||||||
|
|
||||||
|
@ -367,7 +367,7 @@ func TestPredicateMetadata_AddRemovePod(t *testing.T) {
|
|||||||
// are given to the metadata producer.
|
// are given to the metadata producer.
|
||||||
allPodsMeta, _ := getMeta(allPodLister)
|
allPodsMeta, _ := getMeta(allPodLister)
|
||||||
// existingPodsMeta1 is meta data produced for test.existingPods (without test.addedPod).
|
// existingPodsMeta1 is meta data produced for test.existingPods (without test.addedPod).
|
||||||
existingPodsMeta1, nodeInfoMap := getMeta(fakelisters.PodLister(test.existingPods))
|
existingPodsMeta1, nodeInfoMap := getMeta(test.existingPods)
|
||||||
// Add test.addedPod to existingPodsMeta1 and make sure meta is equal to allPodsMeta
|
// Add test.addedPod to existingPodsMeta1 and make sure meta is equal to allPodsMeta
|
||||||
nodeInfo := nodeInfoMap[test.addedPod.Spec.NodeName]
|
nodeInfo := nodeInfoMap[test.addedPod.Spec.NodeName]
|
||||||
if err := existingPodsMeta1.AddPod(test.addedPod, nodeInfo.Node()); err != nil {
|
if err := existingPodsMeta1.AddPod(test.addedPod, nodeInfo.Node()); err != nil {
|
||||||
@ -803,96 +803,9 @@ func TestGetTPMapMatchingIncomingAffinityAntiAffinity(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPodMatchesSpreadConstraint(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
podLabels map[string]string
|
|
||||||
constraint v1.TopologySpreadConstraint
|
|
||||||
want bool
|
|
||||||
wantErr bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "normal match",
|
|
||||||
podLabels: map[string]string{"foo": "", "bar": ""},
|
|
||||||
constraint: v1.TopologySpreadConstraint{
|
|
||||||
LabelSelector: &metav1.LabelSelector{
|
|
||||||
MatchExpressions: []metav1.LabelSelectorRequirement{
|
|
||||||
{
|
|
||||||
Key: "foo",
|
|
||||||
Operator: metav1.LabelSelectorOpExists,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
want: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "normal mismatch",
|
|
||||||
podLabels: map[string]string{"foo": "", "baz": ""},
|
|
||||||
constraint: v1.TopologySpreadConstraint{
|
|
||||||
LabelSelector: &metav1.LabelSelector{
|
|
||||||
MatchExpressions: []metav1.LabelSelectorRequirement{
|
|
||||||
{
|
|
||||||
Key: "foo",
|
|
||||||
Operator: metav1.LabelSelectorOpExists,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Key: "bar",
|
|
||||||
Operator: metav1.LabelSelectorOpExists,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
want: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "podLabels is nil",
|
|
||||||
constraint: v1.TopologySpreadConstraint{
|
|
||||||
LabelSelector: &metav1.LabelSelector{
|
|
||||||
MatchExpressions: []metav1.LabelSelectorRequirement{
|
|
||||||
{
|
|
||||||
Key: "foo",
|
|
||||||
Operator: metav1.LabelSelectorOpExists,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
want: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "constraint.LabelSelector is nil",
|
|
||||||
podLabels: map[string]string{
|
|
||||||
"foo": "",
|
|
||||||
"bar": "",
|
|
||||||
},
|
|
||||||
constraint: v1.TopologySpreadConstraint{
|
|
||||||
MaxSkew: 1,
|
|
||||||
},
|
|
||||||
want: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "both podLabels and constraint.LabelSelector are nil",
|
|
||||||
constraint: v1.TopologySpreadConstraint{
|
|
||||||
MaxSkew: 1,
|
|
||||||
},
|
|
||||||
want: false,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
podLabelSet := labels.Set(tt.podLabels)
|
|
||||||
got, err := PodMatchesSpreadConstraint(podLabelSet, tt.constraint)
|
|
||||||
if (err != nil) != tt.wantErr {
|
|
||||||
t.Errorf("PodMatchesSpreadConstraint() error = %v, wantErr %v", err, tt.wantErr)
|
|
||||||
}
|
|
||||||
if got != tt.want {
|
|
||||||
t.Errorf("PodMatchesSpreadConstraint() = %v, want %v", got, tt.want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
|
func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
|
||||||
|
fooSelector := st.MakeLabelSelector().Exists("foo").Obj()
|
||||||
|
barSelector := st.MakeLabelSelector().Exists("bar").Obj()
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
pod *v1.Pod
|
pod *v1.Pod
|
||||||
@ -903,7 +816,7 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "clean cluster with one spreadConstraint",
|
name: "clean cluster with one spreadConstraint",
|
||||||
pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint(
|
pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint(
|
||||||
1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(),
|
5, "zone", hardSpread, st.MakeLabelSelector().Label("foo", "bar").Obj(),
|
||||||
).Obj(),
|
).Obj(),
|
||||||
nodes: []*v1.Node{
|
nodes: []*v1.Node{
|
||||||
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
|
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
|
||||||
@ -912,6 +825,13 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
|
|||||||
st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Obj(),
|
st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Obj(),
|
||||||
},
|
},
|
||||||
want: &evenPodsSpreadMetadata{
|
want: &evenPodsSpreadMetadata{
|
||||||
|
constraints: []topologySpreadConstraint{
|
||||||
|
{
|
||||||
|
maxSkew: 5,
|
||||||
|
topologyKey: "zone",
|
||||||
|
selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Label("foo", "bar").Obj()),
|
||||||
|
},
|
||||||
|
},
|
||||||
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
||||||
"zone": {{"zone1", 0}, {"zone2", 0}},
|
"zone": {{"zone1", 0}, {"zone2", 0}},
|
||||||
},
|
},
|
||||||
@ -924,7 +844,7 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "normal case with one spreadConstraint",
|
name: "normal case with one spreadConstraint",
|
||||||
pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint(
|
pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint(
|
||||||
1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(),
|
1, "zone", hardSpread, fooSelector,
|
||||||
).Obj(),
|
).Obj(),
|
||||||
nodes: []*v1.Node{
|
nodes: []*v1.Node{
|
||||||
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
|
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
|
||||||
@ -940,6 +860,13 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
|
|||||||
st.MakePod().Name("p-y2").Node("node-y").Label("foo", "").Obj(),
|
st.MakePod().Name("p-y2").Node("node-y").Label("foo", "").Obj(),
|
||||||
},
|
},
|
||||||
want: &evenPodsSpreadMetadata{
|
want: &evenPodsSpreadMetadata{
|
||||||
|
constraints: []topologySpreadConstraint{
|
||||||
|
{
|
||||||
|
maxSkew: 1,
|
||||||
|
topologyKey: "zone",
|
||||||
|
selector: mustConvertLabelSelectorAsSelector(t, fooSelector),
|
||||||
|
},
|
||||||
|
},
|
||||||
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
||||||
"zone": {{"zone2", 2}, {"zone1", 3}},
|
"zone": {{"zone2", 2}, {"zone1", 3}},
|
||||||
},
|
},
|
||||||
@ -970,6 +897,13 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
|
|||||||
st.MakePod().Name("p-y2").Node("node-y").Label("foo", "").Obj(),
|
st.MakePod().Name("p-y2").Node("node-y").Label("foo", "").Obj(),
|
||||||
},
|
},
|
||||||
want: &evenPodsSpreadMetadata{
|
want: &evenPodsSpreadMetadata{
|
||||||
|
constraints: []topologySpreadConstraint{
|
||||||
|
{
|
||||||
|
maxSkew: 1,
|
||||||
|
topologyKey: "zone",
|
||||||
|
selector: mustConvertLabelSelectorAsSelector(t, fooSelector),
|
||||||
|
},
|
||||||
|
},
|
||||||
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
||||||
"zone": {{"zone3", 0}, {"zone2", 2}},
|
"zone": {{"zone3", 0}, {"zone2", 2}},
|
||||||
},
|
},
|
||||||
@ -983,7 +917,7 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "namespace mismatch doesn't count",
|
name: "namespace mismatch doesn't count",
|
||||||
pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint(
|
pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint(
|
||||||
1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(),
|
1, "zone", hardSpread, fooSelector,
|
||||||
).Obj(),
|
).Obj(),
|
||||||
nodes: []*v1.Node{
|
nodes: []*v1.Node{
|
||||||
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
|
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
|
||||||
@ -999,6 +933,13 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
|
|||||||
st.MakePod().Name("p-y2").Node("node-y").Label("foo", "").Obj(),
|
st.MakePod().Name("p-y2").Node("node-y").Label("foo", "").Obj(),
|
||||||
},
|
},
|
||||||
want: &evenPodsSpreadMetadata{
|
want: &evenPodsSpreadMetadata{
|
||||||
|
constraints: []topologySpreadConstraint{
|
||||||
|
{
|
||||||
|
maxSkew: 1,
|
||||||
|
topologyKey: "zone",
|
||||||
|
selector: mustConvertLabelSelectorAsSelector(t, fooSelector),
|
||||||
|
},
|
||||||
|
},
|
||||||
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
||||||
"zone": {{"zone2", 1}, {"zone1", 2}},
|
"zone": {{"zone2", 1}, {"zone1", 2}},
|
||||||
},
|
},
|
||||||
@ -1011,8 +952,8 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "normal case with two spreadConstraints",
|
name: "normal case with two spreadConstraints",
|
||||||
pod: st.MakePod().Name("p").Label("foo", "").
|
pod: st.MakePod().Name("p").Label("foo", "").
|
||||||
SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()).
|
SpreadConstraint(1, "zone", hardSpread, fooSelector).
|
||||||
SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()).
|
SpreadConstraint(1, "node", hardSpread, fooSelector).
|
||||||
Obj(),
|
Obj(),
|
||||||
nodes: []*v1.Node{
|
nodes: []*v1.Node{
|
||||||
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
|
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
|
||||||
@ -1030,6 +971,18 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
|
|||||||
st.MakePod().Name("p-y4").Node("node-y").Label("foo", "").Obj(),
|
st.MakePod().Name("p-y4").Node("node-y").Label("foo", "").Obj(),
|
||||||
},
|
},
|
||||||
want: &evenPodsSpreadMetadata{
|
want: &evenPodsSpreadMetadata{
|
||||||
|
constraints: []topologySpreadConstraint{
|
||||||
|
{
|
||||||
|
maxSkew: 1,
|
||||||
|
topologyKey: "zone",
|
||||||
|
selector: mustConvertLabelSelectorAsSelector(t, fooSelector),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
maxSkew: 1,
|
||||||
|
topologyKey: "node",
|
||||||
|
selector: mustConvertLabelSelectorAsSelector(t, fooSelector),
|
||||||
|
},
|
||||||
|
},
|
||||||
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
||||||
"zone": {{"zone1", 3}, {"zone2", 4}},
|
"zone": {{"zone1", 3}, {"zone2", 4}},
|
||||||
"node": {{"node-x", 0}, {"node-b", 1}},
|
"node": {{"node-x", 0}, {"node-b", 1}},
|
||||||
@ -1047,10 +1000,10 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "soft spreadConstraints should be bypassed",
|
name: "soft spreadConstraints should be bypassed",
|
||||||
pod: st.MakePod().Name("p").Label("foo", "").
|
pod: st.MakePod().Name("p").Label("foo", "").
|
||||||
SpreadConstraint(1, "zone", softSpread, st.MakeLabelSelector().Exists("foo").Obj()).
|
SpreadConstraint(1, "zone", softSpread, fooSelector).
|
||||||
SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()).
|
SpreadConstraint(1, "zone", hardSpread, fooSelector).
|
||||||
SpreadConstraint(1, "zone", softSpread, st.MakeLabelSelector().Exists("foo").Obj()).
|
SpreadConstraint(1, "node", softSpread, fooSelector).
|
||||||
SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()).
|
SpreadConstraint(1, "node", hardSpread, fooSelector).
|
||||||
Obj(),
|
Obj(),
|
||||||
nodes: []*v1.Node{
|
nodes: []*v1.Node{
|
||||||
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
|
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
|
||||||
@ -1067,6 +1020,18 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
|
|||||||
st.MakePod().Name("p-y4").Node("node-y").Label("foo", "").Obj(),
|
st.MakePod().Name("p-y4").Node("node-y").Label("foo", "").Obj(),
|
||||||
},
|
},
|
||||||
want: &evenPodsSpreadMetadata{
|
want: &evenPodsSpreadMetadata{
|
||||||
|
constraints: []topologySpreadConstraint{
|
||||||
|
{
|
||||||
|
maxSkew: 1,
|
||||||
|
topologyKey: "zone",
|
||||||
|
selector: mustConvertLabelSelectorAsSelector(t, fooSelector),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
maxSkew: 1,
|
||||||
|
topologyKey: "node",
|
||||||
|
selector: mustConvertLabelSelectorAsSelector(t, fooSelector),
|
||||||
|
},
|
||||||
|
},
|
||||||
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
||||||
"zone": {{"zone1", 3}, {"zone2", 4}},
|
"zone": {{"zone1", 3}, {"zone2", 4}},
|
||||||
"node": {{"node-b", 1}, {"node-a", 2}},
|
"node": {{"node-b", 1}, {"node-a", 2}},
|
||||||
@ -1083,8 +1048,8 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "different labelSelectors - simple version",
|
name: "different labelSelectors - simple version",
|
||||||
pod: st.MakePod().Name("p").Label("foo", "").Label("bar", "").
|
pod: st.MakePod().Name("p").Label("foo", "").Label("bar", "").
|
||||||
SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()).
|
SpreadConstraint(1, "zone", hardSpread, fooSelector).
|
||||||
SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("bar").Obj()).
|
SpreadConstraint(1, "node", hardSpread, barSelector).
|
||||||
Obj(),
|
Obj(),
|
||||||
nodes: []*v1.Node{
|
nodes: []*v1.Node{
|
||||||
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
|
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
|
||||||
@ -1096,6 +1061,18 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
|
|||||||
st.MakePod().Name("p-b").Node("node-b").Label("bar", "").Obj(),
|
st.MakePod().Name("p-b").Node("node-b").Label("bar", "").Obj(),
|
||||||
},
|
},
|
||||||
want: &evenPodsSpreadMetadata{
|
want: &evenPodsSpreadMetadata{
|
||||||
|
constraints: []topologySpreadConstraint{
|
||||||
|
{
|
||||||
|
maxSkew: 1,
|
||||||
|
topologyKey: "zone",
|
||||||
|
selector: mustConvertLabelSelectorAsSelector(t, fooSelector),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
maxSkew: 1,
|
||||||
|
topologyKey: "node",
|
||||||
|
selector: mustConvertLabelSelectorAsSelector(t, barSelector),
|
||||||
|
},
|
||||||
|
},
|
||||||
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
||||||
"zone": {{"zone2", 0}, {"zone1", 1}},
|
"zone": {{"zone2", 0}, {"zone1", 1}},
|
||||||
"node": {{"node-a", 0}, {"node-y", 0}},
|
"node": {{"node-a", 0}, {"node-y", 0}},
|
||||||
@ -1110,10 +1087,10 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "different labelSelectors - complex version",
|
name: "different labelSelectors - complex pods",
|
||||||
pod: st.MakePod().Name("p").Label("foo", "").Label("bar", "").
|
pod: st.MakePod().Name("p").Label("foo", "").Label("bar", "").
|
||||||
SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()).
|
SpreadConstraint(1, "zone", hardSpread, fooSelector).
|
||||||
SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("bar").Obj()).
|
SpreadConstraint(1, "node", hardSpread, barSelector).
|
||||||
Obj(),
|
Obj(),
|
||||||
nodes: []*v1.Node{
|
nodes: []*v1.Node{
|
||||||
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
|
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
|
||||||
@ -1130,6 +1107,18 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
|
|||||||
st.MakePod().Name("p-y4").Node("node-y").Label("foo", "").Label("bar", "").Obj(),
|
st.MakePod().Name("p-y4").Node("node-y").Label("foo", "").Label("bar", "").Obj(),
|
||||||
},
|
},
|
||||||
want: &evenPodsSpreadMetadata{
|
want: &evenPodsSpreadMetadata{
|
||||||
|
constraints: []topologySpreadConstraint{
|
||||||
|
{
|
||||||
|
maxSkew: 1,
|
||||||
|
topologyKey: "zone",
|
||||||
|
selector: mustConvertLabelSelectorAsSelector(t, fooSelector),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
maxSkew: 1,
|
||||||
|
topologyKey: "node",
|
||||||
|
selector: mustConvertLabelSelectorAsSelector(t, barSelector),
|
||||||
|
},
|
||||||
|
},
|
||||||
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
||||||
"zone": {{"zone1", 3}, {"zone2", 4}},
|
"zone": {{"zone1", 3}, {"zone2", 4}},
|
||||||
"node": {{"node-b", 0}, {"node-a", 1}},
|
"node": {{"node-b", 0}, {"node-a", 1}},
|
||||||
@ -1147,8 +1136,8 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
|
|||||||
name: "two spreadConstraints, and with podAffinity",
|
name: "two spreadConstraints, and with podAffinity",
|
||||||
pod: st.MakePod().Name("p").Label("foo", "").
|
pod: st.MakePod().Name("p").Label("foo", "").
|
||||||
NodeAffinityNotIn("node", []string{"node-x"}). // exclude node-x
|
NodeAffinityNotIn("node", []string{"node-x"}). // exclude node-x
|
||||||
SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()).
|
SpreadConstraint(1, "zone", hardSpread, fooSelector).
|
||||||
SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()).
|
SpreadConstraint(1, "node", hardSpread, fooSelector).
|
||||||
Obj(),
|
Obj(),
|
||||||
nodes: []*v1.Node{
|
nodes: []*v1.Node{
|
||||||
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
|
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
|
||||||
@ -1166,6 +1155,18 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
|
|||||||
st.MakePod().Name("p-y4").Node("node-y").Label("foo", "").Obj(),
|
st.MakePod().Name("p-y4").Node("node-y").Label("foo", "").Obj(),
|
||||||
},
|
},
|
||||||
want: &evenPodsSpreadMetadata{
|
want: &evenPodsSpreadMetadata{
|
||||||
|
constraints: []topologySpreadConstraint{
|
||||||
|
{
|
||||||
|
maxSkew: 1,
|
||||||
|
topologyKey: "zone",
|
||||||
|
selector: mustConvertLabelSelectorAsSelector(t, fooSelector),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
maxSkew: 1,
|
||||||
|
topologyKey: "node",
|
||||||
|
selector: mustConvertLabelSelectorAsSelector(t, fooSelector),
|
||||||
|
},
|
||||||
|
},
|
||||||
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
||||||
"zone": {{"zone1", 3}, {"zone2", 4}},
|
"zone": {{"zone1", 3}, {"zone2", 4}},
|
||||||
"node": {{"node-b", 1}, {"node-a", 2}},
|
"node": {{"node-b", 1}, {"node-a", 2}},
|
||||||
@ -1187,13 +1188,20 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
|
|||||||
got, _ := getEvenPodsSpreadMetadata(tt.pod, l)
|
got, _ := getEvenPodsSpreadMetadata(tt.pod, l)
|
||||||
got.sortCriticalPaths()
|
got.sortCriticalPaths()
|
||||||
if !reflect.DeepEqual(got, tt.want) {
|
if !reflect.DeepEqual(got, tt.want) {
|
||||||
t.Errorf("getEvenPodsSpreadMetadata() = %v, want %v", *got, *tt.want)
|
t.Errorf("getEvenPodsSpreadMetadata() = %#v, want %#v", *got, *tt.want)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPodSpreadCache_addPod(t *testing.T) {
|
func TestPodSpreadCache_addPod(t *testing.T) {
|
||||||
|
nodeConstraint := topologySpreadConstraint{
|
||||||
|
maxSkew: 1,
|
||||||
|
topologyKey: "node",
|
||||||
|
selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()),
|
||||||
|
}
|
||||||
|
zoneConstraint := nodeConstraint
|
||||||
|
zoneConstraint.topologyKey = "zone"
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
preemptor *v1.Pod
|
preemptor *v1.Pod
|
||||||
@ -1216,6 +1224,7 @@ func TestPodSpreadCache_addPod(t *testing.T) {
|
|||||||
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
|
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
|
||||||
},
|
},
|
||||||
want: &evenPodsSpreadMetadata{
|
want: &evenPodsSpreadMetadata{
|
||||||
|
constraints: []topologySpreadConstraint{nodeConstraint},
|
||||||
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
||||||
"node": {{"node-b", 0}, {"node-a", 1}},
|
"node": {{"node-b", 0}, {"node-a", 1}},
|
||||||
},
|
},
|
||||||
@ -1240,6 +1249,7 @@ func TestPodSpreadCache_addPod(t *testing.T) {
|
|||||||
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
|
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
|
||||||
},
|
},
|
||||||
want: &evenPodsSpreadMetadata{
|
want: &evenPodsSpreadMetadata{
|
||||||
|
constraints: []topologySpreadConstraint{nodeConstraint},
|
||||||
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
||||||
"node": {{"node-a", 1}, {"node-b", 1}},
|
"node": {{"node-a", 1}, {"node-b", 1}},
|
||||||
},
|
},
|
||||||
@ -1264,6 +1274,7 @@ func TestPodSpreadCache_addPod(t *testing.T) {
|
|||||||
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
|
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
|
||||||
},
|
},
|
||||||
want: &evenPodsSpreadMetadata{
|
want: &evenPodsSpreadMetadata{
|
||||||
|
constraints: []topologySpreadConstraint{nodeConstraint},
|
||||||
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
||||||
"node": {{"node-a", 0}, {"node-b", 1}},
|
"node": {{"node-a", 0}, {"node-b", 1}},
|
||||||
},
|
},
|
||||||
@ -1288,6 +1299,7 @@ func TestPodSpreadCache_addPod(t *testing.T) {
|
|||||||
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
|
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
|
||||||
},
|
},
|
||||||
want: &evenPodsSpreadMetadata{
|
want: &evenPodsSpreadMetadata{
|
||||||
|
constraints: []topologySpreadConstraint{nodeConstraint},
|
||||||
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
||||||
"node": {{"node-a", 0}, {"node-b", 2}},
|
"node": {{"node-a", 0}, {"node-b", 2}},
|
||||||
},
|
},
|
||||||
@ -1311,6 +1323,7 @@ func TestPodSpreadCache_addPod(t *testing.T) {
|
|||||||
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
|
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
|
||||||
},
|
},
|
||||||
want: &evenPodsSpreadMetadata{
|
want: &evenPodsSpreadMetadata{
|
||||||
|
constraints: []topologySpreadConstraint{zoneConstraint, nodeConstraint},
|
||||||
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
||||||
"zone": {{"zone2", 0}, {"zone1", 1}},
|
"zone": {{"zone2", 0}, {"zone1", 1}},
|
||||||
"node": {{"node-x", 0}, {"node-a", 1}},
|
"node": {{"node-x", 0}, {"node-a", 1}},
|
||||||
@ -1339,6 +1352,7 @@ func TestPodSpreadCache_addPod(t *testing.T) {
|
|||||||
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
|
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
|
||||||
},
|
},
|
||||||
want: &evenPodsSpreadMetadata{
|
want: &evenPodsSpreadMetadata{
|
||||||
|
constraints: []topologySpreadConstraint{zoneConstraint, nodeConstraint},
|
||||||
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
||||||
"zone": {{"zone1", 1}, {"zone2", 1}},
|
"zone": {{"zone1", 1}, {"zone2", 1}},
|
||||||
"node": {{"node-a", 1}, {"node-x", 1}},
|
"node": {{"node-a", 1}, {"node-x", 1}},
|
||||||
@ -1370,6 +1384,7 @@ func TestPodSpreadCache_addPod(t *testing.T) {
|
|||||||
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
|
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
|
||||||
},
|
},
|
||||||
want: &evenPodsSpreadMetadata{
|
want: &evenPodsSpreadMetadata{
|
||||||
|
constraints: []topologySpreadConstraint{zoneConstraint, nodeConstraint},
|
||||||
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
||||||
"zone": {{"zone2", 1}, {"zone1", 3}},
|
"zone": {{"zone2", 1}, {"zone1", 3}},
|
||||||
"node": {{"node-a", 1}, {"node-x", 1}},
|
"node": {{"node-a", 1}, {"node-x", 1}},
|
||||||
@ -1402,6 +1417,14 @@ func TestPodSpreadCache_addPod(t *testing.T) {
|
|||||||
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
|
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
|
||||||
},
|
},
|
||||||
want: &evenPodsSpreadMetadata{
|
want: &evenPodsSpreadMetadata{
|
||||||
|
constraints: []topologySpreadConstraint{
|
||||||
|
zoneConstraint,
|
||||||
|
{
|
||||||
|
maxSkew: 1,
|
||||||
|
topologyKey: "node",
|
||||||
|
selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("bar").Obj()),
|
||||||
|
},
|
||||||
|
},
|
||||||
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
||||||
"zone": {{"zone2", 1}, {"zone1", 2}},
|
"zone": {{"zone2", 1}, {"zone1", 2}},
|
||||||
"node": {{"node-a", 0}, {"node-b", 1}},
|
"node": {{"node-a", 0}, {"node-b", 1}},
|
||||||
@ -1434,6 +1457,14 @@ func TestPodSpreadCache_addPod(t *testing.T) {
|
|||||||
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
|
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
|
||||||
},
|
},
|
||||||
want: &evenPodsSpreadMetadata{
|
want: &evenPodsSpreadMetadata{
|
||||||
|
constraints: []topologySpreadConstraint{
|
||||||
|
zoneConstraint,
|
||||||
|
{
|
||||||
|
maxSkew: 1,
|
||||||
|
topologyKey: "node",
|
||||||
|
selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("bar").Obj()),
|
||||||
|
},
|
||||||
|
},
|
||||||
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
||||||
"zone": {{"zone1", 1}, {"zone2", 1}},
|
"zone": {{"zone1", 1}, {"zone2", 1}},
|
||||||
"node": {{"node-a", 1}, {"node-b", 1}},
|
"node": {{"node-a", 1}, {"node-b", 1}},
|
||||||
@ -1463,6 +1494,13 @@ func TestPodSpreadCache_addPod(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestPodSpreadCache_removePod(t *testing.T) {
|
func TestPodSpreadCache_removePod(t *testing.T) {
|
||||||
|
nodeConstraint := topologySpreadConstraint{
|
||||||
|
maxSkew: 1,
|
||||||
|
topologyKey: "node",
|
||||||
|
selector: mustConvertLabelSelectorAsSelector(t, st.MakeLabelSelector().Exists("foo").Obj()),
|
||||||
|
}
|
||||||
|
zoneConstraint := nodeConstraint
|
||||||
|
zoneConstraint.topologyKey = "zone"
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
preemptor *v1.Pod // preemptor pod
|
preemptor *v1.Pod // preemptor pod
|
||||||
@ -1493,6 +1531,7 @@ func TestPodSpreadCache_removePod(t *testing.T) {
|
|||||||
deletedPodIdx: 0, // remove pod "p-a1"
|
deletedPodIdx: 0, // remove pod "p-a1"
|
||||||
nodeIdx: 0, // node-a
|
nodeIdx: 0, // node-a
|
||||||
want: &evenPodsSpreadMetadata{
|
want: &evenPodsSpreadMetadata{
|
||||||
|
constraints: []topologySpreadConstraint{zoneConstraint},
|
||||||
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
||||||
"zone": {{"zone1", 1}, {"zone2", 1}},
|
"zone": {{"zone1", 1}, {"zone2", 1}},
|
||||||
},
|
},
|
||||||
@ -1522,6 +1561,7 @@ func TestPodSpreadCache_removePod(t *testing.T) {
|
|||||||
deletedPodIdx: 0, // remove pod "p-a1"
|
deletedPodIdx: 0, // remove pod "p-a1"
|
||||||
nodeIdx: 0, // node-a
|
nodeIdx: 0, // node-a
|
||||||
want: &evenPodsSpreadMetadata{
|
want: &evenPodsSpreadMetadata{
|
||||||
|
constraints: []topologySpreadConstraint{zoneConstraint},
|
||||||
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
||||||
"zone": {{"zone1", 1}, {"zone2", 2}},
|
"zone": {{"zone1", 1}, {"zone2", 2}},
|
||||||
},
|
},
|
||||||
@ -1552,6 +1592,7 @@ func TestPodSpreadCache_removePod(t *testing.T) {
|
|||||||
deletedPodIdx: 0, // remove pod "p-a0"
|
deletedPodIdx: 0, // remove pod "p-a0"
|
||||||
nodeIdx: 0, // node-a
|
nodeIdx: 0, // node-a
|
||||||
want: &evenPodsSpreadMetadata{
|
want: &evenPodsSpreadMetadata{
|
||||||
|
constraints: []topologySpreadConstraint{zoneConstraint},
|
||||||
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
||||||
"zone": {{"zone1", 2}, {"zone2", 2}},
|
"zone": {{"zone1", 2}, {"zone2", 2}},
|
||||||
},
|
},
|
||||||
@ -1582,6 +1623,7 @@ func TestPodSpreadCache_removePod(t *testing.T) {
|
|||||||
deletedPod: st.MakePod().Name("p-a0").Node("node-a").Label("bar", "").Obj(),
|
deletedPod: st.MakePod().Name("p-a0").Node("node-a").Label("bar", "").Obj(),
|
||||||
nodeIdx: 0, // node-a
|
nodeIdx: 0, // node-a
|
||||||
want: &evenPodsSpreadMetadata{
|
want: &evenPodsSpreadMetadata{
|
||||||
|
constraints: []topologySpreadConstraint{zoneConstraint},
|
||||||
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
||||||
"zone": {{"zone1", 2}, {"zone2", 2}},
|
"zone": {{"zone1", 2}, {"zone2", 2}},
|
||||||
},
|
},
|
||||||
@ -1612,6 +1654,7 @@ func TestPodSpreadCache_removePod(t *testing.T) {
|
|||||||
deletedPodIdx: 3, // remove pod "p-x1"
|
deletedPodIdx: 3, // remove pod "p-x1"
|
||||||
nodeIdx: 2, // node-x
|
nodeIdx: 2, // node-x
|
||||||
want: &evenPodsSpreadMetadata{
|
want: &evenPodsSpreadMetadata{
|
||||||
|
constraints: []topologySpreadConstraint{zoneConstraint, nodeConstraint},
|
||||||
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
tpKeyToCriticalPaths: map[string]*criticalPaths{
|
||||||
"zone": {{"zone2", 1}, {"zone1", 3}},
|
"zone": {{"zone2", 1}, {"zone1", 3}},
|
||||||
"node": {{"node-b", 1}, {"node-x", 1}},
|
"node": {{"node-b", 1}, {"node-x", 1}},
|
||||||
@ -1703,8 +1746,8 @@ var (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// sortCriticalPaths is only served for testing purpose.
|
// sortCriticalPaths is only served for testing purpose.
|
||||||
func (c *evenPodsSpreadMetadata) sortCriticalPaths() {
|
func (m *evenPodsSpreadMetadata) sortCriticalPaths() {
|
||||||
for _, paths := range c.tpKeyToCriticalPaths {
|
for _, paths := range m.tpKeyToCriticalPaths {
|
||||||
// If two paths both hold minimum matching number, and topologyValue is unordered.
|
// If two paths both hold minimum matching number, and topologyValue is unordered.
|
||||||
if paths[0].matchNum == paths[1].matchNum && paths[0].topologyValue > paths[1].topologyValue {
|
if paths[0].matchNum == paths[1].matchNum && paths[0].topologyValue > paths[1].topologyValue {
|
||||||
// Swap topologyValue to make them sorted alphabetically.
|
// Swap topologyValue to make them sorted alphabetically.
|
||||||
@ -1712,3 +1755,12 @@ func (c *evenPodsSpreadMetadata) sortCriticalPaths() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func mustConvertLabelSelectorAsSelector(t *testing.T, ls *metav1.LabelSelector) labels.Selector {
|
||||||
|
t.Helper()
|
||||||
|
s, err := metav1.LabelSelectorAsSelector(ls)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
@ -1642,55 +1642,47 @@ func EvenPodsSpreadPredicate(pod *v1.Pod, meta Metadata, nodeInfo *schedulernode
|
|||||||
if node == nil {
|
if node == nil {
|
||||||
return false, nil, fmt.Errorf("node not found")
|
return false, nil, fmt.Errorf("node not found")
|
||||||
}
|
}
|
||||||
constraints := getHardTopologySpreadConstraints(pod)
|
|
||||||
if len(constraints) == 0 {
|
|
||||||
return true, nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var evenPodsSpreadMetadata *evenPodsSpreadMetadata
|
var epsMeta *evenPodsSpreadMetadata
|
||||||
if predicateMeta, ok := meta.(*predicateMetadata); ok {
|
if predicateMeta, ok := meta.(*predicateMetadata); ok {
|
||||||
evenPodsSpreadMetadata = predicateMeta.evenPodsSpreadMetadata
|
epsMeta = predicateMeta.evenPodsSpreadMetadata
|
||||||
} else { // We don't have precomputed metadata. We have to follow a slow path to check spread constraints.
|
} else { // We don't have precomputed metadata. We have to follow a slow path to check spread constraints.
|
||||||
// TODO(autoscaler): get it implemented
|
// TODO(autoscaler): get it implemented
|
||||||
return false, nil, errors.New("metadata not pre-computed for EvenPodsSpreadPredicate")
|
return false, nil, errors.New("metadata not pre-computed for EvenPodsSpreadPredicate")
|
||||||
}
|
}
|
||||||
|
|
||||||
if evenPodsSpreadMetadata == nil || len(evenPodsSpreadMetadata.tpPairToMatchNum) == 0 {
|
if epsMeta == nil || len(epsMeta.tpPairToMatchNum) == 0 || len(epsMeta.constraints) == 0 {
|
||||||
return true, nil, nil
|
return true, nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
podLabelSet := labels.Set(pod.Labels)
|
podLabelSet := labels.Set(pod.Labels)
|
||||||
for _, constraint := range constraints {
|
for _, c := range epsMeta.constraints {
|
||||||
tpKey := constraint.TopologyKey
|
tpKey := c.topologyKey
|
||||||
tpVal, ok := node.Labels[constraint.TopologyKey]
|
tpVal, ok := node.Labels[c.topologyKey]
|
||||||
if !ok {
|
if !ok {
|
||||||
klog.V(5).Infof("node '%s' doesn't have required label '%s'", node.Name, tpKey)
|
klog.V(5).Infof("node '%s' doesn't have required label '%s'", node.Name, tpKey)
|
||||||
return false, []PredicateFailureReason{ErrTopologySpreadConstraintsNotMatch}, nil
|
return false, []PredicateFailureReason{ErrTopologySpreadConstraintsNotMatch}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
selfMatch, err := PodMatchesSpreadConstraint(podLabelSet, constraint)
|
|
||||||
if err != nil {
|
|
||||||
return false, nil, err
|
|
||||||
}
|
|
||||||
selfMatchNum := int32(0)
|
selfMatchNum := int32(0)
|
||||||
if selfMatch {
|
if c.selector.Matches(podLabelSet) {
|
||||||
selfMatchNum = 1
|
selfMatchNum = 1
|
||||||
}
|
}
|
||||||
|
|
||||||
pair := topologyPair{key: tpKey, value: tpVal}
|
pair := topologyPair{key: tpKey, value: tpVal}
|
||||||
paths, ok := evenPodsSpreadMetadata.tpKeyToCriticalPaths[tpKey]
|
paths, ok := epsMeta.tpKeyToCriticalPaths[tpKey]
|
||||||
if !ok {
|
if !ok {
|
||||||
// error which should not happen
|
// error which should not happen
|
||||||
klog.Errorf("internal error: get paths from key %q of %#v", tpKey, evenPodsSpreadMetadata.tpKeyToCriticalPaths)
|
klog.Errorf("internal error: get paths from key %q of %#v", tpKey, epsMeta.tpKeyToCriticalPaths)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// judging criteria:
|
// judging criteria:
|
||||||
// 'existing matching num' + 'if self-match (1 or 0)' - 'global min matching num' <= 'maxSkew'
|
// 'existing matching num' + 'if self-match (1 or 0)' - 'global min matching num' <= 'maxSkew'
|
||||||
minMatchNum := paths[0].matchNum
|
minMatchNum := paths[0].matchNum
|
||||||
matchNum := evenPodsSpreadMetadata.tpPairToMatchNum[pair]
|
matchNum := epsMeta.tpPairToMatchNum[pair]
|
||||||
skew := matchNum + selfMatchNum - minMatchNum
|
skew := matchNum + selfMatchNum - minMatchNum
|
||||||
if skew > constraint.MaxSkew {
|
if skew > c.maxSkew {
|
||||||
klog.V(5).Infof("node '%s' failed spreadConstraint[%s]: matchNum(%d) + selfMatchNum(%d) - minMatchNum(%d) > maxSkew(%d)", node.Name, tpKey, matchNum, selfMatchNum, minMatchNum, constraint.MaxSkew)
|
klog.V(5).Infof("node '%s' failed spreadConstraint[%s]: matchNum(%d) + selfMatchNum(%d) - minMatchNum(%d) > maxSkew(%d)", node.Name, tpKey, matchNum, selfMatchNum, minMatchNum, c.maxSkew)
|
||||||
return false, []PredicateFailureReason{ErrTopologySpreadConstraintsNotMatch}, nil
|
return false, []PredicateFailureReason{ErrTopologySpreadConstraintsNotMatch}, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -23,12 +23,13 @@ import (
|
|||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
|
|
||||||
v1 "k8s.io/api/core/v1"
|
v1 "k8s.io/api/core/v1"
|
||||||
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/labels"
|
||||||
"k8s.io/client-go/util/workqueue"
|
"k8s.io/client-go/util/workqueue"
|
||||||
"k8s.io/kubernetes/pkg/scheduler/algorithm/predicates"
|
"k8s.io/kubernetes/pkg/scheduler/algorithm/predicates"
|
||||||
framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
|
framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
|
||||||
schedulerlisters "k8s.io/kubernetes/pkg/scheduler/listers"
|
schedulerlisters "k8s.io/kubernetes/pkg/scheduler/listers"
|
||||||
schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo"
|
schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo"
|
||||||
schedutil "k8s.io/kubernetes/pkg/scheduler/util"
|
|
||||||
|
|
||||||
"k8s.io/klog"
|
"k8s.io/klog"
|
||||||
)
|
)
|
||||||
@ -39,12 +40,21 @@ type topologyPair struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type podTopologySpreadMap struct {
|
type podTopologySpreadMap struct {
|
||||||
|
constraints []topologySpreadConstraint
|
||||||
// nodeNameSet is a string set holding all node names which have all constraints[*].topologyKey present.
|
// nodeNameSet is a string set holding all node names which have all constraints[*].topologyKey present.
|
||||||
nodeNameSet map[string]struct{}
|
nodeNameSet map[string]struct{}
|
||||||
// topologyPairToPodCounts is keyed with topologyPair, and valued with the number of matching pods.
|
// topologyPairToPodCounts is keyed with topologyPair, and valued with the number of matching pods.
|
||||||
topologyPairToPodCounts map[topologyPair]*int64
|
topologyPairToPodCounts map[topologyPair]*int64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// topologySpreadConstraint is an internal version for a soft (ScheduleAnyway
|
||||||
|
// unsatisfiable constraint action) v1.TopologySpreadConstraint and where the
|
||||||
|
// selector is parsed.
|
||||||
|
type topologySpreadConstraint struct {
|
||||||
|
topologyKey string
|
||||||
|
selector labels.Selector
|
||||||
|
}
|
||||||
|
|
||||||
func newTopologySpreadConstraintsMap() *podTopologySpreadMap {
|
func newTopologySpreadConstraintsMap() *podTopologySpreadMap {
|
||||||
return &podTopologySpreadMap{
|
return &podTopologySpreadMap{
|
||||||
nodeNameSet: make(map[string]struct{}),
|
nodeNameSet: make(map[string]struct{}),
|
||||||
@ -54,19 +64,22 @@ func newTopologySpreadConstraintsMap() *podTopologySpreadMap {
|
|||||||
|
|
||||||
// buildPodTopologySpreadMap prepares necessary data (podTopologySpreadMap) for incoming pod on the filteredNodes.
|
// buildPodTopologySpreadMap prepares necessary data (podTopologySpreadMap) for incoming pod on the filteredNodes.
|
||||||
// Later Priority function will use 'podTopologySpreadMap' to perform the Scoring calculations.
|
// Later Priority function will use 'podTopologySpreadMap' to perform the Scoring calculations.
|
||||||
func buildPodTopologySpreadMap(pod *v1.Pod, filteredNodes []*v1.Node, allNodes []*schedulernodeinfo.NodeInfo) *podTopologySpreadMap {
|
func buildPodTopologySpreadMap(pod *v1.Pod, filteredNodes []*v1.Node, allNodes []*schedulernodeinfo.NodeInfo) (*podTopologySpreadMap, error) {
|
||||||
// return if incoming pod doesn't have soft topology spread constraints.
|
if len(filteredNodes) == 0 || len(allNodes) == 0 {
|
||||||
constraints := getSoftTopologySpreadConstraints(pod)
|
return nil, nil
|
||||||
if len(constraints) == 0 || len(filteredNodes) == 0 || len(allNodes) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// initialize podTopologySpreadMap which will be used in Score plugin.
|
// initialize podTopologySpreadMap which will be used in Score plugin.
|
||||||
m := newTopologySpreadConstraintsMap()
|
m := newTopologySpreadConstraintsMap()
|
||||||
m.initialize(pod, filteredNodes)
|
err := m.initialize(pod, filteredNodes)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// return if incoming pod doesn't have soft topology spread constraints.
|
||||||
|
if m.constraints == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
errCh := schedutil.NewErrorChannel()
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
processAllNode := func(i int) {
|
processAllNode := func(i int) {
|
||||||
nodeInfo := allNodes[i]
|
nodeInfo := allNodes[i]
|
||||||
node := nodeInfo.Node()
|
node := nodeInfo.Node()
|
||||||
@ -76,12 +89,12 @@ func buildPodTopologySpreadMap(pod *v1.Pod, filteredNodes []*v1.Node, allNodes [
|
|||||||
// (1) `node` should satisfy incoming pod's NodeSelector/NodeAffinity
|
// (1) `node` should satisfy incoming pod's NodeSelector/NodeAffinity
|
||||||
// (2) All topologyKeys need to be present in `node`
|
// (2) All topologyKeys need to be present in `node`
|
||||||
if !predicates.PodMatchesNodeSelectorAndAffinityTerms(pod, node) ||
|
if !predicates.PodMatchesNodeSelectorAndAffinityTerms(pod, node) ||
|
||||||
!predicates.NodeLabelsMatchSpreadConstraints(node.Labels, constraints) {
|
!nodeLabelsMatchSpreadConstraints(node.Labels, m.constraints) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, constraint := range constraints {
|
for _, c := range m.constraints {
|
||||||
pair := topologyPair{key: constraint.TopologyKey, value: node.Labels[constraint.TopologyKey]}
|
pair := topologyPair{key: c.topologyKey, value: node.Labels[c.topologyKey]}
|
||||||
// If current topology pair is not associated with any candidate node,
|
// If current topology pair is not associated with any candidate node,
|
||||||
// continue to avoid unnecessary calculation.
|
// continue to avoid unnecessary calculation.
|
||||||
if m.topologyPairToPodCounts[pair] == nil {
|
if m.topologyPairToPodCounts[pair] == nil {
|
||||||
@ -91,39 +104,37 @@ func buildPodTopologySpreadMap(pod *v1.Pod, filteredNodes []*v1.Node, allNodes [
|
|||||||
// <matchSum> indicates how many pods (on current node) match the <constraint>.
|
// <matchSum> indicates how many pods (on current node) match the <constraint>.
|
||||||
matchSum := int64(0)
|
matchSum := int64(0)
|
||||||
for _, existingPod := range nodeInfo.Pods() {
|
for _, existingPod := range nodeInfo.Pods() {
|
||||||
match, err := predicates.PodMatchesSpreadConstraint(existingPod.Labels, constraint)
|
if c.selector.Matches(labels.Set(existingPod.Labels)) {
|
||||||
if err != nil {
|
|
||||||
errCh.SendErrorWithCancel(err, cancel)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if match {
|
|
||||||
matchSum++
|
matchSum++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
atomic.AddInt64(m.topologyPairToPodCounts[pair], matchSum)
|
atomic.AddInt64(m.topologyPairToPodCounts[pair], matchSum)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
workqueue.ParallelizeUntil(ctx, 16, len(allNodes), processAllNode)
|
workqueue.ParallelizeUntil(context.Background(), 16, len(allNodes), processAllNode)
|
||||||
if err := errCh.ReceiveError(); err != nil {
|
|
||||||
klog.Error(err)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return m
|
return m, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// initialize iterates "filteredNodes" to filter out the nodes which don't have required topologyKey(s),
|
// initialize iterates "filteredNodes" to filter out the nodes which don't have required topologyKey(s),
|
||||||
// and initialize two maps:
|
// and initialize two maps:
|
||||||
// 1) m.topologyPairToPodCounts: keyed with both eligible topology pair and node names.
|
// 1) m.topologyPairToPodCounts: keyed with both eligible topology pair and node names.
|
||||||
// 2) m.nodeNameSet: keyed with node name, and valued with a *int64 pointer for eligible node only.
|
// 2) m.nodeNameSet: keyed with node name, and valued with a *int64 pointer for eligible node only.
|
||||||
func (m *podTopologySpreadMap) initialize(pod *v1.Pod, filteredNodes []*v1.Node) {
|
func (m *podTopologySpreadMap) initialize(pod *v1.Pod, filteredNodes []*v1.Node) error {
|
||||||
constraints := getSoftTopologySpreadConstraints(pod)
|
constraints, err := filterSoftTopologySpreadConstraints(pod.Spec.TopologySpreadConstraints)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if constraints == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
m.constraints = constraints
|
||||||
for _, node := range filteredNodes {
|
for _, node := range filteredNodes {
|
||||||
if !predicates.NodeLabelsMatchSpreadConstraints(node.Labels, constraints) {
|
if !nodeLabelsMatchSpreadConstraints(node.Labels, m.constraints) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
for _, constraint := range constraints {
|
for _, constraint := range m.constraints {
|
||||||
pair := topologyPair{key: constraint.TopologyKey, value: node.Labels[constraint.TopologyKey]}
|
pair := topologyPair{key: constraint.topologyKey, value: node.Labels[constraint.topologyKey]}
|
||||||
if m.topologyPairToPodCounts[pair] == nil {
|
if m.topologyPairToPodCounts[pair] == nil {
|
||||||
m.topologyPairToPodCounts[pair] = new(int64)
|
m.topologyPairToPodCounts[pair] = new(int64)
|
||||||
}
|
}
|
||||||
@ -132,6 +143,7 @@ func (m *podTopologySpreadMap) initialize(pod *v1.Pod, filteredNodes []*v1.Node)
|
|||||||
// For those nodes which don't have all required topologyKeys present, it's intentional to leave
|
// For those nodes which don't have all required topologyKeys present, it's intentional to leave
|
||||||
// their entries absent in nodeNameSet, so that we're able to score them to 0 afterwards.
|
// their entries absent in nodeNameSet, so that we're able to score them to 0 afterwards.
|
||||||
}
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// CalculateEvenPodsSpreadPriorityMap calculate the number of matching pods on the passed-in "node",
|
// CalculateEvenPodsSpreadPriorityMap calculate the number of matching pods on the passed-in "node",
|
||||||
@ -155,13 +167,12 @@ func CalculateEvenPodsSpreadPriorityMap(pod *v1.Pod, meta interface{}, nodeInfo
|
|||||||
return framework.NodeScore{Name: node.Name, Score: 0}, nil
|
return framework.NodeScore{Name: node.Name, Score: 0}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
constraints := getSoftTopologySpreadConstraints(pod)
|
|
||||||
// For each present <pair>, current node gets a credit of <matchSum>.
|
// For each present <pair>, current node gets a credit of <matchSum>.
|
||||||
// And we sum up <matchSum> and return it as this node's score.
|
// And we sum up <matchSum> and return it as this node's score.
|
||||||
var score int64
|
var score int64
|
||||||
for _, constraint := range constraints {
|
for _, c := range m.constraints {
|
||||||
if tpVal, ok := node.Labels[constraint.TopologyKey]; ok {
|
if tpVal, ok := node.Labels[c.topologyKey]; ok {
|
||||||
pair := topologyPair{key: constraint.TopologyKey, value: tpVal}
|
pair := topologyPair{key: c.topologyKey, value: tpVal}
|
||||||
matchSum := *m.topologyPairToPodCounts[pair]
|
matchSum := *m.topologyPairToPodCounts[pair]
|
||||||
score += matchSum
|
score += matchSum
|
||||||
}
|
}
|
||||||
@ -228,14 +239,29 @@ func CalculateEvenPodsSpreadPriorityReduce(pod *v1.Pod, meta interface{}, shared
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO(Huang-Wei): combine this with getHardTopologySpreadConstraints() in predicates package
|
func filterSoftTopologySpreadConstraints(constraints []v1.TopologySpreadConstraint) ([]topologySpreadConstraint, error) {
|
||||||
func getSoftTopologySpreadConstraints(pod *v1.Pod) (constraints []v1.TopologySpreadConstraint) {
|
var r []topologySpreadConstraint
|
||||||
if pod != nil {
|
for _, c := range constraints {
|
||||||
for _, constraint := range pod.Spec.TopologySpreadConstraints {
|
if c.WhenUnsatisfiable == v1.ScheduleAnyway {
|
||||||
if constraint.WhenUnsatisfiable == v1.ScheduleAnyway {
|
selector, err := metav1.LabelSelectorAsSelector(c.LabelSelector)
|
||||||
constraints = append(constraints, constraint)
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
r = append(r, topologySpreadConstraint{
|
||||||
|
topologyKey: c.TopologyKey,
|
||||||
|
selector: selector,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
return r, nil
|
||||||
return
|
}
|
||||||
|
|
||||||
|
// nodeLabelsMatchSpreadConstraints checks if ALL topology keys in spread constraints are present in node labels.
|
||||||
|
func nodeLabelsMatchSpreadConstraints(nodeLabels map[string]string, constraints []topologySpreadConstraint) bool {
|
||||||
|
for _, c := range constraints {
|
||||||
|
if _, ok := nodeLabels[c.topologyKey]; !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
}
|
}
|
||||||
|
@ -83,7 +83,9 @@ func Test_podTopologySpreadMap_initialize(t *testing.T) {
|
|||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
m := newTopologySpreadConstraintsMap()
|
m := newTopologySpreadConstraintsMap()
|
||||||
m.initialize(tt.pod, tt.nodes)
|
if err := m.initialize(tt.pod, tt.nodes); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
if !reflect.DeepEqual(m.nodeNameSet, tt.wantNodeNameSet) {
|
if !reflect.DeepEqual(m.nodeNameSet, tt.wantNodeNameSet) {
|
||||||
t.Errorf("initilize().nodeNameSet = %#v, want %#v", m.nodeNameSet, tt.wantNodeNameSet)
|
t.Errorf("initilize().nodeNameSet = %#v, want %#v", m.nodeNameSet, tt.wantNodeNameSet)
|
||||||
}
|
}
|
||||||
@ -436,8 +438,12 @@ func TestCalculateEvenPodsSpreadPriority(t *testing.T) {
|
|||||||
allNodes = append(allNodes, tt.failedNodes...)
|
allNodes = append(allNodes, tt.failedNodes...)
|
||||||
snapshot := nodeinfosnapshot.NewSnapshot(nodeinfosnapshot.CreateNodeInfoMap(tt.existingPods, allNodes))
|
snapshot := nodeinfosnapshot.NewSnapshot(nodeinfosnapshot.CreateNodeInfoMap(tt.existingPods, allNodes))
|
||||||
|
|
||||||
|
tpSpreadMap, err := buildPodTopologySpreadMap(tt.pod, tt.nodes, snapshot.NodeInfoList)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
meta := &priorityMetadata{
|
meta := &priorityMetadata{
|
||||||
podTopologySpreadMap: buildPodTopologySpreadMap(tt.pod, tt.nodes, snapshot.NodeInfoList),
|
podTopologySpreadMap: tpSpreadMap,
|
||||||
}
|
}
|
||||||
var gotList framework.NodeScoreList
|
var gotList framework.NodeScoreList
|
||||||
for _, n := range tt.nodes {
|
for _, n := range tt.nodes {
|
||||||
@ -449,7 +455,10 @@ func TestCalculateEvenPodsSpreadPriority(t *testing.T) {
|
|||||||
gotList = append(gotList, nodeScore)
|
gotList = append(gotList, nodeScore)
|
||||||
}
|
}
|
||||||
|
|
||||||
CalculateEvenPodsSpreadPriorityReduce(tt.pod, meta, snapshot, gotList)
|
err = CalculateEvenPodsSpreadPriorityReduce(tt.pod, meta, snapshot, gotList)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
if !reflect.DeepEqual(gotList, tt.want) {
|
if !reflect.DeepEqual(gotList, tt.want) {
|
||||||
t.Errorf("CalculateEvenPodsSpreadPriorityReduce() = %#v, want %#v", gotList, tt.want)
|
t.Errorf("CalculateEvenPodsSpreadPriorityReduce() = %#v, want %#v", gotList, tt.want)
|
||||||
}
|
}
|
||||||
@ -498,8 +507,12 @@ func BenchmarkTestCalculateEvenPodsSpreadPriority(b *testing.B) {
|
|||||||
b.Run(tt.name, func(b *testing.B) {
|
b.Run(tt.name, func(b *testing.B) {
|
||||||
existingPods, allNodes, filteredNodes := st.MakeNodesAndPodsForEvenPodsSpread(tt.pod.Labels, tt.existingPodsNum, tt.allNodesNum, tt.filteredNodesNum)
|
existingPods, allNodes, filteredNodes := st.MakeNodesAndPodsForEvenPodsSpread(tt.pod.Labels, tt.existingPodsNum, tt.allNodesNum, tt.filteredNodesNum)
|
||||||
snapshot := nodeinfosnapshot.NewSnapshot(nodeinfosnapshot.CreateNodeInfoMap(existingPods, allNodes))
|
snapshot := nodeinfosnapshot.NewSnapshot(nodeinfosnapshot.CreateNodeInfoMap(existingPods, allNodes))
|
||||||
|
tpSpreadMap, err := buildPodTopologySpreadMap(tt.pod, filteredNodes, snapshot.NodeInfoList)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
meta := &priorityMetadata{
|
meta := &priorityMetadata{
|
||||||
podTopologySpreadMap: buildPodTopologySpreadMap(tt.pod, filteredNodes, snapshot.NodeInfoList),
|
podTopologySpreadMap: tpSpreadMap,
|
||||||
}
|
}
|
||||||
b.ResetTimer()
|
b.ResetTimer()
|
||||||
|
|
||||||
@ -510,7 +523,10 @@ func BenchmarkTestCalculateEvenPodsSpreadPriority(b *testing.B) {
|
|||||||
nodeScore, _ := CalculateEvenPodsSpreadPriorityMap(tt.pod, meta, snapshot.NodeInfoMap[nodeName])
|
nodeScore, _ := CalculateEvenPodsSpreadPriorityMap(tt.pod, meta, snapshot.NodeInfoMap[nodeName])
|
||||||
gotList = append(gotList, nodeScore)
|
gotList = append(gotList, nodeScore)
|
||||||
}
|
}
|
||||||
CalculateEvenPodsSpreadPriorityReduce(tt.pod, meta, snapshot, gotList)
|
err := CalculateEvenPodsSpreadPriorityReduce(tt.pod, meta, snapshot, gotList)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
@ -22,6 +22,7 @@ import (
|
|||||||
"k8s.io/apimachinery/pkg/labels"
|
"k8s.io/apimachinery/pkg/labels"
|
||||||
appslisters "k8s.io/client-go/listers/apps/v1"
|
appslisters "k8s.io/client-go/listers/apps/v1"
|
||||||
corelisters "k8s.io/client-go/listers/core/v1"
|
corelisters "k8s.io/client-go/listers/core/v1"
|
||||||
|
"k8s.io/klog"
|
||||||
schedulerlisters "k8s.io/kubernetes/pkg/scheduler/listers"
|
schedulerlisters "k8s.io/kubernetes/pkg/scheduler/listers"
|
||||||
schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo"
|
schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo"
|
||||||
)
|
)
|
||||||
@ -84,6 +85,11 @@ func (pmf *MetadataFactory) PriorityMetadata(
|
|||||||
allNodes = l
|
allNodes = l
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
tpSpreadMap, err := buildPodTopologySpreadMap(pod, filteredNodes, allNodes)
|
||||||
|
if err != nil {
|
||||||
|
klog.Errorf("Error building podTopologySpreadMap: %v", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
return &priorityMetadata{
|
return &priorityMetadata{
|
||||||
podLimits: getResourceLimits(pod),
|
podLimits: getResourceLimits(pod),
|
||||||
podTolerations: getAllTolerationPreferNoSchedule(pod.Spec.Tolerations),
|
podTolerations: getAllTolerationPreferNoSchedule(pod.Spec.Tolerations),
|
||||||
@ -92,7 +98,7 @@ func (pmf *MetadataFactory) PriorityMetadata(
|
|||||||
controllerRef: metav1.GetControllerOf(pod),
|
controllerRef: metav1.GetControllerOf(pod),
|
||||||
podFirstServiceSelector: getFirstServiceSelector(pod, pmf.serviceLister),
|
podFirstServiceSelector: getFirstServiceSelector(pod, pmf.serviceLister),
|
||||||
totalNumNodes: totalNumNodes,
|
totalNumNodes: totalNumNodes,
|
||||||
podTopologySpreadMap: buildPodTopologySpreadMap(pod, filteredNodes, allNodes),
|
podTopologySpreadMap: tpSpreadMap,
|
||||||
topologyScore: buildTopologyPairToScore(pod, sharedLister, filteredNodes, pmf.hardPodAffinityWeight),
|
topologyScore: buildTopologyPairToScore(pod, sharedLister, filteredNodes, pmf.hardPodAffinityWeight),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -59,8 +59,12 @@ func BenchmarkTestDefaultEvenPodsSpreadPriority(b *testing.B) {
|
|||||||
b.ResetTimer()
|
b.ResetTimer()
|
||||||
|
|
||||||
for i := 0; i < b.N; i++ {
|
for i := 0; i < b.N; i++ {
|
||||||
|
tpSpreadMap, err := buildPodTopologySpreadMap(pod, filteredNodes, snapshot.NodeInfoList)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
meta := &priorityMetadata{
|
meta := &priorityMetadata{
|
||||||
podTopologySpreadMap: buildPodTopologySpreadMap(pod, filteredNodes, snapshot.NodeInfoList),
|
podTopologySpreadMap: tpSpreadMap,
|
||||||
}
|
}
|
||||||
var gotList framework.NodeScoreList
|
var gotList framework.NodeScoreList
|
||||||
for _, n := range filteredNodes {
|
for _, n := range filteredNodes {
|
||||||
@ -70,7 +74,7 @@ func BenchmarkTestDefaultEvenPodsSpreadPriority(b *testing.B) {
|
|||||||
}
|
}
|
||||||
gotList = append(gotList, score)
|
gotList = append(gotList, score)
|
||||||
}
|
}
|
||||||
err := CalculateEvenPodsSpreadPriorityReduce(pod, meta, snapshot, gotList)
|
err = CalculateEvenPodsSpreadPriorityReduce(pod, meta, snapshot, gotList)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
b.Fatal(err)
|
b.Fatal(err)
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user