github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2026-01-05 07:27:21 +00:00
Code Issues Projects Releases Wiki Activity

[scheduling] Moved node affinity from annotations to api fields. #35518

Browse Source
This commit is contained in:
Robert Rati
2016-11-30 11:51:12 -05:00
parent 1eb9176455
commit 91931c138e
26 changed files with 648 additions and 501 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
plugin/pkg/scheduler/algorithm/predicates/predicates.go
View File

@@ -582,14 +582,6 @@ func podMatchesNodeLabels(pod *v1.Pod, node *v1.Node) bool {
}
}
// Parse required node affinity scheduling requirements
// and check if the current node match the requirements.
affinity, err := v1.GetAffinityFromPodAnnotations(pod.Annotations)
if err != nil {
glog.V(10).Infof("Failed to get Affinity from Pod %+v, err: %+v", podName(pod), err)
return false
}
// 1. nil NodeSelector matches all nodes (i.e. does not filter out any nodes)
// 2. nil []NodeSelectorTerm (equivalent to non-nil empty NodeSelector) matches no nodes
// 3. zero-length non-nil []NodeSelectorTerm matches no nodes also, just for simplicity
@@ -597,6 +589,7 @@ func podMatchesNodeLabels(pod *v1.Pod, node *v1.Node) bool {
// 5. zero-length non-nil []NodeSelectorRequirement matches no nodes also, just for simplicity
// 6. non-nil empty NodeSelectorRequirement is not allowed
nodeAffinityMatches := true
affinity := pod.Spec.Affinity
if affinity != nil && affinity.NodeAffinity != nil {
nodeAffinity := affinity.NodeAffinity
// if no required NodeAffinity requirements, will do no-op, means select all nodes.

458
plugin/pkg/scheduler/algorithm/predicates/predicates_test.go
View File

@@ -866,18 +866,23 @@ func TestPodFitsSelector(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "foo",
"operator": "In",
"values": ["bar", "value2"]
}]
}]
}}}`,
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "foo",
Operator: v1.NodeSelectorOpIn,
Values: []string{"bar", "value2"},
},
},
},
},
},
},
},
},
},
@@ -889,18 +894,23 @@ func TestPodFitsSelector(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "kernel-version",
"operator": "Gt",
"values": ["0204"]
}]
}]
}}}`,
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "kernel-version",
Operator: v1.NodeSelectorOpGt,
Values: []string{"0204"},
},
},
},
},
},
},
},
},
},
@@ -913,18 +923,23 @@ func TestPodFitsSelector(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "mem-type",
"operator": "NotIn",
"values": ["DDR", "DDR2"]
}]
}]
}}}`,
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "mem-type",
Operator: v1.NodeSelectorOpNotIn,
Values: []string{"DDR", "DDR2"},
},
},
},
},
},
},
},
},
},
@@ -936,17 +951,22 @@ func TestPodFitsSelector(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "GPU",
"operator": "Exists"
}]
}]
}}}`,
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "GPU",
Operator: v1.NodeSelectorOpExists,
},
},
},
},
},
},
},
},
},
@@ -958,18 +978,23 @@ func TestPodFitsSelector(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "foo",
"operator": "In",
"values": ["value1", "value2"]
}]
}]
}}}`,
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "foo",
Operator: v1.NodeSelectorOpIn,
Values: []string{"value1", "value2"},
},
},
},
},
},
},
},
},
},
@@ -981,12 +1006,13 @@ func TestPodFitsSelector(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": null
}}}`,
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: nil,
},
},
},
},
},
@@ -998,12 +1024,13 @@ func TestPodFitsSelector(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": []
}}}`,
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{},
},
},
},
},
},
@@ -1013,31 +1040,54 @@ func TestPodFitsSelector(t *testing.T) {
fits: false,
test: "Pod with an empty []NodeSelectorTerm in affinity, can't match the node's labels and won't schedule onto the node",
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{}, {}]
}}}`,
/*
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{}, {}]
}}}`,
},
},
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: {},
},
{
MatchExpressions: {},
},
},
},
},
},
},
},
labels: map[string]string{
"foo": "bar",
},
fits: false,
test: "Pod with invalid NodeSelectTerms in affinity will match no objects and won't schedule onto the node",
},
labels: map[string]string{
"foo": "bar",
},
fits: false,
test: "Pod with invalid NodeSelectTerms in affinity will match no objects and won't schedule onto the node",
},
*/
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{"matchExpressions": [{}]}]
}}}`,
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{},
},
},
},
},
},
},
},
@@ -1048,13 +1098,7 @@ func TestPodFitsSelector(t *testing.T) {
test: "Pod with empty MatchExpressions is not a valid value will match no objects and won't schedule onto the node",
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
"some-key": "some-value",
},
},
},
pod: &v1.Pod{},
labels: map[string]string{
"foo": "bar",
},
@@ -1063,11 +1107,11 @@ func TestPodFitsSelector(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": null
}}`,
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: nil,
},
},
},
},
@@ -1079,21 +1123,26 @@ func TestPodFitsSelector(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "GPU",
"operator": "Exists"
}, {
"key": "GPU",
"operator": "NotIn",
"values": ["AMD", "INTER"]
}]
}]
}}}`,
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "GPU",
Operator: v1.NodeSelectorOpExists,
}, {
Key: "GPU",
Operator: v1.NodeSelectorOpNotIn,
Values: []string{"AMD", "INTER"},
},
},
},
},
},
},
},
},
},
@@ -1105,21 +1154,26 @@ func TestPodFitsSelector(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "GPU",
"operator": "Exists"
}, {
"key": "GPU",
"operator": "In",
"values": ["AMD", "INTER"]
}]
}]
}}}`,
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "GPU",
Operator: v1.NodeSelectorOpExists,
}, {
Key: "GPU",
Operator: v1.NodeSelectorOpIn,
Values: []string{"AMD", "INTER"},
},
},
},
},
},
},
},
},
},
@@ -1131,27 +1185,32 @@ func TestPodFitsSelector(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [
{
"matchExpressions": [{
"key": "foo",
"operator": "In",
"values": ["bar", "value2"]
}]
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "foo",
Operator: v1.NodeSelectorOpIn,
Values: []string{"bar", "value2"},
},
},
},
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "diffkey",
Operator: v1.NodeSelectorOpIn,
Values: []string{"wrong", "value2"},
},
},
},
},
{
"matchExpressions": [{
"key": "diffkey",
"operator": "In",
"values": ["wrong", "value2"]
}]
}
]
}}}`,
},
},
},
},
},
@@ -1199,23 +1258,26 @@ func TestPodFitsSelector(t *testing.T) {
// },
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "foo",
"operator": "Exists"
}]
}]
}}}`,
},
},
Spec: v1.PodSpec{
NodeSelector: map[string]string{
"foo": "bar",
},
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "foo",
Operator: v1.NodeSelectorOpExists,
},
},
},
},
},
},
},
},
},
labels: map[string]string{
@@ -1227,23 +1289,26 @@ func TestPodFitsSelector(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "foo",
"operator": "Exists"
}]
}]
}}}`,
},
},
Spec: v1.PodSpec{
NodeSelector: map[string]string{
"foo": "bar",
},
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "foo",
Operator: v1.NodeSelectorOpExists,
},
},
},
},
},
},
},
},
},
labels: map[string]string{
@@ -2604,17 +2669,6 @@ func TestInterPodAffinityWithMultipleNodes(t *testing.T) {
Annotations: map[string]string{
v1.AffinityAnnotationKey: `
{
"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "hostname",
"operator": "NotIn",
"values": ["h1"]
}]
}]
}
},
"podAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": [{
"labelSelector": {
@@ -2630,6 +2684,25 @@ func TestInterPodAffinityWithMultipleNodes(t *testing.T) {
}`,
},
},
Spec: v1.PodSpec{
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{
{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "hostname",
Operator: v1.NodeSelectorOpNotIn,
Values: []string{"h1"},
},
},
},
},
},
},
},
},
},
pods: []*v1.Pod{
{Spec: v1.PodSpec{NodeName: "nodeA"}, ObjectMeta: v1.ObjectMeta{Labels: map[string]string{"foo": "abc"}}},
@@ -2781,10 +2854,7 @@ func TestInterPodAffinityWithMultipleNodes(t *testing.T) {
if !fits && !reflect.DeepEqual(reasons, affinityExpectedFailureReasons) {
t.Errorf("%s: unexpected failure reasons: %v", test.test, reasons)
}
affinity, err := v1.GetAffinityFromPodAnnotations(test.pod.ObjectMeta.Annotations)
if err != nil {
t.Errorf("%s: unexpected error: %v", test.test, err)
}
affinity := test.pod.Spec.Affinity
if affinity != nil && affinity.NodeAffinity != nil {
nodeInfo := schedulercache.NewNodeInfo()
nodeInfo.SetNode(&node)

7
plugin/pkg/scheduler/algorithm/priorities/metadata.go
View File

@@ -38,10 +38,15 @@ func PriorityMetadata(pod *v1.Pod, nodeNameToInfo map[string]*schedulercache.Nod
if err != nil {
return nil
}
affinity, err := v1.GetAffinityFromPodAnnotations(pod.Annotations)
affinity := pod.Spec.Affinity
annotationAffinity, err := v1.GetAffinityFromPodAnnotations(pod.Annotations)
if err != nil {
return nil
}
if annotationAffinity != nil {
affinity.PodAffinity = annotationAffinity.PodAffinity
affinity.PodAntiAffinity = annotationAffinity.PodAntiAffinity
}
return &priorityMetadata{
nonZeroRequest: getNonZeroRequests(pod),
podTolerations: tolerations,

8
plugin/pkg/scheduler/algorithm/priorities/node_affinity.go
View File

@@ -41,12 +41,8 @@ func CalculateNodeAffinityPriorityMap(pod *v1.Pod, meta interface{}, nodeInfo *s
if priorityMeta, ok := meta.(*priorityMetadata); ok {
affinity = priorityMeta.affinity
} else {
// We couldn't parse metadata - fallback to computing it.
var err error
affinity, err = v1.GetAffinityFromPodAnnotations(pod.Annotations)
if err != nil {
return schedulerapi.HostPriority{}, err
}
// We couldn't parse metadata - fallback to the podspec.
affinity = pod.Spec.Affinity
}
var count int32

126
plugin/pkg/scheduler/algorithm/priorities/node_affinity_test.go
View File

@@ -32,62 +32,72 @@ func TestNodeAffinityPriority(t *testing.T) {
label4 := map[string]string{"abc": "az11", "def": "az22"}
label5 := map[string]string{"foo": "bar", "key": "value", "az": "az1"}
affinity1 := map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": {"preferredDuringSchedulingIgnoredDuringExecution": [
{
"weight": 2,
"preference": {
"matchExpressions": [
{
"key": "foo",
"operator": "In", "values": ["bar"]
}
]
}
}
]}}`,
affinity1 := &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{{
Weight: 2,
Preference: v1.NodeSelectorTerm{
MatchExpressions: []v1.NodeSelectorRequirement{{
Key: "foo",
Operator: v1.NodeSelectorOpIn,
Values: []string{"bar"},
}},
},
}},
},
}
affinity2 := map[string]string{
v1.AffinityAnnotationKey: `
{"nodeAffinity": {"preferredDuringSchedulingIgnoredDuringExecution": [
{
"weight": 2,
"preference": {"matchExpressions": [
{
"key": "foo",
"operator": "In", "values": ["bar"]
}
]}
},
{
"weight": 4,
"preference": {"matchExpressions": [
{
"key": "key",
"operator": "In", "values": ["value"]
}
]}
},
{
"weight": 5,
"preference": {"matchExpressions": [
{
"key": "foo",
"operator": "In", "values": ["bar"]
affinity2 := &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{
{
Weight: 2,
Preference: v1.NodeSelectorTerm{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "foo",
Operator: v1.NodeSelectorOpIn,
Values: []string{"bar"},
},
},
},
{
"key": "key",
"operator": "In", "values": ["value"]
},
{
Weight: 4,
Preference: v1.NodeSelectorTerm{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "key",
Operator: v1.NodeSelectorOpIn,
Values: []string{"value"},
},
},
},
{
"key": "az",
"operator": "In", "values": ["az1"]
}
]}
}
]}}`,
},
{
Weight: 5,
Preference: v1.NodeSelectorTerm{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "foo",
Operator: v1.NodeSelectorOpIn,
Values: []string{"bar"},
},
{
Key: "key",
Operator: v1.NodeSelectorOpIn,
Values: []string{"value"},
},
{
Key: "az",
Operator: v1.NodeSelectorOpIn,
Values: []string{"az1"},
},
},
},
},
},
},
}
tests := []struct {
@@ -112,8 +122,8 @@ func TestNodeAffinityPriority(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: affinity1,
Spec: v1.PodSpec{
Affinity: affinity1,
},
},
nodes: []*v1.Node{
@@ -126,8 +136,8 @@ func TestNodeAffinityPriority(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: affinity1,
Spec: v1.PodSpec{
Affinity: affinity1,
},
},
nodes: []*v1.Node{
@@ -140,8 +150,8 @@ func TestNodeAffinityPriority(t *testing.T) {
},
{
pod: &v1.Pod{
ObjectMeta: v1.ObjectMeta{
Annotations: affinity2,
Spec: v1.PodSpec{
Affinity: affinity2,
},
},
nodes: []*v1.Node{