From e5168cc7bb4cfb924403d21c80efa6ef89b5d88c Mon Sep 17 00:00:00 2001 From: Alex Wang Date: Wed, 25 Dec 2019 17:26:58 +0800 Subject: [PATCH] move nodepreferavoidpods to score plugin --- pkg/scheduler/algorithm/priorities/BUILD | 2 - .../algorithm/priorities/metadata.go | 2 - .../priorities/node_prefer_avoid_pods.go | 67 -------- .../priorities/node_prefer_avoid_pods_test.go | 157 ------------------ .../defaults/register_priorities.go | 2 +- .../plugins/nodepreferavoidpods/BUILD | 4 +- .../node_prefer_avoid_pods.go | 36 +++- 7 files changed, 34 insertions(+), 236 deletions(-) delete mode 100644 pkg/scheduler/algorithm/priorities/node_prefer_avoid_pods.go delete mode 100644 pkg/scheduler/algorithm/priorities/node_prefer_avoid_pods_test.go diff --git a/pkg/scheduler/algorithm/priorities/BUILD b/pkg/scheduler/algorithm/priorities/BUILD index 277f93b71bb..2da5492ac2f 100644 --- a/pkg/scheduler/algorithm/priorities/BUILD +++ b/pkg/scheduler/algorithm/priorities/BUILD @@ -14,7 +14,6 @@ go_library( "least_requested.go", "metadata.go", "most_requested.go", - "node_prefer_avoid_pods.go", "priorities.go", "reduce.go", "requested_to_capacity_ratio.go", @@ -54,7 +53,6 @@ go_test( "least_requested_test.go", "metadata_test.go", "most_requested_test.go", - "node_prefer_avoid_pods_test.go", "requested_to_capacity_ratio_test.go", "resource_limits_test.go", "selector_spreading_test.go", diff --git a/pkg/scheduler/algorithm/priorities/metadata.go b/pkg/scheduler/algorithm/priorities/metadata.go index 5195f15a9f1..4cac2a7c714 100644 --- a/pkg/scheduler/algorithm/priorities/metadata.go +++ b/pkg/scheduler/algorithm/priorities/metadata.go @@ -58,7 +58,6 @@ func NewMetadataFactory( type priorityMetadata struct { podLimits *schedulernodeinfo.Resource podSelector labels.Selector - controllerRef *metav1.OwnerReference podFirstServiceSelector labels.Selector podTopologySpreadMap *podTopologySpreadMap } @@ -87,7 +86,6 @@ func (pmf *MetadataFactory) PriorityMetadata( return &priorityMetadata{ podLimits: getResourceLimits(pod), podSelector: getSelector(pod, pmf.serviceLister, pmf.controllerLister, pmf.replicaSetLister, pmf.statefulSetLister), - controllerRef: metav1.GetControllerOf(pod), podFirstServiceSelector: getFirstServiceSelector(pod, pmf.serviceLister), podTopologySpreadMap: tpSpreadMap, } diff --git a/pkg/scheduler/algorithm/priorities/node_prefer_avoid_pods.go b/pkg/scheduler/algorithm/priorities/node_prefer_avoid_pods.go deleted file mode 100644 index f7c17164ca8..00000000000 --- a/pkg/scheduler/algorithm/priorities/node_prefer_avoid_pods.go +++ /dev/null @@ -1,67 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package priorities - -import ( - "fmt" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" - framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" - schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" -) - -// CalculateNodePreferAvoidPodsPriorityMap priorities nodes according to the node annotation -// "scheduler.alpha.kubernetes.io/preferAvoidPods". -func CalculateNodePreferAvoidPodsPriorityMap(pod *v1.Pod, meta interface{}, nodeInfo *schedulernodeinfo.NodeInfo) (framework.NodeScore, error) { - node := nodeInfo.Node() - if node == nil { - return framework.NodeScore{}, fmt.Errorf("node not found") - } - var controllerRef *metav1.OwnerReference - if priorityMeta, ok := meta.(*priorityMetadata); ok { - controllerRef = priorityMeta.controllerRef - } else { - // We couldn't parse metadata - fallback to the podspec. - controllerRef = metav1.GetControllerOf(pod) - } - - if controllerRef != nil { - // Ignore pods that are owned by other controller than ReplicationController - // or ReplicaSet. - if controllerRef.Kind != "ReplicationController" && controllerRef.Kind != "ReplicaSet" { - controllerRef = nil - } - } - if controllerRef == nil { - return framework.NodeScore{Name: node.Name, Score: framework.MaxNodeScore}, nil - } - - avoids, err := v1helper.GetAvoidPodsFromNodeAnnotations(node.Annotations) - if err != nil { - // If we cannot get annotation, assume it's schedulable there. - return framework.NodeScore{Name: node.Name, Score: framework.MaxNodeScore}, nil - } - for i := range avoids.PreferAvoidPods { - avoid := &avoids.PreferAvoidPods[i] - if avoid.PodSignature.PodController.Kind == controllerRef.Kind && avoid.PodSignature.PodController.UID == controllerRef.UID { - return framework.NodeScore{Name: node.Name, Score: 0}, nil - } - } - return framework.NodeScore{Name: node.Name, Score: framework.MaxNodeScore}, nil -} diff --git a/pkg/scheduler/algorithm/priorities/node_prefer_avoid_pods_test.go b/pkg/scheduler/algorithm/priorities/node_prefer_avoid_pods_test.go deleted file mode 100644 index dcdf95bdc17..00000000000 --- a/pkg/scheduler/algorithm/priorities/node_prefer_avoid_pods_test.go +++ /dev/null @@ -1,157 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package priorities - -import ( - "reflect" - "testing" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" - nodeinfosnapshot "k8s.io/kubernetes/pkg/scheduler/nodeinfo/snapshot" -) - -func TestNodePreferAvoidPriority(t *testing.T) { - annotations1 := map[string]string{ - v1.PreferAvoidPodsAnnotationKey: ` - { - "preferAvoidPods": [ - { - "podSignature": { - "podController": { - "apiVersion": "v1", - "kind": "ReplicationController", - "name": "foo", - "uid": "abcdef123456", - "controller": true - } - }, - "reason": "some reason", - "message": "some message" - } - ] - }`, - } - annotations2 := map[string]string{ - v1.PreferAvoidPodsAnnotationKey: ` - { - "preferAvoidPods": [ - { - "podSignature": { - "podController": { - "apiVersion": "v1", - "kind": "ReplicaSet", - "name": "foo", - "uid": "qwert12345", - "controller": true - } - }, - "reason": "some reason", - "message": "some message" - } - ] - }`, - } - testNodes := []*v1.Node{ - { - ObjectMeta: metav1.ObjectMeta{Name: "machine1", Annotations: annotations1}, - }, - { - ObjectMeta: metav1.ObjectMeta{Name: "machine2", Annotations: annotations2}, - }, - { - ObjectMeta: metav1.ObjectMeta{Name: "machine3"}, - }, - } - trueVar := true - tests := []struct { - pod *v1.Pod - nodes []*v1.Node - expectedList framework.NodeScoreList - name string - }{ - { - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "default", - OwnerReferences: []metav1.OwnerReference{ - {Kind: "ReplicationController", Name: "foo", UID: "abcdef123456", Controller: &trueVar}, - }, - }, - }, - nodes: testNodes, - expectedList: []framework.NodeScore{{Name: "machine1", Score: 0}, {Name: "machine2", Score: framework.MaxNodeScore}, {Name: "machine3", Score: framework.MaxNodeScore}}, - name: "pod managed by ReplicationController should avoid a node, this node get lowest priority score", - }, - { - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "default", - OwnerReferences: []metav1.OwnerReference{ - {Kind: "RandomController", Name: "foo", UID: "abcdef123456", Controller: &trueVar}, - }, - }, - }, - nodes: testNodes, - expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: framework.MaxNodeScore}, {Name: "machine3", Score: framework.MaxNodeScore}}, - name: "ownership by random controller should be ignored", - }, - { - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "default", - OwnerReferences: []metav1.OwnerReference{ - {Kind: "ReplicationController", Name: "foo", UID: "abcdef123456"}, - }, - }, - }, - nodes: testNodes, - expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: framework.MaxNodeScore}, {Name: "machine3", Score: framework.MaxNodeScore}}, - name: "owner without Controller field set should be ignored", - }, - { - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "default", - OwnerReferences: []metav1.OwnerReference{ - {Kind: "ReplicaSet", Name: "foo", UID: "qwert12345", Controller: &trueVar}, - }, - }, - }, - nodes: testNodes, - expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: 0}, {Name: "machine3", Score: framework.MaxNodeScore}}, - name: "pod managed by ReplicaSet should avoid a node, this node get lowest priority score", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - snapshot := nodeinfosnapshot.NewSnapshot(nodeinfosnapshot.CreateNodeInfoMap(nil, test.nodes)) - list, err := runMapReducePriority(CalculateNodePreferAvoidPodsPriorityMap, nil, nil, test.pod, snapshot, test.nodes) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - // sort the two lists to avoid failures on account of different ordering - sortNodeScoreList(test.expectedList) - sortNodeScoreList(list) - if !reflect.DeepEqual(test.expectedList, list) { - t.Errorf("expected %#v, got %#v", test.expectedList, list) - } - }) - } -} diff --git a/pkg/scheduler/algorithmprovider/defaults/register_priorities.go b/pkg/scheduler/algorithmprovider/defaults/register_priorities.go index 66302eaaccf..6e9888282cb 100644 --- a/pkg/scheduler/algorithmprovider/defaults/register_priorities.go +++ b/pkg/scheduler/algorithmprovider/defaults/register_priorities.go @@ -80,7 +80,7 @@ func init() { // Set this weight large enough to override all other priority functions. // TODO: Figure out a better way to do this, maybe at same time as fixing #24720. - scheduler.RegisterPriorityMapReduceFunction(priorities.NodePreferAvoidPodsPriority, priorities.CalculateNodePreferAvoidPodsPriorityMap, nil, 10000) + scheduler.RegisterPriorityMapReduceFunction(priorities.NodePreferAvoidPodsPriority, nil, nil, 10000) // Prioritizes nodes that have labels matching NodeAffinity scheduler.RegisterPriorityMapReduceFunction(priorities.NodeAffinityPriority, nil, nil, 1) diff --git a/pkg/scheduler/framework/plugins/nodepreferavoidpods/BUILD b/pkg/scheduler/framework/plugins/nodepreferavoidpods/BUILD index ee60c6b6728..93926ed1494 100644 --- a/pkg/scheduler/framework/plugins/nodepreferavoidpods/BUILD +++ b/pkg/scheduler/framework/plugins/nodepreferavoidpods/BUILD @@ -6,10 +6,10 @@ go_library( importpath = "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodepreferavoidpods", visibility = ["//visibility:public"], deps = [ - "//pkg/scheduler/algorithm/priorities:go_default_library", - "//pkg/scheduler/framework/plugins/migration:go_default_library", + "//pkg/apis/core/v1/helper:go_default_library", "//pkg/scheduler/framework/v1alpha1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", ], ) diff --git a/pkg/scheduler/framework/plugins/nodepreferavoidpods/node_prefer_avoid_pods.go b/pkg/scheduler/framework/plugins/nodepreferavoidpods/node_prefer_avoid_pods.go index ce62bb34b4e..8450f6b707f 100644 --- a/pkg/scheduler/framework/plugins/nodepreferavoidpods/node_prefer_avoid_pods.go +++ b/pkg/scheduler/framework/plugins/nodepreferavoidpods/node_prefer_avoid_pods.go @@ -21,9 +21,9 @@ import ( "fmt" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/kubernetes/pkg/scheduler/algorithm/priorities" - "k8s.io/kubernetes/pkg/scheduler/framework/plugins/migration" + v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" ) @@ -50,9 +50,35 @@ func (pl *NodePreferAvoidPods) Score(ctx context.Context, state *framework.Cycle return 0, framework.NewStatus(framework.Error, fmt.Sprintf("getting node %q from Snapshot: %v", nodeName, err)) } - meta := migration.PriorityMetadata(state) - s, err := priorities.CalculateNodePreferAvoidPodsPriorityMap(pod, meta, nodeInfo) - return s.Score, migration.ErrorToFrameworkStatus(err) + node := nodeInfo.Node() + if node == nil { + return 0, framework.NewStatus(framework.Error, "node not found") + } + + controllerRef := metav1.GetControllerOf(pod) + if controllerRef != nil { + // Ignore pods that are owned by other controller than ReplicationController + // or ReplicaSet. + if controllerRef.Kind != "ReplicationController" && controllerRef.Kind != "ReplicaSet" { + controllerRef = nil + } + } + if controllerRef == nil { + return framework.MaxNodeScore, nil + } + + avoids, err := v1helper.GetAvoidPodsFromNodeAnnotations(node.Annotations) + if err != nil { + // If we cannot get annotation, assume it's schedulable there. + return framework.MaxNodeScore, nil + } + for i := range avoids.PreferAvoidPods { + avoid := &avoids.PreferAvoidPods[i] + if avoid.PodSignature.PodController.Kind == controllerRef.Kind && avoid.PodSignature.PodController.UID == controllerRef.UID { + return 0, nil + } + } + return framework.MaxNodeScore, nil } // ScoreExtensions of the Score plugin.