From b5934271051d161e3bece12aece53696587f1c6f Mon Sep 17 00:00:00 2001 From: Janet Kuo Date: Wed, 8 Feb 2017 17:33:59 -0800 Subject: [PATCH] Enable PodTolerateNodeTaints predicate in DaemonSet controller --- pkg/controller/daemon/BUILD | 1 + pkg/controller/daemon/daemoncontroller.go | 42 ++++++++++-- .../daemon/daemoncontroller_test.go | 66 +++++++++++++++++++ 3 files changed, 102 insertions(+), 7 deletions(-) diff --git a/pkg/controller/daemon/BUILD b/pkg/controller/daemon/BUILD index a6eb36cdd6a..d019a6d1e8e 100644 --- a/pkg/controller/daemon/BUILD +++ b/pkg/controller/daemon/BUILD @@ -27,6 +27,7 @@ go_library( "//pkg/client/listers/extensions/v1beta1:go_default_library", "//pkg/controller:go_default_library", "//pkg/util/metrics:go_default_library", + "//plugin/pkg/scheduler/algorithm:go_default_library", "//plugin/pkg/scheduler/algorithm/predicates:go_default_library", "//plugin/pkg/scheduler/schedulercache:go_default_library", "//vendor:github.com/golang/glog", diff --git a/pkg/controller/daemon/daemoncontroller.go b/pkg/controller/daemon/daemoncontroller.go index 613baaf3b57..94376e8c167 100644 --- a/pkg/controller/daemon/daemoncontroller.go +++ b/pkg/controller/daemon/daemoncontroller.go @@ -45,6 +45,7 @@ import ( extensionslisters "k8s.io/kubernetes/pkg/client/listers/extensions/v1beta1" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/util/metrics" + "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm" "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/predicates" "k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache" @@ -779,13 +780,14 @@ func (dsc *DaemonSetsController) nodeShouldRunDaemonPod(node *v1.Node, ds *exten nodeInfo := schedulercache.NewNodeInfo(pods...) nodeInfo.SetNode(node) - _, reasons, err := predicates.GeneralPredicates(newPod, nil, nodeInfo) + _, reasons, err := daemonSetPredicates(newPod, nodeInfo) if err != nil { - glog.Warningf("GeneralPredicates failed on ds '%s/%s' due to unexpected error: %v", ds.ObjectMeta.Namespace, ds.ObjectMeta.Name, err) + glog.Warningf("daemonSetPredicates failed on ds '%s/%s' due to unexpected error: %v", ds.ObjectMeta.Namespace, ds.ObjectMeta.Name, err) return false, false, false, err } + for _, r := range reasons { - glog.V(4).Infof("GeneralPredicates failed on ds '%s/%s' for reason: %v", ds.ObjectMeta.Namespace, ds.ObjectMeta.Name, r.GetReason()) + glog.V(4).Infof("daemonSetPredicates failed on ds '%s/%s' for reason: %v", ds.ObjectMeta.Namespace, ds.ObjectMeta.Name, r.GetReason()) switch reason := r.(type) { case *predicates.InsufficientResourceError: dsc.eventRecorder.Eventf(ds, v1.EventTypeNormal, FailedPlacementReason, "failed to place pod on %q: %s", node.ObjectMeta.Name, reason.Error()) @@ -801,7 +803,9 @@ func (dsc *DaemonSetsController) nodeShouldRunDaemonPod(node *v1.Node, ds *exten predicates.ErrNodeLabelPresenceViolated, // this one is probably intentional since it's a workaround for not having // pod hard anti affinity. - predicates.ErrPodNotFitsHostPorts: + predicates.ErrPodNotFitsHostPorts, + // DaemonSet is expected to respect taints and tolerations + predicates.ErrTaintsTolerationsNotMatch: wantToRun, shouldSchedule, shouldContinueRunning = false, false, false // unintentional case @@ -818,9 +822,9 @@ func (dsc *DaemonSetsController) nodeShouldRunDaemonPod(node *v1.Node, ds *exten // unexpected case predicates.ErrPodAffinityNotMatch, - predicates.ErrServiceAffinityViolated, - predicates.ErrTaintsTolerationsNotMatch: - return false, false, false, fmt.Errorf("unexpected reason: GeneralPredicates should not return reason %s", reason.GetReason()) + predicates.ErrServiceAffinityViolated: + glog.Warningf("unexpected predicate failure reason: %s", reason.GetReason()) + return false, false, false, fmt.Errorf("unexpected reason: daemonSetPredicates should not return reason %s", reason.GetReason()) default: glog.V(4).Infof("unknown predicate failure reason: %s", reason.GetReason()) wantToRun, shouldSchedule, shouldContinueRunning = false, false, false @@ -834,6 +838,30 @@ func (dsc *DaemonSetsController) nodeShouldRunDaemonPod(node *v1.Node, ds *exten return } +// daemonSetPredicates checks if a DaemonSet's pod can be scheduled on a node using GeneralPredicates +// and PodToleratesNodeTaints predicate +func daemonSetPredicates(pod *v1.Pod, nodeInfo *schedulercache.NodeInfo) (bool, []algorithm.PredicateFailureReason, error) { + var predicateFails []algorithm.PredicateFailureReason + + fit, reasons, err := predicates.GeneralPredicates(pod, nil, nodeInfo) + if err != nil { + return false, predicateFails, err + } + if !fit { + predicateFails = append(predicateFails, reasons...) + } + + fit, reasons, err = predicates.PodToleratesNodeTaints(pod, nil, nodeInfo) + if err != nil { + return false, predicateFails, err + } + if !fit { + predicateFails = append(predicateFails, reasons...) + } + + return len(predicateFails) == 0, predicateFails, nil +} + // byCreationTimestamp sorts a list by creation timestamp, using their names as a tie breaker. type byCreationTimestamp []*extensions.DaemonSet diff --git a/pkg/controller/daemon/daemoncontroller_test.go b/pkg/controller/daemon/daemoncontroller_test.go index 4e47c2e5237..de7d84de8d0 100644 --- a/pkg/controller/daemon/daemoncontroller_test.go +++ b/pkg/controller/daemon/daemoncontroller_test.go @@ -45,6 +45,15 @@ var ( alwaysReady = func() bool { return true } ) +const ( + noSchedule = ` + [{ + "key": "dedicated", + "value": "user1", + "effect": "NoSchedule" + }]` +) + func getKey(ds *extensions.DaemonSet, t *testing.T) string { if key, err := controller.KeyFunc(ds); err != nil { t.Errorf("Unexpected error getting key for ds %v: %v", ds.Name, err) @@ -708,6 +717,63 @@ func TestDaemonKillFailedPods(t *testing.T) { } } +// DaemonSet should not launch a pod on a tainted node when the pod doesn't tolerate that taint. +func TestTaintedNodeDaemonDoesNotLaunchUntoleratePod(t *testing.T) { + manager, podControl, _ := newTestController() + + node := newNode("tainted", nil) + setNodeTaint(node, noSchedule) + manager.nodeStore.Add(node) + + ds := newDaemonSet("untolerate") + manager.dsStore.Add(ds) + + syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0) +} + +// DaemonSet should launch a pod on a tainted node when the pod can tolerate that taint. +func TestTaintedNodeDaemonLaunchesToleratePod(t *testing.T) { + manager, podControl, _ := newTestController() + + node := newNode("tainted", nil) + setNodeTaint(node, noSchedule) + manager.nodeStore.Add(node) + + ds := newDaemonSet("tolerate") + setDaemonSetToleration(ds, noSchedule) + manager.dsStore.Add(ds) + + syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0) +} + +// DaemonSet should launch a pod on an untainted node when the pod has tolerations. +func TestNodeDaemonLaunchesToleratePod(t *testing.T) { + manager, podControl, _ := newTestController() + + node := newNode("untainted", nil) + manager.nodeStore.Add(node) + + ds := newDaemonSet("tolerate") + setDaemonSetToleration(ds, noSchedule) + manager.dsStore.Add(ds) + + syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0) +} + +func setNodeTaint(node *v1.Node, taint string) { + if node.ObjectMeta.Annotations == nil { + node.ObjectMeta.Annotations = make(map[string]string) + } + node.ObjectMeta.Annotations[v1.TaintsAnnotationKey] = taint +} + +func setDaemonSetToleration(ds *extensions.DaemonSet, toleration string) { + if ds.Spec.Template.ObjectMeta.Annotations == nil { + ds.Spec.Template.ObjectMeta.Annotations = make(map[string]string) + } + ds.Spec.Template.ObjectMeta.Annotations[v1.TolerationsAnnotationKey] = toleration +} + func TestNodeShouldRunDaemonPod(t *testing.T) { cases := []struct { podsOnNode []*v1.Pod