test(scheduler): add more integration tests for TaintNodesByCondition

- refactor test cases to table driven testing style
- more scenarios are covered for TaintNodesByCondition
- update bazel stuff

test/integration/scheduler/BUILD

@@ -96,6 +96,7 @@ go_library(
         "//pkg/scheduler/algorithmprovider:go_default_library",
         "//pkg/scheduler/api:go_default_library",
         "//pkg/scheduler/factory:go_default_library",
+        "//pkg/util/taints:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/api/policy/v1beta1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",

test/integration/scheduler/taint_test.go

@@ -19,7 +19,7 @@ package scheduler
 // This file tests the Taint feature.
 
 import (
-	"reflect"
+	"fmt"
 	"testing"
 	"time"
 
@@ -29,7 +29,6 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	restclient "k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/cache"
 	"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
 	internalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
 	"k8s.io/kubernetes/pkg/controller/nodelifecycle"
@@ -40,10 +39,28 @@ import (
 	pluginapi "k8s.io/kubernetes/plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction"
 )
 
-// TestTaintNodeByCondition verifies:
+func newPod(nsName, name string, req, limit v1.ResourceList) *v1.Pod {
-//   1. MemoryPressure Toleration is added to non-BestEffort Pod by PodTolerationRestriction
+	return &v1.Pod{
-//   2. NodeController taints nodes by node condition
+		ObjectMeta: metav1.ObjectMeta{
-//   3. Scheduler allows pod to tolerate node condition taints, e.g. network unavailable
+			Name:      name,
+			Namespace: nsName,
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:  "busybox",
+					Image: "busybox",
+					Resources: v1.ResourceRequirements{
+						Requests: req,
+						Limits:   limit,
+					},
+				},
+			},
+		},
+	}
+}
+
+// TestTaintNodeByCondition tests related cases for TaintNodeByCondition feature.
 func TestTaintNodeByCondition(t *testing.T) {
 	enabled := utilfeature.DefaultFeatureGate.Enabled("TaintNodesByCondition")
 	defer func() {
@@ -76,7 +93,7 @@ func TestTaintNodeByCondition(t *testing.T) {
 	algorithmprovider.ApplyFeatureGates()
 
 	context = initTestScheduler(t, context, controllerCh, false, nil)
-	clientset := context.clientSet
+	cs := context.clientSet
 	informers := context.informerFactory
 	nsName := context.ns.Name
 
@@ -86,7 +103,7 @@ func TestTaintNodeByCondition(t *testing.T) {
 		informers.Core().V1().Nodes(),
 		informers.Extensions().V1beta1().DaemonSets(),
 		nil, // CloudProvider
-		clientset,
+		cs,
 		time.Second, // Node monitor grace period
 		time.Second, // Node startup grace period
 		time.Second, // Node monitor period
@@ -112,83 +129,369 @@ func TestTaintNodeByCondition(t *testing.T) {
 	// -------------------------------------------
 	// Test TaintNodeByCondition feature.
 	// -------------------------------------------
+	nodeRes := v1.ResourceList{
+		v1.ResourceCPU:    resource.MustParse("4000m"),
+		v1.ResourceMemory: resource.MustParse("16Gi"),
+		v1.ResourcePods:   resource.MustParse("110"),
+	}
+
+	podRes := v1.ResourceList{
+		v1.ResourceCPU:    resource.MustParse("100m"),
+		v1.ResourceMemory: resource.MustParse("100Mi"),
+	}
+
+	notReadyToleration := v1.Toleration{
+		Key:      algorithm.TaintNodeNotReady,
+		Operator: v1.TolerationOpExists,
+		Effect:   v1.TaintEffectNoSchedule,
+	}
+
+	unreachableToleration := v1.Toleration{
+		Key:      algorithm.TaintNodeUnreachable,
+		Operator: v1.TolerationOpExists,
+		Effect:   v1.TaintEffectNoSchedule,
+	}
+
+	unschedulableToleration := v1.Toleration{
+		Key:      algorithm.TaintNodeUnschedulable,
+		Operator: v1.TolerationOpExists,
+		Effect:   v1.TaintEffectNoSchedule,
+	}
+
+	outOfDiskToleration := v1.Toleration{
+		Key:      algorithm.TaintNodeOutOfDisk,
+		Operator: v1.TolerationOpExists,
+		Effect:   v1.TaintEffectNoSchedule,
+	}
+
 	memoryPressureToleration := v1.Toleration{
 		Key:      algorithm.TaintNodeMemoryPressure,
 		Operator: v1.TolerationOpExists,
 		Effect:   v1.TaintEffectNoSchedule,
 	}
 
-	// Case 1: Add MememoryPressure Toleration for non-BestEffort pod.
+	diskPressureToleration := v1.Toleration{
-	burstablePod := &v1.Pod{
+		Key:      algorithm.TaintNodeDiskPressure,
-		ObjectMeta: metav1.ObjectMeta{
+		Operator: v1.TolerationOpExists,
-			Name:      "burstable-pod",
+		Effect:   v1.TaintEffectNoSchedule,
-			Namespace: nsName,
+	}
-		},
+
-		Spec: v1.PodSpec{
+	networkUnavailableToleration := v1.Toleration{
-			Containers: []v1.Container{
+		Key:      algorithm.TaintNodeNetworkUnavailable,
+		Operator: v1.TolerationOpExists,
+		Effect:   v1.TaintEffectNoSchedule,
+	}
+
+	pidPressureToleration := v1.Toleration{
+		Key:      algorithm.TaintNodePIDPressure,
+		Operator: v1.TolerationOpExists,
+		Effect:   v1.TaintEffectNoSchedule,
+	}
+
+	bestEffortPod := newPod(nsName, "besteffort-pod", nil, nil)
+	burstablePod := newPod(nsName, "burstable-pod", podRes, nil)
+	guaranteePod := newPod(nsName, "guarantee-pod", podRes, podRes)
+
+	type podCase struct {
+		pod         *v1.Pod
+		tolerations []v1.Toleration
+		fits        bool
+	}
+
+	// switch to table driven testings
+	tests := []struct {
+		name           string
+		existingTaints []v1.Taint
+		nodeConditions []v1.NodeCondition
+		unschedulable  bool
+		expectedTaints []v1.Taint
+		pods           []podCase
+	}{
 		{
-					Name:  "busybox",
+			name: "not-ready node",
-					Image: "busybox",
+			nodeConditions: []v1.NodeCondition{
-					Resources: v1.ResourceRequirements{
-						Requests: v1.ResourceList{
-							v1.ResourceCPU: resource.MustParse("100m"),
-						},
-					},
-				},
-			},
-		},
-	}
-
-	burstablePodInServ, err := clientset.CoreV1().Pods(nsName).Create(burstablePod)
-	if err != nil {
-		t.Errorf("Case 1: Failed to create pod: %v", err)
-	} else if !reflect.DeepEqual(burstablePodInServ.Spec.Tolerations, []v1.Toleration{memoryPressureToleration}) {
-		t.Errorf("Case 1: Unexpected toleration of non-BestEffort pod, expected: %+v, got: %v",
-			[]v1.Toleration{memoryPressureToleration},
-			burstablePodInServ.Spec.Tolerations)
-	}
-
-	// Case 2: No MemoryPressure Toleration for BestEffort pod.
-	besteffortPod := &v1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "best-effort-pod",
-			Namespace: nsName,
-		},
-		Spec: v1.PodSpec{
-			Containers: []v1.Container{
 				{
-					Name:  "busybox",
+					Type:   v1.NodeReady,
-					Image: "busybox",
+					Status: v1.ConditionFalse,
+				},
+			},
+			expectedTaints: []v1.Taint{
+				{
+					Key:    algorithm.TaintNodeNotReady,
+					Effect: v1.TaintEffectNoSchedule,
+				},
+			},
+			pods: []podCase{
+				{
+					pod:  bestEffortPod,
+					fits: false,
+				},
+				{
+					pod:  burstablePod,
+					fits: false,
+				},
+				{
+					pod:  guaranteePod,
+					fits: false,
+				},
+				{
+					pod:         bestEffortPod,
+					tolerations: []v1.Toleration{notReadyToleration},
+					fits:        true,
 				},
 			},
 		},
-	}
+		{
-
+			name: "unreachable node",
-	besteffortPodInServ, err := clientset.CoreV1().Pods(nsName).Create(besteffortPod)
+			existingTaints: []v1.Taint{
-	if err != nil {
+				{
-		t.Errorf("Case 2: Failed to create pod: %v", err)
+					Key:    algorithm.TaintNodeUnreachable,
-	} else if len(besteffortPodInServ.Spec.Tolerations) != 0 {
+					Effect: v1.TaintEffectNoSchedule,
-		t.Errorf("Case 2: Unexpected toleration # of BestEffort pod, expected: 0, got: %v",
-			len(besteffortPodInServ.Spec.Tolerations))
-	}
-
-	// Case 3: Taint Node by NetworkUnavailable condition.
-	networkUnavailableNode := &v1.Node{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: "node-1",
 				},
-		Status: v1.NodeStatus{
-			Capacity: v1.ResourceList{
-				v1.ResourceCPU:    resource.MustParse("4000m"),
-				v1.ResourceMemory: resource.MustParse("16Gi"),
-				v1.ResourcePods:   resource.MustParse("110"),
 			},
-			Allocatable: v1.ResourceList{
+			nodeConditions: []v1.NodeCondition{
-				v1.ResourceCPU:    resource.MustParse("4000m"),
+				{
-				v1.ResourceMemory: resource.MustParse("16Gi"),
+					Type:   v1.NodeReady,
-				v1.ResourcePods:   resource.MustParse("110"),
+					Status: v1.ConditionUnknown, // node status is "Unknown"
 				},
-			Conditions: []v1.NodeCondition{
+			},
+			expectedTaints: []v1.Taint{
+				{
+					Key:    algorithm.TaintNodeUnreachable,
+					Effect: v1.TaintEffectNoSchedule,
+				},
+			},
+			pods: []podCase{
+				{
+					pod:  bestEffortPod,
+					fits: false,
+				},
+				{
+					pod:  burstablePod,
+					fits: false,
+				},
+				{
+					pod:  guaranteePod,
+					fits: false,
+				},
+				{
+					pod:         bestEffortPod,
+					tolerations: []v1.Toleration{unreachableToleration},
+					fits:        true,
+				},
+			},
+		},
+		{
+			name:          "unschedulable node",
+			unschedulable: true, // node.spec.unschedulable = true
+			nodeConditions: []v1.NodeCondition{
+				{
+					Type:   v1.NodeReady,
+					Status: v1.ConditionTrue,
+				},
+			},
+			expectedTaints: []v1.Taint{
+				{
+					Key:    algorithm.TaintNodeUnschedulable,
+					Effect: v1.TaintEffectNoSchedule,
+				},
+			},
+			pods: []podCase{
+				{
+					pod:  bestEffortPod,
+					fits: false,
+				},
+				{
+					pod:  burstablePod,
+					fits: false,
+				},
+				{
+					pod:  guaranteePod,
+					fits: false,
+				},
+				{
+					pod:         bestEffortPod,
+					tolerations: []v1.Toleration{unschedulableToleration},
+					fits:        true,
+				},
+			},
+		},
+		{
+			name: "out of disk node",
+			nodeConditions: []v1.NodeCondition{
+				{
+					Type:   v1.NodeOutOfDisk,
+					Status: v1.ConditionTrue,
+				},
+				{
+					Type:   v1.NodeReady,
+					Status: v1.ConditionTrue,
+				},
+			},
+			expectedTaints: []v1.Taint{
+				{
+					Key:    algorithm.TaintNodeOutOfDisk,
+					Effect: v1.TaintEffectNoSchedule,
+				},
+			},
+			// In OutOfDisk condition, only pods with toleration can be scheduled.
+			pods: []podCase{
+				{
+					pod:  bestEffortPod,
+					fits: false,
+				},
+				{
+					pod:  burstablePod,
+					fits: false,
+				},
+				{
+					pod:  guaranteePod,
+					fits: false,
+				},
+				{
+					pod:         bestEffortPod,
+					tolerations: []v1.Toleration{outOfDiskToleration},
+					fits:        true,
+				},
+				{
+					pod:         bestEffortPod,
+					tolerations: []v1.Toleration{diskPressureToleration},
+					fits:        false,
+				},
+			},
+		},
+		{
+			name: "memory pressure node",
+			nodeConditions: []v1.NodeCondition{
+				{
+					Type:   v1.NodeMemoryPressure,
+					Status: v1.ConditionTrue,
+				},
+				{
+					Type:   v1.NodeReady,
+					Status: v1.ConditionTrue,
+				},
+			},
+			expectedTaints: []v1.Taint{
+				{
+					Key:    algorithm.TaintNodeMemoryPressure,
+					Effect: v1.TaintEffectNoSchedule,
+				},
+			},
+			// In MemoryPressure condition, both Burstable and Guarantee pods are scheduled;
+			// BestEffort pod with toleration are also scheduled.
+			pods: []podCase{
+				{
+					pod:  bestEffortPod,
+					fits: false,
+				},
+				{
+					pod:         bestEffortPod,
+					tolerations: []v1.Toleration{memoryPressureToleration},
+					fits:        true,
+				},
+				{
+					pod:         bestEffortPod,
+					tolerations: []v1.Toleration{diskPressureToleration},
+					fits:        false,
+				},
+				{
+					pod:  burstablePod,
+					fits: true,
+				},
+				{
+					pod:  guaranteePod,
+					fits: true,
+				},
+			},
+		},
+		{
+			name: "disk pressure node",
+			nodeConditions: []v1.NodeCondition{
+				{
+					Type:   v1.NodeDiskPressure,
+					Status: v1.ConditionTrue,
+				},
+				{
+					Type:   v1.NodeReady,
+					Status: v1.ConditionTrue,
+				},
+			},
+			expectedTaints: []v1.Taint{
+				{
+					Key:    algorithm.TaintNodeDiskPressure,
+					Effect: v1.TaintEffectNoSchedule,
+				},
+			},
+			// In DiskPressure condition, only pods with toleration can be scheduled.
+			pods: []podCase{
+				{
+					pod:  bestEffortPod,
+					fits: false,
+				},
+				{
+					pod:  burstablePod,
+					fits: false,
+				},
+				{
+					pod:  guaranteePod,
+					fits: false,
+				},
+				{
+					pod:         bestEffortPod,
+					tolerations: []v1.Toleration{diskPressureToleration},
+					fits:        true,
+				},
+				{
+					pod:         bestEffortPod,
+					tolerations: []v1.Toleration{memoryPressureToleration},
+					fits:        false,
+				},
+			},
+		},
+		{
+			name: "network unavailable and node is ready",
+			nodeConditions: []v1.NodeCondition{
+				{
+					Type:   v1.NodeNetworkUnavailable,
+					Status: v1.ConditionTrue,
+				},
+				{
+					Type:   v1.NodeReady,
+					Status: v1.ConditionTrue,
+				},
+			},
+			expectedTaints: []v1.Taint{
+				{
+					Key:    algorithm.TaintNodeNetworkUnavailable,
+					Effect: v1.TaintEffectNoSchedule,
+				},
+			},
+			pods: []podCase{
+				{
+					pod:  bestEffortPod,
+					fits: false,
+				},
+				{
+					pod:  burstablePod,
+					fits: false,
+				},
+				{
+					pod:  guaranteePod,
+					fits: false,
+				},
+				{
+					pod: burstablePod,
+					tolerations: []v1.Toleration{
+						networkUnavailableToleration,
+					},
+					fits: true,
+				},
+			},
+		},
+		{
+			name: "network unavailable and node is not ready",
+			nodeConditions: []v1.NodeCondition{
 				{
 					Type:   v1.NodeNetworkUnavailable,
 					Status: v1.ConditionTrue,
@@ -198,121 +501,174 @@ func TestTaintNodeByCondition(t *testing.T) {
 					Status: v1.ConditionFalse,
 				},
 			},
-		},
+			expectedTaints: []v1.Taint{
-	}
-
-	nodeInformerCh := make(chan bool)
-	nodeInformer := informers.Core().V1().Nodes().Informer()
-	nodeInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
-		UpdateFunc: func(old, cur interface{}) {
-			curNode := cur.(*v1.Node)
-			if curNode.Name != "node-1" {
-				return
-			}
-			for _, taint := range curNode.Spec.Taints {
-				if taint.Key == algorithm.TaintNodeNetworkUnavailable &&
-					taint.Effect == v1.TaintEffectNoSchedule {
-					nodeInformerCh <- true
-					break
-				}
-			}
-		},
-	})
-
-	if _, err := clientset.CoreV1().Nodes().Create(networkUnavailableNode); err != nil {
-		t.Errorf("Case 3: Failed to create node: %v", err)
-	} else {
-		select {
-		case <-time.After(60 * time.Second):
-			t.Errorf("Case 3: Failed to taint node after 60s.")
-		case <-nodeInformerCh:
-		}
-	}
-
-	// Case 4: Schedule Pod with NetworkUnavailable and NotReady toleration.
-	networkDaemonPod := &v1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "network-daemon-pod",
-			Namespace: nsName,
-		},
-		Spec: v1.PodSpec{
-			Containers: []v1.Container{
-				{
-					Name:  "busybox",
-					Image: "busybox",
-				},
-			},
-			Tolerations: []v1.Toleration{
 				{
 					Key:    algorithm.TaintNodeNetworkUnavailable,
-					Operator: v1.TolerationOpExists,
 					Effect: v1.TaintEffectNoSchedule,
 				},
 				{
 					Key:    algorithm.TaintNodeNotReady,
-					Operator: v1.TolerationOpExists,
+					Effect: v1.TaintEffectNoSchedule,
+				},
+			},
+			pods: []podCase{
+				{
+					pod:  bestEffortPod,
+					fits: false,
+				},
+				{
+					pod:  burstablePod,
+					fits: false,
+				},
+				{
+					pod:  guaranteePod,
+					fits: false,
+				},
+				{
+					pod: burstablePod,
+					tolerations: []v1.Toleration{
+						networkUnavailableToleration,
+					},
+					fits: false,
+				},
+				{
+					pod: burstablePod,
+					tolerations: []v1.Toleration{
+						networkUnavailableToleration,
+						notReadyToleration,
+					},
+					fits: true,
+				},
+			},
+		},
+		{
+			name: "pid pressure node",
+			nodeConditions: []v1.NodeCondition{
+				{
+					Type:   v1.NodePIDPressure,
+					Status: v1.ConditionTrue,
+				},
+				{
+					Type:   v1.NodeReady,
+					Status: v1.ConditionTrue,
+				},
+			},
+			expectedTaints: []v1.Taint{
+				{
+					Key:    algorithm.TaintNodePIDPressure,
+					Effect: v1.TaintEffectNoSchedule,
+				},
+			},
+			pods: []podCase{
+				{
+					pod:  bestEffortPod,
+					fits: false,
+				},
+				{
+					pod:  burstablePod,
+					fits: false,
+				},
+				{
+					pod:  guaranteePod,
+					fits: false,
+				},
+				{
+					pod:         bestEffortPod,
+					tolerations: []v1.Toleration{pidPressureToleration},
+					fits:        true,
+				},
+			},
+		},
+		{
+			name: "multi taints on node",
+			nodeConditions: []v1.NodeCondition{
+				{
+					Type:   v1.NodePIDPressure,
+					Status: v1.ConditionTrue,
+				},
+				{
+					Type:   v1.NodeMemoryPressure,
+					Status: v1.ConditionTrue,
+				},
+				{
+					Type:   v1.NodeDiskPressure,
+					Status: v1.ConditionTrue,
+				},
+				{
+					Type:   v1.NodeReady,
+					Status: v1.ConditionTrue,
+				},
+			},
+			expectedTaints: []v1.Taint{
+				{
+					Key:    algorithm.TaintNodeDiskPressure,
+					Effect: v1.TaintEffectNoSchedule,
+				},
+				{
+					Key:    algorithm.TaintNodeMemoryPressure,
+					Effect: v1.TaintEffectNoSchedule,
+				},
+				{
+					Key:    algorithm.TaintNodePIDPressure,
 					Effect: v1.TaintEffectNoSchedule,
 				},
 			},
 		},
 	}
 
-	if _, err := clientset.CoreV1().Pods(nsName).Create(networkDaemonPod); err != nil {
+	for _, test := range tests {
-		t.Errorf("Case 4: Failed to create pod for network daemon: %v", err)
+		t.Run(test.name, func(t *testing.T) {
-	} else {
+			node := &v1.Node{
-		if err := waitForPodToScheduleWithTimeout(clientset, networkDaemonPod, time.Second*60); err != nil {
-			t.Errorf("Case 4: Failed to schedule network daemon pod in 60s.")
-		}
-	}
-
-	// Case 5: Taint node by unschedulable condition
-	unschedulableNode := &v1.Node{
 				ObjectMeta: metav1.ObjectMeta{
-			Name: "node-2",
+					Name: "node-1",
 				},
 				Spec: v1.NodeSpec{
-			Unschedulable: true,
+					Unschedulable: test.unschedulable,
+					Taints:        test.existingTaints,
 				},
 				Status: v1.NodeStatus{
-			Capacity: v1.ResourceList{
+					Capacity:    nodeRes,
-				v1.ResourceCPU:    resource.MustParse("4000m"),
+					Allocatable: nodeRes,
-				v1.ResourceMemory: resource.MustParse("16Gi"),
+					Conditions:  test.nodeConditions,
-				v1.ResourcePods:   resource.MustParse("110"),
-			},
-			Allocatable: v1.ResourceList{
-				v1.ResourceCPU:    resource.MustParse("4000m"),
-				v1.ResourceMemory: resource.MustParse("16Gi"),
-				v1.ResourcePods:   resource.MustParse("110"),
-			},
 				},
 			}
 
-	nodeInformerCh2 := make(chan bool)
+			if _, err := cs.CoreV1().Nodes().Create(node); err != nil {
-	nodeInformer2 := informers.Core().V1().Nodes().Informer()
+				t.Errorf("Failed to create node, err: %v", err)
-	nodeInformer2.AddEventHandler(cache.ResourceEventHandlerFuncs{
+			}
-		UpdateFunc: func(old, cur interface{}) {
+			if err := waitForNodeTaints(cs, node, test.expectedTaints); err != nil {
-			curNode := cur.(*v1.Node)
+				t.Errorf("Failed to taint node, err: %v", err)
-			if curNode.Name != "node-2" {
-				return
 			}
 
-			for _, taint := range curNode.Spec.Taints {
+			var pods []*v1.Pod
-				if taint.Key == algorithm.TaintNodeUnschedulable &&
+			for i, p := range test.pods {
-					taint.Effect == v1.TaintEffectNoSchedule {
+				pod := p.pod.DeepCopy()
-					nodeInformerCh2 <- true
+				pod.Name = fmt.Sprintf("%s-%d", pod.Name, i)
-					break
+				pod.Spec.Tolerations = p.tolerations
-				}
-			}
-		},
-	})
 
-	if _, err := clientset.CoreV1().Nodes().Create(unschedulableNode); err != nil {
+				createdPod, err := cs.CoreV1().Pods(pod.Namespace).Create(pod)
-		t.Errorf("Case 5: Failed to create node: %v", err)
+				if err != nil {
+					t.Fatalf("Failed to create pod %s/%s, error: %v",
+						pod.Namespace, pod.Name, err)
+				}
+
+				pods = append(pods, createdPod)
+
+				if p.fits {
+					if err := waitForPodToSchedule(cs, createdPod); err != nil {
+						t.Errorf("Failed to schedule pod %s/%s on the node, err: %v",
+							pod.Namespace, pod.Name, err)
+					}
 				} else {
-		select {
+					if err := waitForPodUnschedulable(cs, createdPod); err != nil {
-		case <-time.After(60 * time.Second):
+						t.Errorf("Unschedulable pod %s/%s gets scheduled on the node, err: %v",
-			t.Errorf("Case 5: Failed to taint node after 60s.")
+							pod.Namespace, pod.Name, err)
-		case <-nodeInformerCh2:
 					}
 				}
+			}
+
+			cleanupPods(cs, t, pods)
+			cleanupNodes(cs, t)
+		})
+	}
 }

test/integration/scheduler/util.go

@@ -51,6 +51,7 @@ import (
 	_ "k8s.io/kubernetes/pkg/scheduler/algorithmprovider"
 	schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
 	"k8s.io/kubernetes/pkg/scheduler/factory"
+	taintutils "k8s.io/kubernetes/pkg/util/taints"
 	"k8s.io/kubernetes/test/integration/framework"
 	imageutils "k8s.io/kubernetes/test/utils/image"
 )
@@ -363,6 +364,44 @@ func createNodes(cs clientset.Interface, prefix string, res *v1.ResourceList, nu
 	return nodes[:], nil
 }
 
+// nodeTainted return a condition function that returns true if the given node contains
+// the taints.
+func nodeTainted(cs clientset.Interface, nodeName string, taints []v1.Taint) wait.ConditionFunc {
+	return func() (bool, error) {
+		node, err := cs.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+
+		if len(taints) != len(node.Spec.Taints) {
+			return false, nil
+		}
+
+		for _, taint := range taints {
+			if !taintutils.TaintExists(node.Spec.Taints, &taint) {
+				return false, nil
+			}
+		}
+
+		return true, nil
+	}
+}
+
+// waitForNodeTaints waits for a node to have the target taints and returns
+// an error if it does not have taints within the given timeout.
+func waitForNodeTaints(cs clientset.Interface, node *v1.Node, taints []v1.Taint) error {
+	return wait.Poll(100*time.Millisecond, 30*time.Second, nodeTainted(cs, node.Name, taints))
+}
+
+// cleanupNodes deletes all nodes.
+func cleanupNodes(cs clientset.Interface, t *testing.T) {
+	err := cs.CoreV1().Nodes().DeleteCollection(
+		metav1.NewDeleteOptions(0), metav1.ListOptions{})
+	if err != nil {
+		t.Errorf("error while deleting all nodes: %v", err)
+	}
+}
+
 type pausePodConfig struct {
 	Name                              string
 	Namespace                         string