use regular client instead of kubectl in scheduler predicate tests when setting/cleanning taints/labels

test/e2e/framework/util.go

@@ -18,6 +18,7 @@ package framework
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -2809,6 +2810,141 @@ func WaitForAllNodesSchedulable(c *client.Client) error {
 	})
 }
 
+func AddOrUpdateLabelOnNode(c *client.Client, nodeName string, labelKey string, labelValue string) {
+	patch := fmt.Sprintf(`{"metadata":{"labels":{"%s":"%s"}}}`, labelKey, labelValue)
+	err := c.Patch(api.MergePatchType).Resource("nodes").Name(nodeName).Body([]byte(patch)).Do().Error()
+	ExpectNoError(err)
+}
+
+func ExpectNodeHasLabel(c *client.Client, nodeName string, labelKey string, labelValue string) {
+	By("verifying the node has the label " + labelKey + " " + labelValue)
+	node, err := c.Nodes().Get(nodeName)
+	ExpectNoError(err)
+	Expect(node.Labels[labelKey]).To(Equal(labelValue))
+}
+
+// RemoveLabelOffNode is for cleaning up labels temporarily added to node,
+// won't fail if target label doesn't exist or has been removed.
+func RemoveLabelOffNode(c *client.Client, nodeName string, labelKey string) {
+	By("removing the label " + labelKey + " off the node " + nodeName)
+	node, err := c.Nodes().Get(nodeName)
+	ExpectNoError(err)
+	if node.Labels == nil || len(node.Labels[labelKey]) == 0 {
+		return
+	}
+	delete(node.Labels, labelKey)
+	nodeUpdated, err := c.Nodes().Update(node)
+	ExpectNoError(err)
+
+	By("verifying the node doesn't have the label " + labelKey)
+	if nodeUpdated.Labels != nil && len(nodeUpdated.Labels[labelKey]) != 0 {
+		Failf("Failed removing label " + labelKey + " of the node " + nodeName)
+	}
+}
+
+func AddOrUpdateTaintOnNode(c *client.Client, nodeName string, taint api.Taint) {
+	node, err := c.Nodes().Get(nodeName)
+	ExpectNoError(err)
+
+	nodeTaints, err := api.GetTaintsFromNodeAnnotations(node.Annotations)
+	ExpectNoError(err)
+
+	var newTaints []api.Taint
+	updated := false
+	for _, existingTaint := range nodeTaints {
+		if existingTaint.Key == taint.Key {
+			newTaints = append(newTaints, taint)
+			updated = true
+			continue
+		}
+
+		newTaints = append(newTaints, existingTaint)
+	}
+
+	if !updated {
+		newTaints = append(newTaints, taint)
+	}
+
+	taintsData, err := json.Marshal(newTaints)
+	ExpectNoError(err)
+	node.Annotations[api.TaintsAnnotationKey] = string(taintsData)
+	_, err = c.Nodes().Update(node)
+	ExpectNoError(err)
+}
+
+func taintExists(taints []api.Taint, taintKey string) bool {
+	for _, taint := range taints {
+		if taint.Key == taintKey {
+			return true
+		}
+	}
+	return false
+}
+
+func ExpectNodeHasTaint(c *client.Client, nodeName string, taintKey string) {
+	By("verifying the node has the taint " + taintKey)
+	node, err := c.Nodes().Get(nodeName)
+	ExpectNoError(err)
+
+	nodeTaints, err := api.GetTaintsFromNodeAnnotations(node.Annotations)
+	ExpectNoError(err)
+
+	if len(nodeTaints) == 0 || !taintExists(nodeTaints, taintKey) {
+		Failf("Failed to find taint %s on node %s", taintKey, nodeName)
+	}
+}
+
+func deleteTaintByKey(taints []api.Taint, taintKey string) ([]api.Taint, error) {
+	newTaints := []api.Taint{}
+	found := false
+	for _, taint := range taints {
+		if taint.Key == taintKey {
+			found = true
+			continue
+		}
+		newTaints = append(newTaints, taint)
+	}
+
+	if !found {
+		return nil, fmt.Errorf("taint key=\"%s\" not found.", taintKey)
+	}
+	return newTaints, nil
+}
+
+// RemoveTaintOffNode is for cleaning up taints temporarily added to node,
+// won't fail if target taint doesn't exist or has been removed.
+func RemoveTaintOffNode(c *client.Client, nodeName string, taintKey string) {
+	By("removing the taint " + taintKey + " off the node " + nodeName)
+	node, err := c.Nodes().Get(nodeName)
+	ExpectNoError(err)
+
+	nodeTaints, err := api.GetTaintsFromNodeAnnotations(node.Annotations)
+	ExpectNoError(err)
+	if len(nodeTaints) == 0 {
+		return
+	}
+
+	if !taintExists(nodeTaints, taintKey) {
+		return
+	}
+
+	newTaints, err := deleteTaintByKey(nodeTaints, taintKey)
+	ExpectNoError(err)
+
+	taintsData, err := json.Marshal(newTaints)
+	ExpectNoError(err)
+	node.Annotations[api.TaintsAnnotationKey] = string(taintsData)
+	nodeUpdated, err := c.Nodes().Update(node)
+	ExpectNoError(err)
+
+	By("verifying the node doesn't have the taint " + taintKey)
+	taintsGot, err := api.GetTaintsFromNodeAnnotations(nodeUpdated.Annotations)
+	ExpectNoError(err)
+	if taintExists(taintsGot, taintKey) {
+		Failf("Failed removing taint " + taintKey + " of the node " + nodeName)
+	}
+}
+
 func ScaleRC(c *client.Client, ns, name string, size uint, wait bool) error {
 	By(fmt.Sprintf("Scaling replication controller %s in namespace %s to %d", name, ns, size))
 	scaler, err := kubectl.ScalerFor(api.Kind("ReplicationController"), c)

test/e2e/scheduler_predicates.go

@@ -19,7 +19,6 @@ package e2e
 import (
 	"fmt"
 	"path/filepath"
-	"strings"
 	"time"
 
 	"k8s.io/kubernetes/pkg/api"
@@ -100,23 +99,6 @@ func cleanupPods(c *client.Client, ns string) {
 	}
 }
 
-func removeLabelOffNode(c *client.Client, nodeName string, labelKey string) {
-	By("removing the label " + labelKey + " off the node " + nodeName)
-	node, err := c.Nodes().Get(nodeName)
-	framework.ExpectNoError(err)
-	if node.Labels == nil || len(node.Labels[labelKey]) == 0 {
-		return
-	}
-	delete(node.Labels, labelKey)
-	nodeUpdated, err := c.Nodes().Update(node)
-	framework.ExpectNoError(err)
-
-	By("verifying the node doesn't have the label " + labelKey)
-	if nodeUpdated.Labels != nil && len(nodeUpdated.Labels[labelKey]) != 0 {
-		framework.Failf("Failed removing label " + labelKey + " of the node " + nodeName)
-	}
-}
-
 // Waits until all existing pods are scheduled and returns their amount.
 func waitForStableCluster(c *client.Client) int {
 	timeout := 10 * time.Minute
@@ -488,14 +470,9 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
 		By("Trying to apply a random label on the found node.")
 		k := fmt.Sprintf("kubernetes.io/e2e-%s", string(util.NewUUID()))
 		v := "42"
-		patch := fmt.Sprintf(`{"metadata":{"labels":{"%s":"%s"}}}`, k, v)
+		framework.AddOrUpdateLabelOnNode(c, nodeName, k, v)
-		err = c.Patch(api.MergePatchType).Resource("nodes").Name(nodeName).Body([]byte(patch)).Do().Error()
+		framework.ExpectNodeHasLabel(c, nodeName, k, v)
-		framework.ExpectNoError(err)
+		defer framework.RemoveLabelOffNode(c, nodeName, k)
-
-		node, err := c.Nodes().Get(nodeName)
-		framework.ExpectNoError(err)
-		Expect(node.Labels[k]).To(Equal(v))
-		defer removeLabelOffNode(c, nodeName, k)
 
 		By("Trying to relaunch the pod, now with labels.")
 		labelPodName := "with-labels"
@@ -626,14 +603,9 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
 		By("Trying to apply a random label on the found node.")
 		k := fmt.Sprintf("kubernetes.io/e2e-%s", string(util.NewUUID()))
 		v := "42"
-		patch := fmt.Sprintf(`{"metadata":{"labels":{"%s":"%s"}}}`, k, v)
+		framework.AddOrUpdateLabelOnNode(c, nodeName, k, v)
-		err = c.Patch(api.MergePatchType).Resource("nodes").Name(nodeName).Body([]byte(patch)).Do().Error()
+		framework.ExpectNodeHasLabel(c, nodeName, k, v)
-		framework.ExpectNoError(err)
+		defer framework.RemoveLabelOffNode(c, nodeName, k)
-
-		node, err := c.Nodes().Get(nodeName)
-		framework.ExpectNoError(err)
-		Expect(node.Labels[k]).To(Equal(v))
-		defer removeLabelOffNode(c, nodeName, k)
 
 		By("Trying to relaunch the pod, now with labels.")
 		labelPodName := "with-labels"
@@ -722,14 +694,9 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
 		By("Trying to apply a label with fake az info on the found node.")
 		k := "kubernetes.io/e2e-az-name"
 		v := "e2e-az1"
-		patch := fmt.Sprintf(`{"metadata":{"labels":{"%s":"%s"}}}`, k, v)
+		framework.AddOrUpdateLabelOnNode(c, nodeName, k, v)
-		err = c.Patch(api.MergePatchType).Resource("nodes").Name(nodeName).Body([]byte(patch)).Do().Error()
+		framework.ExpectNodeHasLabel(c, nodeName, k, v)
-		framework.ExpectNoError(err)
+		defer framework.RemoveLabelOffNode(c, nodeName, k)
-
-		node, err := c.Nodes().Get(nodeName)
-		framework.ExpectNoError(err)
-		Expect(node.Labels[k]).To(Equal(v))
-		defer removeLabelOffNode(c, nodeName, k)
 
 		By("Trying to launch a pod that with NodeAffinity setting as embedded JSON string in the annotation value.")
 		labelPodName := "with-labels"
@@ -887,14 +854,9 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
 		By("Trying to apply a random label on the found node.")
 		k := "e2e.inter-pod-affinity.kubernetes.io/zone"
 		v := "china-e2etest"
-		patch := fmt.Sprintf(`{"metadata":{"labels":{"%s":"%s"}}}`, k, v)
+		framework.AddOrUpdateLabelOnNode(c, nodeName, k, v)
-		err = c.Patch(api.MergePatchType).Resource("nodes").Name(nodeName).Body([]byte(patch)).Do().Error()
+		framework.ExpectNodeHasLabel(c, nodeName, k, v)
-		framework.ExpectNoError(err)
+		defer framework.RemoveLabelOffNode(c, nodeName, k)
-
-		node, err := c.Nodes().Get(nodeName)
-		framework.ExpectNoError(err)
-		Expect(node.Labels[k]).To(Equal(v))
-		defer removeLabelOffNode(c, nodeName, k)
 
 		By("Trying to launch the pod, now with podAffinity.")
 		labelPodName := "with-podaffinity-" + string(util.NewUUID())
@@ -979,14 +941,9 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
 		By("Trying to apply a random label on the found node.")
 		k := "e2e.inter-pod-affinity.kubernetes.io/zone"
 		v := "china-e2etest"
-		patch := fmt.Sprintf(`{"metadata":{"labels":{"%s":"%s"}}}`, k, v)
+		framework.AddOrUpdateLabelOnNode(c, nodeName, k, v)
-		err = c.Patch(api.MergePatchType).Resource("nodes").Name(nodeName).Body([]byte(patch)).Do().Error()
+		framework.ExpectNodeHasLabel(c, nodeName, k, v)
-		framework.ExpectNoError(err)
+		defer framework.RemoveLabelOffNode(c, nodeName, k)
-
-		node, err := c.Nodes().Get(nodeName)
-		framework.ExpectNoError(err)
-		Expect(node.Labels[k]).To(Equal(v))
-		defer removeLabelOffNode(c, nodeName, k)
 
 		By("Trying to launch the pod, now with podAffinity with same Labels.")
 		labelPodName := "with-podaffinity-" + string(util.NewUUID())
@@ -1069,14 +1026,9 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
 		By("Trying to apply a random label on the found node.")
 		k := "e2e.inter-pod-affinity.kubernetes.io/zone"
 		v := "kubernetes-e2e"
-		patch := fmt.Sprintf(`{"metadata":{"labels":{"%s":"%s"}}}`, k, v)
+		framework.AddOrUpdateLabelOnNode(c, nodeName, k, v)
-		err = c.Patch(api.MergePatchType).Resource("nodes").Name(nodeName).Body([]byte(patch)).Do().Error()
+		framework.ExpectNodeHasLabel(c, nodeName, k, v)
-		framework.ExpectNoError(err)
+		defer framework.RemoveLabelOffNode(c, nodeName, k)
-
-		node, err := c.Nodes().Get(nodeName)
-		framework.ExpectNoError(err)
-		Expect(node.Labels[k]).To(Equal(v))
-		defer removeLabelOffNode(c, nodeName, k)
 
 		By("Trying to launch the pod, now with multiple pod affinities with diff LabelOperators.")
 		labelPodName := "with-podaffinity-" + string(util.NewUUID())
@@ -1170,14 +1122,9 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
 		By("Trying to apply a random label on the found node.")
 		k := "e2e.inter-pod-affinity.kubernetes.io/zone"
 		v := "e2e-testing"
-		patch := fmt.Sprintf(`{"metadata":{"labels":{"%s":"%s"}}}`, k, v)
+		framework.AddOrUpdateLabelOnNode(c, nodeName, k, v)
-		err = c.Patch(api.MergePatchType).Resource("nodes").Name(nodeName).Body([]byte(patch)).Do().Error()
+		framework.ExpectNodeHasLabel(c, nodeName, k, v)
-		framework.ExpectNoError(err)
+		defer framework.RemoveLabelOffNode(c, nodeName, k)
-
-		node, err := c.Nodes().Get(nodeName)
-		framework.ExpectNoError(err)
-		Expect(node.Labels[k]).To(Equal(v))
-		defer removeLabelOffNode(c, nodeName, k)
 
 		By("Trying to launch the pod, now with Pod affinity and anti affinity.")
 		labelPodName := "with-podantiaffinity-" + string(util.NewUUID())
@@ -1274,14 +1221,9 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
 		By("Trying to apply a label with fake az info on the found node.")
 		k := "e2e.inter-pod-affinity.kubernetes.io/zone"
 		v := "e2e-az1"
-		patch := fmt.Sprintf(`{"metadata":{"labels":{"%s":"%s"}}}`, k, v)
+		framework.AddOrUpdateLabelOnNode(c, nodeName, k, v)
-		err = c.Patch(api.MergePatchType).Resource("nodes").Name(nodeName).Body([]byte(patch)).Do().Error()
+		framework.ExpectNodeHasLabel(c, nodeName, k, v)
-		framework.ExpectNoError(err)
+		defer framework.RemoveLabelOffNode(c, nodeName, k)
-
-		node, err := c.Nodes().Get(nodeName)
-		framework.ExpectNoError(err)
-		Expect(node.Labels[k]).To(Equal(v))
-		defer removeLabelOffNode(c, nodeName, k)
 
 		By("Trying to launch a pod that with PodAffinity & PodAntiAffinity setting as embedded JSON string in the annotation value.")
 		labelPodName := "with-newlabels"
@@ -1340,39 +1282,17 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
 		By("Trying to apply a random taint on the found node.")
 		taintName := fmt.Sprintf("kubernetes.io/e2e-taint-key-%s", string(util.NewUUID()))
 		taintValue := "testing-taint-value"
-		taintEffect := string(api.TaintEffectNoSchedule)
+		taintEffect := api.TaintEffectNoSchedule
-		framework.RunKubectlOrDie("taint", "nodes", nodeName, taintName+"="+taintValue+":"+taintEffect)
+		framework.AddOrUpdateTaintOnNode(c, nodeName, api.Taint{Key: taintName, Value: taintValue, Effect: taintEffect})
-		defer func() {
+		framework.ExpectNodeHasTaint(c, nodeName, taintName)
-			By("removing the taint " + taintName + " off the node " + nodeName)
+		defer framework.RemoveTaintOffNode(c, nodeName, taintName)
-			framework.RunKubectlOrDie("taint", "nodes", nodeName, taintName+"-")
-			By("verifying the node doesn't have the taint " + taintName)
-			output := framework.RunKubectlOrDie("describe", "node", nodeName)
-			if strings.Contains(output, taintName) {
-				framework.Failf("Failed removing taint " + taintName + " of the node " + nodeName)
-			}
-		}()
-		By("verifying the node has the taint " + taintName + " with the value " + taintValue)
-		output := framework.RunKubectlOrDie("describe", "node", nodeName)
-		requiredStrings := [][]string{
-			{"Name:", nodeName},
-			{"Taints:"},
-			{taintName, taintValue, taintEffect},
-		}
-		checkOutput(output, requiredStrings)
 
 		By("Trying to apply a random label on the found node.")
 		labelKey := fmt.Sprintf("kubernetes.io/e2e-label-key-%s", string(util.NewUUID()))
 		labelValue := "testing-label-value"
-		framework.RunKubectlOrDie("label", "nodes", nodeName, labelKey+"="+labelValue)
+		framework.AddOrUpdateLabelOnNode(c, nodeName, labelKey, labelValue)
-		By("verifying the node has the label " + labelKey + " with the value " + labelValue)
+		framework.ExpectNodeHasLabel(c, nodeName, labelKey, labelValue)
-		labelOutput := framework.RunKubectlOrDie("describe", "node", nodeName)
+		defer framework.RemoveLabelOffNode(c, nodeName, labelKey)
-		labelOutputRequiredStrings := [][]string{
-			{"Name:", nodeName},
-			{"Labels:"},
-			{labelKey + "=" + labelValue},
-		}
-		checkOutput(labelOutput, labelOutputRequiredStrings)
-		defer removeLabelOffNode(c, nodeName, labelKey)
 
 		By("Trying to relaunch the pod, now with tolerations.")
 		tolerationPodName := "with-tolerations"
@@ -1388,7 +1308,7 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
 							{
 								"key": "` + taintName + `",
 								"value": "` + taintValue + `",
-								"effect": "` + taintEffect + `"
+								"effect": "` + string(taintEffect) + `"
 							}
 						]`,
 				},
@@ -1456,39 +1376,17 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
 		By("Trying to apply a random taint on the found node.")
 		taintName := fmt.Sprintf("kubernetes.io/e2e-taint-key-%s", string(util.NewUUID()))
 		taintValue := "testing-taint-value"
-		taintEffect := string(api.TaintEffectNoSchedule)
+		taintEffect := api.TaintEffectNoSchedule
-		framework.RunKubectlOrDie("taint", "nodes", nodeName, taintName+"="+taintValue+":"+taintEffect)
+		framework.AddOrUpdateTaintOnNode(c, nodeName, api.Taint{Key: taintName, Value: taintValue, Effect: taintEffect})
-		defer func() {
+		framework.ExpectNodeHasTaint(c, nodeName, taintName)
-			By("removing the taint " + taintName + " off the node " + nodeName)
+		defer framework.RemoveTaintOffNode(c, nodeName, taintName)
-			framework.RunKubectlOrDie("taint", "nodes", nodeName, taintName+"-")
-			By("verifying the node doesn't have the taint " + taintName)
-			output := framework.RunKubectlOrDie("describe", "node", nodeName)
-			if strings.Contains(output, taintName) {
-				framework.Failf("Failed removing taint " + taintName + " of the node " + nodeName)
-			}
-		}()
-		By("verifying the node has the taint " + taintName + " with the value " + taintValue)
-		output := framework.RunKubectlOrDie("describe", "node", nodeName)
-		requiredStrings := [][]string{
-			{"Name:", nodeName},
-			{"Taints:"},
-			{taintName, taintValue, taintEffect},
-		}
-		checkOutput(output, requiredStrings)
 
 		By("Trying to apply a random label on the found node.")
 		labelKey := fmt.Sprintf("kubernetes.io/e2e-label-key-%s", string(util.NewUUID()))
 		labelValue := "testing-label-value"
-		framework.RunKubectlOrDie("label", "nodes", nodeName, labelKey+"="+labelValue)
+		framework.AddOrUpdateLabelOnNode(c, nodeName, labelKey, labelValue)
-		By("verifying the node has the label " + labelKey + " with the value " + labelValue)
+		framework.ExpectNodeHasLabel(c, nodeName, labelKey, labelValue)
-		labelOutput := framework.RunKubectlOrDie("describe", "node", nodeName)
+		defer framework.RemoveLabelOffNode(c, nodeName, labelKey)
-		labelOutputRequiredStrings := [][]string{
-			{"Name:", nodeName},
-			{"Labels:"},
-			{labelKey + "=" + labelValue},
-		}
-		checkOutput(labelOutput, labelOutputRequiredStrings)
-		defer removeLabelOffNode(c, nodeName, labelKey)
 
 		By("Trying to relaunch the pod, still no tolerations.")
 		podNameNoTolerations := "still-no-tolerations"