From caddb3ffd3f7bebe1676a328dd07e5c6fceb49b6 Mon Sep 17 00:00:00 2001 From: tanjunchen Date: Thu, 30 Apr 2020 23:08:33 +0800 Subject: [PATCH] test/e2e/framework:remove the direct dependency for k8s.io/kubernetes/pkg --- test/e2e/framework/node/BUILD | 4 +- test/e2e/framework/node/resource.go | 104 +++++++++++++++++++++++++++- 2 files changed, 105 insertions(+), 3 deletions(-) diff --git a/test/e2e/framework/node/BUILD b/test/e2e/framework/node/BUILD index 205809288f3..634d0eb3c4f 100644 --- a/test/e2e/framework/node/BUILD +++ b/test/e2e/framework/node/BUILD @@ -10,10 +10,12 @@ go_library( importpath = "k8s.io/kubernetes/test/e2e/framework/node", visibility = ["//visibility:public"], deps = [ - "//pkg/controller:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/conversion:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", diff --git a/test/e2e/framework/node/resource.go b/test/e2e/framework/node/resource.go index b06f71379c3..2829f14b09c 100644 --- a/test/e2e/framework/node/resource.go +++ b/test/e2e/framework/node/resource.go @@ -28,7 +28,11 @@ import ( "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/conversion" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/rand" "k8s.io/apimachinery/pkg/util/sets" @@ -39,7 +43,6 @@ import ( e2elog "k8s.io/kubernetes/test/e2e/framework/log" // TODO remove the direct dependency for internal k8s.io/kubernetes - "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/test/e2e/system" ) @@ -575,10 +578,107 @@ func RemoveTaintOffNode(c clientset.Interface, nodeName string, taint v1.Taint) func AddOrUpdateTaintOnNode(c clientset.Interface, nodeName string, taint v1.Taint) { // TODO use wrapper methods in expect.go after removing the dependency on this // package from the core e2e framework. - err := controller.AddOrUpdateTaintOnNode(c, nodeName, &taint) + err := addOrUpdateTaintOnNode(c, nodeName, &taint) gomega.ExpectWithOffset(2, err).NotTo(gomega.HaveOccurred()) } +// addOrUpdateTaintOnNode add taints to the node. If taint was added into node, it'll issue API calls +// to update nodes; otherwise, no API calls. Return error if any. +// copied from pkg/controller/controller_utils.go AddOrUpdateTaintOnNode() +func addOrUpdateTaintOnNode(c clientset.Interface, nodeName string, taints ...*v1.Taint) error { + if len(taints) == 0 { + return nil + } + firstTry := true + return clientretry.RetryOnConflict(updateTaintBackOff, func() error { + var err error + var oldNode *v1.Node + // First we try getting node from the API server cache, as it's cheaper. If it fails + // we get it from etcd to be sure to have fresh data. + if firstTry { + oldNode, err = c.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{ResourceVersion: "0"}) + firstTry = false + } else { + oldNode, err = c.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) + } + if err != nil { + return err + } + + var newNode *v1.Node + oldNodeCopy := oldNode + updated := false + for _, taint := range taints { + curNewNode, ok, err := addOrUpdateTaint(oldNodeCopy, taint) + if err != nil { + return fmt.Errorf("failed to update taint of node") + } + updated = updated || ok + newNode = curNewNode + oldNodeCopy = curNewNode + } + if !updated { + return nil + } + return patchNodeTaints(c, nodeName, oldNode, newNode) + }) +} + +// addOrUpdateTaint tries to add a taint to annotations list. Returns a new copy of updated Node and true if something was updated +// false otherwise. +// copied from pkg/util/taints/taints.go AddOrUpdateTaint() +func addOrUpdateTaint(node *v1.Node, taint *v1.Taint) (*v1.Node, bool, error) { + newNode := node.DeepCopy() + nodeTaints := newNode.Spec.Taints + + var newTaints []v1.Taint + updated := false + for i := range nodeTaints { + if taint.MatchTaint(&nodeTaints[i]) { + if semantic.DeepEqual(*taint, nodeTaints[i]) { + return newNode, false, nil + } + newTaints = append(newTaints, *taint) + updated = true + continue + } + + newTaints = append(newTaints, nodeTaints[i]) + } + + if !updated { + newTaints = append(newTaints, *taint) + } + + newNode.Spec.Taints = newTaints + return newNode, true, nil +} + +// semantic can do semantic deep equality checks for core objects. +// Example: apiequality.Semantic.DeepEqual(aPod, aPodWithNonNilButEmptyMaps) == true +// copied from pkg/apis/core/helper/helpers.go Semantic +var semantic = conversion.EqualitiesOrDie( + func(a, b resource.Quantity) bool { + // Ignore formatting, only care that numeric value stayed the same. + // TODO: if we decide it's important, it should be safe to start comparing the format. + // + // Uninitialized quantities are equivalent to 0 quantities. + return a.Cmp(b) == 0 + }, + func(a, b metav1.MicroTime) bool { + return a.UTC() == b.UTC() + }, + func(a, b metav1.Time) bool { + return a.UTC() == b.UTC() + }, + func(a, b labels.Selector) bool { + return a.String() == b.String() + }, + func(a, b fields.Selector) bool { + return a.String() == b.String() + }, +) + // removeNodeTaint is for cleaning up taints temporarily added to node, // won't fail if target taint doesn't exist or has been removed. // If passed a node it'll check if there's anything to be done, if taint is not present it won't issue