From c2cf5bbaf69a821d164027173d3c3565998a1b70 Mon Sep 17 00:00:00 2001 From: Harry Zhang Date: Mon, 29 Aug 2016 10:02:53 +0800 Subject: [PATCH] Setup e2e test for no admit --- pkg/kubelet/kubelet.go | 1 + pkg/kubelet/lifecycle/predicate.go | 47 +++++++ .../algorithm/predicates/predicates.go | 2 +- .../algorithm/predicates/predicates_test.go | 2 +- test/e2e_node/noadmit_taint_test.go | 117 ++++++++++++++++++ 5 files changed, 167 insertions(+), 2 deletions(-) create mode 100644 test/e2e_node/noadmit_taint_test.go diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index fba47cc19e4..7f20f0a11de 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -2208,6 +2208,7 @@ func (kl *Kubelet) canAdmitPod(pods []*api.Pod, pod *api.Pod) (bool, string, str return false, result.Reason, result.Message } } + // TODO: When disk space scheduling is implemented (#11976), remove the out-of-disk check here and // add the disk space predicate to predicates.GeneralPredicates. if kl.isOutOfDisk() { diff --git a/pkg/kubelet/lifecycle/predicate.go b/pkg/kubelet/lifecycle/predicate.go index 231ec28d303..addf9d0d85f 100644 --- a/pkg/kubelet/lifecycle/predicate.go +++ b/pkg/kubelet/lifecycle/predicate.go @@ -101,6 +101,53 @@ func (w *predicateAdmitHandler) Admit(attrs *PodAdmitAttributes) PodAdmitResult Message: message, } } + + // Check toleration against taints + // NOTE(harryz) consider move PodToleratesNodeTaints to GeneralPredicates to eliminate duplicate code here + fit, reasons, err = predicates.PodToleratesNodeTaints(pod, nil, nodeInfo) + if err != nil { + message := fmt.Sprintf("PodToleratesNodeTaints failed due to %v, which is unexpected.", err) + glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), message) + return PodAdmitResult{ + Admit: fit, + Reason: "UnexpectedError", + Message: message, + } + + } + if !fit { + var reason string + var message string + if len(reasons) == 0 { + message = fmt.Sprint("PodToleratesNodeTaints failed due to unknown reason, which is unexpected.") + glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), message) + return PodAdmitResult{ + Admit: fit, + Reason: "UnknownReason", + Message: message, + } + } + r := reasons[0] + switch re := r.(type) { + case *predicates.ErrTaintsTolerationsNotMatch: + // if kubelet should not care this unfit + if !re.SomeUntoleratedTaintIsNoAdmit { + return PodAdmitResult{ + Admit: true, + } + } + reason = "PodToleratesNodeTaints" + message = re.Error() + glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), message) + } + + return PodAdmitResult{ + Admit: fit, + Reason: reason, + Message: message, + } + } + return PodAdmitResult{ Admit: true, } diff --git a/plugin/pkg/scheduler/algorithm/predicates/predicates.go b/plugin/pkg/scheduler/algorithm/predicates/predicates.go index 18081030caa..0b8c62d75a4 100644 --- a/plugin/pkg/scheduler/algorithm/predicates/predicates.go +++ b/plugin/pkg/scheduler/algorithm/predicates/predicates.go @@ -1093,7 +1093,7 @@ func PodToleratesNodeTaints(pod *api.Pod, meta interface{}, nodeInfo *schedulerc // tolerationsToleratesTaints checks if given tolerations can live with given taints. // It returns: // 1. whether tolerated or not; -// 2. whether kubelet should be aware of (1). +// 2. whether kubelet should be aware if it's unfit. func tolerationsToleratesTaints(tolerations []api.Toleration, taints []api.Taint) (bool, bool) { // If the taint list is nil/empty, it is tolerated by all tolerations by default. if len(taints) == 0 { diff --git a/plugin/pkg/scheduler/algorithm/predicates/predicates_test.go b/plugin/pkg/scheduler/algorithm/predicates/predicates_test.go index 9b4c9b6462f..10387b453b8 100755 --- a/plugin/pkg/scheduler/algorithm/predicates/predicates_test.go +++ b/plugin/pkg/scheduler/algorithm/predicates/predicates_test.go @@ -2886,7 +2886,7 @@ func TestPodToleratesTaints(t *testing.T) { }, fits: false, expectedFailureReasons: []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(true)}, - test: "node should aware that a pod which can't be scheduled or start on a dedicated node assgined to user2 with effect NoScheduleNoAdmit", + test: "node should aware that a pod which can't be scheduled or start on a dedicated node assigned to user2 with effect NoScheduleNoAdmit", }, } diff --git a/test/e2e_node/noadmit_taint_test.go b/test/e2e_node/noadmit_taint_test.go new file mode 100644 index 00000000000..ee972e284ce --- /dev/null +++ b/test/e2e_node/noadmit_taint_test.go @@ -0,0 +1,117 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_node + +import ( + "fmt" + "strings" + "time" + + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/client/cache" + "k8s.io/kubernetes/pkg/runtime" + "k8s.io/kubernetes/pkg/util/uuid" + "k8s.io/kubernetes/pkg/watch" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +// NOTE(harry): this test will taint a node, which means adding other specs into this context +// may be influenced when testing with -node=N. +var _ = framework.KubeDescribe("[Serial] NoAdmitTaint", func() { + f := framework.NewDefaultFramework("admit-pod") + Context("when create a static pod", func() { + var ns, staticPodName, mirrorPodName, nodeName, taintName, taintValue string + var taint api.Taint + BeforeEach(func() { + nodeName = framework.TestContext.NodeName + ns = f.Namespace.Name + staticPodName = "static-pod-" + string(uuid.NewUUID()) + // we need to check the mirror pod name (suffixed by nodeName) + mirrorPodName = staticPodName + "-" + nodeName + }) + It("should be rejected when node is tainted with NoAdmit effect ", func() { + By("set NoAdmit taint for the node") + taintName = fmt.Sprintf("kubernetes.io/e2e-taint-key-%s", string(uuid.NewUUID())) + taintValue = "testing-taint-value" + taintEffect := api.TaintEffectNoScheduleNoAdmit + taint = api.Taint{ + Key: taintName, + Value: taintValue, + Effect: taintEffect, + } + framework.AddOrUpdateTaintOnNode(f.Client, nodeName, taint) + framework.ExpectNodeHasTaint(f.Client, nodeName, taint) + + By("create the static pod") + err := createStaticPod(framework.TestContext.ManifestPath, staticPodName, ns, "nginx", api.RestartPolicyAlways) + Expect(err).ShouldNot(HaveOccurred()) + + By("Waiting for static pod rejected event") + eventFound := false + + _, controller := cache.NewInformer( + &cache.ListWatch{ + ListFunc: func(options api.ListOptions) (runtime.Object, error) { + return f.Client.Events(f.Namespace.Name).List(options) + }, + WatchFunc: func(options api.ListOptions) (watch.Interface, error) { + return f.Client.Events(f.Namespace.Name).Watch(options) + }, + }, + &api.Event{}, + 0, + cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + if e, ok := obj.(*api.Event); ok { + if e.InvolvedObject.Kind == "Pod" && e.Reason == "PodToleratesNodeTaints" && strings.Contains(e.Message, + "Taint Toleration unmatched with SomeUntoleratedTaintIsNoAdmit is: true") { + By("PodToleratesNodeTaints event found") + eventFound = true + } + } + }, + }, + ) + + stopCh := make(chan struct{}) + defer func() { + close(stopCh) + }() + go controller.Run(stopCh) + + // Check if the PodToleratesNodeTaints event is found + for start := time.Now(); time.Since(start) < 4*time.Minute; time.Sleep(2 * time.Second) { + if eventFound { + break + } + } + Expect(eventFound).Should(Equal(true)) + }) + + AfterEach(func() { + By("delete the static pod") + err := deleteStaticPod(framework.TestContext.ManifestPath, staticPodName, ns) + Expect(err).ShouldNot(HaveOccurred()) + + By("clear taint") + framework.RemoveTaintOffNode(f.Client, nodeName, taint) + }) + }) +})