Setup e2e test for no admit

pkg/kubelet/kubelet.go

@@ -2208,6 +2208,7 @@ func (kl *Kubelet) canAdmitPod(pods []*api.Pod, pod *api.Pod) (bool, string, str
 			return false, result.Reason, result.Message
 		}
 	}
+
 	// TODO: When disk space scheduling is implemented (#11976), remove the out-of-disk check here and
 	// add the disk space predicate to predicates.GeneralPredicates.
 	if kl.isOutOfDisk() {

pkg/kubelet/lifecycle/predicate.go

@@ -101,6 +101,53 @@ func (w *predicateAdmitHandler) Admit(attrs *PodAdmitAttributes) PodAdmitResult
 			Message: message,
 		}
 	}
+
+	// Check toleration against taints
+	// NOTE(harryz) consider move PodToleratesNodeTaints to GeneralPredicates to eliminate duplicate code here
+	fit, reasons, err = predicates.PodToleratesNodeTaints(pod, nil, nodeInfo)
+	if err != nil {
+		message := fmt.Sprintf("PodToleratesNodeTaints failed due to %v, which is unexpected.", err)
+		glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), message)
+		return PodAdmitResult{
+			Admit:   fit,
+			Reason:  "UnexpectedError",
+			Message: message,
+		}
+
+	}
+	if !fit {
+		var reason string
+		var message string
+		if len(reasons) == 0 {
+			message = fmt.Sprint("PodToleratesNodeTaints failed due to unknown reason, which is unexpected.")
+			glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), message)
+			return PodAdmitResult{
+				Admit:   fit,
+				Reason:  "UnknownReason",
+				Message: message,
+			}
+		}
+		r := reasons[0]
+		switch re := r.(type) {
+		case *predicates.ErrTaintsTolerationsNotMatch:
+			// if kubelet should not care this unfit
+			if !re.SomeUntoleratedTaintIsNoAdmit {
+				return PodAdmitResult{
+					Admit: true,
+				}
+			}
+			reason = "PodToleratesNodeTaints"
+			message = re.Error()
+			glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), message)
+		}
+
+		return PodAdmitResult{
+			Admit:   fit,
+			Reason:  reason,
+			Message: message,
+		}
+	}
+
 	return PodAdmitResult{
 		Admit: true,
 	}

plugin/pkg/scheduler/algorithm/predicates/predicates.go

@@ -1093,7 +1093,7 @@ func PodToleratesNodeTaints(pod *api.Pod, meta interface{}, nodeInfo *schedulerc
 // tolerationsToleratesTaints checks if given tolerations can live with given taints.
 // It returns:
 // 1. whether tolerated or not;
-// 2. whether kubelet should be aware of (1).
+// 2. whether kubelet should be aware if it's unfit.
 func tolerationsToleratesTaints(tolerations []api.Toleration, taints []api.Taint) (bool, bool) {
 	// If the taint list is nil/empty, it is tolerated by all tolerations by default.
 	if len(taints) == 0 {

plugin/pkg/scheduler/algorithm/predicates/predicates_test.go

@@ -2886,7 +2886,7 @@ func TestPodToleratesTaints(t *testing.T) {
 			},
 			fits: false,
 			expectedFailureReasons: []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(true)},
-			test: "node should aware that a pod which can't be scheduled or start on a dedicated node assgined to user2 with effect NoScheduleNoAdmit",
+			test: "node should aware that a pod which can't be scheduled or start on a dedicated node assigned to user2 with effect NoScheduleNoAdmit",
 		},
 	}
 

test/e2e_node/noadmit_taint_test.go

@@ -0,0 +1,117 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_node
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/client/cache"
+	"k8s.io/kubernetes/pkg/runtime"
+	"k8s.io/kubernetes/pkg/util/uuid"
+	"k8s.io/kubernetes/pkg/watch"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+// NOTE(harry): this test will taint a node, which means adding other specs into this context
+// may be influenced when testing with -node=N.
+var _ = framework.KubeDescribe("[Serial] NoAdmitTaint", func() {
+	f := framework.NewDefaultFramework("admit-pod")
+	Context("when create a static pod", func() {
+		var ns, staticPodName, mirrorPodName, nodeName, taintName, taintValue string
+		var taint api.Taint
+		BeforeEach(func() {
+			nodeName = framework.TestContext.NodeName
+			ns = f.Namespace.Name
+			staticPodName = "static-pod-" + string(uuid.NewUUID())
+			// we need to check the mirror pod name (suffixed by nodeName)
+			mirrorPodName = staticPodName + "-" + nodeName
+		})
+		It("should be rejected when node is tainted with NoAdmit effect ", func() {
+			By("set NoAdmit taint for the node")
+			taintName = fmt.Sprintf("kubernetes.io/e2e-taint-key-%s", string(uuid.NewUUID()))
+			taintValue = "testing-taint-value"
+			taintEffect := api.TaintEffectNoScheduleNoAdmit
+			taint = api.Taint{
+				Key:    taintName,
+				Value:  taintValue,
+				Effect: taintEffect,
+			}
+			framework.AddOrUpdateTaintOnNode(f.Client, nodeName, taint)
+			framework.ExpectNodeHasTaint(f.Client, nodeName, taint)
+
+			By("create the static pod")
+			err := createStaticPod(framework.TestContext.ManifestPath, staticPodName, ns, "nginx", api.RestartPolicyAlways)
+			Expect(err).ShouldNot(HaveOccurred())
+
+			By("Waiting for static pod rejected event")
+			eventFound := false
+
+			_, controller := cache.NewInformer(
+				&cache.ListWatch{
+					ListFunc: func(options api.ListOptions) (runtime.Object, error) {
+						return f.Client.Events(f.Namespace.Name).List(options)
+					},
+					WatchFunc: func(options api.ListOptions) (watch.Interface, error) {
+						return f.Client.Events(f.Namespace.Name).Watch(options)
+					},
+				},
+				&api.Event{},
+				0,
+				cache.ResourceEventHandlerFuncs{
+					AddFunc: func(obj interface{}) {
+						if e, ok := obj.(*api.Event); ok {
+							if e.InvolvedObject.Kind == "Pod" && e.Reason == "PodToleratesNodeTaints" && strings.Contains(e.Message,
+								"Taint Toleration unmatched with SomeUntoleratedTaintIsNoAdmit is: true") {
+								By("PodToleratesNodeTaints event found")
+								eventFound = true
+							}
+						}
+					},
+				},
+			)
+
+			stopCh := make(chan struct{})
+			defer func() {
+				close(stopCh)
+			}()
+			go controller.Run(stopCh)
+
+			// Check if the PodToleratesNodeTaints event is found
+			for start := time.Now(); time.Since(start) < 4*time.Minute; time.Sleep(2 * time.Second) {
+				if eventFound {
+					break
+				}
+			}
+			Expect(eventFound).Should(Equal(true))
+		})
+
+		AfterEach(func() {
+			By("delete the static pod")
+			err := deleteStaticPod(framework.TestContext.ManifestPath, staticPodName, ns)
+			Expect(err).ShouldNot(HaveOccurred())
+
+			By("clear taint")
+			framework.RemoveTaintOffNode(f.Client, nodeName, taint)
+		})
+	})
+})