Merge pull request #26501 from resouer/scheduler

Automatic merge from submit-queue

Add kubelet awareness to taint tolerant match caculator.

Add kubelet awareness to taint tolerant match caculator.

Ref: #25320

This is required by `TaintEffectNoScheduleNoAdmit` & `TaintEffectNoScheduleNoAdmitNoExecute `, so that node will know if it should expect the taint&tolerant

pkg/api/types.go

@@ -1464,7 +1464,7 @@ type Taint struct {
 	Value string `json:"value,omitempty"`
 	// Required. The effect of the taint on pods
 	// that do not tolerate the taint.
-	// Valid effects are NoSchedule and PreferNoSchedule.
+	// Valid effects are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.
 	Effect TaintEffect `json:"effect"`
 }
 
@@ -1480,12 +1480,11 @@ const (
 	// new pods onto the node, rather than prohibiting new pods from scheduling
 	// onto the node entirely. Enforced by the scheduler.
 	TaintEffectPreferNoSchedule TaintEffect = "PreferNoSchedule"
-	// NOT YET IMPLEMENTED. TODO: Uncomment field once it is implemented.
 	// Do not allow new pods to schedule onto the node unless they tolerate the taint,
 	// do not allow pods to start on Kubelet unless they tolerate the taint,
 	// but allow all already-running pods to continue running.
 	// Enforced by the scheduler and Kubelet.
-	// TaintEffectNoScheduleNoAdmit TaintEffect = "NoScheduleNoAdmit"
+	TaintEffectNoScheduleNoAdmit TaintEffect = "NoScheduleNoAdmit"
 	// NOT YET IMPLEMENTED. TODO: Uncomment field once it is implemented.
 	// Do not allow new pods to schedule onto the node unless they tolerate the taint,
 	// do not allow pods to start on Kubelet unless they tolerate the taint,
@@ -1508,7 +1507,7 @@ type Toleration struct {
 	// If the operator is Exists, the value should be empty, otherwise just a regular string.
 	Value string `json:"value,omitempty"`
 	// Effect indicates the taint effect to match. Empty means match all taint effects.
-	// When specified, allowed values are NoSchedule and PreferNoSchedule.
+	// When specified, allowed values are NoSchedule,NoScheduleNoAdmit and PreferNoSchedule.
 	Effect TaintEffect `json:"effect,omitempty"`
 	// TODO: For forgiveness (#1574), we'd eventually add at least a grace period
 	// here, and possibly an occurrence threshold and period.

pkg/api/v1/generated.proto

@@ -2933,7 +2933,7 @@ message Taint {
 
   // Required. The effect of the taint on pods
   // that do not tolerate the taint.
-  // Valid effects are NoSchedule and PreferNoSchedule.
+  // Valid effects are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.
   optional string effect = 3;
 }
 
@@ -2954,7 +2954,7 @@ message Toleration {
   optional string value = 3;
 
   // Effect indicates the taint effect to match. Empty means match all taint effects.
-  // When specified, allowed values are NoSchedule and PreferNoSchedule.
+  // When specified, allowed values are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.
   optional string effect = 4;
 }
 

pkg/api/v1/types.go

@@ -1683,7 +1683,7 @@ type Taint struct {
 	Value string `json:"value,omitempty" protobuf:"bytes,2,opt,name=value"`
 	// Required. The effect of the taint on pods
 	// that do not tolerate the taint.
-	// Valid effects are NoSchedule and PreferNoSchedule.
+	// Valid effects are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.
 	Effect TaintEffect `json:"effect" protobuf:"bytes,3,opt,name=effect,casttype=TaintEffect"`
 }
 
@@ -1699,12 +1699,11 @@ const (
 	// new pods onto the node, rather than prohibiting new pods from scheduling
 	// onto the node entirely. Enforced by the scheduler.
 	TaintEffectPreferNoSchedule TaintEffect = "PreferNoSchedule"
-	// NOT YET IMPLEMENTED. TODO: Uncomment field once it is implemented.
 	// Do not allow new pods to schedule onto the node unless they tolerate the taint,
 	// do not allow pods to start on Kubelet unless they tolerate the taint,
 	// but allow all already-running pods to continue running.
 	// Enforced by the scheduler and Kubelet.
-	// TaintEffectNoScheduleNoAdmit TaintEffect = "NoScheduleNoAdmit"
+	TaintEffectNoScheduleNoAdmit TaintEffect = "NoScheduleNoAdmit"
 	// NOT YET IMPLEMENTED. TODO: Uncomment field once it is implemented.
 	// Do not allow new pods to schedule onto the node unless they tolerate the taint,
 	// do not allow pods to start on Kubelet unless they tolerate the taint,
@@ -1727,7 +1726,7 @@ type Toleration struct {
 	// If the operator is Exists, the value should be empty, otherwise just a regular string.
 	Value string `json:"value,omitempty" protobuf:"bytes,3,opt,name=value"`
 	// Effect indicates the taint effect to match. Empty means match all taint effects.
-	// When specified, allowed values are NoSchedule and PreferNoSchedule.
+	// When specified, allowed values are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.
 	Effect TaintEffect `json:"effect,omitempty" protobuf:"bytes,4,opt,name=effect,casttype=TaintEffect"`
 	// TODO: For forgiveness (#1574), we'd eventually add at least a grace period
 	// here, and possibly an occurrence threshold and period.

pkg/api/v1/types_swagger_doc_generated.go

@@ -1721,7 +1721,7 @@ var map_Taint = map[string]string{
 	"":       "The node this Taint is attached to has the effect \"effect\" on any pod that that does not tolerate the Taint.",
 	"key":    "Required. The taint key to be applied to a node.",
 	"value":  "Required. The taint value corresponding to the taint key.",
-	"effect": "Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule and PreferNoSchedule.",
+	"effect": "Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.",
 }
 
 func (Taint) SwaggerDoc() map[string]string {
@@ -1733,7 +1733,7 @@ var map_Toleration = map[string]string{
 	"key":      "Required. Key is the taint key that the toleration applies to.",
 	"operator": "operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.",
 	"value":    "Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.",
-	"effect":   "Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule and PreferNoSchedule.",
+	"effect":   "Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.",
 }
 
 func (Toleration) SwaggerDoc() map[string]string {

pkg/api/validation/validation.go

@@ -1780,14 +1780,14 @@ func validateTaintEffect(effect *api.TaintEffect, allowEmpty bool, fldPath *fiel
 	allErrors := field.ErrorList{}
 	switch *effect {
 	// TODO: Replace next line with subsequent commented-out line when implement TaintEffectNoScheduleNoAdmit, TaintEffectNoScheduleNoAdmitNoExecute.
-	case api.TaintEffectNoSchedule, api.TaintEffectPreferNoSchedule:
-		// case api.TaintEffectNoSchedule, api.TaintEffectPreferNoSchedule, api.TaintEffectNoScheduleNoAdmit, api.TaintEffectNoScheduleNoAdmitNoExecute:
+	case api.TaintEffectNoSchedule, api.TaintEffectPreferNoSchedule, api.TaintEffectNoScheduleNoAdmit:
+		// case api.TaintEffectNoSchedule, api.TaintEffectPreferNoSchedule, api.TaintEffectNoScheduleNoAdmitNoExecute:
 	default:
 		validValues := []string{
 			string(api.TaintEffectNoSchedule),
 			string(api.TaintEffectPreferNoSchedule),
+			string(api.TaintEffectNoScheduleNoAdmit),
 			// TODO: Uncomment this block when implement TaintEffectNoScheduleNoAdmit, TaintEffectNoScheduleNoAdmitNoExecute.
-			// string(api.TaintEffectNoScheduleNoAdmit),
 			// string(api.TaintEffectNoScheduleNoAdmitNoExecute),
 		}
 		allErrors = append(allErrors, field.NotSupported(fldPath, effect, validValues))

pkg/generated/openapi/zz_generated.openapi.go

@@ -12502,7 +12502,7 @@ var OpenAPIDefinitions *common.OpenAPIDefinitions = &common.OpenAPIDefinitions{
 					},
 					"effect": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule and PreferNoSchedule.",
+							Description: "Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.",
 							Type:        []string{"string"},
 							Format:      "",
 						},
@@ -12600,7 +12600,7 @@ var OpenAPIDefinitions *common.OpenAPIDefinitions = &common.OpenAPIDefinitions{
 					},
 					"effect": {
 						SchemaProps: spec.SchemaProps{
-							Description: "Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule and PreferNoSchedule.",
+							Description: "Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.",
 							Type:        []string{"string"},
 							Format:      "",
 						},

pkg/kubectl/cmd/taint.go

@@ -181,7 +181,7 @@ func parseTaints(spec []string) ([]api.Taint, []api.Taint, error) {
 				return nil, nil, fmt.Errorf("invalid taint spec: %v, %s", taintSpec, strings.Join(errs, "; "))
 			}
 
-			if parts2[1] != string(api.TaintEffectNoSchedule) && parts2[1] != string(api.TaintEffectPreferNoSchedule) {
+			if parts2[1] != string(api.TaintEffectNoSchedule) && parts2[1] != string(api.TaintEffectPreferNoSchedule) && parts2[1] != string(api.TaintEffectNoScheduleNoAdmit) {
 				return nil, nil, fmt.Errorf("invalid taint spec: %v, unsupported taint effect", taintSpec)
 			}
 

pkg/kubelet/kubelet.go

@@ -1515,6 +1515,7 @@ func (kl *Kubelet) canAdmitPod(pods []*api.Pod, pod *api.Pod) (bool, string, str
 			return false, result.Reason, result.Message
 		}
 	}
+
 	// TODO: When disk space scheduling is implemented (#11976), remove the out-of-disk check here and
 	// add the disk space predicate to predicates.GeneralPredicates.
 	if kl.isOutOfDisk() {

pkg/kubelet/lifecycle/predicate.go

@@ -101,6 +101,53 @@ func (w *predicateAdmitHandler) Admit(attrs *PodAdmitAttributes) PodAdmitResult
 			Message: message,
 		}
 	}
+
+	// Check toleration against taints
+	// NOTE(harryz) consider move PodToleratesNodeTaints to GeneralPredicates to eliminate duplicate code here
+	fit, reasons, err = predicates.PodToleratesNodeTaints(pod, nil, nodeInfo)
+	if err != nil {
+		message := fmt.Sprintf("PodToleratesNodeTaints failed due to %v, which is unexpected.", err)
+		glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), message)
+		return PodAdmitResult{
+			Admit:   fit,
+			Reason:  "UnexpectedError",
+			Message: message,
+		}
+
+	}
+	if !fit {
+		var reason string
+		var message string
+		if len(reasons) == 0 {
+			message = fmt.Sprint("PodToleratesNodeTaints failed due to unknown reason, which is unexpected.")
+			glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), message)
+			return PodAdmitResult{
+				Admit:   fit,
+				Reason:  "UnknownReason",
+				Message: message,
+			}
+		}
+		r := reasons[0]
+		switch re := r.(type) {
+		case *predicates.ErrTaintsTolerationsNotMatch:
+			// if kubelet should not care this unfit
+			if !re.SomeUntoleratedTaintIsNoAdmit {
+				return PodAdmitResult{
+					Admit: true,
+				}
+			}
+			reason = "PodToleratesNodeTaints"
+			message = re.Error()
+			glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), message)
+		}
+
+		return PodAdmitResult{
+			Admit:   fit,
+			Reason:  reason,
+			Message: message,
+		}
+	}
+
 	return PodAdmitResult{
 		Admit: true,
 	}

plugin/pkg/scheduler/algorithm/predicates/error.go

@@ -29,7 +29,6 @@ var (
 	ErrVolumeZoneConflict        = newPredicateFailureError("NoVolumeZoneConflict")
 	ErrNodeSelectorNotMatch      = newPredicateFailureError("MatchNodeSelector")
 	ErrPodAffinityNotMatch       = newPredicateFailureError("MatchInterPodAffinity")
-	ErrTaintsTolerationsNotMatch = newPredicateFailureError("PodToleratesNodeTaints")
 	ErrPodNotMatchHostName       = newPredicateFailureError("HostName")
 	ErrPodNotFitsHostPorts       = newPredicateFailureError("PodFitsHostPorts")
 	ErrNodeLabelPresenceViolated = newPredicateFailureError("CheckNodeLabelPresence")
@@ -43,6 +42,25 @@ var (
 	ErrFakePredicate = newPredicateFailureError("FakePredicateError")
 )
 
+// ErrTaintsTolerationsNotMatch is an error type that indicates with if it should be aware by kubelet.
+type ErrTaintsTolerationsNotMatch struct {
+	SomeUntoleratedTaintIsNoAdmit bool
+}
+
+func newErrTaintsTolerationsNotMatch(someUntoleratedTaintIsNoAdmit bool) *ErrTaintsTolerationsNotMatch {
+	return &ErrTaintsTolerationsNotMatch{
+		SomeUntoleratedTaintIsNoAdmit: someUntoleratedTaintIsNoAdmit,
+	}
+}
+
+func (e *ErrTaintsTolerationsNotMatch) Error() string {
+	return fmt.Sprintf("Taint Toleration unmatched with SomeUntoleratedTaintIsNoAdmit is: %v", e.SomeUntoleratedTaintIsNoAdmit)
+}
+
+func (e *ErrTaintsTolerationsNotMatch) GetReason() string {
+	return fmt.Sprintf("ErrTaintsTolerationsNotMatch, and SomeUntoleratedTaintIsNoAdmit is: %v", e.SomeUntoleratedTaintIsNoAdmit)
+}
+
 // InsufficientResourceError is an error type that indicates what kind of resource limit is
 // hit and caused the unfitting failure.
 type InsufficientResourceError struct {

plugin/pkg/scheduler/algorithm/predicates/predicates.go

@@ -1083,23 +1083,37 @@ func PodToleratesNodeTaints(pod *api.Pod, meta interface{}, nodeInfo *schedulerc
 		return false, nil, err
 	}
 
-	if tolerationsToleratesTaints(tolerations, taints) {
+	if tolerated, someUntoleratedTaintIsNoAdmit := tolerationsToleratesTaints(tolerations, taints); tolerated {
 		return true, nil, nil
+	} else {
+		return false, []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(someUntoleratedTaintIsNoAdmit)}, nil
 	}
-	return false, []algorithm.PredicateFailureReason{ErrTaintsTolerationsNotMatch}, nil
 }
 
-func tolerationsToleratesTaints(tolerations []api.Toleration, taints []api.Taint) bool {
+// tolerationsToleratesTaints checks if given tolerations can live with given taints.
+// It returns:
+// 1. whether tolerated or not;
+// 2. whether kubelet should be aware if it's unfit.
+func tolerationsToleratesTaints(tolerations []api.Toleration, taints []api.Taint) (bool, bool) {
 	// If the taint list is nil/empty, it is tolerated by all tolerations by default.
 	if len(taints) == 0 {
-		return true
+		return true, false
 	}
 
 	// The taint list isn't nil/empty, a nil/empty toleration list can't tolerate them.
 	if len(tolerations) == 0 {
-		return false
+		// if there's taint has TaintEffectNoScheduleNoAdmit, kubelet should also be aware of this.
+		for _, taint := range taints {
+			if taint.Effect == api.TaintEffectNoScheduleNoAdmit {
+				return false, true
+			}
+		}
+		return false, false
 	}
 
+	someUntoleratedTaintIsNoAdmit := false
+	fits := true
+
 	for i := range taints {
 		taint := &taints[i]
 		// skip taints that have effect PreferNoSchedule, since it is for priorities
@@ -1107,12 +1121,16 @@ func tolerationsToleratesTaints(tolerations []api.Toleration, taints []api.Taint
 			continue
 		}
 
-		if !api.TaintToleratedByTolerations(taint, tolerations) {
-			return false
+		if tolerated := api.TaintToleratedByTolerations(taint, tolerations); !tolerated {
+			fits = false
+			if taint.Effect == api.TaintEffectNoScheduleNoAdmit {
+				someUntoleratedTaintIsNoAdmit = true
+				return fits, someUntoleratedTaintIsNoAdmit
+			}
 		}
 	}
 
-	return true
+	return fits, someUntoleratedTaintIsNoAdmit
 }
 
 // Determine if a pod is scheduled with best-effort QoS

plugin/pkg/scheduler/algorithm/predicates/predicates_test.go

@@ -2563,10 +2563,11 @@ func TestInterPodAffinityWithMultipleNodes(t *testing.T) {
 
 func TestPodToleratesTaints(t *testing.T) {
 	podTolerateTaintsTests := []struct {
-		pod  *api.Pod
-		node api.Node
-		fits bool
-		test string
+		pod                    *api.Pod
+		node                   api.Node
+		fits                   bool
+		expectedFailureReasons []algorithm.PredicateFailureReason
+		test                   string
 	}{
 		{
 			pod: &api.Pod{
@@ -2587,6 +2588,7 @@ func TestPodToleratesTaints(t *testing.T) {
 				},
 			},
 			fits: false,
+			expectedFailureReasons: []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(false)},
 			test: "a pod having no tolerations can't be scheduled onto a node with nonempty taints",
 		},
 		{
@@ -2652,6 +2654,7 @@ func TestPodToleratesTaints(t *testing.T) {
 				},
 			},
 			fits: false,
+			expectedFailureReasons: []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(false)},
 			test: "a pod which can't be scheduled on a dedicated node assigned to user2 with effect NoSchedule",
 		},
 		{
@@ -2758,6 +2761,7 @@ func TestPodToleratesTaints(t *testing.T) {
 				},
 			},
 			fits: false,
+			expectedFailureReasons: []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(false)},
 			test: "a pod has a toleration that keys and values match the taint on the node, but (non-empty) effect doesn't match, " +
 				"can't be scheduled onto the node",
 		},
@@ -2791,7 +2795,7 @@ func TestPodToleratesTaints(t *testing.T) {
 				},
 			},
 			fits: true,
-			test: "The pod has a toleration that keys and values match the taint on the node, the effect of toleration is empty, " +
+			test: "the pod has a toleration that keys and values match the taint on the node, the effect of toleration is empty, " +
 				"and the effect of taint is NoSchedule. Pod can be scheduled onto the node",
 		},
 		{
@@ -2828,8 +2832,63 @@ func TestPodToleratesTaints(t *testing.T) {
 			test: "The pod has a toleration that key and value don't match the taint on the node, " +
 				"but the effect of taint on node is PreferNochedule. Pod can be scheduled onto the node",
 		},
+		{
+			pod: &api.Pod{
+				ObjectMeta: api.ObjectMeta{
+					Name: "podadmit1",
+				},
+			},
+			node: api.Node{
+				ObjectMeta: api.ObjectMeta{
+					Annotations: map[string]string{
+						api.TaintsAnnotationKey: `
+						[{
+							"key": "dedicated",
+							"value": "user1",
+							"effect": "NoScheduleNoAdmit"
+						}]`,
+					},
+				},
+			},
+			fits: false,
+			expectedFailureReasons: []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(true)},
+			test: "node should aware that that a pod having no tolerations can't be scheduled or started on a node with nonempty taints",
+		},
+		{
+			pod: &api.Pod{
+				ObjectMeta: api.ObjectMeta{
+					Name: "podadmit2",
+					Annotations: map[string]string{
+						api.TolerationsAnnotationKey: `
+						[{
+							"key": "dedicated",
+							"operator": "Equal",
+							"value": "user2",
+							"effect": "NoScheduleNoAdmit"
+						}]`,
+					},
+				},
+				Spec: api.PodSpec{
+					Containers: []api.Container{{Image: "pod2:V1"}},
+				},
+			},
+			node: api.Node{
+				ObjectMeta: api.ObjectMeta{
+					Annotations: map[string]string{
+						api.TaintsAnnotationKey: `
+						[{
+							"key": "dedicated",
+							"value": "user1",
+							"effect": "NoScheduleNoAdmit"
+						}]`,
+					},
+				},
+			},
+			fits: false,
+			expectedFailureReasons: []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(true)},
+			test: "node should aware that a pod which can't be scheduled or start on a dedicated node assigned to user2 with effect NoScheduleNoAdmit",
+		},
 	}
-	expectedFailureReasons := []algorithm.PredicateFailureReason{ErrTaintsTolerationsNotMatch}
 
 	for _, test := range podTolerateTaintsTests {
 		nodeInfo := schedulercache.NewNodeInfo()
@@ -2838,11 +2897,11 @@ func TestPodToleratesTaints(t *testing.T) {
 		if err != nil {
 			t.Errorf("%s, unexpected error: %v", test.test, err)
 		}
-		if !fits && !reflect.DeepEqual(reasons, expectedFailureReasons) {
-			t.Errorf("%s, unexpected failure reason: %v, want: %v", test.test, reasons, expectedFailureReasons)
+		if !fits && !reflect.DeepEqual(reasons, test.expectedFailureReasons) {
+			t.Errorf("%s, unexpected failure reason: %v, want: %v", test.test, reasons, test.expectedFailureReasons)
 		}
 		if fits != test.fits {
-			t.Errorf("%s, expected: %v got %v", test.test, test.fits, fits)
+			t.Errorf("%s,\n expected: %v got %v", test.test, test.fits, fits)
 		}
 	}
 }

plugin/pkg/scheduler/algorithm/priorities/taint_toleration.go

@@ -34,7 +34,7 @@ func countIntolerableTaintsPreferNoSchedule(taints []api.Taint, tolerations []ap
 			continue
 		}
 
-		if !api.TaintToleratedByTolerations(taint, tolerations) {
+		if tolerable := api.TaintToleratedByTolerations(taint, tolerations); !tolerable {
 			intolerableTaints++
 		}
 	}

test/e2e_node/noadmit_taint_test.go

@@ -0,0 +1,117 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_node
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/client/cache"
+	"k8s.io/kubernetes/pkg/runtime"
+	"k8s.io/kubernetes/pkg/util/uuid"
+	"k8s.io/kubernetes/pkg/watch"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+// NOTE(harry): this test will taint a node, which means adding other specs into this context
+// may be influenced when testing with -node=N.
+var _ = framework.KubeDescribe("[Serial] NoAdmitTaint", func() {
+	f := framework.NewDefaultFramework("admit-pod")
+	Context("when create a static pod", func() {
+		var ns, staticPodName, mirrorPodName, nodeName, taintName, taintValue string
+		var taint api.Taint
+		BeforeEach(func() {
+			nodeName = framework.TestContext.NodeName
+			ns = f.Namespace.Name
+			staticPodName = "static-pod-" + string(uuid.NewUUID())
+			// we need to check the mirror pod name (suffixed by nodeName)
+			mirrorPodName = staticPodName + "-" + nodeName
+		})
+		It("should be rejected when node is tainted with NoAdmit effect ", func() {
+			By("set NoAdmit taint for the node")
+			taintName = fmt.Sprintf("kubernetes.io/e2e-taint-key-%s", string(uuid.NewUUID()))
+			taintValue = "testing-taint-value"
+			taintEffect := api.TaintEffectNoScheduleNoAdmit
+			taint = api.Taint{
+				Key:    taintName,
+				Value:  taintValue,
+				Effect: taintEffect,
+			}
+			framework.AddOrUpdateTaintOnNode(f.Client, nodeName, taint)
+			framework.ExpectNodeHasTaint(f.Client, nodeName, taint)
+
+			By("create the static pod")
+			err := createStaticPod(framework.TestContext.ManifestPath, staticPodName, ns, "nginx", api.RestartPolicyAlways)
+			Expect(err).ShouldNot(HaveOccurred())
+
+			By("Waiting for static pod rejected event")
+			eventFound := false
+
+			_, controller := cache.NewInformer(
+				&cache.ListWatch{
+					ListFunc: func(options api.ListOptions) (runtime.Object, error) {
+						return f.Client.Events(f.Namespace.Name).List(options)
+					},
+					WatchFunc: func(options api.ListOptions) (watch.Interface, error) {
+						return f.Client.Events(f.Namespace.Name).Watch(options)
+					},
+				},
+				&api.Event{},
+				0,
+				cache.ResourceEventHandlerFuncs{
+					AddFunc: func(obj interface{}) {
+						if e, ok := obj.(*api.Event); ok {
+							if e.InvolvedObject.Kind == "Pod" && e.Reason == "PodToleratesNodeTaints" && strings.Contains(e.Message,
+								"Taint Toleration unmatched with SomeUntoleratedTaintIsNoAdmit is: true") {
+								By("PodToleratesNodeTaints event found")
+								eventFound = true
+							}
+						}
+					},
+				},
+			)
+
+			stopCh := make(chan struct{})
+			defer func() {
+				close(stopCh)
+			}()
+			go controller.Run(stopCh)
+
+			// Check if the PodToleratesNodeTaints event is found
+			for start := time.Now(); time.Since(start) < 4*time.Minute; time.Sleep(2 * time.Second) {
+				if eventFound {
+					break
+				}
+			}
+			Expect(eventFound).Should(Equal(true))
+		})
+
+		AfterEach(func() {
+			By("delete the static pod")
+			err := deleteStaticPod(framework.TestContext.ManifestPath, staticPodName, ns)
+			Expect(err).ShouldNot(HaveOccurred())
+
+			By("clear taint")
+			framework.RemoveTaintOffNode(f.Client, nodeName, taint)
+		})
+	})
+})