diff --git a/pkg/api/types.go b/pkg/api/types.go index 8f08fe8dbc5..cf6eebdef4a 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -1464,7 +1464,7 @@ type Taint struct { Value string `json:"value,omitempty"` // Required. The effect of the taint on pods // that do not tolerate the taint. - // Valid effects are NoSchedule and PreferNoSchedule. + // Valid effects are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule. Effect TaintEffect `json:"effect"` } @@ -1480,12 +1480,11 @@ const ( // new pods onto the node, rather than prohibiting new pods from scheduling // onto the node entirely. Enforced by the scheduler. TaintEffectPreferNoSchedule TaintEffect = "PreferNoSchedule" - // NOT YET IMPLEMENTED. TODO: Uncomment field once it is implemented. // Do not allow new pods to schedule onto the node unless they tolerate the taint, // do not allow pods to start on Kubelet unless they tolerate the taint, // but allow all already-running pods to continue running. // Enforced by the scheduler and Kubelet. - // TaintEffectNoScheduleNoAdmit TaintEffect = "NoScheduleNoAdmit" + TaintEffectNoScheduleNoAdmit TaintEffect = "NoScheduleNoAdmit" // NOT YET IMPLEMENTED. TODO: Uncomment field once it is implemented. // Do not allow new pods to schedule onto the node unless they tolerate the taint, // do not allow pods to start on Kubelet unless they tolerate the taint, @@ -1508,7 +1507,7 @@ type Toleration struct { // If the operator is Exists, the value should be empty, otherwise just a regular string. Value string `json:"value,omitempty"` // Effect indicates the taint effect to match. Empty means match all taint effects. - // When specified, allowed values are NoSchedule and PreferNoSchedule. + // When specified, allowed values are NoSchedule,NoScheduleNoAdmit and PreferNoSchedule. Effect TaintEffect `json:"effect,omitempty"` // TODO: For forgiveness (#1574), we'd eventually add at least a grace period // here, and possibly an occurrence threshold and period. diff --git a/pkg/api/v1/generated.proto b/pkg/api/v1/generated.proto index c39e809c85a..2840add2a49 100644 --- a/pkg/api/v1/generated.proto +++ b/pkg/api/v1/generated.proto @@ -2933,7 +2933,7 @@ message Taint { // Required. The effect of the taint on pods // that do not tolerate the taint. - // Valid effects are NoSchedule and PreferNoSchedule. + // Valid effects are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule. optional string effect = 3; } @@ -2954,7 +2954,7 @@ message Toleration { optional string value = 3; // Effect indicates the taint effect to match. Empty means match all taint effects. - // When specified, allowed values are NoSchedule and PreferNoSchedule. + // When specified, allowed values are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule. optional string effect = 4; } diff --git a/pkg/api/v1/types.go b/pkg/api/v1/types.go index e2a6424a47b..ce81d6d2e42 100644 --- a/pkg/api/v1/types.go +++ b/pkg/api/v1/types.go @@ -1683,7 +1683,7 @@ type Taint struct { Value string `json:"value,omitempty" protobuf:"bytes,2,opt,name=value"` // Required. The effect of the taint on pods // that do not tolerate the taint. - // Valid effects are NoSchedule and PreferNoSchedule. + // Valid effects are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule. Effect TaintEffect `json:"effect" protobuf:"bytes,3,opt,name=effect,casttype=TaintEffect"` } @@ -1699,12 +1699,11 @@ const ( // new pods onto the node, rather than prohibiting new pods from scheduling // onto the node entirely. Enforced by the scheduler. TaintEffectPreferNoSchedule TaintEffect = "PreferNoSchedule" - // NOT YET IMPLEMENTED. TODO: Uncomment field once it is implemented. // Do not allow new pods to schedule onto the node unless they tolerate the taint, // do not allow pods to start on Kubelet unless they tolerate the taint, // but allow all already-running pods to continue running. // Enforced by the scheduler and Kubelet. - // TaintEffectNoScheduleNoAdmit TaintEffect = "NoScheduleNoAdmit" + TaintEffectNoScheduleNoAdmit TaintEffect = "NoScheduleNoAdmit" // NOT YET IMPLEMENTED. TODO: Uncomment field once it is implemented. // Do not allow new pods to schedule onto the node unless they tolerate the taint, // do not allow pods to start on Kubelet unless they tolerate the taint, @@ -1727,7 +1726,7 @@ type Toleration struct { // If the operator is Exists, the value should be empty, otherwise just a regular string. Value string `json:"value,omitempty" protobuf:"bytes,3,opt,name=value"` // Effect indicates the taint effect to match. Empty means match all taint effects. - // When specified, allowed values are NoSchedule and PreferNoSchedule. + // When specified, allowed values are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule. Effect TaintEffect `json:"effect,omitempty" protobuf:"bytes,4,opt,name=effect,casttype=TaintEffect"` // TODO: For forgiveness (#1574), we'd eventually add at least a grace period // here, and possibly an occurrence threshold and period. diff --git a/pkg/api/v1/types_swagger_doc_generated.go b/pkg/api/v1/types_swagger_doc_generated.go index 31222110cfc..9fd477829ec 100644 --- a/pkg/api/v1/types_swagger_doc_generated.go +++ b/pkg/api/v1/types_swagger_doc_generated.go @@ -1721,7 +1721,7 @@ var map_Taint = map[string]string{ "": "The node this Taint is attached to has the effect \"effect\" on any pod that that does not tolerate the Taint.", "key": "Required. The taint key to be applied to a node.", "value": "Required. The taint value corresponding to the taint key.", - "effect": "Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule and PreferNoSchedule.", + "effect": "Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.", } func (Taint) SwaggerDoc() map[string]string { @@ -1733,7 +1733,7 @@ var map_Toleration = map[string]string{ "key": "Required. Key is the taint key that the toleration applies to.", "operator": "operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.", "value": "Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.", - "effect": "Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule and PreferNoSchedule.", + "effect": "Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.", } func (Toleration) SwaggerDoc() map[string]string { diff --git a/pkg/api/validation/validation.go b/pkg/api/validation/validation.go index c4680784bcf..478fb8291ea 100644 --- a/pkg/api/validation/validation.go +++ b/pkg/api/validation/validation.go @@ -1780,14 +1780,14 @@ func validateTaintEffect(effect *api.TaintEffect, allowEmpty bool, fldPath *fiel allErrors := field.ErrorList{} switch *effect { // TODO: Replace next line with subsequent commented-out line when implement TaintEffectNoScheduleNoAdmit, TaintEffectNoScheduleNoAdmitNoExecute. - case api.TaintEffectNoSchedule, api.TaintEffectPreferNoSchedule: - // case api.TaintEffectNoSchedule, api.TaintEffectPreferNoSchedule, api.TaintEffectNoScheduleNoAdmit, api.TaintEffectNoScheduleNoAdmitNoExecute: + case api.TaintEffectNoSchedule, api.TaintEffectPreferNoSchedule, api.TaintEffectNoScheduleNoAdmit: + // case api.TaintEffectNoSchedule, api.TaintEffectPreferNoSchedule, api.TaintEffectNoScheduleNoAdmitNoExecute: default: validValues := []string{ string(api.TaintEffectNoSchedule), string(api.TaintEffectPreferNoSchedule), + string(api.TaintEffectNoScheduleNoAdmit), // TODO: Uncomment this block when implement TaintEffectNoScheduleNoAdmit, TaintEffectNoScheduleNoAdmitNoExecute. - // string(api.TaintEffectNoScheduleNoAdmit), // string(api.TaintEffectNoScheduleNoAdmitNoExecute), } allErrors = append(allErrors, field.NotSupported(fldPath, effect, validValues)) diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index 1029bc05cf0..810fefb932f 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -12502,7 +12502,7 @@ var OpenAPIDefinitions *common.OpenAPIDefinitions = &common.OpenAPIDefinitions{ }, "effect": { SchemaProps: spec.SchemaProps{ - Description: "Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule and PreferNoSchedule.", + Description: "Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.", Type: []string{"string"}, Format: "", }, @@ -12600,7 +12600,7 @@ var OpenAPIDefinitions *common.OpenAPIDefinitions = &common.OpenAPIDefinitions{ }, "effect": { SchemaProps: spec.SchemaProps{ - Description: "Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule and PreferNoSchedule.", + Description: "Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.", Type: []string{"string"}, Format: "", }, diff --git a/pkg/kubectl/cmd/taint.go b/pkg/kubectl/cmd/taint.go index caa27758d11..9432b68c821 100644 --- a/pkg/kubectl/cmd/taint.go +++ b/pkg/kubectl/cmd/taint.go @@ -181,7 +181,7 @@ func parseTaints(spec []string) ([]api.Taint, []api.Taint, error) { return nil, nil, fmt.Errorf("invalid taint spec: %v, %s", taintSpec, strings.Join(errs, "; ")) } - if parts2[1] != string(api.TaintEffectNoSchedule) && parts2[1] != string(api.TaintEffectPreferNoSchedule) { + if parts2[1] != string(api.TaintEffectNoSchedule) && parts2[1] != string(api.TaintEffectPreferNoSchedule) && parts2[1] != string(api.TaintEffectNoScheduleNoAdmit) { return nil, nil, fmt.Errorf("invalid taint spec: %v, unsupported taint effect", taintSpec) } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 8cc635ba3e2..fe7eea7e620 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -1515,6 +1515,7 @@ func (kl *Kubelet) canAdmitPod(pods []*api.Pod, pod *api.Pod) (bool, string, str return false, result.Reason, result.Message } } + // TODO: When disk space scheduling is implemented (#11976), remove the out-of-disk check here and // add the disk space predicate to predicates.GeneralPredicates. if kl.isOutOfDisk() { diff --git a/pkg/kubelet/lifecycle/predicate.go b/pkg/kubelet/lifecycle/predicate.go index 231ec28d303..addf9d0d85f 100644 --- a/pkg/kubelet/lifecycle/predicate.go +++ b/pkg/kubelet/lifecycle/predicate.go @@ -101,6 +101,53 @@ func (w *predicateAdmitHandler) Admit(attrs *PodAdmitAttributes) PodAdmitResult Message: message, } } + + // Check toleration against taints + // NOTE(harryz) consider move PodToleratesNodeTaints to GeneralPredicates to eliminate duplicate code here + fit, reasons, err = predicates.PodToleratesNodeTaints(pod, nil, nodeInfo) + if err != nil { + message := fmt.Sprintf("PodToleratesNodeTaints failed due to %v, which is unexpected.", err) + glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), message) + return PodAdmitResult{ + Admit: fit, + Reason: "UnexpectedError", + Message: message, + } + + } + if !fit { + var reason string + var message string + if len(reasons) == 0 { + message = fmt.Sprint("PodToleratesNodeTaints failed due to unknown reason, which is unexpected.") + glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), message) + return PodAdmitResult{ + Admit: fit, + Reason: "UnknownReason", + Message: message, + } + } + r := reasons[0] + switch re := r.(type) { + case *predicates.ErrTaintsTolerationsNotMatch: + // if kubelet should not care this unfit + if !re.SomeUntoleratedTaintIsNoAdmit { + return PodAdmitResult{ + Admit: true, + } + } + reason = "PodToleratesNodeTaints" + message = re.Error() + glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), message) + } + + return PodAdmitResult{ + Admit: fit, + Reason: reason, + Message: message, + } + } + return PodAdmitResult{ Admit: true, } diff --git a/plugin/pkg/scheduler/algorithm/predicates/error.go b/plugin/pkg/scheduler/algorithm/predicates/error.go index 028c2a96d7c..7cee7c1a19e 100644 --- a/plugin/pkg/scheduler/algorithm/predicates/error.go +++ b/plugin/pkg/scheduler/algorithm/predicates/error.go @@ -29,7 +29,6 @@ var ( ErrVolumeZoneConflict = newPredicateFailureError("NoVolumeZoneConflict") ErrNodeSelectorNotMatch = newPredicateFailureError("MatchNodeSelector") ErrPodAffinityNotMatch = newPredicateFailureError("MatchInterPodAffinity") - ErrTaintsTolerationsNotMatch = newPredicateFailureError("PodToleratesNodeTaints") ErrPodNotMatchHostName = newPredicateFailureError("HostName") ErrPodNotFitsHostPorts = newPredicateFailureError("PodFitsHostPorts") ErrNodeLabelPresenceViolated = newPredicateFailureError("CheckNodeLabelPresence") @@ -43,6 +42,25 @@ var ( ErrFakePredicate = newPredicateFailureError("FakePredicateError") ) +// ErrTaintsTolerationsNotMatch is an error type that indicates with if it should be aware by kubelet. +type ErrTaintsTolerationsNotMatch struct { + SomeUntoleratedTaintIsNoAdmit bool +} + +func newErrTaintsTolerationsNotMatch(someUntoleratedTaintIsNoAdmit bool) *ErrTaintsTolerationsNotMatch { + return &ErrTaintsTolerationsNotMatch{ + SomeUntoleratedTaintIsNoAdmit: someUntoleratedTaintIsNoAdmit, + } +} + +func (e *ErrTaintsTolerationsNotMatch) Error() string { + return fmt.Sprintf("Taint Toleration unmatched with SomeUntoleratedTaintIsNoAdmit is: %v", e.SomeUntoleratedTaintIsNoAdmit) +} + +func (e *ErrTaintsTolerationsNotMatch) GetReason() string { + return fmt.Sprintf("ErrTaintsTolerationsNotMatch, and SomeUntoleratedTaintIsNoAdmit is: %v", e.SomeUntoleratedTaintIsNoAdmit) +} + // InsufficientResourceError is an error type that indicates what kind of resource limit is // hit and caused the unfitting failure. type InsufficientResourceError struct { diff --git a/plugin/pkg/scheduler/algorithm/predicates/predicates.go b/plugin/pkg/scheduler/algorithm/predicates/predicates.go index 8dfb3e86558..3cf888011a2 100644 --- a/plugin/pkg/scheduler/algorithm/predicates/predicates.go +++ b/plugin/pkg/scheduler/algorithm/predicates/predicates.go @@ -1083,23 +1083,37 @@ func PodToleratesNodeTaints(pod *api.Pod, meta interface{}, nodeInfo *schedulerc return false, nil, err } - if tolerationsToleratesTaints(tolerations, taints) { + if tolerated, someUntoleratedTaintIsNoAdmit := tolerationsToleratesTaints(tolerations, taints); tolerated { return true, nil, nil + } else { + return false, []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(someUntoleratedTaintIsNoAdmit)}, nil } - return false, []algorithm.PredicateFailureReason{ErrTaintsTolerationsNotMatch}, nil } -func tolerationsToleratesTaints(tolerations []api.Toleration, taints []api.Taint) bool { +// tolerationsToleratesTaints checks if given tolerations can live with given taints. +// It returns: +// 1. whether tolerated or not; +// 2. whether kubelet should be aware if it's unfit. +func tolerationsToleratesTaints(tolerations []api.Toleration, taints []api.Taint) (bool, bool) { // If the taint list is nil/empty, it is tolerated by all tolerations by default. if len(taints) == 0 { - return true + return true, false } // The taint list isn't nil/empty, a nil/empty toleration list can't tolerate them. if len(tolerations) == 0 { - return false + // if there's taint has TaintEffectNoScheduleNoAdmit, kubelet should also be aware of this. + for _, taint := range taints { + if taint.Effect == api.TaintEffectNoScheduleNoAdmit { + return false, true + } + } + return false, false } + someUntoleratedTaintIsNoAdmit := false + fits := true + for i := range taints { taint := &taints[i] // skip taints that have effect PreferNoSchedule, since it is for priorities @@ -1107,12 +1121,16 @@ func tolerationsToleratesTaints(tolerations []api.Toleration, taints []api.Taint continue } - if !api.TaintToleratedByTolerations(taint, tolerations) { - return false + if tolerated := api.TaintToleratedByTolerations(taint, tolerations); !tolerated { + fits = false + if taint.Effect == api.TaintEffectNoScheduleNoAdmit { + someUntoleratedTaintIsNoAdmit = true + return fits, someUntoleratedTaintIsNoAdmit + } } } - return true + return fits, someUntoleratedTaintIsNoAdmit } // Determine if a pod is scheduled with best-effort QoS diff --git a/plugin/pkg/scheduler/algorithm/predicates/predicates_test.go b/plugin/pkg/scheduler/algorithm/predicates/predicates_test.go index 47a1ea714b0..1eb1727c03e 100755 --- a/plugin/pkg/scheduler/algorithm/predicates/predicates_test.go +++ b/plugin/pkg/scheduler/algorithm/predicates/predicates_test.go @@ -2563,10 +2563,11 @@ func TestInterPodAffinityWithMultipleNodes(t *testing.T) { func TestPodToleratesTaints(t *testing.T) { podTolerateTaintsTests := []struct { - pod *api.Pod - node api.Node - fits bool - test string + pod *api.Pod + node api.Node + fits bool + expectedFailureReasons []algorithm.PredicateFailureReason + test string }{ { pod: &api.Pod{ @@ -2587,6 +2588,7 @@ func TestPodToleratesTaints(t *testing.T) { }, }, fits: false, + expectedFailureReasons: []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(false)}, test: "a pod having no tolerations can't be scheduled onto a node with nonempty taints", }, { @@ -2652,6 +2654,7 @@ func TestPodToleratesTaints(t *testing.T) { }, }, fits: false, + expectedFailureReasons: []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(false)}, test: "a pod which can't be scheduled on a dedicated node assigned to user2 with effect NoSchedule", }, { @@ -2758,6 +2761,7 @@ func TestPodToleratesTaints(t *testing.T) { }, }, fits: false, + expectedFailureReasons: []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(false)}, test: "a pod has a toleration that keys and values match the taint on the node, but (non-empty) effect doesn't match, " + "can't be scheduled onto the node", }, @@ -2791,7 +2795,7 @@ func TestPodToleratesTaints(t *testing.T) { }, }, fits: true, - test: "The pod has a toleration that keys and values match the taint on the node, the effect of toleration is empty, " + + test: "the pod has a toleration that keys and values match the taint on the node, the effect of toleration is empty, " + "and the effect of taint is NoSchedule. Pod can be scheduled onto the node", }, { @@ -2828,8 +2832,63 @@ func TestPodToleratesTaints(t *testing.T) { test: "The pod has a toleration that key and value don't match the taint on the node, " + "but the effect of taint on node is PreferNochedule. Pod can be scheduled onto the node", }, + { + pod: &api.Pod{ + ObjectMeta: api.ObjectMeta{ + Name: "podadmit1", + }, + }, + node: api.Node{ + ObjectMeta: api.ObjectMeta{ + Annotations: map[string]string{ + api.TaintsAnnotationKey: ` + [{ + "key": "dedicated", + "value": "user1", + "effect": "NoScheduleNoAdmit" + }]`, + }, + }, + }, + fits: false, + expectedFailureReasons: []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(true)}, + test: "node should aware that that a pod having no tolerations can't be scheduled or started on a node with nonempty taints", + }, + { + pod: &api.Pod{ + ObjectMeta: api.ObjectMeta{ + Name: "podadmit2", + Annotations: map[string]string{ + api.TolerationsAnnotationKey: ` + [{ + "key": "dedicated", + "operator": "Equal", + "value": "user2", + "effect": "NoScheduleNoAdmit" + }]`, + }, + }, + Spec: api.PodSpec{ + Containers: []api.Container{{Image: "pod2:V1"}}, + }, + }, + node: api.Node{ + ObjectMeta: api.ObjectMeta{ + Annotations: map[string]string{ + api.TaintsAnnotationKey: ` + [{ + "key": "dedicated", + "value": "user1", + "effect": "NoScheduleNoAdmit" + }]`, + }, + }, + }, + fits: false, + expectedFailureReasons: []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(true)}, + test: "node should aware that a pod which can't be scheduled or start on a dedicated node assigned to user2 with effect NoScheduleNoAdmit", + }, } - expectedFailureReasons := []algorithm.PredicateFailureReason{ErrTaintsTolerationsNotMatch} for _, test := range podTolerateTaintsTests { nodeInfo := schedulercache.NewNodeInfo() @@ -2838,11 +2897,11 @@ func TestPodToleratesTaints(t *testing.T) { if err != nil { t.Errorf("%s, unexpected error: %v", test.test, err) } - if !fits && !reflect.DeepEqual(reasons, expectedFailureReasons) { - t.Errorf("%s, unexpected failure reason: %v, want: %v", test.test, reasons, expectedFailureReasons) + if !fits && !reflect.DeepEqual(reasons, test.expectedFailureReasons) { + t.Errorf("%s, unexpected failure reason: %v, want: %v", test.test, reasons, test.expectedFailureReasons) } if fits != test.fits { - t.Errorf("%s, expected: %v got %v", test.test, test.fits, fits) + t.Errorf("%s,\n expected: %v got %v", test.test, test.fits, fits) } } } diff --git a/plugin/pkg/scheduler/algorithm/priorities/taint_toleration.go b/plugin/pkg/scheduler/algorithm/priorities/taint_toleration.go index 36fa0312fdb..8777eacc01d 100644 --- a/plugin/pkg/scheduler/algorithm/priorities/taint_toleration.go +++ b/plugin/pkg/scheduler/algorithm/priorities/taint_toleration.go @@ -34,7 +34,7 @@ func countIntolerableTaintsPreferNoSchedule(taints []api.Taint, tolerations []ap continue } - if !api.TaintToleratedByTolerations(taint, tolerations) { + if tolerable := api.TaintToleratedByTolerations(taint, tolerations); !tolerable { intolerableTaints++ } } diff --git a/test/e2e_node/noadmit_taint_test.go b/test/e2e_node/noadmit_taint_test.go new file mode 100644 index 00000000000..ee972e284ce --- /dev/null +++ b/test/e2e_node/noadmit_taint_test.go @@ -0,0 +1,117 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_node + +import ( + "fmt" + "strings" + "time" + + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/client/cache" + "k8s.io/kubernetes/pkg/runtime" + "k8s.io/kubernetes/pkg/util/uuid" + "k8s.io/kubernetes/pkg/watch" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +// NOTE(harry): this test will taint a node, which means adding other specs into this context +// may be influenced when testing with -node=N. +var _ = framework.KubeDescribe("[Serial] NoAdmitTaint", func() { + f := framework.NewDefaultFramework("admit-pod") + Context("when create a static pod", func() { + var ns, staticPodName, mirrorPodName, nodeName, taintName, taintValue string + var taint api.Taint + BeforeEach(func() { + nodeName = framework.TestContext.NodeName + ns = f.Namespace.Name + staticPodName = "static-pod-" + string(uuid.NewUUID()) + // we need to check the mirror pod name (suffixed by nodeName) + mirrorPodName = staticPodName + "-" + nodeName + }) + It("should be rejected when node is tainted with NoAdmit effect ", func() { + By("set NoAdmit taint for the node") + taintName = fmt.Sprintf("kubernetes.io/e2e-taint-key-%s", string(uuid.NewUUID())) + taintValue = "testing-taint-value" + taintEffect := api.TaintEffectNoScheduleNoAdmit + taint = api.Taint{ + Key: taintName, + Value: taintValue, + Effect: taintEffect, + } + framework.AddOrUpdateTaintOnNode(f.Client, nodeName, taint) + framework.ExpectNodeHasTaint(f.Client, nodeName, taint) + + By("create the static pod") + err := createStaticPod(framework.TestContext.ManifestPath, staticPodName, ns, "nginx", api.RestartPolicyAlways) + Expect(err).ShouldNot(HaveOccurred()) + + By("Waiting for static pod rejected event") + eventFound := false + + _, controller := cache.NewInformer( + &cache.ListWatch{ + ListFunc: func(options api.ListOptions) (runtime.Object, error) { + return f.Client.Events(f.Namespace.Name).List(options) + }, + WatchFunc: func(options api.ListOptions) (watch.Interface, error) { + return f.Client.Events(f.Namespace.Name).Watch(options) + }, + }, + &api.Event{}, + 0, + cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + if e, ok := obj.(*api.Event); ok { + if e.InvolvedObject.Kind == "Pod" && e.Reason == "PodToleratesNodeTaints" && strings.Contains(e.Message, + "Taint Toleration unmatched with SomeUntoleratedTaintIsNoAdmit is: true") { + By("PodToleratesNodeTaints event found") + eventFound = true + } + } + }, + }, + ) + + stopCh := make(chan struct{}) + defer func() { + close(stopCh) + }() + go controller.Run(stopCh) + + // Check if the PodToleratesNodeTaints event is found + for start := time.Now(); time.Since(start) < 4*time.Minute; time.Sleep(2 * time.Second) { + if eventFound { + break + } + } + Expect(eventFound).Should(Equal(true)) + }) + + AfterEach(func() { + By("delete the static pod") + err := deleteStaticPod(framework.TestContext.ManifestPath, staticPodName, ns) + Expect(err).ShouldNot(HaveOccurred()) + + By("clear taint") + framework.RemoveTaintOffNode(f.Client, nodeName, taint) + }) + }) +})