mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-25 04:33:26 +00:00
Merge pull request #26501 from resouer/scheduler
Automatic merge from submit-queue Add kubelet awareness to taint tolerant match caculator. Add kubelet awareness to taint tolerant match caculator. Ref: #25320 This is required by `TaintEffectNoScheduleNoAdmit` & `TaintEffectNoScheduleNoAdmitNoExecute `, so that node will know if it should expect the taint&tolerant
This commit is contained in:
commit
21188cadeb
@ -1464,7 +1464,7 @@ type Taint struct {
|
|||||||
Value string `json:"value,omitempty"`
|
Value string `json:"value,omitempty"`
|
||||||
// Required. The effect of the taint on pods
|
// Required. The effect of the taint on pods
|
||||||
// that do not tolerate the taint.
|
// that do not tolerate the taint.
|
||||||
// Valid effects are NoSchedule and PreferNoSchedule.
|
// Valid effects are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.
|
||||||
Effect TaintEffect `json:"effect"`
|
Effect TaintEffect `json:"effect"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1480,12 +1480,11 @@ const (
|
|||||||
// new pods onto the node, rather than prohibiting new pods from scheduling
|
// new pods onto the node, rather than prohibiting new pods from scheduling
|
||||||
// onto the node entirely. Enforced by the scheduler.
|
// onto the node entirely. Enforced by the scheduler.
|
||||||
TaintEffectPreferNoSchedule TaintEffect = "PreferNoSchedule"
|
TaintEffectPreferNoSchedule TaintEffect = "PreferNoSchedule"
|
||||||
// NOT YET IMPLEMENTED. TODO: Uncomment field once it is implemented.
|
|
||||||
// Do not allow new pods to schedule onto the node unless they tolerate the taint,
|
// Do not allow new pods to schedule onto the node unless they tolerate the taint,
|
||||||
// do not allow pods to start on Kubelet unless they tolerate the taint,
|
// do not allow pods to start on Kubelet unless they tolerate the taint,
|
||||||
// but allow all already-running pods to continue running.
|
// but allow all already-running pods to continue running.
|
||||||
// Enforced by the scheduler and Kubelet.
|
// Enforced by the scheduler and Kubelet.
|
||||||
// TaintEffectNoScheduleNoAdmit TaintEffect = "NoScheduleNoAdmit"
|
TaintEffectNoScheduleNoAdmit TaintEffect = "NoScheduleNoAdmit"
|
||||||
// NOT YET IMPLEMENTED. TODO: Uncomment field once it is implemented.
|
// NOT YET IMPLEMENTED. TODO: Uncomment field once it is implemented.
|
||||||
// Do not allow new pods to schedule onto the node unless they tolerate the taint,
|
// Do not allow new pods to schedule onto the node unless they tolerate the taint,
|
||||||
// do not allow pods to start on Kubelet unless they tolerate the taint,
|
// do not allow pods to start on Kubelet unless they tolerate the taint,
|
||||||
@ -1508,7 +1507,7 @@ type Toleration struct {
|
|||||||
// If the operator is Exists, the value should be empty, otherwise just a regular string.
|
// If the operator is Exists, the value should be empty, otherwise just a regular string.
|
||||||
Value string `json:"value,omitempty"`
|
Value string `json:"value,omitempty"`
|
||||||
// Effect indicates the taint effect to match. Empty means match all taint effects.
|
// Effect indicates the taint effect to match. Empty means match all taint effects.
|
||||||
// When specified, allowed values are NoSchedule and PreferNoSchedule.
|
// When specified, allowed values are NoSchedule,NoScheduleNoAdmit and PreferNoSchedule.
|
||||||
Effect TaintEffect `json:"effect,omitempty"`
|
Effect TaintEffect `json:"effect,omitempty"`
|
||||||
// TODO: For forgiveness (#1574), we'd eventually add at least a grace period
|
// TODO: For forgiveness (#1574), we'd eventually add at least a grace period
|
||||||
// here, and possibly an occurrence threshold and period.
|
// here, and possibly an occurrence threshold and period.
|
||||||
|
@ -2933,7 +2933,7 @@ message Taint {
|
|||||||
|
|
||||||
// Required. The effect of the taint on pods
|
// Required. The effect of the taint on pods
|
||||||
// that do not tolerate the taint.
|
// that do not tolerate the taint.
|
||||||
// Valid effects are NoSchedule and PreferNoSchedule.
|
// Valid effects are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.
|
||||||
optional string effect = 3;
|
optional string effect = 3;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -2954,7 +2954,7 @@ message Toleration {
|
|||||||
optional string value = 3;
|
optional string value = 3;
|
||||||
|
|
||||||
// Effect indicates the taint effect to match. Empty means match all taint effects.
|
// Effect indicates the taint effect to match. Empty means match all taint effects.
|
||||||
// When specified, allowed values are NoSchedule and PreferNoSchedule.
|
// When specified, allowed values are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.
|
||||||
optional string effect = 4;
|
optional string effect = 4;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1683,7 +1683,7 @@ type Taint struct {
|
|||||||
Value string `json:"value,omitempty" protobuf:"bytes,2,opt,name=value"`
|
Value string `json:"value,omitempty" protobuf:"bytes,2,opt,name=value"`
|
||||||
// Required. The effect of the taint on pods
|
// Required. The effect of the taint on pods
|
||||||
// that do not tolerate the taint.
|
// that do not tolerate the taint.
|
||||||
// Valid effects are NoSchedule and PreferNoSchedule.
|
// Valid effects are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.
|
||||||
Effect TaintEffect `json:"effect" protobuf:"bytes,3,opt,name=effect,casttype=TaintEffect"`
|
Effect TaintEffect `json:"effect" protobuf:"bytes,3,opt,name=effect,casttype=TaintEffect"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1699,12 +1699,11 @@ const (
|
|||||||
// new pods onto the node, rather than prohibiting new pods from scheduling
|
// new pods onto the node, rather than prohibiting new pods from scheduling
|
||||||
// onto the node entirely. Enforced by the scheduler.
|
// onto the node entirely. Enforced by the scheduler.
|
||||||
TaintEffectPreferNoSchedule TaintEffect = "PreferNoSchedule"
|
TaintEffectPreferNoSchedule TaintEffect = "PreferNoSchedule"
|
||||||
// NOT YET IMPLEMENTED. TODO: Uncomment field once it is implemented.
|
|
||||||
// Do not allow new pods to schedule onto the node unless they tolerate the taint,
|
// Do not allow new pods to schedule onto the node unless they tolerate the taint,
|
||||||
// do not allow pods to start on Kubelet unless they tolerate the taint,
|
// do not allow pods to start on Kubelet unless they tolerate the taint,
|
||||||
// but allow all already-running pods to continue running.
|
// but allow all already-running pods to continue running.
|
||||||
// Enforced by the scheduler and Kubelet.
|
// Enforced by the scheduler and Kubelet.
|
||||||
// TaintEffectNoScheduleNoAdmit TaintEffect = "NoScheduleNoAdmit"
|
TaintEffectNoScheduleNoAdmit TaintEffect = "NoScheduleNoAdmit"
|
||||||
// NOT YET IMPLEMENTED. TODO: Uncomment field once it is implemented.
|
// NOT YET IMPLEMENTED. TODO: Uncomment field once it is implemented.
|
||||||
// Do not allow new pods to schedule onto the node unless they tolerate the taint,
|
// Do not allow new pods to schedule onto the node unless they tolerate the taint,
|
||||||
// do not allow pods to start on Kubelet unless they tolerate the taint,
|
// do not allow pods to start on Kubelet unless they tolerate the taint,
|
||||||
@ -1727,7 +1726,7 @@ type Toleration struct {
|
|||||||
// If the operator is Exists, the value should be empty, otherwise just a regular string.
|
// If the operator is Exists, the value should be empty, otherwise just a regular string.
|
||||||
Value string `json:"value,omitempty" protobuf:"bytes,3,opt,name=value"`
|
Value string `json:"value,omitempty" protobuf:"bytes,3,opt,name=value"`
|
||||||
// Effect indicates the taint effect to match. Empty means match all taint effects.
|
// Effect indicates the taint effect to match. Empty means match all taint effects.
|
||||||
// When specified, allowed values are NoSchedule and PreferNoSchedule.
|
// When specified, allowed values are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.
|
||||||
Effect TaintEffect `json:"effect,omitempty" protobuf:"bytes,4,opt,name=effect,casttype=TaintEffect"`
|
Effect TaintEffect `json:"effect,omitempty" protobuf:"bytes,4,opt,name=effect,casttype=TaintEffect"`
|
||||||
// TODO: For forgiveness (#1574), we'd eventually add at least a grace period
|
// TODO: For forgiveness (#1574), we'd eventually add at least a grace period
|
||||||
// here, and possibly an occurrence threshold and period.
|
// here, and possibly an occurrence threshold and period.
|
||||||
|
@ -1721,7 +1721,7 @@ var map_Taint = map[string]string{
|
|||||||
"": "The node this Taint is attached to has the effect \"effect\" on any pod that that does not tolerate the Taint.",
|
"": "The node this Taint is attached to has the effect \"effect\" on any pod that that does not tolerate the Taint.",
|
||||||
"key": "Required. The taint key to be applied to a node.",
|
"key": "Required. The taint key to be applied to a node.",
|
||||||
"value": "Required. The taint value corresponding to the taint key.",
|
"value": "Required. The taint value corresponding to the taint key.",
|
||||||
"effect": "Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule and PreferNoSchedule.",
|
"effect": "Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.",
|
||||||
}
|
}
|
||||||
|
|
||||||
func (Taint) SwaggerDoc() map[string]string {
|
func (Taint) SwaggerDoc() map[string]string {
|
||||||
@ -1733,7 +1733,7 @@ var map_Toleration = map[string]string{
|
|||||||
"key": "Required. Key is the taint key that the toleration applies to.",
|
"key": "Required. Key is the taint key that the toleration applies to.",
|
||||||
"operator": "operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.",
|
"operator": "operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.",
|
||||||
"value": "Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.",
|
"value": "Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.",
|
||||||
"effect": "Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule and PreferNoSchedule.",
|
"effect": "Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.",
|
||||||
}
|
}
|
||||||
|
|
||||||
func (Toleration) SwaggerDoc() map[string]string {
|
func (Toleration) SwaggerDoc() map[string]string {
|
||||||
|
@ -1780,14 +1780,14 @@ func validateTaintEffect(effect *api.TaintEffect, allowEmpty bool, fldPath *fiel
|
|||||||
allErrors := field.ErrorList{}
|
allErrors := field.ErrorList{}
|
||||||
switch *effect {
|
switch *effect {
|
||||||
// TODO: Replace next line with subsequent commented-out line when implement TaintEffectNoScheduleNoAdmit, TaintEffectNoScheduleNoAdmitNoExecute.
|
// TODO: Replace next line with subsequent commented-out line when implement TaintEffectNoScheduleNoAdmit, TaintEffectNoScheduleNoAdmitNoExecute.
|
||||||
case api.TaintEffectNoSchedule, api.TaintEffectPreferNoSchedule:
|
case api.TaintEffectNoSchedule, api.TaintEffectPreferNoSchedule, api.TaintEffectNoScheduleNoAdmit:
|
||||||
// case api.TaintEffectNoSchedule, api.TaintEffectPreferNoSchedule, api.TaintEffectNoScheduleNoAdmit, api.TaintEffectNoScheduleNoAdmitNoExecute:
|
// case api.TaintEffectNoSchedule, api.TaintEffectPreferNoSchedule, api.TaintEffectNoScheduleNoAdmitNoExecute:
|
||||||
default:
|
default:
|
||||||
validValues := []string{
|
validValues := []string{
|
||||||
string(api.TaintEffectNoSchedule),
|
string(api.TaintEffectNoSchedule),
|
||||||
string(api.TaintEffectPreferNoSchedule),
|
string(api.TaintEffectPreferNoSchedule),
|
||||||
|
string(api.TaintEffectNoScheduleNoAdmit),
|
||||||
// TODO: Uncomment this block when implement TaintEffectNoScheduleNoAdmit, TaintEffectNoScheduleNoAdmitNoExecute.
|
// TODO: Uncomment this block when implement TaintEffectNoScheduleNoAdmit, TaintEffectNoScheduleNoAdmitNoExecute.
|
||||||
// string(api.TaintEffectNoScheduleNoAdmit),
|
|
||||||
// string(api.TaintEffectNoScheduleNoAdmitNoExecute),
|
// string(api.TaintEffectNoScheduleNoAdmitNoExecute),
|
||||||
}
|
}
|
||||||
allErrors = append(allErrors, field.NotSupported(fldPath, effect, validValues))
|
allErrors = append(allErrors, field.NotSupported(fldPath, effect, validValues))
|
||||||
|
@ -12502,7 +12502,7 @@ var OpenAPIDefinitions *common.OpenAPIDefinitions = &common.OpenAPIDefinitions{
|
|||||||
},
|
},
|
||||||
"effect": {
|
"effect": {
|
||||||
SchemaProps: spec.SchemaProps{
|
SchemaProps: spec.SchemaProps{
|
||||||
Description: "Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule and PreferNoSchedule.",
|
Description: "Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.",
|
||||||
Type: []string{"string"},
|
Type: []string{"string"},
|
||||||
Format: "",
|
Format: "",
|
||||||
},
|
},
|
||||||
@ -12600,7 +12600,7 @@ var OpenAPIDefinitions *common.OpenAPIDefinitions = &common.OpenAPIDefinitions{
|
|||||||
},
|
},
|
||||||
"effect": {
|
"effect": {
|
||||||
SchemaProps: spec.SchemaProps{
|
SchemaProps: spec.SchemaProps{
|
||||||
Description: "Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule and PreferNoSchedule.",
|
Description: "Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, NoScheduleNoAdmit and PreferNoSchedule.",
|
||||||
Type: []string{"string"},
|
Type: []string{"string"},
|
||||||
Format: "",
|
Format: "",
|
||||||
},
|
},
|
||||||
|
@ -181,7 +181,7 @@ func parseTaints(spec []string) ([]api.Taint, []api.Taint, error) {
|
|||||||
return nil, nil, fmt.Errorf("invalid taint spec: %v, %s", taintSpec, strings.Join(errs, "; "))
|
return nil, nil, fmt.Errorf("invalid taint spec: %v, %s", taintSpec, strings.Join(errs, "; "))
|
||||||
}
|
}
|
||||||
|
|
||||||
if parts2[1] != string(api.TaintEffectNoSchedule) && parts2[1] != string(api.TaintEffectPreferNoSchedule) {
|
if parts2[1] != string(api.TaintEffectNoSchedule) && parts2[1] != string(api.TaintEffectPreferNoSchedule) && parts2[1] != string(api.TaintEffectNoScheduleNoAdmit) {
|
||||||
return nil, nil, fmt.Errorf("invalid taint spec: %v, unsupported taint effect", taintSpec)
|
return nil, nil, fmt.Errorf("invalid taint spec: %v, unsupported taint effect", taintSpec)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1515,6 +1515,7 @@ func (kl *Kubelet) canAdmitPod(pods []*api.Pod, pod *api.Pod) (bool, string, str
|
|||||||
return false, result.Reason, result.Message
|
return false, result.Reason, result.Message
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: When disk space scheduling is implemented (#11976), remove the out-of-disk check here and
|
// TODO: When disk space scheduling is implemented (#11976), remove the out-of-disk check here and
|
||||||
// add the disk space predicate to predicates.GeneralPredicates.
|
// add the disk space predicate to predicates.GeneralPredicates.
|
||||||
if kl.isOutOfDisk() {
|
if kl.isOutOfDisk() {
|
||||||
|
@ -101,6 +101,53 @@ func (w *predicateAdmitHandler) Admit(attrs *PodAdmitAttributes) PodAdmitResult
|
|||||||
Message: message,
|
Message: message,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Check toleration against taints
|
||||||
|
// NOTE(harryz) consider move PodToleratesNodeTaints to GeneralPredicates to eliminate duplicate code here
|
||||||
|
fit, reasons, err = predicates.PodToleratesNodeTaints(pod, nil, nodeInfo)
|
||||||
|
if err != nil {
|
||||||
|
message := fmt.Sprintf("PodToleratesNodeTaints failed due to %v, which is unexpected.", err)
|
||||||
|
glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), message)
|
||||||
|
return PodAdmitResult{
|
||||||
|
Admit: fit,
|
||||||
|
Reason: "UnexpectedError",
|
||||||
|
Message: message,
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
if !fit {
|
||||||
|
var reason string
|
||||||
|
var message string
|
||||||
|
if len(reasons) == 0 {
|
||||||
|
message = fmt.Sprint("PodToleratesNodeTaints failed due to unknown reason, which is unexpected.")
|
||||||
|
glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), message)
|
||||||
|
return PodAdmitResult{
|
||||||
|
Admit: fit,
|
||||||
|
Reason: "UnknownReason",
|
||||||
|
Message: message,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
r := reasons[0]
|
||||||
|
switch re := r.(type) {
|
||||||
|
case *predicates.ErrTaintsTolerationsNotMatch:
|
||||||
|
// if kubelet should not care this unfit
|
||||||
|
if !re.SomeUntoleratedTaintIsNoAdmit {
|
||||||
|
return PodAdmitResult{
|
||||||
|
Admit: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
reason = "PodToleratesNodeTaints"
|
||||||
|
message = re.Error()
|
||||||
|
glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), message)
|
||||||
|
}
|
||||||
|
|
||||||
|
return PodAdmitResult{
|
||||||
|
Admit: fit,
|
||||||
|
Reason: reason,
|
||||||
|
Message: message,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return PodAdmitResult{
|
return PodAdmitResult{
|
||||||
Admit: true,
|
Admit: true,
|
||||||
}
|
}
|
||||||
|
@ -29,7 +29,6 @@ var (
|
|||||||
ErrVolumeZoneConflict = newPredicateFailureError("NoVolumeZoneConflict")
|
ErrVolumeZoneConflict = newPredicateFailureError("NoVolumeZoneConflict")
|
||||||
ErrNodeSelectorNotMatch = newPredicateFailureError("MatchNodeSelector")
|
ErrNodeSelectorNotMatch = newPredicateFailureError("MatchNodeSelector")
|
||||||
ErrPodAffinityNotMatch = newPredicateFailureError("MatchInterPodAffinity")
|
ErrPodAffinityNotMatch = newPredicateFailureError("MatchInterPodAffinity")
|
||||||
ErrTaintsTolerationsNotMatch = newPredicateFailureError("PodToleratesNodeTaints")
|
|
||||||
ErrPodNotMatchHostName = newPredicateFailureError("HostName")
|
ErrPodNotMatchHostName = newPredicateFailureError("HostName")
|
||||||
ErrPodNotFitsHostPorts = newPredicateFailureError("PodFitsHostPorts")
|
ErrPodNotFitsHostPorts = newPredicateFailureError("PodFitsHostPorts")
|
||||||
ErrNodeLabelPresenceViolated = newPredicateFailureError("CheckNodeLabelPresence")
|
ErrNodeLabelPresenceViolated = newPredicateFailureError("CheckNodeLabelPresence")
|
||||||
@ -43,6 +42,25 @@ var (
|
|||||||
ErrFakePredicate = newPredicateFailureError("FakePredicateError")
|
ErrFakePredicate = newPredicateFailureError("FakePredicateError")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ErrTaintsTolerationsNotMatch is an error type that indicates with if it should be aware by kubelet.
|
||||||
|
type ErrTaintsTolerationsNotMatch struct {
|
||||||
|
SomeUntoleratedTaintIsNoAdmit bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func newErrTaintsTolerationsNotMatch(someUntoleratedTaintIsNoAdmit bool) *ErrTaintsTolerationsNotMatch {
|
||||||
|
return &ErrTaintsTolerationsNotMatch{
|
||||||
|
SomeUntoleratedTaintIsNoAdmit: someUntoleratedTaintIsNoAdmit,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *ErrTaintsTolerationsNotMatch) Error() string {
|
||||||
|
return fmt.Sprintf("Taint Toleration unmatched with SomeUntoleratedTaintIsNoAdmit is: %v", e.SomeUntoleratedTaintIsNoAdmit)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *ErrTaintsTolerationsNotMatch) GetReason() string {
|
||||||
|
return fmt.Sprintf("ErrTaintsTolerationsNotMatch, and SomeUntoleratedTaintIsNoAdmit is: %v", e.SomeUntoleratedTaintIsNoAdmit)
|
||||||
|
}
|
||||||
|
|
||||||
// InsufficientResourceError is an error type that indicates what kind of resource limit is
|
// InsufficientResourceError is an error type that indicates what kind of resource limit is
|
||||||
// hit and caused the unfitting failure.
|
// hit and caused the unfitting failure.
|
||||||
type InsufficientResourceError struct {
|
type InsufficientResourceError struct {
|
||||||
|
@ -1083,23 +1083,37 @@ func PodToleratesNodeTaints(pod *api.Pod, meta interface{}, nodeInfo *schedulerc
|
|||||||
return false, nil, err
|
return false, nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if tolerationsToleratesTaints(tolerations, taints) {
|
if tolerated, someUntoleratedTaintIsNoAdmit := tolerationsToleratesTaints(tolerations, taints); tolerated {
|
||||||
return true, nil, nil
|
return true, nil, nil
|
||||||
|
} else {
|
||||||
|
return false, []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(someUntoleratedTaintIsNoAdmit)}, nil
|
||||||
}
|
}
|
||||||
return false, []algorithm.PredicateFailureReason{ErrTaintsTolerationsNotMatch}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func tolerationsToleratesTaints(tolerations []api.Toleration, taints []api.Taint) bool {
|
// tolerationsToleratesTaints checks if given tolerations can live with given taints.
|
||||||
|
// It returns:
|
||||||
|
// 1. whether tolerated or not;
|
||||||
|
// 2. whether kubelet should be aware if it's unfit.
|
||||||
|
func tolerationsToleratesTaints(tolerations []api.Toleration, taints []api.Taint) (bool, bool) {
|
||||||
// If the taint list is nil/empty, it is tolerated by all tolerations by default.
|
// If the taint list is nil/empty, it is tolerated by all tolerations by default.
|
||||||
if len(taints) == 0 {
|
if len(taints) == 0 {
|
||||||
return true
|
return true, false
|
||||||
}
|
}
|
||||||
|
|
||||||
// The taint list isn't nil/empty, a nil/empty toleration list can't tolerate them.
|
// The taint list isn't nil/empty, a nil/empty toleration list can't tolerate them.
|
||||||
if len(tolerations) == 0 {
|
if len(tolerations) == 0 {
|
||||||
return false
|
// if there's taint has TaintEffectNoScheduleNoAdmit, kubelet should also be aware of this.
|
||||||
|
for _, taint := range taints {
|
||||||
|
if taint.Effect == api.TaintEffectNoScheduleNoAdmit {
|
||||||
|
return false, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false, false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
someUntoleratedTaintIsNoAdmit := false
|
||||||
|
fits := true
|
||||||
|
|
||||||
for i := range taints {
|
for i := range taints {
|
||||||
taint := &taints[i]
|
taint := &taints[i]
|
||||||
// skip taints that have effect PreferNoSchedule, since it is for priorities
|
// skip taints that have effect PreferNoSchedule, since it is for priorities
|
||||||
@ -1107,12 +1121,16 @@ func tolerationsToleratesTaints(tolerations []api.Toleration, taints []api.Taint
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if !api.TaintToleratedByTolerations(taint, tolerations) {
|
if tolerated := api.TaintToleratedByTolerations(taint, tolerations); !tolerated {
|
||||||
return false
|
fits = false
|
||||||
|
if taint.Effect == api.TaintEffectNoScheduleNoAdmit {
|
||||||
|
someUntoleratedTaintIsNoAdmit = true
|
||||||
|
return fits, someUntoleratedTaintIsNoAdmit
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return true
|
return fits, someUntoleratedTaintIsNoAdmit
|
||||||
}
|
}
|
||||||
|
|
||||||
// Determine if a pod is scheduled with best-effort QoS
|
// Determine if a pod is scheduled with best-effort QoS
|
||||||
|
@ -2563,10 +2563,11 @@ func TestInterPodAffinityWithMultipleNodes(t *testing.T) {
|
|||||||
|
|
||||||
func TestPodToleratesTaints(t *testing.T) {
|
func TestPodToleratesTaints(t *testing.T) {
|
||||||
podTolerateTaintsTests := []struct {
|
podTolerateTaintsTests := []struct {
|
||||||
pod *api.Pod
|
pod *api.Pod
|
||||||
node api.Node
|
node api.Node
|
||||||
fits bool
|
fits bool
|
||||||
test string
|
expectedFailureReasons []algorithm.PredicateFailureReason
|
||||||
|
test string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
pod: &api.Pod{
|
pod: &api.Pod{
|
||||||
@ -2587,6 +2588,7 @@ func TestPodToleratesTaints(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
fits: false,
|
fits: false,
|
||||||
|
expectedFailureReasons: []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(false)},
|
||||||
test: "a pod having no tolerations can't be scheduled onto a node with nonempty taints",
|
test: "a pod having no tolerations can't be scheduled onto a node with nonempty taints",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -2652,6 +2654,7 @@ func TestPodToleratesTaints(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
fits: false,
|
fits: false,
|
||||||
|
expectedFailureReasons: []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(false)},
|
||||||
test: "a pod which can't be scheduled on a dedicated node assigned to user2 with effect NoSchedule",
|
test: "a pod which can't be scheduled on a dedicated node assigned to user2 with effect NoSchedule",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -2758,6 +2761,7 @@ func TestPodToleratesTaints(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
fits: false,
|
fits: false,
|
||||||
|
expectedFailureReasons: []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(false)},
|
||||||
test: "a pod has a toleration that keys and values match the taint on the node, but (non-empty) effect doesn't match, " +
|
test: "a pod has a toleration that keys and values match the taint on the node, but (non-empty) effect doesn't match, " +
|
||||||
"can't be scheduled onto the node",
|
"can't be scheduled onto the node",
|
||||||
},
|
},
|
||||||
@ -2791,7 +2795,7 @@ func TestPodToleratesTaints(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
fits: true,
|
fits: true,
|
||||||
test: "The pod has a toleration that keys and values match the taint on the node, the effect of toleration is empty, " +
|
test: "the pod has a toleration that keys and values match the taint on the node, the effect of toleration is empty, " +
|
||||||
"and the effect of taint is NoSchedule. Pod can be scheduled onto the node",
|
"and the effect of taint is NoSchedule. Pod can be scheduled onto the node",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -2828,8 +2832,63 @@ func TestPodToleratesTaints(t *testing.T) {
|
|||||||
test: "The pod has a toleration that key and value don't match the taint on the node, " +
|
test: "The pod has a toleration that key and value don't match the taint on the node, " +
|
||||||
"but the effect of taint on node is PreferNochedule. Pod can be scheduled onto the node",
|
"but the effect of taint on node is PreferNochedule. Pod can be scheduled onto the node",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
pod: &api.Pod{
|
||||||
|
ObjectMeta: api.ObjectMeta{
|
||||||
|
Name: "podadmit1",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
node: api.Node{
|
||||||
|
ObjectMeta: api.ObjectMeta{
|
||||||
|
Annotations: map[string]string{
|
||||||
|
api.TaintsAnnotationKey: `
|
||||||
|
[{
|
||||||
|
"key": "dedicated",
|
||||||
|
"value": "user1",
|
||||||
|
"effect": "NoScheduleNoAdmit"
|
||||||
|
}]`,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
fits: false,
|
||||||
|
expectedFailureReasons: []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(true)},
|
||||||
|
test: "node should aware that that a pod having no tolerations can't be scheduled or started on a node with nonempty taints",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
pod: &api.Pod{
|
||||||
|
ObjectMeta: api.ObjectMeta{
|
||||||
|
Name: "podadmit2",
|
||||||
|
Annotations: map[string]string{
|
||||||
|
api.TolerationsAnnotationKey: `
|
||||||
|
[{
|
||||||
|
"key": "dedicated",
|
||||||
|
"operator": "Equal",
|
||||||
|
"value": "user2",
|
||||||
|
"effect": "NoScheduleNoAdmit"
|
||||||
|
}]`,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Spec: api.PodSpec{
|
||||||
|
Containers: []api.Container{{Image: "pod2:V1"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
node: api.Node{
|
||||||
|
ObjectMeta: api.ObjectMeta{
|
||||||
|
Annotations: map[string]string{
|
||||||
|
api.TaintsAnnotationKey: `
|
||||||
|
[{
|
||||||
|
"key": "dedicated",
|
||||||
|
"value": "user1",
|
||||||
|
"effect": "NoScheduleNoAdmit"
|
||||||
|
}]`,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
fits: false,
|
||||||
|
expectedFailureReasons: []algorithm.PredicateFailureReason{newErrTaintsTolerationsNotMatch(true)},
|
||||||
|
test: "node should aware that a pod which can't be scheduled or start on a dedicated node assigned to user2 with effect NoScheduleNoAdmit",
|
||||||
|
},
|
||||||
}
|
}
|
||||||
expectedFailureReasons := []algorithm.PredicateFailureReason{ErrTaintsTolerationsNotMatch}
|
|
||||||
|
|
||||||
for _, test := range podTolerateTaintsTests {
|
for _, test := range podTolerateTaintsTests {
|
||||||
nodeInfo := schedulercache.NewNodeInfo()
|
nodeInfo := schedulercache.NewNodeInfo()
|
||||||
@ -2838,11 +2897,11 @@ func TestPodToleratesTaints(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("%s, unexpected error: %v", test.test, err)
|
t.Errorf("%s, unexpected error: %v", test.test, err)
|
||||||
}
|
}
|
||||||
if !fits && !reflect.DeepEqual(reasons, expectedFailureReasons) {
|
if !fits && !reflect.DeepEqual(reasons, test.expectedFailureReasons) {
|
||||||
t.Errorf("%s, unexpected failure reason: %v, want: %v", test.test, reasons, expectedFailureReasons)
|
t.Errorf("%s, unexpected failure reason: %v, want: %v", test.test, reasons, test.expectedFailureReasons)
|
||||||
}
|
}
|
||||||
if fits != test.fits {
|
if fits != test.fits {
|
||||||
t.Errorf("%s, expected: %v got %v", test.test, test.fits, fits)
|
t.Errorf("%s,\n expected: %v got %v", test.test, test.fits, fits)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -34,7 +34,7 @@ func countIntolerableTaintsPreferNoSchedule(taints []api.Taint, tolerations []ap
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if !api.TaintToleratedByTolerations(taint, tolerations) {
|
if tolerable := api.TaintToleratedByTolerations(taint, tolerations); !tolerable {
|
||||||
intolerableTaints++
|
intolerableTaints++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
117
test/e2e_node/noadmit_taint_test.go
Normal file
117
test/e2e_node/noadmit_taint_test.go
Normal file
@ -0,0 +1,117 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2016 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package e2e_node
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"k8s.io/kubernetes/pkg/api"
|
||||||
|
"k8s.io/kubernetes/pkg/client/cache"
|
||||||
|
"k8s.io/kubernetes/pkg/runtime"
|
||||||
|
"k8s.io/kubernetes/pkg/util/uuid"
|
||||||
|
"k8s.io/kubernetes/pkg/watch"
|
||||||
|
"k8s.io/kubernetes/test/e2e/framework"
|
||||||
|
|
||||||
|
. "github.com/onsi/ginkgo"
|
||||||
|
. "github.com/onsi/gomega"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NOTE(harry): this test will taint a node, which means adding other specs into this context
|
||||||
|
// may be influenced when testing with -node=N.
|
||||||
|
var _ = framework.KubeDescribe("[Serial] NoAdmitTaint", func() {
|
||||||
|
f := framework.NewDefaultFramework("admit-pod")
|
||||||
|
Context("when create a static pod", func() {
|
||||||
|
var ns, staticPodName, mirrorPodName, nodeName, taintName, taintValue string
|
||||||
|
var taint api.Taint
|
||||||
|
BeforeEach(func() {
|
||||||
|
nodeName = framework.TestContext.NodeName
|
||||||
|
ns = f.Namespace.Name
|
||||||
|
staticPodName = "static-pod-" + string(uuid.NewUUID())
|
||||||
|
// we need to check the mirror pod name (suffixed by nodeName)
|
||||||
|
mirrorPodName = staticPodName + "-" + nodeName
|
||||||
|
})
|
||||||
|
It("should be rejected when node is tainted with NoAdmit effect ", func() {
|
||||||
|
By("set NoAdmit taint for the node")
|
||||||
|
taintName = fmt.Sprintf("kubernetes.io/e2e-taint-key-%s", string(uuid.NewUUID()))
|
||||||
|
taintValue = "testing-taint-value"
|
||||||
|
taintEffect := api.TaintEffectNoScheduleNoAdmit
|
||||||
|
taint = api.Taint{
|
||||||
|
Key: taintName,
|
||||||
|
Value: taintValue,
|
||||||
|
Effect: taintEffect,
|
||||||
|
}
|
||||||
|
framework.AddOrUpdateTaintOnNode(f.Client, nodeName, taint)
|
||||||
|
framework.ExpectNodeHasTaint(f.Client, nodeName, taint)
|
||||||
|
|
||||||
|
By("create the static pod")
|
||||||
|
err := createStaticPod(framework.TestContext.ManifestPath, staticPodName, ns, "nginx", api.RestartPolicyAlways)
|
||||||
|
Expect(err).ShouldNot(HaveOccurred())
|
||||||
|
|
||||||
|
By("Waiting for static pod rejected event")
|
||||||
|
eventFound := false
|
||||||
|
|
||||||
|
_, controller := cache.NewInformer(
|
||||||
|
&cache.ListWatch{
|
||||||
|
ListFunc: func(options api.ListOptions) (runtime.Object, error) {
|
||||||
|
return f.Client.Events(f.Namespace.Name).List(options)
|
||||||
|
},
|
||||||
|
WatchFunc: func(options api.ListOptions) (watch.Interface, error) {
|
||||||
|
return f.Client.Events(f.Namespace.Name).Watch(options)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
&api.Event{},
|
||||||
|
0,
|
||||||
|
cache.ResourceEventHandlerFuncs{
|
||||||
|
AddFunc: func(obj interface{}) {
|
||||||
|
if e, ok := obj.(*api.Event); ok {
|
||||||
|
if e.InvolvedObject.Kind == "Pod" && e.Reason == "PodToleratesNodeTaints" && strings.Contains(e.Message,
|
||||||
|
"Taint Toleration unmatched with SomeUntoleratedTaintIsNoAdmit is: true") {
|
||||||
|
By("PodToleratesNodeTaints event found")
|
||||||
|
eventFound = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
stopCh := make(chan struct{})
|
||||||
|
defer func() {
|
||||||
|
close(stopCh)
|
||||||
|
}()
|
||||||
|
go controller.Run(stopCh)
|
||||||
|
|
||||||
|
// Check if the PodToleratesNodeTaints event is found
|
||||||
|
for start := time.Now(); time.Since(start) < 4*time.Minute; time.Sleep(2 * time.Second) {
|
||||||
|
if eventFound {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Expect(eventFound).Should(Equal(true))
|
||||||
|
})
|
||||||
|
|
||||||
|
AfterEach(func() {
|
||||||
|
By("delete the static pod")
|
||||||
|
err := deleteStaticPod(framework.TestContext.ManifestPath, staticPodName, ns)
|
||||||
|
Expect(err).ShouldNot(HaveOccurred())
|
||||||
|
|
||||||
|
By("clear taint")
|
||||||
|
framework.RemoveTaintOffNode(f.Client, nodeName, taint)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
Loading…
Reference in New Issue
Block a user