From 72a19819a6164c6b8a76fd599e195340708243a5 Mon Sep 17 00:00:00 2001 From: Kevin Date: Wed, 4 Jan 2017 19:45:50 +0800 Subject: [PATCH] api changes of forgiveness phase1 --- pkg/api/types.go | 37 ++++--- pkg/api/v1/types.go | 37 ++++--- pkg/api/validation/validation.go | 28 +++-- pkg/api/validation/validation_test.go | 104 +++++++++++++++++- .../pkg/apis/meta/v1/well_known_labels.go | 11 ++ 5 files changed, 174 insertions(+), 43 deletions(-) diff --git a/pkg/api/types.go b/pkg/api/types.go index 0dac64e8fdb..334cd507da4 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -1783,8 +1783,12 @@ type Taint struct { Value string // Required. The effect of the taint on pods // that do not tolerate the taint. - // Valid effects are NoSchedule and PreferNoSchedule. + // Valid effects are NoSchedule, PreferNoSchedule and NoExecute. Effect TaintEffect + // TimeAdded represents the time at which the taint was added. + // It is only written for NoExecute taints. + // +optional + TimeAdded metav1.Time } type TaintEffect string @@ -1800,26 +1804,23 @@ const ( // onto the node entirely. Enforced by the scheduler. TaintEffectPreferNoSchedule TaintEffect = "PreferNoSchedule" // NOT YET IMPLEMENTED. TODO: Uncomment field once it is implemented. - // Do not allow new pods to schedule onto the node unless they tolerate the taint, - // do not allow pods to start on Kubelet unless they tolerate the taint, - // but allow all already-running pods to continue running. - // Enforced by the scheduler and Kubelet. + // Like TaintEffectNoSchedule, but additionally do not allow pods submitted to + // Kubelet without going through the scheduler to start. + // Enforced by Kubelet and the scheduler. // TaintEffectNoScheduleNoAdmit TaintEffect = "NoScheduleNoAdmit" - // NOT YET IMPLEMENTED. TODO: Uncomment field once it is implemented. - // Do not allow new pods to schedule onto the node unless they tolerate the taint, - // do not allow pods to start on Kubelet unless they tolerate the taint, - // and evict any already-running pods that do not tolerate the taint. - // Enforced by the scheduler and Kubelet. - // TaintEffectNoScheduleNoAdmitNoExecute = "NoScheduleNoAdmitNoExecute" + // Evict any already-running pods that do not tolerate the taint. + // Currently enforced by NodeController. + TaintEffectNoExecute TaintEffect = "NoExecute" ) // The pod this Toleration is attached to tolerates any taint that matches // the triple using the matching operator . type Toleration struct { - // Required. Key is the taint key that the toleration applies to. + // Key is the taint key that the toleration applies to. Empty means match all taint keys. + // If the key is empty, operator must be Exists; this combination means to match all values and all keys. // +optional Key string - // operator represents a key's relationship to the value. + // Operator represents a key's relationship to the value. // Valid operators are Exists and Equal. Defaults to Equal. // Exists is equivalent to wildcard for value, so that a pod can // tolerate all taints of a particular category. @@ -1830,11 +1831,15 @@ type Toleration struct { // +optional Value string // Effect indicates the taint effect to match. Empty means match all taint effects. - // When specified, allowed values are NoSchedule and PreferNoSchedule. + // When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. // +optional Effect TaintEffect - // TODO: For forgiveness (#1574), we'd eventually add at least a grace period - // here, and possibly an occurrence threshold and period. + // TolerationSeconds represents the period of time the toleration (which must be + // of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + // it is not set, which means tolerate the taint forever (do not evict). Zero and + // negative values will be treated as 0 (evict immediately) by the system. + // +optional + TolerationSeconds *int64 } // A toleration operator is the set of operators that can be used in a toleration. diff --git a/pkg/api/v1/types.go b/pkg/api/v1/types.go index 297dda69e53..c664ebdfd2d 100644 --- a/pkg/api/v1/types.go +++ b/pkg/api/v1/types.go @@ -2002,8 +2002,12 @@ type Taint struct { Value string `json:"value,omitempty" protobuf:"bytes,2,opt,name=value"` // Required. The effect of the taint on pods // that do not tolerate the taint. - // Valid effects are NoSchedule and PreferNoSchedule. + // Valid effects are NoSchedule, PreferNoSchedule and NoExecute. Effect TaintEffect `json:"effect" protobuf:"bytes,3,opt,name=effect,casttype=TaintEffect"` + // TimeAdded represents the time at which the taint was added. + // It is only written for NoExecute taints. + // +optional + TimeAdded metav1.Time `json:"timeAdded,omitempty" protobuf:"bytes,4,opt,name=timeAdded"` } type TaintEffect string @@ -2019,26 +2023,23 @@ const ( // onto the node entirely. Enforced by the scheduler. TaintEffectPreferNoSchedule TaintEffect = "PreferNoSchedule" // NOT YET IMPLEMENTED. TODO: Uncomment field once it is implemented. - // Do not allow new pods to schedule onto the node unless they tolerate the taint, - // do not allow pods to start on Kubelet unless they tolerate the taint, - // but allow all already-running pods to continue running. - // Enforced by the scheduler and Kubelet. + // Like TaintEffectNoSchedule, but additionally do not allow pods submitted to + // Kubelet without going through the scheduler to start. + // Enforced by Kubelet and the scheduler. // TaintEffectNoScheduleNoAdmit TaintEffect = "NoScheduleNoAdmit" - // NOT YET IMPLEMENTED. TODO: Uncomment field once it is implemented. - // Do not allow new pods to schedule onto the node unless they tolerate the taint, - // do not allow pods to start on Kubelet unless they tolerate the taint, - // and evict any already-running pods that do not tolerate the taint. - // Enforced by the scheduler and Kubelet. - // TaintEffectNoScheduleNoAdmitNoExecute = "NoScheduleNoAdmitNoExecute" + // Evict any already-running pods that do not tolerate the taint. + // Currently enforced by NodeController. + TaintEffectNoExecute TaintEffect = "NoExecute" ) // The pod this Toleration is attached to tolerates any taint that matches // the triple using the matching operator . type Toleration struct { - // Required. Key is the taint key that the toleration applies to. + // Key is the taint key that the toleration applies to. Empty means match all taint keys. + // If the key is empty, operator must be Exists; this combination means to match all values and all keys. // +optional Key string `json:"key,omitempty" patchStrategy:"merge" patchMergeKey:"key" protobuf:"bytes,1,opt,name=key"` - // operator represents a key's relationship to the value. + // Operator represents a key's relationship to the value. // Valid operators are Exists and Equal. Defaults to Equal. // Exists is equivalent to wildcard for value, so that a pod can // tolerate all taints of a particular category. @@ -2049,11 +2050,15 @@ type Toleration struct { // +optional Value string `json:"value,omitempty" protobuf:"bytes,3,opt,name=value"` // Effect indicates the taint effect to match. Empty means match all taint effects. - // When specified, allowed values are NoSchedule and PreferNoSchedule. + // When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. // +optional Effect TaintEffect `json:"effect,omitempty" protobuf:"bytes,4,opt,name=effect,casttype=TaintEffect"` - // TODO: For forgiveness (#1574), we'd eventually add at least a grace period - // here, and possibly an occurrence threshold and period. + // TolerationSeconds represents the period of time the toleration (which must be + // of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + // it is not set, which means tolerate the taint forever (do not evict). Zero and + // negative values will be treated as 0 (evict immediately) by the system. + // +optional + TolerationSeconds *int64 `json:"tolerationSeconds,omitempty" protobuf:"varint,5,opt,name=tolerationSeconds"` } // A toleration operator is the set of operators that can be used in a toleration. diff --git a/pkg/api/validation/validation.go b/pkg/api/validation/validation.go index eefc981e3fb..8265c23e0c5 100644 --- a/pkg/api/validation/validation.go +++ b/pkg/api/validation/validation.go @@ -1717,16 +1717,16 @@ func validateTaintEffect(effect *api.TaintEffect, allowEmpty bool, fldPath *fiel allErrors := field.ErrorList{} switch *effect { - // TODO: Replace next line with subsequent commented-out line when implement TaintEffectNoScheduleNoAdmit, TaintEffectNoScheduleNoAdmitNoExecute. - case api.TaintEffectNoSchedule, api.TaintEffectPreferNoSchedule: - // case api.TaintEffectNoSchedule, api.TaintEffectPreferNoSchedule, api.TaintEffectNoScheduleNoAdmit, api.TaintEffectNoScheduleNoAdmitNoExecute: + // TODO: Replace next line with subsequent commented-out line when implement TaintEffectNoScheduleNoAdmit. + case api.TaintEffectNoSchedule, api.TaintEffectPreferNoSchedule, api.TaintEffectNoExecute: + // case api.TaintEffectNoSchedule, api.TaintEffectPreferNoSchedule, api.TaintEffectNoScheduleNoAdmit, api.TaintEffectNoExecute: default: validValues := []string{ string(api.TaintEffectNoSchedule), string(api.TaintEffectPreferNoSchedule), - // TODO: Uncomment this block when implement TaintEffectNoScheduleNoAdmit, TaintEffectNoScheduleNoAdmitNoExecute. + string(api.TaintEffectNoExecute), + // TODO: Uncomment this block when implement TaintEffectNoScheduleNoAdmit. // string(api.TaintEffectNoScheduleNoAdmit), - // string(api.TaintEffectNoScheduleNoAdmitNoExecute), } allErrors = append(allErrors, field.NotSupported(fldPath, effect, validValues)) } @@ -1739,10 +1739,24 @@ func validateTolerations(tolerations []api.Toleration, fldPath *field.Path) fiel for i, toleration := range tolerations { idxPath := fldPath.Index(i) // validate the toleration key - allErrors = append(allErrors, unversionedvalidation.ValidateLabelName(toleration.Key, idxPath.Child("key"))...) + if len(toleration.Key) > 0 { + allErrors = append(allErrors, unversionedvalidation.ValidateLabelName(toleration.Key, idxPath.Child("key"))...) + } + + // empty toleration key with Exists operator and empty value means match all taints + if len(toleration.Key) == 0 && toleration.Operator != api.TolerationOpExists { + allErrors = append(allErrors, field.Invalid(idxPath.Child("operator"), toleration.Operator, + "operator must be Exists when `key` is empty, which means \"match all values and all keys\"")) + } + + if toleration.TolerationSeconds != nil && toleration.Effect != api.TaintEffectNoExecute { + allErrors = append(allErrors, field.Invalid(idxPath.Child("effect"), toleration.Effect, + "effect must be 'NoExecute' when `tolerationSeconds` is set")) + } // validate toleration operator and value switch toleration.Operator { + // empty operator means Equal case api.TolerationOpEqual, "": if errs := validation.IsValidLabelValue(toleration.Value); len(errs) != 0 { allErrors = append(allErrors, field.Invalid(idxPath.Child("operator"), toleration.Value, strings.Join(errs, ";"))) @@ -1756,7 +1770,7 @@ func validateTolerations(tolerations []api.Toleration, fldPath *field.Path) fiel allErrors = append(allErrors, field.NotSupported(idxPath.Child("operator"), toleration.Operator, validValues)) } - // validate toleration effect + // validate toleration effect, empty toleration effect means match all taint effects if len(toleration.Effect) > 0 { allErrors = append(allErrors, validateTaintEffect(&toleration.Effect, true, idxPath.Child("effect"))...) } diff --git a/pkg/api/validation/validation_test.go b/pkg/api/validation/validation_test.go index 8e5382b1c8b..9209b69a6be 100644 --- a/pkg/api/validation/validation_test.go +++ b/pkg/api/validation/validation_test.go @@ -3378,6 +3378,40 @@ func TestValidatePod(t *testing.T) { }, }), }, + { // populate forgiveness tolerations with exists operator in annotations. + ObjectMeta: metav1.ObjectMeta{ + Name: "123", + Namespace: "ns", + Annotations: map[string]string{ + api.TolerationsAnnotationKey: ` + [{ + "key": "foo", + "operator": "Exists", + "value": "", + "effect": "NoExecute", + "tolerationSeconds": 60 + }]`, + }, + }, + Spec: validPodSpec(nil), + }, + { // populate forgiveness tolerations with equal operator in annotations. + ObjectMeta: metav1.ObjectMeta{ + Name: "123", + Namespace: "ns", + Annotations: map[string]string{ + api.TolerationsAnnotationKey: ` + [{ + "key": "foo", + "operator": "Equal", + "value": "bar", + "effect": "NoExecute", + "tolerationSeconds": 60 + }]`, + }, + }, + Spec: validPodSpec(nil), + }, { // populate tolerations equal operator in annotations. ObjectMeta: metav1.ObjectMeta{ Name: "123", @@ -3409,7 +3443,21 @@ func TestValidatePod(t *testing.T) { }, Spec: validPodSpec(nil), }, - { // empty operator is ok for toleration + { // empty key with Exists operator is OK for toleration, empty toleration key means match all taint keys. + ObjectMeta: metav1.ObjectMeta{ + Name: "123", + Namespace: "ns", + Annotations: map[string]string{ + api.TolerationsAnnotationKey: ` + [{ + "operator": "Exists", + "effect": "NoSchedule" + }]`, + }, + }, + Spec: validPodSpec(nil), + }, + { // empty operator is OK for toleration, defaults to Equal. ObjectMeta: metav1.ObjectMeta{ Name: "123", Namespace: "ns", @@ -3424,7 +3472,7 @@ func TestValidatePod(t *testing.T) { }, Spec: validPodSpec(nil), }, - { // empty efffect is ok for toleration + { // empty effect is OK for toleration, empty toleration effect means match all taint effects. ObjectMeta: metav1.ObjectMeta{ Name: "123", Namespace: "ns", @@ -3439,6 +3487,22 @@ func TestValidatePod(t *testing.T) { }, Spec: validPodSpec(nil), }, + { // negative tolerationSeconds is OK for toleration. + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-forgiveness-invalid", + Namespace: "ns", + Annotations: map[string]string{ + api.TolerationsAnnotationKey: ` + [{ + "key": "node.alpha.kubernetes.io/notReady", + "operator": "Exists", + "effect": "NoExecute", + "tolerationSeconds": -2 + }]`, + }, + }, + Spec: validPodSpec(nil), + }, { // docker default seccomp profile ObjectMeta: metav1.ObjectMeta{ Name: "123", @@ -3898,6 +3962,38 @@ func TestValidatePod(t *testing.T) { }, Spec: validPodSpec(nil), }, + + "operator must be 'Exists' when `key` is empty": { + ObjectMeta: metav1.ObjectMeta{ + Name: "123", + Namespace: "ns", + Annotations: map[string]string{ + api.TolerationsAnnotationKey: ` + [{ + "operator": "Equal", + "value": "bar", + "effect": "NoSchedule" + }]`, + }, + }, + Spec: validPodSpec(nil), + }, + "effect must be 'NoExecute' when `TolerationSeconds` is set": { + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-forgiveness-invalid", + Namespace: "ns", + Annotations: map[string]string{ + api.TolerationsAnnotationKey: ` + [{ + "key": "node.alpha.kubernetes.io/notReady", + "operator": "Exists", + "effect": "NoSchedule", + "tolerationSeconds": 20 + }]`, + }, + }, + Spec: validPodSpec(nil), + }, "must be a valid pod seccomp profile": { ObjectMeta: metav1.ObjectMeta{ Name: "123", @@ -5927,7 +6023,7 @@ func TestValidateNode(t *testing.T) { ExternalID: "external", }, }, - "invalide-taint-effect": { + "invalid-taint-effect": { ObjectMeta: metav1.ObjectMeta{ Name: "dedicated-node3", // Add a taint with an empty effect to a node @@ -5936,7 +6032,7 @@ func TestValidateNode(t *testing.T) { [{ "key": "dedicated", "value": "special-user-3", - "effect": "NoExecute" + "effect": "NoScheduleNoAdmit" }]`, }, }, diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/well_known_labels.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/well_known_labels.go index da003cd6568..a84ed35edb8 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/well_known_labels.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/well_known_labels.go @@ -33,6 +33,17 @@ const ( // of fluentd running on a node, kubelet need to mark node on which // fluentd in not running as a manifest pod with LabelFluentdDsReady. LabelFluentdDsReady = "alpha.kubernetes.io/fluentd-ds-ready" + + // When the --use-taint-based-evictions flag is enabled, + // TaintNodeNotReady would be automatically added by node controller + // when node is not ready, and removed when node becomes ready. + TaintNodeNotReady = "node.alpha.kubernetes.io/notReady" + + // When the --use-taint-based-evictions flag is enabled, + // TaintNodeUnreachable would be automatically added by node controller + // when node becomes unreachable (corresponding to NodeReady status ConditionUnknown) + // and removed when node becomes reachable (NodeReady status ConditionTrue). + TaintNodeUnreachable = "node.alpha.kubernetes.io/unreachable" ) // Role labels are applied to Nodes to mark their purpose. In particular, we