diff --git a/cmd/kubeadm/app/master/selfhosted.go b/cmd/kubeadm/app/master/selfhosted.go index b3d4df8cdad..5708f696211 100644 --- a/cmd/kubeadm/app/master/selfhosted.go +++ b/cmd/kubeadm/app/master/selfhosted.go @@ -17,7 +17,6 @@ limitations under the License. package master import ( - "encoding/json" "fmt" "os" "path" @@ -203,9 +202,6 @@ func getAPIServerDS(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, vo "component": kubeAPIServer, "tier": "control-plane", }, - Annotations: map[string]string{ - v1.TolerationsAnnotationKey: getMasterToleration(), - }, }, Spec: v1.PodSpec{ NodeSelector: map[string]string{metav1.NodeLabelKubeadmAlphaRole: metav1.NodeLabelRoleMaster}, @@ -222,6 +218,7 @@ func getAPIServerDS(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, vo Resources: componentResources("250m"), }, }, + Tolerations: getMasterToleration(), }, }, }, @@ -256,9 +253,6 @@ func getControllerManagerDeployment(cfg *kubeadmapi.MasterConfiguration, volumes "component": kubeControllerManager, "tier": "control-plane", }, - Annotations: map[string]string{ - v1.TolerationsAnnotationKey: getMasterToleration(), - }, }, Spec: v1.PodSpec{ NodeSelector: map[string]string{metav1.NodeLabelKubeadmAlphaRole: metav1.NodeLabelRoleMaster}, @@ -275,7 +269,8 @@ func getControllerManagerDeployment(cfg *kubeadmapi.MasterConfiguration, volumes Env: getProxyEnvVars(), }, }, - DNSPolicy: v1.DNSDefault, + Tolerations: getMasterToleration(), + DNSPolicy: v1.DNSDefault, }, }, }, @@ -310,9 +305,6 @@ func getSchedulerDeployment(cfg *kubeadmapi.MasterConfiguration) ext.Deployment "component": kubeScheduler, "tier": "control-plane", }, - Annotations: map[string]string{ - v1.TolerationsAnnotationKey: getMasterToleration(), - }, }, Spec: v1.PodSpec{ NodeSelector: map[string]string{metav1.NodeLabelKubeadmAlphaRole: metav1.NodeLabelRoleMaster}, @@ -327,6 +319,7 @@ func getSchedulerDeployment(cfg *kubeadmapi.MasterConfiguration) ext.Deployment Env: getProxyEnvVars(), }, }, + Tolerations: getMasterToleration(), }, }, }, @@ -338,15 +331,14 @@ func buildStaticManifestFilepath(name string) string { return path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "manifests", name+".yaml") } -func getMasterToleration() string { +func getMasterToleration() []v1.Toleration { // Tolerate the master taint we add to our master nodes, as this can and should // run there. // TODO: Duplicated above - masterToleration, _ := json.Marshal([]v1.Toleration{{ + return []v1.Toleration{{ Key: "dedicated", Value: "master", Operator: v1.TolerationOpEqual, Effect: v1.TaintEffectNoSchedule, - }}) - return string(masterToleration) + }} } diff --git a/cmd/kubeadm/app/master/templates.go b/cmd/kubeadm/app/master/templates.go index 97b45bdb039..fbdd748e252 100644 --- a/cmd/kubeadm/app/master/templates.go +++ b/cmd/kubeadm/app/master/templates.go @@ -59,9 +59,6 @@ spec: k8s-app: kube-discovery # TODO: I guess we can remove all these cluster-service labels... kubernetes.io/cluster-service: "true" - annotations: - # TODO: Move this to the beta tolerations field below as soon as the Tolerations field exists in PodSpec - scheduler.alpha.kubernetes.io/tolerations: '[{"key":"dedicated","value":"master","effect":"NoSchedule"}]' spec: containers: - name: kube-discovery @@ -78,10 +75,10 @@ spec: name: clusterinfo readOnly: true hostNetwork: true - # tolerations: - # - key: dedicated - # value: master - # effect: NoSchedule + tolerations: + - key: "dedicated" + value: "master" + effect: "NoSchedule" securityContext: seLinuxOptions: type: spc_t diff --git a/cmd/kubeadm/app/phases/addons/manifests.go b/cmd/kubeadm/app/phases/addons/manifests.go index 44d9cf297ad..0b716ad793e 100644 --- a/cmd/kubeadm/app/phases/addons/manifests.go +++ b/cmd/kubeadm/app/phases/addons/manifests.go @@ -63,9 +63,6 @@ spec: metadata: labels: k8s-app: kube-proxy - annotations: - # TODO: Move this to the beta tolerations field below as soon as the Tolerations field exists in PodSpec - scheduler.alpha.kubernetes.io/tolerations: '[{"key":"dedicated","value":"master","effect":"NoSchedule"}]' spec: containers: - name: kube-proxy @@ -82,11 +79,10 @@ spec: name: kube-proxy hostNetwork: true serviceAccountName: kube-proxy - # Tolerate running on the master - # tolerations: - # - key: dedicated - # value: master - # effect: NoSchedule + tolerations: + - key: dedicated + value: master + effect: NoSchedule volumes: - name: kube-proxy configMap: @@ -122,8 +118,6 @@ spec: k8s-app: kube-dns annotations: scheduler.alpha.kubernetes.io/critical-pod: '' - # TODO: Move this to the beta tolerations field below as soon as the Tolerations field exists in PodSpec - scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}, {"key":"dedicated","value":"master","effect":"NoSchedule"}]' spec: volumes: - name: kube-dns-config @@ -242,10 +236,12 @@ spec: cpu: 10m dnsPolicy: Default # Don't use cluster DNS. serviceAccountName: kube-dns - # tolerations: - # - key: dedicated - # value: master - # effect: NoSchedule + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - key: "dedicated" + value: "master" + effect: "NoSchedule" # TODO: Remove this affinity field as soon as we are using manifest lists affinity: nodeAffinity: diff --git a/cmd/kubeadm/app/phases/apiconfig/setupmaster.go b/cmd/kubeadm/app/phases/apiconfig/setupmaster.go index 39c056009a0..02d9f6d1522 100644 --- a/cmd/kubeadm/app/phases/apiconfig/setupmaster.go +++ b/cmd/kubeadm/app/phases/apiconfig/setupmaster.go @@ -61,9 +61,7 @@ func attemptToUpdateMasterRoleLabelsAndTaints(client *clientset.Clientset) error // TODO: Switch to the new master label defined in https://github.com/kubernetes/kubernetes/pull/39112 n.ObjectMeta.Labels[metav1.NodeLabelKubeadmAlphaRole] = metav1.NodeLabelRoleMaster - // TODO: Use the Taints beta field on the NodeSpec now - taintsAnnotation, _ := json.Marshal([]v1.Taint{{Key: "dedicated", Value: "master", Effect: "NoSchedule"}}) - n.ObjectMeta.Annotations[v1.TaintsAnnotationKey] = string(taintsAnnotation) + n.Spec.Taints = []v1.Taint{{Key: "dedicated", Value: "master", Effect: "NoSchedule"}} newData, err := json.Marshal(n) if err != nil { diff --git a/pkg/api/helpers.go b/pkg/api/helpers.go index 256cfbcd4fe..6672ba2e288 100644 --- a/pkg/api/helpers.go +++ b/pkg/api/helpers.go @@ -429,14 +429,6 @@ func NodeSelectorRequirementsAsSelector(nsm []NodeSelectorRequirement) (labels.S } const ( - // TolerationsAnnotationKey represents the key of tolerations data (json serialized) - // in the Annotations of a Pod. - TolerationsAnnotationKey string = "scheduler.alpha.kubernetes.io/tolerations" - - // TaintsAnnotationKey represents the key of taints data (json serialized) - // in the Annotations of a Node. - TaintsAnnotationKey string = "scheduler.alpha.kubernetes.io/taints" - // SeccompPodAnnotationKey represents the key of a seccomp profile applied // to all containers of a pod. SeccompPodAnnotationKey string = "seccomp.security.alpha.kubernetes.io/pod" diff --git a/pkg/api/json.go b/pkg/api/json.go index fbc7444e472..3a6e04c182f 100644 --- a/pkg/api/json.go +++ b/pkg/api/json.go @@ -21,18 +21,6 @@ import "encoding/json" // This file implements json marshaling/unmarshaling interfaces on objects that are currently marshaled into annotations // to prevent anyone from marshaling these internal structs. -var _ = json.Marshaler(Taint{}) -var _ = json.Unmarshaler(&Taint{}) - -func (Taint) MarshalJSON() ([]byte, error) { panic("do not marshal internal struct") } -func (*Taint) UnmarshalJSON([]byte) error { panic("do not unmarshal to internal struct") } - -var _ = json.Marshaler(Toleration{}) -var _ = json.Unmarshaler(&Toleration{}) - -func (Toleration) MarshalJSON() ([]byte, error) { panic("do not marshal internal struct") } -func (*Toleration) UnmarshalJSON([]byte) error { panic("do not unmarshal to internal struct") } - var _ = json.Marshaler(&AvoidPods{}) var _ = json.Unmarshaler(&AvoidPods{}) diff --git a/pkg/api/types.go b/pkg/api/types.go index 6a0d4a99c78..e0527e7115d 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -2001,6 +2001,9 @@ type PodSpec struct { // If not specified, the pod will be dispatched by default scheduler. // +optional SchedulerName string + // If specified, the pod's tolerations. + // +optional + Tolerations []Toleration } // Sysctl defines a kernel parameter to be set @@ -2623,6 +2626,10 @@ type NodeSpec struct { // Unschedulable controls node schedulability of new pods. By default node is schedulable. // +optional Unschedulable bool + + // If specified, the node's taints. + // +optional + Taints []Taint } // DaemonEndpoint contains information about a single Daemon endpoint. diff --git a/pkg/api/v1/helpers.go b/pkg/api/v1/helpers.go index 4608399043a..2bb238cacf6 100644 --- a/pkg/api/v1/helpers.go +++ b/pkg/api/v1/helpers.go @@ -235,14 +235,6 @@ func NodeSelectorRequirementsAsSelector(nsm []NodeSelectorRequirement) (labels.S } const ( - // TolerationsAnnotationKey represents the key of tolerations data (json serialized) - // in the Annotations of a Pod. - TolerationsAnnotationKey string = "scheduler.alpha.kubernetes.io/tolerations" - - // TaintsAnnotationKey represents the key of taints data (json serialized) - // in the Annotations of a Node. - TaintsAnnotationKey string = "scheduler.alpha.kubernetes.io/taints" - // SeccompPodAnnotationKey represents the key of a seccomp profile applied // to all containers of a pod. SeccompPodAnnotationKey string = "seccomp.security.alpha.kubernetes.io/pod" @@ -284,79 +276,34 @@ const ( AffinityAnnotationKey string = "scheduler.alpha.kubernetes.io/affinity" ) -// GetTolerationsFromPodAnnotations gets the json serialized tolerations data from Pod.Annotations -// and converts it to the []Toleration type in api. -func GetTolerationsFromPodAnnotations(annotations map[string]string) ([]Toleration, error) { - var tolerations []Toleration - if len(annotations) > 0 && annotations[TolerationsAnnotationKey] != "" { - err := json.Unmarshal([]byte(annotations[TolerationsAnnotationKey]), &tolerations) - if err != nil { - return tolerations, err - } - } - return tolerations, nil -} - -func GetPodTolerations(pod *Pod) ([]Toleration, error) { - return GetTolerationsFromPodAnnotations(pod.Annotations) -} - // Tries to add a toleration to annotations list. Returns true if something was updated // false otherwise. func AddOrUpdateTolerationInPod(pod *Pod, toleration *Toleration) (bool, error) { - podTolerations, err := GetPodTolerations(pod) - if err != nil { - return false, err - } + podTolerations := pod.Spec.Tolerations - var newTolerations []*Toleration + var newTolerations []Toleration updated := false for i := range podTolerations { if toleration.MatchToleration(&podTolerations[i]) { if api.Semantic.DeepEqual(toleration, podTolerations[i]) { return false, nil } - newTolerations = append(newTolerations, toleration) + newTolerations = append(newTolerations, *toleration) updated = true continue } - newTolerations = append(newTolerations, &podTolerations[i]) + newTolerations = append(newTolerations, podTolerations[i]) } if !updated { - newTolerations = append(newTolerations, toleration) + newTolerations = append(newTolerations, *toleration) } - tolerationsData, err := json.Marshal(newTolerations) - if err != nil { - return false, err - } - - if pod.Annotations == nil { - pod.Annotations = make(map[string]string) - } - pod.Annotations[TolerationsAnnotationKey] = string(tolerationsData) + pod.Spec.Tolerations = newTolerations return true, nil } -// GetTaintsFromNodeAnnotations gets the json serialized taints data from Pod.Annotations -// and converts it to the []Taint type in api. -func GetTaintsFromNodeAnnotations(annotations map[string]string) ([]Taint, error) { - var taints []Taint - if len(annotations) > 0 && annotations[TaintsAnnotationKey] != "" { - err := json.Unmarshal([]byte(annotations[TaintsAnnotationKey]), &taints) - if err != nil { - return []Taint{}, err - } - } - return taints, nil -} - -func GetNodeTaints(node *Node) ([]Taint, error) { - return GetTaintsFromNodeAnnotations(node.Annotations) -} - // MatchToleration checks if the toleration matches tolerationToMatch. Tolerations are unique by , // if the two tolerations have same combination, regard as they match. // TODO: uniqueness check for tolerations in api validations. @@ -574,39 +521,28 @@ func AddOrUpdateTaint(node *Node, taint *Taint) (*Node, bool, error) { return nil, false, err } newNode := objCopy.(*Node) - nodeTaints, err := GetTaintsFromNodeAnnotations(newNode.Annotations) - if err != nil { - return newNode, false, err - } + nodeTaints := newNode.Spec.Taints - var newTaints []*Taint + var newTaints []Taint updated := false for i := range nodeTaints { if taint.MatchTaint(&nodeTaints[i]) { if api.Semantic.DeepEqual(taint, nodeTaints[i]) { return newNode, false, nil } - newTaints = append(newTaints, taint) + newTaints = append(newTaints, *taint) updated = true continue } - newTaints = append(newTaints, &nodeTaints[i]) + newTaints = append(newTaints, nodeTaints[i]) } if !updated { - newTaints = append(newTaints, taint) + newTaints = append(newTaints, *taint) } - taintsData, err := json.Marshal(newTaints) - if err != nil { - return nil, false, err - } - - if newNode.Annotations == nil { - newNode.Annotations = make(map[string]string) - } - newNode.Annotations[TaintsAnnotationKey] = string(taintsData) + newNode.Spec.Taints = newTaints return newNode, true, nil } @@ -627,10 +563,7 @@ func RemoveTaint(node *Node, taint *Taint) (*Node, bool, error) { return nil, false, err } newNode := objCopy.(*Node) - nodeTaints, err := GetTaintsFromNodeAnnotations(newNode.Annotations) - if err != nil { - return newNode, false, err - } + nodeTaints := newNode.Spec.Taints if len(nodeTaints) == 0 { return newNode, false, nil } @@ -640,15 +573,7 @@ func RemoveTaint(node *Node, taint *Taint) (*Node, bool, error) { } newTaints, _ := DeleteTaint(nodeTaints, taint) - if len(newTaints) == 0 { - delete(newNode.Annotations, TaintsAnnotationKey) - } else { - taintsData, err := json.Marshal(newTaints) - if err != nil { - return newNode, false, err - } - newNode.Annotations[TaintsAnnotationKey] = string(taintsData) - } + newNode.Spec.Taints = newTaints return newNode, true, nil } diff --git a/pkg/api/v1/types.go b/pkg/api/v1/types.go index f0307b02425..9304ad40eaf 100644 --- a/pkg/api/v1/types.go +++ b/pkg/api/v1/types.go @@ -2286,6 +2286,9 @@ type PodSpec struct { // If not specified, the pod will be dispatched by default scheduler. // +optional SchedulerName string `json:"schedulername,omitempty" protobuf:"bytes,19,opt,name=schedulername"` + // If specified, the pod's tolerations. + // +optional + Tolerations []Toleration `json:"tolerations,omitempty" protobuf:"bytes,22,opt,name=tolerations"` } // PodSecurityContext holds pod-level security attributes and common container settings. @@ -3022,6 +3025,9 @@ type NodeSpec struct { // More info: http://releases.k8s.io/HEAD/docs/admin/node.md#manual-node-administration // +optional Unschedulable bool `json:"unschedulable,omitempty" protobuf:"varint,4,opt,name=unschedulable"` + // If specified, the node's taints. + // +optional + Taints []Taint `json:"taints,omitempty" protobuf:"bytes,5,opt,name=taints"` } // DaemonEndpoint contains information about a single Daemon endpoint. diff --git a/pkg/api/validation/validation.go b/pkg/api/validation/validation.go index 01474a2e80f..880f15fb4c8 100644 --- a/pkg/api/validation/validation.go +++ b/pkg/api/validation/validation.go @@ -105,9 +105,6 @@ func ValidateDNS1123Subdomain(value string, fldPath *field.Path) field.ErrorList func ValidatePodSpecificAnnotations(annotations map[string]string, spec *api.PodSpec, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} - if annotations[api.TolerationsAnnotationKey] != "" { - allErrs = append(allErrs, ValidateTolerationsInPodAnnotations(annotations, fldPath)...) - } // TODO: remove these after we EOL the annotations. if hostname, exists := annotations[utilpod.PodHostnameAnnotation]; exists { @@ -1948,6 +1945,10 @@ func ValidatePodSpec(spec *api.PodSpec, fldPath *field.Path) field.ErrorList { allErrs = append(allErrs, ValidateDNS1123Label(spec.Subdomain, fldPath.Child("subdomain"))...) } + if len(spec.Tolerations) > 0 { + allErrs = append(allErrs, validateTolerations(spec.Tolerations, fldPath.Child("tolerations"))...) + } + return allErrs } @@ -2139,29 +2140,6 @@ func validatePodAffinity(podAffinity *api.PodAffinity, fldPath *field.Path) fiel return allErrs } -// ValidateTolerationsInPodAnnotations tests that the serialized tolerations in Pod.Annotations has valid data -func ValidateTolerationsInPodAnnotations(annotations map[string]string, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - - v1Tolerations, err := v1.GetTolerationsFromPodAnnotations(annotations) - if err != nil { - allErrs = append(allErrs, field.Invalid(fldPath, api.TolerationsAnnotationKey, err.Error())) - return allErrs - } - tolerations := make([]api.Toleration, len(v1Tolerations)) - for i := range v1Tolerations { - if err := v1.Convert_v1_Toleration_To_api_Toleration(&v1Tolerations[i], &tolerations[i], nil); err != nil { - allErrs = append(allErrs, field.Invalid(fldPath, api.TolerationsAnnotationKey, err.Error())) - return allErrs - } - } - if len(tolerations) > 0 { - allErrs = append(allErrs, validateTolerations(tolerations, fldPath.Child(api.TolerationsAnnotationKey))...) - } - - return allErrs -} - func ValidateSeccompProfile(p string, fldPath *field.Path) field.ErrorList { if p == "docker/default" { return nil @@ -2813,8 +2791,8 @@ func ValidateReadOnlyPersistentDisks(volumes []api.Volume, fldPath *field.Path) return allErrs } -// validateTaints tests if given taints have valid data. -func validateTaints(taints []api.Taint, fldPath *field.Path) field.ErrorList { +// validateNodeTaints tests if given taints have valid data. +func validateNodeTaints(taints []api.Taint, fldPath *field.Path) field.ErrorList { allErrors := field.ErrorList{} uniqueTaints := map[api.TaintEffect]sets.String{} @@ -2847,37 +2825,11 @@ func validateTaints(taints []api.Taint, fldPath *field.Path) field.ErrorList { return allErrors } -// ValidateTaintsInNodeAnnotations tests that the serialized taints in Node.Annotations has valid data -func ValidateTaintsInNodeAnnotations(annotations map[string]string, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - - v1Taints, err := v1.GetTaintsFromNodeAnnotations(annotations) - if err != nil { - allErrs = append(allErrs, field.Invalid(fldPath, api.TaintsAnnotationKey, err.Error())) - return allErrs - } - taints := make([]api.Taint, len(v1Taints)) - for i := range v1Taints { - if err := v1.Convert_v1_Taint_To_api_Taint(&v1Taints[i], &taints[i], nil); err != nil { - allErrs = append(allErrs, field.Invalid(fldPath, api.TaintsAnnotationKey, err.Error())) - return allErrs - } - } - if len(taints) > 0 { - allErrs = append(allErrs, validateTaints(taints, fldPath.Child(api.TaintsAnnotationKey))...) - } - - return allErrs -} - func ValidateNodeSpecificAnnotations(annotations map[string]string, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} if annotations[api.PreferAvoidPodsAnnotationKey] != "" { allErrs = append(allErrs, ValidateAvoidPodsInNodeAnnotations(annotations, fldPath)...) } - if annotations[api.TaintsAnnotationKey] != "" { - allErrs = append(allErrs, ValidateTaintsInNodeAnnotations(annotations, fldPath)...) - } return allErrs } @@ -2886,6 +2838,9 @@ func ValidateNode(node *api.Node) field.ErrorList { fldPath := field.NewPath("metadata") allErrs := ValidateObjectMeta(&node.ObjectMeta, false, ValidateNodeName, fldPath) allErrs = append(allErrs, ValidateNodeSpecificAnnotations(node.ObjectMeta.Annotations, fldPath.Child("annotations"))...) + if len(node.Spec.Taints) > 0 { + allErrs = append(allErrs, validateNodeTaints(node.Spec.Taints, fldPath.Child("taints"))...) + } // Only validate spec. All status fields are optional and can be updated later. @@ -2948,10 +2903,16 @@ func ValidateNodeUpdate(node, oldNode *api.Node) field.ErrorList { // Clear status oldNode.Status = node.Status + // update taints + if len(node.Spec.Taints) > 0 { + allErrs = append(allErrs, validateNodeTaints(node.Spec.Taints, fldPath.Child("taints"))...) + } + oldNode.Spec.Taints = node.Spec.Taints + // TODO: Add a 'real' error type for this error and provide print actual diffs. if !apiequality.Semantic.DeepEqual(oldNode, node) { glog.V(4).Infof("Update failed validation %#v vs %#v", oldNode, node) - allErrs = append(allErrs, field.Forbidden(field.NewPath(""), "node updates may only change labels or capacity")) + allErrs = append(allErrs, field.Forbidden(field.NewPath(""), "node updates may only change labels, taints or capacity")) } return allErrs diff --git a/pkg/controller/node/taint_controller.go b/pkg/controller/node/taint_controller.go index f81d1049053..75b759d09c7 100644 --- a/pkg/controller/node/taint_controller.go +++ b/pkg/controller/node/taint_controller.go @@ -280,22 +280,13 @@ func (tc *NoExecuteTaintManager) Run(stopCh <-chan struct{}) { // PodUpdated is used to notify NoExecuteTaintManager about Pod changes. func (tc *NoExecuteTaintManager) PodUpdated(oldPod *v1.Pod, newPod *v1.Pod) { - var err error oldTolerations := []v1.Toleration{} if oldPod != nil { - oldTolerations, err = v1.GetPodTolerations(oldPod) - if err != nil { - glog.Errorf("Failed to get Tolerations from the old Pod: %v", err) - return - } + oldTolerations = oldPod.Spec.Tolerations } newTolerations := []v1.Toleration{} if newPod != nil { - newTolerations, err = v1.GetPodTolerations(newPod) - if err != nil { - glog.Errorf("Failed to get Tolerations from the new Pod: %v", err) - return - } + newTolerations = newPod.Spec.Tolerations } if oldPod != nil && newPod != nil && api.Semantic.DeepEqual(oldTolerations, newTolerations) && oldPod.Spec.NodeName == newPod.Spec.NodeName { @@ -312,24 +303,15 @@ func (tc *NoExecuteTaintManager) PodUpdated(oldPod *v1.Pod, newPod *v1.Pod) { // NodeUpdated is used to notify NoExecuteTaintManager about Node changes. func (tc *NoExecuteTaintManager) NodeUpdated(oldNode *v1.Node, newNode *v1.Node) { - var err error oldTaints := []v1.Taint{} if oldNode != nil { - oldTaints, err = v1.GetNodeTaints(oldNode) - if err != nil { - glog.Errorf("Failed to get Taints from the old Node: %v", err) - return - } + oldTaints = oldNode.Spec.Taints } oldTaints = getNoExecuteTaints(oldTaints) newTaints := []v1.Taint{} if newNode != nil { - newTaints, err = v1.GetNodeTaints(newNode) - if err != nil { - glog.Errorf("Failed to get Taints from the new Node: %v", err) - return - } + newTaints = newNode.Spec.Taints } newTaints = getNoExecuteTaints(newTaints) @@ -466,12 +448,7 @@ func (tc *NoExecuteTaintManager) handleNodeUpdate(nodeUpdate *nodeUpdateItem) { for i := range pods { pod := &pods[i] podNamespacedName := types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name} - tolerations, err := v1.GetPodTolerations(pod) - if err != nil { - glog.Errorf("Failed to get Tolerations from Pod %v: %v", podNamespacedName.String(), err) - continue - } - tc.processPodOnNode(podNamespacedName, node.Name, tolerations, taints, now) + tc.processPodOnNode(podNamespacedName, node.Name, pod.Spec.Tolerations, taints, now) } } diff --git a/pkg/kubectl/cmd/taint.go b/pkg/kubectl/cmd/taint.go index 0008b1de929..c606078c04b 100644 --- a/pkg/kubectl/cmd/taint.go +++ b/pkg/kubectl/cmd/taint.go @@ -25,8 +25,6 @@ import ( "github.com/golang/glog" "github.com/spf13/cobra" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" utilerrors "k8s.io/apimachinery/pkg/util/errors" @@ -115,18 +113,15 @@ func NewCmdTaint(f cmdutil.Factory, out io.Writer) *cobra.Command { // reorganizeTaints returns the updated set of taints, taking into account old taints that were not updated, // old taints that were updated, old taints that were deleted, and new taints. -func reorganizeTaints(accessor metav1.Object, overwrite bool, taintsToAdd []v1.Taint, taintsToRemove []v1.Taint) ([]v1.Taint, error) { - newTaints := append([]v1.Taint{}, taintsToAdd...) - - var oldTaints []v1.Taint - var err error - annotations := accessor.GetAnnotations() - if annotations != nil { - if oldTaints, err = v1.GetTaintsFromNodeAnnotations(annotations); err != nil { - return nil, err - } +func reorganizeTaints(obj runtime.Object, overwrite bool, taintsToAdd []v1.Taint, taintsToRemove []v1.Taint) ([]v1.Taint, error) { + node, ok := obj.(*v1.Node) + if !ok { + return nil, fmt.Errorf("unexpected type %T, expected Node", obj) } + newTaints := append([]v1.Taint{}, taintsToAdd...) + + oldTaints := node.Spec.Taints // add taints that already existing but not updated to newTaints for _, oldTaint := range oldTaints { existsInNew := false @@ -351,23 +346,18 @@ func (o TaintOptions) RunTaint() error { } // validateNoTaintOverwrites validates that when overwrite is false, to-be-updated taints don't exist in the node taint list (yet) -func validateNoTaintOverwrites(accessor metav1.Object, taints []v1.Taint) error { - annotations := accessor.GetAnnotations() - if annotations == nil { - return nil +func validateNoTaintOverwrites(obj runtime.Object, taints []v1.Taint) error { + node, ok := obj.(*v1.Node) + if !ok { + return fmt.Errorf("unexpected type %T, expected Node", obj) } allErrs := []error{} - oldTaints, err := v1.GetTaintsFromNodeAnnotations(annotations) - if err != nil { - allErrs = append(allErrs, err) - return utilerrors.NewAggregate(allErrs) - } - + oldTaints := node.Spec.Taints for _, taint := range taints { for _, oldTaint := range oldTaints { if taint.Key == oldTaint.Key && taint.Effect == oldTaint.Effect { - allErrs = append(allErrs, fmt.Errorf("Node '%s' already has a taint with key (%s) and effect (%v), and --overwrite is false", accessor.GetName(), taint.Key, taint.Effect)) + allErrs = append(allErrs, fmt.Errorf("Node '%s' already has a taint with key (%s) and effect (%v), and --overwrite is false", node.Name, taint.Key, taint.Effect)) break } } @@ -377,31 +367,22 @@ func validateNoTaintOverwrites(accessor metav1.Object, taints []v1.Taint) error // updateTaints updates taints of obj func (o TaintOptions) updateTaints(obj runtime.Object) error { - accessor, err := meta.Accessor(obj) - if err != nil { - return err - } if !o.overwrite { - if err := validateNoTaintOverwrites(accessor, o.taintsToAdd); err != nil { + if err := validateNoTaintOverwrites(obj, o.taintsToAdd); err != nil { return err } } - annotations := accessor.GetAnnotations() - if annotations == nil { - annotations = make(map[string]string) + newTaints, err := reorganizeTaints(obj, o.overwrite, o.taintsToAdd, o.taintsToRemove) + if err != nil { + return err } - newTaints, err := reorganizeTaints(accessor, o.overwrite, o.taintsToAdd, o.taintsToRemove) - if err != nil { - return err + node, ok := obj.(*v1.Node) + if !ok { + return fmt.Errorf("unexpected type %T, expected Node", obj) } - taintsData, err := json.Marshal(newTaints) - if err != nil { - return err - } - annotations[v1.TaintsAnnotationKey] = string(taintsData) - accessor.SetAnnotations(annotations) + node.Spec.Taints = newTaints return nil } diff --git a/pkg/kubectl/describe.go b/pkg/kubectl/describe.go index dcc4c566135..c6167a5aeae 100644 --- a/pkg/kubectl/describe.go +++ b/pkg/kubectl/describe.go @@ -45,7 +45,6 @@ import ( "k8s.io/kubernetes/pkg/api/annotations" "k8s.io/kubernetes/pkg/api/events" - "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/apis/apps" "k8s.io/kubernetes/pkg/apis/autoscaling" "k8s.io/kubernetes/pkg/apis/batch" @@ -582,7 +581,7 @@ func describePod(pod *api.Pod, events *api.EventList) (string, error) { describeVolumes(pod.Spec.Volumes, w, "") w.Write(LEVEL_0, "QoS Class:\t%s\n", pod.Status.QOSClass) printLabelsMultiline(w, "Node-Selectors", pod.Spec.NodeSelector) - printTolerationsInAnnotationMultiline(w, "Tolerations", pod.Annotations) + printPodTolerationsMultiline(w, "Tolerations", pod.Spec.Tolerations) if events != nil { DescribeEvents(events, w) } @@ -1987,7 +1986,7 @@ func describeNode(node *api.Node, nodeNonTerminatedPodsList *api.PodList, events w.Write(LEVEL_0, "Name:\t%s\n", node.Name) w.Write(LEVEL_0, "Role:\t%s\n", findNodeRole(node)) printLabelsMultiline(w, "Labels", node.Labels) - printTaintsInAnnotationMultiline(w, "Taints", node.Annotations) + printNodeTaintsMultiline(w, "Taints", node.Spec.Taints) w.Write(LEVEL_0, "CreationTimestamp:\t%s\n", node.CreationTimestamp.Time.Format(time.RFC1123Z)) w.Write(LEVEL_0, "Phase:\t%v\n", node.Status.Phase) if len(node.Status.Conditions) > 0 { @@ -2854,17 +2853,7 @@ func printLabelsMultilineWithIndent(w *PrefixWriter, initialIndent, title, inner } // printTaintsMultiline prints multiple taints with a proper alignment. -func printTaintsInAnnotationMultiline(w *PrefixWriter, title string, annotations map[string]string) { - v1Taints, err := v1.GetTaintsFromNodeAnnotations(annotations) - if err != nil { - v1Taints = []v1.Taint{} - } - taints := make([]api.Taint, len(v1Taints)) - for i := range v1Taints { - if err := v1.Convert_v1_Taint_To_api_Taint(&v1Taints[i], &taints[i], nil); err != nil { - panic(err) - } - } +func printNodeTaintsMultiline(w *PrefixWriter, title string, taints []api.Taint) { printTaintsMultilineWithIndent(w, "", title, "\t", taints) } @@ -2898,18 +2887,8 @@ func printTaintsMultilineWithIndent(w *PrefixWriter, initialIndent, title, inner } } -// printTolerationsMultiline prints multiple tolerations with a proper alignment. -func printTolerationsInAnnotationMultiline(w *PrefixWriter, title string, annotations map[string]string) { - v1Tolerations, err := v1.GetTolerationsFromPodAnnotations(annotations) - if err != nil { - v1Tolerations = []v1.Toleration{} - } - tolerations := make([]api.Toleration, len(v1Tolerations)) - for i := range v1Tolerations { - if err := v1.Convert_v1_Toleration_To_api_Toleration(&v1Tolerations[i], &tolerations[i], nil); err != nil { - panic(err) - } - } +// printPodTolerationsMultiline prints multiple tolerations with a proper alignment. +func printPodTolerationsMultiline(w *PrefixWriter, title string, tolerations []api.Toleration) { printTolerationsMultilineWithIndent(w, "", title, "\t", tolerations) } diff --git a/pkg/kubelet/kubelet_node_status.go b/pkg/kubelet/kubelet_node_status.go index 2148851cb81..b3064c26ab4 100644 --- a/pkg/kubelet/kubelet_node_status.go +++ b/pkg/kubelet/kubelet_node_status.go @@ -17,7 +17,6 @@ limitations under the License. package kubelet import ( - "encoding/json" "fmt" "math" "net" @@ -204,19 +203,13 @@ func (kl *Kubelet) initialNode() (*v1.Node, error) { }, } if len(kl.kubeletConfiguration.RegisterWithTaints) > 0 { - annotations := make(map[string]string) taints := make([]v1.Taint, len(kl.kubeletConfiguration.RegisterWithTaints)) for i := range kl.kubeletConfiguration.RegisterWithTaints { if err := v1.Convert_api_Taint_To_v1_Taint(&kl.kubeletConfiguration.RegisterWithTaints[i], &taints[i], nil); err != nil { return nil, err } } - b, err := json.Marshal(taints) - if err != nil { - return nil, err - } - annotations[v1.TaintsAnnotationKey] = string(b) - node.ObjectMeta.Annotations = annotations + node.Spec.Taints = taints } // Initially, set NodeNetworkUnavailable to true. diff --git a/plugin/pkg/admission/defaulttolerationseconds/admission.go b/plugin/pkg/admission/defaulttolerationseconds/admission.go index 0cd692cb8f3..ac20a7b249a 100644 --- a/plugin/pkg/admission/defaulttolerationseconds/admission.go +++ b/plugin/pkg/admission/defaulttolerationseconds/admission.go @@ -72,11 +72,7 @@ func (p *plugin) Admit(attributes admission.Attributes) (err error) { return nil } - tolerations, err := v1.GetPodTolerations(pod) - if err != nil { - glog.V(5).Infof("Invalid pod tolerations detected, but we will leave handling of this to validation phase") - return nil - } + tolerations := pod.Spec.Tolerations toleratesNodeNotReady := false toleratesNodeUnreachable := false diff --git a/plugin/pkg/scheduler/algorithm/predicates/predicates.go b/plugin/pkg/scheduler/algorithm/predicates/predicates.go index 75fa310a87a..4be7fe0ca86 100644 --- a/plugin/pkg/scheduler/algorithm/predicates/predicates.go +++ b/plugin/pkg/scheduler/algorithm/predicates/predicates.go @@ -1160,13 +1160,8 @@ func PodToleratesNodeTaints(pod *v1.Pod, meta interface{}, nodeInfo *schedulerca return false, nil, err } - tolerations, err := v1.GetTolerationsFromPodAnnotations(pod.Annotations) - if err != nil { - return false, nil, err - } - - if v1.TolerationsTolerateTaintsWithFilter(tolerations, taints, func(t *v1.Taint) bool { - // PodToleratesNodeTaints is only interested in NoSchedule and NoExecute taints. + if v1.TolerationsTolerateTaintsWithFilter(pod.Spec.Tolerations, taints, func(t *v1.Taint) bool { + // PodToleratesNodeTaints is only interested in NoSchedule taints. return t.Effect == v1.TaintEffectNoSchedule || t.Effect == v1.TaintEffectNoExecute }) { return true, nil, nil diff --git a/plugin/pkg/scheduler/algorithm/priorities/taint_toleration.go b/plugin/pkg/scheduler/algorithm/priorities/taint_toleration.go index bebc30564ec..b88db85585a 100644 --- a/plugin/pkg/scheduler/algorithm/priorities/taint_toleration.go +++ b/plugin/pkg/scheduler/algorithm/priorities/taint_toleration.go @@ -53,11 +53,7 @@ func getAllTolerationPreferNoSchedule(tolerations []v1.Toleration) (tolerationLi } func getTolerationListFromPod(pod *v1.Pod) ([]v1.Toleration, error) { - tolerations, err := v1.GetTolerationsFromPodAnnotations(pod.Annotations) - if err != nil { - return nil, err - } - return getAllTolerationPreferNoSchedule(tolerations), nil + return getAllTolerationPreferNoSchedule(pod.Spec.Tolerations), nil } // ComputeTaintTolerationPriority prepares the priority list for all the nodes based on the number of intolerable taints on the node @@ -78,13 +74,9 @@ func ComputeTaintTolerationPriorityMap(pod *v1.Pod, meta interface{}, nodeInfo * } } - taints, err := v1.GetTaintsFromNodeAnnotations(node.Annotations) - if err != nil { - return schedulerapi.HostPriority{}, err - } return schedulerapi.HostPriority{ Host: node.Name, - Score: countIntolerableTaintsPreferNoSchedule(taints, tolerationList), + Score: countIntolerableTaintsPreferNoSchedule(node.Spec.Taints, tolerationList), }, nil } diff --git a/plugin/pkg/scheduler/schedulercache/node_info.go b/plugin/pkg/scheduler/schedulercache/node_info.go index dd4ccf02b2c..4fca801e419 100644 --- a/plugin/pkg/scheduler/schedulercache/node_info.go +++ b/plugin/pkg/scheduler/schedulercache/node_info.go @@ -337,7 +337,7 @@ func (n *NodeInfo) SetNode(node *v1.Node) error { } } } - n.taints, n.taintsErr = v1.GetTaintsFromNodeAnnotations(node.Annotations) + n.taints = node.Spec.Taints for i := range node.Status.Conditions { cond := &node.Status.Conditions[i] switch cond.Type { diff --git a/staging/src/k8s.io/client-go/pkg/api/helpers.go b/staging/src/k8s.io/client-go/pkg/api/helpers.go index 256cfbcd4fe..6672ba2e288 100644 --- a/staging/src/k8s.io/client-go/pkg/api/helpers.go +++ b/staging/src/k8s.io/client-go/pkg/api/helpers.go @@ -429,14 +429,6 @@ func NodeSelectorRequirementsAsSelector(nsm []NodeSelectorRequirement) (labels.S } const ( - // TolerationsAnnotationKey represents the key of tolerations data (json serialized) - // in the Annotations of a Pod. - TolerationsAnnotationKey string = "scheduler.alpha.kubernetes.io/tolerations" - - // TaintsAnnotationKey represents the key of taints data (json serialized) - // in the Annotations of a Node. - TaintsAnnotationKey string = "scheduler.alpha.kubernetes.io/taints" - // SeccompPodAnnotationKey represents the key of a seccomp profile applied // to all containers of a pod. SeccompPodAnnotationKey string = "seccomp.security.alpha.kubernetes.io/pod" diff --git a/staging/src/k8s.io/client-go/pkg/api/types.go b/staging/src/k8s.io/client-go/pkg/api/types.go index 80e6f430421..e18968a6022 100644 --- a/staging/src/k8s.io/client-go/pkg/api/types.go +++ b/staging/src/k8s.io/client-go/pkg/api/types.go @@ -1913,6 +1913,9 @@ type PodSpec struct { // If not specified, the pod will be dispatched by default scheduler. // +optional SchedulerName string + // If specified, the pod's tolerations. + // +optional + Tolerations []Toleration } // Sysctl defines a kernel parameter to be set @@ -2530,6 +2533,10 @@ type NodeSpec struct { // Unschedulable controls node schedulability of new pods. By default node is schedulable. // +optional Unschedulable bool + + // If specified, the node's taints. + // +optional + Taints []Taint } // DaemonEndpoint contains information about a single Daemon endpoint. diff --git a/staging/src/k8s.io/client-go/pkg/api/v1/helpers.go b/staging/src/k8s.io/client-go/pkg/api/v1/helpers.go index 564671a5a21..d705cfadb96 100644 --- a/staging/src/k8s.io/client-go/pkg/api/v1/helpers.go +++ b/staging/src/k8s.io/client-go/pkg/api/v1/helpers.go @@ -234,14 +234,6 @@ func NodeSelectorRequirementsAsSelector(nsm []NodeSelectorRequirement) (labels.S } const ( - // TolerationsAnnotationKey represents the key of tolerations data (json serialized) - // in the Annotations of a Pod. - TolerationsAnnotationKey string = "scheduler.alpha.kubernetes.io/tolerations" - - // TaintsAnnotationKey represents the key of taints data (json serialized) - // in the Annotations of a Node. - TaintsAnnotationKey string = "scheduler.alpha.kubernetes.io/taints" - // SeccompPodAnnotationKey represents the key of a seccomp profile applied // to all containers of a pod. SeccompPodAnnotationKey string = "seccomp.security.alpha.kubernetes.io/pod" @@ -278,40 +270,6 @@ const ( ObjectTTLAnnotationKey string = "node.alpha.kubernetes.io/ttl" ) -// GetTolerationsFromPodAnnotations gets the json serialized tolerations data from Pod.Annotations -// and converts it to the []Toleration type in api. -func GetTolerationsFromPodAnnotations(annotations map[string]string) ([]Toleration, error) { - var tolerations []Toleration - if len(annotations) > 0 && annotations[TolerationsAnnotationKey] != "" { - err := json.Unmarshal([]byte(annotations[TolerationsAnnotationKey]), &tolerations) - if err != nil { - return tolerations, err - } - } - return tolerations, nil -} - -func GetPodTolerations(pod *Pod) ([]Toleration, error) { - return GetTolerationsFromPodAnnotations(pod.Annotations) -} - -// GetTaintsFromNodeAnnotations gets the json serialized taints data from Pod.Annotations -// and converts it to the []Taint type in api. -func GetTaintsFromNodeAnnotations(annotations map[string]string) ([]Taint, error) { - var taints []Taint - if len(annotations) > 0 && annotations[TaintsAnnotationKey] != "" { - err := json.Unmarshal([]byte(annotations[TaintsAnnotationKey]), &taints) - if err != nil { - return []Taint{}, err - } - } - return taints, nil -} - -func GetNodeTaints(node *Node) ([]Taint, error) { - return GetTaintsFromNodeAnnotations(node.Annotations) -} - // ToleratesTaint checks if the toleration tolerates the taint. // The matching follows the rules below: // (1) Empty toleration.effect means to match all taint effects, diff --git a/staging/src/k8s.io/client-go/pkg/api/v1/types.go b/staging/src/k8s.io/client-go/pkg/api/v1/types.go index e90ef580e4e..885b1cb01ef 100644 --- a/staging/src/k8s.io/client-go/pkg/api/v1/types.go +++ b/staging/src/k8s.io/client-go/pkg/api/v1/types.go @@ -2203,6 +2203,9 @@ type PodSpec struct { // If not specified, the pod will be dispatched by default scheduler. // +optional SchedulerName string `json:"schedulername,omitempty" protobuf:"bytes,19,opt,name=schedulername"` + // If specified, the pod's tolerations. + // +optional + Tolerations []Toleration `json:"tolerations,omitempty" protobuf:"bytes,21,opt,name=tolerations"` } // PodSecurityContext holds pod-level security attributes and common container settings. @@ -2934,6 +2937,9 @@ type NodeSpec struct { // More info: http://releases.k8s.io/HEAD/docs/admin/node.md#manual-node-administration // +optional Unschedulable bool `json:"unschedulable,omitempty" protobuf:"varint,4,opt,name=unschedulable"` + // If specified, the node's taints. + // +optional + Taints []Taint `json:"taints,omitempty" protobuf:"bytes,5,opt,name=taints"` } // DaemonEndpoint contains information about a single Daemon endpoint.