github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-31 07:20:13 +00:00
Code Issues Projects Releases Wiki Activity

Change taint/toleration annotations to api fields.

Browse Source
This commit is contained in:
Avesh Agarwal 2017-02-20 11:43:05 -05:00
parent eef16cf141
commit 9b640838a5
22 changed files with 116 additions and 378 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
cmd/kubeadm/app/master/selfhosted.go
View File

@ -17,7 +17,6 @@ limitations under the License.
package master
import (
"encoding/json"
"fmt"
"os"
"path"
@ -203,9 +202,6 @@ func getAPIServerDS(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, vo
"component": kubeAPIServer,
"tier": "control-plane",
},
Annotations: map[string]string{
v1.TolerationsAnnotationKey: getMasterToleration(),
},
},
Spec: v1.PodSpec{
NodeSelector: map[string]string{metav1.NodeLabelKubeadmAlphaRole: metav1.NodeLabelRoleMaster},
@ -222,6 +218,7 @@ func getAPIServerDS(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, vo
Resources: componentResources("250m"),
},
},
Tolerations: getMasterToleration(),
},
},
},
@ -256,9 +253,6 @@ func getControllerManagerDeployment(cfg *kubeadmapi.MasterConfiguration, volumes
"component": kubeControllerManager,
"tier": "control-plane",
},
Annotations: map[string]string{
v1.TolerationsAnnotationKey: getMasterToleration(),
},
},
Spec: v1.PodSpec{
NodeSelector: map[string]string{metav1.NodeLabelKubeadmAlphaRole: metav1.NodeLabelRoleMaster},
@ -275,7 +269,8 @@ func getControllerManagerDeployment(cfg *kubeadmapi.MasterConfiguration, volumes
Env: getProxyEnvVars(),
},
},
DNSPolicy: v1.DNSDefault,
Tolerations: getMasterToleration(),
DNSPolicy: v1.DNSDefault,
},
},
},
@ -310,9 +305,6 @@ func getSchedulerDeployment(cfg *kubeadmapi.MasterConfiguration) ext.Deployment
"component": kubeScheduler,
"tier": "control-plane",
},
Annotations: map[string]string{
v1.TolerationsAnnotationKey: getMasterToleration(),
},
},
Spec: v1.PodSpec{
NodeSelector: map[string]string{metav1.NodeLabelKubeadmAlphaRole: metav1.NodeLabelRoleMaster},
@ -327,6 +319,7 @@ func getSchedulerDeployment(cfg *kubeadmapi.MasterConfiguration) ext.Deployment
Env: getProxyEnvVars(),
},
},
Tolerations: getMasterToleration(),
},
},
},
@ -338,15 +331,14 @@ func buildStaticManifestFilepath(name string) string {
return path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "manifests", name+".yaml")
}
func getMasterToleration() string {
func getMasterToleration() []v1.Toleration {
// Tolerate the master taint we add to our master nodes, as this can and should
// run there.
// TODO: Duplicated above
masterToleration, _ := json.Marshal([]v1.Toleration{{
return []v1.Toleration{{
Key: "dedicated",
Value: "master",
Operator: v1.TolerationOpEqual,
Effect: v1.TaintEffectNoSchedule,
}})
return string(masterToleration)
}}
}

11
cmd/kubeadm/app/master/templates.go
View File

@ -59,9 +59,6 @@ spec:
k8s-app: kube-discovery
# TODO: I guess we can remove all these cluster-service labels...
kubernetes.io/cluster-service: "true"
annotations:
# TODO: Move this to the beta tolerations field below as soon as the Tolerations field exists in PodSpec
scheduler.alpha.kubernetes.io/tolerations: '[{"key":"dedicated","value":"master","effect":"NoSchedule"}]'
spec:
containers:
- name: kube-discovery
@ -78,10 +75,10 @@ spec:
name: clusterinfo
readOnly: true
hostNetwork: true
# tolerations:
# - key: dedicated
# value: master
# effect: NoSchedule
tolerations:
- key: "dedicated"
value: "master"
effect: "NoSchedule"
securityContext:
seLinuxOptions:
type: spc_t

24
cmd/kubeadm/app/phases/addons/manifests.go
View File

@ -63,9 +63,6 @@ spec:
metadata:
labels:
k8s-app: kube-proxy
annotations:
# TODO: Move this to the beta tolerations field below as soon as the Tolerations field exists in PodSpec
scheduler.alpha.kubernetes.io/tolerations: '[{"key":"dedicated","value":"master","effect":"NoSchedule"}]'
spec:
containers:
- name: kube-proxy
@ -82,11 +79,10 @@ spec:
name: kube-proxy
hostNetwork: true
serviceAccountName: kube-proxy
# Tolerate running on the master
# tolerations:
# - key: dedicated
# value: master
# effect: NoSchedule
tolerations:
- key: dedicated
value: master
effect: NoSchedule
volumes:
- name: kube-proxy
configMap:
@ -122,8 +118,6 @@ spec:
k8s-app: kube-dns
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
# TODO: Move this to the beta tolerations field below as soon as the Tolerations field exists in PodSpec
scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}, {"key":"dedicated","value":"master","effect":"NoSchedule"}]'
spec:
volumes:
- name: kube-dns-config
@ -242,10 +236,12 @@ spec:
cpu: 10m
dnsPolicy: Default # Don't use cluster DNS.
serviceAccountName: kube-dns
# tolerations:
# - key: dedicated
# value: master
# effect: NoSchedule
tolerations:
- key: "CriticalAddonsOnly"
operator: "Exists"
- key: "dedicated"
value: "master"
effect: "NoSchedule"
# TODO: Remove this affinity field as soon as we are using manifest lists
affinity:
nodeAffinity:

4
cmd/kubeadm/app/phases/apiconfig/setupmaster.go
View File

@ -61,9 +61,7 @@ func attemptToUpdateMasterRoleLabelsAndTaints(client *clientset.Clientset) error
// TODO: Switch to the new master label defined in https://github.com/kubernetes/kubernetes/pull/39112
n.ObjectMeta.Labels[metav1.NodeLabelKubeadmAlphaRole] = metav1.NodeLabelRoleMaster
// TODO: Use the Taints beta field on the NodeSpec now
taintsAnnotation, _ := json.Marshal([]v1.Taint{{Key: "dedicated", Value: "master", Effect: "NoSchedule"}})
n.ObjectMeta.Annotations[v1.TaintsAnnotationKey] = string(taintsAnnotation)
n.Spec.Taints = []v1.Taint{{Key: "dedicated", Value: "master", Effect: "NoSchedule"}}
newData, err := json.Marshal(n)
if err != nil {

8
pkg/api/helpers.go
View File

@ -429,14 +429,6 @@ func NodeSelectorRequirementsAsSelector(nsm []NodeSelectorRequirement) (labels.S
}
const (
// TolerationsAnnotationKey represents the key of tolerations data (json serialized)
// in the Annotations of a Pod.
TolerationsAnnotationKey string = "scheduler.alpha.kubernetes.io/tolerations"
// TaintsAnnotationKey represents the key of taints data (json serialized)
// in the Annotations of a Node.
TaintsAnnotationKey string = "scheduler.alpha.kubernetes.io/taints"
// SeccompPodAnnotationKey represents the key of a seccomp profile applied
// to all containers of a pod.
SeccompPodAnnotationKey string = "seccomp.security.alpha.kubernetes.io/pod"

12
pkg/api/json.go
View File

@ -21,18 +21,6 @@ import "encoding/json"
// This file implements json marshaling/unmarshaling interfaces on objects that are currently marshaled into annotations
// to prevent anyone from marshaling these internal structs.
var _ = json.Marshaler(Taint{})
var _ = json.Unmarshaler(&Taint{})
func (Taint) MarshalJSON() ([]byte, error) { panic("do not marshal internal struct") }
func (*Taint) UnmarshalJSON([]byte) error { panic("do not unmarshal to internal struct") }
var _ = json.Marshaler(Toleration{})
var _ = json.Unmarshaler(&Toleration{})
func (Toleration) MarshalJSON() ([]byte, error) { panic("do not marshal internal struct") }
func (*Toleration) UnmarshalJSON([]byte) error { panic("do not unmarshal to internal struct") }
var _ = json.Marshaler(&AvoidPods{})
var _ = json.Unmarshaler(&AvoidPods{})

7
pkg/api/types.go
View File

@ -2001,6 +2001,9 @@ type PodSpec struct {
// If not specified, the pod will be dispatched by default scheduler.
// +optional
SchedulerName string
// If specified, the pod's tolerations.
// +optional
Tolerations []Toleration
}
// Sysctl defines a kernel parameter to be set
@ -2623,6 +2626,10 @@ type NodeSpec struct {
// Unschedulable controls node schedulability of new pods. By default node is schedulable.
// +optional
Unschedulable bool
// If specified, the node's taints.
// +optional
Taints []Taint
}
// DaemonEndpoint contains information about a single Daemon endpoint.

103
pkg/api/v1/helpers.go
View File

@ -235,14 +235,6 @@ func NodeSelectorRequirementsAsSelector(nsm []NodeSelectorRequirement) (labels.S
}
const (
// TolerationsAnnotationKey represents the key of tolerations data (json serialized)
// in the Annotations of a Pod.
TolerationsAnnotationKey string = "scheduler.alpha.kubernetes.io/tolerations"
// TaintsAnnotationKey represents the key of taints data (json serialized)
// in the Annotations of a Node.
TaintsAnnotationKey string = "scheduler.alpha.kubernetes.io/taints"
// SeccompPodAnnotationKey represents the key of a seccomp profile applied
// to all containers of a pod.
SeccompPodAnnotationKey string = "seccomp.security.alpha.kubernetes.io/pod"
@ -284,79 +276,34 @@ const (
AffinityAnnotationKey string = "scheduler.alpha.kubernetes.io/affinity"
)
// GetTolerationsFromPodAnnotations gets the json serialized tolerations data from Pod.Annotations
// and converts it to the []Toleration type in api.
func GetTolerationsFromPodAnnotations(annotations map[string]string) ([]Toleration, error) {
var tolerations []Toleration
if len(annotations) > 0 && annotations[TolerationsAnnotationKey] != "" {
err := json.Unmarshal([]byte(annotations[TolerationsAnnotationKey]), &tolerations)
if err != nil {
return tolerations, err
}
}
return tolerations, nil
}
func GetPodTolerations(pod *Pod) ([]Toleration, error) {
return GetTolerationsFromPodAnnotations(pod.Annotations)
}
// Tries to add a toleration to annotations list. Returns true if something was updated
// false otherwise.
func AddOrUpdateTolerationInPod(pod *Pod, toleration *Toleration) (bool, error) {
podTolerations, err := GetPodTolerations(pod)
if err != nil {
return false, err
}
podTolerations := pod.Spec.Tolerations
var newTolerations []*Toleration
var newTolerations []Toleration
updated := false
for i := range podTolerations {
if toleration.MatchToleration(&podTolerations[i]) {
if api.Semantic.DeepEqual(toleration, podTolerations[i]) {
return false, nil
}
newTolerations = append(newTolerations, toleration)
newTolerations = append(newTolerations, *toleration)
updated = true
continue
}
newTolerations = append(newTolerations, &podTolerations[i])
newTolerations = append(newTolerations, podTolerations[i])
}
if !updated {
newTolerations = append(newTolerations, toleration)
newTolerations = append(newTolerations, *toleration)
}
tolerationsData, err := json.Marshal(newTolerations)
if err != nil {
return false, err
}
if pod.Annotations == nil {
pod.Annotations = make(map[string]string)
}
pod.Annotations[TolerationsAnnotationKey] = string(tolerationsData)
pod.Spec.Tolerations = newTolerations
return true, nil
}
// GetTaintsFromNodeAnnotations gets the json serialized taints data from Pod.Annotations
// and converts it to the []Taint type in api.
func GetTaintsFromNodeAnnotations(annotations map[string]string) ([]Taint, error) {
var taints []Taint
if len(annotations) > 0 && annotations[TaintsAnnotationKey] != "" {
err := json.Unmarshal([]byte(annotations[TaintsAnnotationKey]), &taints)
if err != nil {
return []Taint{}, err
}
}
return taints, nil
}
func GetNodeTaints(node *Node) ([]Taint, error) {
return GetTaintsFromNodeAnnotations(node.Annotations)
}
// MatchToleration checks if the toleration matches tolerationToMatch. Tolerations are unique by <key,effect,operator,value>,
// if the two tolerations have same <key,effect,operator,value> combination, regard as they match.
// TODO: uniqueness check for tolerations in api validations.
@ -574,39 +521,28 @@ func AddOrUpdateTaint(node *Node, taint *Taint) (*Node, bool, error) {
return nil, false, err
}
newNode := objCopy.(*Node)
nodeTaints, err := GetTaintsFromNodeAnnotations(newNode.Annotations)
if err != nil {
return newNode, false, err
}
nodeTaints := newNode.Spec.Taints
var newTaints []*Taint
var newTaints []Taint
updated := false
for i := range nodeTaints {
if taint.MatchTaint(&nodeTaints[i]) {
if api.Semantic.DeepEqual(taint, nodeTaints[i]) {
return newNode, false, nil
}
newTaints = append(newTaints, taint)
newTaints = append(newTaints, *taint)
updated = true
continue
}
newTaints = append(newTaints, &nodeTaints[i])
newTaints = append(newTaints, nodeTaints[i])
}
if !updated {
newTaints = append(newTaints, taint)
newTaints = append(newTaints, *taint)
}
taintsData, err := json.Marshal(newTaints)
if err != nil {
return nil, false, err
}
if newNode.Annotations == nil {
newNode.Annotations = make(map[string]string)
}
newNode.Annotations[TaintsAnnotationKey] = string(taintsData)
newNode.Spec.Taints = newTaints
return newNode, true, nil
}
@ -627,10 +563,7 @@ func RemoveTaint(node *Node, taint *Taint) (*Node, bool, error) {
return nil, false, err
}
newNode := objCopy.(*Node)
nodeTaints, err := GetTaintsFromNodeAnnotations(newNode.Annotations)
if err != nil {
return newNode, false, err
}
nodeTaints := newNode.Spec.Taints
if len(nodeTaints) == 0 {
return newNode, false, nil
}
@ -640,15 +573,7 @@ func RemoveTaint(node *Node, taint *Taint) (*Node, bool, error) {
}
newTaints, _ := DeleteTaint(nodeTaints, taint)
if len(newTaints) == 0 {
delete(newNode.Annotations, TaintsAnnotationKey)
} else {
taintsData, err := json.Marshal(newTaints)
if err != nil {
return newNode, false, err
}
newNode.Annotations[TaintsAnnotationKey] = string(taintsData)
}
newNode.Spec.Taints = newTaints
return newNode, true, nil
}

6
pkg/api/v1/types.go
View File

@ -2286,6 +2286,9 @@ type PodSpec struct {
// If not specified, the pod will be dispatched by default scheduler.
// +optional
SchedulerName string `json:"schedulername,omitempty" protobuf:"bytes,19,opt,name=schedulername"`
// If specified, the pod's tolerations.
// +optional
Tolerations []Toleration `json:"tolerations,omitempty" protobuf:"bytes,22,opt,name=tolerations"`
}
// PodSecurityContext holds pod-level security attributes and common container settings.
@ -3022,6 +3025,9 @@ type NodeSpec struct {
// More info: http://releases.k8s.io/HEAD/docs/admin/node.md#manual-node-administration
// +optional
Unschedulable bool `json:"unschedulable,omitempty" protobuf:"varint,4,opt,name=unschedulable"`
// If specified, the node's taints.
// +optional
Taints []Taint `json:"taints,omitempty" protobuf:"bytes,5,opt,name=taints"`
}
// DaemonEndpoint contains information about a single Daemon endpoint.

71
pkg/api/validation/validation.go
View File

@ -105,9 +105,6 @@ func ValidateDNS1123Subdomain(value string, fldPath *field.Path) field.ErrorList
func ValidatePodSpecificAnnotations(annotations map[string]string, spec *api.PodSpec, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
if annotations[api.TolerationsAnnotationKey] != "" {
allErrs = append(allErrs, ValidateTolerationsInPodAnnotations(annotations, fldPath)...)
}
// TODO: remove these after we EOL the annotations.
if hostname, exists := annotations[utilpod.PodHostnameAnnotation]; exists {
@ -1948,6 +1945,10 @@ func ValidatePodSpec(spec *api.PodSpec, fldPath *field.Path) field.ErrorList {
allErrs = append(allErrs, ValidateDNS1123Label(spec.Subdomain, fldPath.Child("subdomain"))...)
}
if len(spec.Tolerations) > 0 {
allErrs = append(allErrs, validateTolerations(spec.Tolerations, fldPath.Child("tolerations"))...)
}
return allErrs
}
@ -2139,29 +2140,6 @@ func validatePodAffinity(podAffinity *api.PodAffinity, fldPath *field.Path) fiel
return allErrs
}
// ValidateTolerationsInPodAnnotations tests that the serialized tolerations in Pod.Annotations has valid data
func ValidateTolerationsInPodAnnotations(annotations map[string]string, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
v1Tolerations, err := v1.GetTolerationsFromPodAnnotations(annotations)
if err != nil {
allErrs = append(allErrs, field.Invalid(fldPath, api.TolerationsAnnotationKey, err.Error()))
return allErrs
}
tolerations := make([]api.Toleration, len(v1Tolerations))
for i := range v1Tolerations {
if err := v1.Convert_v1_Toleration_To_api_Toleration(&v1Tolerations[i], &tolerations[i], nil); err != nil {
allErrs = append(allErrs, field.Invalid(fldPath, api.TolerationsAnnotationKey, err.Error()))
return allErrs
}
}
if len(tolerations) > 0 {
allErrs = append(allErrs, validateTolerations(tolerations, fldPath.Child(api.TolerationsAnnotationKey))...)
}
return allErrs
}
func ValidateSeccompProfile(p string, fldPath *field.Path) field.ErrorList {
if p == "docker/default" {
return nil
@ -2813,8 +2791,8 @@ func ValidateReadOnlyPersistentDisks(volumes []api.Volume, fldPath *field.Path)
return allErrs
}
// validateTaints tests if given taints have valid data.
func validateTaints(taints []api.Taint, fldPath *field.Path) field.ErrorList {
// validateNodeTaints tests if given taints have valid data.
func validateNodeTaints(taints []api.Taint, fldPath *field.Path) field.ErrorList {
allErrors := field.ErrorList{}
uniqueTaints := map[api.TaintEffect]sets.String{}
@ -2847,37 +2825,11 @@ func validateTaints(taints []api.Taint, fldPath *field.Path) field.ErrorList {
return allErrors
}
// ValidateTaintsInNodeAnnotations tests that the serialized taints in Node.Annotations has valid data
func ValidateTaintsInNodeAnnotations(annotations map[string]string, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
v1Taints, err := v1.GetTaintsFromNodeAnnotations(annotations)
if err != nil {
allErrs = append(allErrs, field.Invalid(fldPath, api.TaintsAnnotationKey, err.Error()))
return allErrs
}
taints := make([]api.Taint, len(v1Taints))
for i := range v1Taints {
if err := v1.Convert_v1_Taint_To_api_Taint(&v1Taints[i], &taints[i], nil); err != nil {
allErrs = append(allErrs, field.Invalid(fldPath, api.TaintsAnnotationKey, err.Error()))
return allErrs
}
}
if len(taints) > 0 {
allErrs = append(allErrs, validateTaints(taints, fldPath.Child(api.TaintsAnnotationKey))...)
}
return allErrs
}
func ValidateNodeSpecificAnnotations(annotations map[string]string, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
if annotations[api.PreferAvoidPodsAnnotationKey] != "" {
allErrs = append(allErrs, ValidateAvoidPodsInNodeAnnotations(annotations, fldPath)...)
}
if annotations[api.TaintsAnnotationKey] != "" {
allErrs = append(allErrs, ValidateTaintsInNodeAnnotations(annotations, fldPath)...)
}
return allErrs
}
@ -2886,6 +2838,9 @@ func ValidateNode(node *api.Node) field.ErrorList {
fldPath := field.NewPath("metadata")
allErrs := ValidateObjectMeta(&node.ObjectMeta, false, ValidateNodeName, fldPath)
allErrs = append(allErrs, ValidateNodeSpecificAnnotations(node.ObjectMeta.Annotations, fldPath.Child("annotations"))...)
if len(node.Spec.Taints) > 0 {
allErrs = append(allErrs, validateNodeTaints(node.Spec.Taints, fldPath.Child("taints"))...)
}
// Only validate spec. All status fields are optional and can be updated later.
@ -2948,10 +2903,16 @@ func ValidateNodeUpdate(node, oldNode *api.Node) field.ErrorList {
// Clear status
oldNode.Status = node.Status
// update taints
if len(node.Spec.Taints) > 0 {
allErrs = append(allErrs, validateNodeTaints(node.Spec.Taints, fldPath.Child("taints"))...)
}
oldNode.Spec.Taints = node.Spec.Taints
// TODO: Add a 'real' error type for this error and provide print actual diffs.
if !apiequality.Semantic.DeepEqual(oldNode, node) {
glog.V(4).Infof("Update failed validation %#v vs %#v", oldNode, node)
allErrs = append(allErrs, field.Forbidden(field.NewPath(""), "node updates may only change labels or capacity"))
allErrs = append(allErrs, field.Forbidden(field.NewPath(""), "node updates may only change labels, taints or capacity"))
}
return allErrs

33
pkg/controller/node/taint_controller.go
View File

@ -280,22 +280,13 @@ func (tc *NoExecuteTaintManager) Run(stopCh <-chan struct{}) {
// PodUpdated is used to notify NoExecuteTaintManager about Pod changes.
func (tc *NoExecuteTaintManager) PodUpdated(oldPod *v1.Pod, newPod *v1.Pod) {
var err error
oldTolerations := []v1.Toleration{}
if oldPod != nil {
oldTolerations, err = v1.GetPodTolerations(oldPod)
if err != nil {
glog.Errorf("Failed to get Tolerations from the old Pod: %v", err)
return
}
oldTolerations = oldPod.Spec.Tolerations
}
newTolerations := []v1.Toleration{}
if newPod != nil {
newTolerations, err = v1.GetPodTolerations(newPod)
if err != nil {
glog.Errorf("Failed to get Tolerations from the new Pod: %v", err)
return
}
newTolerations = newPod.Spec.Tolerations
}
if oldPod != nil && newPod != nil && api.Semantic.DeepEqual(oldTolerations, newTolerations) && oldPod.Spec.NodeName == newPod.Spec.NodeName {
@ -312,24 +303,15 @@ func (tc *NoExecuteTaintManager) PodUpdated(oldPod *v1.Pod, newPod *v1.Pod) {
// NodeUpdated is used to notify NoExecuteTaintManager about Node changes.
func (tc *NoExecuteTaintManager) NodeUpdated(oldNode *v1.Node, newNode *v1.Node) {
var err error
oldTaints := []v1.Taint{}
if oldNode != nil {
oldTaints, err = v1.GetNodeTaints(oldNode)
if err != nil {
glog.Errorf("Failed to get Taints from the old Node: %v", err)
return
}
oldTaints = oldNode.Spec.Taints
}
oldTaints = getNoExecuteTaints(oldTaints)
newTaints := []v1.Taint{}
if newNode != nil {
newTaints, err = v1.GetNodeTaints(newNode)
if err != nil {
glog.Errorf("Failed to get Taints from the new Node: %v", err)
return
}
newTaints = newNode.Spec.Taints
}
newTaints = getNoExecuteTaints(newTaints)
@ -466,12 +448,7 @@ func (tc *NoExecuteTaintManager) handleNodeUpdate(nodeUpdate *nodeUpdateItem) {
for i := range pods {
pod := &pods[i]
podNamespacedName := types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}
tolerations, err := v1.GetPodTolerations(pod)
if err != nil {
glog.Errorf("Failed to get Tolerations from Pod %v: %v", podNamespacedName.String(), err)
continue
}
tc.processPodOnNode(podNamespacedName, node.Name, tolerations, taints, now)
tc.processPodOnNode(podNamespacedName, node.Name, pod.Spec.Tolerations, taints, now)
}
}

61
pkg/kubectl/cmd/taint.go
View File

@ -25,8 +25,6 @@ import (
"github.com/golang/glog"
"github.com/spf13/cobra"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
@ -115,18 +113,15 @@ func NewCmdTaint(f cmdutil.Factory, out io.Writer) *cobra.Command {
// reorganizeTaints returns the updated set of taints, taking into account old taints that were not updated,
// old taints that were updated, old taints that were deleted, and new taints.
func reorganizeTaints(accessor metav1.Object, overwrite bool, taintsToAdd []v1.Taint, taintsToRemove []v1.Taint) ([]v1.Taint, error) {
newTaints := append([]v1.Taint{}, taintsToAdd...)
var oldTaints []v1.Taint
var err error
annotations := accessor.GetAnnotations()
if annotations != nil {
if oldTaints, err = v1.GetTaintsFromNodeAnnotations(annotations); err != nil {
return nil, err
}
func reorganizeTaints(obj runtime.Object, overwrite bool, taintsToAdd []v1.Taint, taintsToRemove []v1.Taint) ([]v1.Taint, error) {
node, ok := obj.(*v1.Node)
if !ok {
return nil, fmt.Errorf("unexpected type %T, expected Node", obj)
}
newTaints := append([]v1.Taint{}, taintsToAdd...)
oldTaints := node.Spec.Taints
// add taints that already existing but not updated to newTaints
for _, oldTaint := range oldTaints {
existsInNew := false
@ -351,23 +346,18 @@ func (o TaintOptions) RunTaint() error {
}
// validateNoTaintOverwrites validates that when overwrite is false, to-be-updated taints don't exist in the node taint list (yet)
func validateNoTaintOverwrites(accessor metav1.Object, taints []v1.Taint) error {
annotations := accessor.GetAnnotations()
if annotations == nil {
return nil
func validateNoTaintOverwrites(obj runtime.Object, taints []v1.Taint) error {
node, ok := obj.(*v1.Node)
if !ok {
return fmt.Errorf("unexpected type %T, expected Node", obj)
}
allErrs := []error{}
oldTaints, err := v1.GetTaintsFromNodeAnnotations(annotations)
if err != nil {
allErrs = append(allErrs, err)
return utilerrors.NewAggregate(allErrs)
}
oldTaints := node.Spec.Taints
for _, taint := range taints {
for _, oldTaint := range oldTaints {
if taint.Key == oldTaint.Key && taint.Effect == oldTaint.Effect {
allErrs = append(allErrs, fmt.Errorf("Node '%s' already has a taint with key (%s) and effect (%v), and --overwrite is false", accessor.GetName(), taint.Key, taint.Effect))
allErrs = append(allErrs, fmt.Errorf("Node '%s' already has a taint with key (%s) and effect (%v), and --overwrite is false", node.Name, taint.Key, taint.Effect))
break
}
}
@ -377,31 +367,22 @@ func validateNoTaintOverwrites(accessor metav1.Object, taints []v1.Taint) error
// updateTaints updates taints of obj
func (o TaintOptions) updateTaints(obj runtime.Object) error {
accessor, err := meta.Accessor(obj)
if err != nil {
return err
}
if !o.overwrite {
if err := validateNoTaintOverwrites(accessor, o.taintsToAdd); err != nil {
if err := validateNoTaintOverwrites(obj, o.taintsToAdd); err != nil {
return err
}
}
annotations := accessor.GetAnnotations()
if annotations == nil {
annotations = make(map[string]string)
newTaints, err := reorganizeTaints(obj, o.overwrite, o.taintsToAdd, o.taintsToRemove)
if err != nil {
return err
}
newTaints, err := reorganizeTaints(accessor, o.overwrite, o.taintsToAdd, o.taintsToRemove)
if err != nil {
return err
node, ok := obj.(*v1.Node)
if !ok {
return fmt.Errorf("unexpected type %T, expected Node", obj)
}
taintsData, err := json.Marshal(newTaints)
if err != nil {
return err
}
annotations[v1.TaintsAnnotationKey] = string(taintsData)
accessor.SetAnnotations(annotations)
node.Spec.Taints = newTaints
return nil
}

31
pkg/kubectl/describe.go
View File

@ -45,7 +45,6 @@ import (
"k8s.io/kubernetes/pkg/api/annotations"
"k8s.io/kubernetes/pkg/api/events"
"k8s.io/kubernetes/pkg/api/v1"
"k8s.io/kubernetes/pkg/apis/apps"
"k8s.io/kubernetes/pkg/apis/autoscaling"
"k8s.io/kubernetes/pkg/apis/batch"
@ -582,7 +581,7 @@ func describePod(pod *api.Pod, events *api.EventList) (string, error) {
describeVolumes(pod.Spec.Volumes, w, "")
w.Write(LEVEL_0, "QoS Class:\t%s\n", pod.Status.QOSClass)
printLabelsMultiline(w, "Node-Selectors", pod.Spec.NodeSelector)
printTolerationsInAnnotationMultiline(w, "Tolerations", pod.Annotations)
printPodTolerationsMultiline(w, "Tolerations", pod.Spec.Tolerations)
if events != nil {
DescribeEvents(events, w)
}
@ -1987,7 +1986,7 @@ func describeNode(node *api.Node, nodeNonTerminatedPodsList *api.PodList, events
w.Write(LEVEL_0, "Name:\t%s\n", node.Name)
w.Write(LEVEL_0, "Role:\t%s\n", findNodeRole(node))
printLabelsMultiline(w, "Labels", node.Labels)
printTaintsInAnnotationMultiline(w, "Taints", node.Annotations)
printNodeTaintsMultiline(w, "Taints", node.Spec.Taints)
w.Write(LEVEL_0, "CreationTimestamp:\t%s\n", node.CreationTimestamp.Time.Format(time.RFC1123Z))
w.Write(LEVEL_0, "Phase:\t%v\n", node.Status.Phase)
if len(node.Status.Conditions) > 0 {
@ -2854,17 +2853,7 @@ func printLabelsMultilineWithIndent(w *PrefixWriter, initialIndent, title, inner
}
// printTaintsMultiline prints multiple taints with a proper alignment.
func printTaintsInAnnotationMultiline(w *PrefixWriter, title string, annotations map[string]string) {
v1Taints, err := v1.GetTaintsFromNodeAnnotations(annotations)
if err != nil {
v1Taints = []v1.Taint{}
}
taints := make([]api.Taint, len(v1Taints))
for i := range v1Taints {
if err := v1.Convert_v1_Taint_To_api_Taint(&v1Taints[i], &taints[i], nil); err != nil {
panic(err)
}
}
func printNodeTaintsMultiline(w *PrefixWriter, title string, taints []api.Taint) {
printTaintsMultilineWithIndent(w, "", title, "\t", taints)
}
@ -2898,18 +2887,8 @@ func printTaintsMultilineWithIndent(w *PrefixWriter, initialIndent, title, inner
}
}
// printTolerationsMultiline prints multiple tolerations with a proper alignment.
func printTolerationsInAnnotationMultiline(w *PrefixWriter, title string, annotations map[string]string) {
v1Tolerations, err := v1.GetTolerationsFromPodAnnotations(annotations)
if err != nil {
v1Tolerations = []v1.Toleration{}
}
tolerations := make([]api.Toleration, len(v1Tolerations))
for i := range v1Tolerations {
if err := v1.Convert_v1_Toleration_To_api_Toleration(&v1Tolerations[i], &tolerations[i], nil); err != nil {
panic(err)
}
}
// printPodTolerationsMultiline prints multiple tolerations with a proper alignment.
func printPodTolerationsMultiline(w *PrefixWriter, title string, tolerations []api.Toleration) {
printTolerationsMultilineWithIndent(w, "", title, "\t", tolerations)
}

9
pkg/kubelet/kubelet_node_status.go
View File

@ -17,7 +17,6 @@ limitations under the License.
package kubelet
import (
"encoding/json"
"fmt"
"math"
"net"
@ -204,19 +203,13 @@ func (kl *Kubelet) initialNode() (*v1.Node, error) {
},
}
if len(kl.kubeletConfiguration.RegisterWithTaints) > 0 {
annotations := make(map[string]string)
taints := make([]v1.Taint, len(kl.kubeletConfiguration.RegisterWithTaints))
for i := range kl.kubeletConfiguration.RegisterWithTaints {
if err := v1.Convert_api_Taint_To_v1_Taint(&kl.kubeletConfiguration.RegisterWithTaints[i], &taints[i], nil); err != nil {
return nil, err
}
}
b, err := json.Marshal(taints)
if err != nil {
return nil, err
}
annotations[v1.TaintsAnnotationKey] = string(b)
node.ObjectMeta.Annotations = annotations
node.Spec.Taints = taints
}
// Initially, set NodeNetworkUnavailable to true.

6
plugin/pkg/admission/defaulttolerationseconds/admission.go
View File

@ -72,11 +72,7 @@ func (p *plugin) Admit(attributes admission.Attributes) (err error) {
return nil
}
tolerations, err := v1.GetPodTolerations(pod)
if err != nil {
glog.V(5).Infof("Invalid pod tolerations detected, but we will leave handling of this to validation phase")
return nil
}
tolerations := pod.Spec.Tolerations
toleratesNodeNotReady := false
toleratesNodeUnreachable := false

9
plugin/pkg/scheduler/algorithm/predicates/predicates.go
View File

@ -1160,13 +1160,8 @@ func PodToleratesNodeTaints(pod *v1.Pod, meta interface{}, nodeInfo *schedulerca
return false, nil, err
}
tolerations, err := v1.GetTolerationsFromPodAnnotations(pod.Annotations)
if err != nil {
return false, nil, err
}
if v1.TolerationsTolerateTaintsWithFilter(tolerations, taints, func(t *v1.Taint) bool {
// PodToleratesNodeTaints is only interested in NoSchedule and NoExecute taints.
if v1.TolerationsTolerateTaintsWithFilter(pod.Spec.Tolerations, taints, func(t *v1.Taint) bool {
// PodToleratesNodeTaints is only interested in NoSchedule taints.
return t.Effect == v1.TaintEffectNoSchedule || t.Effect == v1.TaintEffectNoExecute
}) {
return true, nil, nil

12
plugin/pkg/scheduler/algorithm/priorities/taint_toleration.go
View File

@ -53,11 +53,7 @@ func getAllTolerationPreferNoSchedule(tolerations []v1.Toleration) (tolerationLi
}
func getTolerationListFromPod(pod *v1.Pod) ([]v1.Toleration, error) {
tolerations, err := v1.GetTolerationsFromPodAnnotations(pod.Annotations)
if err != nil {
return nil, err
}
return getAllTolerationPreferNoSchedule(tolerations), nil
return getAllTolerationPreferNoSchedule(pod.Spec.Tolerations), nil
}
// ComputeTaintTolerationPriority prepares the priority list for all the nodes based on the number of intolerable taints on the node
@ -78,13 +74,9 @@ func ComputeTaintTolerationPriorityMap(pod *v1.Pod, meta interface{}, nodeInfo *
}
}
taints, err := v1.GetTaintsFromNodeAnnotations(node.Annotations)
if err != nil {
return schedulerapi.HostPriority{}, err
}
return schedulerapi.HostPriority{
Host: node.Name,
Score: countIntolerableTaintsPreferNoSchedule(taints, tolerationList),
Score: countIntolerableTaintsPreferNoSchedule(node.Spec.Taints, tolerationList),
}, nil
}

2
plugin/pkg/scheduler/schedulercache/node_info.go
View File

@ -337,7 +337,7 @@ func (n *NodeInfo) SetNode(node *v1.Node) error {
}
}
}
n.taints, n.taintsErr = v1.GetTaintsFromNodeAnnotations(node.Annotations)
n.taints = node.Spec.Taints
for i := range node.Status.Conditions {
cond := &node.Status.Conditions[i]
switch cond.Type {

8
staging/src/k8s.io/client-go/pkg/api/helpers.go
View File

@ -429,14 +429,6 @@ func NodeSelectorRequirementsAsSelector(nsm []NodeSelectorRequirement) (labels.S
}
const (
// TolerationsAnnotationKey represents the key of tolerations data (json serialized)
// in the Annotations of a Pod.
TolerationsAnnotationKey string = "scheduler.alpha.kubernetes.io/tolerations"
// TaintsAnnotationKey represents the key of taints data (json serialized)
// in the Annotations of a Node.
TaintsAnnotationKey string = "scheduler.alpha.kubernetes.io/taints"
// SeccompPodAnnotationKey represents the key of a seccomp profile applied
// to all containers of a pod.
SeccompPodAnnotationKey string = "seccomp.security.alpha.kubernetes.io/pod"

7
staging/src/k8s.io/client-go/pkg/api/types.go
View File

@ -1913,6 +1913,9 @@ type PodSpec struct {
// If not specified, the pod will be dispatched by default scheduler.
// +optional
SchedulerName string
// If specified, the pod's tolerations.
// +optional
Tolerations []Toleration
}
// Sysctl defines a kernel parameter to be set
@ -2530,6 +2533,10 @@ type NodeSpec struct {
// Unschedulable controls node schedulability of new pods. By default node is schedulable.
// +optional
Unschedulable bool
// If specified, the node's taints.
// +optional
Taints []Taint
}
// DaemonEndpoint contains information about a single Daemon endpoint.

42
staging/src/k8s.io/client-go/pkg/api/v1/helpers.go
View File

@ -234,14 +234,6 @@ func NodeSelectorRequirementsAsSelector(nsm []NodeSelectorRequirement) (labels.S
}
const (
// TolerationsAnnotationKey represents the key of tolerations data (json serialized)
// in the Annotations of a Pod.
TolerationsAnnotationKey string = "scheduler.alpha.kubernetes.io/tolerations"
// TaintsAnnotationKey represents the key of taints data (json serialized)
// in the Annotations of a Node.
TaintsAnnotationKey string = "scheduler.alpha.kubernetes.io/taints"
// SeccompPodAnnotationKey represents the key of a seccomp profile applied
// to all containers of a pod.
SeccompPodAnnotationKey string = "seccomp.security.alpha.kubernetes.io/pod"
@ -278,40 +270,6 @@ const (
ObjectTTLAnnotationKey string = "node.alpha.kubernetes.io/ttl"
)
// GetTolerationsFromPodAnnotations gets the json serialized tolerations data from Pod.Annotations
// and converts it to the []Toleration type in api.
func GetTolerationsFromPodAnnotations(annotations map[string]string) ([]Toleration, error) {
var tolerations []Toleration
if len(annotations) > 0 && annotations[TolerationsAnnotationKey] != "" {
err := json.Unmarshal([]byte(annotations[TolerationsAnnotationKey]), &tolerations)
if err != nil {
return tolerations, err
}
}
return tolerations, nil
}
func GetPodTolerations(pod *Pod) ([]Toleration, error) {
return GetTolerationsFromPodAnnotations(pod.Annotations)
}
// GetTaintsFromNodeAnnotations gets the json serialized taints data from Pod.Annotations
// and converts it to the []Taint type in api.
func GetTaintsFromNodeAnnotations(annotations map[string]string) ([]Taint, error) {
var taints []Taint
if len(annotations) > 0 && annotations[TaintsAnnotationKey] != "" {
err := json.Unmarshal([]byte(annotations[TaintsAnnotationKey]), &taints)
if err != nil {
return []Taint{}, err
}
}
return taints, nil
}
func GetNodeTaints(node *Node) ([]Taint, error) {
return GetTaintsFromNodeAnnotations(node.Annotations)
}
// ToleratesTaint checks if the toleration tolerates the taint.
// The matching follows the rules below:
// (1) Empty toleration.effect means to match all taint effects,

6
staging/src/k8s.io/client-go/pkg/api/v1/types.go
View File

@ -2203,6 +2203,9 @@ type PodSpec struct {
// If not specified, the pod will be dispatched by default scheduler.
// +optional
SchedulerName string `json:"schedulername,omitempty" protobuf:"bytes,19,opt,name=schedulername"`
// If specified, the pod's tolerations.
// +optional
Tolerations []Toleration `json:"tolerations,omitempty" protobuf:"bytes,21,opt,name=tolerations"`
}
// PodSecurityContext holds pod-level security attributes and common container settings.
@ -2934,6 +2937,9 @@ type NodeSpec struct {
// More info: http://releases.k8s.io/HEAD/docs/admin/node.md#manual-node-administration
// +optional
Unschedulable bool `json:"unschedulable,omitempty" protobuf:"varint,4,opt,name=unschedulable"`
// If specified, the node's taints.
// +optional
Taints []Taint `json:"taints,omitempty" protobuf:"bytes,5,opt,name=taints"`
}
// DaemonEndpoint contains information about a single Daemon endpoint.