diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index e1ce0d07db1..f5050ce572b 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -507,6 +507,7 @@ function start-kubelet { if [[ ! -z "${KUBELET_APISERVER:-}" && ! -z "${KUBELET_CERT:-}" && ! -z "${KUBELET_KEY:-}" ]]; then flags+=" --api-servers=https://${KUBELET_APISERVER}" flags+=" --register-schedulable=false" + flags+=" --register-with-taints=node.alpha.kubernetes.io/ismaster=:NoSchedule" else # Standalone mode (not widely used?) flags+=" --pod-cidr=${MASTER_IP_RANGE}" diff --git a/cluster/saltbase/salt/kubelet/default b/cluster/saltbase/salt/kubelet/default index a24e91590f2..5ac6cf331ab 100644 --- a/cluster/saltbase/salt/kubelet/default +++ b/cluster/saltbase/salt/kubelet/default @@ -33,7 +33,7 @@ # running on the master. {% if grains.kubelet_api_servers is defined -%} {% set api_servers_with_port = "--api-servers=https://" + grains.kubelet_api_servers -%} - {% set master_kubelet_args = master_kubelet_args + "--register-schedulable=false" -%} + {% set master_kubelet_args = master_kubelet_args + "--register-schedulable=false --register-with-taints=node.alpha.kubernetes.io/ismaster=:NoSchedule" -%} {% else -%} {% set api_servers_with_port = "" -%} {% endif -%} diff --git a/cmd/kubelet/app/options/BUILD b/cmd/kubelet/app/options/BUILD index 4dee7ec8469..c684a9b8c8c 100644 --- a/cmd/kubelet/app/options/BUILD +++ b/cmd/kubelet/app/options/BUILD @@ -20,6 +20,7 @@ go_library( "//pkg/apis/componentconfig/v1alpha1:go_default_library", "//pkg/util/config:go_default_library", "//pkg/util/flag:go_default_library", + "//pkg/util/taints:go_default_library", "//vendor:github.com/spf13/pflag", ], ) diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 610701e8eb2..6e86e343057 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -26,6 +26,7 @@ import ( "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1" utilconfig "k8s.io/kubernetes/pkg/util/config" "k8s.io/kubernetes/pkg/util/flag" + utiltaints "k8s.io/kubernetes/pkg/util/taints" "github.com/spf13/pflag" ) @@ -232,6 +233,8 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { fs.Var(&s.SystemReserved, "system-reserved", "A set of ResourceName=ResourceQuantity (e.g. cpu=200m,memory=150G) pairs that describe resources reserved for non-kubernetes components. Currently only cpu and memory are supported. See http://kubernetes.io/docs/user-guide/compute-resources for more detail. [default=none]") fs.Var(&s.KubeReserved, "kube-reserved", "A set of ResourceName=ResourceQuantity (e.g. cpu=200m,memory=150G) pairs that describe resources reserved for kubernetes system components. Currently only cpu and memory are supported. See http://kubernetes.io/docs/user-guide/compute-resources for more detail. [default=none]") fs.BoolVar(&s.RegisterSchedulable, "register-schedulable", s.RegisterSchedulable, "Register the node as schedulable. Won't have any effect if register-node is false. [default=true]") + fs.MarkDeprecated("register-schedulable", "will be removed in a future version") + fs.Var(utiltaints.NewTaintsVar(&s.RegisterWithTaints), "register-with-taints", "Register the node with the given list of taints (comma seperated \"=:\"). No-op if register-node is false.") fs.StringVar(&s.ContentType, "kube-api-content-type", s.ContentType, "Content type of requests sent to apiserver.") fs.Int32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver") fs.Int32Var(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver") diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 54ac50c9b47..d9b19bad7b9 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -485,6 +485,7 @@ reconcile-interval register-node register-retry-count register-schedulable +register-with-taints registry-burst registry-qps reject-methods diff --git a/pkg/apis/componentconfig/types.go b/pkg/apis/componentconfig/types.go index b34de61372c..cd87c983671 100644 --- a/pkg/apis/componentconfig/types.go +++ b/pkg/apis/componentconfig/types.go @@ -17,6 +17,7 @@ limitations under the License. package componentconfig import ( + "k8s.io/kubernetes/pkg/api" metav1 "k8s.io/kubernetes/pkg/apis/meta/v1" utilconfig "k8s.io/kubernetes/pkg/util/config" ) @@ -379,7 +380,12 @@ type KubeletConfiguration struct { ReconcileCIDR bool `json:"reconcileCIDR"` // registerSchedulable tells the kubelet to register the node as // schedulable. Won't have any effect if register-node is false. + // DEPRECATED: use registerWithTaints instead RegisterSchedulable bool `json:"registerSchedulable"` + // registerWithTaints are an array of taints to add to a node object when + // the kubelet registers itself. This only takes effect when registerNode + // is true and upon the initial registration of the node. + RegisterWithTaints []api.Taint `json:"registerWithTaints"` // contentType is contentType of requests sent to apiserver. ContentType string `json:"contentType"` // kubeAPIQPS is the QPS to use while talking with kubernetes apiserver diff --git a/pkg/apis/componentconfig/v1alpha1/types.go b/pkg/apis/componentconfig/v1alpha1/types.go index 24514b8f515..16b1f24fec0 100644 --- a/pkg/apis/componentconfig/v1alpha1/types.go +++ b/pkg/apis/componentconfig/v1alpha1/types.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha1 import ( + "k8s.io/kubernetes/pkg/api/v1" metav1 "k8s.io/kubernetes/pkg/apis/meta/v1" ) @@ -429,7 +430,12 @@ type KubeletConfiguration struct { ReconcileCIDR *bool `json:"reconcileCIDR"` // registerSchedulable tells the kubelet to register the node as // schedulable. Won't have any effect if register-node is false. + // DEPRECATED: use registerWithTaints instead RegisterSchedulable *bool `json:"registerSchedulable"` + // registerWithTaints are an array of taints to add to a node object when + // the kubelet registers itself. This only takes effect when registerNode + // is true and upon the initial registration of the node. + RegisterWithTaints []v1.Taint `json:"registerWithTaints"` // contentType is contentType of requests sent to apiserver. ContentType string `json:"contentType"` // kubeAPIQPS is the QPS to use while talking with kubernetes apiserver diff --git a/pkg/kubectl/cmd/BUILD b/pkg/kubectl/cmd/BUILD index 3e07f32abaa..a561e34ded7 100644 --- a/pkg/kubectl/cmd/BUILD +++ b/pkg/kubectl/cmd/BUILD @@ -106,6 +106,7 @@ go_library( "//pkg/util/intstr:go_default_library", "//pkg/util/sets:go_default_library", "//pkg/util/strategicpatch:go_default_library", + "//pkg/util/taints:go_default_library", "//pkg/util/term:go_default_library", "//pkg/util/validation:go_default_library", "//pkg/util/validation/field:go_default_library", diff --git a/pkg/kubectl/cmd/taint.go b/pkg/kubectl/cmd/taint.go index 6961dc31145..fdb1ba2ed50 100644 --- a/pkg/kubectl/cmd/taint.go +++ b/pkg/kubectl/cmd/taint.go @@ -35,6 +35,7 @@ import ( utilerrors "k8s.io/kubernetes/pkg/util/errors" "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/util/strategicpatch" + utiltaints "k8s.io/kubernetes/pkg/util/taints" "k8s.io/kubernetes/pkg/util/validation" ) @@ -171,28 +172,10 @@ func parseTaints(spec []string) ([]api.Taint, []api.Taint, error) { for _, taintSpec := range spec { if strings.Index(taintSpec, "=") != -1 && strings.Index(taintSpec, ":") != -1 { - parts := strings.Split(taintSpec, "=") - if len(parts) != 2 || len(parts[1]) == 0 || len(validation.IsQualifiedName(parts[0])) > 0 { - return nil, nil, fmt.Errorf("invalid taint spec: %v", taintSpec) + newTaint, err := utiltaints.ParseTaint(taintSpec) + if err != nil { + return nil, nil, err } - - parts2 := strings.Split(parts[1], ":") - errs := validation.IsValidLabelValue(parts2[0]) - if len(parts2) != 2 || len(errs) != 0 { - return nil, nil, fmt.Errorf("invalid taint spec: %v, %s", taintSpec, strings.Join(errs, "; ")) - } - - if parts2[1] != string(api.TaintEffectNoSchedule) && parts2[1] != string(api.TaintEffectPreferNoSchedule) { - return nil, nil, fmt.Errorf("invalid taint spec: %v, unsupported taint effect", taintSpec) - } - - effect := api.TaintEffect(parts2[1]) - newTaint := api.Taint{ - Key: parts[0], - Value: parts2[0], - Effect: effect, - } - // validate if taint is unique by if len(uniqueTaints[newTaint.Effect]) > 0 && uniqueTaints[newTaint.Effect].Has(newTaint.Key) { return nil, nil, fmt.Errorf("duplicated taints with the same key and effect: %v", newTaint) diff --git a/pkg/kubelet/kubelet_node_status.go b/pkg/kubelet/kubelet_node_status.go index 4c938fb4a83..e9b589bafae 100644 --- a/pkg/kubelet/kubelet_node_status.go +++ b/pkg/kubelet/kubelet_node_status.go @@ -17,6 +17,7 @@ limitations under the License. package kubelet import ( + "encoding/json" "fmt" "math" "net" @@ -185,6 +186,16 @@ func (kl *Kubelet) initialNode() (*v1.Node, error) { Unschedulable: !kl.registerSchedulable, }, } + if len(kl.kubeletConfiguration.RegisterWithTaints) > 0 { + annotations := make(map[string]string) + b, err := json.Marshal(kl.kubeletConfiguration.RegisterWithTaints) + if err != nil { + return nil, err + } + annotations[v1.TaintsAnnotationKey] = string(b) + node.ObjectMeta.Annotations = annotations + + } // Initially, set NodeNetworkUnavailable to true. if kl.providerRequiresNetworkingConfiguration() { node.Status.Conditions = append(node.Status.Conditions, v1.NodeCondition{ diff --git a/pkg/util/taints/BUILD b/pkg/util/taints/BUILD new file mode 100644 index 00000000000..64c4aa24b46 --- /dev/null +++ b/pkg/util/taints/BUILD @@ -0,0 +1,32 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_binary", + "go_library", + "go_test", + "cgo_library", +) + +go_library( + name = "go_default_library", + srcs = ["taints.go"], + tags = ["automanaged"], + deps = [ + "//pkg/api:go_default_library", + "//pkg/util/validation:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["taints_test.go"], + library = "go_default_library", + tags = ["automanaged"], + deps = [ + "//pkg/api:go_default_library", + "//vendor:github.com/spf13/pflag", + ], +) diff --git a/pkg/util/taints/taints.go b/pkg/util/taints/taints.go new file mode 100644 index 00000000000..b36680bd392 --- /dev/null +++ b/pkg/util/taints/taints.go @@ -0,0 +1,95 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// package taints implements uitilites for working with taints +package taints + +import ( + "fmt" + "strings" + + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/util/validation" +) + +// ParseTaint parses a taint from a string. Taint must be off the format '=:'. +func ParseTaint(st string) (api.Taint, error) { + var taint api.Taint + parts := strings.Split(st, "=") + if len(parts) != 2 || len(parts[1]) == 0 || len(validation.IsQualifiedName(parts[0])) > 0 { + return taint, fmt.Errorf("invalid taint spec: %v", st) + } + + parts2 := strings.Split(parts[1], ":") + + effect := api.TaintEffect(parts2[1]) + + errs := validation.IsValidLabelValue(parts2[0]) + if len(parts2) != 2 || len(errs) != 0 { + return taint, fmt.Errorf("invalid taint spec: %v, %s", st, strings.Join(errs, "; ")) + } + + if effect != api.TaintEffectNoSchedule && effect != api.TaintEffectPreferNoSchedule { + return taint, fmt.Errorf("invalid taint spec: %v, unsupported taint effect", st) + } + + taint.Key = parts[0] + taint.Value = parts2[0] + taint.Effect = effect + + return taint, nil +} + +// NewTaintsVar wraps []api.Taint in a struct that implements flag.Value to allow taints to be +// bound to command line flags. +func NewTaintsVar(ptr *[]api.Taint) taintsVar { + return taintsVar{ + ptr: ptr, + } +} + +type taintsVar struct { + ptr *[]api.Taint +} + +func (t taintsVar) Set(s string) error { + sts := strings.Split(s, ",") + var taints []api.Taint + for _, st := range sts { + taint, err := ParseTaint(st) + if err != nil { + return err + } + taints = append(taints, taint) + } + *t.ptr = taints + return nil +} + +func (t taintsVar) String() string { + if len(*t.ptr) == 0 { + return "" + } + var taints []string + for _, taint := range *t.ptr { + taints = append(taints, fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect)) + } + return strings.Join(taints, ",") +} + +func (t taintsVar) Type() string { + return "[]api.Taint" +} diff --git a/pkg/util/taints/taints_test.go b/pkg/util/taints/taints_test.go new file mode 100644 index 00000000000..51edcff1957 --- /dev/null +++ b/pkg/util/taints/taints_test.go @@ -0,0 +1,72 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package taints + +import ( + "reflect" + "strings" + "testing" + + "k8s.io/kubernetes/pkg/api" + + "github.com/spf13/pflag" +) + +func TestTaintsVar(t *testing.T) { + cases := []struct { + f string + err bool + t []api.Taint + }{ + { + f: "", + t: []api.Taint(nil), + }, + { + f: "--t=foo=bar:NoSchedule", + t: []api.Taint{{Key: "foo", Value: "bar", Effect: "NoSchedule"}}, + }, + { + f: "--t=foo=bar:NoSchedule,bing=bang:PreferNoSchedule", + t: []api.Taint{ + {Key: "foo", Value: "bar", Effect: api.TaintEffectNoSchedule}, + {Key: "bing", Value: "bang", Effect: api.TaintEffectPreferNoSchedule}, + }, + }, + } + + for i, c := range cases { + args := append([]string{"test"}, strings.Fields(c.f)...) + cli := pflag.NewFlagSet("test", pflag.ContinueOnError) + var taints []api.Taint + cli.Var(NewTaintsVar(&taints), "t", "bar") + + err := cli.Parse(args) + if err == nil && c.err { + t.Errorf("[%v] expected error", i) + continue + } + if err != nil && !c.err { + t.Errorf("[%v] unexpected error: %v", i, err) + continue + } + if !reflect.DeepEqual(c.t, taints) { + t.Errorf("[%v] unexpected taints:\n\texpected:\n\t\t%#v\n\tgot:\n\t\t%#v", i, c.t, taints) + } + } + +}