From bad4faf1b9d69d3cb6e8de90dc5eee1d8cf0b6d3 Mon Sep 17 00:00:00 2001 From: caozhiyuan Date: Mon, 4 Oct 2021 09:17:10 +0800 Subject: [PATCH] migrate --register-with-taints to KubeletConfiguration --- api/api-rules/violation_exceptions.list | 1 + cmd/kubelet/app/options/options.go | 18 +-- cmd/kubelet/app/server.go | 3 +- cmd/kubemark/hollow-node.go | 7 +- pkg/kubelet/apis/config/helpers_test.go | 2 + pkg/kubelet/apis/config/register_test.go | 1 + .../KubeletConfiguration/after/v1beta1.yaml | 1 + .../roundtrip/default/v1beta1.yaml | 1 + pkg/kubelet/apis/config/types.go | 9 ++ pkg/kubelet/apis/config/v1beta1/defaults.go | 3 + .../apis/config/v1beta1/defaults_test.go | 6 + .../config/v1beta1/zz_generated.conversion.go | 8 ++ .../apis/config/validation/validation.go | 11 ++ .../apis/config/zz_generated.deepcopy.go | 7 ++ pkg/kubelet/kubelet.go | 5 +- pkg/kubelet/kubelet_node_status.go | 14 +-- pkg/kubemark/hollow_kubelet.go | 8 +- pkg/util/flag/flags.go | 43 +++++++ pkg/util/flag/flags_test.go | 60 ++++++++++ pkg/util/taints/taints.go | 68 ++++------- pkg/util/taints/taints_test.go | 111 ++++++++---------- .../k8s.io/kubelet/config/v1beta1/types.go | 10 ++ .../config/v1beta1/zz_generated.deepcopy.go | 12 ++ 23 files changed, 260 insertions(+), 149 deletions(-) diff --git a/api/api-rules/violation_exceptions.list b/api/api-rules/violation_exceptions.list index 27ff722cdd0..d8ea748b82e 100644 --- a/api/api-rules/violation_exceptions.list +++ b/api/api-rules/violation_exceptions.list @@ -379,6 +379,7 @@ API rule violation: list_type_missing,k8s.io/kubelet/config/v1alpha1,CredentialP API rule violation: list_type_missing,k8s.io/kubelet/config/v1beta1,KubeletConfiguration,AllowedUnsafeSysctls API rule violation: list_type_missing,k8s.io/kubelet/config/v1beta1,KubeletConfiguration,ClusterDNS API rule violation: list_type_missing,k8s.io/kubelet/config/v1beta1,KubeletConfiguration,EnforceNodeAllocatable +API rule violation: list_type_missing,k8s.io/kubelet/config/v1beta1,KubeletConfiguration,RegisterWithTaints API rule violation: list_type_missing,k8s.io/kubelet/config/v1beta1,KubeletConfiguration,ReservedMemory API rule violation: list_type_missing,k8s.io/kubelet/config/v1beta1,KubeletConfiguration,TLSCipherSuites API rule violation: list_type_missing,k8s.io/metrics/pkg/apis/metrics/v1alpha1,PodMetrics,Containers diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index f5ad53f6634..8f096e4f277 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -32,7 +32,6 @@ import ( "k8s.io/component-base/logs" "k8s.io/kubelet/config/v1beta1" kubeletapis "k8s.io/kubelet/pkg/apis" - "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/cluster/ports" "k8s.io/kubernetes/pkg/features" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" @@ -40,7 +39,6 @@ import ( kubeletconfigvalidation "k8s.io/kubernetes/pkg/kubelet/apis/config/validation" "k8s.io/kubernetes/pkg/kubelet/config" utilflag "k8s.io/kubernetes/pkg/util/flag" - utiltaints "k8s.io/kubernetes/pkg/util/taints" ) const defaultRootDir = "/var/lib/kubelet" @@ -97,14 +95,6 @@ type KubeletFlags struct { // Omit this flag to use the combination of built-in default configuration values and flags. KubeletConfigFile string - // registerNode enables automatic registration with the apiserver. - RegisterNode bool - - // registerWithTaints are an array of taints to add to a node object when - // the kubelet registers itself. This only takes effect when registerNode - // is true and upon the initial registration of the node. - RegisterWithTaints []core.Taint - // WindowsService should be set to true if kubelet is running as a service on Windows. // Its corresponding flag only gets registered in Windows builds. WindowsService bool @@ -188,7 +178,6 @@ func NewKubeletFlags() *KubeletFlags { RegisterSchedulable: true, RemoteRuntimeEndpoint: remoteRuntimeEndpoint, NodeLabels: make(map[string]string), - RegisterNode: true, } } @@ -339,9 +328,6 @@ func (f *KubeletFlags) AddFlags(mainfs *pflag.FlagSet) { fs.Var(&f.DynamicConfigDir, "dynamic-config-dir", "The Kubelet will use this directory for checkpointing downloaded configurations and tracking configuration health. The Kubelet will create this directory if it does not already exist. The path may be absolute or relative; relative paths start at the Kubelet's current working directory. Providing this flag enables dynamic Kubelet configuration. The DynamicKubeletConfig feature gate must be enabled to pass this flag.") fs.MarkDeprecated("dynamic-config-dir", "Feature DynamicKubeletConfig is deprecated in 1.22 and will not move to GA. It is planned to be removed from Kubernetes in the version 1.23. Please use alternative ways to update kubelet configuration.") - fs.BoolVar(&f.RegisterNode, "register-node", f.RegisterNode, "Register the node with the apiserver. If --kubeconfig is not provided, this flag is irrelevant, as the Kubelet won't have an apiserver to register with.") - fs.Var(utiltaints.NewTaintsVar(&f.RegisterWithTaints), "register-with-taints", "Register the node with the given list of taints (comma separated \"=:\"). No-op if register-node is false.") - // EXPERIMENTAL FLAGS fs.StringVar(&f.RemoteRuntimeEndpoint, "container-runtime-endpoint", f.RemoteRuntimeEndpoint, "[Experimental] The endpoint of remote runtime service. Currently unix socket endpoint is supported on Linux, while npipe and tcp endpoints are supported on windows. Note: When using docker as container runtime this specifies the dockershim socket location which kubelet itself creates. Examples:'unix:///var/run/dockershim.sock', 'npipe:////./pipe/dockershim'") fs.StringVar(&f.RemoteImageEndpoint, "image-service-endpoint", f.RemoteImageEndpoint, "[Experimental] The endpoint of remote image service. If not specified, it will be the same with container-runtime-endpoint by default. Currently unix socket endpoint is supported on Linux, while npipe and tcp endpoints are supported on windows. Examples:'unix:///var/run/dockershim.sock', 'npipe:////./pipe/dockershim'") @@ -556,4 +542,8 @@ func AddKubeletConfigFlags(mainfs *pflag.FlagSet, c *kubeletconfig.KubeletConfig // Memory Manager Flags fs.StringVar(&c.MemoryManagerPolicy, "memory-manager-policy", c.MemoryManagerPolicy, "Memory Manager policy to use. Possible values: 'None', 'Static'.") fs.Var(&utilflag.ReservedMemoryVar{Value: &c.ReservedMemory}, "reserved-memory", "A comma separated list of memory reservations for NUMA nodes. (e.g. --reserved-memory 0:memory=1Gi,hugepages-1M=2Gi --reserved-memory 1:memory=2Gi). The total sum for each memory type should be equal to the sum of kube-reserved, system-reserved and eviction-threshold. See https://kubernetes.io/docs/tasks/administer-cluster/memory-manager/#reserved-memory-flag for more details.") + + fs.BoolVar(&c.RegisterNode, "register-node", c.RegisterNode, "Register the node with the apiserver. If --kubeconfig is not provided, this flag is irrelevant, as the Kubelet won't have an apiserver to register with.") + + fs.Var(&utilflag.RegisterWithTaintsVar{Value: &c.RegisterWithTaints}, "register-with-taints", "Register the node with the given list of taints (comma separated \"=:\"). No-op if register-node is false.") } diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 5648659699e..fa04ac29b21 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -71,7 +71,6 @@ import ( kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" "k8s.io/kubernetes/cmd/kubelet/app/options" "k8s.io/kubernetes/pkg/api/legacyscheme" - api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/capabilities" "k8s.io/kubernetes/pkg/credentialprovider" "k8s.io/kubernetes/pkg/features" @@ -1250,7 +1249,7 @@ func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, imageCredentialProviderConfigFile string, imageCredentialProviderBinDir string, registerNode bool, - registerWithTaints []api.Taint, + registerWithTaints []v1.Taint, allowedUnsafeSysctls []string, experimentalMounterPath string, kernelMemcgNotification bool, diff --git a/cmd/kubemark/hollow-node.go b/cmd/kubemark/hollow-node.go index c0e08143f2a..54258bfa326 100644 --- a/cmd/kubemark/hollow-node.go +++ b/cmd/kubemark/hollow-node.go @@ -43,15 +43,14 @@ import ( "k8s.io/component-base/version/verflag" fakesysctl "k8s.io/component-helpers/node/util/sysctl/testing" "k8s.io/kubernetes/pkg/api/legacyscheme" - "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/cluster/ports" cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing" "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/pkg/kubelet/cri/remote" fakeremote "k8s.io/kubernetes/pkg/kubelet/cri/remote/fake" "k8s.io/kubernetes/pkg/kubemark" + utilflag "k8s.io/kubernetes/pkg/util/flag" fakeiptables "k8s.io/kubernetes/pkg/util/iptables/testing" - utiltaints "k8s.io/kubernetes/pkg/util/taints" fakeexec "k8s.io/utils/exec/testing" ) @@ -67,7 +66,7 @@ type hollowNodeConfig struct { ProxierSyncPeriod time.Duration ProxierMinSyncPeriod time.Duration NodeLabels map[string]string - RegisterWithTaints []core.Taint + RegisterWithTaints []v1.Taint MaxPods int ExtendedResources map[string]string UseHostImageService bool @@ -95,7 +94,7 @@ func (c *hollowNodeConfig) addFlags(fs *pflag.FlagSet) { fs.DurationVar(&c.ProxierMinSyncPeriod, "proxier-min-sync-period", 0, "Minimum period that proxy rules are refreshed in hollow-proxy.") bindableNodeLabels := cliflag.ConfigurationMap(c.NodeLabels) fs.Var(&bindableNodeLabels, "node-labels", "Additional node labels") - fs.Var(utiltaints.NewTaintsVar(&c.RegisterWithTaints), "register-with-taints", "Register the node with the given list of taints (comma separated \"=:\"). No-op if register-node is false.") + fs.Var(utilflag.RegisterWithTaintsVar{Value: &c.RegisterWithTaints}, "register-with-taints", "Register the node with the given list of taints (comma separated \"=:\"). No-op if register-node is false.") fs.IntVar(&c.MaxPods, "max-pods", maxPods, "Number of pods that can run on this Kubelet.") bindableExtendedResources := cliflag.ConfigurationMap(c.ExtendedResources) fs.Var(&bindableExtendedResources, "extended-resources", "Register the node with extended resources (comma separated \"=\")") diff --git a/pkg/kubelet/apis/config/helpers_test.go b/pkg/kubelet/apis/config/helpers_test.go index aadeba2f7c7..29361abd450 100644 --- a/pkg/kubelet/apis/config/helpers_test.go +++ b/pkg/kubelet/apis/config/helpers_test.go @@ -256,10 +256,12 @@ var ( "ProtectKernelDefaults", "ProviderID", "ReadOnlyPort", + "RegisterNode", "RegistryBurst", "RegistryPullQPS", "ReservedMemory", "ReservedSystemCPUs", + "RegisterWithTaints", "RuntimeRequestTimeout.Duration", "RunOnce", "SeccompDefault", diff --git a/pkg/kubelet/apis/config/register_test.go b/pkg/kubelet/apis/config/register_test.go index 2c439b09270..87f7adbda15 100644 --- a/pkg/kubelet/apis/config/register_test.go +++ b/pkg/kubelet/apis/config/register_test.go @@ -35,6 +35,7 @@ func TestComponentConfigSetup(t *testing.T) { reflect.TypeOf(metav1.TypeMeta{}): true, reflect.TypeOf(metav1.Duration{}): true, reflect.TypeOf(v1.NodeConfigSource{}): true, + reflect.TypeOf(v1.Taint{}): true, }, } diff --git a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml index 53772e91ffc..3cb76b89923 100644 --- a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml +++ b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml @@ -72,6 +72,7 @@ nodeStatusUpdateFrequency: 10s oomScoreAdj: -999 podPidsLimit: -1 port: 10250 +registerNode: true registryBurst: 10 registryPullQPS: 5 resolvConf: /etc/resolv.conf diff --git a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml index 53772e91ffc..3cb76b89923 100644 --- a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml +++ b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml @@ -72,6 +72,7 @@ nodeStatusUpdateFrequency: 10s oomScoreAdj: -999 podPidsLimit: -1 port: 10250 +registerNode: true registryBurst: 10 registryPullQPS: 5 resolvConf: /etc/resolv.conf diff --git a/pkg/kubelet/apis/config/types.go b/pkg/kubelet/apis/config/types.go index c407769a017..4855b3e2a04 100644 --- a/pkg/kubelet/apis/config/types.go +++ b/pkg/kubelet/apis/config/types.go @@ -426,6 +426,15 @@ type KubeletConfiguration struct { // +featureGate=MemoryQoS // +optional MemoryThrottlingFactor *float64 + // registerWithTaints are an array of taints to add to a node object when + // the kubelet registers itself. This only takes effect when registerNode + // is true and upon the initial registration of the node. + // +optional + RegisterWithTaints []v1.Taint + + // registerNode enables automatic registration with the apiserver. + // +optional + RegisterNode bool } // KubeletAuthorizationMode denotes the authorization mode for the kubelet diff --git a/pkg/kubelet/apis/config/v1beta1/defaults.go b/pkg/kubelet/apis/config/v1beta1/defaults.go index 9a89c8b0eff..eee1d0945da 100644 --- a/pkg/kubelet/apis/config/v1beta1/defaults.go +++ b/pkg/kubelet/apis/config/v1beta1/defaults.go @@ -261,4 +261,7 @@ func SetDefaults_KubeletConfiguration(obj *kubeletconfigv1beta1.KubeletConfigura if obj.MemoryThrottlingFactor == nil { obj.MemoryThrottlingFactor = utilpointer.Float64Ptr(DefaultMemoryThrottlingFactor) } + if obj.RegisterNode == nil { + obj.RegisterNode = utilpointer.BoolPtr(true) + } } diff --git a/pkg/kubelet/apis/config/v1beta1/defaults_test.go b/pkg/kubelet/apis/config/v1beta1/defaults_test.go index 0949badabfe..63ae4db9b5e 100644 --- a/pkg/kubelet/apis/config/v1beta1/defaults_test.go +++ b/pkg/kubelet/apis/config/v1beta1/defaults_test.go @@ -120,6 +120,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { EnableDebugFlagsHandler: utilpointer.BoolPtr(true), SeccompDefault: utilpointer.BoolPtr(false), MemoryThrottlingFactor: utilpointer.Float64Ptr(DefaultMemoryThrottlingFactor), + RegisterNode: utilpointer.BoolPtr(true), }, }, { @@ -244,6 +245,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { EnableDebugFlagsHandler: utilpointer.Bool(false), SeccompDefault: utilpointer.Bool(false), MemoryThrottlingFactor: utilpointer.Float64(0), + RegisterNode: utilpointer.BoolPtr(false), }, &v1beta1.KubeletConfiguration{ EnableServer: utilpointer.BoolPtr(false), @@ -339,6 +341,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { EnableDebugFlagsHandler: utilpointer.Bool(false), SeccompDefault: utilpointer.Bool(false), MemoryThrottlingFactor: utilpointer.Float64(0), + RegisterNode: utilpointer.BoolPtr(false), }, }, { @@ -488,6 +491,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { EnableDebugFlagsHandler: utilpointer.Bool(true), SeccompDefault: utilpointer.Bool(true), MemoryThrottlingFactor: utilpointer.Float64(1), + RegisterNode: utilpointer.BoolPtr(true), }, &v1beta1.KubeletConfiguration{ EnableServer: utilpointer.BoolPtr(true), @@ -634,6 +638,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { EnableDebugFlagsHandler: utilpointer.Bool(true), SeccompDefault: utilpointer.Bool(true), MemoryThrottlingFactor: utilpointer.Float64(1), + RegisterNode: utilpointer.BoolPtr(true), }, }, { @@ -719,6 +724,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { EnableDebugFlagsHandler: utilpointer.BoolPtr(true), SeccompDefault: utilpointer.BoolPtr(false), MemoryThrottlingFactor: utilpointer.Float64Ptr(DefaultMemoryThrottlingFactor), + RegisterNode: utilpointer.BoolPtr(true), }, }, } diff --git a/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go b/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go index 45e8364e542..7c4dbe4203d 100644 --- a/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go +++ b/pkg/kubelet/apis/config/v1beta1/zz_generated.conversion.go @@ -392,6 +392,10 @@ func autoConvert_v1beta1_KubeletConfiguration_To_config_KubeletConfiguration(in return err } out.MemoryThrottlingFactor = (*float64)(unsafe.Pointer(in.MemoryThrottlingFactor)) + out.RegisterWithTaints = *(*[]corev1.Taint)(unsafe.Pointer(&in.RegisterWithTaints)) + if err := v1.Convert_Pointer_bool_To_bool(&in.RegisterNode, &out.RegisterNode, s); err != nil { + return err + } return nil } @@ -563,6 +567,10 @@ func autoConvert_config_KubeletConfiguration_To_v1beta1_KubeletConfiguration(in return err } out.MemoryThrottlingFactor = (*float64)(unsafe.Pointer(in.MemoryThrottlingFactor)) + out.RegisterWithTaints = *(*[]corev1.Taint)(unsafe.Pointer(&in.RegisterWithTaints)) + if err := v1.Convert_bool_To_Pointer_bool(&in.RegisterNode, &out.RegisterNode, s); err != nil { + return err + } return nil } diff --git a/pkg/kubelet/apis/config/validation/validation.go b/pkg/kubelet/apis/config/validation/validation.go index 700b8cf0da1..585e6639af5 100644 --- a/pkg/kubelet/apis/config/validation/validation.go +++ b/pkg/kubelet/apis/config/validation/validation.go @@ -31,6 +31,7 @@ import ( kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" + utiltaints "k8s.io/kubernetes/pkg/util/taints" ) var ( @@ -126,6 +127,16 @@ func ValidateKubeletConfiguration(kc *kubeletconfig.KubeletConfiguration) error if kc.TopologyManagerPolicy != kubeletconfig.NoneTopologyManagerPolicy && !localFeatureGate.Enabled(features.TopologyManager) { allErrors = append(allErrors, fmt.Errorf("invalid configuration: topologyManagerPolicy %v requires feature gate TopologyManager", kc.TopologyManagerPolicy)) } + + for _, nodeTaint := range kc.RegisterWithTaints { + if err := utiltaints.CheckTaintValidation(nodeTaint); err != nil { + allErrors = append(allErrors, fmt.Errorf("invalid taint: %v", nodeTaint)) + } + if nodeTaint.TimeAdded != nil { + allErrors = append(allErrors, fmt.Errorf("taint TimeAdded is not nil")) + } + } + switch kc.TopologyManagerPolicy { case kubeletconfig.NoneTopologyManagerPolicy: case kubeletconfig.BestEffortTopologyManagerPolicy: diff --git a/pkg/kubelet/apis/config/zz_generated.deepcopy.go b/pkg/kubelet/apis/config/zz_generated.deepcopy.go index 82d958c6447..f483437bf2d 100644 --- a/pkg/kubelet/apis/config/zz_generated.deepcopy.go +++ b/pkg/kubelet/apis/config/zz_generated.deepcopy.go @@ -295,6 +295,13 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { *out = new(float64) **out = **in } + if in.RegisterWithTaints != nil { + in, out := &in.RegisterWithTaints, &out.RegisterWithTaints + *out = make([]corev1.Taint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } return } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 2b286285411..cbb79413977 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -62,7 +62,6 @@ import ( "k8s.io/klog/v2" pluginwatcherapi "k8s.io/kubelet/pkg/apis/pluginregistration/v1" statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1" - api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/features" kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config" "k8s.io/kubernetes/pkg/kubelet/apis/podresources" @@ -360,7 +359,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, imageCredentialProviderConfigFile string, imageCredentialProviderBinDir string, registerNode bool, - registerWithTaints []api.Taint, + registerWithTaints []v1.Taint, allowedUnsafeSysctls []string, experimentalMounterPath string, kernelMemcgNotification bool, @@ -944,7 +943,7 @@ type Kubelet struct { // Set to true to have the node register itself with the apiserver. registerNode bool // List of taints to add to a node object when the kubelet registers itself. - registerWithTaints []api.Taint + registerWithTaints []v1.Taint // Set to true to have the node register itself as schedulable. registerSchedulable bool // for internal book keeping; access only from within registerWithApiserver diff --git a/pkg/kubelet/kubelet_node_status.go b/pkg/kubelet/kubelet_node_status.go index 39d0099d932..aabb75bcd2e 100644 --- a/pkg/kubelet/kubelet_node_status.go +++ b/pkg/kubelet/kubelet_node_status.go @@ -37,7 +37,6 @@ import ( nodeutil "k8s.io/component-helpers/node/util" "k8s.io/klog/v2" kubeletapis "k8s.io/kubelet/pkg/apis" - k8s_api_v1 "k8s.io/kubernetes/pkg/apis/core/v1" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" "k8s.io/kubernetes/pkg/kubelet/events" "k8s.io/kubernetes/pkg/kubelet/nodestatus" @@ -308,17 +307,8 @@ func (kl *Kubelet) initialNode(ctx context.Context) (*v1.Node, error) { node.Labels[label] = value } - nodeTaints := make([]v1.Taint, 0) - if len(kl.registerWithTaints) > 0 { - taints := make([]v1.Taint, len(kl.registerWithTaints)) - for i := range kl.registerWithTaints { - if err := k8s_api_v1.Convert_core_Taint_To_v1_Taint(&kl.registerWithTaints[i], &taints[i], nil); err != nil { - return nil, err - } - } - nodeTaints = append(nodeTaints, taints...) - } - + nodeTaints := make([]v1.Taint, len(kl.registerWithTaints)) + copy(nodeTaints, kl.registerWithTaints) unschedulableTaint := v1.Taint{ Key: v1.TaintNodeUnschedulable, Effect: v1.TaintEffectNoSchedule, diff --git a/pkg/kubemark/hollow_kubelet.go b/pkg/kubemark/hollow_kubelet.go index cd789e7c3e4..3108f610c16 100644 --- a/pkg/kubemark/hollow_kubelet.go +++ b/pkg/kubemark/hollow_kubelet.go @@ -20,6 +20,7 @@ import ( "fmt" "time" + v1 "k8s.io/api/core/v1" "k8s.io/klog/v2" "k8s.io/mount-utils" @@ -28,7 +29,6 @@ import ( internalapi "k8s.io/cri-api/pkg/apis" kubeletapp "k8s.io/kubernetes/cmd/kubelet/app" "k8s.io/kubernetes/cmd/kubelet/app/options" - "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/kubelet" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" "k8s.io/kubernetes/pkg/kubelet/cadvisor" @@ -143,7 +143,7 @@ type HollowKubletOptions struct { MaxPods int PodsPerCore int NodeLabels map[string]string - RegisterWithTaints []core.Taint + RegisterWithTaints []v1.Taint } // Builds a KubeletConfiguration for the HollowKubelet, ensuring that the @@ -162,9 +162,7 @@ func GetHollowKubeletConfig(opt *HollowKubletOptions) (*options.KubeletFlags, *k f.MaxPerPodContainerCount = 2 f.NodeLabels = opt.NodeLabels f.ContainerRuntimeOptions.ContainerRuntime = kubetypes.RemoteContainerRuntime - f.RegisterNode = true f.RegisterSchedulable = true - f.RegisterWithTaints = opt.RegisterWithTaints f.RemoteImageEndpoint = "unix:///run/containerd/containerd.sock" // Config struct @@ -209,6 +207,8 @@ func GetHollowKubeletConfig(opt *HollowKubletOptions) (*options.KubeletFlags, *k c.SerializeImagePulls = true c.SystemCgroups = "" c.ProtectKernelDefaults = false + c.RegisterWithTaints = opt.RegisterWithTaints + c.RegisterNode = true return f, c } diff --git a/pkg/util/flag/flags.go b/pkg/util/flag/flags.go index 9113769fcf9..5ecc283974e 100644 --- a/pkg/util/flag/flags.go +++ b/pkg/util/flag/flags.go @@ -30,6 +30,7 @@ import ( utilnet "k8s.io/apimachinery/pkg/util/net" corev1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" + utiltaints "k8s.io/kubernetes/pkg/util/taints" netutils "k8s.io/utils/net" ) @@ -40,6 +41,7 @@ var ( _ pflag.Value = &IPPortVar{} _ pflag.Value = &PortRangeVar{} _ pflag.Value = &ReservedMemoryVar{} + _ pflag.Value = &RegisterWithTaintsVar{} ) // IPVar is used for validating a command line option that represents an IP. It implements the pflag.Value interface @@ -255,3 +257,44 @@ func (v *ReservedMemoryVar) String() string { func (v *ReservedMemoryVar) Type() string { return "reserved-memory" } + +// RegisterWithTaintsVar is used for validating a command line option that represents a register with taints. It implements the pflag.Value interface +type RegisterWithTaintsVar struct { + Value *[]v1.Taint +} + +// Set sets the flag value +func (t RegisterWithTaintsVar) Set(s string) error { + if len(s) == 0 { + *t.Value = nil + return nil + } + sts := strings.Split(s, ",") + corev1Taints, _, err := utiltaints.ParseTaints(sts) + if err != nil { + return err + } + var taints []v1.Taint + for _, ct := range corev1Taints { + taints = append(taints, v1.Taint{Key: ct.Key, Value: ct.Value, Effect: v1.TaintEffect(ct.Effect)}) + } + *t.Value = taints + return nil +} + +// String returns the flag value +func (t RegisterWithTaintsVar) String() string { + if len(*t.Value) == 0 { + return "" + } + var taints []string + for _, taint := range *t.Value { + taints = append(taints, fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect)) + } + return strings.Join(taints, ",") +} + +// Type gets the flag type +func (t RegisterWithTaintsVar) Type() string { + return "[]v1.Taint" +} diff --git a/pkg/util/flag/flags_test.go b/pkg/util/flag/flags_test.go index cd1edd0f5e8..04daa8ddb2f 100644 --- a/pkg/util/flag/flags_test.go +++ b/pkg/util/flag/flags_test.go @@ -18,6 +18,7 @@ package flag import ( "fmt" + "reflect" "strings" "testing" @@ -287,3 +288,62 @@ func TestReservedMemoryVar(t *testing.T) { } } } + +func TestTaintsVar(t *testing.T) { + cases := []struct { + f string + err bool + t []v1.Taint + }{ + { + f: "", + t: []v1.Taint(nil), + }, + { + f: "--t=foo=bar:NoSchedule", + t: []v1.Taint{{Key: "foo", Value: "bar", Effect: "NoSchedule"}}, + }, + { + f: "--t=baz:NoSchedule", + t: []v1.Taint{{Key: "baz", Value: "", Effect: "NoSchedule"}}, + }, + { + f: "--t=foo=bar:NoSchedule,baz:NoSchedule,bing=bang:PreferNoSchedule,qux=:NoSchedule", + t: []v1.Taint{ + {Key: "foo", Value: "bar", Effect: v1.TaintEffectNoSchedule}, + {Key: "baz", Value: "", Effect: "NoSchedule"}, + {Key: "bing", Value: "bang", Effect: v1.TaintEffectPreferNoSchedule}, + {Key: "qux", Value: "", Effect: "NoSchedule"}, + }, + }, + { + f: "--t=dedicated-for=user1:NoExecute,baz:NoSchedule,foo-bar=:NoSchedule", + t: []v1.Taint{ + {Key: "dedicated-for", Value: "user1", Effect: "NoExecute"}, + {Key: "baz", Value: "", Effect: "NoSchedule"}, + {Key: "foo-bar", Value: "", Effect: "NoSchedule"}, + }, + }, + } + + for i, c := range cases { + args := append([]string{"test"}, strings.Fields(c.f)...) + cli := pflag.NewFlagSet("test", pflag.ContinueOnError) + var taints []v1.Taint + cli.Var(RegisterWithTaintsVar{Value: &taints}, "t", "bar") + + err := cli.Parse(args) + if err == nil && c.err { + t.Errorf("[%v] expected error", i) + continue + } + if err != nil && !c.err { + t.Errorf("[%v] unexpected error: %v", i, err) + continue + } + if !reflect.DeepEqual(c.t, taints) { + t.Errorf("[%v] unexpected taints:\n\texpected:\n\t\t%#v\n\tgot:\n\t\t%#v", i, c.t, taints) + } + } + +} diff --git a/pkg/util/taints/taints.go b/pkg/util/taints/taints.go index 3d7d560506f..4cfe227f8b8 100644 --- a/pkg/util/taints/taints.go +++ b/pkg/util/taints/taints.go @@ -21,11 +21,10 @@ import ( "fmt" "strings" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation" - api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/core/helper" ) @@ -88,51 +87,6 @@ func validateTaintEffect(effect v1.TaintEffect) error { return nil } -// NewTaintsVar wraps []api.Taint in a struct that implements flag.Value to allow taints to be -// bound to command line flags. -func NewTaintsVar(ptr *[]api.Taint) taintsVar { - return taintsVar{ - ptr: ptr, - } -} - -type taintsVar struct { - ptr *[]api.Taint -} - -func (t taintsVar) Set(s string) error { - if len(s) == 0 { - *t.ptr = nil - return nil - } - sts := strings.Split(s, ",") - var taints []api.Taint - for _, st := range sts { - taint, err := parseTaint(st) - if err != nil { - return err - } - taints = append(taints, api.Taint{Key: taint.Key, Value: taint.Value, Effect: api.TaintEffect(taint.Effect)}) - } - *t.ptr = taints - return nil -} - -func (t taintsVar) String() string { - if len(*t.ptr) == 0 { - return "" - } - var taints []string - for _, taint := range *t.ptr { - taints = append(taints, fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect)) - } - return strings.Join(taints, ",") -} - -func (t taintsVar) Type() string { - return "[]api.Taint" -} - // ParseTaints takes a spec which is an array and creates slices for new taints to be added, taints to be deleted. // It also validates the spec. For example, the form `` may be used to remove a taint, but not to add one. func ParseTaints(spec []string) ([]v1.Taint, []v1.Taint, error) { @@ -350,3 +304,23 @@ func TaintSetFilter(taints []v1.Taint, fn func(*v1.Taint) bool) []v1.Taint { return res } + +// CheckTaintValidation checks if the given taint is valid. +// Returns error if the given taint is invalid. +func CheckTaintValidation(taint v1.Taint) error { + if errs := validation.IsQualifiedName(taint.Key); len(errs) > 0 { + return fmt.Errorf("invalid taint key: %s", strings.Join(errs, "; ")) + } + if taint.Value != "" { + if errs := validation.IsValidLabelValue(taint.Value); len(errs) > 0 { + return fmt.Errorf("invalid taint value: %s", strings.Join(errs, "; ")) + } + } + if taint.Effect != "" { + if err := validateTaintEffect(taint.Effect); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/util/taints/taints_test.go b/pkg/util/taints/taints_test.go index a2dfca67c85..966dd9ee5b2 100644 --- a/pkg/util/taints/taints_test.go +++ b/pkg/util/taints/taints_test.go @@ -21,71 +21,9 @@ import ( "strings" "testing" - "k8s.io/api/core/v1" - api "k8s.io/kubernetes/pkg/apis/core" - - "github.com/spf13/pflag" + v1 "k8s.io/api/core/v1" ) -func TestTaintsVar(t *testing.T) { - cases := []struct { - f string - err bool - t []api.Taint - }{ - { - f: "", - t: []api.Taint(nil), - }, - { - f: "--t=foo=bar:NoSchedule", - t: []api.Taint{{Key: "foo", Value: "bar", Effect: "NoSchedule"}}, - }, - { - f: "--t=baz:NoSchedule", - t: []api.Taint{{Key: "baz", Value: "", Effect: "NoSchedule"}}, - }, - { - f: "--t=foo=bar:NoSchedule,baz:NoSchedule,bing=bang:PreferNoSchedule,qux=:NoSchedule", - t: []api.Taint{ - {Key: "foo", Value: "bar", Effect: api.TaintEffectNoSchedule}, - {Key: "baz", Value: "", Effect: "NoSchedule"}, - {Key: "bing", Value: "bang", Effect: api.TaintEffectPreferNoSchedule}, - {Key: "qux", Value: "", Effect: "NoSchedule"}, - }, - }, - { - f: "--t=dedicated-for=user1:NoExecute,baz:NoSchedule,foo-bar=:NoSchedule", - t: []api.Taint{ - {Key: "dedicated-for", Value: "user1", Effect: "NoExecute"}, - {Key: "baz", Value: "", Effect: "NoSchedule"}, - {Key: "foo-bar", Value: "", Effect: "NoSchedule"}, - }, - }, - } - - for i, c := range cases { - args := append([]string{"test"}, strings.Fields(c.f)...) - cli := pflag.NewFlagSet("test", pflag.ContinueOnError) - var taints []api.Taint - cli.Var(NewTaintsVar(&taints), "t", "bar") - - err := cli.Parse(args) - if err == nil && c.err { - t.Errorf("[%v] expected error", i) - continue - } - if err != nil && !c.err { - t.Errorf("[%v] unexpected error: %v", i, err) - continue - } - if !reflect.DeepEqual(c.t, taints) { - t.Errorf("[%v] unexpected taints:\n\texpected:\n\t\t%#v\n\tgot:\n\t\t%#v", i, c.t, taints) - } - } - -} - func TestAddOrUpdateTaint(t *testing.T) { node := &v1.Node{} @@ -757,3 +695,50 @@ func TestParseTaints(t *testing.T) { } } } + +func TestValidateTaint(t *testing.T) { + cases := []struct { + name string + taintsToCheck v1.Taint + expectedErr bool + }{ + { + name: "taint invalid key", + taintsToCheck: v1.Taint{Key: "", Value: "bar_1", Effect: v1.TaintEffectNoExecute}, + expectedErr: true, + }, + { + name: "taint invalid value", + taintsToCheck: v1.Taint{Key: "foo_1", Value: strings.Repeat("a", 64), Effect: v1.TaintEffectNoExecute}, + expectedErr: true, + }, + { + name: "taint invalid effect", + taintsToCheck: v1.Taint{Key: "foo_2", Value: "bar_2", Effect: "no_such_effect"}, + expectedErr: true, + }, + { + name: "valid taint", + taintsToCheck: v1.Taint{Key: "foo_3", Value: "bar_3", Effect: v1.TaintEffectNoExecute}, + expectedErr: false, + }, + { + name: "valid taint", + taintsToCheck: v1.Taint{Key: "foo_4", Effect: v1.TaintEffectNoExecute}, + expectedErr: false, + }, + { + name: "valid taint", + taintsToCheck: v1.Taint{Key: "foo_5", Value: "bar_5"}, + expectedErr: false, + }, + } + + for _, c := range cases { + err := CheckTaintValidation(c.taintsToCheck) + + if c.expectedErr && err == nil { + t.Errorf("[%s] expected error for spec %+v, but got nothing", c.name, c.taintsToCheck) + } + } +} diff --git a/staging/src/k8s.io/kubelet/config/v1beta1/types.go b/staging/src/k8s.io/kubelet/config/v1beta1/types.go index 6567ff679b7..30eb22f7b77 100644 --- a/staging/src/k8s.io/kubelet/config/v1beta1/types.go +++ b/staging/src/k8s.io/kubelet/config/v1beta1/types.go @@ -1033,6 +1033,16 @@ type KubeletConfiguration struct { // +featureGate=MemoryQoS // +optional MemoryThrottlingFactor *float64 `json:"memoryThrottlingFactor,omitempty"` + // registerWithTaints are an array of taints to add to a node object when + // the kubelet registers itself. This only takes effect when registerNode + // is true and upon the initial registration of the node. + // Default: nil + // +optional + RegisterWithTaints []v1.Taint `json:"registerWithTaints,omitempty"` + // registerNode enables automatic registration with the apiserver. + // Default: true + // +optional + RegisterNode *bool `json:"registerNode,omitempty"` } type KubeletAuthorizationMode string diff --git a/staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go b/staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go index de561fd5996..114cba363f4 100644 --- a/staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/kubelet/config/v1beta1/zz_generated.deepcopy.go @@ -345,6 +345,18 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { *out = new(float64) **out = **in } + if in.RegisterWithTaints != nil { + in, out := &in.RegisterWithTaints, &out.RegisterWithTaints + *out = make([]corev1.Taint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.RegisterNode != nil { + in, out := &in.RegisterNode, &out.RegisterNode + *out = new(bool) + **out = **in + } return }