From b48f23b786f026edb407c27866c667df2809fe4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Tue, 29 May 2018 17:51:39 +0300 Subject: [PATCH] kubeadm: Move .NodeName and .CRISocket to a common sub-struct --- cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go | 18 +++- cmd/kubeadm/app/apis/kubeadm/types.go | 51 +++++---- .../app/apis/kubeadm/v1alpha1/conversion.go | 63 +++++++++++ .../app/apis/kubeadm/v1alpha2/defaults.go | 16 +-- .../app/apis/kubeadm/v1alpha2/types.go | 45 +++++--- .../app/apis/kubeadm/validation/validation.go | 20 +++- cmd/kubeadm/app/cmd/config.go | 2 +- cmd/kubeadm/app/cmd/init.go | 18 ++-- cmd/kubeadm/app/cmd/join.go | 14 ++- cmd/kubeadm/app/cmd/phases/kubeconfig.go | 2 +- cmd/kubeadm/app/cmd/phases/kubelet.go | 5 +- cmd/kubeadm/app/cmd/phases/markmaster.go | 4 +- cmd/kubeadm/app/constants/constants.go | 7 +- .../app/phases/certs/pkiutil/pki_helpers.go | 4 +- .../app/phases/kubeconfig/kubeconfig.go | 2 +- cmd/kubeadm/app/phases/kubelet/config.go | 17 +-- cmd/kubeadm/app/phases/kubelet/dynamic.go | 71 ++++-------- cmd/kubeadm/app/phases/kubelet/flags.go | 23 ++-- .../app/phases/markmaster/markmaster.go | 101 +++--------------- .../app/phases/selfhosting/selfhosting.go | 2 +- cmd/kubeadm/app/phases/upgrade/staticpods.go | 6 +- cmd/kubeadm/app/preflight/checks.go | 2 +- cmd/kubeadm/app/util/apiclient/idempotency.go | 52 +++++++++ cmd/kubeadm/app/util/config/masterconfig.go | 8 +- cmd/kubeadm/app/util/config/nodeconfig.go | 2 +- cmd/kubeadm/test/util.go | 7 +- 26 files changed, 309 insertions(+), 253 deletions(-) diff --git a/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go b/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go index 2ede4cb22ba..b640d6e533e 100644 --- a/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go +++ b/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go @@ -21,6 +21,7 @@ import ( fuzz "github.com/google/gofuzz" + "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" @@ -43,7 +44,6 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { obj.APIServerCertSANs = []string{"foo"} obj.Token = "foo" - obj.CRISocket = "foo" obj.TokenTTL = &metav1.Duration{Duration: 1 * time.Hour} obj.TokenUsages = []string{"foo"} obj.TokenGroups = []string{"foo"} @@ -59,6 +59,9 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { MountPath: "foo", Writable: false, }} + // Note: We don't set values here for obj.Etcd.External, as these are mutually exlusive. + // And to make sure the fuzzer doesn't set a random value for obj.Etcd.External, we let + // kubeadmapi.Etcd implement fuzz.Interface (we handle that ourselves) obj.Etcd.Local = &kubeadm.LocalEtcd{ Image: "foo", DataDir: "foo", @@ -66,9 +69,11 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { PeerCertSANs: []string{"foo"}, ExtraArgs: map[string]string{"foo": "foo"}, } - // Note: We don't set values here for obj.Etcd.External, as these are mutually exlusive. - // And to make sure the fuzzer doesn't set a random value for obj.Etcd.External, we let - // kubeadmapi.Etcd implement fuzz.Interface (we handle that ourselves) + obj.NodeRegistration = kubeadm.NodeRegistrationOptions{ + CRISocket: "foo", + Name: "foo", + Taints: []v1.Taint{}, + } obj.KubeletConfiguration = kubeadm.KubeletConfiguration{ BaseConfig: &kubeletconfigv1beta1.KubeletConfiguration{ StaticPodPath: "foo", @@ -139,8 +144,11 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { obj.DiscoveryTimeout = &metav1.Duration{Duration: 1} obj.TLSBootstrapToken = "foo" obj.Token = "foo" - obj.CRISocket = "foo" obj.ClusterName = "foo" + obj.NodeRegistration = kubeadm.NodeRegistrationOptions{ + CRISocket: "foo", + Name: "foo", + } }, } } diff --git a/cmd/kubeadm/app/apis/kubeadm/types.go b/cmd/kubeadm/app/apis/kubeadm/types.go index ea062c62184..f83d63e9bd6 100644 --- a/cmd/kubeadm/app/apis/kubeadm/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/types.go @@ -44,13 +44,9 @@ type MasterConfiguration struct { Networking Networking // KubernetesVersion is the target version of the control plane. KubernetesVersion string - // NodeName is the name of the node that will host the k8s control plane. - // Defaults to the hostname if not provided. - NodeName string - // NoTaintMaster will, if set, suppress the tainting of the - // master node allowing workloads to be run on it (e.g. in - // single node configurations). - NoTaintMaster bool + + // NodeRegistration holds fields that relate to registering the new master node to the cluster + NodeRegistration NodeRegistrationOptions // Token is used for establishing bidirectional trust between nodes and masters. // Used for joining nodes in the cluster. @@ -62,9 +58,6 @@ type MasterConfiguration struct { // Extra groups that this token will authenticate as when used for authentication TokenGroups []string - // CRISocket is used to retrieve container runtime info. - CRISocket string - // APIServerExtraArgs is a set of extra flags to pass to the API Server or override // default ones in form of =. // TODO: This is temporary and ideally we would like to switch all components to @@ -138,6 +131,28 @@ type API struct { BindPort int32 } +// NodeRegistrationOptions holds fields that relate to registering a new master or node to the cluster, either via "kubeadm init" or "kubeadm join" +type NodeRegistrationOptions struct { + + // Name is the `.Metadata.Name` field of the Node API object that will be created in this `kubeadm init` or `kubeadm joiń` operation. + // This field is also used in the CommonName field of the kubelet's client certificate to the API server. + // Defaults to the hostname of the node if not provided. + Name string + + // CRISocket is used to retrieve container runtime info. This information will be annotated to the Node API object, for later re-use + CRISocket string + + // Taints specifies the taints the Node API object should be registered with. If this field is unset, i.e. nil, in the `kubeadm init` process + // it will be defaulted to []v1.Taint{'node-role.kubernetes.io/master=""'}. If you don't want to taint your master node, set this field to an + // empty slice, i.e. `taints: {}` in the YAML file. This field is solely used for Node registration. + Taints []v1.Taint + + // ExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file + // kubeadm writes at runtime for the kubelet to source. This overrides the generic base-level configuration in the kubelet-config-1.X ConfigMap + // Flags have higher higher priority when parsing. These values are local and specific to the node kubeadm is executing on. + ExtraArgs map[string]string +} + // TokenDiscovery contains elements needed for token discovery. type TokenDiscovery struct { // ID is the first part of a bootstrap token. Considered public information. @@ -223,6 +238,9 @@ type ExternalEtcd struct { type NodeConfiguration struct { metav1.TypeMeta + // NodeRegistration holds fields that relate to registering the new master node to the cluster + NodeRegistration NodeRegistrationOptions + // CACertPath is the path to the SSL certificate authority used to // secure comunications between node and master. // Defaults to "/etc/kubernetes/pki/ca.crt". @@ -239,16 +257,11 @@ type NodeConfiguration struct { DiscoveryTokenAPIServers []string // DiscoveryTimeout modifies the discovery timeout DiscoveryTimeout *metav1.Duration - // NodeName is the name of the node to join the cluster. Defaults - // to the name of the host. - NodeName string // TLSBootstrapToken is a token used for TLS bootstrapping. // Defaults to Token. TLSBootstrapToken string // Token is used for both discovery and TLS bootstrapping. Token string - // CRISocket is used to retrieve container runtime info. - CRISocket string // The cluster name ClusterName string @@ -332,13 +345,13 @@ type CommonConfiguration interface { // GetCRISocket will return the CRISocket that is defined for the MasterConfiguration. // This is used internally to deduplicate the kubeadm preflight checks. func (cfg *MasterConfiguration) GetCRISocket() string { - return cfg.CRISocket + return cfg.NodeRegistration.CRISocket } // GetNodeName will return the NodeName that is defined for the MasterConfiguration. // This is used internally to deduplicate the kubeadm preflight checks. func (cfg *MasterConfiguration) GetNodeName() string { - return cfg.NodeName + return cfg.NodeRegistration.Name } // GetKubernetesVersion will return the KubernetesVersion that is defined for the MasterConfiguration. @@ -350,13 +363,13 @@ func (cfg *MasterConfiguration) GetKubernetesVersion() string { // GetCRISocket will return the CRISocket that is defined for the NodeConfiguration. // This is used internally to deduplicate the kubeadm preflight checks. func (cfg *NodeConfiguration) GetCRISocket() string { - return cfg.CRISocket + return cfg.NodeRegistration.CRISocket } // GetNodeName will return the NodeName that is defined for the NodeConfiguration. // This is used internally to deduplicate the kubeadm preflight checks. func (cfg *NodeConfiguration) GetNodeName() string { - return cfg.NodeName + return cfg.NodeRegistration.Name } // GetKubernetesVersion will return an empty string since KubernetesVersion is not a diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/conversion.go index d5492c3333a..54a598af1c9 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/conversion.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/conversion.go @@ -20,15 +20,20 @@ import ( "reflect" "strings" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/conversion" "k8s.io/apimachinery/pkg/runtime" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" ) func addConversionFuncs(scheme *runtime.Scheme) error { // Add non-generated conversion functions err := scheme.AddConversionFuncs( Convert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration, + Convert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration, + Convert_v1alpha1_NodeConfiguration_To_kubeadm_NodeConfiguration, + Convert_kubeadm_NodeConfiguration_To_v1alpha1_NodeConfiguration, Convert_v1alpha1_Etcd_To_kubeadm_Etcd, Convert_kubeadm_Etcd_To_v1alpha1_Etcd, ) @@ -39,6 +44,8 @@ func addConversionFuncs(scheme *runtime.Scheme) error { return nil } +// Upgrades below + func Convert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration(in *MasterConfiguration, out *kubeadm.MasterConfiguration, s conversion.Scope) error { if err := autoConvert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration(in, out, s); err != nil { return err @@ -46,12 +53,26 @@ func Convert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration(in *Mas UpgradeCloudProvider(in, out) UpgradeAuthorizationModes(in, out) + UpgradeNodeRegistrationOptionsForMaster(in, out) // We don't support migrating information from the .PrivilegedPods field which was removed in v1alpha2 // We don't support migrating information from the .ImagePullPolicy field which was removed in v1alpha2 return nil } +func Convert_v1alpha1_NodeConfiguration_To_kubeadm_NodeConfiguration(in *NodeConfiguration, out *kubeadm.NodeConfiguration, s conversion.Scope) error { + if err := autoConvert_v1alpha1_NodeConfiguration_To_kubeadm_NodeConfiguration(in, out, s); err != nil { + return err + } + + // .NodeName has moved to .NodeRegistration.Name + out.NodeRegistration.Name = in.NodeName + // .CRISocket has moved to .NodeRegistration.CRISocket + out.NodeRegistration.CRISocket = in.CRISocket + + return nil +} + func Convert_v1alpha1_Etcd_To_kubeadm_Etcd(in *Etcd, out *kubeadm.Etcd, s conversion.Scope) error { if err := autoConvert_v1alpha1_Etcd_To_kubeadm_Etcd(in, out, s); err != nil { return err @@ -123,3 +144,45 @@ func UpgradeAuthorizationModes(in *MasterConfiguration, out *kubeadm.MasterConfi out.APIServerExtraArgs["authorization-mode"] = strings.Join(in.AuthorizationModes, ",") } } + +func UpgradeNodeRegistrationOptionsForMaster(in *MasterConfiguration, out *kubeadm.MasterConfiguration) { + // .NodeName has moved to .NodeRegistration.Name + out.NodeRegistration.Name = in.NodeName + + // .CRISocket has moved to .NodeRegistration.CRISocket + out.NodeRegistration.CRISocket = in.CRISocket + + // Transfer the information from .NoTaintMaster to the new layout + if in.NoTaintMaster { + out.NodeRegistration.Taints = []v1.Taint{} + } else { + out.NodeRegistration.Taints = []v1.Taint{constants.MasterTaint} + } +} + +// Downgrades below + +func Convert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration(in *kubeadm.MasterConfiguration, out *MasterConfiguration, s conversion.Scope) error { + if err := autoConvert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration(in, out, s); err != nil { + return err + } + + // Converting from newer API version to an older API version isn't supported. This is here only for the roundtrip tests meanwhile. + out.NodeName = in.NodeRegistration.Name + out.CRISocket = in.NodeRegistration.CRISocket + out.NoTaintMaster = in.NodeRegistration.Taints != nil && len(in.NodeRegistration.Taints) == 0 + + return nil +} + +func Convert_kubeadm_NodeConfiguration_To_v1alpha1_NodeConfiguration(in *kubeadm.NodeConfiguration, out *NodeConfiguration, s conversion.Scope) error { + if err := autoConvert_kubeadm_NodeConfiguration_To_v1alpha1_NodeConfiguration(in, out, s); err != nil { + return err + } + + // Converting from newer API version to an older API version isn't supported. This is here only for the roundtrip tests meanwhile. + out.NodeName = in.NodeRegistration.Name + out.CRISocket = in.NodeRegistration.CRISocket + + return nil +} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha2/defaults.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha2/defaults.go index f9c4000198f..d9467e27f87 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha2/defaults.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha2/defaults.go @@ -103,10 +103,6 @@ func SetDefaults_MasterConfiguration(obj *MasterConfiguration) { } } - if obj.CRISocket == "" { - obj.CRISocket = DefaultCRISocket - } - if len(obj.TokenUsages) == 0 { obj.TokenUsages = constants.DefaultTokenUsages } @@ -123,6 +119,7 @@ func SetDefaults_MasterConfiguration(obj *MasterConfiguration) { obj.ClusterName = DefaultClusterName } + SetDefaults_NodeRegistrationOptions(&obj.NodeRegistration) SetDefaults_KubeletConfiguration(obj) SetDefaults_Etcd(obj) SetDefaults_ProxyConfiguration(obj) @@ -168,9 +165,6 @@ func SetDefaults_NodeConfiguration(obj *NodeConfiguration) { if len(obj.DiscoveryToken) == 0 && len(obj.DiscoveryFile) == 0 { obj.DiscoveryToken = obj.Token } - if obj.CRISocket == "" { - obj.CRISocket = DefaultCRISocket - } // Make sure file URLs become paths if len(obj.DiscoveryFile) != 0 { u, err := url.Parse(obj.DiscoveryFile) @@ -186,6 +180,8 @@ func SetDefaults_NodeConfiguration(obj *NodeConfiguration) { if obj.ClusterName == "" { obj.ClusterName = DefaultClusterName } + + SetDefaults_NodeRegistrationOptions(&obj.NodeRegistration) } // SetDefaults_KubeletConfiguration assigns default values to kubelet @@ -237,6 +233,12 @@ func SetDefaults_KubeletConfiguration(obj *MasterConfiguration) { } } +func SetDefaults_NodeRegistrationOptions(obj *NodeRegistrationOptions) { + if obj.CRISocket == "" { + obj.CRISocket = DefaultCRISocket + } +} + // SetDefaults_AuditPolicyConfiguration sets default values for the AuditPolicyConfiguration func SetDefaults_AuditPolicyConfiguration(obj *MasterConfiguration) { if obj.AuditPolicyConfiguration.LogDir == "" { diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha2/types.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha2/types.go index dd48f2b9277..f1ec6f51079 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha2/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha2/types.go @@ -40,15 +40,12 @@ type MasterConfiguration struct { KubeletConfiguration KubeletConfiguration `json:"kubeletConfiguration"` // Networking holds configuration for the networking topology of the cluster. Networking Networking `json:"networking"` + + // NodeRegistration holds fields that relate to registering the new master node to the cluster + NodeRegistration NodeRegistrationOptions `json:"nodeRegistration"` + // KubernetesVersion is the target version of the control plane. KubernetesVersion string `json:"kubernetesVersion"` - // NodeName is the name of the node that will host the k8s control plane. - // Defaults to the hostname if not provided. - NodeName string `json:"nodeName"` - // NoTaintMaster will, if set, suppress the tainting of the - // master node allowing workloads to be run on it (e.g. in - // single node configurations). - NoTaintMaster bool `json:"noTaintMaster,omitempty"` // Token is used for establishing bidirectional trust between nodes and masters. // Used for joining nodes in the cluster. @@ -60,9 +57,6 @@ type MasterConfiguration struct { // Extra groups that this token will authenticate as when used for authentication TokenGroups []string `json:"tokenGroups,omitempty"` - // CRISocket is used to retrieve container runtime info. - CRISocket string `json:"criSocket,omitempty"` - // APIServerExtraArgs is a set of extra flags to pass to the API Server or override // default ones in form of =. // TODO: This is temporary and ideally we would like to switch all components to @@ -129,6 +123,28 @@ type API struct { BindPort int32 `json:"bindPort"` } +// NodeRegistrationOptions holds fields that relate to registering a new master or node to the cluster, either via "kubeadm init" or "kubeadm join" +type NodeRegistrationOptions struct { + + // Name is the `.Metadata.Name` field of the Node API object that will be created in this `kubeadm init` or `kubeadm joiń` operation. + // This field is also used in the CommonName field of the kubelet's client certificate to the API server. + // Defaults to the hostname of the node if not provided. + Name string `json:"name"` + + // CRISocket is used to retrieve container runtime info. This information will be annotated to the Node API object, for later re-use + CRISocket string `json:"criSocket"` + + // Taints specifies the taints the Node API object should be registered with. If this field is unset, i.e. nil, in the `kubeadm init` process + // it will be defaulted to []v1.Taint{'node-role.kubernetes.io/master=""'}. If you don't want to taint your master node, set this field to an + // empty slice, i.e. `taints: {}` in the YAML file. This field is solely used for Node registration. + Taints []v1.Taint `json:"taints,omitempty"` + + // ExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file + // kubeadm writes at runtime for the kubelet to source. This overrides the generic base-level configuration in the kubelet-config-1.X ConfigMap + // Flags have higher higher priority when parsing. These values are local and specific to the node kubeadm is executing on. + ExtraArgs map[string]string `json:"kubeletExtraArgs,omitempty"` +} + // TokenDiscovery contains elements needed for token discovery. type TokenDiscovery struct { // ID is the first part of a bootstrap token. Considered public information. @@ -206,6 +222,9 @@ type ExternalEtcd struct { type NodeConfiguration struct { metav1.TypeMeta `json:",inline"` + // NodeRegistration holds fields that relate to registering the new master node to the cluster + NodeRegistration NodeRegistrationOptions `json:"nodeRegistration"` + // CACertPath is the path to the SSL certificate authority used to // secure comunications between node and master. // Defaults to "/etc/kubernetes/pki/ca.crt". @@ -222,16 +241,12 @@ type NodeConfiguration struct { DiscoveryTokenAPIServers []string `json:"discoveryTokenAPIServers,omitempty"` // DiscoveryTimeout modifies the discovery timeout DiscoveryTimeout *metav1.Duration `json:"discoveryTimeout,omitempty"` - // NodeName is the name of the node to join the cluster. Defaults - // to the name of the host. - NodeName string `json:"nodeName"` // TLSBootstrapToken is a token used for TLS bootstrapping. // Defaults to Token. TLSBootstrapToken string `json:"tlsBootstrapToken"` // Token is used for both discovery and TLS bootstrapping. Token string `json:"token"` - // CRISocket is used to retrieve container runtime info. - CRISocket string `json:"criSocket,omitempty"` + // ClusterName is the name for the cluster in kubeconfig. ClusterName string `json:"clusterName,omitempty"` diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go index 116bfce6c8d..f51f539ac8f 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go +++ b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go @@ -54,7 +54,7 @@ func ValidateMasterConfiguration(c *kubeadm.MasterConfiguration) field.ErrorList allErrs = append(allErrs, ValidateNetworking(&c.Networking, field.NewPath("networking"))...) allErrs = append(allErrs, ValidateCertSANs(c.APIServerCertSANs, field.NewPath("apiServerCertSANs"))...) allErrs = append(allErrs, ValidateAbsolutePath(c.CertificatesDir, field.NewPath("certificatesDir"))...) - allErrs = append(allErrs, ValidateNodeName(c.NodeName, field.NewPath("nodeName"))...) + allErrs = append(allErrs, ValidateNodeRegistrationOptions(&c.NodeRegistration, field.NewPath("nodeRegistration"))...) allErrs = append(allErrs, ValidateToken(c.Token, field.NewPath("token"))...) allErrs = append(allErrs, ValidateTokenUsages(c.TokenUsages, field.NewPath("tokenUsages"))...) allErrs = append(allErrs, ValidateTokenGroups(c.TokenUsages, c.TokenGroups, field.NewPath("tokenGroups"))...) @@ -62,9 +62,7 @@ func ValidateMasterConfiguration(c *kubeadm.MasterConfiguration) field.ErrorList allErrs = append(allErrs, ValidateAPIEndpoint(&c.API, field.NewPath("api"))...) allErrs = append(allErrs, ValidateProxy(c.KubeProxy.Config, field.NewPath("kubeProxy").Child("config"))...) allErrs = append(allErrs, ValidateEtcd(&c.Etcd, field.NewPath("etcd"))...) - if features.Enabled(c.FeatureGates, features.DynamicKubeletConfig) { - allErrs = append(allErrs, ValidateKubeletConfiguration(&c.KubeletConfiguration, field.NewPath("kubeletConfiguration"))...) - } + allErrs = append(allErrs, ValidateKubeletConfiguration(&c.KubeletConfiguration, field.NewPath("kubeletConfiguration"))...) return allErrs } @@ -86,6 +84,7 @@ func ValidateProxy(c *kubeproxyconfigv1alpha1.KubeProxyConfiguration, fldPath *f func ValidateNodeConfiguration(c *kubeadm.NodeConfiguration) field.ErrorList { allErrs := field.ErrorList{} allErrs = append(allErrs, ValidateDiscovery(c)...) + allErrs = append(allErrs, ValidateNodeRegistrationOptions(&c.NodeRegistration, field.NewPath("nodeRegistration"))...) if !filepath.IsAbs(c.CACertPath) || !strings.HasSuffix(c.CACertPath, ".crt") { allErrs = append(allErrs, field.Invalid(field.NewPath("caCertPath"), c.CACertPath, "the ca certificate path must be an absolute path")) @@ -93,6 +92,15 @@ func ValidateNodeConfiguration(c *kubeadm.NodeConfiguration) field.ErrorList { return allErrs } +// ValidateNodeRegistrationOptions validates the NodeRegistrationOptions object +func ValidateNodeRegistrationOptions(nro *kubeadm.NodeRegistrationOptions, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + allErrs = append(allErrs, ValidateNodeName(nro.Name, fldPath.Child("name"))...) + allErrs = append(allErrs, ValidateAbsolutePath(nro.CRISocket, fldPath.Child("criSocket"))...) + // TODO: Maybe validate .Taints as well in the future using something like validateNodeTaints() in pkg/apis/core/validation + return allErrs +} + // ValidateDiscovery validates discovery related configuration and collects all encountered errors func ValidateDiscovery(c *kubeadm.NodeConfiguration) field.ErrorList { allErrs := field.ErrorList{} @@ -422,6 +430,10 @@ func ValidateIgnorePreflightErrors(ignorePreflightErrors []string, skipPreflight func ValidateKubeletConfiguration(c *kubeadm.KubeletConfiguration, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} + if c.BaseConfig == nil { + return allErrs + } + scheme, _, err := kubeletscheme.NewSchemeAndCodecs() if err != nil { allErrs = append(allErrs, field.Invalid(fldPath, "kubeletConfiguration", err.Error())) diff --git a/cmd/kubeadm/app/cmd/config.go b/cmd/kubeadm/app/cmd/config.go index 62f89e92788..8b45f8407fe 100644 --- a/cmd/kubeadm/app/cmd/config.go +++ b/cmd/kubeadm/app/cmd/config.go @@ -401,5 +401,5 @@ func AddImagesCommonConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiv1alpha2.M // AddImagesPullFlags adds flags related to the `kubeadm config images pull` command func AddImagesPullFlags(flagSet *flag.FlagSet, cfg *kubeadmapiv1alpha2.MasterConfiguration) { - flagSet.StringVar(&cfg.CRISocket, "cri-socket-path", cfg.CRISocket, "Path to the CRI socket.") + flagSet.StringVar(&cfg.NodeRegistration.CRISocket, "cri-socket-path", cfg.NodeRegistration.CRISocket, "Path to the CRI socket.") } diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 50d3beaa62a..68df174b7b9 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -189,7 +189,7 @@ func AddInitConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiv1alpha2.MasterCon `Optional extra Subject Alternative Names (SANs) to use for the API Server serving certificate. Can be both IP addresses and DNS names.`, ) flagSet.StringVar( - &cfg.NodeName, "node-name", cfg.NodeName, + &cfg.NodeRegistration.Name, "node-name", cfg.NodeRegistration.Name, `Specify the node name.`, ) flagSet.StringVar( @@ -201,7 +201,7 @@ func AddInitConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiv1alpha2.MasterCon "The duration before the bootstrap token is automatically deleted. If set to '0', the token will never expire.", ) flagSet.StringVar( - &cfg.CRISocket, "cri-socket", cfg.CRISocket, + &cfg.NodeRegistration.CRISocket, "cri-socket", cfg.NodeRegistration.CRISocket, `Specify the CRI socket to connect to.`, ) flagSet.StringVar(featureGatesString, "feature-gates", *featureGatesString, "A set of key=value pairs that describe feature gates for various features. "+ @@ -276,8 +276,10 @@ type Init struct { // Run executes master node provisioning, including certificates, needed static pod manifests, etc. func (i *Init) Run(out io.Writer) error { - // Write env file with flags for the kubelet to use - if err := kubeletphase.WriteKubeletDynamicEnvFile(i.cfg); err != nil { + // Write env file with flags for the kubelet to use. We do not need to write the --register-with-taints for the master, + // as we handle that ourselves in the markmaster phase + // TODO: Maybe we want to do that some time in the future, in order to remove some logic from the markmaster phase? + if err := kubeletphase.WriteKubeletDynamicEnvFile(&i.cfg.NodeRegistration, false); err != nil { return err } @@ -362,7 +364,7 @@ func (i *Init) Run(out io.Writer) error { } // Write the kubelet configuration to disk. - if err := kubeletphase.WriteConfigToDisk(i.cfg.KubeletConfiguration.BaseConfig, kubeletVersion); err != nil { + if err := kubeletphase.WriteConfigToDisk(i.cfg.KubeletConfiguration.BaseConfig); err != nil { return fmt.Errorf("error writing kubelet configuration to disk: %v", err) } @@ -411,7 +413,7 @@ func (i *Init) Run(out io.Writer) error { // PHASE 4: Mark the master with the right label/taint glog.V(1).Infof("[init] marking the master with right label") - if err := markmasterphase.MarkMaster(client, i.cfg.NodeName, !i.cfg.NoTaintMaster); err != nil { + if err := markmasterphase.MarkMaster(client, i.cfg.NodeRegistration.Name, i.cfg.NodeRegistration.Taints); err != nil { return fmt.Errorf("error marking master: %v", err) } @@ -419,7 +421,7 @@ func (i *Init) Run(out io.Writer) error { // This feature is disabled by default, as it is alpha still if features.Enabled(i.cfg.FeatureGates, features.DynamicKubeletConfig) { // Enable dynamic kubelet configuration for the node. - if err := kubeletphase.EnableDynamicConfigForNode(client, i.cfg.NodeName, kubeletVersion); err != nil { + if err := kubeletphase.EnableDynamicConfigForNode(client, i.cfg.NodeRegistration.Name, kubeletVersion); err != nil { return fmt.Errorf("error enabling dynamic kubelet configuration: %v", err) } } @@ -507,7 +509,7 @@ func (i *Init) Run(out io.Writer) error { func createClient(cfg *kubeadmapi.MasterConfiguration, dryRun bool) (clientset.Interface, error) { if dryRun { // If we're dry-running; we should create a faked client that answers some GETs in order to be able to do the full init flow and just logs the rest of requests - dryRunGetter := apiclient.NewInitDryRunGetter(cfg.NodeName, cfg.Networking.ServiceSubnet) + dryRunGetter := apiclient.NewInitDryRunGetter(cfg.NodeRegistration.Name, cfg.Networking.ServiceSubnet) return apiclient.NewDryRunClient(dryRunGetter, os.Stdout), nil } diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index 77fcb6697ca..0a3239fe4e0 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -155,7 +155,7 @@ func AddJoinConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiv1alpha2.NodeConfi &cfg.DiscoveryToken, "discovery-token", "", "A token used to validate cluster information fetched from the master.") flagSet.StringVar( - &cfg.NodeName, "node-name", "", + &cfg.NodeRegistration.Name, "node-name", cfg.NodeRegistration.Name, "Specify the node name.") flagSet.StringVar( &cfg.TLSBootstrapToken, "tls-bootstrap-token", "", @@ -174,7 +174,7 @@ func AddJoinConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiv1alpha2.NodeConfi "A set of key=value pairs that describe feature gates for various features. "+ "Options are:\n"+strings.Join(features.KnownFeatures(&features.InitFeatureGates), "\n")) flagSet.StringVar( - &cfg.CRISocket, "cri-socket", cfg.CRISocket, + &cfg.NodeRegistration.CRISocket, "cri-socket", cfg.NodeRegistration.CRISocket, `Specify the CRI socket to connect to.`, ) } @@ -204,7 +204,7 @@ type Join struct { // NewJoin instantiates Join struct with given arguments func NewJoin(cfgPath string, args []string, defaultcfg *kubeadmapiv1alpha2.NodeConfiguration, ignorePreflightErrors sets.String) (*Join, error) { - if defaultcfg.NodeName == "" { + if defaultcfg.NodeRegistration.Name == "" { glog.V(1).Infoln("[join] found NodeName empty") glog.V(1).Infoln("[join] considered OS hostname as NodeName") } @@ -231,6 +231,12 @@ func NewJoin(cfgPath string, args []string, defaultcfg *kubeadmapiv1alpha2.NodeC // Run executes worker node provisioning and tries to join an existing cluster. func (j *Join) Run(out io.Writer) error { + + // Write env file with flags for the kubelet to use. Also register taints + if err := kubeletphase.WriteKubeletDynamicEnvFile(&j.cfg.NodeRegistration, true); err != nil { + return err + } + glog.V(1).Infoln("[join] retrieving KubeConfig objects") cfg, err := discovery.For(j.cfg) if err != nil { @@ -273,7 +279,7 @@ func (j *Join) Run(out io.Writer) error { return err } - if err := kubeletphase.EnableDynamicConfigForNode(client, j.cfg.NodeName, kubeletVersion); err != nil { + if err := kubeletphase.EnableDynamicConfigForNode(client, j.cfg.NodeRegistration.Name, kubeletVersion); err != nil { return fmt.Errorf("error consuming base kubelet configuration: %v", err) } } diff --git a/cmd/kubeadm/app/cmd/phases/kubeconfig.go b/cmd/kubeadm/app/cmd/phases/kubeconfig.go index 3f3e185299d..7a0dcb0ce5b 100644 --- a/cmd/kubeadm/app/cmd/phases/kubeconfig.go +++ b/cmd/kubeadm/app/cmd/phases/kubeconfig.go @@ -184,7 +184,7 @@ func getKubeConfigSubCommands(out io.Writer, outDir, defaultKubernetesVersion st cmd.Flags().Int32Var(&cfg.API.BindPort, "apiserver-bind-port", cfg.API.BindPort, "The port the API server is accessible on") cmd.Flags().StringVar(&outDir, "kubeconfig-dir", outDir, "The path where to save the kubeconfig file") if properties.use == "all" || properties.use == "kubelet" { - cmd.Flags().StringVar(&cfg.NodeName, "node-name", cfg.NodeName, `The node name that should be used for the kubelet client certificate`) + cmd.Flags().StringVar(&cfg.NodeRegistration.Name, "node-name", cfg.NodeRegistration.Name, `The node name that should be used for the kubelet client certificate`) } if properties.use == "user" { cmd.Flags().StringVar(&token, "token", token, "The token that should be used as the authentication mechanism for this kubeconfig, instead of client certificates") diff --git a/cmd/kubeadm/app/cmd/phases/kubelet.go b/cmd/kubeadm/app/cmd/phases/kubelet.go index c07257d9b7f..bd979d60492 100644 --- a/cmd/kubeadm/app/cmd/phases/kubelet.go +++ b/cmd/kubeadm/app/cmd/phases/kubelet.go @@ -134,9 +134,6 @@ func NewCmdKubeletWriteConfigToDisk(kubeConfigFile *string) *cobra.Command { kubeadmutil.CheckErr(fmt.Errorf("The --kubelet-version argument is required")) } - kubeletVersion, err := version.ParseSemantic(kubeletVersionStr) - kubeadmutil.CheckErr(err) - client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile) kubeadmutil.CheckErr(err) @@ -144,7 +141,7 @@ func NewCmdKubeletWriteConfigToDisk(kubeConfigFile *string) *cobra.Command { internalcfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "kubelet", cfgPath) kubeadmutil.CheckErr(err) - err = kubeletphase.WriteConfigToDisk(internalcfg.KubeletConfiguration.BaseConfig, kubeletVersion) + err = kubeletphase.WriteConfigToDisk(internalcfg.KubeletConfiguration.BaseConfig) kubeadmutil.CheckErr(err) }, } diff --git a/cmd/kubeadm/app/cmd/phases/markmaster.go b/cmd/kubeadm/app/cmd/phases/markmaster.go index 5a2f11ddbb5..13ec0a314ce 100644 --- a/cmd/kubeadm/app/cmd/phases/markmaster.go +++ b/cmd/kubeadm/app/cmd/phases/markmaster.go @@ -75,14 +75,14 @@ func NewCmdMarkMaster() *cobra.Command { client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile) kubeadmutil.CheckErr(err) - err = markmasterphase.MarkMaster(client, internalcfg.NodeName, !internalcfg.NoTaintMaster) + err = markmasterphase.MarkMaster(client, internalcfg.NodeRegistration.Name, internalcfg.NodeRegistration.Taints) kubeadmutil.CheckErr(err) }, } cmd.Flags().StringVar(&kubeConfigFile, "kubeconfig", "/etc/kubernetes/admin.conf", "The KubeConfig file to use when talking to the cluster") cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file. WARNING: Usage of a configuration file is experimental") - cmd.Flags().StringVar(&cfg.NodeName, "node-name", cfg.NodeName, `The node name to which label and taints should apply`) + cmd.Flags().StringVar(&cfg.NodeRegistration.Name, "node-name", cfg.NodeRegistration.Name, `The node name to which label and taints should apply`) return cmd } diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index afd5ce5ba5c..3a6f4233f4f 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -165,8 +165,8 @@ const ( APICallRetryInterval = 500 * time.Millisecond // DiscoveryRetryInterval specifies how long kubeadm should wait before retrying to connect to the master when doing discovery DiscoveryRetryInterval = 5 * time.Second - // MarkMasterTimeout specifies how long kubeadm should wait for applying the label and taint on the master before timing out - MarkMasterTimeout = 2 * time.Minute + // PatchNodeTimeout specifies how long kubeadm should wait for applying the label and taint on the master before timing out + PatchNodeTimeout = 2 * time.Minute // UpdateNodeTimeout specifies how long kubeadm should wait for updating node with the initial remote configuration of kubelet before timing out UpdateNodeTimeout = 2 * time.Minute @@ -295,9 +295,6 @@ var ( // MinimumKubeletVersion specifies the minimum version of kubelet which kubeadm supports MinimumKubeletVersion = version.MustParseSemantic("v1.10.0") - // MinimumKubeletConfigVersion specifies the minimum version of Kubernetes where kubeadm supports specifying --config to the kubelet - MinimumKubeletConfigVersion = version.MustParseSemantic("v1.11.0-alpha.1") - // SupportedEtcdVersion lists officially supported etcd versions with corresponding kubernetes releases SupportedEtcdVersion = map[uint8]string{ 10: "3.1.12", diff --git a/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers.go b/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers.go index 39ca139cba7..337656afd73 100644 --- a/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers.go +++ b/cmd/kubeadm/app/phases/certs/pkiutil/pki_helpers.go @@ -275,7 +275,7 @@ func GetAPIServerAltNames(cfg *kubeadmapi.MasterConfiguration) (*certutil.AltNam // create AltNames with defaults DNSNames/IPs altNames := &certutil.AltNames{ DNSNames: []string{ - cfg.NodeName, + cfg.NodeRegistration.Name, "kubernetes", "kubernetes.default", "kubernetes.default.svc", @@ -336,7 +336,7 @@ func GetEtcdPeerAltNames(cfg *kubeadmapi.MasterConfiguration) (*certutil.AltName // create AltNames with defaults DNSNames/IPs altNames := &certutil.AltNames{ - DNSNames: []string{cfg.NodeName}, + DNSNames: []string{cfg.NodeRegistration.Name}, IPs: []net.IP{advertiseAddress}, } diff --git a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go index 02194885a97..6a21c8f772d 100644 --- a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go +++ b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go @@ -160,7 +160,7 @@ func getKubeConfigSpecs(cfg *kubeadmapi.MasterConfiguration) (map[string]*kubeCo kubeadmconstants.KubeletKubeConfigFileName: { CACert: caCert, APIServer: masterEndpoint, - ClientName: fmt.Sprintf("system:node:%s", cfg.NodeName), + ClientName: fmt.Sprintf("system:node:%s", cfg.NodeRegistration.Name), ClientCertAuth: &clientCertAuth{ CAKey: caKey, Organizations: []string{kubeadmconstants.NodesGroup}, diff --git a/cmd/kubeadm/app/phases/kubelet/config.go b/cmd/kubeadm/app/phases/kubelet/config.go index 5503a16a7fc..e515d649860 100644 --- a/cmd/kubeadm/app/phases/kubelet/config.go +++ b/cmd/kubeadm/app/phases/kubelet/config.go @@ -37,12 +37,7 @@ import ( // WriteConfigToDisk writes the kubelet config object down to a file // Used at "kubeadm init" and "kubeadm upgrade" time -func WriteConfigToDisk(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration, kubeletVersion *version.Version) error { - - // If the kubelet version is v1.10.x, exit - if kubeletVersion.LessThan(kubeadmconstants.MinimumKubeletConfigVersion) { - return nil - } +func WriteConfigToDisk(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration) error { kubeletBytes, err := getConfigBytes(kubeletConfig) if err != nil { @@ -60,11 +55,6 @@ func CreateConfigMap(cfg *kubeadmapi.MasterConfiguration, client clientset.Inter return err } - // If Kubernetes version is v1.10.x, exit - if k8sVersion.LessThan(kubeadmconstants.MinimumKubeletConfigVersion) { - return nil - } - configMapName := configMapName(k8sVersion) fmt.Printf("[kubelet] Creating a ConfigMap %q in namespace %s with the configuration for the kubelets in the cluster\n", configMapName, metav1.NamespaceSystem) @@ -132,11 +122,6 @@ func createConfigMapRBACRules(client clientset.Interface, k8sVersion *version.Ve // Used at "kubeadm join" time func DownloadConfig(kubeletKubeConfig string, kubeletVersion *version.Version) error { - // If the kubelet version is v1.10.x, exit - if kubeletVersion.LessThan(kubeadmconstants.MinimumKubeletConfigVersion) { - return nil - } - // Download the ConfigMap from the cluster based on what version the kubelet is configMapName := configMapName(kubeletVersion) diff --git a/cmd/kubeadm/app/phases/kubelet/dynamic.go b/cmd/kubeadm/app/phases/kubelet/dynamic.go index b65d7460370..2f49209182a 100644 --- a/cmd/kubeadm/app/phases/kubelet/dynamic.go +++ b/cmd/kubeadm/app/phases/kubelet/dynamic.go @@ -17,19 +17,17 @@ limitations under the License. package kubelet import ( - "encoding/json" "fmt" "os" "path/filepath" "k8s.io/api/core/v1" - apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" "k8s.io/kubernetes/pkg/util/version" ) @@ -39,64 +37,33 @@ import ( // This func is ONLY run if the user enables the `DynamicKubeletConfig` feature gate, which is by default off func EnableDynamicConfigForNode(client clientset.Interface, nodeName string, kubeletVersion *version.Version) error { - // If the kubelet version is v1.10.x, exit - if kubeletVersion.LessThan(kubeadmconstants.MinimumKubeletConfigVersion) { - return nil - } - configMapName := configMapName(kubeletVersion) fmt.Printf("[kubelet] Enabling Dynamic Kubelet Config for Node %q; config sourced from ConfigMap %q in namespace %s\n", nodeName, configMapName, metav1.NamespaceSystem) fmt.Println("[kubelet] WARNING: The Dynamic Kubelet Config feature is alpha and off by default. It hasn't been well-tested yet at this stage, use with caution.") + kubeletConfigMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(configMapName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("couldn't get the kubelet configuration ConfigMap: %v", err) + } + // Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned. - return wait.Poll(kubeadmconstants.APICallRetryInterval, kubeadmconstants.UpdateNodeTimeout, func() (bool, error) { - node, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) - if err != nil { - return false, nil - } - - oldData, err := json.Marshal(node) - if err != nil { - return false, err - } - - kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(configMapName, metav1.GetOptions{}) - if err != nil { - return false, nil - } - - node.Spec.ConfigSource = &v1.NodeConfigSource{ - ConfigMap: &v1.ConfigMapNodeConfigSource{ - Name: configMapName, - Namespace: metav1.NamespaceSystem, - UID: kubeletCfg.UID, - KubeletConfigKey: kubeadmconstants.KubeletBaseConfigurationConfigMapKey, - }, - } - - newData, err := json.Marshal(node) - if err != nil { - return false, err - } - - patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{}) - if err != nil { - return false, err - } - - if _, err := client.CoreV1().Nodes().Patch(node.Name, types.StrategicMergePatchType, patchBytes); err != nil { - if apierrs.IsConflict(err) { - fmt.Println("Temporarily unable to update node metadata due to conflict (will retry)") - return false, nil - } - return false, err - } - - return true, nil + return apiclient.PatchNode(client, nodeName, func(n *v1.Node) { + patchNodeForDynamicConfig(n, configMapName, kubeletConfigMap.UID) }) } +func patchNodeForDynamicConfig(n *v1.Node, configMapName string, configMapUID types.UID) { + n.Spec.ConfigSource = &v1.NodeConfigSource{ + ConfigMap: &v1.ConfigMapNodeConfigSource{ + Name: configMapName, + Namespace: metav1.NamespaceSystem, + UID: configMapUID, + KubeletConfigKey: kubeadmconstants.KubeletBaseConfigurationConfigMapKey, + }, + } +} + // GetLocalNodeTLSBootstrappedClient waits for the kubelet to perform the TLS bootstrap // and then creates a client from config file /etc/kubernetes/kubelet.conf func GetLocalNodeTLSBootstrappedClient() (clientset.Interface, error) { diff --git a/cmd/kubeadm/app/phases/kubelet/flags.go b/cmd/kubeadm/app/phases/kubelet/flags.go index 167966793aa..926c4b5022e 100644 --- a/cmd/kubeadm/app/phases/kubelet/flags.go +++ b/cmd/kubeadm/app/phases/kubelet/flags.go @@ -31,10 +31,9 @@ import ( // WriteKubeletDynamicEnvFile writes a environment file with dynamic flags to the kubelet. // Used at "kubeadm init" and "kubeadm join" time. -func WriteKubeletDynamicEnvFile(cfg *kubeadmapi.MasterConfiguration) error { +func WriteKubeletDynamicEnvFile(nodeRegOpts *kubeadmapi.NodeRegistrationOptions, registerTaintsUsingFlags bool) error { - // TODO: Pass through extra arguments from the config file here in the future - argList := kubeadmutil.BuildArgumentListFromMap(buildKubeletArgMap(cfg), map[string]string{}) + argList := kubeadmutil.BuildArgumentListFromMap(buildKubeletArgMap(nodeRegOpts, registerTaintsUsingFlags), nodeRegOpts.ExtraArgs) envFileContent := fmt.Sprintf("%s=%s\n", constants.KubeletEnvFileVariableName, strings.Join(argList, " ")) return writeKubeletFlagBytesToDisk([]byte(envFileContent)) @@ -42,20 +41,28 @@ func WriteKubeletDynamicEnvFile(cfg *kubeadmapi.MasterConfiguration) error { // buildKubeletArgMap takes a MasterConfiguration object and builds based on that a string-string map with flags // that should be given to the local kubelet daemon. -func buildKubeletArgMap(cfg *kubeadmapi.MasterConfiguration) map[string]string { +func buildKubeletArgMap(nodeRegOpts *kubeadmapi.NodeRegistrationOptions, registerTaintsUsingFlags bool) map[string]string { kubeletFlags := map[string]string{} - if cfg.CRISocket == kubeadmapiv1alpha2.DefaultCRISocket { + if nodeRegOpts.CRISocket == kubeadmapiv1alpha2.DefaultCRISocket { // These flags should only be set when running docker kubeletFlags["network-plugin"] = "cni" kubeletFlags["cni-conf-dir"] = "/etc/cni/net.d" kubeletFlags["cni-bin-dir"] = "/opt/cni/bin" } else { kubeletFlags["container-runtime"] = "remote" - kubeletFlags["container-runtime-endpoint"] = cfg.CRISocket + kubeletFlags["container-runtime-endpoint"] = nodeRegOpts.CRISocket } - // TODO: Add support for registering custom Taints and Labels - // TODO: Add support for overriding flags with ExtraArgs + + if registerTaintsUsingFlags && nodeRegOpts.Taints != nil && len(nodeRegOpts.Taints) > 0 { + taintStrs := []string{} + for _, taint := range nodeRegOpts.Taints { + taintStrs = append(taintStrs, taint.ToString()) + } + + kubeletFlags["register-with-taints"] = strings.Join(taintStrs, ",") + } + // TODO: Pass through --hostname-override if a custom name is used? // TODO: Check if `systemd-resolved` is running, and set `--resolv-conf` based on that // TODO: Conditionally set `--cgroup-driver` to either `systemd` or `cgroupfs` diff --git a/cmd/kubeadm/app/phases/markmaster/markmaster.go b/cmd/kubeadm/app/phases/markmaster/markmaster.go index 9f8f52237d9..39ee9174da5 100644 --- a/cmd/kubeadm/app/phases/markmaster/markmaster.go +++ b/cmd/kubeadm/app/phases/markmaster/markmaster.go @@ -17,107 +17,30 @@ limitations under the License. package markmaster import ( - "encoding/json" - "fmt" - "github.com/golang/glog" "k8s.io/api/core/v1" - apierrs "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/strategicpatch" - "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" ) // MarkMaster taints the master and sets the master label -func MarkMaster(client clientset.Interface, masterName string, taint bool) error { +func MarkMaster(client clientset.Interface, masterName string, taints []v1.Taint) error { - if taint { - glog.Infof("[markmaster] will mark node %s as master by adding a label and a taint\n", masterName) - } else { - glog.Infof("[markmaster] will mark node %s as master by adding a label\n", masterName) + glog.Infof("[markmaster] Marking the node %s as master by adding the label \"%s=''\"\n", masterName, constants.LabelNodeRoleMaster) + + if taints != nil && len(taints) > 0 { + glog.Infof("[markmaster] Marking the node %s as master by adding the taints %v\n", masterName, taints) } - // Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned. - return wait.Poll(kubeadmconstants.APICallRetryInterval, kubeadmconstants.MarkMasterTimeout, func() (bool, error) { - // First get the node object - n, err := client.CoreV1().Nodes().Get(masterName, metav1.GetOptions{}) - if err != nil { - return false, nil - } - - // The node may appear to have no labels at first, - // so we wait for it to get hostname label. - if _, found := n.ObjectMeta.Labels[kubeletapis.LabelHostname]; !found { - return false, nil - } - - oldData, err := json.Marshal(n) - if err != nil { - return false, err - } - - // The master node should be tainted and labelled accordingly - markMasterNode(n, taint) - - newData, err := json.Marshal(n) - if err != nil { - return false, err - } - - patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{}) - if err != nil { - return false, err - } - - if _, err := client.CoreV1().Nodes().Patch(n.Name, types.StrategicMergePatchType, patchBytes); err != nil { - if apierrs.IsConflict(err) { - fmt.Println("[markmaster] Temporarily unable to update master node metadata due to conflict (will retry)") - return false, nil - } - return false, err - } - - if taint { - fmt.Printf("[markmaster] Master %s tainted and labelled with key/value: %s=%q\n", masterName, kubeadmconstants.LabelNodeRoleMaster, "") - } else { - fmt.Printf("[markmaster] Master %s labelled with key/value: %s=%q\n", masterName, kubeadmconstants.LabelNodeRoleMaster, "") - } - - return true, nil + return apiclient.PatchNode(client, masterName, func(n *v1.Node) { + markMasterNode(n, taints) }) } -func markMasterNode(n *v1.Node, taint bool) { - n.ObjectMeta.Labels[kubeadmconstants.LabelNodeRoleMaster] = "" - if taint { - addTaintIfNotExists(n, kubeadmconstants.MasterTaint) - } else { - delTaintIfExists(n, kubeadmconstants.MasterTaint) - } -} - -func addTaintIfNotExists(n *v1.Node, t v1.Taint) { - for _, taint := range n.Spec.Taints { - if taint == t { - return - } - } - - n.Spec.Taints = append(n.Spec.Taints, t) -} - -func delTaintIfExists(n *v1.Node, t v1.Taint) { - var taints []v1.Taint - for _, taint := range n.Spec.Taints { - if taint == t { - continue - } - taints = append(taints, t) - } +func markMasterNode(n *v1.Node, taints []v1.Taint) { + n.ObjectMeta.Labels[constants.LabelNodeRoleMaster] = "" + // TODO: Append taints, don't override? n.Spec.Taints = taints } diff --git a/cmd/kubeadm/app/phases/selfhosting/selfhosting.go b/cmd/kubeadm/app/phases/selfhosting/selfhosting.go index 3d20c959dd8..5bd90886e4e 100644 --- a/cmd/kubeadm/app/phases/selfhosting/selfhosting.go +++ b/cmd/kubeadm/app/phases/selfhosting/selfhosting.go @@ -118,7 +118,7 @@ func CreateSelfHostedControlPlane(manifestsDir, kubeConfigDir string, cfg *kubea // Wait for the mirror Pod hash to be removed; otherwise we'll run into race conditions here when the kubelet hasn't had time to // remove the Static Pod (or the mirror Pod respectively). This implicitly also tests that the API server endpoint is healthy, // because this blocks until the API server returns a 404 Not Found when getting the Static Pod - staticPodName := fmt.Sprintf("%s-%s", componentName, cfg.NodeName) + staticPodName := fmt.Sprintf("%s-%s", componentName, cfg.NodeRegistration.Name) if err := waiter.WaitForPodToDisappear(staticPodName); err != nil { return err } diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods.go b/cmd/kubeadm/app/phases/upgrade/staticpods.go index a041c9669cf..77056dcadda 100644 --- a/cmd/kubeadm/app/phases/upgrade/staticpods.go +++ b/cmd/kubeadm/app/phases/upgrade/staticpods.go @@ -206,7 +206,7 @@ func upgradeComponent(component string, waiter apiclient.Waiter, pathMgr StaticP // notice the removal of the Static Pod, leading to a false positive below where we check that the API endpoint is healthy // If we don't do this, there is a case where we remove the Static Pod manifest, kubelet is slow to react, kubeadm checks the // API endpoint below of the OLD Static Pod component and proceeds quickly enough, which might lead to unexpected results. - if err := waiter.WaitForStaticPodHashChange(cfg.NodeName, component, beforePodHash); err != nil { + if err := waiter.WaitForStaticPodHashChange(cfg.NodeRegistration.Name, component, beforePodHash); err != nil { return rollbackOldManifests(recoverManifests, err, pathMgr, recoverEtcd) } @@ -266,7 +266,7 @@ func performEtcdStaticPodUpgrade(waiter apiclient.Waiter, pathMgr StaticPodPathM return false, nil } - beforeEtcdPodHash, err := waiter.WaitForStaticPodSingleHash(cfg.NodeName, constants.Etcd) + beforeEtcdPodHash, err := waiter.WaitForStaticPodSingleHash(cfg.NodeRegistration.Name, constants.Etcd) if err != nil { return true, fmt.Errorf("failed to get etcd pod's hash: %v", err) } @@ -376,7 +376,7 @@ func StaticPodControlPlane(waiter apiclient.Waiter, pathMgr StaticPodPathManager var isTLSUpgrade bool var isExternalEtcd bool - beforePodHashMap, err := waiter.WaitForStaticPodControlPlaneHashes(cfg.NodeName) + beforePodHashMap, err := waiter.WaitForStaticPodControlPlaneHashes(cfg.NodeRegistration.Name) if err != nil { return err } diff --git a/cmd/kubeadm/app/preflight/checks.go b/cmd/kubeadm/app/preflight/checks.go index 922ca00825d..6f0cdf4c876 100644 --- a/cmd/kubeadm/app/preflight/checks.go +++ b/cmd/kubeadm/app/preflight/checks.go @@ -906,7 +906,7 @@ func RunInitMasterChecks(execer utilsexec.Interface, cfg *kubeadmapi.MasterConfi checks = addCommonChecks(execer, cfg, checks) // Check ipvs required kernel module once we use ipvs kube-proxy mode - if cfg.KubeProxy.Config.Mode == ipvsutil.IPVSProxyMode { + if cfg.KubeProxy.Config != nil && cfg.KubeProxy.Config.Mode == ipvsutil.IPVSProxyMode { checks = append(checks, ipvsutil.RequiredIPVSKernelModulesAvailableCheck{Executor: execer}, ) diff --git a/cmd/kubeadm/app/util/apiclient/idempotency.go b/cmd/kubeadm/app/util/apiclient/idempotency.go index ffe42df077a..bc36f65ebdd 100644 --- a/cmd/kubeadm/app/util/apiclient/idempotency.go +++ b/cmd/kubeadm/app/util/apiclient/idempotency.go @@ -17,6 +17,7 @@ limitations under the License. package apiclient import ( + "encoding/json" "fmt" apps "k8s.io/api/apps/v1" @@ -24,7 +25,12 @@ import ( rbac "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/strategicpatch" + "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" + kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" ) // TODO: We should invent a dynamic mechanism for this using the dynamic client instead of hard-coding these functions per-type @@ -186,3 +192,49 @@ func CreateOrUpdateClusterRoleBinding(client clientset.Interface, clusterRoleBin } return nil } + +// PatchNode tries to patch a node using the following client, executing patchFn for the actual mutating logic +func PatchNode(client clientset.Interface, nodeName string, patchFn func(*v1.Node)) error { + // Loop on every false return. Return with an error if raised. Exit successfully if true is returned. + return wait.Poll(constants.APICallRetryInterval, constants.PatchNodeTimeout, func() (bool, error) { + // First get the node object + n, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) + if err != nil { + return false, nil + } + + // The node may appear to have no labels at first, + // so we wait for it to get hostname label. + if _, found := n.ObjectMeta.Labels[kubeletapis.LabelHostname]; !found { + return false, nil + } + + oldData, err := json.Marshal(n) + if err != nil { + return false, err + } + + // Execute the mutating function + patchFn(n) + + newData, err := json.Marshal(n) + if err != nil { + return false, err + } + + patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{}) + if err != nil { + return false, err + } + + if _, err := client.CoreV1().Nodes().Patch(n.Name, types.StrategicMergePatchType, patchBytes); err != nil { + if apierrors.IsConflict(err) { + fmt.Println("[patchnode] Temporarily unable to update node metadata due to conflict (will retry)") + return false, nil + } + return false, err + } + + return true, nil + }) +} diff --git a/cmd/kubeadm/app/util/config/masterconfig.go b/cmd/kubeadm/app/util/config/masterconfig.go index ec2f84dec46..a78149e211c 100644 --- a/cmd/kubeadm/app/util/config/masterconfig.go +++ b/cmd/kubeadm/app/util/config/masterconfig.go @@ -23,6 +23,7 @@ import ( "github.com/golang/glog" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" netutil "k8s.io/apimachinery/pkg/util/net" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" @@ -72,7 +73,12 @@ func SetInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error { } } - cfg.NodeName = node.GetHostname(cfg.NodeName) + cfg.NodeRegistration.Name = node.GetHostname(cfg.NodeRegistration.Name) + + // Only if the slice is nil, we should append the master taint. This allows the user to specify an empty slice for no default master taint + if cfg.NodeRegistration.Taints == nil { + cfg.NodeRegistration.Taints = []v1.Taint{kubeadmconstants.MasterTaint} + } return nil } diff --git a/cmd/kubeadm/app/util/config/nodeconfig.go b/cmd/kubeadm/app/util/config/nodeconfig.go index b8ffc64d932..77bcf8b32f6 100644 --- a/cmd/kubeadm/app/util/config/nodeconfig.go +++ b/cmd/kubeadm/app/util/config/nodeconfig.go @@ -33,7 +33,7 @@ import ( // SetJoinDynamicDefaults checks and sets configuration values for the NodeConfiguration object func SetJoinDynamicDefaults(cfg *kubeadmapi.NodeConfiguration) error { - cfg.NodeName = node.GetHostname(cfg.NodeName) + cfg.NodeRegistration.Name = node.GetHostname(cfg.NodeRegistration.Name) return nil } diff --git a/cmd/kubeadm/test/util.go b/cmd/kubeadm/test/util.go index fdfdae5592b..cb7ae225a68 100644 --- a/cmd/kubeadm/test/util.go +++ b/cmd/kubeadm/test/util.go @@ -57,9 +57,10 @@ func SetupMasterConfigurationFile(t *testing.T, tmpdir string, cfg *kubeadmapi.M kind: MasterConfiguration certificatesDir: {{.CertificatesDir}} api: - advertiseAddress: {{.API.AdvertiseAddress}} - bindPort: {{.API.BindPort}} - nodeName: {{.NodeName}} + advertiseAddress: {{.API.AdvertiseAddress}} + bindPort: {{.API.BindPort}} + nodeRegistration: + name: {{.NodeRegistration.Name}} `))) f, err := os.Create(cfgPath)