diff --git a/cmd/kubeadm/app/apis/kubeadm/types.go b/cmd/kubeadm/app/apis/kubeadm/types.go index 2cad0226185..4391cfe6447 100644 --- a/cmd/kubeadm/app/apis/kubeadm/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/types.go @@ -32,6 +32,21 @@ import ( type MasterConfiguration struct { metav1.TypeMeta + // `kubeadm init`-only information. These fields are solely used the first time `kubeadm init` runs. + // After that, the information in the fields ARE NOT uploaded to the `kubeadm-config` ConfigMap + // that is used by `kubeadm upgrade` for instance. + + // BootstrapTokens is respected at `kubeadm init` time and describes a set of Bootstrap Tokens to create. + // This information IS NOT uploaded to the kubeadm cluster configmap, partly because of its sensitive nature + BootstrapTokens []BootstrapToken + + // NodeRegistration holds fields that relate to registering the new master node to the cluster + NodeRegistration NodeRegistrationOptions + + // Cluster-wide configuration + // TODO: Move these fields under some kind of ClusterConfiguration or similar struct that describes + // one cluster. Eventually we want this kind of spec to align well with the Cluster API spec. + // API holds configuration for the k8s apiserver. API API // KubeProxy holds configuration for the k8s service proxy. @@ -45,13 +60,6 @@ type MasterConfiguration struct { // KubernetesVersion is the target version of the control plane. KubernetesVersion string - // NodeRegistration holds fields that relate to registering the new master node to the cluster - NodeRegistration NodeRegistrationOptions - - // BootstrapTokens is respected at `kubeadm init` time and describes a set of Bootstrap Tokens to create. - // This information IS NOT uploaded to the kubeadm cluster configmap, due to its sensitive nature - BootstrapTokens []BootstrapToken - // APIServerExtraArgs is a set of extra flags to pass to the API Server or override // default ones in form of =. // TODO: This is temporary and ideally we would like to switch all components to @@ -141,10 +149,10 @@ type NodeRegistrationOptions struct { // empty slice, i.e. `taints: {}` in the YAML file. This field is solely used for Node registration. Taints []v1.Taint - // ExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file + // KubeletExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file // kubeadm writes at runtime for the kubelet to source. This overrides the generic base-level configuration in the kubelet-config-1.X ConfigMap // Flags have higher higher priority when parsing. These values are local and specific to the node kubeadm is executing on. - ExtraArgs map[string]string + KubeletExtraArgs map[string]string } // Networking contains elements describing cluster's networking configuration. diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go index 81ab257038b..66fa3997405 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go @@ -244,6 +244,8 @@ func autoConvert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration(in } func autoConvert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration(in *kubeadm.MasterConfiguration, out *MasterConfiguration, s conversion.Scope) error { + // WARNING: in.BootstrapTokens requires manual conversion: does not exist in peer-type + // WARNING: in.NodeRegistration requires manual conversion: does not exist in peer-type if err := Convert_kubeadm_API_To_v1alpha1_API(&in.API, &out.API, s); err != nil { return err } @@ -260,8 +262,6 @@ func autoConvert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration(in return err } out.KubernetesVersion = in.KubernetesVersion - // WARNING: in.NodeRegistration requires manual conversion: does not exist in peer-type - // WARNING: in.BootstrapTokens requires manual conversion: does not exist in peer-type out.APIServerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.APIServerExtraArgs)) out.ControllerManagerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ControllerManagerExtraArgs)) out.SchedulerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.SchedulerExtraArgs)) diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha2/types.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha2/types.go index 9d755d5f8ed..b68184150ca 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha2/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha2/types.go @@ -30,6 +30,21 @@ import ( type MasterConfiguration struct { metav1.TypeMeta `json:",inline"` + // `kubeadm init`-only information. These fields are solely used the first time `kubeadm init` runs. + // After that, the information in the fields ARE NOT uploaded to the `kubeadm-config` ConfigMap + // that is used by `kubeadm upgrade` for instance. These fields must be omitempty. + + // BootstrapTokens is respected at `kubeadm init` time and describes a set of Bootstrap Tokens to create. + // This information IS NOT uploaded to the kubeadm cluster configmap, partly because of its sensitive nature + BootstrapTokens []BootstrapToken `json:"bootstrapTokens,omitempty"` + + // NodeRegistration holds fields that relate to registering the new master node to the cluster + NodeRegistration NodeRegistrationOptions `json:"nodeRegistration,omitempty"` + + // Cluster-wide configuration + // TODO: Move these fields under some kind of ClusterConfiguration or similar struct that describes + // one cluster. Eventually we want this kind of spec to align well with the Cluster API spec. + // API holds configuration for the k8s apiserver. API API `json:"api"` // KubeProxy holds configuration for the k8s service proxy. @@ -41,16 +56,9 @@ type MasterConfiguration struct { // Networking holds configuration for the networking topology of the cluster. Networking Networking `json:"networking"` - // NodeRegistration holds fields that relate to registering the new master node to the cluster - NodeRegistration NodeRegistrationOptions `json:"nodeRegistration"` - // KubernetesVersion is the target version of the control plane. KubernetesVersion string `json:"kubernetesVersion"` - // BootstrapTokens is respected at `kubeadm init` time and describes a set of Bootstrap Tokens to create. - // This information IS NOT uploaded to the kubeadm cluster configmap, due to its sensitive nature - BootstrapTokens []BootstrapToken `json:"bootstrapTokens,omitempty"` - // APIServerExtraArgs is a set of extra flags to pass to the API Server or override // default ones in form of =. // TODO: This is temporary and ideally we would like to switch all components to @@ -133,10 +141,10 @@ type NodeRegistrationOptions struct { // empty slice, i.e. `taints: {}` in the YAML file. This field is solely used for Node registration. Taints []v1.Taint `json:"taints,omitempty"` - // ExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file + // KubeletExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file // kubeadm writes at runtime for the kubelet to source. This overrides the generic base-level configuration in the kubelet-config-1.X ConfigMap // Flags have higher higher priority when parsing. These values are local and specific to the node kubeadm is executing on. - ExtraArgs map[string]string `json:"kubeletExtraArgs,omitempty"` + KubeletExtraArgs map[string]string `json:"kubeletExtraArgs,omitempty"` } // Networking contains elements describing cluster's networking configuration diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha2/zz_generated.conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha2/zz_generated.conversion.go index 07b5c5fc284..22ef44a1464 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha2/zz_generated.conversion.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha2/zz_generated.conversion.go @@ -316,6 +316,10 @@ func Convert_kubeadm_LocalEtcd_To_v1alpha2_LocalEtcd(in *kubeadm.LocalEtcd, out } func autoConvert_v1alpha2_MasterConfiguration_To_kubeadm_MasterConfiguration(in *MasterConfiguration, out *kubeadm.MasterConfiguration, s conversion.Scope) error { + out.BootstrapTokens = *(*[]kubeadm.BootstrapToken)(unsafe.Pointer(&in.BootstrapTokens)) + if err := Convert_v1alpha2_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(&in.NodeRegistration, &out.NodeRegistration, s); err != nil { + return err + } if err := Convert_v1alpha2_API_To_kubeadm_API(&in.API, &out.API, s); err != nil { return err } @@ -331,11 +335,7 @@ func autoConvert_v1alpha2_MasterConfiguration_To_kubeadm_MasterConfiguration(in if err := Convert_v1alpha2_Networking_To_kubeadm_Networking(&in.Networking, &out.Networking, s); err != nil { return err } - if err := Convert_v1alpha2_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(&in.NodeRegistration, &out.NodeRegistration, s); err != nil { - return err - } out.KubernetesVersion = in.KubernetesVersion - out.BootstrapTokens = *(*[]kubeadm.BootstrapToken)(unsafe.Pointer(&in.BootstrapTokens)) out.APIServerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.APIServerExtraArgs)) out.ControllerManagerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ControllerManagerExtraArgs)) out.SchedulerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.SchedulerExtraArgs)) @@ -360,6 +360,10 @@ func Convert_v1alpha2_MasterConfiguration_To_kubeadm_MasterConfiguration(in *Mas } func autoConvert_kubeadm_MasterConfiguration_To_v1alpha2_MasterConfiguration(in *kubeadm.MasterConfiguration, out *MasterConfiguration, s conversion.Scope) error { + out.BootstrapTokens = *(*[]BootstrapToken)(unsafe.Pointer(&in.BootstrapTokens)) + if err := Convert_kubeadm_NodeRegistrationOptions_To_v1alpha2_NodeRegistrationOptions(&in.NodeRegistration, &out.NodeRegistration, s); err != nil { + return err + } if err := Convert_kubeadm_API_To_v1alpha2_API(&in.API, &out.API, s); err != nil { return err } @@ -376,10 +380,6 @@ func autoConvert_kubeadm_MasterConfiguration_To_v1alpha2_MasterConfiguration(in return err } out.KubernetesVersion = in.KubernetesVersion - if err := Convert_kubeadm_NodeRegistrationOptions_To_v1alpha2_NodeRegistrationOptions(&in.NodeRegistration, &out.NodeRegistration, s); err != nil { - return err - } - out.BootstrapTokens = *(*[]BootstrapToken)(unsafe.Pointer(&in.BootstrapTokens)) out.APIServerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.APIServerExtraArgs)) out.ControllerManagerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ControllerManagerExtraArgs)) out.SchedulerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.SchedulerExtraArgs)) @@ -478,7 +478,7 @@ func autoConvert_v1alpha2_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOpt out.Name = in.Name out.CRISocket = in.CRISocket out.Taints = *(*[]core_v1.Taint)(unsafe.Pointer(&in.Taints)) - out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) + out.KubeletExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.KubeletExtraArgs)) return nil } @@ -491,7 +491,7 @@ func autoConvert_kubeadm_NodeRegistrationOptions_To_v1alpha2_NodeRegistrationOpt out.Name = in.Name out.CRISocket = in.CRISocket out.Taints = *(*[]core_v1.Taint)(unsafe.Pointer(&in.Taints)) - out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) + out.KubeletExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.KubeletExtraArgs)) return nil } diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha2/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha2/zz_generated.deepcopy.go index 00d78c57efe..d63be5c14c8 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha2/zz_generated.deepcopy.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha2/zz_generated.deepcopy.go @@ -295,12 +295,6 @@ func (in *LocalEtcd) DeepCopy() *LocalEtcd { func (in *MasterConfiguration) DeepCopyInto(out *MasterConfiguration) { *out = *in out.TypeMeta = in.TypeMeta - out.API = in.API - in.KubeProxy.DeepCopyInto(&out.KubeProxy) - in.Etcd.DeepCopyInto(&out.Etcd) - in.KubeletConfiguration.DeepCopyInto(&out.KubeletConfiguration) - out.Networking = in.Networking - in.NodeRegistration.DeepCopyInto(&out.NodeRegistration) if in.BootstrapTokens != nil { in, out := &in.BootstrapTokens, &out.BootstrapTokens *out = make([]BootstrapToken, len(*in)) @@ -308,6 +302,12 @@ func (in *MasterConfiguration) DeepCopyInto(out *MasterConfiguration) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + in.NodeRegistration.DeepCopyInto(&out.NodeRegistration) + out.API = in.API + in.KubeProxy.DeepCopyInto(&out.KubeProxy) + in.Etcd.DeepCopyInto(&out.Etcd) + in.KubeletConfiguration.DeepCopyInto(&out.KubeletConfiguration) + out.Networking = in.Networking if in.APIServerExtraArgs != nil { in, out := &in.APIServerExtraArgs, &out.APIServerExtraArgs *out = make(map[string]string, len(*in)) @@ -456,8 +456,8 @@ func (in *NodeRegistrationOptions) DeepCopyInto(out *NodeRegistrationOptions) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.ExtraArgs != nil { - in, out := &in.ExtraArgs, &out.ExtraArgs + if in.KubeletExtraArgs != nil { + in, out := &in.KubeletExtraArgs, &out.KubeletExtraArgs *out = make(map[string]string, len(*in)) for key, val := range *in { (*out)[key] = val diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha2/zz_generated.defaults.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha2/zz_generated.defaults.go index 46560609bcb..940e5621f8c 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha2/zz_generated.defaults.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha2/zz_generated.defaults.go @@ -37,17 +37,17 @@ func RegisterDefaults(scheme *runtime.Scheme) error { func SetObjectDefaults_MasterConfiguration(in *MasterConfiguration) { SetDefaults_MasterConfiguration(in) + for i := range in.BootstrapTokens { + a := &in.BootstrapTokens[i] + SetDefaults_BootstrapToken(a) + } + SetDefaults_NodeRegistrationOptions(&in.NodeRegistration) if in.KubeProxy.Config != nil { v1alpha1.SetDefaults_KubeProxyConfiguration(in.KubeProxy.Config) } if in.KubeletConfiguration.BaseConfig != nil { v1beta1.SetDefaults_KubeletConfiguration(in.KubeletConfiguration.BaseConfig) } - SetDefaults_NodeRegistrationOptions(&in.NodeRegistration) - for i := range in.BootstrapTokens { - a := &in.BootstrapTokens[i] - SetDefaults_BootstrapToken(a) - } } func SetObjectDefaults_NodeConfiguration(in *NodeConfiguration) { diff --git a/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go index 2c2a23323f8..2295df2419f 100644 --- a/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go +++ b/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go @@ -295,12 +295,6 @@ func (in *LocalEtcd) DeepCopy() *LocalEtcd { func (in *MasterConfiguration) DeepCopyInto(out *MasterConfiguration) { *out = *in out.TypeMeta = in.TypeMeta - out.API = in.API - in.KubeProxy.DeepCopyInto(&out.KubeProxy) - in.Etcd.DeepCopyInto(&out.Etcd) - in.KubeletConfiguration.DeepCopyInto(&out.KubeletConfiguration) - out.Networking = in.Networking - in.NodeRegistration.DeepCopyInto(&out.NodeRegistration) if in.BootstrapTokens != nil { in, out := &in.BootstrapTokens, &out.BootstrapTokens *out = make([]BootstrapToken, len(*in)) @@ -308,6 +302,12 @@ func (in *MasterConfiguration) DeepCopyInto(out *MasterConfiguration) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + in.NodeRegistration.DeepCopyInto(&out.NodeRegistration) + out.API = in.API + in.KubeProxy.DeepCopyInto(&out.KubeProxy) + in.Etcd.DeepCopyInto(&out.Etcd) + in.KubeletConfiguration.DeepCopyInto(&out.KubeletConfiguration) + out.Networking = in.Networking if in.APIServerExtraArgs != nil { in, out := &in.APIServerExtraArgs, &out.APIServerExtraArgs *out = make(map[string]string, len(*in)) @@ -456,8 +456,8 @@ func (in *NodeRegistrationOptions) DeepCopyInto(out *NodeRegistrationOptions) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.ExtraArgs != nil { - in, out := &in.ExtraArgs, &out.ExtraArgs + if in.KubeletExtraArgs != nil { + in, out := &in.KubeletExtraArgs, &out.KubeletExtraArgs *out = make(map[string]string, len(*in)) for key, val := range *in { (*out)[key] = val diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 637d2a6fc0c..e3c3aafd9eb 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -277,24 +277,35 @@ type Init struct { // Run executes master node provisioning, including certificates, needed static pod manifests, etc. func (i *Init) Run(out io.Writer) error { - // Write env file with flags for the kubelet to use. We do not need to write the --register-with-taints for the master, - // as we handle that ourselves in the markmaster phase - // TODO: Maybe we want to do that some time in the future, in order to remove some logic from the markmaster phase? - if err := kubeletphase.WriteKubeletDynamicEnvFile(&i.cfg.NodeRegistration, false); err != nil { - return err - } - - // Try to start the kubelet service in case it's inactive - glog.V(1).Infof("Starting kubelet") - preflight.TryStartKubelet(i.ignorePreflightErrors) - // Get directories to write files to; can be faked if we're dry-running glog.V(1).Infof("[init] Getting certificates directory from configuration") realCertsDir := i.cfg.CertificatesDir - certsDirToWriteTo, kubeConfigDir, manifestDir, err := getDirectoriesToUse(i.dryRun, i.cfg.CertificatesDir) + certsDirToWriteTo, kubeConfigDir, manifestDir, kubeletDir, err := getDirectoriesToUse(i.dryRun, i.cfg.CertificatesDir) if err != nil { return fmt.Errorf("error getting directories to use: %v", err) } + + // First off, configure the kubelet. In this short timeframe, kubeadm is trying to stop/restart the kubelet + // Try to stop the kubelet service so no race conditions occur when configuring it + glog.V(1).Infof("Stopping the kubelet") + preflight.TryStopKubelet(i.ignorePreflightErrors) + + // Write env file with flags for the kubelet to use. We do not need to write the --register-with-taints for the master, + // as we handle that ourselves in the markmaster phase + // TODO: Maybe we want to do that some time in the future, in order to remove some logic from the markmaster phase? + if err := kubeletphase.WriteKubeletDynamicEnvFile(&i.cfg.NodeRegistration, i.cfg.FeatureGates, false, kubeletDir); err != nil { + return fmt.Errorf("error writing a dynamic environment file for the kubelet: %v", err) + } + + // Write the kubelet configuration file to disk. + if err := kubeletphase.WriteConfigToDisk(i.cfg.KubeletConfiguration.BaseConfig, kubeletDir); err != nil { + return fmt.Errorf("error writing kubelet configuration to disk: %v", err) + } + + // Try to start the kubelet service in case it's inactive + glog.V(1).Infof("Starting the kubelet") + preflight.TryStartKubelet(i.ignorePreflightErrors) + // certsDirToWriteTo is gonna equal cfg.CertificatesDir in the normal case, but gonna be a temp directory if dryrunning i.cfg.CertificatesDir = certsDirToWriteTo @@ -359,16 +370,6 @@ func (i *Init) Run(out io.Writer) error { return fmt.Errorf("error printing files on dryrun: %v", err) } - kubeletVersion, err := preflight.GetKubeletVersion(utilsexec.New()) - if err != nil { - return err - } - - // Write the kubelet configuration to disk. - if err := kubeletphase.WriteConfigToDisk(i.cfg.KubeletConfiguration.BaseConfig); err != nil { - return fmt.Errorf("error writing kubelet configuration to disk: %v", err) - } - // Create a kubernetes client and wait for the API server to be healthy (if not dryrunning) glog.V(1).Infof("creating Kubernetes client") client, err := createClient(i.cfg, i.dryRun) @@ -426,9 +427,13 @@ func (i *Init) Run(out io.Writer) error { return fmt.Errorf("error uploading crisocket: %v", err) } - // NOTE: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf - // This feature is disabled by default, as it is alpha still + // This feature is disabled by default if features.Enabled(i.cfg.FeatureGates, features.DynamicKubeletConfig) { + kubeletVersion, err := preflight.GetKubeletVersion(utilsexec.New()) + if err != nil { + return err + } + // Enable dynamic kubelet configuration for the node. if err := kubeletphase.EnableDynamicConfigForNode(client, i.cfg.NodeRegistration.Name, kubeletVersion); err != nil { return fmt.Errorf("error enabling dynamic kubelet configuration: %v", err) @@ -544,17 +549,17 @@ func createClient(cfg *kubeadmapi.MasterConfiguration, dryRun bool) (clientset.I // getDirectoriesToUse returns the (in order) certificates, kubeconfig and Static Pod manifest directories, followed by a possible error // This behaves differently when dry-running vs the normal flow -func getDirectoriesToUse(dryRun bool, defaultPkiDir string) (string, string, string, error) { +func getDirectoriesToUse(dryRun bool, defaultPkiDir string) (string, string, string, string, error) { if dryRun { dryRunDir, err := ioutil.TempDir("", "kubeadm-init-dryrun") if err != nil { - return "", "", "", fmt.Errorf("couldn't create a temporary directory: %v", err) + return "", "", "", "", fmt.Errorf("couldn't create a temporary directory: %v", err) } // Use the same temp dir for all - return dryRunDir, dryRunDir, dryRunDir, nil + return dryRunDir, dryRunDir, dryRunDir, dryRunDir, nil } - return defaultPkiDir, kubeadmconstants.KubernetesDir, kubeadmconstants.GetStaticPodDirectory(), nil + return defaultPkiDir, kubeadmconstants.KubernetesDir, kubeadmconstants.GetStaticPodDirectory(), kubeadmconstants.KubeletRunDirectory, nil } // printFilesIfDryRunning prints the Static Pod manifests to stdout and informs about the temporary directory to go and lookup @@ -569,11 +574,19 @@ func printFilesIfDryRunning(dryRun bool, manifestDir string) error { // Print the contents of the upgraded manifests and pretend like they were in /etc/kubernetes/manifests files := []dryrunutil.FileToPrint{} + // Print static pod manifests for _, component := range kubeadmconstants.MasterComponents { realPath := kubeadmconstants.GetStaticPodFilepath(component, manifestDir) outputPath := kubeadmconstants.GetStaticPodFilepath(component, kubeadmconstants.GetStaticPodDirectory()) files = append(files, dryrunutil.NewFileToPrint(realPath, outputPath)) } + // Print kubelet config manifests + kubeletConfigFiles := []string{kubeadmconstants.KubeletConfigurationFileName, kubeadmconstants.KubeletEnvFileName} + for _, filename := range kubeletConfigFiles { + realPath := filepath.Join(manifestDir, filename) + outputPath := filepath.Join(kubeadmconstants.KubeletRunDirectory, filename) + files = append(files, dryrunutil.NewFileToPrint(realPath, outputPath)) + } return dryrunutil.PrintDryRunFiles(files, os.Stdout) } diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index 00fdd5e9942..4c1f32c180a 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -215,7 +215,8 @@ func AddJoinOtherFlags(flagSet *flag.FlagSet, cfgPath *string, skipPreFlight *bo // Join defines struct used by kubeadm join command type Join struct { - cfg *kubeadmapi.NodeConfiguration + cfg *kubeadmapi.NodeConfiguration + ignorePreflightErrors sets.String } // NewJoin instantiates Join struct with given arguments @@ -239,39 +240,31 @@ func NewJoin(cfgPath string, args []string, defaultcfg *kubeadmapiv1alpha2.NodeC return nil, err } - // Try to start the kubelet service in case it's inactive - glog.V(1).Infoln("[preflight] starting kubelet service if it's inactive") - preflight.TryStartKubelet(ignorePreflightErrors) - - return &Join{cfg: internalcfg}, nil + return &Join{cfg: internalcfg, ignorePreflightErrors: ignorePreflightErrors}, nil } // Run executes worker node provisioning and tries to join an existing cluster. func (j *Join) Run(out io.Writer) error { - // Write env file with flags for the kubelet to use. Also register taints - if err := kubeletphase.WriteKubeletDynamicEnvFile(&j.cfg.NodeRegistration, true); err != nil { - return err - } - + // Perform the Discovery, which turns a Bootstrap Token and optionally (and preferably) a CA cert hash into a KubeConfig + // file that may be used for the TLS Bootstrapping process the kubelet performs using the Certificates API. glog.V(1).Infoln("[join] retrieving KubeConfig objects") cfg, err := discovery.For(j.cfg) if err != nil { return err } - kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletBootstrapKubeConfigFileName) + bootstrapKubeConfigFile := kubeadmconstants.GetBootstrapKubeletKubeConfigPath() // Write the bootstrap kubelet config file or the TLS-Boostrapped kubelet config file down to disk - glog.V(1).Infoln("[join] writing bootstrap kubelet config file at", kubeconfigFile) - if err := kubeconfigutil.WriteToDisk(kubeconfigFile, cfg); err != nil { + glog.V(1).Infoln("[join] writing bootstrap kubelet config file at", bootstrapKubeConfigFile) + if err := kubeconfigutil.WriteToDisk(bootstrapKubeConfigFile, cfg); err != nil { return fmt.Errorf("couldn't save bootstrap-kubelet.conf to disk: %v", err) } // Write the ca certificate to disk so kubelet can use it for authentication cluster := cfg.Contexts[cfg.CurrentContext].Cluster - err = certutil.WriteCert(j.cfg.CACertPath, cfg.Clusters[cluster].CertificateAuthorityData) - if err != nil { + if err := certutil.WriteCert(j.cfg.CACertPath, cfg.Clusters[cluster].CertificateAuthorityData); err != nil { return fmt.Errorf("couldn't save the CA certificate to disk: %v", err) } @@ -280,12 +273,31 @@ func (j *Join) Run(out io.Writer) error { return err } - // Write the configuration for the kubelet down to disk so the kubelet can start - if err := kubeletphase.DownloadConfig(kubeconfigFile, kubeletVersion); err != nil { + bootstrapClient, err := kubeconfigutil.ClientSetFromFile(bootstrapKubeConfigFile) + if err != nil { + return fmt.Errorf("couldn't create client from kubeconfig file %q", bootstrapKubeConfigFile) + } + + // Configure the kubelet. In this short timeframe, kubeadm is trying to stop/restart the kubelet + // Try to stop the kubelet service so no race conditions occur when configuring it + glog.V(1).Infof("Stopping the kubelet") + preflight.TryStopKubelet(j.ignorePreflightErrors) + + // Write the configuration for the kubelet (using the bootstrap token credentials) to disk so the kubelet can start + if err := kubeletphase.DownloadConfig(bootstrapClient, kubeletVersion, kubeadmconstants.KubeletRunDirectory); err != nil { return err } - // Now the kubelet will perform the TLS Bootstrap, transforming bootstrap-kubeconfig.conf to kubeconfig.conf in /etc/kubernetes + // Write env file with flags for the kubelet to use. Also register taints + if err := kubeletphase.WriteKubeletDynamicEnvFile(&j.cfg.NodeRegistration, j.cfg.FeatureGates, true, kubeadmconstants.KubeletRunDirectory); err != nil { + return err + } + + // Try to start the kubelet service in case it's inactive + glog.V(1).Infof("Starting the kubelet") + preflight.TryStartKubelet(j.ignorePreflightErrors) + + // Now the kubelet will perform the TLS Bootstrap, transforming /etc/kubernetes/bootstrap-kubelet.conf to /etc/kubernetes/kubelet.conf // Wait for the kubelet to create the /etc/kubernetes/kubelet.conf KubeConfig file. If this process // times out, display a somewhat user-friendly message. waiter := apiclient.NewKubeWaiter(nil, kubeadmconstants.TLSBootstrapTimeout, os.Stdout) @@ -295,8 +307,7 @@ func (j *Join) Run(out io.Writer) error { } // When we know the /etc/kubernetes/kubelet.conf file is available, get the client - kubeletKubeConfig := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName) - client, err := kubeconfigutil.ClientSetFromFile(kubeletKubeConfig) + client, err := kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetKubeletKubeConfigPath()) if err != nil { return err } @@ -306,9 +317,7 @@ func (j *Join) Run(out io.Writer) error { return fmt.Errorf("error uploading crisocket: %v", err) } - // NOTE: the "--dynamic-config-dir" flag should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf for this to work - // This feature is disabled by default, as it is alpha still - glog.V(1).Infoln("[join] enabling dynamic kubelet configuration") + // This feature is disabled by default in kubeadm if features.Enabled(j.cfg.FeatureGates, features.DynamicKubeletConfig) { if err := kubeletphase.EnableDynamicConfigForNode(client, j.cfg.NodeRegistration.Name, kubeletVersion); err != nil { return fmt.Errorf("error consuming base kubelet configuration: %v", err) diff --git a/cmd/kubeadm/app/cmd/options/BUILD b/cmd/kubeadm/app/cmd/options/BUILD index 6c2d59b5c07..c04f725e520 100644 --- a/cmd/kubeadm/app/cmd/options/BUILD +++ b/cmd/kubeadm/app/cmd/options/BUILD @@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", - srcs = ["token.go"], + srcs = [ + "generic.go", + "token.go", + ], importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options", visibility = ["//visibility:public"], deps = [ diff --git a/cmd/kubeadm/app/cmd/options/generic.go b/cmd/kubeadm/app/cmd/options/generic.go new file mode 100644 index 00000000000..7aa8d82f98c --- /dev/null +++ b/cmd/kubeadm/app/cmd/options/generic.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import "github.com/spf13/pflag" + +// AddKubeConfigFlag adds the --kubeconfig flag to the given flagset +func AddKubeConfigFlag(fs *pflag.FlagSet, kubeConfigFile *string) { + fs.StringVar(kubeConfigFile, "kubeconfig", *kubeConfigFile, "The KubeConfig file to use when talking to the cluster") +} + +// AddConfigFlag adds the --config flag to the given flagset +func AddConfigFlag(fs *pflag.FlagSet, cfgPath *string) { + fs.StringVar(cfgPath, "config", *cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)") +} diff --git a/cmd/kubeadm/app/cmd/phases/kubelet.go b/cmd/kubeadm/app/cmd/phases/kubelet.go index bd979d60492..f18171370e0 100644 --- a/cmd/kubeadm/app/cmd/phases/kubelet.go +++ b/cmd/kubeadm/app/cmd/phases/kubelet.go @@ -18,45 +18,81 @@ package phases import ( "fmt" - "os" + "io/ioutil" "github.com/spf13/cobra" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" + "k8s.io/kubernetes/cmd/kubeadm/app/preflight" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" "k8s.io/kubernetes/pkg/util/normalizer" "k8s.io/kubernetes/pkg/util/version" + utilsexec "k8s.io/utils/exec" +) + +const ( + // TODO: Figure out how to get these constants from the API machinery + masterConfig = "MasterConfiguration" + nodeConfig = "NodeConfiguration" ) var ( - kubeletWriteConfigToDiskLongDesc = normalizer.LongDesc(` - Writes kubelet configuration to disk, either based on the kubelet-config-1.X ConfigMap in the cluster, or from the - configuration passed to the command via "--config". + kubeletWriteEnvFileLongDesc = normalizer.LongDesc(` + Writes an environment file with flags that should be passed to the kubelet executing on the master or node. + This --config flag can either consume a MasterConfiguration object or a NodeConfiguration one, as this + function is used for both "kubeadm init" and "kubeadm join". ` + cmdutil.AlphaDisclaimer) - kubeletWriteConfigToDiskExample = normalizer.Examples(` - # Writes kubelet configuration for a node to disk. The information is fetched from the cluster ConfigMap - kubeadm alpha phase kubelet write-config-to-disk --kubelet-version v1.11.0 --kubeconfig /etc/kubernetes/kubelet.conf + kubeletWriteEnvFileExample = normalizer.Examples(` + # Writes a dynamic environment file with kubelet flags from a MasterConfiguration file. + kubeadm alpha phase kubelet write-env-file --config masterconfig.yaml - # Writes kubelet configuration down to disk, based on the configuration flag passed to --config - kubeadm alpha phase kubelet write-config-to-disk --kubelet-version v1.11.0 --config kubeadm.yaml + # Writes a dynamic environment file with kubelet flags from a NodeConfiguration file. + kubeadm alpha phase kubelet write-env-file --config nodeConfig.yaml `) - kubeletUploadDynamicConfigLongDesc = normalizer.LongDesc(` + kubeletConfigUploadLongDesc = normalizer.LongDesc(` Uploads kubelet configuration extracted from the kubeadm MasterConfiguration object to a ConfigMap - of the form kubelet-config-1.X in the cluster, where X is the minor version of the current Kubernetes version + of the form kubelet-config-1.X in the cluster, where X is the minor version of the current (API Server) Kubernetes version. ` + cmdutil.AlphaDisclaimer) - kubeletUploadDynamicConfigExample = normalizer.Examples(` + kubeletConfigUploadExample = normalizer.Examples(` # Uploads the kubelet configuration from the kubeadm Config file to a ConfigMap in the cluster. - kubeadm alpha phase kubelet upload-config --config kubeadm.yaml + kubeadm alpha phase kubelet config upload --config kubeadm.yaml `) - kubeletEnableDynamicConfigLongDesc = normalizer.LongDesc(` + kubeletConfigDownloadLongDesc = normalizer.LongDesc(` + Downloads the kubelet configuration from a ConfigMap of the form "kubelet-config-1.X" in the cluster, + where X is the minor version of the kubelet. Either kubeadm autodetects the kubelet version by exec-ing + "kubelet --version" or respects the --kubelet-version parameter. + ` + cmdutil.AlphaDisclaimer) + + kubeletConfigDownloadExample = normalizer.Examples(` + # Downloads the kubelet configuration from the ConfigMap in the cluster. Autodetects the kubelet version. + kubeadm alpha phase kubelet config download + + # Downloads the kubelet configuration from the ConfigMap in the cluster. Uses a specific desired kubelet version. + kubeadm alpha phase kubelet config download --kubelet-version v1.11.0 + `) + + kubeletConfigWriteToDiskLongDesc = normalizer.LongDesc(` + Writes kubelet configuration to disk, based on the kubeadm configuration passed via "--config". + ` + cmdutil.AlphaDisclaimer) + + kubeletConfigWriteToDiskExample = normalizer.Examples(` + # Extracts the kubelet configuration from a kubeadm configuration file + kubeadm alpha phase kubelet config write-to-disk --config kubeadm.yaml + `) + + kubeletConfigEnableDynamicLongDesc = normalizer.LongDesc(` Enables or updates dynamic kubelet configuration for a Node, against the kubelet-config-1.X ConfigMap in the cluster, where X is the minor version of the desired kubelet version. @@ -65,7 +101,7 @@ var ( ` + cmdutil.AlphaDisclaimer) - kubeletEnableDynamicConfigExample = normalizer.Examples(` + kubeletConfigEnableDynamicExample = normalizer.Examples(` # Enables dynamic kubelet configuration for a Node. kubeadm alpha phase kubelet enable-dynamic-config --node-name node-1 --kubelet-version v1.11.0 @@ -74,32 +110,110 @@ var ( `) ) -// NewCmdKubelet returns main command for Kubelet phase +// NewCmdKubelet returns command for `kubeadm phase kubelet` func NewCmdKubelet() *cobra.Command { - var kubeConfigFile string cmd := &cobra.Command{ Use: "kubelet", + Short: "Commands related to handling the kubelet.", + Long: cmdutil.MacroCommandLongDescription, + } + + cmd.AddCommand(NewCmdKubeletConfig()) + cmd.AddCommand(NewCmdKubeletWriteEnvFile()) + return cmd +} + +// NewCmdKubeletWriteEnvFile calls cobra.Command for writing the dynamic kubelet env file based on a MasterConfiguration or NodeConfiguration object +func NewCmdKubeletWriteEnvFile() *cobra.Command { + var cfgPath string + + cmd := &cobra.Command{ + Use: "write-env-file", + Short: "Writes an environment file with runtime flags for the kubelet.", + Long: kubeletWriteEnvFileLongDesc, + Example: kubeletWriteEnvFileExample, + Run: func(cmd *cobra.Command, args []string) { + err := RunKubeletWriteEnvFile(cfgPath) + kubeadmutil.CheckErr(err) + }, + } + + options.AddConfigFlag(cmd.Flags(), &cfgPath) + return cmd +} + +// RunKubeletWriteEnvFile is the function that is run when "kubeadm phase kubelet write-env-file" is executed +func RunKubeletWriteEnvFile(cfgPath string) error { + b, err := ioutil.ReadFile(cfgPath) + if err != nil { + return err + } + + gvk, err := kubeadmutil.GroupVersionKindFromBytes(b, kubeadmscheme.Codecs) + if err != nil { + return err + } + + var nodeRegistrationObj *kubeadmapi.NodeRegistrationOptions + var featureGates map[string]bool + var registerWithTaints bool + switch gvk.Kind { + case masterConfig: + internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, &kubeadmapiv1alpha2.MasterConfiguration{}) + if err != nil { + return err + } + nodeRegistrationObj = &internalcfg.NodeRegistration + featureGates = internalcfg.FeatureGates + registerWithTaints = false + case nodeConfig: + internalcfg, err := configutil.NodeConfigFileAndDefaultsToInternalConfig(cfgPath, &kubeadmapiv1alpha2.NodeConfiguration{}) + if err != nil { + return err + } + nodeRegistrationObj = &internalcfg.NodeRegistration + featureGates = internalcfg.FeatureGates + registerWithTaints = true + default: + if err != nil { + return fmt.Errorf("Didn't recognize type with GroupVersionKind: %v", gvk) + } + } + if nodeRegistrationObj == nil { + return fmt.Errorf("couldn't load nodeRegistration field from config file") + } + + if err := kubeletphase.WriteKubeletDynamicEnvFile(nodeRegistrationObj, featureGates, registerWithTaints, constants.KubeletRunDirectory); err != nil { + return fmt.Errorf("error writing a dynamic environment file for the kubelet: %v", err) + } + return nil +} + +// NewCmdKubeletConfig returns command for `kubeadm phase kubelet config` +func NewCmdKubeletConfig() *cobra.Command { + cmd := &cobra.Command{ + Use: "config", Short: "Handles kubelet configuration.", Long: cmdutil.MacroCommandLongDescription, } - cmd.PersistentFlags().StringVar(&kubeConfigFile, "kubeconfig", "/etc/kubernetes/admin.conf", "The KubeConfig file to use when talking to the cluster") - - cmd.AddCommand(NewCmdKubeletWriteConfigToDisk(&kubeConfigFile)) - cmd.AddCommand(NewCmdKubeletUploadConfig(&kubeConfigFile)) - cmd.AddCommand(NewCmdKubeletEnableDynamicConfig(&kubeConfigFile)) + cmd.AddCommand(NewCmdKubeletConfigUpload()) + cmd.AddCommand(NewCmdKubeletConfigDownload()) + cmd.AddCommand(NewCmdKubeletConfigWriteToDisk()) + cmd.AddCommand(NewCmdKubeletConfigEnableDynamic()) return cmd } -// NewCmdKubeletUploadConfig calls cobra.Command for uploading dynamic kubelet configuration -func NewCmdKubeletUploadConfig(kubeConfigFile *string) *cobra.Command { +// NewCmdKubeletConfigUpload calls cobra.Command for uploading dynamic kubelet configuration +func NewCmdKubeletConfigUpload() *cobra.Command { var cfgPath string + kubeConfigFile := constants.GetAdminKubeConfigPath() cmd := &cobra.Command{ - Use: "upload-config", - Short: "Uploads kubelet configuration to a ConfigMap", - Long: kubeletUploadDynamicConfigLongDesc, - Example: kubeletUploadDynamicConfigExample, + Use: "upload", + Short: "Uploads kubelet configuration to a ConfigMap based on a kubeadm MasterConfiguration file.", + Long: kubeletConfigUploadLongDesc, + Example: kubeletConfigUploadExample, Run: func(cmd *cobra.Command, args []string) { if len(cfgPath) == 0 { kubeadmutil.CheckErr(fmt.Errorf("The --config argument is required")) @@ -109,7 +223,7 @@ func NewCmdKubeletUploadConfig(kubeConfigFile *string) *cobra.Command { internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, &kubeadmapiv1alpha2.MasterConfiguration{}) kubeadmutil.CheckErr(err) - client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile) + client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile) kubeadmutil.CheckErr(err) err = kubeletphase.CreateConfigMap(internalcfg, client) @@ -117,50 +231,83 @@ func NewCmdKubeletUploadConfig(kubeConfigFile *string) *cobra.Command { }, } - cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)") + options.AddKubeConfigFlag(cmd.Flags(), &kubeConfigFile) + options.AddConfigFlag(cmd.Flags(), &cfgPath) return cmd } -// NewCmdKubeletWriteConfigToDisk calls cobra.Command for writing init kubelet configuration -func NewCmdKubeletWriteConfigToDisk(kubeConfigFile *string) *cobra.Command { - var cfgPath, kubeletVersionStr string +// NewCmdKubeletConfigDownload calls cobra.Command for downloading the kubelet configuration from the kubelet-config-1.X ConfigMap in the cluster +func NewCmdKubeletConfigDownload() *cobra.Command { + var kubeletVersionStr string + // TODO: Be smarter about this and be able to load multiple kubeconfig files in different orders of precedence + kubeConfigFile := constants.GetKubeletKubeConfigPath() + cmd := &cobra.Command{ - Use: "write-config-to-disk", - Short: "Writes kubelet configuration to disk, either based on the --config argument or the kubeadm-config ConfigMap.", - Long: kubeletWriteConfigToDiskLongDesc, - Example: kubeletWriteConfigToDiskExample, + Use: "download", + Short: "Downloads the kubelet configuration from the cluster ConfigMap kubelet-config-1.X, where X is the minor version of the kubelet.", + Long: kubeletConfigDownloadLongDesc, + Example: kubeletConfigDownloadExample, Run: func(cmd *cobra.Command, args []string) { - if len(kubeletVersionStr) == 0 { - kubeadmutil.CheckErr(fmt.Errorf("The --kubelet-version argument is required")) - } - - client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile) + kubeletVersion, err := getKubeletVersion(kubeletVersionStr) kubeadmutil.CheckErr(err) - // This call returns the ready-to-use configuration based on the configuration file - internalcfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "kubelet", cfgPath) + client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile) kubeadmutil.CheckErr(err) - err = kubeletphase.WriteConfigToDisk(internalcfg.KubeletConfiguration.BaseConfig) + err = kubeletphase.DownloadConfig(client, kubeletVersion, constants.KubeletRunDirectory) kubeadmutil.CheckErr(err) }, } - cmd.Flags().StringVar(&kubeletVersionStr, "kubelet-version", kubeletVersionStr, "The desired version for the kubelet") - cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)") + options.AddKubeConfigFlag(cmd.Flags(), &kubeConfigFile) + cmd.Flags().StringVar(&kubeletVersionStr, "kubelet-version", kubeletVersionStr, "The desired version for the kubelet. Defaults to being autodetected from 'kubelet --version'.") return cmd } -// NewCmdKubeletEnableDynamicConfig calls cobra.Command for enabling dynamic kubelet configuration on node +func getKubeletVersion(kubeletVersionStr string) (*version.Version, error) { + if len(kubeletVersionStr) > 0 { + return version.ParseSemantic(kubeletVersionStr) + } + return preflight.GetKubeletVersion(utilsexec.New()) +} + +// NewCmdKubeletConfigWriteToDisk calls cobra.Command for writing init kubelet configuration +func NewCmdKubeletConfigWriteToDisk() *cobra.Command { + var cfgPath string + cmd := &cobra.Command{ + Use: "write-to-disk", + Short: "Writes kubelet configuration to disk, either based on the --config argument.", + Long: kubeletConfigWriteToDiskLongDesc, + Example: kubeletConfigWriteToDiskExample, + Run: func(cmd *cobra.Command, args []string) { + if len(cfgPath) == 0 { + kubeadmutil.CheckErr(fmt.Errorf("The --config argument is required")) + } + + // This call returns the ready-to-use configuration based on the configuration file + internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, &kubeadmapiv1alpha2.MasterConfiguration{}) + kubeadmutil.CheckErr(err) + + err = kubeletphase.WriteConfigToDisk(internalcfg.KubeletConfiguration.BaseConfig, constants.KubeletRunDirectory) + kubeadmutil.CheckErr(err) + }, + } + + options.AddConfigFlag(cmd.Flags(), &cfgPath) + return cmd +} + +// NewCmdKubeletConfigEnableDynamic calls cobra.Command for enabling dynamic kubelet configuration on node // This feature is still in alpha and an experimental state -func NewCmdKubeletEnableDynamicConfig(kubeConfigFile *string) *cobra.Command { +func NewCmdKubeletConfigEnableDynamic() *cobra.Command { var nodeName, kubeletVersionStr string + kubeConfigFile := constants.GetAdminKubeConfigPath() cmd := &cobra.Command{ - Use: "enable-dynamic-config", + Use: "enable-dynamic", Short: "EXPERIMENTAL: Enables or updates dynamic kubelet configuration for a Node", - Long: kubeletEnableDynamicConfigLongDesc, - Example: kubeletEnableDynamicConfigExample, + Long: kubeletConfigEnableDynamicLongDesc, + Example: kubeletConfigEnableDynamicExample, Run: func(cmd *cobra.Command, args []string) { if len(nodeName) == 0 { kubeadmutil.CheckErr(fmt.Errorf("The --node-name argument is required")) @@ -172,7 +319,7 @@ func NewCmdKubeletEnableDynamicConfig(kubeConfigFile *string) *cobra.Command { kubeletVersion, err := version.ParseSemantic(kubeletVersionStr) kubeadmutil.CheckErr(err) - client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile) + client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile) kubeadmutil.CheckErr(err) err = kubeletphase.EnableDynamicConfigForNode(client, nodeName, kubeletVersion) @@ -180,6 +327,7 @@ func NewCmdKubeletEnableDynamicConfig(kubeConfigFile *string) *cobra.Command { }, } + options.AddKubeConfigFlag(cmd.Flags(), &kubeConfigFile) cmd.Flags().StringVar(&nodeName, "node-name", nodeName, "Name of the node that should enable the dynamic kubelet configuration") cmd.Flags().StringVar(&kubeletVersionStr, "kubelet-version", kubeletVersionStr, "The desired version for the kubelet") return cmd diff --git a/cmd/kubeadm/app/cmd/phases/kubelet_test.go b/cmd/kubeadm/app/cmd/phases/kubelet_test.go index 89159d26105..4abfdd5d7cb 100644 --- a/cmd/kubeadm/app/cmd/phases/kubelet_test.go +++ b/cmd/kubeadm/app/cmd/phases/kubelet_test.go @@ -25,11 +25,12 @@ import ( ) func TestKubeletSubCommandsHasFlags(t *testing.T) { - kubeConfigFile := "foo" subCmds := []*cobra.Command{ - NewCmdKubeletUploadConfig(&kubeConfigFile), - NewCmdKubeletWriteConfigToDisk(&kubeConfigFile), - NewCmdKubeletEnableDynamicConfig(&kubeConfigFile), + NewCmdKubeletWriteEnvFile(), + NewCmdKubeletConfigUpload(), + NewCmdKubeletConfigDownload(), + NewCmdKubeletConfigWriteToDisk(), + NewCmdKubeletConfigEnableDynamic(), } commonFlags := []string{} @@ -39,21 +40,35 @@ func TestKubeletSubCommandsHasFlags(t *testing.T) { additionalFlags []string }{ { - command: "upload-config", + command: "write-env-file", additionalFlags: []string{ "config", }, }, { - command: "write-config-to-disk", + command: "upload", additionalFlags: []string{ + "kubeconfig", + "config", + }, + }, + { + command: "download", + additionalFlags: []string{ + "kubeconfig", "kubelet-version", + }, + }, + { + command: "write-to-disk", + additionalFlags: []string{ "config", }, }, { - command: "enable-dynamic-config", + command: "enable-dynamic", additionalFlags: []string{ + "kubeconfig", "node-name", "kubelet-version", }, diff --git a/cmd/kubeadm/app/cmd/upgrade/BUILD b/cmd/kubeadm/app/cmd/upgrade/BUILD index 4018c1c876b..254d16c0872 100644 --- a/cmd/kubeadm/app/cmd/upgrade/BUILD +++ b/cmd/kubeadm/app/cmd/upgrade/BUILD @@ -6,6 +6,7 @@ go_library( "apply.go", "common.go", "diff.go", + "node.go", "plan.go", "upgrade.go", ], @@ -20,6 +21,7 @@ go_library( "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/phases/controlplane:go_default_library", + "//cmd/kubeadm/app/phases/kubelet:go_default_library", "//cmd/kubeadm/app/phases/upgrade:go_default_library", "//cmd/kubeadm/app/preflight:go_default_library", "//cmd/kubeadm/app/util:go_default_library", @@ -28,6 +30,7 @@ go_library( "//cmd/kubeadm/app/util/dryrun:go_default_library", "//cmd/kubeadm/app/util/etcd:go_default_library", "//cmd/kubeadm/app/util/kubeconfig:go_default_library", + "//pkg/util/normalizer:go_default_library", "//pkg/util/version:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pmezard/go-difflib/difflib:go_default_library", diff --git a/cmd/kubeadm/app/cmd/upgrade/common.go b/cmd/kubeadm/app/cmd/upgrade/common.go index 1f5f2336ca8..23269ec9ddf 100644 --- a/cmd/kubeadm/app/cmd/upgrade/common.go +++ b/cmd/kubeadm/app/cmd/upgrade/common.go @@ -54,6 +54,12 @@ type upgradeVariables struct { // enforceRequirements verifies that it's okay to upgrade and then returns the variables needed for the rest of the procedure func enforceRequirements(flags *cmdUpgradeFlags, dryRun bool, newK8sVersion string) (*upgradeVariables, error) { + + // Set the default for the kubeconfig path if the user didn't override with the flags + if flags.kubeConfigPath == "" { + flags.kubeConfigPath = "/etc/kubernetes/admin.conf" + } + client, err := getClient(flags.kubeConfigPath, dryRun) if err != nil { return nil, fmt.Errorf("couldn't create a Kubernetes client from file %q: %v", flags.kubeConfigPath, err) @@ -154,7 +160,10 @@ func getClient(file string, dryRun bool) (clientset.Interface, error) { } // Get the fake clientset - fakeclient := apiclient.NewDryRunClient(dryRunGetter, os.Stdout) + dryRunOpts := apiclient.GetDefaultDryRunClientOptions(dryRunGetter, os.Stdout) + // Print GET and LIST requests + dryRunOpts.PrintGETAndLIST = true + fakeclient := apiclient.NewDryRunClientWithOpts(dryRunOpts) // As we know the return of Discovery() of the fake clientset is of type *fakediscovery.FakeDiscovery // we can convert it to that struct. fakeclientDiscovery, ok := fakeclient.Discovery().(*fakediscovery.FakeDiscovery) diff --git a/cmd/kubeadm/app/cmd/upgrade/node.go b/cmd/kubeadm/app/cmd/upgrade/node.go new file mode 100644 index 00000000000..28415640d73 --- /dev/null +++ b/cmd/kubeadm/app/cmd/upgrade/node.go @@ -0,0 +1,162 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "github.com/spf13/cobra" + + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" + kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" + "k8s.io/kubernetes/pkg/util/normalizer" + "k8s.io/kubernetes/pkg/util/version" +) + +var ( + upgradeNodeConfigLongDesc = normalizer.LongDesc(` + Downloads the kubelet configuration from a ConfigMap of the form "kubelet-config-1.X" in the cluster, + where X is the minor version of the kubelet. kubeadm uses the --kubelet-version parameter to determine + what the _desired_ kubelet version is. Give + `) + + upgradeNodeConfigExample = normalizer.Examples(` + # Downloads the kubelet configuration from the ConfigMap in the cluster. Uses a specific desired kubelet version. + kubeadm upgrade node config --kubelet-version v1.11.0 + + # Simulates the downloading of the kubelet configuration from the ConfigMap in the cluster with a specific desired + # version. Does not change any state locally on the node. + kubeadm upgrade node config --kubelet-version v1.11.0 --dry-run + `) +) + +type nodeUpgradeFlags struct { + parent *cmdUpgradeFlags + kubeletVersionStr string + dryRun bool +} + +// NewCmdNode returns the cobra command for `kubeadm upgrade node` +func NewCmdNode(parentFlags *cmdUpgradeFlags) *cobra.Command { + cmd := &cobra.Command{ + Use: "node", + Short: "Upgrade commands for a node in the cluster. Currently only supports upgrading the configuration, not the kubelet itself.", + RunE: cmdutil.SubCmdRunE("node"), + } + cmd.AddCommand(NewCmdUpgradeNodeConfig(parentFlags)) + return cmd +} + +// NewCmdUpgradeNodeConfig returns the cobra.Command for downloading the new/upgrading the kubelet configuration from the kubelet-config-1.X +// ConfigMap in the cluster +func NewCmdUpgradeNodeConfig(parentFlags *cmdUpgradeFlags) *cobra.Command { + flags := &nodeUpgradeFlags{ + parent: parentFlags, + kubeletVersionStr: "", + } + + cmd := &cobra.Command{ + Use: "config", + Short: "Downloads the kubelet configuration from the cluster ConfigMap kubelet-config-1.X, where X is the minor version of the kubelet.", + Long: upgradeNodeConfigLongDesc, + Example: upgradeNodeConfigExample, + Run: func(cmd *cobra.Command, args []string) { + err := RunUpgradeNodeConfig(flags) + kubeadmutil.CheckErr(err) + }, + } + + // TODO: Unify the registration of common flags + cmd.Flags().BoolVar(&flags.dryRun, "dry-run", flags.dryRun, "Do not change any state, just output the actions that would be performed.") + cmd.Flags().StringVar(&flags.kubeletVersionStr, "kubelet-version", flags.kubeletVersionStr, "The *desired* version for the kubelet after the upgrade.") + return cmd +} + +// RunUpgradeNodeConfig is executed when `kubeadm upgrade node config` runs. +func RunUpgradeNodeConfig(flags *nodeUpgradeFlags) error { + if len(flags.kubeletVersionStr) == 0 { + return fmt.Errorf("The --kubelet-version argument is required") + } + + // Set up the kubelet directory to use. If dry-running, use a fake directory + kubeletDir, err := getKubeletDir(flags.dryRun) + if err != nil { + return err + } + + // Set the default for the kubeconfig path if the user didn't override with the flags + // TODO: Be smarter about this and be able to load multiple kubeconfig files in different orders of precedence + if flags.parent.kubeConfigPath == "" { + flags.parent.kubeConfigPath = constants.GetKubeletKubeConfigPath() + } + + client, err := getClient(flags.parent.kubeConfigPath, flags.dryRun) + if err != nil { + return fmt.Errorf("couldn't create a Kubernetes client from file %q: %v", flags.parent.kubeConfigPath, err) + } + + // Parse the desired kubelet version + kubeletVersion, err := version.ParseSemantic(flags.kubeletVersionStr) + if err != nil { + return err + } + // TODO: Checkpoint the current configuration first so that if something goes wrong it can be recovered + if err := kubeletphase.DownloadConfig(client, kubeletVersion, kubeletDir); err != nil { + return err + } + + // If we're dry-running, print the generated manifests, otherwise do nothing + if err := printFilesIfDryRunning(flags.dryRun, kubeletDir); err != nil { + return fmt.Errorf("error printing files on dryrun: %v", err) + } + + fmt.Println("[upgrade] The configuration for this node was successfully updated!") + fmt.Println("[upgrade] Now you should go ahead and upgrade the kubelet package using your package manager.") + return nil +} + +// getKubeletDir gets the kubelet directory based on whether the user is dry-running this command or not. +func getKubeletDir(dryRun bool) (string, error) { + if dryRun { + dryRunDir, err := ioutil.TempDir("", "kubeadm-init-dryrun") + if err != nil { + return "", fmt.Errorf("couldn't create a temporary directory: %v", err) + } + return dryRunDir, nil + } + return constants.KubeletRunDirectory, nil +} + +// printFilesIfDryRunning prints the Static Pod manifests to stdout and informs about the temporary directory to go and lookup +func printFilesIfDryRunning(dryRun bool, kubeletDir string) error { + if !dryRun { + return nil + } + + // Print the contents of the upgraded file and pretend like they were in kubeadmconstants.KubeletRunDirectory + fileToPrint := dryrunutil.FileToPrint{ + RealPath: filepath.Join(kubeletDir, constants.KubeletConfigurationFileName), + PrintPath: filepath.Join(constants.KubeletRunDirectory, constants.KubeletConfigurationFileName), + } + return dryrunutil.PrintDryRunFiles([]dryrunutil.FileToPrint{fileToPrint}, os.Stdout) +} diff --git a/cmd/kubeadm/app/cmd/upgrade/upgrade.go b/cmd/kubeadm/app/cmd/upgrade/upgrade.go index a7928aa01bc..ad2c59b0c2d 100644 --- a/cmd/kubeadm/app/cmd/upgrade/upgrade.go +++ b/cmd/kubeadm/app/cmd/upgrade/upgrade.go @@ -74,6 +74,7 @@ func NewCmdUpgrade(out io.Writer) *cobra.Command { cmd.AddCommand(NewCmdApply(flags)) cmd.AddCommand(NewCmdPlan(flags)) cmd.AddCommand(NewCmdDiff(flags)) + cmd.AddCommand(NewCmdNode(flags)) return cmd } diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index 4b5f8da9731..5ded24e0701 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -205,8 +205,26 @@ const ( // KubeletBaseConfigMapRolePrefix defines the base kubelet configuration ConfigMap. KubeletBaseConfigMapRolePrefix = "kubeadm:kubelet-config-" - // KubeletConfigurationFile specifies the file name on the node which stores initial remote configuration of kubelet - KubeletConfigurationFile = "/var/lib/kubelet/config.yaml" + // KubeletRunDirectory specifies the directory where the kubelet runtime information is stored. + // TODO: Make hard-coded "/var/lib/kubelet" strings reference this constant. + KubeletRunDirectory = "/var/lib/kubelet" + + // KubeletConfigurationFileName specifies the file name on the node which stores initial remote configuration of kubelet + // This file should exist under KubeletRunDirectory + KubeletConfigurationFileName = "config.yaml" + + // DynamicKubeletConfigurationDirectoryName specifies the directory which stores the dynamic configuration checkpoints for the kubelet + // This directory should exist under KubeletRunDirectory + DynamicKubeletConfigurationDirectoryName = "dynamic-config" + + // KubeletEnvFileName is a file "kubeadm init" writes at runtime. Using that interface, kubeadm can customize certain + // kubelet flags conditionally based on the environment at runtime. Also, parameters given to the configuration file + // might be passed through this file. "kubeadm init" writes one variable, with the name ${KubeletEnvFileVariableName}. + // This file should exist under KubeletRunDirectory + KubeletEnvFileName = "kubeadm-flags.env" + + // KubeletEnvFileVariableName specifies the shell script variable name "kubeadm init" should write a value to in KubeletEnvFile + KubeletEnvFileVariableName = "KUBELET_KUBEADM_ARGS" // MinExternalEtcdVersion indicates minimum external etcd version which kubeadm supports MinExternalEtcdVersion = "3.2.17" @@ -266,14 +284,6 @@ const ( // TODO: Import this constant from a consts only package, that does not pull any further dependencies. LeaseEndpointReconcilerType = "lease" - // KubeletEnvFile is a file "kubeadm init" writes at runtime. Using that interface, kubeadm can customize certain - // kubelet flags conditionally based on the environment at runtime. Also, parameters given to the configuration file - // might be passed through this file. "kubeadm init" writes one variable, with the name ${KubeletEnvFileVariableName}. - KubeletEnvFile = "/var/lib/kubelet/kubeadm-flags.env" - - // KubeletEnvFileVariableName specifies the shell script variable name "kubeadm init" should write a value to in KubeletEnvFile - KubeletEnvFileVariableName = "KUBELET_KUBEADM_ARGS" - // KubeDNSVersion is the version of kube-dns to be deployed if it is used KubeDNSVersion = "1.14.10" @@ -350,6 +360,16 @@ func GetAdminKubeConfigPath() string { return filepath.Join(KubernetesDir, AdminKubeConfigFileName) } +// GetBootstrapKubeletKubeConfigPath returns the location on the disk where bootstrap kubelet kubeconfig is located by default +func GetBootstrapKubeletKubeConfigPath() string { + return filepath.Join(KubernetesDir, KubeletBootstrapKubeConfigFileName) +} + +// GetKubeletKubeConfigPath returns the location on the disk where kubelet kubeconfig is located by default +func GetKubeletKubeConfigPath() string { + return filepath.Join(KubernetesDir, KubeletKubeConfigFileName) +} + // AddSelfHostedPrefix adds the self-hosted- prefix to the component name func AddSelfHostedPrefix(componentName string) string { return fmt.Sprintf("%s%s", SelfHostingPrefix, componentName) diff --git a/cmd/kubeadm/app/constants/constants_test.go b/cmd/kubeadm/app/constants/constants_test.go index b65475ef0ee..7bcec426224 100644 --- a/cmd/kubeadm/app/constants/constants_test.go +++ b/cmd/kubeadm/app/constants/constants_test.go @@ -50,6 +50,32 @@ func TestGetAdminKubeConfigPath(t *testing.T) { } } +func TestGetBootstrapKubeletKubeConfigPath(t *testing.T) { + expected := "/etc/kubernetes/bootstrap-kubelet.conf" + actual := GetBootstrapKubeletKubeConfigPath() + + if actual != expected { + t.Errorf( + "failed GetBootstrapKubeletKubeConfigPath:\n\texpected: %s\n\t actual: %s", + expected, + actual, + ) + } +} + +func TestGetKubeletKubeConfigPath(t *testing.T) { + expected := "/etc/kubernetes/kubelet.conf" + actual := GetKubeletKubeConfigPath() + + if actual != expected { + t.Errorf( + "failed GetKubeletKubeConfigPath:\n\texpected: %s\n\t actual: %s", + expected, + actual, + ) + } +} + func TestGetStaticPodFilepath(t *testing.T) { var tests = []struct { componentName, manifestsDir, expected string diff --git a/cmd/kubeadm/app/phases/kubelet/BUILD b/cmd/kubeadm/app/phases/kubelet/BUILD index 3d2166d65b3..171771d880c 100644 --- a/cmd/kubeadm/app/phases/kubelet/BUILD +++ b/cmd/kubeadm/app/phases/kubelet/BUILD @@ -13,9 +13,9 @@ go_library( "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/apis/kubeadm/v1alpha2:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", + "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/util:go_default_library", "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//cmd/kubeadm/app/util/kubeconfig:go_default_library", "//pkg/apis/rbac/v1:go_default_library", "//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library", "//pkg/kubelet/apis/kubeletconfig/v1beta1:go_default_library", @@ -44,13 +44,13 @@ go_test( "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//pkg/kubelet/apis:go_default_library", "//pkg/kubelet/apis/kubeletconfig/v1beta1:go_default_library", - "//pkg/util/node:go_default_library", "//pkg/util/version:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/client-go/kubernetes/fake:go_default_library", "//vendor/k8s.io/client-go/testing:go_default_library", + "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/cmd/kubeadm/app/phases/kubelet/config.go b/cmd/kubeadm/app/phases/kubelet/config.go index e515d649860..723f92a69db 100644 --- a/cmd/kubeadm/app/phases/kubelet/config.go +++ b/cmd/kubeadm/app/phases/kubelet/config.go @@ -19,6 +19,7 @@ package kubelet import ( "fmt" "io/ioutil" + "path/filepath" "k8s.io/api/core/v1" rbac "k8s.io/api/rbac/v1" @@ -28,7 +29,6 @@ import ( kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1" kubeletconfigscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme" kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1" @@ -37,13 +37,13 @@ import ( // WriteConfigToDisk writes the kubelet config object down to a file // Used at "kubeadm init" and "kubeadm upgrade" time -func WriteConfigToDisk(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration) error { +func WriteConfigToDisk(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration, kubeletDir string) error { kubeletBytes, err := getConfigBytes(kubeletConfig) if err != nil { return err } - return writeConfigBytesToDisk(kubeletBytes) + return writeConfigBytesToDisk(kubeletBytes, kubeletDir) } // CreateConfigMap creates a ConfigMap with the generic kubelet configuration. @@ -120,7 +120,7 @@ func createConfigMapRBACRules(client clientset.Interface, k8sVersion *version.Ve // DownloadConfig downloads the kubelet configuration from a ConfigMap and writes it to disk. // Used at "kubeadm join" time -func DownloadConfig(kubeletKubeConfig string, kubeletVersion *version.Version) error { +func DownloadConfig(client clientset.Interface, kubeletVersion *version.Version, kubeletDir string) error { // Download the ConfigMap from the cluster based on what version the kubelet is configMapName := configMapName(kubeletVersion) @@ -128,17 +128,12 @@ func DownloadConfig(kubeletKubeConfig string, kubeletVersion *version.Version) e fmt.Printf("[kubelet] Downloading configuration for the kubelet from the %q ConfigMap in the %s namespace\n", configMapName, metav1.NamespaceSystem) - client, err := kubeconfigutil.ClientSetFromFile(kubeletKubeConfig) - if err != nil { - return fmt.Errorf("couldn't create client from kubeconfig file %q", kubeletKubeConfig) - } - kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(configMapName, metav1.GetOptions{}) if err != nil { return err } - return writeConfigBytesToDisk([]byte(kubeletCfg.Data[kubeadmconstants.KubeletBaseConfigurationConfigMapKey])) + return writeConfigBytesToDisk([]byte(kubeletCfg.Data[kubeadmconstants.KubeletBaseConfigurationConfigMapKey]), kubeletDir) } // configMapName returns the right ConfigMap name for the right branch of k8s @@ -162,11 +157,12 @@ func getConfigBytes(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration) ([ } // writeConfigBytesToDisk writes a byte slice down to disk at the specific location of the kubelet config file -func writeConfigBytesToDisk(b []byte) error { - fmt.Printf("[kubelet] Writing kubelet configuration to file %q\n", kubeadmconstants.KubeletConfigurationFile) +func writeConfigBytesToDisk(b []byte, kubeletDir string) error { + configFile := filepath.Join(kubeletDir, kubeadmconstants.KubeletConfigurationFileName) + fmt.Printf("[kubelet] Writing kubelet configuration to file %q\n", configFile) - if err := ioutil.WriteFile(kubeadmconstants.KubeletConfigurationFile, b, 0644); err != nil { - return fmt.Errorf("failed to write kubelet configuration to the file %q: %v", kubeadmconstants.KubeletConfigurationFile, err) + if err := ioutil.WriteFile(configFile, b, 0644); err != nil { + return fmt.Errorf("failed to write kubelet configuration to the file %q: %v", configFile, err) } return nil } diff --git a/cmd/kubeadm/app/phases/kubelet/flags.go b/cmd/kubeadm/app/phases/kubelet/flags.go index 91f876e0c47..51277b81e73 100644 --- a/cmd/kubeadm/app/phases/kubelet/flags.go +++ b/cmd/kubeadm/app/phases/kubelet/flags.go @@ -27,34 +27,52 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2" "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/features" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" nodeutil "k8s.io/kubernetes/pkg/util/node" "k8s.io/kubernetes/pkg/util/procfs" utilsexec "k8s.io/utils/exec" ) +type kubeletFlagsOpts struct { + nodeRegOpts *kubeadmapi.NodeRegistrationOptions + featureGates map[string]bool + registerTaintsUsingFlags bool + execer utilsexec.Interface + pidOfFunc func(string) ([]int, error) + defaultHostname string +} + // WriteKubeletDynamicEnvFile writes a environment file with dynamic flags to the kubelet. // Used at "kubeadm init" and "kubeadm join" time. -func WriteKubeletDynamicEnvFile(nodeRegOpts *kubeadmapi.NodeRegistrationOptions, registerTaintsUsingFlags bool) error { +func WriteKubeletDynamicEnvFile(nodeRegOpts *kubeadmapi.NodeRegistrationOptions, featureGates map[string]bool, registerTaintsUsingFlags bool, kubeletDir string) error { - argList := kubeadmutil.BuildArgumentListFromMap(buildKubeletArgMap(nodeRegOpts, registerTaintsUsingFlags), nodeRegOpts.ExtraArgs) + flagOpts := kubeletFlagsOpts{ + nodeRegOpts: nodeRegOpts, + featureGates: featureGates, + registerTaintsUsingFlags: registerTaintsUsingFlags, + execer: utilsexec.New(), + pidOfFunc: procfs.PidOf, + defaultHostname: nodeutil.GetHostname(""), + } + stringMap := buildKubeletArgMap(flagOpts) + argList := kubeadmutil.BuildArgumentListFromMap(stringMap, nodeRegOpts.KubeletExtraArgs) envFileContent := fmt.Sprintf("%s=%s\n", constants.KubeletEnvFileVariableName, strings.Join(argList, " ")) - return writeKubeletFlagBytesToDisk([]byte(envFileContent)) + return writeKubeletFlagBytesToDisk([]byte(envFileContent), kubeletDir) } // buildKubeletArgMap takes a MasterConfiguration object and builds based on that a string-string map with flags // that should be given to the local kubelet daemon. -func buildKubeletArgMap(nodeRegOpts *kubeadmapi.NodeRegistrationOptions, registerTaintsUsingFlags bool) map[string]string { +func buildKubeletArgMap(opts kubeletFlagsOpts) map[string]string { kubeletFlags := map[string]string{} - if nodeRegOpts.CRISocket == kubeadmapiv1alpha2.DefaultCRISocket { + if opts.nodeRegOpts.CRISocket == kubeadmapiv1alpha2.DefaultCRISocket { // These flags should only be set when running docker kubeletFlags["network-plugin"] = "cni" kubeletFlags["cni-conf-dir"] = "/etc/cni/net.d" kubeletFlags["cni-bin-dir"] = "/opt/cni/bin" - execer := utilsexec.New() - driver, err := kubeadmutil.GetCgroupDriverDocker(execer) + driver, err := kubeadmutil.GetCgroupDriverDocker(opts.execer) if err != nil { glog.Warningf("cannot automatically assign a '--cgroup-driver' value when starting the Kubelet: %v\n", err) } else { @@ -62,27 +80,33 @@ func buildKubeletArgMap(nodeRegOpts *kubeadmapi.NodeRegistrationOptions, registe } } else { kubeletFlags["container-runtime"] = "remote" - kubeletFlags["container-runtime-endpoint"] = nodeRegOpts.CRISocket + kubeletFlags["container-runtime-endpoint"] = opts.nodeRegOpts.CRISocket } - if registerTaintsUsingFlags && nodeRegOpts.Taints != nil && len(nodeRegOpts.Taints) > 0 { + if opts.registerTaintsUsingFlags && opts.nodeRegOpts.Taints != nil && len(opts.nodeRegOpts.Taints) > 0 { taintStrs := []string{} - for _, taint := range nodeRegOpts.Taints { + for _, taint := range opts.nodeRegOpts.Taints { taintStrs = append(taintStrs, taint.ToString()) } kubeletFlags["register-with-taints"] = strings.Join(taintStrs, ",") } - if pids, _ := procfs.PidOf("systemd-resolved"); len(pids) > 0 { + if pids, _ := opts.pidOfFunc("systemd-resolved"); len(pids) > 0 { // procfs.PidOf only returns an error if the regex is empty or doesn't compile, so we can ignore it kubeletFlags["resolv-conf"] = "/run/systemd/resolve/resolv.conf" } // Make sure the node name we're passed will work with Kubelet - if nodeRegOpts.Name != "" && nodeRegOpts.Name != nodeutil.GetHostname("") { - glog.V(1).Info("setting kubelet hostname-override to %q", nodeRegOpts.Name) - kubeletFlags["hostname-override"] = nodeRegOpts.Name + if opts.nodeRegOpts.Name != "" && opts.nodeRegOpts.Name != opts.defaultHostname { + glog.V(1).Info("setting kubelet hostname-override to %q", opts.nodeRegOpts.Name) + kubeletFlags["hostname-override"] = opts.nodeRegOpts.Name + } + + // If the user enabled Dynamic Kubelet Configuration (which is disabled by default), set the directory + // in the CLI flags so that the feature actually gets enabled + if features.Enabled(opts.featureGates, features.DynamicKubeletConfig) { + kubeletFlags["dynamic-config-dir"] = filepath.Join(constants.KubeletRunDirectory, constants.DynamicKubeletConfigurationDirectoryName) } // TODO: Conditionally set `--cgroup-driver` to either `systemd` or `cgroupfs` for CRI other than Docker @@ -91,15 +115,16 @@ func buildKubeletArgMap(nodeRegOpts *kubeadmapi.NodeRegistrationOptions, registe } // writeKubeletFlagBytesToDisk writes a byte slice down to disk at the specific location of the kubelet flag overrides file -func writeKubeletFlagBytesToDisk(b []byte) error { - fmt.Printf("[kubelet] Writing kubelet environment file with flags to file %q\n", constants.KubeletEnvFile) +func writeKubeletFlagBytesToDisk(b []byte, kubeletDir string) error { + kubeletEnvFilePath := filepath.Join(kubeletDir, constants.KubeletEnvFileName) + fmt.Printf("[kubelet] Writing kubelet environment file with flags to file %q\n", kubeletEnvFilePath) // creates target folder if not already exists - if err := os.MkdirAll(filepath.Dir(constants.KubeletEnvFile), 0700); err != nil { - return fmt.Errorf("failed to create directory %q: %v", filepath.Dir(constants.KubeletEnvFile), err) + if err := os.MkdirAll(kubeletDir, 0700); err != nil { + return fmt.Errorf("failed to create directory %q: %v", kubeletDir, err) } - if err := ioutil.WriteFile(constants.KubeletEnvFile, b, 0644); err != nil { - return fmt.Errorf("failed to write kubelet configuration to the file %q: %v", constants.KubeletEnvFile, err) + if err := ioutil.WriteFile(kubeletEnvFilePath, b, 0644); err != nil { + return fmt.Errorf("failed to write kubelet configuration to the file %q: %v", kubeletEnvFilePath, err) } return nil } diff --git a/cmd/kubeadm/app/phases/kubelet/flags_test.go b/cmd/kubeadm/app/phases/kubelet/flags_test.go index 32646647d43..b50a658df53 100644 --- a/cmd/kubeadm/app/phases/kubelet/flags_test.go +++ b/cmd/kubeadm/app/phases/kubelet/flags_test.go @@ -17,45 +17,260 @@ limitations under the License. package kubelet import ( + "context" + "errors" + "fmt" + "io" + "reflect" + "strings" "testing" + "k8s.io/api/core/v1" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - nodeutil "k8s.io/kubernetes/pkg/util/node" + "k8s.io/utils/exec" ) +type fakeCmd struct { + b []byte + err error +} + +func (f fakeCmd) Run() error { return f.err } +func (f fakeCmd) CombinedOutput() ([]byte, error) { return f.b, f.err } +func (f fakeCmd) Output() ([]byte, error) { return f.b, f.err } +func (f fakeCmd) SetDir(dir string) {} +func (f fakeCmd) SetStdin(in io.Reader) {} +func (f fakeCmd) SetStdout(out io.Writer) {} +func (f fakeCmd) SetStderr(out io.Writer) {} +func (f fakeCmd) Stop() {} + +type fakeExecer struct { + ioMap map[string]fakeCmd +} + +func (f fakeExecer) Command(cmd string, args ...string) exec.Cmd { + cmds := []string{cmd} + cmds = append(cmds, args...) + return f.ioMap[strings.Join(cmds, " ")] +} +func (f fakeExecer) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd { + return f.Command(cmd, args...) +} +func (f fakeExecer) LookPath(file string) (string, error) { return "", errors.New("unknown binary") } + +var ( + systemdCgroupExecer = fakeExecer{ + ioMap: map[string]fakeCmd{ + "docker info": { + b: []byte(`Cgroup Driver: systemd`), + }, + }, + } + + cgroupfsCgroupExecer = fakeExecer{ + ioMap: map[string]fakeCmd{ + "docker info": { + b: []byte(`Cgroup Driver: cgroupfs`), + }, + }, + } + + errCgroupExecer = fakeExecer{ + ioMap: map[string]fakeCmd{ + "docker info": { + err: fmt.Errorf("no such binary: docker"), + }, + }, + } +) + +func binaryRunningPidOfFunc(_ string) ([]int, error) { + return []int{1, 2, 3}, nil +} + +func binaryNotRunningPidOfFunc(_ string) ([]int, error) { + return []int{}, nil +} + func TestBuildKubeletArgMap(t *testing.T) { tests := []struct { - name string - hostname string - expectedHostname string + name string + opts kubeletFlagsOpts + expected map[string]string }{ { - name: "manually set to current hostname", - hostname: nodeutil.GetHostname(""), - expectedHostname: "", + name: "the simplest case", + opts: kubeletFlagsOpts{ + nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ + CRISocket: "/var/run/dockershim.sock", + Name: "foo", + Taints: []v1.Taint{ // This should be ignored as registerTaintsUsingFlags is false + { + Key: "foo", + Value: "bar", + Effect: "baz", + }, + }, + }, + execer: errCgroupExecer, + pidOfFunc: binaryNotRunningPidOfFunc, + defaultHostname: "foo", + }, + expected: map[string]string{ + "network-plugin": "cni", + "cni-conf-dir": "/etc/cni/net.d", + "cni-bin-dir": "/opt/cni/bin", + }, }, { - name: "unset hostname", - hostname: "", - expectedHostname: "", + name: "nodeRegOpts.Name != default hostname", + opts: kubeletFlagsOpts{ + nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ + CRISocket: "/var/run/dockershim.sock", + Name: "override-name", + }, + execer: errCgroupExecer, + pidOfFunc: binaryNotRunningPidOfFunc, + defaultHostname: "default", + }, + expected: map[string]string{ + "network-plugin": "cni", + "cni-conf-dir": "/etc/cni/net.d", + "cni-bin-dir": "/opt/cni/bin", + "hostname-override": "override-name", + }, }, { - name: "override hostname", - hostname: "my-node", - expectedHostname: "my-node", + name: "systemd cgroup driver", + opts: kubeletFlagsOpts{ + nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ + CRISocket: "/var/run/dockershim.sock", + Name: "foo", + }, + execer: systemdCgroupExecer, + pidOfFunc: binaryNotRunningPidOfFunc, + defaultHostname: "foo", + }, + expected: map[string]string{ + "network-plugin": "cni", + "cni-conf-dir": "/etc/cni/net.d", + "cni-bin-dir": "/opt/cni/bin", + "cgroup-driver": "systemd", + }, + }, + { + name: "cgroupfs cgroup driver", + opts: kubeletFlagsOpts{ + nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ + CRISocket: "/var/run/dockershim.sock", + Name: "foo", + }, + execer: cgroupfsCgroupExecer, + pidOfFunc: binaryNotRunningPidOfFunc, + defaultHostname: "foo", + }, + expected: map[string]string{ + "network-plugin": "cni", + "cni-conf-dir": "/etc/cni/net.d", + "cni-bin-dir": "/opt/cni/bin", + "cgroup-driver": "cgroupfs", + }, + }, + { + name: "external CRI runtime", + opts: kubeletFlagsOpts{ + nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ + CRISocket: "/var/run/containerd.sock", + Name: "foo", + }, + execer: cgroupfsCgroupExecer, + pidOfFunc: binaryNotRunningPidOfFunc, + defaultHostname: "foo", + }, + expected: map[string]string{ + "container-runtime": "remote", + "container-runtime-endpoint": "/var/run/containerd.sock", + }, + }, + { + name: "register with taints", + opts: kubeletFlagsOpts{ + nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ + CRISocket: "/var/run/containerd.sock", + Name: "foo", + Taints: []v1.Taint{ + { + Key: "foo", + Value: "bar", + Effect: "baz", + }, + { + Key: "key", + Value: "val", + Effect: "eff", + }, + }, + }, + registerTaintsUsingFlags: true, + execer: cgroupfsCgroupExecer, + pidOfFunc: binaryNotRunningPidOfFunc, + defaultHostname: "foo", + }, + expected: map[string]string{ + "container-runtime": "remote", + "container-runtime-endpoint": "/var/run/containerd.sock", + "register-with-taints": "foo=bar:baz,key=val:eff", + }, + }, + { + name: "systemd-resolved running", + opts: kubeletFlagsOpts{ + nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ + CRISocket: "/var/run/containerd.sock", + Name: "foo", + }, + execer: cgroupfsCgroupExecer, + pidOfFunc: binaryRunningPidOfFunc, + defaultHostname: "foo", + }, + expected: map[string]string{ + "container-runtime": "remote", + "container-runtime-endpoint": "/var/run/containerd.sock", + "resolv-conf": "/run/systemd/resolve/resolv.conf", + }, + }, + { + name: "dynamic kubelet config enabled", + opts: kubeletFlagsOpts{ + nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ + CRISocket: "/var/run/containerd.sock", + Name: "foo", + }, + featureGates: map[string]bool{ + "DynamicKubeletConfig": true, + }, + execer: cgroupfsCgroupExecer, + pidOfFunc: binaryNotRunningPidOfFunc, + defaultHostname: "foo", + }, + expected: map[string]string{ + "container-runtime": "remote", + "container-runtime-endpoint": "/var/run/containerd.sock", + "dynamic-config-dir": "/var/lib/kubelet/dynamic-config", + }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - opts := &kubeadmapi.NodeRegistrationOptions{ - Name: test.hostname, - } - - m := buildKubeletArgMap(opts, false) - if m["hostname-override"] != test.expectedHostname { - t.Errorf("expected hostname %q, got %q", test.expectedHostname, m["hostname-override"]) + actual := buildKubeletArgMap(test.opts) + if !reflect.DeepEqual(actual, test.expected) { + t.Errorf( + "failed buildKubeletArgMap:\n\texpected: %v\n\t actual: %v", + test.expected, + actual, + ) } }) } diff --git a/cmd/kubeadm/app/phases/markmaster/markmaster.go b/cmd/kubeadm/app/phases/markmaster/markmaster.go index bb1ff4210e0..c9d36989fec 100644 --- a/cmd/kubeadm/app/phases/markmaster/markmaster.go +++ b/cmd/kubeadm/app/phases/markmaster/markmaster.go @@ -31,7 +31,11 @@ func MarkMaster(client clientset.Interface, masterName string, taints []v1.Taint fmt.Printf("[markmaster] Marking the node %s as master by adding the label \"%s=''\"\n", masterName, constants.LabelNodeRoleMaster) if taints != nil && len(taints) > 0 { - fmt.Printf("[markmaster] Marking the node %s as master by adding the taints %v\n", masterName, taints) + taintStrs := []string{} + for _, taint := range taints { + taintStrs = append(taintStrs, taint.ToString()) + } + fmt.Printf("[markmaster] Marking the node %s as master by adding the taints %v\n", masterName, taintStrs) } return apiclient.PatchNode(client, masterName, func(n *v1.Node) { diff --git a/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go b/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go index c1c33c2a45d..7445b436167 100644 --- a/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go +++ b/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go @@ -42,6 +42,8 @@ func UploadConfiguration(cfg *kubeadmapi.MasterConfiguration, client clientset.I // Removes sensitive info from the data that will be stored in the config map externalcfg.BootstrapTokens = nil + // Clear the NodeRegistration object. + externalcfg.NodeRegistration = kubeadmapiv1alpha2.NodeRegistrationOptions{} cfgYaml, err := util.MarshalToYamlForCodecs(externalcfg, kubeadmapiv1alpha2.SchemeGroupVersion, scheme.Codecs) if err != nil { diff --git a/cmd/kubeadm/app/preflight/checks.go b/cmd/kubeadm/app/preflight/checks.go index 18c0025f638..6b26e73b673 100644 --- a/cmd/kubeadm/app/preflight/checks.go +++ b/cmd/kubeadm/app/preflight/checks.go @@ -1063,15 +1063,41 @@ func TryStartKubelet(ignorePreflightErrors sets.String) { // If we notice that the kubelet service is inactive, try to start it initSystem, err := initsystem.GetInitSystem() if err != nil { - fmt.Println("[preflight] no supported init system detected, won't ensure kubelet is running.") - } else if initSystem.ServiceExists("kubelet") { + fmt.Println("[preflight] no supported init system detected, won't make sure the kubelet is running properly.") + return + } - fmt.Println("[preflight] Activating the kubelet service") - // This runs "systemctl daemon-reload && systemctl restart kubelet" - if err := initSystem.ServiceRestart("kubelet"); err != nil { - glog.Warningf("[preflight] unable to start the kubelet service: [%v]\n", err) - glog.Warningf("[preflight] please ensure kubelet is running manually.") - } + if !initSystem.ServiceExists("kubelet") { + fmt.Println("[preflight] couldn't detect a kubelet service, can't make sure the kubelet is running properly.") + } + + fmt.Println("[preflight] Activating the kubelet service") + // This runs "systemctl daemon-reload && systemctl restart kubelet" + if err := initSystem.ServiceRestart("kubelet"); err != nil { + fmt.Printf("[preflight] WARNING: unable to start the kubelet service: [%v]\n", err) + fmt.Printf("[preflight] please ensure kubelet is reloaded and running manually.\n") + } +} + +// TryStopKubelet attempts to bring down the kubelet service momentarily +func TryStopKubelet(ignorePreflightErrors sets.String) { + if setHasItemOrAll(ignorePreflightErrors, "StopKubelet") { + return + } + // If we notice that the kubelet service is inactive, try to start it + initSystem, err := initsystem.GetInitSystem() + if err != nil { + fmt.Println("[preflight] no supported init system detected, won't make sure the kubelet not running for a short period of time while setting up configuration for it.") + return + } + + if !initSystem.ServiceExists("kubelet") { + fmt.Println("[preflight] couldn't detect a kubelet service, can't make sure the kubelet not running for a short period of time while setting up configuration for it.") + } + + // This runs "systemctl daemon-reload && systemctl stop kubelet" + if err := initSystem.ServiceStop("kubelet"); err != nil { + fmt.Printf("[preflight] WARNING: unable to stop the kubelet service momentarily: [%v]\n", err) } } diff --git a/cmd/kubeadm/app/util/apiclient/dryrunclient.go b/cmd/kubeadm/app/util/apiclient/dryrunclient.go index 6c0d796e6fa..4d4a52d706a 100644 --- a/cmd/kubeadm/app/util/apiclient/dryrunclient.go +++ b/cmd/kubeadm/app/util/apiclient/dryrunclient.go @@ -55,6 +55,18 @@ type DryRunClientOptions struct { PrintGETAndLIST bool } +// GetDefaultDryRunClientOptions returns the default DryRunClientOptions values +func GetDefaultDryRunClientOptions(drg DryRunGetter, w io.Writer) DryRunClientOptions { + return DryRunClientOptions{ + Writer: w, + Getter: drg, + PrependReactors: []core.Reactor{}, + AppendReactors: []core.Reactor{}, + MarshalFunc: DefaultMarshalFunc, + PrintGETAndLIST: false, + } +} + // actionWithName is the generic interface for an action that has a name associated with it // This just makes it easier to catch all actions that has a name; instead of hard-coding all request that has it associated type actionWithName interface { @@ -71,14 +83,7 @@ type actionWithObject interface { // NewDryRunClient is a wrapper for NewDryRunClientWithOpts using some default values func NewDryRunClient(drg DryRunGetter, w io.Writer) clientset.Interface { - return NewDryRunClientWithOpts(DryRunClientOptions{ - Writer: w, - Getter: drg, - PrependReactors: []core.Reactor{}, - AppendReactors: []core.Reactor{}, - MarshalFunc: DefaultMarshalFunc, - PrintGETAndLIST: false, - }) + return NewDryRunClientWithOpts(GetDefaultDryRunClientOptions(drg, w)) } // NewDryRunClientWithOpts returns a clientset.Interface that can be used normally for talking to the Kubernetes API. diff --git a/cmd/kubeadm/app/util/apiclient/init_dryrun.go b/cmd/kubeadm/app/util/apiclient/init_dryrun.go index b21ec7810d0..875ff3f6aee 100644 --- a/cmd/kubeadm/app/util/apiclient/init_dryrun.go +++ b/cmd/kubeadm/app/util/apiclient/init_dryrun.go @@ -61,6 +61,7 @@ func (idr *InitDryRunGetter) HandleGetAction(action core.GetAction) (bool, runti idr.handleGetNode, idr.handleSystemNodesClusterRoleBinding, idr.handleGetBootstrapToken, + idr.handleGetKubeDNSConfigMap, } for _, f := range funcs { handled, obj, err := f(action) @@ -157,3 +158,20 @@ func (idr *InitDryRunGetter) handleGetBootstrapToken(action core.GetAction) (boo // We can safely return a NotFound error here as the code will just proceed normally and create the Bootstrap Token return true, nil, apierrors.NewNotFound(action.GetResource().GroupResource(), "secret not found") } + +// handleGetKubeDNSConfigMap handles the case where kubeadm init will try to read the kube-dns ConfigMap in the cluster +// in order to transform information there to core-dns configuration. We can safely return an empty configmap here +func (idr *InitDryRunGetter) handleGetKubeDNSConfigMap(action core.GetAction) (bool, runtime.Object, error) { + if !strings.HasPrefix(action.GetName(), "kube-dns") || action.GetNamespace() != metav1.NamespaceSystem || action.GetResource().Resource != "configmaps" { + // We can't handle this event + return false, nil, nil + } + // We can safely return an empty configmap here, as we don't have any kube-dns specific config to convert to coredns config + return true, &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-dns", + Namespace: metav1.NamespaceSystem, + }, + Data: map[string]string{}, + }, nil +} diff --git a/cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml b/cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml index dc8f00274b0..9996fa33f24 100644 --- a/cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml +++ b/cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml @@ -151,7 +151,7 @@ Networking: ServiceSubnet: 10.96.0.0/12 NodeRegistration: CRISocket: /var/run/dockershim.sock - ExtraArgs: null + KubeletExtraArgs: null Name: master-1 Taints: - effect: NoSchedule diff --git a/cmd/kubeadm/app/util/config/testdata/conversion/node/internal.yaml b/cmd/kubeadm/app/util/config/testdata/conversion/node/internal.yaml index 517a1def2ca..cd820682b78 100644 --- a/cmd/kubeadm/app/util/config/testdata/conversion/node/internal.yaml +++ b/cmd/kubeadm/app/util/config/testdata/conversion/node/internal.yaml @@ -10,7 +10,7 @@ DiscoveryTokenUnsafeSkipCAVerification: true FeatureGates: null NodeRegistration: CRISocket: /var/run/dockershim.sock - ExtraArgs: null + KubeletExtraArgs: null Name: master-1 Taints: null TLSBootstrapToken: abcdef.0123456789abcdef diff --git a/docs/.generated_docs b/docs/.generated_docs index f6b821fb3fb..0fd748bcfc2 100644 --- a/docs/.generated_docs +++ b/docs/.generated_docs @@ -46,9 +46,12 @@ docs/admin/kubeadm_alpha_phase_kubeconfig_kubelet.md docs/admin/kubeadm_alpha_phase_kubeconfig_scheduler.md docs/admin/kubeadm_alpha_phase_kubeconfig_user.md docs/admin/kubeadm_alpha_phase_kubelet.md -docs/admin/kubeadm_alpha_phase_kubelet_enable-dynamic-config.md -docs/admin/kubeadm_alpha_phase_kubelet_upload-config.md -docs/admin/kubeadm_alpha_phase_kubelet_write-config-to-disk.md +docs/admin/kubeadm_alpha_phase_kubelet_config.md +docs/admin/kubeadm_alpha_phase_kubelet_config_download.md +docs/admin/kubeadm_alpha_phase_kubelet_config_enable-dynamic.md +docs/admin/kubeadm_alpha_phase_kubelet_config_upload.md +docs/admin/kubeadm_alpha_phase_kubelet_config_write-to-disk.md +docs/admin/kubeadm_alpha_phase_kubelet_write-env-file.md docs/admin/kubeadm_alpha_phase_mark-master.md docs/admin/kubeadm_alpha_phase_preflight.md docs/admin/kubeadm_alpha_phase_preflight_master.md @@ -78,6 +81,8 @@ docs/admin/kubeadm_token_list.md docs/admin/kubeadm_upgrade.md docs/admin/kubeadm_upgrade_apply.md docs/admin/kubeadm_upgrade_diff.md +docs/admin/kubeadm_upgrade_node.md +docs/admin/kubeadm_upgrade_node_config.md docs/admin/kubeadm_upgrade_plan.md docs/admin/kubeadm_version.md docs/admin/kubelet.md @@ -124,9 +129,12 @@ docs/man/man1/kubeadm-alpha-phase-kubeconfig-kubelet.1 docs/man/man1/kubeadm-alpha-phase-kubeconfig-scheduler.1 docs/man/man1/kubeadm-alpha-phase-kubeconfig-user.1 docs/man/man1/kubeadm-alpha-phase-kubeconfig.1 -docs/man/man1/kubeadm-alpha-phase-kubelet-enable-dynamic-config.1 -docs/man/man1/kubeadm-alpha-phase-kubelet-upload-config.1 -docs/man/man1/kubeadm-alpha-phase-kubelet-write-config-to-disk.1 +docs/man/man1/kubeadm-alpha-phase-kubelet-config-download.1 +docs/man/man1/kubeadm-alpha-phase-kubelet-config-enable-dynamic.1 +docs/man/man1/kubeadm-alpha-phase-kubelet-config-upload.1 +docs/man/man1/kubeadm-alpha-phase-kubelet-config-write-to-disk.1 +docs/man/man1/kubeadm-alpha-phase-kubelet-config.1 +docs/man/man1/kubeadm-alpha-phase-kubelet-write-env-file.1 docs/man/man1/kubeadm-alpha-phase-kubelet.1 docs/man/man1/kubeadm-alpha-phase-mark-master.1 docs/man/man1/kubeadm-alpha-phase-preflight-master.1 @@ -158,6 +166,8 @@ docs/man/man1/kubeadm-token-list.1 docs/man/man1/kubeadm-token.1 docs/man/man1/kubeadm-upgrade-apply.1 docs/man/man1/kubeadm-upgrade-diff.1 +docs/man/man1/kubeadm-upgrade-node-config.1 +docs/man/man1/kubeadm-upgrade-node.1 docs/man/man1/kubeadm-upgrade-plan.1 docs/man/man1/kubeadm-upgrade.1 docs/man/man1/kubeadm-version.1 diff --git a/docs/admin/kubeadm_alpha_phase_kubelet_enable-dynamic-config.md b/docs/admin/kubeadm_alpha_phase_kubelet_config.md similarity index 100% rename from docs/admin/kubeadm_alpha_phase_kubelet_enable-dynamic-config.md rename to docs/admin/kubeadm_alpha_phase_kubelet_config.md diff --git a/docs/admin/kubeadm_alpha_phase_kubelet_upload-config.md b/docs/admin/kubeadm_alpha_phase_kubelet_config_download.md similarity index 100% rename from docs/admin/kubeadm_alpha_phase_kubelet_upload-config.md rename to docs/admin/kubeadm_alpha_phase_kubelet_config_download.md diff --git a/docs/admin/kubeadm_alpha_phase_kubelet_write-config-to-disk.md b/docs/admin/kubeadm_alpha_phase_kubelet_config_enable-dynamic.md similarity index 100% rename from docs/admin/kubeadm_alpha_phase_kubelet_write-config-to-disk.md rename to docs/admin/kubeadm_alpha_phase_kubelet_config_enable-dynamic.md diff --git a/docs/man/man1/kubeadm-alpha-phase-kubelet-enable-dynamic-config.1 b/docs/admin/kubeadm_alpha_phase_kubelet_config_upload.md similarity index 100% rename from docs/man/man1/kubeadm-alpha-phase-kubelet-enable-dynamic-config.1 rename to docs/admin/kubeadm_alpha_phase_kubelet_config_upload.md diff --git a/docs/man/man1/kubeadm-alpha-phase-kubelet-upload-config.1 b/docs/admin/kubeadm_alpha_phase_kubelet_config_write-to-disk.md similarity index 100% rename from docs/man/man1/kubeadm-alpha-phase-kubelet-upload-config.1 rename to docs/admin/kubeadm_alpha_phase_kubelet_config_write-to-disk.md diff --git a/docs/man/man1/kubeadm-alpha-phase-kubelet-write-config-to-disk.1 b/docs/admin/kubeadm_alpha_phase_kubelet_write-env-file.md similarity index 100% rename from docs/man/man1/kubeadm-alpha-phase-kubelet-write-config-to-disk.1 rename to docs/admin/kubeadm_alpha_phase_kubelet_write-env-file.md diff --git a/docs/admin/kubeadm_upgrade_node.md b/docs/admin/kubeadm_upgrade_node.md new file mode 100644 index 00000000000..b6fd7a0f989 --- /dev/null +++ b/docs/admin/kubeadm_upgrade_node.md @@ -0,0 +1,3 @@ +This file is autogenerated, but we've stopped checking such files into the +repository to reduce the need for rebases. Please run hack/generate-docs.sh to +populate this file. diff --git a/docs/admin/kubeadm_upgrade_node_config.md b/docs/admin/kubeadm_upgrade_node_config.md new file mode 100644 index 00000000000..b6fd7a0f989 --- /dev/null +++ b/docs/admin/kubeadm_upgrade_node_config.md @@ -0,0 +1,3 @@ +This file is autogenerated, but we've stopped checking such files into the +repository to reduce the need for rebases. Please run hack/generate-docs.sh to +populate this file. diff --git a/docs/man/man1/kubeadm-alpha-phase-kubelet-config-download.1 b/docs/man/man1/kubeadm-alpha-phase-kubelet-config-download.1 new file mode 100644 index 00000000000..b6fd7a0f989 --- /dev/null +++ b/docs/man/man1/kubeadm-alpha-phase-kubelet-config-download.1 @@ -0,0 +1,3 @@ +This file is autogenerated, but we've stopped checking such files into the +repository to reduce the need for rebases. Please run hack/generate-docs.sh to +populate this file. diff --git a/docs/man/man1/kubeadm-alpha-phase-kubelet-config-enable-dynamic.1 b/docs/man/man1/kubeadm-alpha-phase-kubelet-config-enable-dynamic.1 new file mode 100644 index 00000000000..b6fd7a0f989 --- /dev/null +++ b/docs/man/man1/kubeadm-alpha-phase-kubelet-config-enable-dynamic.1 @@ -0,0 +1,3 @@ +This file is autogenerated, but we've stopped checking such files into the +repository to reduce the need for rebases. Please run hack/generate-docs.sh to +populate this file. diff --git a/docs/man/man1/kubeadm-alpha-phase-kubelet-config-upload.1 b/docs/man/man1/kubeadm-alpha-phase-kubelet-config-upload.1 new file mode 100644 index 00000000000..b6fd7a0f989 --- /dev/null +++ b/docs/man/man1/kubeadm-alpha-phase-kubelet-config-upload.1 @@ -0,0 +1,3 @@ +This file is autogenerated, but we've stopped checking such files into the +repository to reduce the need for rebases. Please run hack/generate-docs.sh to +populate this file. diff --git a/docs/man/man1/kubeadm-alpha-phase-kubelet-config-write-to-disk.1 b/docs/man/man1/kubeadm-alpha-phase-kubelet-config-write-to-disk.1 new file mode 100644 index 00000000000..b6fd7a0f989 --- /dev/null +++ b/docs/man/man1/kubeadm-alpha-phase-kubelet-config-write-to-disk.1 @@ -0,0 +1,3 @@ +This file is autogenerated, but we've stopped checking such files into the +repository to reduce the need for rebases. Please run hack/generate-docs.sh to +populate this file. diff --git a/docs/man/man1/kubeadm-alpha-phase-kubelet-config.1 b/docs/man/man1/kubeadm-alpha-phase-kubelet-config.1 new file mode 100644 index 00000000000..b6fd7a0f989 --- /dev/null +++ b/docs/man/man1/kubeadm-alpha-phase-kubelet-config.1 @@ -0,0 +1,3 @@ +This file is autogenerated, but we've stopped checking such files into the +repository to reduce the need for rebases. Please run hack/generate-docs.sh to +populate this file. diff --git a/docs/man/man1/kubeadm-alpha-phase-kubelet-write-env-file.1 b/docs/man/man1/kubeadm-alpha-phase-kubelet-write-env-file.1 new file mode 100644 index 00000000000..b6fd7a0f989 --- /dev/null +++ b/docs/man/man1/kubeadm-alpha-phase-kubelet-write-env-file.1 @@ -0,0 +1,3 @@ +This file is autogenerated, but we've stopped checking such files into the +repository to reduce the need for rebases. Please run hack/generate-docs.sh to +populate this file. diff --git a/docs/man/man1/kubeadm-upgrade-node-config.1 b/docs/man/man1/kubeadm-upgrade-node-config.1 new file mode 100644 index 00000000000..b6fd7a0f989 --- /dev/null +++ b/docs/man/man1/kubeadm-upgrade-node-config.1 @@ -0,0 +1,3 @@ +This file is autogenerated, but we've stopped checking such files into the +repository to reduce the need for rebases. Please run hack/generate-docs.sh to +populate this file. diff --git a/docs/man/man1/kubeadm-upgrade-node.1 b/docs/man/man1/kubeadm-upgrade-node.1 new file mode 100644 index 00000000000..b6fd7a0f989 --- /dev/null +++ b/docs/man/man1/kubeadm-upgrade-node.1 @@ -0,0 +1,3 @@ +This file is autogenerated, but we've stopped checking such files into the +repository to reduce the need for rebases. Please run hack/generate-docs.sh to +populate this file. diff --git a/pkg/util/initsystem/initsystem.go b/pkg/util/initsystem/initsystem.go index aca7a41b860..6638bab5eb2 100644 --- a/pkg/util/initsystem/initsystem.go +++ b/pkg/util/initsystem/initsystem.go @@ -44,15 +44,26 @@ type InitSystem interface { type SystemdInitSystem struct{} +func (sysd SystemdInitSystem) reloadSystemd() error { + if err := exec.Command("systemctl", "daemon-reload").Run(); err != nil { + return fmt.Errorf("failed to reload systemd: %v", err) + } + return nil +} + func (sysd SystemdInitSystem) ServiceStart(service string) error { + // Before we try to start any service, make sure that systemd is ready + if err := sysd.reloadSystemd(); err != nil { + return err + } args := []string{"start", service} - err := exec.Command("systemctl", args...).Run() - return err + return exec.Command("systemctl", args...).Run() } func (sysd SystemdInitSystem) ServiceRestart(service string) error { - if err := exec.Command("systemctl", "daemon-reload").Run(); err != nil { - return fmt.Errorf("failed to reload systemd: %v", err) + // Before we try to restart any service, make sure that systemd is ready + if err := sysd.reloadSystemd(); err != nil { + return err } args := []string{"restart", service} return exec.Command("systemctl", args...).Run() @@ -60,8 +71,7 @@ func (sysd SystemdInitSystem) ServiceRestart(service string) error { func (sysd SystemdInitSystem) ServiceStop(service string) error { args := []string{"stop", service} - err := exec.Command("systemctl", args...).Run() - return err + return exec.Command("systemctl", args...).Run() } func (sysd SystemdInitSystem) ServiceExists(service string) bool {