Merge pull request #64624 from luxas/kubeadm_kubelet_final

Automatic merge from submit-queue (batch tested with PRs 63386, 64624, 62297, 64847). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Final kubeadm-kubelet integration refactor PR

**What this PR does / why we need it**:
Note: Work in progress
This PR:
 - [x] Updates the debs/rpms to do the "right thing" with the new integration flow
    - Broken out into https://github.com/kubernetes/kubernetes/pull/64780
 - [x] Uploads the `CRISocket` information to the Node object as an annotation
   - Broken out into: https://github.com/kubernetes/kubernetes/pull/64792
 - [x] Makes the `kubeadm init` / `kubeadm join` flow to be preflight, stop kubelet, write config/env files, daemon-reload, start kubelet
 - [x] Renames `.NodeRegistration.ExtraArgs` to `.NodeRegistration.KubeletExtraArgs` as discussed in the SIG meeting
 - [x] Adds a `kubeadm upgrade node config` command for fetching the latest configuration and writing it down to the node before upgrading the kubelet
 - [x] Makes dynamic kubelet config actually get enabled when the feature gate in kubeadm is specifically opted into by the user
 - [x] Fixes misc. minor bugs
 - [x] Makes sure `kubeadm init --dry-run` works, so the dry-run functionality works for the kubelet integration as well

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #

**Special notes for your reviewer**:

**Release note**:

```release-note
kubeadm: Add a new `kubeadm upgrade node config` command
```
@kubernetes/sig-cluster-lifecycle-pr-reviews

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -32,6 +32,21 @@ import (
 type MasterConfiguration struct {
 	metav1.TypeMeta
 
+	// `kubeadm init`-only information. These fields are solely used the first time `kubeadm init` runs.
+	// After that, the information in the fields ARE NOT uploaded to the `kubeadm-config` ConfigMap
+	// that is used by `kubeadm upgrade` for instance.
+
+	// BootstrapTokens is respected at `kubeadm init` time and describes a set of Bootstrap Tokens to create.
+	// This information IS NOT uploaded to the kubeadm cluster configmap, partly because of its sensitive nature
+	BootstrapTokens []BootstrapToken
+
+	// NodeRegistration holds fields that relate to registering the new master node to the cluster
+	NodeRegistration NodeRegistrationOptions
+
+	// Cluster-wide configuration
+	// TODO: Move these fields under some kind of ClusterConfiguration or similar struct that describes
+	// one cluster. Eventually we want this kind of spec to align well with the Cluster API spec.
+
 	// API holds configuration for the k8s apiserver.
 	API API
 	// KubeProxy holds configuration for the k8s service proxy.
@@ -45,13 +60,6 @@ type MasterConfiguration struct {
 	// KubernetesVersion is the target version of the control plane.
 	KubernetesVersion string
 
-	// NodeRegistration holds fields that relate to registering the new master node to the cluster
-	NodeRegistration NodeRegistrationOptions
-
-	// BootstrapTokens is respected at `kubeadm init` time and describes a set of Bootstrap Tokens to create.
-	// This information IS NOT uploaded to the kubeadm cluster configmap, due to its sensitive nature
-	BootstrapTokens []BootstrapToken
-
 	// APIServerExtraArgs is a set of extra flags to pass to the API Server or override
 	// default ones in form of <flagname>=<value>.
 	// TODO: This is temporary and ideally we would like to switch all components to
@@ -141,10 +149,10 @@ type NodeRegistrationOptions struct {
 	// empty slice, i.e. `taints: {}` in the YAML file. This field is solely used for Node registration.
 	Taints []v1.Taint
 
-	// ExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file
+	// KubeletExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file
 	// kubeadm writes at runtime for the kubelet to source. This overrides the generic base-level configuration in the kubelet-config-1.X ConfigMap
 	// Flags have higher higher priority when parsing. These values are local and specific to the node kubeadm is executing on.
-	ExtraArgs map[string]string
+	KubeletExtraArgs map[string]string
 }
 
 // Networking contains elements describing cluster's networking configuration.

cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go

@@ -244,6 +244,8 @@ func autoConvert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration(in
 }
 
 func autoConvert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration(in *kubeadm.MasterConfiguration, out *MasterConfiguration, s conversion.Scope) error {
+	// WARNING: in.BootstrapTokens requires manual conversion: does not exist in peer-type
+	// WARNING: in.NodeRegistration requires manual conversion: does not exist in peer-type
 	if err := Convert_kubeadm_API_To_v1alpha1_API(&in.API, &out.API, s); err != nil {
 		return err
 	}
@@ -260,8 +262,6 @@ func autoConvert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration(in
 		return err
 	}
 	out.KubernetesVersion = in.KubernetesVersion
-	// WARNING: in.NodeRegistration requires manual conversion: does not exist in peer-type
-	// WARNING: in.BootstrapTokens requires manual conversion: does not exist in peer-type
 	out.APIServerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.APIServerExtraArgs))
 	out.ControllerManagerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ControllerManagerExtraArgs))
 	out.SchedulerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.SchedulerExtraArgs))

cmd/kubeadm/app/apis/kubeadm/v1alpha2/types.go

@@ -30,6 +30,21 @@ import (
 type MasterConfiguration struct {
 	metav1.TypeMeta `json:",inline"`
 
+	// `kubeadm init`-only information. These fields are solely used the first time `kubeadm init` runs.
+	// After that, the information in the fields ARE NOT uploaded to the `kubeadm-config` ConfigMap
+	// that is used by `kubeadm upgrade` for instance. These fields must be omitempty.
+
+	// BootstrapTokens is respected at `kubeadm init` time and describes a set of Bootstrap Tokens to create.
+	// This information IS NOT uploaded to the kubeadm cluster configmap, partly because of its sensitive nature
+	BootstrapTokens []BootstrapToken `json:"bootstrapTokens,omitempty"`
+
+	// NodeRegistration holds fields that relate to registering the new master node to the cluster
+	NodeRegistration NodeRegistrationOptions `json:"nodeRegistration,omitempty"`
+
+	// Cluster-wide configuration
+	// TODO: Move these fields under some kind of ClusterConfiguration or similar struct that describes
+	// one cluster. Eventually we want this kind of spec to align well with the Cluster API spec.
+
 	// API holds configuration for the k8s apiserver.
 	API API `json:"api"`
 	// KubeProxy holds configuration for the k8s service proxy.
@@ -41,16 +56,9 @@ type MasterConfiguration struct {
 	// Networking holds configuration for the networking topology of the cluster.
 	Networking Networking `json:"networking"`
 
-	// NodeRegistration holds fields that relate to registering the new master node to the cluster
-	NodeRegistration NodeRegistrationOptions `json:"nodeRegistration"`
-
 	// KubernetesVersion is the target version of the control plane.
 	KubernetesVersion string `json:"kubernetesVersion"`
 
-	// BootstrapTokens is respected at `kubeadm init` time and describes a set of Bootstrap Tokens to create.
-	// This information IS NOT uploaded to the kubeadm cluster configmap, due to its sensitive nature
-	BootstrapTokens []BootstrapToken `json:"bootstrapTokens,omitempty"`
-
 	// APIServerExtraArgs is a set of extra flags to pass to the API Server or override
 	// default ones in form of <flagname>=<value>.
 	// TODO: This is temporary and ideally we would like to switch all components to
@@ -133,10 +141,10 @@ type NodeRegistrationOptions struct {
 	// empty slice, i.e. `taints: {}` in the YAML file. This field is solely used for Node registration.
 	Taints []v1.Taint `json:"taints,omitempty"`
 
-	// ExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file
+	// KubeletExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file
 	// kubeadm writes at runtime for the kubelet to source. This overrides the generic base-level configuration in the kubelet-config-1.X ConfigMap
 	// Flags have higher higher priority when parsing. These values are local and specific to the node kubeadm is executing on.
-	ExtraArgs map[string]string `json:"kubeletExtraArgs,omitempty"`
+	KubeletExtraArgs map[string]string `json:"kubeletExtraArgs,omitempty"`
 }
 
 // Networking contains elements describing cluster's networking configuration

cmd/kubeadm/app/apis/kubeadm/v1alpha2/zz_generated.conversion.go

@@ -316,6 +316,10 @@ func Convert_kubeadm_LocalEtcd_To_v1alpha2_LocalEtcd(in *kubeadm.LocalEtcd, out
 }
 
 func autoConvert_v1alpha2_MasterConfiguration_To_kubeadm_MasterConfiguration(in *MasterConfiguration, out *kubeadm.MasterConfiguration, s conversion.Scope) error {
+	out.BootstrapTokens = *(*[]kubeadm.BootstrapToken)(unsafe.Pointer(&in.BootstrapTokens))
+	if err := Convert_v1alpha2_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(&in.NodeRegistration, &out.NodeRegistration, s); err != nil {
+		return err
+	}
 	if err := Convert_v1alpha2_API_To_kubeadm_API(&in.API, &out.API, s); err != nil {
 		return err
 	}
@@ -331,11 +335,7 @@ func autoConvert_v1alpha2_MasterConfiguration_To_kubeadm_MasterConfiguration(in
 	if err := Convert_v1alpha2_Networking_To_kubeadm_Networking(&in.Networking, &out.Networking, s); err != nil {
 		return err
 	}
-	if err := Convert_v1alpha2_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(&in.NodeRegistration, &out.NodeRegistration, s); err != nil {
-		return err
-	}
 	out.KubernetesVersion = in.KubernetesVersion
-	out.BootstrapTokens = *(*[]kubeadm.BootstrapToken)(unsafe.Pointer(&in.BootstrapTokens))
 	out.APIServerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.APIServerExtraArgs))
 	out.ControllerManagerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ControllerManagerExtraArgs))
 	out.SchedulerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.SchedulerExtraArgs))
@@ -360,6 +360,10 @@ func Convert_v1alpha2_MasterConfiguration_To_kubeadm_MasterConfiguration(in *Mas
 }
 
 func autoConvert_kubeadm_MasterConfiguration_To_v1alpha2_MasterConfiguration(in *kubeadm.MasterConfiguration, out *MasterConfiguration, s conversion.Scope) error {
+	out.BootstrapTokens = *(*[]BootstrapToken)(unsafe.Pointer(&in.BootstrapTokens))
+	if err := Convert_kubeadm_NodeRegistrationOptions_To_v1alpha2_NodeRegistrationOptions(&in.NodeRegistration, &out.NodeRegistration, s); err != nil {
+		return err
+	}
 	if err := Convert_kubeadm_API_To_v1alpha2_API(&in.API, &out.API, s); err != nil {
 		return err
 	}
@@ -376,10 +380,6 @@ func autoConvert_kubeadm_MasterConfiguration_To_v1alpha2_MasterConfiguration(in
 		return err
 	}
 	out.KubernetesVersion = in.KubernetesVersion
-	if err := Convert_kubeadm_NodeRegistrationOptions_To_v1alpha2_NodeRegistrationOptions(&in.NodeRegistration, &out.NodeRegistration, s); err != nil {
-		return err
-	}
-	out.BootstrapTokens = *(*[]BootstrapToken)(unsafe.Pointer(&in.BootstrapTokens))
 	out.APIServerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.APIServerExtraArgs))
 	out.ControllerManagerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ControllerManagerExtraArgs))
 	out.SchedulerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.SchedulerExtraArgs))
@@ -478,7 +478,7 @@ func autoConvert_v1alpha2_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOpt
 	out.Name = in.Name
 	out.CRISocket = in.CRISocket
 	out.Taints = *(*[]core_v1.Taint)(unsafe.Pointer(&in.Taints))
-	out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs))
+	out.KubeletExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.KubeletExtraArgs))
 	return nil
 }
 
@@ -491,7 +491,7 @@ func autoConvert_kubeadm_NodeRegistrationOptions_To_v1alpha2_NodeRegistrationOpt
 	out.Name = in.Name
 	out.CRISocket = in.CRISocket
 	out.Taints = *(*[]core_v1.Taint)(unsafe.Pointer(&in.Taints))
-	out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs))
+	out.KubeletExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.KubeletExtraArgs))
 	return nil
 }
 

cmd/kubeadm/app/apis/kubeadm/v1alpha2/zz_generated.deepcopy.go

@@ -295,12 +295,6 @@ func (in *LocalEtcd) DeepCopy() *LocalEtcd {
 func (in *MasterConfiguration) DeepCopyInto(out *MasterConfiguration) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta
-	out.API = in.API
-	in.KubeProxy.DeepCopyInto(&out.KubeProxy)
-	in.Etcd.DeepCopyInto(&out.Etcd)
-	in.KubeletConfiguration.DeepCopyInto(&out.KubeletConfiguration)
-	out.Networking = in.Networking
-	in.NodeRegistration.DeepCopyInto(&out.NodeRegistration)
 	if in.BootstrapTokens != nil {
 		in, out := &in.BootstrapTokens, &out.BootstrapTokens
 		*out = make([]BootstrapToken, len(*in))
@@ -308,6 +302,12 @@ func (in *MasterConfiguration) DeepCopyInto(out *MasterConfiguration) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	in.NodeRegistration.DeepCopyInto(&out.NodeRegistration)
+	out.API = in.API
+	in.KubeProxy.DeepCopyInto(&out.KubeProxy)
+	in.Etcd.DeepCopyInto(&out.Etcd)
+	in.KubeletConfiguration.DeepCopyInto(&out.KubeletConfiguration)
+	out.Networking = in.Networking
 	if in.APIServerExtraArgs != nil {
 		in, out := &in.APIServerExtraArgs, &out.APIServerExtraArgs
 		*out = make(map[string]string, len(*in))
@@ -456,8 +456,8 @@ func (in *NodeRegistrationOptions) DeepCopyInto(out *NodeRegistrationOptions) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
-	if in.ExtraArgs != nil {
+	if in.KubeletExtraArgs != nil {
-		in, out := &in.ExtraArgs, &out.ExtraArgs
+		in, out := &in.KubeletExtraArgs, &out.KubeletExtraArgs
 		*out = make(map[string]string, len(*in))
 		for key, val := range *in {
 			(*out)[key] = val

cmd/kubeadm/app/apis/kubeadm/v1alpha2/zz_generated.defaults.go

@@ -37,17 +37,17 @@ func RegisterDefaults(scheme *runtime.Scheme) error {
 
 func SetObjectDefaults_MasterConfiguration(in *MasterConfiguration) {
 	SetDefaults_MasterConfiguration(in)
+	for i := range in.BootstrapTokens {
+		a := &in.BootstrapTokens[i]
+		SetDefaults_BootstrapToken(a)
+	}
+	SetDefaults_NodeRegistrationOptions(&in.NodeRegistration)
 	if in.KubeProxy.Config != nil {
 		v1alpha1.SetDefaults_KubeProxyConfiguration(in.KubeProxy.Config)
 	}
 	if in.KubeletConfiguration.BaseConfig != nil {
 		v1beta1.SetDefaults_KubeletConfiguration(in.KubeletConfiguration.BaseConfig)
 	}
-	SetDefaults_NodeRegistrationOptions(&in.NodeRegistration)
-	for i := range in.BootstrapTokens {
-		a := &in.BootstrapTokens[i]
-		SetDefaults_BootstrapToken(a)
-	}
 }
 
 func SetObjectDefaults_NodeConfiguration(in *NodeConfiguration) {

cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go

@@ -295,12 +295,6 @@ func (in *LocalEtcd) DeepCopy() *LocalEtcd {
 func (in *MasterConfiguration) DeepCopyInto(out *MasterConfiguration) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta
-	out.API = in.API
-	in.KubeProxy.DeepCopyInto(&out.KubeProxy)
-	in.Etcd.DeepCopyInto(&out.Etcd)
-	in.KubeletConfiguration.DeepCopyInto(&out.KubeletConfiguration)
-	out.Networking = in.Networking
-	in.NodeRegistration.DeepCopyInto(&out.NodeRegistration)
 	if in.BootstrapTokens != nil {
 		in, out := &in.BootstrapTokens, &out.BootstrapTokens
 		*out = make([]BootstrapToken, len(*in))
@@ -308,6 +302,12 @@ func (in *MasterConfiguration) DeepCopyInto(out *MasterConfiguration) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	in.NodeRegistration.DeepCopyInto(&out.NodeRegistration)
+	out.API = in.API
+	in.KubeProxy.DeepCopyInto(&out.KubeProxy)
+	in.Etcd.DeepCopyInto(&out.Etcd)
+	in.KubeletConfiguration.DeepCopyInto(&out.KubeletConfiguration)
+	out.Networking = in.Networking
 	if in.APIServerExtraArgs != nil {
 		in, out := &in.APIServerExtraArgs, &out.APIServerExtraArgs
 		*out = make(map[string]string, len(*in))
@@ -456,8 +456,8 @@ func (in *NodeRegistrationOptions) DeepCopyInto(out *NodeRegistrationOptions) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
-	if in.ExtraArgs != nil {
+	if in.KubeletExtraArgs != nil {
-		in, out := &in.ExtraArgs, &out.ExtraArgs
+		in, out := &in.KubeletExtraArgs, &out.KubeletExtraArgs
 		*out = make(map[string]string, len(*in))
 		for key, val := range *in {
 			(*out)[key] = val

cmd/kubeadm/app/cmd/init.go

@@ -277,24 +277,35 @@ type Init struct {
 // Run executes master node provisioning, including certificates, needed static pod manifests, etc.
 func (i *Init) Run(out io.Writer) error {
 
-	// Write env file with flags for the kubelet to use. We do not need to write the --register-with-taints for the master,
-	// as we handle that ourselves in the markmaster phase
-	// TODO: Maybe we want to do that some time in the future, in order to remove some logic from the markmaster phase?
-	if err := kubeletphase.WriteKubeletDynamicEnvFile(&i.cfg.NodeRegistration, false); err != nil {
-		return err
-	}
-
-	// Try to start the kubelet service in case it's inactive
-	glog.V(1).Infof("Starting kubelet")
-	preflight.TryStartKubelet(i.ignorePreflightErrors)
-
 	// Get directories to write files to; can be faked if we're dry-running
 	glog.V(1).Infof("[init] Getting certificates directory from configuration")
 	realCertsDir := i.cfg.CertificatesDir
-	certsDirToWriteTo, kubeConfigDir, manifestDir, err := getDirectoriesToUse(i.dryRun, i.cfg.CertificatesDir)
+	certsDirToWriteTo, kubeConfigDir, manifestDir, kubeletDir, err := getDirectoriesToUse(i.dryRun, i.cfg.CertificatesDir)
 	if err != nil {
 		return fmt.Errorf("error getting directories to use: %v", err)
 	}
+
+	// First off, configure the kubelet. In this short timeframe, kubeadm is trying to stop/restart the kubelet
+	// Try to stop the kubelet service so no race conditions occur when configuring it
+	glog.V(1).Infof("Stopping the kubelet")
+	preflight.TryStopKubelet(i.ignorePreflightErrors)
+
+	// Write env file with flags for the kubelet to use. We do not need to write the --register-with-taints for the master,
+	// as we handle that ourselves in the markmaster phase
+	// TODO: Maybe we want to do that some time in the future, in order to remove some logic from the markmaster phase?
+	if err := kubeletphase.WriteKubeletDynamicEnvFile(&i.cfg.NodeRegistration, i.cfg.FeatureGates, false, kubeletDir); err != nil {
+		return fmt.Errorf("error writing a dynamic environment file for the kubelet: %v", err)
+	}
+
+	// Write the kubelet configuration file to disk.
+	if err := kubeletphase.WriteConfigToDisk(i.cfg.KubeletConfiguration.BaseConfig, kubeletDir); err != nil {
+		return fmt.Errorf("error writing kubelet configuration to disk: %v", err)
+	}
+
+	// Try to start the kubelet service in case it's inactive
+	glog.V(1).Infof("Starting the kubelet")
+	preflight.TryStartKubelet(i.ignorePreflightErrors)
+
 	// certsDirToWriteTo is gonna equal cfg.CertificatesDir in the normal case, but gonna be a temp directory if dryrunning
 	i.cfg.CertificatesDir = certsDirToWriteTo
 
@@ -359,16 +370,6 @@ func (i *Init) Run(out io.Writer) error {
 		return fmt.Errorf("error printing files on dryrun: %v", err)
 	}
 
-	kubeletVersion, err := preflight.GetKubeletVersion(utilsexec.New())
-	if err != nil {
-		return err
-	}
-
-	// Write the kubelet configuration to disk.
-	if err := kubeletphase.WriteConfigToDisk(i.cfg.KubeletConfiguration.BaseConfig); err != nil {
-		return fmt.Errorf("error writing kubelet configuration to disk: %v", err)
-	}
-
 	// Create a kubernetes client and wait for the API server to be healthy (if not dryrunning)
 	glog.V(1).Infof("creating Kubernetes client")
 	client, err := createClient(i.cfg, i.dryRun)
@@ -426,9 +427,13 @@ func (i *Init) Run(out io.Writer) error {
 		return fmt.Errorf("error uploading crisocket: %v", err)
 	}
 
-	// NOTE: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+	// This feature is disabled by default
-	// This feature is disabled by default, as it is alpha still
 	if features.Enabled(i.cfg.FeatureGates, features.DynamicKubeletConfig) {
+		kubeletVersion, err := preflight.GetKubeletVersion(utilsexec.New())
+		if err != nil {
+			return err
+		}
+
 		// Enable dynamic kubelet configuration for the node.
 		if err := kubeletphase.EnableDynamicConfigForNode(client, i.cfg.NodeRegistration.Name, kubeletVersion); err != nil {
 			return fmt.Errorf("error enabling dynamic kubelet configuration: %v", err)
@@ -544,17 +549,17 @@ func createClient(cfg *kubeadmapi.MasterConfiguration, dryRun bool) (clientset.I
 
 // getDirectoriesToUse returns the (in order) certificates, kubeconfig and Static Pod manifest directories, followed by a possible error
 // This behaves differently when dry-running vs the normal flow
-func getDirectoriesToUse(dryRun bool, defaultPkiDir string) (string, string, string, error) {
+func getDirectoriesToUse(dryRun bool, defaultPkiDir string) (string, string, string, string, error) {
 	if dryRun {
 		dryRunDir, err := ioutil.TempDir("", "kubeadm-init-dryrun")
 		if err != nil {
-			return "", "", "", fmt.Errorf("couldn't create a temporary directory: %v", err)
+			return "", "", "", "", fmt.Errorf("couldn't create a temporary directory: %v", err)
 		}
 		// Use the same temp dir for all
-		return dryRunDir, dryRunDir, dryRunDir, nil
+		return dryRunDir, dryRunDir, dryRunDir, dryRunDir, nil
 	}
 
-	return defaultPkiDir, kubeadmconstants.KubernetesDir, kubeadmconstants.GetStaticPodDirectory(), nil
+	return defaultPkiDir, kubeadmconstants.KubernetesDir, kubeadmconstants.GetStaticPodDirectory(), kubeadmconstants.KubeletRunDirectory, nil
 }
 
 // printFilesIfDryRunning prints the Static Pod manifests to stdout and informs about the temporary directory to go and lookup
@@ -569,11 +574,19 @@ func printFilesIfDryRunning(dryRun bool, manifestDir string) error {
 
 	// Print the contents of the upgraded manifests and pretend like they were in /etc/kubernetes/manifests
 	files := []dryrunutil.FileToPrint{}
+	// Print static pod manifests
 	for _, component := range kubeadmconstants.MasterComponents {
 		realPath := kubeadmconstants.GetStaticPodFilepath(component, manifestDir)
 		outputPath := kubeadmconstants.GetStaticPodFilepath(component, kubeadmconstants.GetStaticPodDirectory())
 		files = append(files, dryrunutil.NewFileToPrint(realPath, outputPath))
 	}
+	// Print kubelet config manifests
+	kubeletConfigFiles := []string{kubeadmconstants.KubeletConfigurationFileName, kubeadmconstants.KubeletEnvFileName}
+	for _, filename := range kubeletConfigFiles {
+		realPath := filepath.Join(manifestDir, filename)
+		outputPath := filepath.Join(kubeadmconstants.KubeletRunDirectory, filename)
+		files = append(files, dryrunutil.NewFileToPrint(realPath, outputPath))
+	}
 
 	return dryrunutil.PrintDryRunFiles(files, os.Stdout)
 }

cmd/kubeadm/app/cmd/join.go

@@ -215,7 +215,8 @@ func AddJoinOtherFlags(flagSet *flag.FlagSet, cfgPath *string, skipPreFlight *bo
 
 // Join defines struct used by kubeadm join command
 type Join struct {
-	cfg *kubeadmapi.NodeConfiguration
+	cfg                   *kubeadmapi.NodeConfiguration
+	ignorePreflightErrors sets.String
 }
 
 // NewJoin instantiates Join struct with given arguments
@@ -239,39 +240,31 @@ func NewJoin(cfgPath string, args []string, defaultcfg *kubeadmapiv1alpha2.NodeC
 		return nil, err
 	}
 
-	// Try to start the kubelet service in case it's inactive
+	return &Join{cfg: internalcfg, ignorePreflightErrors: ignorePreflightErrors}, nil
-	glog.V(1).Infoln("[preflight] starting kubelet service if it's inactive")
-	preflight.TryStartKubelet(ignorePreflightErrors)
-
-	return &Join{cfg: internalcfg}, nil
 }
 
 // Run executes worker node provisioning and tries to join an existing cluster.
 func (j *Join) Run(out io.Writer) error {
 
-	// Write env file with flags for the kubelet to use. Also register taints
+	// Perform the Discovery, which turns a Bootstrap Token and optionally (and preferably) a CA cert hash into a KubeConfig
-	if err := kubeletphase.WriteKubeletDynamicEnvFile(&j.cfg.NodeRegistration, true); err != nil {
+	// file that may be used for the TLS Bootstrapping process the kubelet performs using the Certificates API.
-		return err
-	}
-
 	glog.V(1).Infoln("[join] retrieving KubeConfig objects")
 	cfg, err := discovery.For(j.cfg)
 	if err != nil {
 		return err
 	}
 
-	kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletBootstrapKubeConfigFileName)
+	bootstrapKubeConfigFile := kubeadmconstants.GetBootstrapKubeletKubeConfigPath()
 
 	// Write the bootstrap kubelet config file or the TLS-Boostrapped kubelet config file down to disk
-	glog.V(1).Infoln("[join] writing bootstrap kubelet config file at", kubeconfigFile)
+	glog.V(1).Infoln("[join] writing bootstrap kubelet config file at", bootstrapKubeConfigFile)
-	if err := kubeconfigutil.WriteToDisk(kubeconfigFile, cfg); err != nil {
+	if err := kubeconfigutil.WriteToDisk(bootstrapKubeConfigFile, cfg); err != nil {
 		return fmt.Errorf("couldn't save bootstrap-kubelet.conf to disk: %v", err)
 	}
 
 	// Write the ca certificate to disk so kubelet can use it for authentication
 	cluster := cfg.Contexts[cfg.CurrentContext].Cluster
-	err = certutil.WriteCert(j.cfg.CACertPath, cfg.Clusters[cluster].CertificateAuthorityData)
+	if err := certutil.WriteCert(j.cfg.CACertPath, cfg.Clusters[cluster].CertificateAuthorityData); err != nil {
-	if err != nil {
 		return fmt.Errorf("couldn't save the CA certificate to disk: %v", err)
 	}
 
@@ -280,12 +273,31 @@ func (j *Join) Run(out io.Writer) error {
 		return err
 	}
 
-	// Write the configuration for the kubelet down to disk so the kubelet can start
+	bootstrapClient, err := kubeconfigutil.ClientSetFromFile(bootstrapKubeConfigFile)
-	if err := kubeletphase.DownloadConfig(kubeconfigFile, kubeletVersion); err != nil {
+	if err != nil {
+		return fmt.Errorf("couldn't create client from kubeconfig file %q", bootstrapKubeConfigFile)
+	}
+
+	// Configure the kubelet. In this short timeframe, kubeadm is trying to stop/restart the kubelet
+	// Try to stop the kubelet service so no race conditions occur when configuring it
+	glog.V(1).Infof("Stopping the kubelet")
+	preflight.TryStopKubelet(j.ignorePreflightErrors)
+
+	// Write the configuration for the kubelet (using the bootstrap token credentials) to disk so the kubelet can start
+	if err := kubeletphase.DownloadConfig(bootstrapClient, kubeletVersion, kubeadmconstants.KubeletRunDirectory); err != nil {
 		return err
 	}
 
-	// Now the kubelet will perform the TLS Bootstrap, transforming bootstrap-kubeconfig.conf to kubeconfig.conf in /etc/kubernetes
+	// Write env file with flags for the kubelet to use. Also register taints
+	if err := kubeletphase.WriteKubeletDynamicEnvFile(&j.cfg.NodeRegistration, j.cfg.FeatureGates, true, kubeadmconstants.KubeletRunDirectory); err != nil {
+		return err
+	}
+
+	// Try to start the kubelet service in case it's inactive
+	glog.V(1).Infof("Starting the kubelet")
+	preflight.TryStartKubelet(j.ignorePreflightErrors)
+
+	// Now the kubelet will perform the TLS Bootstrap, transforming /etc/kubernetes/bootstrap-kubelet.conf to /etc/kubernetes/kubelet.conf
 	// Wait for the kubelet to create the /etc/kubernetes/kubelet.conf KubeConfig file. If this process
 	// times out, display a somewhat user-friendly message.
 	waiter := apiclient.NewKubeWaiter(nil, kubeadmconstants.TLSBootstrapTimeout, os.Stdout)
@@ -295,8 +307,7 @@ func (j *Join) Run(out io.Writer) error {
 	}
 
 	// When we know the /etc/kubernetes/kubelet.conf file is available, get the client
-	kubeletKubeConfig := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName)
+	client, err := kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetKubeletKubeConfigPath())
-	client, err := kubeconfigutil.ClientSetFromFile(kubeletKubeConfig)
 	if err != nil {
 		return err
 	}
@@ -306,9 +317,7 @@ func (j *Join) Run(out io.Writer) error {
 		return fmt.Errorf("error uploading crisocket: %v", err)
 	}
 
-	// NOTE: the "--dynamic-config-dir" flag should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf for this to work
+	// This feature is disabled by default in kubeadm
-	// This feature is disabled by default, as it is alpha still
-	glog.V(1).Infoln("[join] enabling dynamic kubelet configuration")
 	if features.Enabled(j.cfg.FeatureGates, features.DynamicKubeletConfig) {
 		if err := kubeletphase.EnableDynamicConfigForNode(client, j.cfg.NodeRegistration.Name, kubeletVersion); err != nil {
 			return fmt.Errorf("error consuming base kubelet configuration: %v", err)

cmd/kubeadm/app/cmd/options/BUILD

@@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
 
 go_library(
     name = "go_default_library",
-    srcs = ["token.go"],
+    srcs = [
+        "generic.go",
+        "token.go",
+    ],
     importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options",
     visibility = ["//visibility:public"],
     deps = [

cmd/kubeadm/app/cmd/options/generic.go

@@ -0,0 +1,29 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import "github.com/spf13/pflag"
+
+// AddKubeConfigFlag adds the --kubeconfig flag to the given flagset
+func AddKubeConfigFlag(fs *pflag.FlagSet, kubeConfigFile *string) {
+	fs.StringVar(kubeConfigFile, "kubeconfig", *kubeConfigFile, "The KubeConfig file to use when talking to the cluster")
+}
+
+// AddConfigFlag adds the --config flag to the given flagset
+func AddConfigFlag(fs *pflag.FlagSet, cfgPath *string) {
+	fs.StringVar(cfgPath, "config", *cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)")
+}

cmd/kubeadm/app/cmd/phases/kubelet.go

@@ -18,45 +18,81 @@ package phases
 
 import (
 	"fmt"
-	"os"
+	"io/ioutil"
 
 	"github.com/spf13/cobra"
 
+	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
 	kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
+	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
+	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
 	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
 	"k8s.io/kubernetes/pkg/util/normalizer"
 	"k8s.io/kubernetes/pkg/util/version"
+	utilsexec "k8s.io/utils/exec"
+)
+
+const (
+	// TODO: Figure out how to get these constants from the API machinery
+	masterConfig = "MasterConfiguration"
+	nodeConfig   = "NodeConfiguration"
 )
 
 var (
-	kubeletWriteConfigToDiskLongDesc = normalizer.LongDesc(`
+	kubeletWriteEnvFileLongDesc = normalizer.LongDesc(`
-		Writes kubelet configuration to disk, either based on the kubelet-config-1.X ConfigMap in the cluster, or from the
+		Writes an environment file with flags that should be passed to the kubelet executing on the master or node.
-		configuration passed to the command via "--config".
+		This --config flag can either consume a MasterConfiguration object or a NodeConfiguration one, as this
+		function is used for both "kubeadm init" and "kubeadm join".
 		` + cmdutil.AlphaDisclaimer)
 
-	kubeletWriteConfigToDiskExample = normalizer.Examples(`
+	kubeletWriteEnvFileExample = normalizer.Examples(`
-		# Writes kubelet configuration for a node to disk. The information is fetched from the cluster ConfigMap
+		# Writes a dynamic environment file with kubelet flags from a MasterConfiguration file.
-		kubeadm alpha phase kubelet write-config-to-disk --kubelet-version v1.11.0 --kubeconfig /etc/kubernetes/kubelet.conf
+		kubeadm alpha phase kubelet write-env-file --config masterconfig.yaml
 
-		# Writes kubelet configuration down to disk, based on the configuration flag passed to --config
+		# Writes a dynamic environment file with kubelet flags from a NodeConfiguration file.
-		kubeadm alpha phase kubelet write-config-to-disk --kubelet-version v1.11.0 --config kubeadm.yaml
+		kubeadm alpha phase kubelet write-env-file --config nodeConfig.yaml
 		`)
 
-	kubeletUploadDynamicConfigLongDesc = normalizer.LongDesc(`
+	kubeletConfigUploadLongDesc = normalizer.LongDesc(`
 		Uploads kubelet configuration extracted from the kubeadm MasterConfiguration object to a ConfigMap
-		of the form kubelet-config-1.X in the cluster, where X is the minor version of the current Kubernetes version
+		of the form kubelet-config-1.X in the cluster, where X is the minor version of the current (API Server) Kubernetes version.
 		` + cmdutil.AlphaDisclaimer)
 
-	kubeletUploadDynamicConfigExample = normalizer.Examples(`
+	kubeletConfigUploadExample = normalizer.Examples(`
 		# Uploads the kubelet configuration from the kubeadm Config file to a ConfigMap in the cluster.
-		kubeadm alpha phase kubelet upload-config --config kubeadm.yaml
+		kubeadm alpha phase kubelet config upload --config kubeadm.yaml
 		`)
 
-	kubeletEnableDynamicConfigLongDesc = normalizer.LongDesc(`
+	kubeletConfigDownloadLongDesc = normalizer.LongDesc(`
+		Downloads the kubelet configuration from a ConfigMap of the form "kubelet-config-1.X" in the cluster,
+		where X is the minor version of the kubelet. Either kubeadm autodetects the kubelet version by exec-ing
+		"kubelet --version" or respects the --kubelet-version parameter.
+		` + cmdutil.AlphaDisclaimer)
+
+	kubeletConfigDownloadExample = normalizer.Examples(`
+		# Downloads the kubelet configuration from the ConfigMap in the cluster. Autodetects the kubelet version.
+		kubeadm alpha phase kubelet config download
+
+		# Downloads the kubelet configuration from the ConfigMap in the cluster. Uses a specific desired kubelet version.
+		kubeadm alpha phase kubelet config download --kubelet-version v1.11.0
+		`)
+
+	kubeletConfigWriteToDiskLongDesc = normalizer.LongDesc(`
+		Writes kubelet configuration to disk, based on the kubeadm configuration passed via "--config".
+		` + cmdutil.AlphaDisclaimer)
+
+	kubeletConfigWriteToDiskExample = normalizer.Examples(`
+		# Extracts the kubelet configuration from a kubeadm configuration file
+		kubeadm alpha phase kubelet config write-to-disk --config kubeadm.yaml
+		`)
+
+	kubeletConfigEnableDynamicLongDesc = normalizer.LongDesc(`
 		Enables or updates dynamic kubelet configuration for a Node, against the kubelet-config-1.X ConfigMap in the cluster,
 		where X is the minor version of the desired kubelet version.
 
@@ -65,7 +101,7 @@ var (
 
 		` + cmdutil.AlphaDisclaimer)
 
-	kubeletEnableDynamicConfigExample = normalizer.Examples(`
+	kubeletConfigEnableDynamicExample = normalizer.Examples(`
 		# Enables dynamic kubelet configuration for a Node.
 		kubeadm alpha phase kubelet enable-dynamic-config --node-name node-1 --kubelet-version v1.11.0
 
@@ -74,32 +110,110 @@ var (
 		`)
 )
 
-// NewCmdKubelet returns main command for Kubelet phase
+// NewCmdKubelet returns command for `kubeadm phase kubelet`
 func NewCmdKubelet() *cobra.Command {
-	var kubeConfigFile string
 	cmd := &cobra.Command{
 		Use:   "kubelet",
+		Short: "Commands related to handling the kubelet.",
+		Long:  cmdutil.MacroCommandLongDescription,
+	}
+
+	cmd.AddCommand(NewCmdKubeletConfig())
+	cmd.AddCommand(NewCmdKubeletWriteEnvFile())
+	return cmd
+}
+
+// NewCmdKubeletWriteEnvFile calls cobra.Command for writing the dynamic kubelet env file based on a MasterConfiguration or NodeConfiguration object
+func NewCmdKubeletWriteEnvFile() *cobra.Command {
+	var cfgPath string
+
+	cmd := &cobra.Command{
+		Use:     "write-env-file",
+		Short:   "Writes an environment file with runtime flags for the kubelet.",
+		Long:    kubeletWriteEnvFileLongDesc,
+		Example: kubeletWriteEnvFileExample,
+		Run: func(cmd *cobra.Command, args []string) {
+			err := RunKubeletWriteEnvFile(cfgPath)
+			kubeadmutil.CheckErr(err)
+		},
+	}
+
+	options.AddConfigFlag(cmd.Flags(), &cfgPath)
+	return cmd
+}
+
+// RunKubeletWriteEnvFile is the function that is run when "kubeadm phase kubelet write-env-file" is executed
+func RunKubeletWriteEnvFile(cfgPath string) error {
+	b, err := ioutil.ReadFile(cfgPath)
+	if err != nil {
+		return err
+	}
+
+	gvk, err := kubeadmutil.GroupVersionKindFromBytes(b, kubeadmscheme.Codecs)
+	if err != nil {
+		return err
+	}
+
+	var nodeRegistrationObj *kubeadmapi.NodeRegistrationOptions
+	var featureGates map[string]bool
+	var registerWithTaints bool
+	switch gvk.Kind {
+	case masterConfig:
+		internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, &kubeadmapiv1alpha2.MasterConfiguration{})
+		if err != nil {
+			return err
+		}
+		nodeRegistrationObj = &internalcfg.NodeRegistration
+		featureGates = internalcfg.FeatureGates
+		registerWithTaints = false
+	case nodeConfig:
+		internalcfg, err := configutil.NodeConfigFileAndDefaultsToInternalConfig(cfgPath, &kubeadmapiv1alpha2.NodeConfiguration{})
+		if err != nil {
+			return err
+		}
+		nodeRegistrationObj = &internalcfg.NodeRegistration
+		featureGates = internalcfg.FeatureGates
+		registerWithTaints = true
+	default:
+		if err != nil {
+			return fmt.Errorf("Didn't recognize type with GroupVersionKind: %v", gvk)
+		}
+	}
+	if nodeRegistrationObj == nil {
+		return fmt.Errorf("couldn't load nodeRegistration field from config file")
+	}
+
+	if err := kubeletphase.WriteKubeletDynamicEnvFile(nodeRegistrationObj, featureGates, registerWithTaints, constants.KubeletRunDirectory); err != nil {
+		return fmt.Errorf("error writing a dynamic environment file for the kubelet: %v", err)
+	}
+	return nil
+}
+
+// NewCmdKubeletConfig returns command for `kubeadm phase kubelet config`
+func NewCmdKubeletConfig() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "config",
 		Short: "Handles kubelet configuration.",
 		Long:  cmdutil.MacroCommandLongDescription,
 	}
 
-	cmd.PersistentFlags().StringVar(&kubeConfigFile, "kubeconfig", "/etc/kubernetes/admin.conf", "The KubeConfig file to use when talking to the cluster")
+	cmd.AddCommand(NewCmdKubeletConfigUpload())
-
+	cmd.AddCommand(NewCmdKubeletConfigDownload())
-	cmd.AddCommand(NewCmdKubeletWriteConfigToDisk(&kubeConfigFile))
+	cmd.AddCommand(NewCmdKubeletConfigWriteToDisk())
-	cmd.AddCommand(NewCmdKubeletUploadConfig(&kubeConfigFile))
+	cmd.AddCommand(NewCmdKubeletConfigEnableDynamic())
-	cmd.AddCommand(NewCmdKubeletEnableDynamicConfig(&kubeConfigFile))
 	return cmd
 }
 
-// NewCmdKubeletUploadConfig calls cobra.Command for uploading dynamic kubelet configuration
+// NewCmdKubeletConfigUpload calls cobra.Command for uploading dynamic kubelet configuration
-func NewCmdKubeletUploadConfig(kubeConfigFile *string) *cobra.Command {
+func NewCmdKubeletConfigUpload() *cobra.Command {
 	var cfgPath string
+	kubeConfigFile := constants.GetAdminKubeConfigPath()
 
 	cmd := &cobra.Command{
-		Use:     "upload-config",
+		Use:     "upload",
-		Short:   "Uploads kubelet configuration to a ConfigMap",
+		Short:   "Uploads kubelet configuration to a ConfigMap based on a kubeadm MasterConfiguration file.",
-		Long:    kubeletUploadDynamicConfigLongDesc,
+		Long:    kubeletConfigUploadLongDesc,
-		Example: kubeletUploadDynamicConfigExample,
+		Example: kubeletConfigUploadExample,
 		Run: func(cmd *cobra.Command, args []string) {
 			if len(cfgPath) == 0 {
 				kubeadmutil.CheckErr(fmt.Errorf("The --config argument is required"))
@@ -109,7 +223,7 @@ func NewCmdKubeletUploadConfig(kubeConfigFile *string) *cobra.Command {
 			internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, &kubeadmapiv1alpha2.MasterConfiguration{})
 			kubeadmutil.CheckErr(err)
 
-			client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile)
+			client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile)
 			kubeadmutil.CheckErr(err)
 
 			err = kubeletphase.CreateConfigMap(internalcfg, client)
@@ -117,50 +231,83 @@ func NewCmdKubeletUploadConfig(kubeConfigFile *string) *cobra.Command {
 		},
 	}
 
-	cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)")
+	options.AddKubeConfigFlag(cmd.Flags(), &kubeConfigFile)
+	options.AddConfigFlag(cmd.Flags(), &cfgPath)
 	return cmd
 }
 
-// NewCmdKubeletWriteConfigToDisk calls cobra.Command for writing init kubelet configuration
+// NewCmdKubeletConfigDownload calls cobra.Command for downloading the kubelet configuration from the kubelet-config-1.X ConfigMap in the cluster
-func NewCmdKubeletWriteConfigToDisk(kubeConfigFile *string) *cobra.Command {
+func NewCmdKubeletConfigDownload() *cobra.Command {
-	var cfgPath, kubeletVersionStr string
+	var kubeletVersionStr string
+	// TODO: Be smarter about this and be able to load multiple kubeconfig files in different orders of precedence
+	kubeConfigFile := constants.GetKubeletKubeConfigPath()
+
 	cmd := &cobra.Command{
-		Use:     "write-config-to-disk",
+		Use:     "download",
-		Short:   "Writes kubelet configuration to disk, either based on the --config argument or the kubeadm-config ConfigMap.",
+		Short:   "Downloads the kubelet configuration from the cluster ConfigMap kubelet-config-1.X, where X is the minor version of the kubelet.",
-		Long:    kubeletWriteConfigToDiskLongDesc,
+		Long:    kubeletConfigDownloadLongDesc,
-		Example: kubeletWriteConfigToDiskExample,
+		Example: kubeletConfigDownloadExample,
 		Run: func(cmd *cobra.Command, args []string) {
-			if len(kubeletVersionStr) == 0 {
+			kubeletVersion, err := getKubeletVersion(kubeletVersionStr)
-				kubeadmutil.CheckErr(fmt.Errorf("The --kubelet-version argument is required"))
-			}
-
-			client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile)
 			kubeadmutil.CheckErr(err)
 
-			// This call returns the ready-to-use configuration based on the configuration file
+			client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile)
-			internalcfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "kubelet", cfgPath)
 			kubeadmutil.CheckErr(err)
 
-			err = kubeletphase.WriteConfigToDisk(internalcfg.KubeletConfiguration.BaseConfig)
+			err = kubeletphase.DownloadConfig(client, kubeletVersion, constants.KubeletRunDirectory)
 			kubeadmutil.CheckErr(err)
 		},
 	}
 
-	cmd.Flags().StringVar(&kubeletVersionStr, "kubelet-version", kubeletVersionStr, "The desired version for the kubelet")
+	options.AddKubeConfigFlag(cmd.Flags(), &kubeConfigFile)
-	cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)")
+	cmd.Flags().StringVar(&kubeletVersionStr, "kubelet-version", kubeletVersionStr, "The desired version for the kubelet. Defaults to being autodetected from 'kubelet --version'.")
 	return cmd
 }
 
-// NewCmdKubeletEnableDynamicConfig calls cobra.Command for enabling dynamic kubelet configuration on node
+func getKubeletVersion(kubeletVersionStr string) (*version.Version, error) {
+	if len(kubeletVersionStr) > 0 {
+		return version.ParseSemantic(kubeletVersionStr)
+	}
+	return preflight.GetKubeletVersion(utilsexec.New())
+}
+
+// NewCmdKubeletConfigWriteToDisk calls cobra.Command for writing init kubelet configuration
+func NewCmdKubeletConfigWriteToDisk() *cobra.Command {
+	var cfgPath string
+	cmd := &cobra.Command{
+		Use:     "write-to-disk",
+		Short:   "Writes kubelet configuration to disk, either based on the --config argument.",
+		Long:    kubeletConfigWriteToDiskLongDesc,
+		Example: kubeletConfigWriteToDiskExample,
+		Run: func(cmd *cobra.Command, args []string) {
+			if len(cfgPath) == 0 {
+				kubeadmutil.CheckErr(fmt.Errorf("The --config argument is required"))
+			}
+
+			// This call returns the ready-to-use configuration based on the configuration file
+			internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, &kubeadmapiv1alpha2.MasterConfiguration{})
+			kubeadmutil.CheckErr(err)
+
+			err = kubeletphase.WriteConfigToDisk(internalcfg.KubeletConfiguration.BaseConfig, constants.KubeletRunDirectory)
+			kubeadmutil.CheckErr(err)
+		},
+	}
+
+	options.AddConfigFlag(cmd.Flags(), &cfgPath)
+	return cmd
+}
+
+// NewCmdKubeletConfigEnableDynamic calls cobra.Command for enabling dynamic kubelet configuration on node
 // This feature is still in alpha and an experimental state
-func NewCmdKubeletEnableDynamicConfig(kubeConfigFile *string) *cobra.Command {
+func NewCmdKubeletConfigEnableDynamic() *cobra.Command {
 	var nodeName, kubeletVersionStr string
+	kubeConfigFile := constants.GetAdminKubeConfigPath()
 
 	cmd := &cobra.Command{
-		Use:     "enable-dynamic-config",
+		Use:     "enable-dynamic",
 		Short:   "EXPERIMENTAL: Enables or updates dynamic kubelet configuration for a Node",
-		Long:    kubeletEnableDynamicConfigLongDesc,
+		Long:    kubeletConfigEnableDynamicLongDesc,
-		Example: kubeletEnableDynamicConfigExample,
+		Example: kubeletConfigEnableDynamicExample,
 		Run: func(cmd *cobra.Command, args []string) {
 			if len(nodeName) == 0 {
 				kubeadmutil.CheckErr(fmt.Errorf("The --node-name argument is required"))
@@ -172,7 +319,7 @@ func NewCmdKubeletEnableDynamicConfig(kubeConfigFile *string) *cobra.Command {
 			kubeletVersion, err := version.ParseSemantic(kubeletVersionStr)
 			kubeadmutil.CheckErr(err)
 
-			client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile)
+			client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile)
 			kubeadmutil.CheckErr(err)
 
 			err = kubeletphase.EnableDynamicConfigForNode(client, nodeName, kubeletVersion)
@@ -180,6 +327,7 @@ func NewCmdKubeletEnableDynamicConfig(kubeConfigFile *string) *cobra.Command {
 		},
 	}
 
+	options.AddKubeConfigFlag(cmd.Flags(), &kubeConfigFile)
 	cmd.Flags().StringVar(&nodeName, "node-name", nodeName, "Name of the node that should enable the dynamic kubelet configuration")
 	cmd.Flags().StringVar(&kubeletVersionStr, "kubelet-version", kubeletVersionStr, "The desired version for the kubelet")
 	return cmd

cmd/kubeadm/app/cmd/phases/kubelet_test.go

@@ -25,11 +25,12 @@ import (
 )
 
 func TestKubeletSubCommandsHasFlags(t *testing.T) {
-	kubeConfigFile := "foo"
 	subCmds := []*cobra.Command{
-		NewCmdKubeletUploadConfig(&kubeConfigFile),
+		NewCmdKubeletWriteEnvFile(),
-		NewCmdKubeletWriteConfigToDisk(&kubeConfigFile),
+		NewCmdKubeletConfigUpload(),
-		NewCmdKubeletEnableDynamicConfig(&kubeConfigFile),
+		NewCmdKubeletConfigDownload(),
+		NewCmdKubeletConfigWriteToDisk(),
+		NewCmdKubeletConfigEnableDynamic(),
 	}
 
 	commonFlags := []string{}
@@ -39,21 +40,35 @@ func TestKubeletSubCommandsHasFlags(t *testing.T) {
 		additionalFlags []string
 	}{
 		{
-			command: "upload-config",
+			command: "write-env-file",
 			additionalFlags: []string{
 				"config",
 			},
 		},
 		{
-			command: "write-config-to-disk",
+			command: "upload",
 			additionalFlags: []string{
+				"kubeconfig",
+				"config",
+			},
+		},
+		{
+			command: "download",
+			additionalFlags: []string{
+				"kubeconfig",
 				"kubelet-version",
+			},
+		},
+		{
+			command: "write-to-disk",
+			additionalFlags: []string{
 				"config",
 			},
 		},
 		{
-			command: "enable-dynamic-config",
+			command: "enable-dynamic",
 			additionalFlags: []string{
+				"kubeconfig",
 				"node-name",
 				"kubelet-version",
 			},

cmd/kubeadm/app/cmd/upgrade/BUILD

@@ -6,6 +6,7 @@ go_library(
         "apply.go",
         "common.go",
         "diff.go",
+        "node.go",
         "plan.go",
         "upgrade.go",
     ],
@@ -20,6 +21,7 @@ go_library(
         "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/features:go_default_library",
         "//cmd/kubeadm/app/phases/controlplane:go_default_library",
+        "//cmd/kubeadm/app/phases/kubelet:go_default_library",
         "//cmd/kubeadm/app/phases/upgrade:go_default_library",
         "//cmd/kubeadm/app/preflight:go_default_library",
         "//cmd/kubeadm/app/util:go_default_library",
@@ -28,6 +30,7 @@ go_library(
         "//cmd/kubeadm/app/util/dryrun:go_default_library",
         "//cmd/kubeadm/app/util/etcd:go_default_library",
         "//cmd/kubeadm/app/util/kubeconfig:go_default_library",
+        "//pkg/util/normalizer:go_default_library",
         "//pkg/util/version:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/github.com/pmezard/go-difflib/difflib:go_default_library",

cmd/kubeadm/app/cmd/upgrade/common.go

@@ -54,6 +54,12 @@ type upgradeVariables struct {
 
 // enforceRequirements verifies that it's okay to upgrade and then returns the variables needed for the rest of the procedure
 func enforceRequirements(flags *cmdUpgradeFlags, dryRun bool, newK8sVersion string) (*upgradeVariables, error) {
+
+	// Set the default for the kubeconfig path if the user didn't override with the flags
+	if flags.kubeConfigPath == "" {
+		flags.kubeConfigPath = "/etc/kubernetes/admin.conf"
+	}
+
 	client, err := getClient(flags.kubeConfigPath, dryRun)
 	if err != nil {
 		return nil, fmt.Errorf("couldn't create a Kubernetes client from file %q: %v", flags.kubeConfigPath, err)
@@ -154,7 +160,10 @@ func getClient(file string, dryRun bool) (clientset.Interface, error) {
 		}
 
 		// Get the fake clientset
-		fakeclient := apiclient.NewDryRunClient(dryRunGetter, os.Stdout)
+		dryRunOpts := apiclient.GetDefaultDryRunClientOptions(dryRunGetter, os.Stdout)
+		// Print GET and LIST requests
+		dryRunOpts.PrintGETAndLIST = true
+		fakeclient := apiclient.NewDryRunClientWithOpts(dryRunOpts)
 		// As we know the return of Discovery() of the fake clientset is of type *fakediscovery.FakeDiscovery
 		// we can convert it to that struct.
 		fakeclientDiscovery, ok := fakeclient.Discovery().(*fakediscovery.FakeDiscovery)

cmd/kubeadm/app/cmd/upgrade/node.go

@@ -0,0 +1,162 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package upgrade
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+
+	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
+	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
+	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
+	dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
+	"k8s.io/kubernetes/pkg/util/normalizer"
+	"k8s.io/kubernetes/pkg/util/version"
+)
+
+var (
+	upgradeNodeConfigLongDesc = normalizer.LongDesc(`
+		Downloads the kubelet configuration from a ConfigMap of the form "kubelet-config-1.X" in the cluster,
+		where X is the minor version of the kubelet. kubeadm uses the --kubelet-version parameter to determine
+		what the _desired_ kubelet version is. Give 
+		`)
+
+	upgradeNodeConfigExample = normalizer.Examples(`
+		# Downloads the kubelet configuration from the ConfigMap in the cluster. Uses a specific desired kubelet version.
+		kubeadm upgrade node config --kubelet-version v1.11.0
+
+		# Simulates the downloading of the kubelet configuration from the ConfigMap in the cluster with a specific desired
+		# version. Does not change any state locally on the node.
+		kubeadm upgrade node config --kubelet-version v1.11.0 --dry-run
+		`)
+)
+
+type nodeUpgradeFlags struct {
+	parent            *cmdUpgradeFlags
+	kubeletVersionStr string
+	dryRun            bool
+}
+
+// NewCmdNode returns the cobra command for `kubeadm upgrade node`
+func NewCmdNode(parentFlags *cmdUpgradeFlags) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "node",
+		Short: "Upgrade commands for a node in the cluster. Currently only supports upgrading the configuration, not the kubelet itself.",
+		RunE:  cmdutil.SubCmdRunE("node"),
+	}
+	cmd.AddCommand(NewCmdUpgradeNodeConfig(parentFlags))
+	return cmd
+}
+
+// NewCmdUpgradeNodeConfig returns the cobra.Command for downloading the new/upgrading the kubelet configuration from the kubelet-config-1.X
+// ConfigMap in the cluster
+func NewCmdUpgradeNodeConfig(parentFlags *cmdUpgradeFlags) *cobra.Command {
+	flags := &nodeUpgradeFlags{
+		parent:            parentFlags,
+		kubeletVersionStr: "",
+	}
+
+	cmd := &cobra.Command{
+		Use:     "config",
+		Short:   "Downloads the kubelet configuration from the cluster ConfigMap kubelet-config-1.X, where X is the minor version of the kubelet.",
+		Long:    upgradeNodeConfigLongDesc,
+		Example: upgradeNodeConfigExample,
+		Run: func(cmd *cobra.Command, args []string) {
+			err := RunUpgradeNodeConfig(flags)
+			kubeadmutil.CheckErr(err)
+		},
+	}
+
+	// TODO: Unify the registration of common flags
+	cmd.Flags().BoolVar(&flags.dryRun, "dry-run", flags.dryRun, "Do not change any state, just output the actions that would be performed.")
+	cmd.Flags().StringVar(&flags.kubeletVersionStr, "kubelet-version", flags.kubeletVersionStr, "The *desired* version for the kubelet after the upgrade.")
+	return cmd
+}
+
+// RunUpgradeNodeConfig is executed when `kubeadm upgrade node config` runs.
+func RunUpgradeNodeConfig(flags *nodeUpgradeFlags) error {
+	if len(flags.kubeletVersionStr) == 0 {
+		return fmt.Errorf("The --kubelet-version argument is required")
+	}
+
+	// Set up the kubelet directory to use. If dry-running, use a fake directory
+	kubeletDir, err := getKubeletDir(flags.dryRun)
+	if err != nil {
+		return err
+	}
+
+	// Set the default for the kubeconfig path if the user didn't override with the flags
+	// TODO: Be smarter about this and be able to load multiple kubeconfig files in different orders of precedence
+	if flags.parent.kubeConfigPath == "" {
+		flags.parent.kubeConfigPath = constants.GetKubeletKubeConfigPath()
+	}
+
+	client, err := getClient(flags.parent.kubeConfigPath, flags.dryRun)
+	if err != nil {
+		return fmt.Errorf("couldn't create a Kubernetes client from file %q: %v", flags.parent.kubeConfigPath, err)
+	}
+
+	// Parse the desired kubelet version
+	kubeletVersion, err := version.ParseSemantic(flags.kubeletVersionStr)
+	if err != nil {
+		return err
+	}
+	// TODO: Checkpoint the current configuration first so that if something goes wrong it can be recovered
+	if err := kubeletphase.DownloadConfig(client, kubeletVersion, kubeletDir); err != nil {
+		return err
+	}
+
+	// If we're dry-running, print the generated manifests, otherwise do nothing
+	if err := printFilesIfDryRunning(flags.dryRun, kubeletDir); err != nil {
+		return fmt.Errorf("error printing files on dryrun: %v", err)
+	}
+
+	fmt.Println("[upgrade] The configuration for this node was successfully updated!")
+	fmt.Println("[upgrade] Now you should go ahead and upgrade the kubelet package using your package manager.")
+	return nil
+}
+
+// getKubeletDir gets the kubelet directory based on whether the user is dry-running this command or not.
+func getKubeletDir(dryRun bool) (string, error) {
+	if dryRun {
+		dryRunDir, err := ioutil.TempDir("", "kubeadm-init-dryrun")
+		if err != nil {
+			return "", fmt.Errorf("couldn't create a temporary directory: %v", err)
+		}
+		return dryRunDir, nil
+	}
+	return constants.KubeletRunDirectory, nil
+}
+
+// printFilesIfDryRunning prints the Static Pod manifests to stdout and informs about the temporary directory to go and lookup
+func printFilesIfDryRunning(dryRun bool, kubeletDir string) error {
+	if !dryRun {
+		return nil
+	}
+
+	// Print the contents of the upgraded file and pretend like they were in kubeadmconstants.KubeletRunDirectory
+	fileToPrint := dryrunutil.FileToPrint{
+		RealPath:  filepath.Join(kubeletDir, constants.KubeletConfigurationFileName),
+		PrintPath: filepath.Join(constants.KubeletRunDirectory, constants.KubeletConfigurationFileName),
+	}
+	return dryrunutil.PrintDryRunFiles([]dryrunutil.FileToPrint{fileToPrint}, os.Stdout)
+}

cmd/kubeadm/app/cmd/upgrade/upgrade.go

@@ -74,6 +74,7 @@ func NewCmdUpgrade(out io.Writer) *cobra.Command {
 	cmd.AddCommand(NewCmdApply(flags))
 	cmd.AddCommand(NewCmdPlan(flags))
 	cmd.AddCommand(NewCmdDiff(flags))
+	cmd.AddCommand(NewCmdNode(flags))
 
 	return cmd
 }

cmd/kubeadm/app/constants/constants.go

@@ -205,8 +205,26 @@ const (
 	// KubeletBaseConfigMapRolePrefix defines the base kubelet configuration ConfigMap.
 	KubeletBaseConfigMapRolePrefix = "kubeadm:kubelet-config-"
 
-	// KubeletConfigurationFile specifies the file name on the node which stores initial remote configuration of kubelet
+	// KubeletRunDirectory specifies the directory where the kubelet runtime information is stored.
-	KubeletConfigurationFile = "/var/lib/kubelet/config.yaml"
+	// TODO: Make hard-coded "/var/lib/kubelet" strings reference this constant.
+	KubeletRunDirectory = "/var/lib/kubelet"
+
+	// KubeletConfigurationFileName specifies the file name on the node which stores initial remote configuration of kubelet
+	// This file should exist under KubeletRunDirectory
+	KubeletConfigurationFileName = "config.yaml"
+
+	// DynamicKubeletConfigurationDirectoryName specifies the directory which stores the dynamic configuration checkpoints for the kubelet
+	// This directory should exist under KubeletRunDirectory
+	DynamicKubeletConfigurationDirectoryName = "dynamic-config"
+
+	// KubeletEnvFileName is a file "kubeadm init" writes at runtime. Using that interface, kubeadm can customize certain
+	// kubelet flags conditionally based on the environment at runtime. Also, parameters given to the configuration file
+	// might be passed through this file. "kubeadm init" writes one variable, with the name ${KubeletEnvFileVariableName}.
+	// This file should exist under KubeletRunDirectory
+	KubeletEnvFileName = "kubeadm-flags.env"
+
+	// KubeletEnvFileVariableName specifies the shell script variable name "kubeadm init" should write a value to in KubeletEnvFile
+	KubeletEnvFileVariableName = "KUBELET_KUBEADM_ARGS"
 
 	// MinExternalEtcdVersion indicates minimum external etcd version which kubeadm supports
 	MinExternalEtcdVersion = "3.2.17"
@@ -266,14 +284,6 @@ const (
 	// TODO: Import this constant from a consts only package, that does not pull any further dependencies.
 	LeaseEndpointReconcilerType = "lease"
 
-	// KubeletEnvFile is a file "kubeadm init" writes at runtime. Using that interface, kubeadm can customize certain
-	// kubelet flags conditionally based on the environment at runtime. Also, parameters given to the configuration file
-	// might be passed through this file. "kubeadm init" writes one variable, with the name ${KubeletEnvFileVariableName}.
-	KubeletEnvFile = "/var/lib/kubelet/kubeadm-flags.env"
-
-	// KubeletEnvFileVariableName specifies the shell script variable name "kubeadm init" should write a value to in KubeletEnvFile
-	KubeletEnvFileVariableName = "KUBELET_KUBEADM_ARGS"
-
 	// KubeDNSVersion is the version of kube-dns to be deployed if it is used
 	KubeDNSVersion = "1.14.10"
 
@@ -350,6 +360,16 @@ func GetAdminKubeConfigPath() string {
 	return filepath.Join(KubernetesDir, AdminKubeConfigFileName)
 }
 
+// GetBootstrapKubeletKubeConfigPath returns the location on the disk where bootstrap kubelet kubeconfig is located by default
+func GetBootstrapKubeletKubeConfigPath() string {
+	return filepath.Join(KubernetesDir, KubeletBootstrapKubeConfigFileName)
+}
+
+// GetKubeletKubeConfigPath returns the location on the disk where kubelet kubeconfig is located by default
+func GetKubeletKubeConfigPath() string {
+	return filepath.Join(KubernetesDir, KubeletKubeConfigFileName)
+}
+
 // AddSelfHostedPrefix adds the self-hosted- prefix to the component name
 func AddSelfHostedPrefix(componentName string) string {
 	return fmt.Sprintf("%s%s", SelfHostingPrefix, componentName)

cmd/kubeadm/app/constants/constants_test.go

@@ -50,6 +50,32 @@ func TestGetAdminKubeConfigPath(t *testing.T) {
 	}
 }
 
+func TestGetBootstrapKubeletKubeConfigPath(t *testing.T) {
+	expected := "/etc/kubernetes/bootstrap-kubelet.conf"
+	actual := GetBootstrapKubeletKubeConfigPath()
+
+	if actual != expected {
+		t.Errorf(
+			"failed GetBootstrapKubeletKubeConfigPath:\n\texpected: %s\n\t  actual: %s",
+			expected,
+			actual,
+		)
+	}
+}
+
+func TestGetKubeletKubeConfigPath(t *testing.T) {
+	expected := "/etc/kubernetes/kubelet.conf"
+	actual := GetKubeletKubeConfigPath()
+
+	if actual != expected {
+		t.Errorf(
+			"failed GetKubeletKubeConfigPath:\n\texpected: %s\n\t  actual: %s",
+			expected,
+			actual,
+		)
+	}
+}
+
 func TestGetStaticPodFilepath(t *testing.T) {
 	var tests = []struct {
 		componentName, manifestsDir, expected string

cmd/kubeadm/app/phases/kubelet/BUILD

@@ -13,9 +13,9 @@ go_library(
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/apis/kubeadm/v1alpha2:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
+        "//cmd/kubeadm/app/features:go_default_library",
         "//cmd/kubeadm/app/util:go_default_library",
         "//cmd/kubeadm/app/util/apiclient:go_default_library",
-        "//cmd/kubeadm/app/util/kubeconfig:go_default_library",
         "//pkg/apis/rbac/v1:go_default_library",
         "//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library",
         "//pkg/kubelet/apis/kubeletconfig/v1beta1:go_default_library",
@@ -44,13 +44,13 @@ go_test(
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//pkg/kubelet/apis:go_default_library",
         "//pkg/kubelet/apis/kubeletconfig/v1beta1:go_default_library",
-        "//pkg/util/node:go_default_library",
         "//pkg/util/version:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
         "//vendor/k8s.io/client-go/testing:go_default_library",
+        "//vendor/k8s.io/utils/exec:go_default_library",
     ],
 )
 

cmd/kubeadm/app/phases/kubelet/config.go

@@ -19,6 +19,7 @@ package kubelet
 import (
 	"fmt"
 	"io/ioutil"
+	"path/filepath"
 
 	"k8s.io/api/core/v1"
 	rbac "k8s.io/api/rbac/v1"
@@ -28,7 +29,6 @@ import (
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
-	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
 	rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1"
 	kubeletconfigscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
 	kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
@@ -37,13 +37,13 @@ import (
 
 // WriteConfigToDisk writes the kubelet config object down to a file
 // Used at "kubeadm init" and "kubeadm upgrade" time
-func WriteConfigToDisk(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration) error {
+func WriteConfigToDisk(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration, kubeletDir string) error {
 
 	kubeletBytes, err := getConfigBytes(kubeletConfig)
 	if err != nil {
 		return err
 	}
-	return writeConfigBytesToDisk(kubeletBytes)
+	return writeConfigBytesToDisk(kubeletBytes, kubeletDir)
 }
 
 // CreateConfigMap creates a ConfigMap with the generic kubelet configuration.
@@ -120,7 +120,7 @@ func createConfigMapRBACRules(client clientset.Interface, k8sVersion *version.Ve
 
 // DownloadConfig downloads the kubelet configuration from a ConfigMap and writes it to disk.
 // Used at "kubeadm join" time
-func DownloadConfig(kubeletKubeConfig string, kubeletVersion *version.Version) error {
+func DownloadConfig(client clientset.Interface, kubeletVersion *version.Version, kubeletDir string) error {
 
 	// Download the ConfigMap from the cluster based on what version the kubelet is
 	configMapName := configMapName(kubeletVersion)
@@ -128,17 +128,12 @@ func DownloadConfig(kubeletKubeConfig string, kubeletVersion *version.Version) e
 	fmt.Printf("[kubelet] Downloading configuration for the kubelet from the %q ConfigMap in the %s namespace\n",
 		configMapName, metav1.NamespaceSystem)
 
-	client, err := kubeconfigutil.ClientSetFromFile(kubeletKubeConfig)
-	if err != nil {
-		return fmt.Errorf("couldn't create client from kubeconfig file %q", kubeletKubeConfig)
-	}
-
 	kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(configMapName, metav1.GetOptions{})
 	if err != nil {
 		return err
 	}
 
-	return writeConfigBytesToDisk([]byte(kubeletCfg.Data[kubeadmconstants.KubeletBaseConfigurationConfigMapKey]))
+	return writeConfigBytesToDisk([]byte(kubeletCfg.Data[kubeadmconstants.KubeletBaseConfigurationConfigMapKey]), kubeletDir)
 }
 
 // configMapName returns the right ConfigMap name for the right branch of k8s
@@ -162,11 +157,12 @@ func getConfigBytes(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration) ([
 }
 
 // writeConfigBytesToDisk writes a byte slice down to disk at the specific location of the kubelet config file
-func writeConfigBytesToDisk(b []byte) error {
+func writeConfigBytesToDisk(b []byte, kubeletDir string) error {
-	fmt.Printf("[kubelet] Writing kubelet configuration to file %q\n", kubeadmconstants.KubeletConfigurationFile)
+	configFile := filepath.Join(kubeletDir, kubeadmconstants.KubeletConfigurationFileName)
+	fmt.Printf("[kubelet] Writing kubelet configuration to file %q\n", configFile)
 
-	if err := ioutil.WriteFile(kubeadmconstants.KubeletConfigurationFile, b, 0644); err != nil {
+	if err := ioutil.WriteFile(configFile, b, 0644); err != nil {
-		return fmt.Errorf("failed to write kubelet configuration to the file %q: %v", kubeadmconstants.KubeletConfigurationFile, err)
+		return fmt.Errorf("failed to write kubelet configuration to the file %q: %v", configFile, err)
 	}
 	return nil
 }

cmd/kubeadm/app/phases/kubelet/flags.go

@@ -27,34 +27,52 @@ import (
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	nodeutil "k8s.io/kubernetes/pkg/util/node"
 	"k8s.io/kubernetes/pkg/util/procfs"
 	utilsexec "k8s.io/utils/exec"
 )
 
+type kubeletFlagsOpts struct {
+	nodeRegOpts              *kubeadmapi.NodeRegistrationOptions
+	featureGates             map[string]bool
+	registerTaintsUsingFlags bool
+	execer                   utilsexec.Interface
+	pidOfFunc                func(string) ([]int, error)
+	defaultHostname          string
+}
+
 // WriteKubeletDynamicEnvFile writes a environment file with dynamic flags to the kubelet.
 // Used at "kubeadm init" and "kubeadm join" time.
-func WriteKubeletDynamicEnvFile(nodeRegOpts *kubeadmapi.NodeRegistrationOptions, registerTaintsUsingFlags bool) error {
+func WriteKubeletDynamicEnvFile(nodeRegOpts *kubeadmapi.NodeRegistrationOptions, featureGates map[string]bool, registerTaintsUsingFlags bool, kubeletDir string) error {
 
-	argList := kubeadmutil.BuildArgumentListFromMap(buildKubeletArgMap(nodeRegOpts, registerTaintsUsingFlags), nodeRegOpts.ExtraArgs)
+	flagOpts := kubeletFlagsOpts{
+		nodeRegOpts:              nodeRegOpts,
+		featureGates:             featureGates,
+		registerTaintsUsingFlags: registerTaintsUsingFlags,
+		execer:          utilsexec.New(),
+		pidOfFunc:       procfs.PidOf,
+		defaultHostname: nodeutil.GetHostname(""),
+	}
+	stringMap := buildKubeletArgMap(flagOpts)
+	argList := kubeadmutil.BuildArgumentListFromMap(stringMap, nodeRegOpts.KubeletExtraArgs)
 	envFileContent := fmt.Sprintf("%s=%s\n", constants.KubeletEnvFileVariableName, strings.Join(argList, " "))
 
-	return writeKubeletFlagBytesToDisk([]byte(envFileContent))
+	return writeKubeletFlagBytesToDisk([]byte(envFileContent), kubeletDir)
 }
 
 // buildKubeletArgMap takes a MasterConfiguration object and builds based on that a string-string map with flags
 // that should be given to the local kubelet daemon.
-func buildKubeletArgMap(nodeRegOpts *kubeadmapi.NodeRegistrationOptions, registerTaintsUsingFlags bool) map[string]string {
+func buildKubeletArgMap(opts kubeletFlagsOpts) map[string]string {
 	kubeletFlags := map[string]string{}
 
-	if nodeRegOpts.CRISocket == kubeadmapiv1alpha2.DefaultCRISocket {
+	if opts.nodeRegOpts.CRISocket == kubeadmapiv1alpha2.DefaultCRISocket {
 		// These flags should only be set when running docker
 		kubeletFlags["network-plugin"] = "cni"
 		kubeletFlags["cni-conf-dir"] = "/etc/cni/net.d"
 		kubeletFlags["cni-bin-dir"] = "/opt/cni/bin"
-		execer := utilsexec.New()
+		driver, err := kubeadmutil.GetCgroupDriverDocker(opts.execer)
-		driver, err := kubeadmutil.GetCgroupDriverDocker(execer)
 		if err != nil {
 			glog.Warningf("cannot automatically assign a '--cgroup-driver' value when starting the Kubelet: %v\n", err)
 		} else {
@@ -62,27 +80,33 @@ func buildKubeletArgMap(nodeRegOpts *kubeadmapi.NodeRegistrationOptions, registe
 		}
 	} else {
 		kubeletFlags["container-runtime"] = "remote"
-		kubeletFlags["container-runtime-endpoint"] = nodeRegOpts.CRISocket
+		kubeletFlags["container-runtime-endpoint"] = opts.nodeRegOpts.CRISocket
 	}
 
-	if registerTaintsUsingFlags && nodeRegOpts.Taints != nil && len(nodeRegOpts.Taints) > 0 {
+	if opts.registerTaintsUsingFlags && opts.nodeRegOpts.Taints != nil && len(opts.nodeRegOpts.Taints) > 0 {
 		taintStrs := []string{}
-		for _, taint := range nodeRegOpts.Taints {
+		for _, taint := range opts.nodeRegOpts.Taints {
 			taintStrs = append(taintStrs, taint.ToString())
 		}
 
 		kubeletFlags["register-with-taints"] = strings.Join(taintStrs, ",")
 	}
 
-	if pids, _ := procfs.PidOf("systemd-resolved"); len(pids) > 0 {
+	if pids, _ := opts.pidOfFunc("systemd-resolved"); len(pids) > 0 {
 		// procfs.PidOf only returns an error if the regex is empty or doesn't compile, so we can ignore it
 		kubeletFlags["resolv-conf"] = "/run/systemd/resolve/resolv.conf"
 	}
 
 	// Make sure the node name we're passed will work with Kubelet
-	if nodeRegOpts.Name != "" && nodeRegOpts.Name != nodeutil.GetHostname("") {
+	if opts.nodeRegOpts.Name != "" && opts.nodeRegOpts.Name != opts.defaultHostname {
-		glog.V(1).Info("setting kubelet hostname-override to %q", nodeRegOpts.Name)
+		glog.V(1).Info("setting kubelet hostname-override to %q", opts.nodeRegOpts.Name)
-		kubeletFlags["hostname-override"] = nodeRegOpts.Name
+		kubeletFlags["hostname-override"] = opts.nodeRegOpts.Name
+	}
+
+	// If the user enabled Dynamic Kubelet Configuration (which is disabled by default), set the directory
+	// in the CLI flags so that the feature actually gets enabled
+	if features.Enabled(opts.featureGates, features.DynamicKubeletConfig) {
+		kubeletFlags["dynamic-config-dir"] = filepath.Join(constants.KubeletRunDirectory, constants.DynamicKubeletConfigurationDirectoryName)
 	}
 
 	// TODO: Conditionally set `--cgroup-driver` to either `systemd` or `cgroupfs` for CRI other than Docker
@@ -91,15 +115,16 @@ func buildKubeletArgMap(nodeRegOpts *kubeadmapi.NodeRegistrationOptions, registe
 }
 
 // writeKubeletFlagBytesToDisk writes a byte slice down to disk at the specific location of the kubelet flag overrides file
-func writeKubeletFlagBytesToDisk(b []byte) error {
+func writeKubeletFlagBytesToDisk(b []byte, kubeletDir string) error {
-	fmt.Printf("[kubelet] Writing kubelet environment file with flags to file %q\n", constants.KubeletEnvFile)
+	kubeletEnvFilePath := filepath.Join(kubeletDir, constants.KubeletEnvFileName)
+	fmt.Printf("[kubelet] Writing kubelet environment file with flags to file %q\n", kubeletEnvFilePath)
 
 	// creates target folder if not already exists
-	if err := os.MkdirAll(filepath.Dir(constants.KubeletEnvFile), 0700); err != nil {
+	if err := os.MkdirAll(kubeletDir, 0700); err != nil {
-		return fmt.Errorf("failed to create directory %q: %v", filepath.Dir(constants.KubeletEnvFile), err)
+		return fmt.Errorf("failed to create directory %q: %v", kubeletDir, err)
 	}
-	if err := ioutil.WriteFile(constants.KubeletEnvFile, b, 0644); err != nil {
+	if err := ioutil.WriteFile(kubeletEnvFilePath, b, 0644); err != nil {
-		return fmt.Errorf("failed to write kubelet configuration to the file %q: %v", constants.KubeletEnvFile, err)
+		return fmt.Errorf("failed to write kubelet configuration to the file %q: %v", kubeletEnvFilePath, err)
 	}
 	return nil
 }

cmd/kubeadm/app/phases/kubelet/flags_test.go

@@ -17,45 +17,260 @@ limitations under the License.
 package kubelet
 
 import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"reflect"
+	"strings"
 	"testing"
 
+	"k8s.io/api/core/v1"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
-	nodeutil "k8s.io/kubernetes/pkg/util/node"
+	"k8s.io/utils/exec"
 )
 
+type fakeCmd struct {
+	b   []byte
+	err error
+}
+
+func (f fakeCmd) Run() error                      { return f.err }
+func (f fakeCmd) CombinedOutput() ([]byte, error) { return f.b, f.err }
+func (f fakeCmd) Output() ([]byte, error)         { return f.b, f.err }
+func (f fakeCmd) SetDir(dir string)               {}
+func (f fakeCmd) SetStdin(in io.Reader)           {}
+func (f fakeCmd) SetStdout(out io.Writer)         {}
+func (f fakeCmd) SetStderr(out io.Writer)         {}
+func (f fakeCmd) Stop()                           {}
+
+type fakeExecer struct {
+	ioMap map[string]fakeCmd
+}
+
+func (f fakeExecer) Command(cmd string, args ...string) exec.Cmd {
+	cmds := []string{cmd}
+	cmds = append(cmds, args...)
+	return f.ioMap[strings.Join(cmds, " ")]
+}
+func (f fakeExecer) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd {
+	return f.Command(cmd, args...)
+}
+func (f fakeExecer) LookPath(file string) (string, error) { return "", errors.New("unknown binary") }
+
+var (
+	systemdCgroupExecer = fakeExecer{
+		ioMap: map[string]fakeCmd{
+			"docker info": {
+				b: []byte(`Cgroup Driver: systemd`),
+			},
+		},
+	}
+
+	cgroupfsCgroupExecer = fakeExecer{
+		ioMap: map[string]fakeCmd{
+			"docker info": {
+				b: []byte(`Cgroup Driver: cgroupfs`),
+			},
+		},
+	}
+
+	errCgroupExecer = fakeExecer{
+		ioMap: map[string]fakeCmd{
+			"docker info": {
+				err: fmt.Errorf("no such binary: docker"),
+			},
+		},
+	}
+)
+
+func binaryRunningPidOfFunc(_ string) ([]int, error) {
+	return []int{1, 2, 3}, nil
+}
+
+func binaryNotRunningPidOfFunc(_ string) ([]int, error) {
+	return []int{}, nil
+}
+
 func TestBuildKubeletArgMap(t *testing.T) {
 
 	tests := []struct {
-		name             string
+		name     string
-		hostname         string
+		opts     kubeletFlagsOpts
-		expectedHostname string
+		expected map[string]string
 	}{
 		{
-			name:             "manually set to current hostname",
+			name: "the simplest case",
-			hostname:         nodeutil.GetHostname(""),
+			opts: kubeletFlagsOpts{
-			expectedHostname: "",
+				nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{
+					CRISocket: "/var/run/dockershim.sock",
+					Name:      "foo",
+					Taints: []v1.Taint{ // This should be ignored as registerTaintsUsingFlags is false
+						{
+							Key:    "foo",
+							Value:  "bar",
+							Effect: "baz",
+						},
+					},
+				},
+				execer:          errCgroupExecer,
+				pidOfFunc:       binaryNotRunningPidOfFunc,
+				defaultHostname: "foo",
+			},
+			expected: map[string]string{
+				"network-plugin": "cni",
+				"cni-conf-dir":   "/etc/cni/net.d",
+				"cni-bin-dir":    "/opt/cni/bin",
+			},
 		},
 		{
-			name:             "unset hostname",
+			name: "nodeRegOpts.Name != default hostname",
-			hostname:         "",
+			opts: kubeletFlagsOpts{
-			expectedHostname: "",
+				nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{
+					CRISocket: "/var/run/dockershim.sock",
+					Name:      "override-name",
+				},
+				execer:          errCgroupExecer,
+				pidOfFunc:       binaryNotRunningPidOfFunc,
+				defaultHostname: "default",
+			},
+			expected: map[string]string{
+				"network-plugin":    "cni",
+				"cni-conf-dir":      "/etc/cni/net.d",
+				"cni-bin-dir":       "/opt/cni/bin",
+				"hostname-override": "override-name",
+			},
 		},
 		{
-			name:             "override hostname",
+			name: "systemd cgroup driver",
-			hostname:         "my-node",
+			opts: kubeletFlagsOpts{
-			expectedHostname: "my-node",
+				nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{
+					CRISocket: "/var/run/dockershim.sock",
+					Name:      "foo",
+				},
+				execer:          systemdCgroupExecer,
+				pidOfFunc:       binaryNotRunningPidOfFunc,
+				defaultHostname: "foo",
+			},
+			expected: map[string]string{
+				"network-plugin": "cni",
+				"cni-conf-dir":   "/etc/cni/net.d",
+				"cni-bin-dir":    "/opt/cni/bin",
+				"cgroup-driver":  "systemd",
+			},
+		},
+		{
+			name: "cgroupfs cgroup driver",
+			opts: kubeletFlagsOpts{
+				nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{
+					CRISocket: "/var/run/dockershim.sock",
+					Name:      "foo",
+				},
+				execer:          cgroupfsCgroupExecer,
+				pidOfFunc:       binaryNotRunningPidOfFunc,
+				defaultHostname: "foo",
+			},
+			expected: map[string]string{
+				"network-plugin": "cni",
+				"cni-conf-dir":   "/etc/cni/net.d",
+				"cni-bin-dir":    "/opt/cni/bin",
+				"cgroup-driver":  "cgroupfs",
+			},
+		},
+		{
+			name: "external CRI runtime",
+			opts: kubeletFlagsOpts{
+				nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{
+					CRISocket: "/var/run/containerd.sock",
+					Name:      "foo",
+				},
+				execer:          cgroupfsCgroupExecer,
+				pidOfFunc:       binaryNotRunningPidOfFunc,
+				defaultHostname: "foo",
+			},
+			expected: map[string]string{
+				"container-runtime":          "remote",
+				"container-runtime-endpoint": "/var/run/containerd.sock",
+			},
+		},
+		{
+			name: "register with taints",
+			opts: kubeletFlagsOpts{
+				nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{
+					CRISocket: "/var/run/containerd.sock",
+					Name:      "foo",
+					Taints: []v1.Taint{
+						{
+							Key:    "foo",
+							Value:  "bar",
+							Effect: "baz",
+						},
+						{
+							Key:    "key",
+							Value:  "val",
+							Effect: "eff",
+						},
+					},
+				},
+				registerTaintsUsingFlags: true,
+				execer:          cgroupfsCgroupExecer,
+				pidOfFunc:       binaryNotRunningPidOfFunc,
+				defaultHostname: "foo",
+			},
+			expected: map[string]string{
+				"container-runtime":          "remote",
+				"container-runtime-endpoint": "/var/run/containerd.sock",
+				"register-with-taints":       "foo=bar:baz,key=val:eff",
+			},
+		},
+		{
+			name: "systemd-resolved running",
+			opts: kubeletFlagsOpts{
+				nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{
+					CRISocket: "/var/run/containerd.sock",
+					Name:      "foo",
+				},
+				execer:          cgroupfsCgroupExecer,
+				pidOfFunc:       binaryRunningPidOfFunc,
+				defaultHostname: "foo",
+			},
+			expected: map[string]string{
+				"container-runtime":          "remote",
+				"container-runtime-endpoint": "/var/run/containerd.sock",
+				"resolv-conf":                "/run/systemd/resolve/resolv.conf",
+			},
+		},
+		{
+			name: "dynamic kubelet config enabled",
+			opts: kubeletFlagsOpts{
+				nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{
+					CRISocket: "/var/run/containerd.sock",
+					Name:      "foo",
+				},
+				featureGates: map[string]bool{
+					"DynamicKubeletConfig": true,
+				},
+				execer:          cgroupfsCgroupExecer,
+				pidOfFunc:       binaryNotRunningPidOfFunc,
+				defaultHostname: "foo",
+			},
+			expected: map[string]string{
+				"container-runtime":          "remote",
+				"container-runtime-endpoint": "/var/run/containerd.sock",
+				"dynamic-config-dir":         "/var/lib/kubelet/dynamic-config",
+			},
 		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			opts := &kubeadmapi.NodeRegistrationOptions{
+			actual := buildKubeletArgMap(test.opts)
-				Name: test.hostname,
+			if !reflect.DeepEqual(actual, test.expected) {
-			}
+				t.Errorf(
-
+					"failed buildKubeletArgMap:\n\texpected: %v\n\t  actual: %v",
-			m := buildKubeletArgMap(opts, false)
+					test.expected,
-			if m["hostname-override"] != test.expectedHostname {
+					actual,
-				t.Errorf("expected hostname %q, got %q", test.expectedHostname, m["hostname-override"])
+				)
 			}
 		})
 	}

cmd/kubeadm/app/phases/markmaster/markmaster.go

@@ -31,7 +31,11 @@ func MarkMaster(client clientset.Interface, masterName string, taints []v1.Taint
 	fmt.Printf("[markmaster] Marking the node %s as master by adding the label \"%s=''\"\n", masterName, constants.LabelNodeRoleMaster)
 
 	if taints != nil && len(taints) > 0 {
-		fmt.Printf("[markmaster] Marking the node %s as master by adding the taints %v\n", masterName, taints)
+		taintStrs := []string{}
+		for _, taint := range taints {
+			taintStrs = append(taintStrs, taint.ToString())
+		}
+		fmt.Printf("[markmaster] Marking the node %s as master by adding the taints %v\n", masterName, taintStrs)
 	}
 
 	return apiclient.PatchNode(client, masterName, func(n *v1.Node) {

cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go

@@ -42,6 +42,8 @@ func UploadConfiguration(cfg *kubeadmapi.MasterConfiguration, client clientset.I
 
 	// Removes sensitive info from the data that will be stored in the config map
 	externalcfg.BootstrapTokens = nil
+	// Clear the NodeRegistration object.
+	externalcfg.NodeRegistration = kubeadmapiv1alpha2.NodeRegistrationOptions{}
 
 	cfgYaml, err := util.MarshalToYamlForCodecs(externalcfg, kubeadmapiv1alpha2.SchemeGroupVersion, scheme.Codecs)
 	if err != nil {

cmd/kubeadm/app/preflight/checks.go

@@ -1063,15 +1063,41 @@ func TryStartKubelet(ignorePreflightErrors sets.String) {
 	// If we notice that the kubelet service is inactive, try to start it
 	initSystem, err := initsystem.GetInitSystem()
 	if err != nil {
-		fmt.Println("[preflight] no supported init system detected, won't ensure kubelet is running.")
+		fmt.Println("[preflight] no supported init system detected, won't make sure the kubelet is running properly.")
-	} else if initSystem.ServiceExists("kubelet") {
+		return
+	}
 
-		fmt.Println("[preflight] Activating the kubelet service")
+	if !initSystem.ServiceExists("kubelet") {
-		// This runs "systemctl daemon-reload && systemctl restart kubelet"
+		fmt.Println("[preflight] couldn't detect a kubelet service, can't make sure the kubelet is running properly.")
-		if err := initSystem.ServiceRestart("kubelet"); err != nil {
+	}
-			glog.Warningf("[preflight] unable to start the kubelet service: [%v]\n", err)
+
-			glog.Warningf("[preflight] please ensure kubelet is running manually.")
+	fmt.Println("[preflight] Activating the kubelet service")
-		}
+	// This runs "systemctl daemon-reload && systemctl restart kubelet"
+	if err := initSystem.ServiceRestart("kubelet"); err != nil {
+		fmt.Printf("[preflight] WARNING: unable to start the kubelet service: [%v]\n", err)
+		fmt.Printf("[preflight] please ensure kubelet is reloaded and running manually.\n")
+	}
+}
+
+// TryStopKubelet attempts to bring down the kubelet service momentarily
+func TryStopKubelet(ignorePreflightErrors sets.String) {
+	if setHasItemOrAll(ignorePreflightErrors, "StopKubelet") {
+		return
+	}
+	// If we notice that the kubelet service is inactive, try to start it
+	initSystem, err := initsystem.GetInitSystem()
+	if err != nil {
+		fmt.Println("[preflight] no supported init system detected, won't make sure the kubelet not running for a short period of time while setting up configuration for it.")
+		return
+	}
+
+	if !initSystem.ServiceExists("kubelet") {
+		fmt.Println("[preflight] couldn't detect a kubelet service, can't make sure the kubelet not running for a short period of time while setting up configuration for it.")
+	}
+
+	// This runs "systemctl daemon-reload && systemctl stop kubelet"
+	if err := initSystem.ServiceStop("kubelet"); err != nil {
+		fmt.Printf("[preflight] WARNING: unable to stop the kubelet service momentarily: [%v]\n", err)
 	}
 }
 

cmd/kubeadm/app/util/apiclient/dryrunclient.go

@@ -55,6 +55,18 @@ type DryRunClientOptions struct {
 	PrintGETAndLIST bool
 }
 
+// GetDefaultDryRunClientOptions returns the default DryRunClientOptions values
+func GetDefaultDryRunClientOptions(drg DryRunGetter, w io.Writer) DryRunClientOptions {
+	return DryRunClientOptions{
+		Writer:          w,
+		Getter:          drg,
+		PrependReactors: []core.Reactor{},
+		AppendReactors:  []core.Reactor{},
+		MarshalFunc:     DefaultMarshalFunc,
+		PrintGETAndLIST: false,
+	}
+}
+
 // actionWithName is the generic interface for an action that has a name associated with it
 // This just makes it easier to catch all actions that has a name; instead of hard-coding all request that has it associated
 type actionWithName interface {
@@ -71,14 +83,7 @@ type actionWithObject interface {
 
 // NewDryRunClient is a wrapper for NewDryRunClientWithOpts using some default values
 func NewDryRunClient(drg DryRunGetter, w io.Writer) clientset.Interface {
-	return NewDryRunClientWithOpts(DryRunClientOptions{
+	return NewDryRunClientWithOpts(GetDefaultDryRunClientOptions(drg, w))
-		Writer:          w,
-		Getter:          drg,
-		PrependReactors: []core.Reactor{},
-		AppendReactors:  []core.Reactor{},
-		MarshalFunc:     DefaultMarshalFunc,
-		PrintGETAndLIST: false,
-	})
 }
 
 // NewDryRunClientWithOpts returns a clientset.Interface that can be used normally for talking to the Kubernetes API.

cmd/kubeadm/app/util/apiclient/init_dryrun.go

@@ -61,6 +61,7 @@ func (idr *InitDryRunGetter) HandleGetAction(action core.GetAction) (bool, runti
 		idr.handleGetNode,
 		idr.handleSystemNodesClusterRoleBinding,
 		idr.handleGetBootstrapToken,
+		idr.handleGetKubeDNSConfigMap,
 	}
 	for _, f := range funcs {
 		handled, obj, err := f(action)
@@ -157,3 +158,20 @@ func (idr *InitDryRunGetter) handleGetBootstrapToken(action core.GetAction) (boo
 	// We can safely return a NotFound error here as the code will just proceed normally and create the Bootstrap Token
 	return true, nil, apierrors.NewNotFound(action.GetResource().GroupResource(), "secret not found")
 }
+
+// handleGetKubeDNSConfigMap handles the case where kubeadm init will try to read the kube-dns ConfigMap in the cluster
+// in order to transform information there to core-dns configuration. We can safely return an empty configmap here
+func (idr *InitDryRunGetter) handleGetKubeDNSConfigMap(action core.GetAction) (bool, runtime.Object, error) {
+	if !strings.HasPrefix(action.GetName(), "kube-dns") || action.GetNamespace() != metav1.NamespaceSystem || action.GetResource().Resource != "configmaps" {
+		// We can't handle this event
+		return false, nil, nil
+	}
+	// We can safely return an empty configmap here, as we don't have any kube-dns specific config to convert to coredns config
+	return true, &v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "kube-dns",
+			Namespace: metav1.NamespaceSystem,
+		},
+		Data: map[string]string{},
+	}, nil
+}

cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml

@@ -151,7 +151,7 @@ Networking:
   ServiceSubnet: 10.96.0.0/12
 NodeRegistration:
   CRISocket: /var/run/dockershim.sock
-  ExtraArgs: null
+  KubeletExtraArgs: null
   Name: master-1
   Taints:
   - effect: NoSchedule

cmd/kubeadm/app/util/config/testdata/conversion/node/internal.yaml

@@ -10,7 +10,7 @@ DiscoveryTokenUnsafeSkipCAVerification: true
 FeatureGates: null
 NodeRegistration:
   CRISocket: /var/run/dockershim.sock
-  ExtraArgs: null
+  KubeletExtraArgs: null
   Name: master-1
   Taints: null
 TLSBootstrapToken: abcdef.0123456789abcdef

docs/.generated_docs

@@ -46,9 +46,12 @@ docs/admin/kubeadm_alpha_phase_kubeconfig_kubelet.md
 docs/admin/kubeadm_alpha_phase_kubeconfig_scheduler.md
 docs/admin/kubeadm_alpha_phase_kubeconfig_user.md
 docs/admin/kubeadm_alpha_phase_kubelet.md
-docs/admin/kubeadm_alpha_phase_kubelet_enable-dynamic-config.md
+docs/admin/kubeadm_alpha_phase_kubelet_config.md
-docs/admin/kubeadm_alpha_phase_kubelet_upload-config.md
+docs/admin/kubeadm_alpha_phase_kubelet_config_download.md
-docs/admin/kubeadm_alpha_phase_kubelet_write-config-to-disk.md
+docs/admin/kubeadm_alpha_phase_kubelet_config_enable-dynamic.md
+docs/admin/kubeadm_alpha_phase_kubelet_config_upload.md
+docs/admin/kubeadm_alpha_phase_kubelet_config_write-to-disk.md
+docs/admin/kubeadm_alpha_phase_kubelet_write-env-file.md
 docs/admin/kubeadm_alpha_phase_mark-master.md
 docs/admin/kubeadm_alpha_phase_preflight.md
 docs/admin/kubeadm_alpha_phase_preflight_master.md
@@ -78,6 +81,8 @@ docs/admin/kubeadm_token_list.md
 docs/admin/kubeadm_upgrade.md
 docs/admin/kubeadm_upgrade_apply.md
 docs/admin/kubeadm_upgrade_diff.md
+docs/admin/kubeadm_upgrade_node.md
+docs/admin/kubeadm_upgrade_node_config.md
 docs/admin/kubeadm_upgrade_plan.md
 docs/admin/kubeadm_version.md
 docs/admin/kubelet.md
@@ -124,9 +129,12 @@ docs/man/man1/kubeadm-alpha-phase-kubeconfig-kubelet.1
 docs/man/man1/kubeadm-alpha-phase-kubeconfig-scheduler.1
 docs/man/man1/kubeadm-alpha-phase-kubeconfig-user.1
 docs/man/man1/kubeadm-alpha-phase-kubeconfig.1
-docs/man/man1/kubeadm-alpha-phase-kubelet-enable-dynamic-config.1
+docs/man/man1/kubeadm-alpha-phase-kubelet-config-download.1
-docs/man/man1/kubeadm-alpha-phase-kubelet-upload-config.1
+docs/man/man1/kubeadm-alpha-phase-kubelet-config-enable-dynamic.1
-docs/man/man1/kubeadm-alpha-phase-kubelet-write-config-to-disk.1
+docs/man/man1/kubeadm-alpha-phase-kubelet-config-upload.1
+docs/man/man1/kubeadm-alpha-phase-kubelet-config-write-to-disk.1
+docs/man/man1/kubeadm-alpha-phase-kubelet-config.1
+docs/man/man1/kubeadm-alpha-phase-kubelet-write-env-file.1
 docs/man/man1/kubeadm-alpha-phase-kubelet.1
 docs/man/man1/kubeadm-alpha-phase-mark-master.1
 docs/man/man1/kubeadm-alpha-phase-preflight-master.1
@@ -158,6 +166,8 @@ docs/man/man1/kubeadm-token-list.1
 docs/man/man1/kubeadm-token.1
 docs/man/man1/kubeadm-upgrade-apply.1
 docs/man/man1/kubeadm-upgrade-diff.1
+docs/man/man1/kubeadm-upgrade-node-config.1
+docs/man/man1/kubeadm-upgrade-node.1
 docs/man/man1/kubeadm-upgrade-plan.1
 docs/man/man1/kubeadm-upgrade.1
 docs/man/man1/kubeadm-version.1

docs/admin/kubeadm_upgrade_node.md

@@ -0,0 +1,3 @@
+This file is autogenerated, but we've stopped checking such files into the
+repository to reduce the need for rebases. Please run hack/generate-docs.sh to
+populate this file.

docs/admin/kubeadm_upgrade_node_config.md

@@ -0,0 +1,3 @@
+This file is autogenerated, but we've stopped checking such files into the
+repository to reduce the need for rebases. Please run hack/generate-docs.sh to
+populate this file.

docs/man/man1/kubeadm-alpha-phase-kubelet-config-download.1

@@ -0,0 +1,3 @@
+This file is autogenerated, but we've stopped checking such files into the
+repository to reduce the need for rebases. Please run hack/generate-docs.sh to
+populate this file.

docs/man/man1/kubeadm-alpha-phase-kubelet-config-enable-dynamic.1

@@ -0,0 +1,3 @@
+This file is autogenerated, but we've stopped checking such files into the
+repository to reduce the need for rebases. Please run hack/generate-docs.sh to
+populate this file.

docs/man/man1/kubeadm-alpha-phase-kubelet-config-upload.1

@@ -0,0 +1,3 @@
+This file is autogenerated, but we've stopped checking such files into the
+repository to reduce the need for rebases. Please run hack/generate-docs.sh to
+populate this file.

docs/man/man1/kubeadm-alpha-phase-kubelet-config-write-to-disk.1

@@ -0,0 +1,3 @@
+This file is autogenerated, but we've stopped checking such files into the
+repository to reduce the need for rebases. Please run hack/generate-docs.sh to
+populate this file.

docs/man/man1/kubeadm-alpha-phase-kubelet-config.1

@@ -0,0 +1,3 @@
+This file is autogenerated, but we've stopped checking such files into the
+repository to reduce the need for rebases. Please run hack/generate-docs.sh to
+populate this file.

docs/man/man1/kubeadm-alpha-phase-kubelet-write-env-file.1

@@ -0,0 +1,3 @@
+This file is autogenerated, but we've stopped checking such files into the
+repository to reduce the need for rebases. Please run hack/generate-docs.sh to
+populate this file.

docs/man/man1/kubeadm-upgrade-node-config.1

@@ -0,0 +1,3 @@
+This file is autogenerated, but we've stopped checking such files into the
+repository to reduce the need for rebases. Please run hack/generate-docs.sh to
+populate this file.

docs/man/man1/kubeadm-upgrade-node.1

@@ -0,0 +1,3 @@
+This file is autogenerated, but we've stopped checking such files into the
+repository to reduce the need for rebases. Please run hack/generate-docs.sh to
+populate this file.

pkg/util/initsystem/initsystem.go

@@ -44,15 +44,26 @@ type InitSystem interface {
 
 type SystemdInitSystem struct{}
 
+func (sysd SystemdInitSystem) reloadSystemd() error {
+	if err := exec.Command("systemctl", "daemon-reload").Run(); err != nil {
+		return fmt.Errorf("failed to reload systemd: %v", err)
+	}
+	return nil
+}
+
 func (sysd SystemdInitSystem) ServiceStart(service string) error {
+	// Before we try to start any service, make sure that systemd is ready
+	if err := sysd.reloadSystemd(); err != nil {
+		return err
+	}
 	args := []string{"start", service}
-	err := exec.Command("systemctl", args...).Run()
+	return exec.Command("systemctl", args...).Run()
-	return err
 }
 
 func (sysd SystemdInitSystem) ServiceRestart(service string) error {
-	if err := exec.Command("systemctl", "daemon-reload").Run(); err != nil {
+	// Before we try to restart any service, make sure that systemd is ready
-		return fmt.Errorf("failed to reload systemd: %v", err)
+	if err := sysd.reloadSystemd(); err != nil {
+		return err
 	}
 	args := []string{"restart", service}
 	return exec.Command("systemctl", args...).Run()
@@ -60,8 +71,7 @@ func (sysd SystemdInitSystem) ServiceRestart(service string) error {
 
 func (sysd SystemdInitSystem) ServiceStop(service string) error {
 	args := []string{"stop", service}
-	err := exec.Command("systemctl", args...).Run()
+	return exec.Command("systemctl", args...).Run()
-	return err
 }
 
 func (sysd SystemdInitSystem) ServiceExists(service string) bool {