Merge pull request #65104 from luxas/kubeadm_v111_more_bugs

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

kubeadm: Fix an upgrading issue wrt the dynamic kubelet dropin

**What this PR does / why we need it**:
At the moment (will change), this PR does two things:
 - Makes `kubeletphase.WriteKubeletDynamicEnvFile` run at upgrade time if needed for making v1.10 -> v1.11 upgrades work end to end
 - Fixes so that `--dry-run` works smoothly on `init/upgrade` ==> separate PR

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #

**Special notes for your reviewer**:

**Release note**:

```release-note
NONE
```
@kubernetes/sig-cluster-lifecycle-pr-reviews 
/kind bug
/priority critical-urgent
/status approved-for-milestone
/assign @timothysc

cmd/kubeadm/app/phases/upgrade/postupgrade.go

@@ -65,19 +65,8 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterC
 		errs = append(errs, fmt.Errorf("error creating kubelet configuration ConfigMap: %v", err))
 	}
 
-	kubeletDir, err := getKubeletDir(dryRun)
-	if err == nil {
-		// Write the configuration for the kubelet down to disk so the upgraded kubelet can start with fresh config
-		if err := kubeletphase.DownloadConfig(client, newK8sVer, kubeletDir); err != nil {
-			// Tolerate the error being NotFound when dryrunning, as there is a pretty common scenario: the dryrun process
-			// *would* post the new kubelet-config-1.X configmap that doesn't exist now when we're trying to download it
-			// again.
-			if !(apierrors.IsNotFound(err) && dryRun) {
-				errs = append(errs, fmt.Errorf("error downloading kubelet configuration from the ConfigMap: %v", err))
-			}
-		}
-	} else {
-		// The error here should never occur in reality, would only be thrown if /tmp doesn't exist on the machine.
+	// Write the new kubelet config down to disk and the env file if needed
+	if err := writeKubeletConfigFiles(client, cfg, newK8sVer, dryRun); err != nil {
 		errs = append(errs, err)
 	}
 
@@ -209,6 +198,43 @@ func backupAPIServerCertIfNeeded(cfg *kubeadmapi.MasterConfiguration, dryRun boo
 	return certsphase.CreateAPIServerCertAndKeyFiles(cfg)
 }
 
+func writeKubeletConfigFiles(client clientset.Interface, cfg *kubeadmapi.MasterConfiguration, newK8sVer *version.Version, dryRun bool) error {
+	kubeletDir, err := getKubeletDir(dryRun)
+	if err != nil {
+		// The error here should never occur in reality, would only be thrown if /tmp doesn't exist on the machine.
+		return err
+	}
+	errs := []error{}
+	// Write the configuration for the kubelet down to disk so the upgraded kubelet can start with fresh config
+	if err := kubeletphase.DownloadConfig(client, newK8sVer, kubeletDir); err != nil {
+		// Tolerate the error being NotFound when dryrunning, as there is a pretty common scenario: the dryrun process
+		// *would* post the new kubelet-config-1.X configmap that doesn't exist now when we're trying to download it
+		// again.
+		if !(apierrors.IsNotFound(err) && dryRun) {
+			errs = append(errs, fmt.Errorf("error downloading kubelet configuration from the ConfigMap: %v", err))
+		}
+	}
+
+	if dryRun { // Print what contents would be written
+		dryrunutil.PrintDryRunFile(kubeadmconstants.KubeletConfigurationFileName, kubeletDir, kubeadmconstants.KubeletRunDirectory, os.Stdout)
+	}
+
+	envFilePath := filepath.Join(kubeadmconstants.KubeletRunDirectory, kubeadmconstants.KubeletEnvFileName)
+	if _, err := os.Stat(envFilePath); os.IsNotExist(err) {
+		// Write env file with flags for the kubelet to use. We do not need to write the --register-with-taints for the master,
+		// as we handle that ourselves in the markmaster phase
+		// TODO: Maybe we want to do that some time in the future, in order to remove some logic from the markmaster phase?
+		if err := kubeletphase.WriteKubeletDynamicEnvFile(&cfg.NodeRegistration, cfg.FeatureGates, false, kubeletDir); err != nil {
+			errs = append(errs, fmt.Errorf("error writing a dynamic environment file for the kubelet: %v", err))
+		}
+
+		if dryRun { // Print what contents would be written
+			dryrunutil.PrintDryRunFile(kubeadmconstants.KubeletEnvFileName, kubeletDir, kubeadmconstants.KubeletRunDirectory, os.Stdout)
+		}
+	}
+	return errors.NewAggregate(errs)
+}
+
 // getWaiter gets the right waiter implementation for the right occasion
 // TODO: Consolidate this with what's in init.go?
 func getWaiter(dryRun bool, client clientset.Interface) apiclient.Waiter {

cmd/kubeadm/app/util/dryrun/dryrun.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"path/filepath"
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -43,6 +44,13 @@ func NewFileToPrint(realPath, printPath string) FileToPrint {
 	}
 }
 
+// PrintDryRunFile is a helper method around PrintDryRunFiles
+func PrintDryRunFile(fileName, realDir, printDir string, w io.Writer) error {
+	return PrintDryRunFiles([]FileToPrint{
+		NewFileToPrint(filepath.Join(realDir, fileName), filepath.Join(printDir, fileName)),
+	}, w)
+}
+
 // PrintDryRunFiles prints the contents of the FileToPrints given to it to the writer w
 func PrintDryRunFiles(files []FileToPrint, w io.Writer) error {
 	errs := []error{}