From 2ee787c5837561dc35d18bbf9a5f02d9f86e0053 Mon Sep 17 00:00:00 2001
From: Devan Goodwin <dgoodwin@redhat.com>
Date: Thu, 27 Oct 2016 10:26:26 -0300
Subject: [PATCH] kubeadm: Empty directories during reset, but do not delete
 them.

This will allow packages to maintain ownership of config and data
directories, which may carry selinux or other attributes that should be
preserved, but we do not wish to manage within kubeadm itself.
---
 cmd/kubeadm/app/cmd/reset.go      | 39 +++++++++++++++++++++++--------
 cmd/kubeadm/app/cmd/reset_test.go | 18 ++++++++++++--
 2 files changed, 45 insertions(+), 12 deletions(-)

diff --git a/cmd/kubeadm/app/cmd/reset.go b/cmd/kubeadm/app/cmd/reset.go
index 04d9e175b48..a44afbefd9d 100644
--- a/cmd/kubeadm/app/cmd/reset.go
+++ b/cmd/kubeadm/app/cmd/reset.go
@@ -67,18 +67,40 @@ func NewReset(skipPreFlight bool) (*Reset, error) {
 	return &Reset{}, nil
 }
 
+// cleanDir removes everything in a directory, but not the directory itself:
+func cleanDir(path string) {
+	// If the directory doesn't even exist there's nothing to do, and we do
+	// not consider this an error:
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return
+	}
+
+	d, err := os.Open(path)
+	if err != nil {
+		fmt.Printf("failed to remove directory: [%v]\n", err)
+	}
+	defer d.Close()
+	names, err := d.Readdirnames(-1)
+	if err != nil {
+		fmt.Printf("failed to remove directory: [%v]\n", err)
+	}
+	for _, name := range names {
+		err = os.RemoveAll(filepath.Join(path, name))
+		if err != nil {
+			fmt.Printf("failed to remove directory: [%v]\n", err)
+		}
+	}
+}
+
 // resetConfigDir is used to cleanup the files kubeadm writes in /etc/kubernetes/.
 func resetConfigDir(configDirPath string) {
 	dirsToClean := []string{
 		filepath.Join(configDirPath, "manifests"),
 		filepath.Join(configDirPath, "pki"),
 	}
-	fmt.Printf("Deleting config directories: %v\n", dirsToClean)
+	fmt.Printf("Deleting contents of config directories: %v\n", dirsToClean)
 	for _, dir := range dirsToClean {
-		err := os.RemoveAll(dir)
-		if err != nil {
-			fmt.Printf("failed to remove directory: [%v]\n", err)
-		}
+		cleanDir(dir)
 	}
 
 	filesToClean := []string{
@@ -112,12 +134,9 @@ func (r *Reset) Run(out io.Writer) error {
 	resetConfigDir("/etc/kubernetes/")
 
 	dirsToClean := []string{"/var/lib/kubelet", "/var/lib/etcd"}
-	fmt.Printf("Deleting stateful directories: %v\n", dirsToClean)
+	fmt.Printf("Deleting contents of stateful directories: %v\n", dirsToClean)
 	for _, dir := range dirsToClean {
-		err := os.RemoveAll(dir)
-		if err != nil {
-			fmt.Printf("failed to remove directory: [%v]\n", err)
-		}
+		cleanDir(dir)
 	}
 
 	dockerCheck := preflight.ServiceCheck{Service: "docker"}
diff --git a/cmd/kubeadm/app/cmd/reset_test.go b/cmd/kubeadm/app/cmd/reset_test.go
index 7113cb460af..d9355e51935 100644
--- a/cmd/kubeadm/app/cmd/reset_test.go
+++ b/cmd/kubeadm/app/cmd/reset_test.go
@@ -66,6 +66,10 @@ func TestConfigDirCleaner(t *testing.T) {
 				"admin.conf",
 				"kubelet.conf",
 			},
+			verifyExists: []string{
+				"manifests",
+				"pki",
+			},
 		},
 		"partial reset": {
 			setupDirs: []string{
@@ -75,6 +79,12 @@ func TestConfigDirCleaner(t *testing.T) {
 				"pki/ca.pem",
 				"kubelet.conf",
 			},
+			verifyExists: []string{
+				"pki",
+			},
+			verifyNotExists: []string{
+				"manifests",
+			},
 		},
 		"preserve cloud-config.json": {
 			setupDirs: []string{
@@ -90,6 +100,8 @@ func TestConfigDirCleaner(t *testing.T) {
 				"cloud-config.json",
 			},
 			verifyExists: []string{
+				"manifests",
+				"pki",
 				"cloud-config.json",
 			},
 		},
@@ -109,6 +121,8 @@ func TestConfigDirCleaner(t *testing.T) {
 				".mydir/.myfile",
 			},
 			verifyExists: []string{
+				"manifests",
+				"pki",
 				".cloud-config.json",
 				".mydir",
 				".mydir/.myfile",
@@ -154,8 +168,8 @@ func TestConfigDirCleaner(t *testing.T) {
 		assertExists(t, tmpDir)
 		assertNotExists(t, filepath.Join(tmpDir, "admin.conf"))
 		assertNotExists(t, filepath.Join(tmpDir, "kubelet.conf"))
-		assertNotExists(t, filepath.Join(tmpDir, "manifests"))
-		assertNotExists(t, filepath.Join(tmpDir, "pki"))
+		assertDirEmpty(t, filepath.Join(tmpDir, "manifests"))
+		assertDirEmpty(t, filepath.Join(tmpDir, "pki"))
 
 		// Verify the files as requested by the test:
 		for _, path := range test.verifyExists {