Cleanup etcd data dir on best effort basis

Signed-off-by: Dave Chen <dave.chen@arm.com>

cmd/kubeadm/app/cmd/phases/reset/cleanupnode.go

@@ -19,6 +19,7 @@ package phases
 import (
 	"errors"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 
@@ -201,3 +202,16 @@ func CleanDir(filePath string) error {
 	}
 	return nil
 }
+
+func IsDirEmpty(dir string) (bool, error) {
+	d, err := os.Open(dir)
+	if err != nil {
+		return false, err
+	}
+	defer d.Close()
+	_, err = d.Readdirnames(1)
+	if err == io.EOF {
+		return true, nil
+	}
+	return false, nil
+}

cmd/kubeadm/app/cmd/phases/reset/removeetcdmember.go

@@ -19,11 +19,14 @@ package phases
 import (
 	"errors"
 	"fmt"
+	"os"
 	"path/filepath"
 
 	"k8s.io/klog/v2"
 
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
+	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
@@ -63,7 +66,9 @@ func runRemoveETCDMemberPhase(c workflow.RunData) error {
 					klog.Warningf("[reset] Failed to remove etcd member: %v, please manually remove this etcd member using etcdctl", err)
 				} else {
 					if err := CleanDir(etcdDataDir); err != nil {
-						klog.Warningf("[reset] Failed to delete contents of %q directory: %v", etcdDataDir, err)
+						klog.Warningf("[reset] Failed to delete contents of the etcd directory: %q, error: %v", etcdDataDir, err)
+					} else {
+						fmt.Printf("[reset] Deleted contents of the etcd data directory: %v\n", etcdDataDir)
 					}
 				}
 			} else {
@@ -71,6 +76,16 @@ func runRemoveETCDMemberPhase(c workflow.RunData) error {
 				fmt.Printf("[reset] Would delete contents of the etcd data directory: %v\n", etcdDataDir)
 			}
 		}
+		// This could happen if the phase `cleanup-node` is run before the `remove-etcd-member`.
+		// Cleanup the data in the etcd data dir to avoid some stale files which might cause the failure to build cluster in the next time.
+		empty, _ := IsDirEmpty(etcdDataDir)
+		if !empty && !r.DryRun() {
+			if err := CleanDir(etcdDataDir); err != nil {
+				klog.Warningf("[reset] Failed to delete contents of the etcd directory: %q, error: %v", etcdDataDir, err)
+			} else {
+				fmt.Printf("[reset] Deleted contents of the etcd data directory: %v\n", etcdDataDir)
+			}
+		}
 	} else {
 		fmt.Println("[reset] No etcd config found. Assuming external etcd")
 		fmt.Println("[reset] Please, manually reset etcd to prevent further issues")
@@ -88,11 +103,17 @@ func getEtcdDataDir(manifestPath string, cfg *kubeadmapi.InitConfiguration) (str
 	}
 	klog.Warningln("[reset] No kubeadm config, using etcd pod spec to get data directory")
 
+	if _, err := os.Stat(manifestPath); os.IsNotExist(err) {
+		// Fall back to use the default cluster config if etcd.yaml doesn't exist, this could happen that
+		// etcd.yaml is removed by other reset phases, e.g. cleanup-node.
+		cfg := &v1beta3.ClusterConfiguration{}
+		scheme.Scheme.Default(cfg)
+		return cfg.Etcd.Local.DataDir, nil
+	}
 	etcdPod, err := utilstaticpod.ReadStaticPodFromDisk(manifestPath)
 	if err != nil {
 		return "", err
 	}
-
 	for _, volumeMount := range etcdPod.Spec.Volumes {
 		if volumeMount.Name == etcdVolumeName {
 			dataDir = volumeMount.HostPath.Path

cmd/kubeadm/app/cmd/phases/reset/removeetcdmember_test.go

@@ -63,10 +63,11 @@ func TestGetEtcdDataDir(t *testing.T) {
 		writeManifest bool
 		validConfig   bool
 	}{
-		"non-existent file returns error": {
-			expectErr:     true,
+		"non-existent file returns default data dir": {
+			expectErr:     false,
 			writeManifest: false,
 			validConfig:   true,
+			dataDir:       "/var/lib/etcd",
 		},
 		"return etcd data dir": {
 			dataDir:       "/path/to/etcd",