Merge pull request #70003 from yagonobre/kubeadm-clean-etcd-dir

Get the etcd data path from kubeadm config or etcd pod manifest on kubeadm reset

cmd/kubeadm/app/cmd/BUILD

@@ -56,6 +56,7 @@ go_library(
         "//cmd/kubeadm/app/util/dryrun:go_default_library",
         "//cmd/kubeadm/app/util/kubeconfig:go_default_library",
         "//cmd/kubeadm/app/util/runtime:go_default_library",
+        "//cmd/kubeadm/app/util/staticpod:go_default_library",
         "//pkg/util/initsystem:go_default_library",
         "//pkg/version:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
@@ -102,6 +103,7 @@ go_test(
         "//cmd/kubeadm/app/util:go_default_library",
         "//cmd/kubeadm/app/util/config:go_default_library",
         "//cmd/kubeadm/app/util/runtime:go_default_library",
+        "//cmd/kubeadm/test:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",

cmd/kubeadm/app/cmd/reset.go

@@ -30,13 +30,17 @@ import (
 	"github.com/spf13/cobra"
 
 	"k8s.io/apimachinery/pkg/util/sets"
+	clientset "k8s.io/client-go/kubernetes"
 	kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
+	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
+	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
 	utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime"
+	utilstaticpod "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
 	"k8s.io/kubernetes/pkg/util/initsystem"
 	utilsexec "k8s.io/utils/exec"
 )
@@ -47,6 +51,7 @@ func NewCmdReset(in io.Reader, out io.Writer) *cobra.Command {
 	var criSocketPath string
 	var ignorePreflightErrors []string
 	var forceReset bool
+	kubeConfigFile := kubeadmconstants.GetAdminKubeConfigPath()
 
 	cmd := &cobra.Command{
 		Use:   "reset",
@@ -55,13 +60,18 @@ func NewCmdReset(in io.Reader, out io.Writer) *cobra.Command {
 			ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(ignorePreflightErrors)
 			kubeadmutil.CheckErr(err)
 
+			kubeConfigFile = cmdutil.FindExistingKubeConfig(kubeConfigFile)
+			client, err := getClientset(kubeConfigFile, false)
+			kubeadmutil.CheckErr(err)
+
 			r, err := NewReset(in, ignorePreflightErrorsSet, forceReset, certsDir, criSocketPath)
 			kubeadmutil.CheckErr(err)
-			kubeadmutil.CheckErr(r.Run(out))
+			kubeadmutil.CheckErr(r.Run(out, client))
 		},
 	}
 
 	options.AddIgnorePreflightErrorsFlag(cmd.PersistentFlags(), &ignorePreflightErrors)
+	options.AddKubeConfigFlag(cmd.PersistentFlags(), &kubeConfigFile)
 
 	cmd.PersistentFlags().StringVar(
 		&certsDir, "cert-dir", kubeadmapiv1beta1.DefaultCertificatesDir,
@@ -114,7 +124,19 @@ func NewReset(in io.Reader, ignorePreflightErrors sets.String, forceReset bool,
 }
 
 // Run reverts any changes made to this host by "kubeadm init" or "kubeadm join".
-func (r *Reset) Run(out io.Writer) error {
+func (r *Reset) Run(out io.Writer, client clientset.Interface) error {
+	var dirsToClean []string
+	// Only clear etcd data when using local etcd.
+	etcdManifestPath := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName, "etcd.yaml")
+
+	glog.V(1).Infof("[reset] checking for etcd config")
+	etcdDataDir, err := getEtcdDataDir(etcdManifestPath, client)
+	if err == nil {
+		dirsToClean = append(dirsToClean, etcdDataDir)
+	} else {
+		fmt.Println("[reset] no etcd config found. Assuming external etcd")
+		fmt.Println("[reset] please manually reset etcd to prevent further issues")
+	}
 
 	// Try to stop the kubelet service
 	glog.V(1).Infof("[reset] getting init system")
@@ -144,19 +166,7 @@ func (r *Reset) Run(out io.Writer) error {
 	if err := removeContainers(utilsexec.New(), r.criSocketPath); err != nil {
 		glog.Errorf("[reset] failed to remove containers: %+v", err)
 	}
-	dirsToClean := []string{kubeadmconstants.KubeletRunDirectory, "/etc/cni/net.d", "/var/lib/dockershim", "/var/run/kubernetes"}
+	dirsToClean = append(dirsToClean, []string{kubeadmconstants.KubeletRunDirectory, "/etc/cni/net.d", "/var/lib/dockershim", "/var/run/kubernetes"}...)
-
-	// Only clear etcd data when the etcd manifest is found. In case it is not found, we must assume that the user
-	// provided external etcd endpoints. In that case, it is their own responsibility to reset etcd
-	etcdManifestPath := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName, "etcd.yaml")
-	glog.V(1).Infof("[reset] checking for etcd manifest")
-	if _, err := os.Stat(etcdManifestPath); err == nil {
-		glog.V(1).Infof("Found one at %s", etcdManifestPath)
-		dirsToClean = append(dirsToClean, "/var/lib/etcd")
-	} else {
-		fmt.Printf("[reset] no etcd manifest found in %q. Assuming external etcd\n", etcdManifestPath)
-		fmt.Println("[reset] please manually reset etcd to prevent further issues")
-	}
 
 	// Then clean contents from the stateful kubelet, etcd and cni directories
 	fmt.Printf("[reset] deleting contents of stateful directories: %v\n", dirsToClean)
@@ -175,6 +185,33 @@ func (r *Reset) Run(out io.Writer) error {
 	return nil
 }
 
+func getEtcdDataDir(manifestPath string, client clientset.Interface) (string, error) {
+	const etcdVolumeName = "etcd-data"
+	var dataDir string
+
+	cfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "reset", "", false)
+	if err == nil {
+		return cfg.Etcd.Local.DataDir, nil
+	}
+	glog.Warningf("[reset] Unable to fetch the kubeadm-config ConfigMap, using etcd pod spec as fallback: %v", err)
+
+	etcdPod, err := utilstaticpod.ReadStaticPodFromDisk(manifestPath)
+	if err != nil {
+		return "", err
+	}
+
+	for _, volumeMount := range etcdPod.Spec.Volumes {
+		if volumeMount.Name == etcdVolumeName {
+			dataDir = volumeMount.HostPath.Path
+			break
+		}
+	}
+	if dataDir == "" {
+		return dataDir, fmt.Errorf("invalid etcd pod manifest")
+	}
+	return dataDir, nil
+}
+
 func removeContainers(execer utilsexec.Interface, criSocketPath string) error {
 	containerRuntime, err := utilruntime.NewContainerRuntime(execer, criSocketPath)
 	if err != nil {

cmd/kubeadm/app/cmd/reset_test.go

@@ -23,14 +23,46 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/renstrom/dedent"
+
+	clientsetfake "k8s.io/client-go/kubernetes/fake"
 	kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
+	testutil "k8s.io/kubernetes/cmd/kubeadm/test"
 	"k8s.io/utils/exec"
 	fakeexec "k8s.io/utils/exec/testing"
 )
 
+const (
+	etcdPod = `apiVersion: v1
+kind: Pod
+metadata:
+spec:
+  volumes:
+  - hostPath:
+      path: /path/to/etcd
+      type: DirectoryOrCreate
+    name: etcd-data
+  - hostPath:
+      path: /etc/kubernetes/pki/etcd
+      type: DirectoryOrCreate
+    name: etcd-certs`
+
+	etcdPodWithoutDataVolume = `apiVersion: v1
+kind: Pod
+metadata:
+spec:
+  volumes:
+  - hostPath:
+      path: /etc/kubernetes/pki/etcd
+      type: DirectoryOrCreate
+    name: etcd-certs`
+
+	etcdPodInvalid = `invalid pod`
+)
+
 func assertExists(t *testing.T, path string) {
 	if _, err := os.Stat(path); os.IsNotExist(err) {
 		t.Errorf("file/directory does not exist; error: %s", err)
@@ -234,3 +266,66 @@ func TestRemoveContainers(t *testing.T) {
 
 	removeContainers(&fexec, "unix:///var/run/crio/crio.sock")
 }
+
+func TestGetEtcdDataDir(t *testing.T) {
+	tests := map[string]struct {
+		dataDir       string
+		podYaml       string
+		expectErr     bool
+		writeManifest bool
+	}{
+		"non-existent file returns error": {
+			dataDir:       "",
+			podYaml:       "",
+			expectErr:     true,
+			writeManifest: false,
+		},
+		"return etcd data dir": {
+			dataDir:       "/path/to/etcd",
+			podYaml:       etcdPod,
+			expectErr:     false,
+			writeManifest: true,
+		},
+		"invalid etcd pod": {
+			dataDir:       "",
+			podYaml:       etcdPodInvalid,
+			expectErr:     true,
+			writeManifest: true,
+		},
+		"etcd pod spec without data volume": {
+			dataDir:       "",
+			podYaml:       etcdPodWithoutDataVolume,
+			expectErr:     true,
+			writeManifest: true,
+		},
+	}
+
+	for name, test := range tests {
+		tmpdir := testutil.SetupTempDir(t)
+		defer os.RemoveAll(tmpdir)
+
+		manifestPath := filepath.Join(tmpdir, "etcd.yaml")
+		if test.writeManifest {
+			err := ioutil.WriteFile(manifestPath, []byte(test.podYaml), 0644)
+			if err != nil {
+				t.Fatalf(dedent.Dedent("failed to write pod manifest\n%s\n\tfatal error: %v"), name, err)
+			}
+		}
+
+		client := clientsetfake.NewSimpleClientset()
+		dataDir, err := getEtcdDataDir(manifestPath, client)
+		if (err != nil) != test.expectErr {
+			t.Fatalf(dedent.Dedent(
+				"getEtcdDataDir failed\n%s\nexpected error: %t\n\tgot: %t\nerror: %v"),
+				name,
+				test.expectErr,
+				(err != nil),
+				err,
+			)
+		}
+
+		if dataDir != test.dataDir {
+			t.Fatalf(dedent.Dedent("getEtcdDataDir failed\n%s\n\texpected: %s\ngot: %s"), name, test.dataDir, dataDir)
+		}
+	}
+}