From 51d733f418ab2cb29de0b9effa1d3bbd8254d342 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Sun, 4 Dec 2016 19:50:52 +0200 Subject: [PATCH 1/3] Remove the cni directory when resetting; otherwise kubelet can pick up the wrong config on the next kubeadm init run --- cmd/kubeadm/app/cmd/reset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/kubeadm/app/cmd/reset.go b/cmd/kubeadm/app/cmd/reset.go index c2325b789db..86fb8526869 100644 --- a/cmd/kubeadm/app/cmd/reset.go +++ b/cmd/kubeadm/app/cmd/reset.go @@ -136,7 +136,7 @@ func (r *Reset) Run(out io.Writer) error { fmt.Printf("failed to unmount directories in /var/lib/kubelet, %s", string(umountOutputBytes)) } - dirsToClean := []string{"/var/lib/kubelet"} + dirsToClean := []string{"/var/lib/kubelet", "/etc/cni/net.d"} // Only clear etcd data when the etcd manifest is found. In case it is not found, we must assume that the user // provided external etcd endpoints. In that case, it is his own responsibility to reset etcd From 67d4ddaf5945fa91a69b65fe9e29ad98e07ec6c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Sun, 4 Dec 2016 20:07:33 +0200 Subject: [PATCH 2/3] Improve the kubeadm reset command. Reorder the functions and log more user-friendly output --- cmd/kubeadm/app/cmd/reset.go | 144 ++++++++++++++++-------------- cmd/kubeadm/app/cmd/reset_test.go | 2 +- 2 files changed, 77 insertions(+), 69 deletions(-) diff --git a/cmd/kubeadm/app/cmd/reset.go b/cmd/kubeadm/app/cmd/reset.go index 86fb8526869..912ad237036 100644 --- a/cmd/kubeadm/app/cmd/reset.go +++ b/cmd/kubeadm/app/cmd/reset.go @@ -1,5 +1,5 @@ /* -Copyright 2014 The Kubernetes Authors. +Copyright 2016 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,16 +21,17 @@ import ( "io" "os" "os/exec" - "path/filepath" + "path" "github.com/spf13/cobra" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/preflight" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/pkg/util/initsystem" ) -// NewCmdReset returns "kubeadm reset" command. +// NewCmdReset returns the "kubeadm reset" command func NewCmdReset(out io.Writer) *cobra.Command { var skipPreFlight bool cmd := &cobra.Command{ @@ -55,111 +56,118 @@ type Reset struct{} func NewReset(skipPreFlight bool) (*Reset, error) { if !skipPreFlight { - fmt.Println("Running pre-flight checks") - err := preflight.RunResetCheck() - if err != nil { + fmt.Println("[preflight] Running pre-flight checks...") + + if err := preflight.RunResetCheck(); err != nil { return nil, &preflight.PreFlightError{Msg: err.Error()} } } else { - fmt.Println("Skipping pre-flight checks") + fmt.Println("[preflight] Skipping pre-flight checks...") } return &Reset{}, nil } -// cleanDir removes everything in a directory, but not the directory itself: -func cleanDir(path string) { - // If the directory doesn't even exist there's nothing to do, and we do - // not consider this an error: - if _, err := os.Stat(path); os.IsNotExist(err) { - return - } - - d, err := os.Open(path) - if err != nil { - fmt.Printf("failed to remove directory: [%v]\n", err) - } - defer d.Close() - names, err := d.Readdirnames(-1) - if err != nil { - fmt.Printf("failed to remove directory: [%v]\n", err) - } - for _, name := range names { - err = os.RemoveAll(filepath.Join(path, name)) - if err != nil { - fmt.Printf("failed to remove directory: [%v]\n", err) - } - } -} - -// resetConfigDir is used to cleanup the files kubeadm writes in /etc/kubernetes/. -func resetConfigDir(configDirPath string) { - dirsToClean := []string{ - filepath.Join(configDirPath, "manifests"), - filepath.Join(configDirPath, "pki"), - } - fmt.Printf("Deleting contents of config directories: %v\n", dirsToClean) - for _, dir := range dirsToClean { - cleanDir(dir) - } - - filesToClean := []string{ - filepath.Join(configDirPath, "admin.conf"), - filepath.Join(configDirPath, "kubelet.conf"), - } - fmt.Printf("Deleting files: %v\n", filesToClean) - for _, path := range filesToClean { - err := os.RemoveAll(path) - if err != nil { - fmt.Printf("failed to remove file: [%v]\n", err) - } - } -} - // Run reverts any changes made to this host by "kubeadm init" or "kubeadm join". func (r *Reset) Run(out io.Writer) error { serviceToStop := "kubelet" initSystem, err := initsystem.GetInitSystem() if err != nil { - fmt.Printf("%v", err) + fmt.Printf("[reset] Failed to detect init system and stop the kubelet service: %v\n", err) } else { - fmt.Printf("Stopping the %s service...\n", serviceToStop) + fmt.Printf("[reset] Stopping the %s service...\n", serviceToStop) if err := initSystem.ServiceStop(serviceToStop); err != nil { - fmt.Printf("failed to stop the %s service", serviceToStop) + fmt.Printf("[reset] Failed to stop the %s service", serviceToStop) } } - fmt.Printf("Unmounting directories in /var/lib/kubelet...\n") + fmt.Println("[reset] Unmounting directories in /var/lib/kubelet...") umountDirsCmd := "cat /proc/mounts | awk '{print $2}' | grep '/var/lib/kubelet' | xargs -r umount" umountOutputBytes, err := exec.Command("sh", "-c", umountDirsCmd).Output() if err != nil { - fmt.Printf("failed to unmount directories in /var/lib/kubelet, %s", string(umountOutputBytes)) + fmt.Printf("[reset] Failed to unmount directories in /var/lib/kubelet: %s\n", string(umountOutputBytes)) } + // Remove contents from the config and pki directories + resetConfigDir(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmapi.GlobalEnvParams.HostPKIPath) + dirsToClean := []string{"/var/lib/kubelet", "/etc/cni/net.d"} // Only clear etcd data when the etcd manifest is found. In case it is not found, we must assume that the user // provided external etcd endpoints. In that case, it is his own responsibility to reset etcd if _, err := os.Stat("/etc/kubernetes/manifests/etcd.json"); os.IsNotExist(err) { dirsToClean = append(dirsToClean, "/var/lib/etcd") + } else { + fmt.Println("[reset] No etcd manifest found in %q, assuming external etcd.", "/etc/kubernetes/manifests/etcd.json") } - resetConfigDir("/etc/kubernetes/") - - fmt.Printf("Deleting contents of stateful directories: %v\n", dirsToClean) + fmt.Printf("[reset] Deleting contents of stateful directories: %v\n", dirsToClean) for _, dir := range dirsToClean { cleanDir(dir) } dockerCheck := preflight.ServiceCheck{Service: "docker"} if warnings, errors := dockerCheck.Check(); len(warnings) == 0 && len(errors) == 0 { - fmt.Println("Stopping all running docker containers...") - if err := exec.Command("sh", "-c", "docker ps | grep 'k8s_' | awk '{print $1}' | xargs docker rm --force --volumes").Run(); err != nil { - fmt.Println("failed to stop the running containers") + fmt.Println("[reset] Stopping all running docker containers...") + if err := exec.Command("sh", "-c", "docker ps | grep 'k8s_' | awk '{print $1}' | xargs -r docker rm --force --volumes").Run(); err != nil { + fmt.Println("[reset] Failed to stop the running containers") } } else { - fmt.Println("docker doesn't seem to be running, skipping the removal of kubernetes containers") + fmt.Println("[reset] docker doesn't seem to be running, skipping the removal of running kubernetes containers") } return nil } + +// cleanDir removes everything in a directory, but not the directory itself +func cleanDir(filepath string) error { + // If the directory doesn't even exist there's nothing to do, and we do + // not consider this an error + if _, err := os.Stat(filepath); os.IsNotExist(err) { + return nil + } + + d, err := os.Open(filepath) + if err != nil { + return err + } + defer d.Close() + names, err := d.Readdirnames(-1) + if err != nil { + return err + } + for _, name := range names { + err = os.RemoveAll(path.Join(filepath, name)) + if err != nil { + return err + } + } + return nil +} + +// resetConfigDir is used to cleanup the files kubeadm writes in /etc/kubernetes/. +func resetConfigDir(configPathDir, pkiPathDir string) { + dirsToClean := []string{ + path.Join(configPathDir, "manifests"), + pkiPathDir, + } + fmt.Printf("[reset] Deleting contents of config directories: %v\n", dirsToClean) + for _, dir := range dirsToClean { + err := cleanDir(dir) + if err != nil { + fmt.Printf("[reset] Failed to remove directory: %q [%v]", dir, err) + } + } + + filesToClean := []string{ + path.Join(configPathDir, "admin.conf"), + path.Join(configPathDir, "kubelet.conf"), + } + fmt.Printf("[reset] Deleting files: %v\n", filesToClean) + for _, path := range filesToClean { + err := os.RemoveAll(path) + if err != nil { + fmt.Printf("[reset] Failed to remove file: %q [%v]\n", path, err) + } + } +} diff --git a/cmd/kubeadm/app/cmd/reset_test.go b/cmd/kubeadm/app/cmd/reset_test.go index d9355e51935..37dc4631826 100644 --- a/cmd/kubeadm/app/cmd/reset_test.go +++ b/cmd/kubeadm/app/cmd/reset_test.go @@ -162,7 +162,7 @@ func TestConfigDirCleaner(t *testing.T) { } } - resetConfigDir(tmpDir) + resetConfigDir(tmpDir, filepath.Join(tmpDir, "pki")) // Verify the files we cleanup implicitly in every test: assertExists(t, tmpDir) From 7a463eff0877ad68f6a22b3befe0ac815034a3ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Tue, 6 Dec 2016 15:43:59 +0200 Subject: [PATCH 3/3] Drain node on kubeadm reset and make it possible to specify if the node should be removed from the cluster as well --- cmd/kubeadm/app/cmd/reset.go | 72 +++++++++++++++++++++++++++---- hack/verify-flags/known-flags.txt | 1 + 2 files changed, 64 insertions(+), 9 deletions(-) diff --git a/cmd/kubeadm/app/cmd/reset.go b/cmd/kubeadm/app/cmd/reset.go index 912ad237036..802b03f8c8d 100644 --- a/cmd/kubeadm/app/cmd/reset.go +++ b/cmd/kubeadm/app/cmd/reset.go @@ -22,6 +22,7 @@ import ( "os" "os/exec" "path" + "strings" "github.com/spf13/cobra" @@ -33,12 +34,12 @@ import ( // NewCmdReset returns the "kubeadm reset" command func NewCmdReset(out io.Writer) *cobra.Command { - var skipPreFlight bool + var skipPreFlight, removeNode bool cmd := &cobra.Command{ Use: "reset", Short: "Run this to revert any changes made to this host by 'kubeadm init' or 'kubeadm join'.", Run: func(cmd *cobra.Command, args []string) { - r, err := NewReset(skipPreFlight) + r, err := NewReset(skipPreFlight, removeNode) kubeadmutil.CheckErr(err) kubeadmutil.CheckErr(r.Run(out)) }, @@ -46,15 +47,22 @@ func NewCmdReset(out io.Writer) *cobra.Command { cmd.PersistentFlags().BoolVar( &skipPreFlight, "skip-preflight-checks", false, - "skip preflight checks normally run before modifying the system", + "Skip preflight checks normally run before modifying the system", + ) + + cmd.PersistentFlags().BoolVar( + &removeNode, "remove-node", true, + "Remove this node from the pool of nodes in this cluster", ) return cmd } -type Reset struct{} +type Reset struct { + removeNode bool +} -func NewReset(skipPreFlight bool) (*Reset, error) { +func NewReset(skipPreFlight, removeNode bool) (*Reset, error) { if !skipPreFlight { fmt.Println("[preflight] Running pre-flight checks...") @@ -65,11 +73,20 @@ func NewReset(skipPreFlight bool) (*Reset, error) { fmt.Println("[preflight] Skipping pre-flight checks...") } - return &Reset{}, nil + return &Reset{ + removeNode: removeNode, + }, nil } // Run reverts any changes made to this host by "kubeadm init" or "kubeadm join". func (r *Reset) Run(out io.Writer) error { + + // Drain and maybe remove the node from the cluster + err := drainAndRemoveNode(r.removeNode) + if err != nil { + fmt.Printf("[reset] Failed to cleanup node: [%v]\n", err) + } + serviceToStop := "kubelet" initSystem, err := initsystem.GetInitSystem() if err != nil { @@ -77,7 +94,7 @@ func (r *Reset) Run(out io.Writer) error { } else { fmt.Printf("[reset] Stopping the %s service...\n", serviceToStop) if err := initSystem.ServiceStop(serviceToStop); err != nil { - fmt.Printf("[reset] Failed to stop the %s service", serviceToStop) + fmt.Printf("[reset] Failed to stop the %s service\n", serviceToStop) } } @@ -98,7 +115,7 @@ func (r *Reset) Run(out io.Writer) error { if _, err := os.Stat("/etc/kubernetes/manifests/etcd.json"); os.IsNotExist(err) { dirsToClean = append(dirsToClean, "/var/lib/etcd") } else { - fmt.Println("[reset] No etcd manifest found in %q, assuming external etcd.", "/etc/kubernetes/manifests/etcd.json") + fmt.Printf("[reset] No etcd manifest found in %q, assuming external etcd.\n", "/etc/kubernetes/manifests/etcd.json") } fmt.Printf("[reset] Deleting contents of stateful directories: %v\n", dirsToClean) @@ -119,6 +136,43 @@ func (r *Reset) Run(out io.Writer) error { return nil } +func drainAndRemoveNode(removeNode bool) error { + + hostname, err := os.Hostname() + if err != nil { + return fmt.Errorf("failed to detect node hostname") + } + hostname = strings.ToLower(hostname) + + // TODO: Use the "native" k8s client for this once we're confident the versioned is working + kubeConfigPath := path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "kubelet.conf") + + getNodesCmd := fmt.Sprintf("kubectl --kubeconfig %s get nodes | grep %s", kubeConfigPath, hostname) + output, err := exec.Command("sh", "-c", getNodesCmd).Output() + if err != nil || len(output) == 0 { + // kubeadm shouldn't drain and/or remove the node when it doesn't exist anymore + return nil + } + + fmt.Printf("[reset] Draining node: %q\n", hostname) + + output, err = exec.Command("kubectl", "--kubeconfig", kubeConfigPath, "drain", hostname, "--delete-local-data", "--force", "--ignore-daemonsets").Output() + if err != nil { + return fmt.Errorf("failed to drain node %q [%s]", hostname, output) + } + + if removeNode { + fmt.Printf("[reset] Removing node: %q\n", hostname) + + output, err = exec.Command("kubectl", "--kubeconfig", kubeConfigPath, "delete", "node", hostname).Output() + if err != nil { + return fmt.Errorf("failed to remove node %q [%s]", hostname, output) + } + } + + return nil +} + // cleanDir removes everything in a directory, but not the directory itself func cleanDir(filepath string) error { // If the directory doesn't even exist there's nothing to do, and we do @@ -155,7 +209,7 @@ func resetConfigDir(configPathDir, pkiPathDir string) { for _, dir := range dirsToClean { err := cleanDir(dir) if err != nil { - fmt.Printf("[reset] Failed to remove directory: %q [%v]", dir, err) + fmt.Printf("[reset] Failed to remove directory: %q [%v]\n", dir, err) } } diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 54ac50c9b47..d6994d64368 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -489,6 +489,7 @@ registry-burst registry-qps reject-methods reject-paths +remove-node repair-malformed-updates replicaset-lookup-cache-size replication-controller-lookup-cache-size