diff --git a/cmd/kubeadm/app/cmd/BUILD b/cmd/kubeadm/app/cmd/BUILD index 5617b19a668..d34b8e2eb89 100644 --- a/cmd/kubeadm/app/cmd/BUILD +++ b/cmd/kubeadm/app/cmd/BUILD @@ -73,6 +73,7 @@ go_library( "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/util/cert:go_default_library", + "//vendor/k8s.io/utils/exec:go_default_library", ], ) @@ -87,6 +88,8 @@ go_test( deps = [ "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/preflight:go_default_library", + "//vendor/k8s.io/utils/exec:go_default_library", + "//vendor/k8s.io/utils/exec/testing:go_default_library", ], ) diff --git a/cmd/kubeadm/app/cmd/reset.go b/cmd/kubeadm/app/cmd/reset.go index 44edae99a56..619a45b1404 100644 --- a/cmd/kubeadm/app/cmd/reset.go +++ b/cmd/kubeadm/app/cmd/reset.go @@ -30,17 +30,23 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/preflight" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/pkg/util/initsystem" + utilsexec "k8s.io/utils/exec" +) + +var ( + crictlParamsFormat = "%s -r %s sandboxes --quiet | xargs -r %s -r %s rms" ) // NewCmdReset returns the "kubeadm reset" command func NewCmdReset(out io.Writer) *cobra.Command { var skipPreFlight bool var certsDir string + var criSocketPath string cmd := &cobra.Command{ Use: "reset", Short: "Run this to revert any changes made to this host by 'kubeadm init' or 'kubeadm join'.", Run: func(cmd *cobra.Command, args []string) { - r, err := NewReset(skipPreFlight, certsDir) + r, err := NewReset(skipPreFlight, certsDir, criSocketPath) kubeadmutil.CheckErr(err) kubeadmutil.CheckErr(r.Run(out)) }, @@ -56,16 +62,22 @@ func NewCmdReset(out io.Writer) *cobra.Command { "The path to the directory where the certificates are stored. If specified, clean this directory.", ) + cmd.PersistentFlags().StringVar( + &criSocketPath, "cri-socket", "/var/run/dockershim.sock", + "The path to the CRI socket to use with crictl when cleaning up containers.", + ) + return cmd } // Reset defines struct used for kubeadm reset command type Reset struct { - certsDir string + certsDir string + criSocketPath string } // NewReset instantiate Reset struct -func NewReset(skipPreFlight bool, certsDir string) (*Reset, error) { +func NewReset(skipPreFlight bool, certsDir, criSocketPath string) (*Reset, error) { if !skipPreFlight { fmt.Println("[preflight] Running pre-flight checks.") @@ -77,7 +89,8 @@ func NewReset(skipPreFlight bool, certsDir string) (*Reset, error) { } return &Reset{ - certsDir: certsDir, + certsDir: certsDir, + criSocketPath: criSocketPath, }, nil } @@ -105,15 +118,11 @@ func (r *Reset) Run(out io.Writer) error { fmt.Printf("[reset] Failed to unmount mounted directories in /var/lib/kubelet: %s\n", string(umountOutputBytes)) } + fmt.Println("[reset] Removing kubernetes-managed containers.") dockerCheck := preflight.ServiceCheck{Service: "docker", CheckIfActive: true} - if _, errors := dockerCheck.Check(); len(errors) == 0 { - fmt.Println("[reset] Removing kubernetes-managed containers.") - if err := exec.Command("sh", "-c", "docker ps -a --filter name=k8s_ -q | xargs -r docker rm --force --volumes").Run(); err != nil { - fmt.Println("[reset] Failed to stop the running containers.") - } - } else { - fmt.Println("[reset] Docker doesn't seem to be running. Skipping the removal of running Kubernetes containers.") - } + execer := utilsexec.New() + + reset(execer, dockerCheck, r.criSocketPath) dirsToClean := []string{"/var/lib/kubelet", "/etc/cni/net.d", "/var/lib/dockershim", "/var/run/kubernetes"} @@ -141,6 +150,39 @@ func (r *Reset) Run(out io.Writer) error { return nil } +func reset(execer utilsexec.Interface, dockerCheck preflight.Checker, criSocketPath string) { + crictlPath, err := execer.LookPath("crictl") + if err == nil { + resetWithCrictl(execer, dockerCheck, criSocketPath, crictlPath) + } else { + resetWithDocker(execer, dockerCheck) + } +} + +func resetWithDocker(execer utilsexec.Interface, dockerCheck preflight.Checker) { + if _, errors := dockerCheck.Check(); len(errors) == 0 { + if err := execer.Command("sh", "-c", "docker ps -a --filter name=k8s_ -q | xargs -r docker rm --force --volumes").Run(); err != nil { + fmt.Println("[reset] Failed to stop the running containers.") + } + } else { + fmt.Println("[reset] Docker doesn't seem to be running. Skipping the removal of running Kubernetes containers.") + } +} + +func resetWithCrictl(execer utilsexec.Interface, dockerCheck preflight.Checker, criSocketPath, crictlPath string) { + if criSocketPath != "" { + fmt.Printf("[reset] Cleaning up running containers using crictl with socket %s\n", criSocketPath) + cmd := fmt.Sprintf(crictlParamsFormat, crictlPath, criSocketPath, crictlPath, criSocketPath) + if err := execer.Command("sh", "-c", cmd).Run(); err != nil { + fmt.Println("[reset] Failed to stop the running containers using crictl. Trying using docker instead.") + resetWithDocker(execer, dockerCheck) + } + } else { + fmt.Println("[reset] CRI socket path not provided for crictl. Trying docker instead.") + resetWithDocker(execer, dockerCheck) + } +} + // cleanDir removes everything in a directory, but not the directory itself func cleanDir(filePath string) error { // If the directory doesn't even exist there's nothing to do, and we do diff --git a/cmd/kubeadm/app/cmd/reset_test.go b/cmd/kubeadm/app/cmd/reset_test.go index 1eeb0076dc1..a0e479c8b20 100644 --- a/cmd/kubeadm/app/cmd/reset_test.go +++ b/cmd/kubeadm/app/cmd/reset_test.go @@ -17,13 +17,17 @@ limitations under the License. package cmd import ( + "errors" "io/ioutil" "os" "path/filepath" + "strings" "testing" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/preflight" + "k8s.io/utils/exec" + fakeexec "k8s.io/utils/exec/testing" ) func assertExists(t *testing.T, path string) { @@ -181,3 +185,129 @@ func TestConfigDirCleaner(t *testing.T) { } } } + +type fakeDockerChecker struct { + warnings []error + errors []error +} + +func (c *fakeDockerChecker) Check() (warnings, errors []error) { + return c.warnings, c.errors +} + +func newFakeDockerChecker(warnings, errors []error) preflight.Checker { + return &fakeDockerChecker{warnings: warnings, errors: errors} +} + +func TestResetWithDocker(t *testing.T) { + fcmd := fakeexec.FakeCmd{ + RunScript: []fakeexec.FakeRunAction{ + func() ([]byte, []byte, error) { return nil, nil, nil }, + func() ([]byte, []byte, error) { return nil, nil, errors.New("docker error") }, + func() ([]byte, []byte, error) { return nil, nil, nil }, + }, + } + fexec := fakeexec.FakeExec{ + CommandScript: []fakeexec.FakeCommandAction{ + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + }, + } + resetWithDocker(&fexec, newFakeDockerChecker(nil, nil)) + if fcmd.RunCalls != 1 { + t.Errorf("expected 1 call to Run, got %d", fcmd.RunCalls) + } + resetWithDocker(&fexec, newFakeDockerChecker(nil, nil)) + if fcmd.RunCalls != 2 { + t.Errorf("expected 2 calls to Run, got %d", fcmd.RunCalls) + } + resetWithDocker(&fexec, newFakeDockerChecker(nil, []error{errors.New("test error")})) + if fcmd.RunCalls != 2 { + t.Errorf("expected 2 calls to Run, got %d", fcmd.RunCalls) + } +} + +func TestResetWithCrictl(t *testing.T) { + fcmd := fakeexec.FakeCmd{ + RunScript: []fakeexec.FakeRunAction{ + func() ([]byte, []byte, error) { return nil, nil, nil }, + func() ([]byte, []byte, error) { return nil, nil, nil }, + func() ([]byte, []byte, error) { return nil, nil, errors.New("crictl error") }, + func() ([]byte, []byte, error) { return nil, nil, nil }, + }, + } + fexec := fakeexec.FakeExec{ + CommandScript: []fakeexec.FakeCommandAction{ + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + }, + } + + resetWithCrictl(&fexec, newFakeDockerChecker(nil, nil), "", "crictl") + if fcmd.RunCalls != 1 { + t.Errorf("expected 1 call to Run, got %d", fcmd.RunCalls) + } + if !strings.Contains(fcmd.RunLog[0][2], "docker") { + t.Errorf("expected a call to docker, got %v", fcmd.RunLog[0]) + } + + resetWithCrictl(&fexec, newFakeDockerChecker(nil, nil), "/test.sock", "crictl") + if fcmd.RunCalls != 2 { + t.Errorf("expected 2 calls to Run, got %d", fcmd.RunCalls) + } + if !strings.Contains(fcmd.RunLog[1][2], "crictl") { + t.Errorf("expected a call to crictl, got %v", fcmd.RunLog[0]) + } + + resetWithCrictl(&fexec, newFakeDockerChecker(nil, nil), "/test.sock", "crictl") + if fcmd.RunCalls != 4 { + t.Errorf("expected 4 calls to Run, got %d", fcmd.RunCalls) + } + if !strings.Contains(fcmd.RunLog[2][2], "crictl") { + t.Errorf("expected a call to crictl, got %v", fcmd.RunLog[0]) + } + if !strings.Contains(fcmd.RunLog[3][2], "docker") { + t.Errorf("expected a call to docker, got %v", fcmd.RunLog[0]) + } + + resetWithCrictl(&fexec, newFakeDockerChecker(nil, []error{errors.New("test error")}), "", "crictl") + if fcmd.RunCalls != 4 { + t.Errorf("expected 4 calls to Run, got %d", fcmd.RunCalls) + } +} + +func TestReset(t *testing.T) { + fcmd := fakeexec.FakeCmd{ + RunScript: []fakeexec.FakeRunAction{ + func() ([]byte, []byte, error) { return nil, nil, nil }, + func() ([]byte, []byte, error) { return nil, nil, nil }, + }, + } + fexec := fakeexec.FakeExec{ + CommandScript: []fakeexec.FakeCommandAction{ + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + }, + LookPathFunc: func(cmd string) (string, error) { return cmd, nil }, + } + + reset(&fexec, newFakeDockerChecker(nil, nil), "/test.sock") + if fcmd.RunCalls != 1 { + t.Errorf("expected 1 call to Run, got %d", fcmd.RunCalls) + } + if !strings.Contains(fcmd.RunLog[0][2], "crictl") { + t.Errorf("expected a call to crictl, got %v", fcmd.RunLog[0]) + } + + fexec.LookPathFunc = func(cmd string) (string, error) { return "", errors.New("no crictl") } + reset(&fexec, newFakeDockerChecker(nil, nil), "/test.sock") + if fcmd.RunCalls != 2 { + t.Errorf("expected 2 calls to Run, got %d", fcmd.RunCalls) + } + if !strings.Contains(fcmd.RunLog[1][2], "docker") { + t.Errorf("expected a call to docker, got %v", fcmd.RunLog[0]) + } +}