kubeadm: reset: use crictl to reset containers

Signed-off-by: Antonio Murdaca <runcom@redhat.com>

cmd/kubeadm/app/cmd/BUILD

@@ -73,6 +73,7 @@ go_library(
         "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
         "//vendor/k8s.io/client-go/util/cert:go_default_library",
+        "//vendor/k8s.io/utils/exec:go_default_library",
     ],
 )
 
@@ -87,6 +88,8 @@ go_test(
     deps = [
         "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/preflight:go_default_library",
+        "//vendor/k8s.io/utils/exec:go_default_library",
+        "//vendor/k8s.io/utils/exec/testing:go_default_library",
     ],
 )
 

cmd/kubeadm/app/cmd/reset.go

@@ -30,17 +30,23 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	"k8s.io/kubernetes/pkg/util/initsystem"
+	utilsexec "k8s.io/utils/exec"
+)
+
+var (
+	crictlParamsFormat = "%s -r %s sandboxes --quiet | xargs -r %s -r %s rms"
 )
 
 // NewCmdReset returns the "kubeadm reset" command
 func NewCmdReset(out io.Writer) *cobra.Command {
 	var skipPreFlight bool
 	var certsDir string
+	var criSocketPath string
 	cmd := &cobra.Command{
 		Use:   "reset",
 		Short: "Run this to revert any changes made to this host by 'kubeadm init' or 'kubeadm join'.",
 		Run: func(cmd *cobra.Command, args []string) {
-			r, err := NewReset(skipPreFlight, certsDir)
+			r, err := NewReset(skipPreFlight, certsDir, criSocketPath)
 			kubeadmutil.CheckErr(err)
 			kubeadmutil.CheckErr(r.Run(out))
 		},
@@ -56,16 +62,22 @@ func NewCmdReset(out io.Writer) *cobra.Command {
 		"The path to the directory where the certificates are stored. If specified, clean this directory.",
 	)
 
+	cmd.PersistentFlags().StringVar(
+		&criSocketPath, "cri-socket", "/var/run/dockershim.sock",
+		"The path to the CRI socket to use with crictl when cleaning up containers.",
+	)
+
 	return cmd
 }
 
 // Reset defines struct used for kubeadm reset command
 type Reset struct {
-	certsDir string
+	certsDir      string
+	criSocketPath string
 }
 
 // NewReset instantiate Reset struct
-func NewReset(skipPreFlight bool, certsDir string) (*Reset, error) {
+func NewReset(skipPreFlight bool, certsDir, criSocketPath string) (*Reset, error) {
 	if !skipPreFlight {
 		fmt.Println("[preflight] Running pre-flight checks.")
 
@@ -77,7 +89,8 @@ func NewReset(skipPreFlight bool, certsDir string) (*Reset, error) {
 	}
 
 	return &Reset{
-		certsDir: certsDir,
+		certsDir:      certsDir,
+		criSocketPath: criSocketPath,
 	}, nil
 }
 
@@ -105,15 +118,11 @@ func (r *Reset) Run(out io.Writer) error {
 		fmt.Printf("[reset] Failed to unmount mounted directories in /var/lib/kubelet: %s\n", string(umountOutputBytes))
 	}
 
+	fmt.Println("[reset] Removing kubernetes-managed containers.")
 	dockerCheck := preflight.ServiceCheck{Service: "docker", CheckIfActive: true}
-	if _, errors := dockerCheck.Check(); len(errors) == 0 {
+	execer := utilsexec.New()
-		fmt.Println("[reset] Removing kubernetes-managed containers.")
+
-		if err := exec.Command("sh", "-c", "docker ps -a --filter name=k8s_ -q | xargs -r docker rm --force --volumes").Run(); err != nil {
+	reset(execer, dockerCheck, r.criSocketPath)
-			fmt.Println("[reset] Failed to stop the running containers.")
-		}
-	} else {
-		fmt.Println("[reset] Docker doesn't seem to be running. Skipping the removal of running Kubernetes containers.")
-	}
 
 	dirsToClean := []string{"/var/lib/kubelet", "/etc/cni/net.d", "/var/lib/dockershim", "/var/run/kubernetes"}
 
@@ -141,6 +150,39 @@ func (r *Reset) Run(out io.Writer) error {
 	return nil
 }
 
+func reset(execer utilsexec.Interface, dockerCheck preflight.Checker, criSocketPath string) {
+	crictlPath, err := execer.LookPath("crictl")
+	if err == nil {
+		resetWithCrictl(execer, dockerCheck, criSocketPath, crictlPath)
+	} else {
+		resetWithDocker(execer, dockerCheck)
+	}
+}
+
+func resetWithDocker(execer utilsexec.Interface, dockerCheck preflight.Checker) {
+	if _, errors := dockerCheck.Check(); len(errors) == 0 {
+		if err := execer.Command("sh", "-c", "docker ps -a --filter name=k8s_ -q | xargs -r docker rm --force --volumes").Run(); err != nil {
+			fmt.Println("[reset] Failed to stop the running containers.")
+		}
+	} else {
+		fmt.Println("[reset] Docker doesn't seem to be running. Skipping the removal of running Kubernetes containers.")
+	}
+}
+
+func resetWithCrictl(execer utilsexec.Interface, dockerCheck preflight.Checker, criSocketPath, crictlPath string) {
+	if criSocketPath != "" {
+		fmt.Printf("[reset] Cleaning up running containers using crictl with socket %s\n", criSocketPath)
+		cmd := fmt.Sprintf(crictlParamsFormat, crictlPath, criSocketPath, crictlPath, criSocketPath)
+		if err := execer.Command("sh", "-c", cmd).Run(); err != nil {
+			fmt.Println("[reset] Failed to stop the running containers using crictl. Trying using docker instead.")
+			resetWithDocker(execer, dockerCheck)
+		}
+	} else {
+		fmt.Println("[reset] CRI socket path not provided for crictl. Trying docker instead.")
+		resetWithDocker(execer, dockerCheck)
+	}
+}
+
 // cleanDir removes everything in a directory, but not the directory itself
 func cleanDir(filePath string) error {
 	// If the directory doesn't even exist there's nothing to do, and we do

cmd/kubeadm/app/cmd/reset_test.go

@@ -17,13 +17,17 @@ limitations under the License.
 package cmd
 
 import (
+	"errors"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
+	"k8s.io/utils/exec"
+	fakeexec "k8s.io/utils/exec/testing"
 )
 
 func assertExists(t *testing.T, path string) {
@@ -181,3 +185,129 @@ func TestConfigDirCleaner(t *testing.T) {
 		}
 	}
 }
+
+type fakeDockerChecker struct {
+	warnings []error
+	errors   []error
+}
+
+func (c *fakeDockerChecker) Check() (warnings, errors []error) {
+	return c.warnings, c.errors
+}
+
+func newFakeDockerChecker(warnings, errors []error) preflight.Checker {
+	return &fakeDockerChecker{warnings: warnings, errors: errors}
+}
+
+func TestResetWithDocker(t *testing.T) {
+	fcmd := fakeexec.FakeCmd{
+		RunScript: []fakeexec.FakeRunAction{
+			func() ([]byte, []byte, error) { return nil, nil, nil },
+			func() ([]byte, []byte, error) { return nil, nil, errors.New("docker error") },
+			func() ([]byte, []byte, error) { return nil, nil, nil },
+		},
+	}
+	fexec := fakeexec.FakeExec{
+		CommandScript: []fakeexec.FakeCommandAction{
+			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
+			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
+			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
+		},
+	}
+	resetWithDocker(&fexec, newFakeDockerChecker(nil, nil))
+	if fcmd.RunCalls != 1 {
+		t.Errorf("expected 1 call to Run, got %d", fcmd.RunCalls)
+	}
+	resetWithDocker(&fexec, newFakeDockerChecker(nil, nil))
+	if fcmd.RunCalls != 2 {
+		t.Errorf("expected 2 calls to Run, got %d", fcmd.RunCalls)
+	}
+	resetWithDocker(&fexec, newFakeDockerChecker(nil, []error{errors.New("test error")}))
+	if fcmd.RunCalls != 2 {
+		t.Errorf("expected 2 calls to Run, got %d", fcmd.RunCalls)
+	}
+}
+
+func TestResetWithCrictl(t *testing.T) {
+	fcmd := fakeexec.FakeCmd{
+		RunScript: []fakeexec.FakeRunAction{
+			func() ([]byte, []byte, error) { return nil, nil, nil },
+			func() ([]byte, []byte, error) { return nil, nil, nil },
+			func() ([]byte, []byte, error) { return nil, nil, errors.New("crictl error") },
+			func() ([]byte, []byte, error) { return nil, nil, nil },
+		},
+	}
+	fexec := fakeexec.FakeExec{
+		CommandScript: []fakeexec.FakeCommandAction{
+			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
+			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
+			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
+			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
+		},
+	}
+
+	resetWithCrictl(&fexec, newFakeDockerChecker(nil, nil), "", "crictl")
+	if fcmd.RunCalls != 1 {
+		t.Errorf("expected 1 call to Run, got %d", fcmd.RunCalls)
+	}
+	if !strings.Contains(fcmd.RunLog[0][2], "docker") {
+		t.Errorf("expected a call to docker, got %v", fcmd.RunLog[0])
+	}
+
+	resetWithCrictl(&fexec, newFakeDockerChecker(nil, nil), "/test.sock", "crictl")
+	if fcmd.RunCalls != 2 {
+		t.Errorf("expected 2 calls to Run, got %d", fcmd.RunCalls)
+	}
+	if !strings.Contains(fcmd.RunLog[1][2], "crictl") {
+		t.Errorf("expected a call to crictl, got %v", fcmd.RunLog[0])
+	}
+
+	resetWithCrictl(&fexec, newFakeDockerChecker(nil, nil), "/test.sock", "crictl")
+	if fcmd.RunCalls != 4 {
+		t.Errorf("expected 4 calls to Run, got %d", fcmd.RunCalls)
+	}
+	if !strings.Contains(fcmd.RunLog[2][2], "crictl") {
+		t.Errorf("expected a call to crictl, got %v", fcmd.RunLog[0])
+	}
+	if !strings.Contains(fcmd.RunLog[3][2], "docker") {
+		t.Errorf("expected a call to docker, got %v", fcmd.RunLog[0])
+	}
+
+	resetWithCrictl(&fexec, newFakeDockerChecker(nil, []error{errors.New("test error")}), "", "crictl")
+	if fcmd.RunCalls != 4 {
+		t.Errorf("expected 4 calls to Run, got %d", fcmd.RunCalls)
+	}
+}
+
+func TestReset(t *testing.T) {
+	fcmd := fakeexec.FakeCmd{
+		RunScript: []fakeexec.FakeRunAction{
+			func() ([]byte, []byte, error) { return nil, nil, nil },
+			func() ([]byte, []byte, error) { return nil, nil, nil },
+		},
+	}
+	fexec := fakeexec.FakeExec{
+		CommandScript: []fakeexec.FakeCommandAction{
+			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
+			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
+		},
+		LookPathFunc: func(cmd string) (string, error) { return cmd, nil },
+	}
+
+	reset(&fexec, newFakeDockerChecker(nil, nil), "/test.sock")
+	if fcmd.RunCalls != 1 {
+		t.Errorf("expected 1 call to Run, got %d", fcmd.RunCalls)
+	}
+	if !strings.Contains(fcmd.RunLog[0][2], "crictl") {
+		t.Errorf("expected a call to crictl, got %v", fcmd.RunLog[0])
+	}
+
+	fexec.LookPathFunc = func(cmd string) (string, error) { return "", errors.New("no crictl") }
+	reset(&fexec, newFakeDockerChecker(nil, nil), "/test.sock")
+	if fcmd.RunCalls != 2 {
+		t.Errorf("expected 2 calls to Run, got %d", fcmd.RunCalls)
+	}
+	if !strings.Contains(fcmd.RunLog[1][2], "docker") {
+		t.Errorf("expected a call to docker, got %v", fcmd.RunLog[0])
+	}
+}