diff --git a/cmd/kubeadm/app/cmd/BUILD b/cmd/kubeadm/app/cmd/BUILD index 83d9dcddab0..a6c63813eff 100644 --- a/cmd/kubeadm/app/cmd/BUILD +++ b/cmd/kubeadm/app/cmd/BUILD @@ -54,6 +54,7 @@ go_library( "//cmd/kubeadm/app/util/config:go_default_library", "//cmd/kubeadm/app/util/dryrun:go_default_library", "//cmd/kubeadm/app/util/kubeconfig:go_default_library", + "//cmd/kubeadm/app/util/runtime:go_default_library", "//pkg/util/initsystem:go_default_library", "//pkg/version:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", @@ -96,6 +97,7 @@ go_test( "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/preflight:go_default_library", "//cmd/kubeadm/app/util/config:go_default_library", + "//cmd/kubeadm/app/util/runtime:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/cmd/kubeadm/app/cmd/config.go b/cmd/kubeadm/app/cmd/config.go index d2cece6fcd3..3789392ce3c 100644 --- a/cmd/kubeadm/app/cmd/config.go +++ b/cmd/kubeadm/app/cmd/config.go @@ -43,6 +43,7 @@ import ( kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" + utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime" utilsexec "k8s.io/utils/exec" ) @@ -407,9 +408,9 @@ func NewCmdConfigImagesPull() *cobra.Command { kubeadmutil.CheckErr(err) internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, cfg) kubeadmutil.CheckErr(err) - puller, err := images.NewCRInterfacer(utilsexec.New(), internalcfg.GetCRISocket()) + containerRuntime, err := utilruntime.NewContainerRuntime(utilsexec.New(), internalcfg.GetCRISocket()) kubeadmutil.CheckErr(err) - imagesPull := NewImagesPull(puller, images.GetAllImages(internalcfg)) + imagesPull := NewImagesPull(containerRuntime, images.GetAllImages(internalcfg)) kubeadmutil.CheckErr(imagesPull.PullAll()) }, } @@ -421,22 +422,22 @@ func NewCmdConfigImagesPull() *cobra.Command { // ImagesPull is the struct used to hold information relating to image pulling type ImagesPull struct { - puller images.Puller - images []string + runtime utilruntime.ContainerRuntime + images []string } // NewImagesPull initializes and returns the `kubeadm config images pull` command -func NewImagesPull(puller images.Puller, images []string) *ImagesPull { +func NewImagesPull(runtime utilruntime.ContainerRuntime, images []string) *ImagesPull { return &ImagesPull{ - puller: puller, - images: images, + runtime: runtime, + images: images, } } // PullAll pulls all images that the ImagesPull knows about func (ip *ImagesPull) PullAll() error { for _, image := range ip.images { - if err := ip.puller.Pull(image); err != nil { + if err := ip.runtime.PullImage(image); err != nil { return fmt.Errorf("failed to pull image %q: %v", image, err) } fmt.Printf("[config/images] Pulled %s\n", image) diff --git a/cmd/kubeadm/app/cmd/config_test.go b/cmd/kubeadm/app/cmd/config_test.go index 980e1ec723f..e66e41939c8 100644 --- a/cmd/kubeadm/app/cmd/config_test.go +++ b/cmd/kubeadm/app/cmd/config_test.go @@ -30,6 +30,9 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/cmd" "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/util/config" + utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime" + "k8s.io/utils/exec" + fakeexec "k8s.io/utils/exec/testing" ) const ( @@ -181,27 +184,43 @@ func TestConfigImagesListRunWithoutPath(t *testing.T) { } } -type fakePuller struct { - count map[string]int -} - -func (f *fakePuller) Pull(image string) error { - f.count[image]++ - return nil -} - func TestImagesPull(t *testing.T) { - puller := &fakePuller{ - count: make(map[string]int), + fcmd := fakeexec.FakeCmd{ + RunScript: []fakeexec.FakeRunAction{ + func() ([]byte, []byte, error) { return nil, nil, nil }, + func() ([]byte, []byte, error) { return nil, nil, nil }, + func() ([]byte, []byte, error) { return nil, nil, nil }, + func() ([]byte, []byte, error) { return nil, nil, nil }, + func() ([]byte, []byte, error) { return nil, nil, nil }, + }, } + + fexec := fakeexec.FakeExec{ + CommandScript: []fakeexec.FakeCommandAction{ + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + }, + LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/docker", nil }, + } + + containerRuntime, err := utilruntime.NewContainerRuntime(&fexec, kubeadmapiv1alpha3.DefaultCRISocket) + if err != nil { + t.Errorf("unexpected NewContainerRuntime error: %v", err) + } + images := []string{"a", "b", "c", "d", "a"} - ip := cmd.NewImagesPull(puller, images) - err := ip.PullAll() + ip := cmd.NewImagesPull(containerRuntime, images) + + err = ip.PullAll() if err != nil { t.Fatalf("expected nil but found %v", err) } - if puller.count["a"] != 2 { - t.Fatalf("expected 2 but found %v", puller.count["a"]) + + if fcmd.RunCalls != len(images) { + t.Errorf("expected %d docker calls, got %d", len(images), fcmd.RunCalls) } } diff --git a/cmd/kubeadm/app/cmd/reset.go b/cmd/kubeadm/app/cmd/reset.go index c5f59530937..60a5307ffe4 100644 --- a/cmd/kubeadm/app/cmd/reset.go +++ b/cmd/kubeadm/app/cmd/reset.go @@ -35,6 +35,7 @@ import ( kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/preflight" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime" "k8s.io/kubernetes/pkg/util/initsystem" utilsexec "k8s.io/utils/exec" ) @@ -147,12 +148,10 @@ func (r *Reset) Run(out io.Writer) error { glog.Errorf("[reset] failed to unmount mounted directories in /var/lib/kubelet: %s\n", string(umountOutputBytes)) } - fmt.Println("[reset] removing kubernetes-managed containers") - dockerCheck := preflight.ServiceCheck{Service: "docker", CheckIfActive: true} - execer := utilsexec.New() - - reset(execer, dockerCheck, r.criSocketPath) - + glog.V(1).Info("[reset] removing kubernetes-managed containers") + if err := removeContainers(utilsexec.New(), r.criSocketPath); err != nil { + glog.Errorf("[reset] failed to remove containers: %+v", err) + } dirsToClean := []string{"/var/lib/kubelet", "/etc/cni/net.d", "/var/lib/dockershim", "/var/run/kubernetes"} // Only clear etcd data when the etcd manifest is found. In case it is not found, we must assume that the user @@ -184,63 +183,19 @@ func (r *Reset) Run(out io.Writer) error { return nil } -func reset(execer utilsexec.Interface, dockerCheck preflight.Checker, criSocketPath string) { - crictlPath, err := execer.LookPath("crictl") - if err == nil { - resetWithCrictl(execer, dockerCheck, criSocketPath, crictlPath) - } else { - resetWithDocker(execer, dockerCheck) +func removeContainers(execer utilsexec.Interface, criSocketPath string) error { + containerRuntime, err := utilruntime.NewContainerRuntime(execer, criSocketPath) + if err != nil { + return err } -} - -func resetWithDocker(execer utilsexec.Interface, dockerCheck preflight.Checker) { - if _, errors := dockerCheck.Check(); len(errors) == 0 { - if err := execer.Command("sh", "-c", "docker ps -a --filter name=k8s_ -q | xargs -r docker rm --force --volumes").Run(); err != nil { - glog.Errorln("[reset] failed to stop the running containers") - } - } else { - fmt.Println("[reset] docker doesn't seem to be running. Skipping the removal of running Kubernetes containers") + containers, err := containerRuntime.ListKubeContainers() + if err != nil { + return err } -} - -func resetWithCrictl(execer utilsexec.Interface, dockerCheck preflight.Checker, criSocketPath, crictlPath string) { - if criSocketPath != "" { - fmt.Printf("[reset] cleaning up running containers using crictl with socket %s\n", criSocketPath) - glog.V(1).Infoln("[reset] listing running pods using crictl") - - params := []string{"-r", criSocketPath, "pods", "--quiet"} - glog.V(1).Infof("[reset] Executing command %s %s", crictlPath, strings.Join(params, " ")) - output, err := execer.Command(crictlPath, params...).CombinedOutput() - if err != nil { - fmt.Printf("[reset] failed to list running pods using crictl: %v. Trying to use docker instead", err) - resetWithDocker(execer, dockerCheck) - return - } - sandboxes := strings.Fields(string(output)) - glog.V(1).Infoln("[reset] Stopping and removing running containers using crictl") - for _, s := range sandboxes { - if strings.TrimSpace(s) == "" { - continue - } - params = []string{"-r", criSocketPath, "stopp", s} - glog.V(1).Infof("[reset] Executing command %s %s", crictlPath, strings.Join(params, " ")) - if err := execer.Command(crictlPath, params...).Run(); err != nil { - fmt.Printf("[reset] failed to stop the running containers using crictl: %v. Trying to use docker instead", err) - resetWithDocker(execer, dockerCheck) - return - } - params = []string{"-r", criSocketPath, "rmp", s} - glog.V(1).Infof("[reset] Executing command %s %s", crictlPath, strings.Join(params, " ")) - if err := execer.Command(crictlPath, params...).Run(); err != nil { - fmt.Printf("[reset] failed to remove the running containers using crictl: %v. Trying to use docker instead", err) - resetWithDocker(execer, dockerCheck) - return - } - } - } else { - fmt.Println("[reset] CRI socket path not provided for crictl. Trying to use docker instead") - resetWithDocker(execer, dockerCheck) + if err := containerRuntime.RemoveContainers(containers); err != nil { + return err } + return nil } // cleanDir removes everything in a directory, but not the directory itself diff --git a/cmd/kubeadm/app/cmd/reset_test.go b/cmd/kubeadm/app/cmd/reset_test.go index 8029e3e6721..6ab732eef89 100644 --- a/cmd/kubeadm/app/cmd/reset_test.go +++ b/cmd/kubeadm/app/cmd/reset_test.go @@ -17,12 +17,10 @@ limitations under the License. package cmd import ( - "errors" "io" "io/ioutil" "os" "path/filepath" - "strings" "testing" kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3" @@ -247,112 +245,13 @@ func newFakeDockerChecker(warnings, errors []error) preflight.Checker { return &fakeDockerChecker{warnings: warnings, errors: errors} } -func TestResetWithDocker(t *testing.T) { - fcmd := fakeexec.FakeCmd{ - RunScript: []fakeexec.FakeRunAction{ - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, errors.New("docker error") }, - func() ([]byte, []byte, error) { return nil, nil, nil }, - }, - } - fexec := fakeexec.FakeExec{ - CommandScript: []fakeexec.FakeCommandAction{ - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - }, - } - resetWithDocker(&fexec, newFakeDockerChecker(nil, nil)) - if fcmd.RunCalls != 1 { - t.Errorf("expected 1 call to Run, got %d", fcmd.RunCalls) - } - resetWithDocker(&fexec, newFakeDockerChecker(nil, nil)) - if fcmd.RunCalls != 2 { - t.Errorf("expected 2 calls to Run, got %d", fcmd.RunCalls) - } - resetWithDocker(&fexec, newFakeDockerChecker(nil, []error{errors.New("test error")})) - if fcmd.RunCalls != 2 { - t.Errorf("expected 2 calls to Run, got %d", fcmd.RunCalls) - } -} - -func TestResetWithCrictl(t *testing.T) { +func TestRemoveContainers(t *testing.T) { fcmd := fakeexec.FakeCmd{ CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{ - // 2: socket path provided, not running with crictl (1x CombinedOutput, 2x Run) - func() ([]byte, error) { return []byte("1"), nil }, - // 3: socket path provided, crictl fails, reset with docker (1x CombinedOuput fail, 1x Run) - func() ([]byte, error) { return nil, errors.New("crictl list err") }, + func() ([]byte, error) { return []byte("id1\nid2"), nil }, }, RunScript: []fakeexec.FakeRunAction{ - // 1: socket path not provided, running with docker func() ([]byte, []byte, error) { return nil, nil, nil }, - // 2: socket path provided, now running with crictl (1x CombinedOutput, 2x Run) - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, nil }, - // 3: socket path provided, crictl fails, reset with docker (1x CombinedOuput, 1x Run) - func() ([]byte, []byte, error) { return nil, nil, nil }, - // 4: running with no socket and docker fails (1x Run) - func() ([]byte, []byte, error) { return nil, nil, nil }, - }, - } - fexec := fakeexec.FakeExec{ - CommandScript: []fakeexec.FakeCommandAction{ - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - }, - } - - // 1: socket path not provided, running with docker - resetWithCrictl(&fexec, newFakeDockerChecker(nil, nil), "", "crictl") - if fcmd.RunCalls != 1 { - t.Errorf("expected 1 call to Run, got %d", fcmd.RunCalls) - } - if !strings.Contains(fcmd.RunLog[0][2], "docker") { - t.Errorf("expected a call to docker, got %v", fcmd.RunLog[0]) - } - - // 2: socket path provided, now running with crictl (1x CombinedOutput, 2x Run) - resetWithCrictl(&fexec, newFakeDockerChecker(nil, nil), "/test.sock", "crictl") - if fcmd.RunCalls != 3 { - t.Errorf("expected 3 calls to Run, got %d", fcmd.RunCalls) - } - if !strings.Contains(fcmd.RunLog[1][0], "crictl") { - t.Errorf("expected a call to crictl, got %v", fcmd.RunLog[0]) - } - if !strings.Contains(fcmd.RunLog[2][0], "crictl") { - t.Errorf("expected a call to crictl, got %v", fcmd.RunLog[0]) - } - - // 3: socket path provided, crictl fails, reset with docker - resetWithCrictl(&fexec, newFakeDockerChecker(nil, nil), "/test.sock", "crictl") - if fcmd.RunCalls != 4 { - t.Errorf("expected 4 calls to Run, got %d", fcmd.RunCalls) - } - if !strings.Contains(fcmd.RunLog[3][2], "docker") { - t.Errorf("expected a call to docker, got %v", fcmd.RunLog[0]) - } - - // 4: running with no socket and docker fails (1x Run) - resetWithCrictl(&fexec, newFakeDockerChecker(nil, []error{errors.New("test error")}), "", "crictl") - if fcmd.RunCalls != 4 { - t.Errorf("expected 4 calls to Run, got %d", fcmd.RunCalls) - } -} - -func TestReset(t *testing.T) { - fcmd := fakeexec.FakeCmd{ - CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{ - func() ([]byte, error) { return []byte("1"), nil }, - func() ([]byte, error) { return []byte("1"), nil }, - func() ([]byte, error) { return []byte("1"), nil }, - }, - RunScript: []fakeexec.FakeRunAction{ func() ([]byte, []byte, error) { return nil, nil, nil }, func() ([]byte, []byte, error) { return nil, nil, nil }, func() ([]byte, []byte, error) { return nil, nil, nil }, @@ -365,25 +264,9 @@ func TestReset(t *testing.T) { func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, - LookPathFunc: func(cmd string) (string, error) { return cmd, nil }, + LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, } - reset(&fexec, newFakeDockerChecker(nil, nil), "/test.sock") - if fcmd.RunCalls != 2 { - t.Errorf("expected 2 call to Run, got %d", fcmd.RunCalls) - } - if !strings.Contains(fcmd.RunLog[0][0], "crictl") { - t.Errorf("expected a call to crictl, got %v", fcmd.RunLog[0]) - } - - fexec.LookPathFunc = func(cmd string) (string, error) { return "", errors.New("no crictl") } - reset(&fexec, newFakeDockerChecker(nil, nil), "/test.sock") - if fcmd.RunCalls != 3 { - t.Errorf("expected 3 calls to Run, got %d", fcmd.RunCalls) - } - if !strings.Contains(fcmd.RunLog[2][2], "docker") { - t.Errorf("expected a call to docker, got %v", fcmd.RunLog[0]) - } + removeContainers(&fexec, "unix:///var/run/crio/crio.sock") } diff --git a/cmd/kubeadm/app/images/BUILD b/cmd/kubeadm/app/images/BUILD index d8d5c745b09..a7ef57fb287 100644 --- a/cmd/kubeadm/app/images/BUILD +++ b/cmd/kubeadm/app/images/BUILD @@ -8,33 +8,23 @@ load( go_library( name = "go_default_library", - srcs = [ - "images.go", - "interface.go", - ], + srcs = ["images.go"], importpath = "k8s.io/kubernetes/cmd/kubeadm/app/images", deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/v1alpha3:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/features:go_default_library", "//cmd/kubeadm/app/util:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", ], ) go_test( name = "go_default_test", - srcs = [ - "images_test.go", - "interface_test.go", - ], + srcs = ["images_test.go"], embed = [":go_default_library"], deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/v1alpha3:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", ], ) diff --git a/cmd/kubeadm/app/images/interface.go b/cmd/kubeadm/app/images/interface.go deleted file mode 100644 index e52ad2c8061..00000000000 --- a/cmd/kubeadm/app/images/interface.go +++ /dev/null @@ -1,89 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package images - -import ( - "fmt" - - kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3" - utilsexec "k8s.io/utils/exec" -) - -// Puller is an interface for pulling images -type Puller interface { - Pull(string) error -} - -// Existence is an interface to determine if an image exists on the system -// A nil error means the image was found -type Existence interface { - Exists(string) error -} - -// Images defines the set of behaviors needed for images relating to the CRI -type Images interface { - Puller - Existence -} - -// CRInterfacer is a struct that interfaces with the container runtime -type CRInterfacer struct { - criSocket string - exec utilsexec.Interface - crictlPath string - dockerPath string -} - -// NewCRInterfacer sets up and returns a CRInterfacer -func NewCRInterfacer(execer utilsexec.Interface, criSocket string) (*CRInterfacer, error) { - var crictlPath, dockerPath string - var err error - if criSocket != kubeadmapiv1alpha3.DefaultCRISocket { - if crictlPath, err = execer.LookPath("crictl"); err != nil { - return nil, fmt.Errorf("crictl is required for non docker container runtimes: %v", err) - } - } else { - // use the dockershim - if dockerPath, err = execer.LookPath("docker"); err != nil { - return nil, fmt.Errorf("`docker` is required when docker is the container runtime and the kubelet is not running: %v", err) - } - } - - return &CRInterfacer{ - exec: execer, - criSocket: criSocket, - crictlPath: crictlPath, - dockerPath: dockerPath, - }, nil -} - -// Pull pulls the actual image using either crictl or docker -func (cri *CRInterfacer) Pull(image string) error { - if cri.criSocket != kubeadmapiv1alpha3.DefaultCRISocket { - return cri.exec.Command(cri.crictlPath, "-r", cri.criSocket, "pull", image).Run() - } - return cri.exec.Command(cri.dockerPath, "pull", image).Run() -} - -// Exists checks to see if the image exists on the system already -// Returns an error if the image is not found. -func (cri *CRInterfacer) Exists(image string) error { - if cri.criSocket != kubeadmapiv1alpha3.DefaultCRISocket { - return cri.exec.Command(cri.crictlPath, "-r", cri.criSocket, "inspecti", image).Run() - } - return cri.exec.Command(cri.dockerPath, "inspect", image).Run() -} diff --git a/cmd/kubeadm/app/images/interface_test.go b/cmd/kubeadm/app/images/interface_test.go deleted file mode 100644 index 890aa6397f8..00000000000 --- a/cmd/kubeadm/app/images/interface_test.go +++ /dev/null @@ -1,266 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package images_test - -import ( - "context" - "errors" - "io" - "testing" - - kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3" - "k8s.io/kubernetes/cmd/kubeadm/app/images" - "k8s.io/utils/exec" -) - -type fakeCmd struct { - err error -} - -func (f *fakeCmd) Run() error { - return f.err -} -func (f *fakeCmd) CombinedOutput() ([]byte, error) { return nil, nil } -func (f *fakeCmd) Output() ([]byte, error) { return nil, nil } -func (f *fakeCmd) SetDir(dir string) {} -func (f *fakeCmd) SetStdin(in io.Reader) {} -func (f *fakeCmd) SetStdout(out io.Writer) {} -func (f *fakeCmd) SetStderr(out io.Writer) {} -func (f *fakeCmd) Stop() {} - -type fakeExecer struct { - cmd exec.Cmd - findCrictl bool - findDocker bool -} - -func (f *fakeExecer) Command(cmd string, args ...string) exec.Cmd { return f.cmd } -func (f *fakeExecer) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd { - return f.cmd -} -func (f *fakeExecer) LookPath(file string) (string, error) { - if file == "crictl" { - if f.findCrictl { - return "/path", nil - } - return "", errors.New("no crictl for you") - } - if file == "docker" { - if f.findDocker { - return "/path", nil - } - return "", errors.New("no docker for you") - } - return "", errors.New("unknown binary") -} - -func TestNewCRInterfacer(t *testing.T) { - testcases := []struct { - name string - criSocket string - findCrictl bool - findDocker bool - expectError bool - }{ - { - name: "need crictl but can only find docker should return an error", - criSocket: "/not/docker", - findCrictl: false, - findDocker: true, - expectError: true, - }, - { - name: "need crictl and cannot find either should return an error", - criSocket: "/not/docker", - findCrictl: false, - findDocker: false, - expectError: true, - }, - { - name: "need crictl and cannot find docker should return no error", - criSocket: "/not/docker", - findCrictl: true, - findDocker: false, - expectError: false, - }, - { - name: "need crictl and can find both should return no error", - criSocket: "/not/docker", - findCrictl: true, - findDocker: true, - expectError: false, - }, - { - name: "need docker and cannot find crictl should return no error", - criSocket: kubeadmapiv1alpha3.DefaultCRISocket, - findCrictl: false, - findDocker: true, - expectError: false, - }, - { - name: "need docker and cannot find docker should return an error", - criSocket: kubeadmapiv1alpha3.DefaultCRISocket, - findCrictl: false, - findDocker: false, - expectError: true, - }, - { - name: "need docker and can find both should return no error", - criSocket: kubeadmapiv1alpha3.DefaultCRISocket, - findCrictl: true, - findDocker: true, - expectError: false, - }, - { - name: "need docker and can only find crictl should return an error", - criSocket: kubeadmapiv1alpha3.DefaultCRISocket, - findCrictl: true, - findDocker: false, - expectError: true, - }, - } - - for _, tc := range testcases { - t.Run(tc.name, func(t *testing.T) { - fe := &fakeExecer{ - findCrictl: tc.findCrictl, - findDocker: tc.findDocker, - } - _, err := images.NewCRInterfacer(fe, tc.criSocket) - if tc.expectError && err == nil { - t.Fatal("expected an error but did not get one") - } - if !tc.expectError && err != nil { - t.Fatalf("did not expedt an error but got an error: %v", err) - } - }) - } -} - -func TestImagePuller(t *testing.T) { - testcases := []struct { - name string - criSocket string - pullFails bool - errorExpected bool - }{ - { - name: "using docker and pull fails", - criSocket: kubeadmapiv1alpha3.DefaultCRISocket, - pullFails: true, - errorExpected: true, - }, - { - name: "using docker and pull succeeds", - criSocket: kubeadmapiv1alpha3.DefaultCRISocket, - pullFails: false, - errorExpected: false, - }, - { - name: "using crictl pull fails", - criSocket: "/not/default", - pullFails: true, - errorExpected: true, - }, - { - name: "using crictl and pull succeeds", - criSocket: "/not/default", - pullFails: false, - errorExpected: false, - }, - } - for _, tc := range testcases { - t.Run(tc.name, func(t *testing.T) { - var err error - if tc.pullFails { - err = errors.New("error") - } - - fe := &fakeExecer{ - cmd: &fakeCmd{err}, - findCrictl: true, - findDocker: true, - } - ip, _ := images.NewCRInterfacer(fe, tc.criSocket) - - err = ip.Pull("imageName") - if tc.errorExpected && err == nil { - t.Fatal("expected an error and did not get one") - } - if !tc.errorExpected && err != nil { - t.Fatalf("expected no error but got one: %v", err) - } - }) - } -} - -func TestImageExists(t *testing.T) { - testcases := []struct { - name string - criSocket string - existFails bool - errorExpected bool - }{ - { - name: "using docker and exist fails", - criSocket: kubeadmapiv1alpha3.DefaultCRISocket, - existFails: true, - errorExpected: true, - }, - { - name: "using docker and exist succeeds", - criSocket: kubeadmapiv1alpha3.DefaultCRISocket, - existFails: false, - errorExpected: false, - }, - { - name: "using crictl exist fails", - criSocket: "/not/default", - existFails: true, - errorExpected: true, - }, - { - name: "using crictl and exist succeeds", - criSocket: "/not/default", - existFails: false, - errorExpected: false, - }, - } - for _, tc := range testcases { - t.Run(tc.name, func(t *testing.T) { - var err error - if tc.existFails { - err = errors.New("error") - } - - fe := &fakeExecer{ - cmd: &fakeCmd{err}, - findCrictl: true, - findDocker: true, - } - ip, _ := images.NewCRInterfacer(fe, tc.criSocket) - - err = ip.Exists("imageName") - if tc.errorExpected && err == nil { - t.Fatal("expected an error and did not get one") - } - if !tc.errorExpected && err != nil { - t.Fatalf("expected no error but got one: %v", err) - } - }) - } -} diff --git a/cmd/kubeadm/app/preflight/BUILD b/cmd/kubeadm/app/preflight/BUILD index 81b4e616c89..287db1c2e79 100644 --- a/cmd/kubeadm/app/preflight/BUILD +++ b/cmd/kubeadm/app/preflight/BUILD @@ -17,9 +17,9 @@ go_library( importpath = "k8s.io/kubernetes/cmd/kubeadm/app/preflight", deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/v1alpha3:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/images:go_default_library", + "//cmd/kubeadm/app/util/runtime:go_default_library", "//cmd/kubeadm/app/util/system:go_default_library", "//pkg/registry/core/service/ipallocator:go_default_library", "//pkg/util/initsystem:go_default_library", @@ -44,6 +44,8 @@ go_test( embed = [":go_default_library"], deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//cmd/kubeadm/app/apis/kubeadm/v1alpha3:go_default_library", + "//cmd/kubeadm/app/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/github.com/renstrom/dedent:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", diff --git a/cmd/kubeadm/app/preflight/checks.go b/cmd/kubeadm/app/preflight/checks.go index 9de7cd6c99a..d4734972985 100644 --- a/cmd/kubeadm/app/preflight/checks.go +++ b/cmd/kubeadm/app/preflight/checks.go @@ -42,9 +42,9 @@ import ( netutil "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/sets" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/images" + utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime" "k8s.io/kubernetes/cmd/kubeadm/app/util/system" "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" "k8s.io/kubernetes/pkg/util/initsystem" @@ -90,28 +90,21 @@ type Checker interface { Name() string } -// CRICheck verifies the container runtime through the CRI. -type CRICheck struct { - socket string - exec utilsexec.Interface +// ContainerRuntimeCheck verifies the container runtime. +type ContainerRuntimeCheck struct { + runtime utilruntime.ContainerRuntime } -// Name returns label for CRICheck. -func (CRICheck) Name() string { +// Name returns label for RuntimeCheck. +func (ContainerRuntimeCheck) Name() string { return "CRI" } -// Check validates the container runtime through the CRI. -func (criCheck CRICheck) Check() (warnings, errors []error) { - glog.V(1).Infoln("validating the container runtime through the CRI") - crictlPath, err := criCheck.exec.LookPath("crictl") - if err != nil { - errors = append(errors, fmt.Errorf("unable to find command crictl: %s", err)) - return warnings, errors - } - if err := criCheck.exec.Command(crictlPath, "-r", criCheck.socket, "info").Run(); err != nil { - errors = append(errors, fmt.Errorf("unable to check if the container runtime at %q is running: %s", criCheck.socket, err)) - return warnings, errors +// Check validates the container runtime +func (crc ContainerRuntimeCheck) Check() (warnings, errors []error) { + glog.V(1).Infoln("validating the container runtime") + if err := crc.runtime.IsRunning(); err != nil { + errors = append(errors, err) } return warnings, errors } @@ -510,7 +503,7 @@ func (subnet HTTPProxyCIDRCheck) Check() (warnings, errors []error) { // SystemVerificationCheck defines struct used for for running the system verification node check in test/e2e_node/system type SystemVerificationCheck struct { - CRISocket string + IsDocker bool } // Name will return SystemVerification as name for SystemVerificationCheck @@ -532,9 +525,8 @@ func (sysver SystemVerificationCheck) Check() (warnings, errors []error) { var validators = []system.Validator{ &system.KernelValidator{Reporter: reporter}} - // run the docker validator only with dockershim - if sysver.CRISocket == kubeadmapiv1alpha3.DefaultCRISocket { - // https://github.com/kubernetes/kubeadm/issues/533 + // run the docker validator only with docker runtime + if sysver.IsDocker { validators = append(validators, &system.DockerValidator{Reporter: reporter}) } @@ -825,8 +817,8 @@ func getEtcdVersionResponse(client *http.Client, url string, target interface{}) // ImagePullCheck will pull container images used by kubeadm type ImagePullCheck struct { - Images images.Images - ImageList []string + runtime utilruntime.ContainerRuntime + imageList []string } // Name returns the label for ImagePullCheck @@ -835,14 +827,18 @@ func (ImagePullCheck) Name() string { } // Check pulls images required by kubeadm. This is a mutating check -func (i ImagePullCheck) Check() (warnings, errors []error) { - for _, image := range i.ImageList { +func (ipc ImagePullCheck) Check() (warnings, errors []error) { + for _, image := range ipc.imageList { glog.V(1).Infoln("pulling ", image) - if err := i.Images.Exists(image); err == nil { + ret, err := ipc.runtime.ImageExists(image) + if ret && err == nil { continue } - if err := i.Images.Pull(image); err != nil { - errors = append(errors, fmt.Errorf("failed to pull image [%s]: %v", image, err)) + if err != nil { + errors = append(errors, fmt.Errorf("failed to check if image %s exists: %v", image, err)) + } + if err := ipc.runtime.PullImage(image); err != nil { + errors = append(errors, fmt.Errorf("failed to pull image %s: %v", image, err)) } } return warnings, errors @@ -957,11 +953,16 @@ func RunJoinNodeChecks(execer utilsexec.Interface, cfg *kubeadmapi.JoinConfigura // addCommonChecks is a helper function to deplicate checks that are common between both the // kubeadm init and join commands func addCommonChecks(execer utilsexec.Interface, cfg kubeadmapi.CommonConfiguration, checks []Checker) []Checker { - // Check whether or not the CRI socket defined is the default - if cfg.GetCRISocket() != kubeadmapiv1alpha3.DefaultCRISocket { - checks = append(checks, CRICheck{socket: cfg.GetCRISocket(), exec: execer}) + containerRuntime, err := utilruntime.NewContainerRuntime(execer, cfg.GetCRISocket()) + isDocker := false + if err != nil { + fmt.Printf("[preflight] WARNING: Couldn't create the interface used for talking to the container runtime: %v\n", err) } else { - checks = append(checks, ServiceCheck{Service: "docker", CheckIfActive: true}) + checks = append(checks, ContainerRuntimeCheck{runtime: containerRuntime}) + if containerRuntime.IsDocker() { + isDocker = true + checks = append(checks, ServiceCheck{Service: "docker", CheckIfActive: true}) + } } // non-windows checks @@ -982,7 +983,7 @@ func addCommonChecks(execer utilsexec.Interface, cfg kubeadmapi.CommonConfigurat InPathCheck{executable: "touch", mandatory: false, exec: execer}) } checks = append(checks, - SystemVerificationCheck{CRISocket: cfg.GetCRISocket()}, + SystemVerificationCheck{IsDocker: isDocker}, IsPrivilegedUserCheck{}, HostnameCheck{nodeName: cfg.GetNodeName()}, KubeletVersionCheck{KubernetesVersion: cfg.GetKubernetesVersion(), exec: execer}, @@ -1002,13 +1003,13 @@ func RunRootCheckOnly(ignorePreflightErrors sets.String) error { // RunPullImagesCheck will pull images kubeadm needs if the are not found on the system func RunPullImagesCheck(execer utilsexec.Interface, cfg *kubeadmapi.InitConfiguration, ignorePreflightErrors sets.String) error { - criInterfacer, err := images.NewCRInterfacer(execer, cfg.GetCRISocket()) + containerRuntime, err := utilruntime.NewContainerRuntime(utilsexec.New(), cfg.GetCRISocket()) if err != nil { return err } checks := []Checker{ - ImagePullCheck{Images: criInterfacer, ImageList: images.GetAllImages(cfg)}, + ImagePullCheck{runtime: containerRuntime, imageList: images.GetAllImages(cfg)}, } return RunChecks(checks, os.Stderr, ignorePreflightErrors) } diff --git a/cmd/kubeadm/app/preflight/checks_test.go b/cmd/kubeadm/app/preflight/checks_test.go index cd4c8d39266..609c6a8f3ff 100644 --- a/cmd/kubeadm/app/preflight/checks_test.go +++ b/cmd/kubeadm/app/preflight/checks_test.go @@ -18,7 +18,6 @@ package preflight import ( "bytes" - "errors" "fmt" "io/ioutil" "strings" @@ -31,6 +30,8 @@ import ( "k8s.io/apimachinery/pkg/util/sets" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3" + utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime" "k8s.io/utils/exec" fakeexec "k8s.io/utils/exec/testing" ) @@ -698,27 +699,50 @@ func TestSetHasItemOrAll(t *testing.T) { } } -type imgs struct{} - -func (i *imgs) Pull(image string) error { - if image == "bad pull" { - return errors.New("pull error") - } - return nil -} -func (i *imgs) Exists(image string) error { - if image == "found" { - return nil - } - return errors.New("error") -} - func TestImagePullCheck(t *testing.T) { - i := ImagePullCheck{ - Images: &imgs{}, - ImageList: []string{"found", "not found", "bad pull"}, + fcmd := fakeexec.FakeCmd{ + RunScript: []fakeexec.FakeRunAction{ + func() ([]byte, []byte, error) { return nil, nil, nil }, // Test case 1 + func() ([]byte, []byte, error) { return nil, nil, nil }, + func() ([]byte, []byte, error) { return nil, nil, nil }, + func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, // Test case 2 + func() ([]byte, []byte, error) { return nil, nil, nil }, + func() ([]byte, []byte, error) { return nil, nil, nil }, + func() ([]byte, []byte, error) { return nil, nil, nil }, + }, } - warnings, errors := i.Check() + + fexec := fakeexec.FakeExec{ + CommandScript: []fakeexec.FakeCommandAction{ + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + }, + LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/docker", nil }, + } + + containerRuntime, err := utilruntime.NewContainerRuntime(&fexec, kubeadmapiv1alpha3.DefaultCRISocket) + if err != nil { + t.Errorf("unexpected NewContainerRuntime error: %v", err) + } + + check := ImagePullCheck{ + runtime: containerRuntime, + imageList: []string{"img1", "img2", "img3"}, + } + warnings, errors := check.Check() + if len(warnings) != 0 { + t.Fatalf("did not expect any warnings but got %q", warnings) + } + if len(errors) != 0 { + t.Fatalf("expected 1 errors but got %d: %q", len(errors), errors) + } + + warnings, errors = check.Check() if len(warnings) != 0 { t.Fatalf("did not expect any warnings but got %q", warnings) } diff --git a/cmd/kubeadm/app/util/BUILD b/cmd/kubeadm/app/util/BUILD index 08eb8158a91..78013cc8624 100644 --- a/cmd/kubeadm/app/util/BUILD +++ b/cmd/kubeadm/app/util/BUILD @@ -77,6 +77,7 @@ filegroup( "//cmd/kubeadm/app/util/etcd:all-srcs", "//cmd/kubeadm/app/util/kubeconfig:all-srcs", "//cmd/kubeadm/app/util/pubkeypin:all-srcs", + "//cmd/kubeadm/app/util/runtime:all-srcs", "//cmd/kubeadm/app/util/staticpod:all-srcs", "//cmd/kubeadm/app/util/system:all-srcs", ], diff --git a/cmd/kubeadm/app/util/runtime/BUILD b/cmd/kubeadm/app/util/runtime/BUILD new file mode 100644 index 00000000000..78c8e654b5f --- /dev/null +++ b/cmd/kubeadm/app/util/runtime/BUILD @@ -0,0 +1,38 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["runtime.go"], + importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime", + visibility = ["//visibility:public"], + deps = [ + "//cmd/kubeadm/app/apis/kubeadm/v1alpha3:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", + "//vendor/k8s.io/utils/exec:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["runtime_test.go"], + embed = [":go_default_library"], + deps = [ + "//cmd/kubeadm/app/apis/kubeadm/v1alpha3:go_default_library", + "//vendor/k8s.io/utils/exec:go_default_library", + "//vendor/k8s.io/utils/exec/testing:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/cmd/kubeadm/app/util/runtime/runtime.go b/cmd/kubeadm/app/util/runtime/runtime.go new file mode 100644 index 00000000000..2edb57215c2 --- /dev/null +++ b/cmd/kubeadm/app/util/runtime/runtime.go @@ -0,0 +1,175 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + "path/filepath" + goruntime "runtime" + "strings" + + "k8s.io/apimachinery/pkg/util/errors" + kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3" + utilsexec "k8s.io/utils/exec" +) + +// ContainerRuntime is an interface for working with container runtimes +type ContainerRuntime interface { + IsDocker() bool + IsRunning() error + ListKubeContainers() ([]string, error) + RemoveContainers(containers []string) error + PullImage(image string) error + ImageExists(image string) (bool, error) +} + +// CRIRuntime is a struct that interfaces with the CRI +type CRIRuntime struct { + exec utilsexec.Interface + criSocket string +} + +// DockerRuntime is a struct that interfaces with the Docker daemon +type DockerRuntime struct { + exec utilsexec.Interface +} + +// NewContainerRuntime sets up and returns a ContainerRuntime struct +func NewContainerRuntime(execer utilsexec.Interface, criSocket string) (ContainerRuntime, error) { + var toolName string + var runtime ContainerRuntime + + if criSocket != kubeadmapiv1alpha3.DefaultCRISocket { + toolName = "crictl" + // !!! temporary work around crictl warning: + // Using "/var/run/crio/crio.sock" as endpoint is deprecated, + // please consider using full url format "unix:///var/run/crio/crio.sock" + if filepath.IsAbs(criSocket) && goruntime.GOOS != "windows" { + criSocket = "unix://" + criSocket + } + runtime = &CRIRuntime{execer, criSocket} + } else { + toolName = "docker" + runtime = &DockerRuntime{execer} + } + + if _, err := execer.LookPath(toolName); err != nil { + return nil, fmt.Errorf("%s is required for container runtime: %v", toolName, err) + } + + return runtime, nil +} + +// IsDocker returns true if the runtime is docker +func (runtime *CRIRuntime) IsDocker() bool { + return false +} + +// IsDocker returns true if the runtime is docker +func (runtime *DockerRuntime) IsDocker() bool { + return true +} + +// IsRunning checks if runtime is running +func (runtime *CRIRuntime) IsRunning() error { + if err := runtime.exec.Command("crictl", "-r", runtime.criSocket, "info").Run(); err != nil { + return fmt.Errorf("container runtime is not running: %v", err) + } + return nil +} + +// IsRunning checks if runtime is running +func (runtime *DockerRuntime) IsRunning() error { + if err := runtime.exec.Command("docker", "info").Run(); err != nil { + return fmt.Errorf("container runtime is not running: %v", err) + } + return nil +} + +// ListKubeContainers lists running k8s CRI pods +func (runtime *CRIRuntime) ListKubeContainers() ([]string, error) { + output, err := runtime.exec.Command("crictl", "-r", runtime.criSocket, "pods", "-q").CombinedOutput() + if err != nil { + return nil, err + } + pods := []string{} + for _, pod := range strings.Fields(string(output)) { + if strings.HasPrefix(pod, "k8s_") { + pods = append(pods, pod) + } + } + return pods, nil +} + +// ListKubeContainers lists running k8s containers +func (runtime *DockerRuntime) ListKubeContainers() ([]string, error) { + output, err := runtime.exec.Command("docker", "ps", "-a", "--filter", "name=k8s_", "-q").CombinedOutput() + return strings.Fields(string(output)), err +} + +// RemoveContainers removes running k8s pods +func (runtime *CRIRuntime) RemoveContainers(containers []string) error { + errs := []error{} + for _, container := range containers { + err := runtime.exec.Command("crictl", "-r", runtime.criSocket, "stopp", container).Run() + if err != nil { + // don't stop on errors, try to remove as many containers as possible + errs = append(errs, fmt.Errorf("failed to stop running pod %s: %v", container, err)) + } else { + err = runtime.exec.Command("crictl", "-r", runtime.criSocket, "rmp", container).Run() + if err != nil { + errs = append(errs, fmt.Errorf("failed to remove running container %s: %v", container, err)) + } + } + } + return errors.NewAggregate(errs) +} + +// RemoveContainers removes running containers +func (runtime *DockerRuntime) RemoveContainers(containers []string) error { + errs := []error{} + for _, container := range containers { + err := runtime.exec.Command("docker", "rm", "--force", "--volumes", container).Run() + if err != nil { + // don't stop on errors, try to remove as many containers as possible + errs = append(errs, fmt.Errorf("failed to remove running container %s: %v", container, err)) + } + } + return errors.NewAggregate(errs) +} + +// PullImage pulls the image +func (runtime *CRIRuntime) PullImage(image string) error { + return runtime.exec.Command("crictl", "-r", runtime.criSocket, "pull", image).Run() +} + +// PullImage pulls the image +func (runtime *DockerRuntime) PullImage(image string) error { + return runtime.exec.Command("docker", "pull", image).Run() +} + +// ImageExists checks to see if the image exists on the system +func (runtime *CRIRuntime) ImageExists(image string) (bool, error) { + err := runtime.exec.Command("crictl", "-r", runtime.criSocket, "inspecti", image).Run() + return err == nil, err +} + +// ImageExists checks to see if the image exists on the system +func (runtime *DockerRuntime) ImageExists(image string) (bool, error) { + err := runtime.exec.Command("docker", "inspect", image).Run() + return err == nil, err +} diff --git a/cmd/kubeadm/app/util/runtime/runtime_test.go b/cmd/kubeadm/app/util/runtime/runtime_test.go new file mode 100644 index 00000000000..dbb6e517a5c --- /dev/null +++ b/cmd/kubeadm/app/util/runtime/runtime_test.go @@ -0,0 +1,301 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + "reflect" + "testing" + + kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3" + "k8s.io/utils/exec" + fakeexec "k8s.io/utils/exec/testing" +) + +func TestNewContainerRuntime(t *testing.T) { + execLookPathOK := fakeexec.FakeExec{ + LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, + } + execLookPathErr := fakeexec.FakeExec{ + LookPathFunc: func(cmd string) (string, error) { return "", fmt.Errorf("%s not found", cmd) }, + } + cases := []struct { + execer fakeexec.FakeExec + criSocket string + isDocker bool + isError bool + }{ + {execLookPathOK, kubeadmapiv1alpha3.DefaultCRISocket, true, false}, + {execLookPathOK, "unix:///var/run/crio/crio.sock", false, false}, + {execLookPathOK, "/var/run/crio/crio.sock", false, false}, + {execLookPathErr, "unix:///var/run/crio/crio.sock", false, true}, + } + + for _, tc := range cases { + runtime, err := NewContainerRuntime(&tc.execer, tc.criSocket) + if err != nil { + if !tc.isError { + t.Errorf("unexpected NewContainerRuntime error. criSocket: %s, error: %v", tc.criSocket, err) + } + continue // expected error occurs, impossible to test runtime further + } + if tc.isError && err == nil { + t.Errorf("unexpected NewContainerRuntime success. criSocket: %s", tc.criSocket) + } + isDocker := runtime.IsDocker() + if tc.isDocker != isDocker { + t.Errorf("unexpected isDocker() result %v for the criSocket %s", isDocker, tc.criSocket) + } + } +} + +func genFakeActions(fcmd *fakeexec.FakeCmd, num int) []fakeexec.FakeCommandAction { + var actions []fakeexec.FakeCommandAction + for i := 0; i < num; i++ { + actions = append(actions, func(cmd string, args ...string) exec.Cmd { + return fakeexec.InitFakeCmd(fcmd, cmd, args...) + }) + } + return actions +} + +func TestIsRunning(t *testing.T) { + fcmd := fakeexec.FakeCmd{ + RunScript: []fakeexec.FakeRunAction{ + func() ([]byte, []byte, error) { return nil, nil, nil }, + func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, + func() ([]byte, []byte, error) { return nil, nil, nil }, + func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, + }, + } + criExecer := fakeexec.FakeExec{ + CommandScript: genFakeActions(&fcmd, len(fcmd.RunScript)), + LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, + } + + dockerExecer := fakeexec.FakeExec{ + CommandScript: genFakeActions(&fcmd, len(fcmd.RunScript)), + LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/docker", nil }, + } + + cases := []struct { + criSocket string + execer fakeexec.FakeExec + isError bool + runCalls int + }{ + {"unix:///var/run/crio/crio.sock", criExecer, false, 1}, + {"unix:///var/run/crio/crio.sock", criExecer, true, 2}, + {kubeadmapiv1alpha3.DefaultCRISocket, dockerExecer, false, 3}, + {kubeadmapiv1alpha3.DefaultCRISocket, dockerExecer, true, 4}, + } + + for _, tc := range cases { + runtime, err := NewContainerRuntime(&tc.execer, tc.criSocket) + if err != nil { + t.Fatalf("unexpected NewContainerRuntime error: %v", err) + } + isRunning := runtime.IsRunning() + if tc.isError && isRunning == nil { + t.Error("unexpected IsRunning() success") + } + if !tc.isError && isRunning != nil { + t.Error("unexpected IsRunning() error") + } + if fcmd.RunCalls != tc.runCalls { + t.Errorf("expected %d Run() calls, got %d", tc.runCalls, fcmd.RunCalls) + } + } +} + +func TestListKubeContainers(t *testing.T) { + fcmd := fakeexec.FakeCmd{ + CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{ + func() ([]byte, error) { return []byte("k8s_p1\nk8s_p2\nid3"), nil }, + func() ([]byte, error) { return nil, &fakeexec.FakeExitError{Status: 1} }, + func() ([]byte, error) { return []byte("k8s_p1\nk8s_p2"), nil }, + }, + } + execer := fakeexec.FakeExec{ + CommandScript: genFakeActions(&fcmd, len(fcmd.CombinedOutputScript)), + LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, + } + + cases := []struct { + criSocket string + isError bool + }{ + {"unix:///var/run/crio/crio.sock", false}, + {"unix:///var/run/crio/crio.sock", true}, + {kubeadmapiv1alpha3.DefaultCRISocket, false}, + } + + for _, tc := range cases { + runtime, err := NewContainerRuntime(&execer, tc.criSocket) + if err != nil { + t.Errorf("unexpected NewContainerRuntime error: %v", err) + continue + } + + containers, err := runtime.ListKubeContainers() + if tc.isError { + if err == nil { + t.Errorf("unexpected ListKubeContainers success") + } + continue + } else if err != nil { + t.Errorf("unexpected ListKubeContainers error: %v", err) + } + + if !reflect.DeepEqual(containers, []string{"k8s_p1", "k8s_p2"}) { + t.Errorf("unexpected ListKubeContainers output: %v", containers) + } + } +} + +func TestRemoveContainers(t *testing.T) { + fakeOK := func() ([]byte, []byte, error) { return nil, nil, nil } + fakeErr := func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} } + fcmd := fakeexec.FakeCmd{ + RunScript: []fakeexec.FakeRunAction{ + fakeOK, fakeOK, fakeOK, fakeOK, fakeOK, fakeOK, // Test case 1 + fakeOK, fakeOK, fakeOK, fakeErr, fakeOK, fakeOK, + fakeErr, fakeOK, fakeOK, fakeErr, fakeOK, + fakeOK, fakeOK, fakeOK, + fakeOK, fakeErr, fakeOK, + }, + } + execer := fakeexec.FakeExec{ + CommandScript: genFakeActions(&fcmd, len(fcmd.RunScript)), + LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, + } + + cases := []struct { + criSocket string + containers []string + isError bool + }{ + {"unix:///var/run/crio/crio.sock", []string{"k8s_p1", "k8s_p2", "k8s_p3"}, false}, // Test case 1 + {"unix:///var/run/crio/crio.sock", []string{"k8s_p1", "k8s_p2", "k8s_p3"}, true}, + {"unix:///var/run/crio/crio.sock", []string{"k8s_p1", "k8s_p2", "k8s_p3"}, true}, + {kubeadmapiv1alpha3.DefaultCRISocket, []string{"k8s_c1", "k8s_c2", "k8s_c3"}, false}, + {kubeadmapiv1alpha3.DefaultCRISocket, []string{"k8s_c1", "k8s_c2", "k8s_c3"}, true}, + } + + for _, tc := range cases { + runtime, err := NewContainerRuntime(&execer, tc.criSocket) + if err != nil { + t.Errorf("unexpected NewContainerRuntime error: %v, criSocket: %s", err, tc.criSocket) + continue + } + + err = runtime.RemoveContainers(tc.containers) + if !tc.isError && err != nil { + t.Errorf("unexpected RemoveContainers errors: %v, criSocket: %s, containers: %v", err, tc.criSocket, tc.containers) + } + if tc.isError && err == nil { + t.Errorf("unexpected RemoveContnainers success, criSocket: %s, containers: %v", tc.criSocket, tc.containers) + } + } +} + +func TestPullImage(t *testing.T) { + fcmd := fakeexec.FakeCmd{ + RunScript: []fakeexec.FakeRunAction{ + func() ([]byte, []byte, error) { return nil, nil, nil }, + func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, + func() ([]byte, []byte, error) { return nil, nil, nil }, + func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, + }, + } + execer := fakeexec.FakeExec{ + CommandScript: genFakeActions(&fcmd, len(fcmd.RunScript)), + LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, + } + + cases := []struct { + criSocket string + image string + isError bool + }{ + {"unix:///var/run/crio/crio.sock", "image1", false}, + {"unix:///var/run/crio/crio.sock", "image2", true}, + {kubeadmapiv1alpha3.DefaultCRISocket, "image1", false}, + {kubeadmapiv1alpha3.DefaultCRISocket, "image2", true}, + } + + for _, tc := range cases { + runtime, err := NewContainerRuntime(&execer, tc.criSocket) + if err != nil { + t.Errorf("unexpected NewContainerRuntime error: %v, criSocket: %s", err, tc.criSocket) + continue + } + + err = runtime.PullImage(tc.image) + if !tc.isError && err != nil { + t.Errorf("unexpected PullImage error: %v, criSocket: %s, image: %s", err, tc.criSocket, tc.image) + } + if tc.isError && err == nil { + t.Errorf("unexpected PullImage success, criSocket: %s, image: %s", tc.criSocket, tc.image) + } + } +} + +func TestImageExists(t *testing.T) { + fcmd := fakeexec.FakeCmd{ + RunScript: []fakeexec.FakeRunAction{ + func() ([]byte, []byte, error) { return nil, nil, nil }, + func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, + func() ([]byte, []byte, error) { return nil, nil, nil }, + func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, + }, + } + execer := fakeexec.FakeExec{ + CommandScript: genFakeActions(&fcmd, len(fcmd.RunScript)), + LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, + } + + cases := []struct { + criSocket string + image string + isError bool + }{ + {"unix:///var/run/crio/crio.sock", "image1", false}, + {"unix:///var/run/crio/crio.sock", "image2", true}, + {kubeadmapiv1alpha3.DefaultCRISocket, "image1", false}, + {kubeadmapiv1alpha3.DefaultCRISocket, "image2", true}, + } + + for _, tc := range cases { + runtime, err := NewContainerRuntime(&execer, tc.criSocket) + if err != nil { + t.Errorf("unexpected NewContainerRuntime error: %v, criSocket: %s", err, tc.criSocket) + continue + } + + result, err := runtime.ImageExists(tc.image) + if result && err != nil { + t.Errorf("unexpected ImageExists result %v and error: %v", result, err) + } + if !tc.isError && err != nil { + t.Errorf("unexpected ImageExists error: %v, criSocket: %s, image: %s", err, tc.criSocket, tc.image) + } + if tc.isError && err == nil { + t.Errorf("unexpected ImageExists success, criSocket: %s, image: %s", tc.criSocket, tc.image) + } + } +}