diff --git a/cmd/kubeadm/app/cmd/config.go b/cmd/kubeadm/app/cmd/config.go index 068bfbdf590..35b81e2e760 100644 --- a/cmd/kubeadm/app/cmd/config.go +++ b/cmd/kubeadm/app/cmd/config.go @@ -32,7 +32,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/cli-runtime/pkg/genericclioptions" - utilsexec "k8s.io/utils/exec" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" @@ -372,8 +371,8 @@ func newCmdConfigImagesPull() *cobra.Command { if err != nil { return err } - containerRuntime, err := utilruntime.NewContainerRuntime(utilsexec.New(), internalcfg.NodeRegistration.CRISocket) - if err != nil { + containerRuntime := utilruntime.NewContainerRuntime(internalcfg.NodeRegistration.CRISocket) + if err := containerRuntime.Connect(); err != nil { return err } return PullControlPlaneImages(containerRuntime, &internalcfg.ClusterConfiguration) diff --git a/cmd/kubeadm/app/cmd/config_test.go b/cmd/kubeadm/app/cmd/config_test.go index df9cd2cc3f2..044d09572f7 100644 --- a/cmd/kubeadm/app/cmd/config_test.go +++ b/cmd/kubeadm/app/cmd/config_test.go @@ -31,15 +31,11 @@ import ( "github.com/lithammer/dedent" "github.com/spf13/cobra" - "k8s.io/utils/exec" - fakeexec "k8s.io/utils/exec/testing" - kubeadmapiv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" outputapischeme "k8s.io/kubernetes/cmd/kubeadm/app/apis/output/scheme" "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/output" - utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime" ) const ( @@ -339,46 +335,6 @@ registry.k8s.io/etcd:{{.EtcdVersion}} } } -func TestImagesPull(t *testing.T) { - fcmd := fakeexec.FakeCmd{ - CombinedOutputScript: []fakeexec.FakeAction{ - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, nil }, - }, - } - - fexec := &fakeexec.FakeExec{ - CommandScript: []fakeexec.FakeCommandAction{ - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - }, - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, - } - - containerRuntime, err := utilruntime.NewContainerRuntime(fexec, constants.DefaultCRISocket) - if err != nil { - t.Errorf("unexpected NewContainerRuntime error: %v", err) - } - - images := []string{"a", "b", "c", "d", "a"} - for _, image := range images { - if err := containerRuntime.PullImage(image); err != nil { - t.Fatalf("expected nil but found %v", err) - } - fmt.Printf("[config/images] Pulled %s\n", image) - } - - if fcmd.CombinedOutputCalls != len(images) { - t.Errorf("expected %d calls, got %d", len(images), fcmd.CombinedOutputCalls) - } -} - func TestNewCmdConfigPrintActionDefaults(t *testing.T) { tests := []struct { name string diff --git a/cmd/kubeadm/app/cmd/phases/reset/cleanupnode.go b/cmd/kubeadm/app/cmd/phases/reset/cleanupnode.go index d33a85c5761..f5f87d26b38 100644 --- a/cmd/kubeadm/app/cmd/phases/reset/cleanupnode.go +++ b/cmd/kubeadm/app/cmd/phases/reset/cleanupnode.go @@ -26,7 +26,6 @@ import ( "github.com/pkg/errors" "k8s.io/klog/v2" - utilsexec "k8s.io/utils/exec" kubeadmapiv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" @@ -102,7 +101,7 @@ func runCleanupNode(c workflow.RunData) error { if !r.DryRun() { klog.V(1).Info("[reset] Removing Kubernetes-managed containers") - if err := removeContainers(utilsexec.New(), r.CRISocketPath()); err != nil { + if err := removeContainers(r.CRISocketPath()); err != nil { klog.Warningf("[reset] Failed to remove containers: %v\n", err) } } else { @@ -135,9 +134,9 @@ func runCleanupNode(c workflow.RunData) error { return nil } -func removeContainers(execer utilsexec.Interface, criSocketPath string) error { - containerRuntime, err := utilruntime.NewContainerRuntime(execer, criSocketPath) - if err != nil { +func removeContainers(criSocketPath string) error { + containerRuntime := utilruntime.NewContainerRuntime(criSocketPath) + if err := containerRuntime.Connect(); err != nil { return err } containers, err := containerRuntime.ListKubeContainers() diff --git a/cmd/kubeadm/app/cmd/phases/reset/cleanupnode_test.go b/cmd/kubeadm/app/cmd/phases/reset/cleanupnode_test.go index de714cc8156..511e3e5fb40 100644 --- a/cmd/kubeadm/app/cmd/phases/reset/cleanupnode_test.go +++ b/cmd/kubeadm/app/cmd/phases/reset/cleanupnode_test.go @@ -21,9 +21,6 @@ import ( "path/filepath" "testing" - "k8s.io/utils/exec" - fakeexec "k8s.io/utils/exec/testing" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/preflight" ) @@ -212,27 +209,3 @@ func TestConfigDirCleaner(t *testing.T) { }) } } - -func TestRemoveContainers(t *testing.T) { - fcmd := fakeexec.FakeCmd{ - CombinedOutputScript: []fakeexec.FakeAction{ - func() ([]byte, []byte, error) { return []byte("id1\nid2"), nil, nil }, - func() ([]byte, []byte, error) { return []byte(""), nil, nil }, - func() ([]byte, []byte, error) { return []byte(""), nil, nil }, - func() ([]byte, []byte, error) { return []byte(""), nil, nil }, - func() ([]byte, []byte, error) { return []byte(""), nil, nil }, - }, - } - fexec := &fakeexec.FakeExec{ - CommandScript: []fakeexec.FakeCommandAction{ - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - }, - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, - } - - removeContainers(fexec, "unix:///var/run/crio/crio.sock") -} diff --git a/cmd/kubeadm/app/preflight/checks.go b/cmd/kubeadm/app/preflight/checks.go index 201e8913065..5f635c05c6a 100644 --- a/cmd/kubeadm/app/preflight/checks.go +++ b/cmd/kubeadm/app/preflight/checks.go @@ -857,8 +857,7 @@ func (ipc ImagePullCheck) Check() (warnings, errorList []error) { for _, image := range ipc.imageList { switch policy { case v1.PullIfNotPresent: - ret, err := ipc.runtime.ImageExists(image) - if ret && err == nil { + if ipc.runtime.ImageExists(image) { klog.V(1).Infof("image exists: %s", image) continue } @@ -1074,8 +1073,8 @@ func RunJoinNodeChecks(execer utilsexec.Interface, cfg *kubeadmapi.JoinConfigura // addCommonChecks is a helper function to duplicate checks that are common between both the // kubeadm init and join commands func addCommonChecks(execer utilsexec.Interface, k8sVersion string, nodeReg *kubeadmapi.NodeRegistrationOptions, checks []Checker) []Checker { - containerRuntime, err := utilruntime.NewContainerRuntime(execer, nodeReg.CRISocket) - if err != nil { + containerRuntime := utilruntime.NewContainerRuntime(nodeReg.CRISocket) + if err := containerRuntime.Connect(); err != nil { klog.Warningf("[preflight] WARNING: Couldn't create the interface used for talking to the container runtime: %v\n", err) } else { checks = append(checks, ContainerRuntimeCheck{runtime: containerRuntime}) @@ -1104,8 +1103,8 @@ func RunRootCheckOnly(ignorePreflightErrors sets.Set[string]) error { // RunPullImagesCheck will pull images kubeadm needs if they are not found on the system func RunPullImagesCheck(execer utilsexec.Interface, cfg *kubeadmapi.InitConfiguration, ignorePreflightErrors sets.Set[string]) error { - containerRuntime, err := utilruntime.NewContainerRuntime(utilsexec.New(), cfg.NodeRegistration.CRISocket) - if err != nil { + containerRuntime := utilruntime.NewContainerRuntime(cfg.NodeRegistration.CRISocket) + if err := containerRuntime.Connect(); err != nil { return &Error{Msg: err.Error()} } diff --git a/cmd/kubeadm/app/preflight/checks_linux.go b/cmd/kubeadm/app/preflight/checks_linux.go index 3d23bf7730e..eb87fe69612 100644 --- a/cmd/kubeadm/app/preflight/checks_linux.go +++ b/cmd/kubeadm/app/preflight/checks_linux.go @@ -74,7 +74,7 @@ func addSwapCheck(checks []Checker) []Checker { // addExecChecks adds checks that verify if certain binaries are in PATH func addExecChecks(checks []Checker, execer utilsexec.Interface) []Checker { checks = append(checks, - InPathCheck{executable: "crictl", mandatory: true, exec: execer}, + InPathCheck{executable: "crictl", mandatory: false, exec: execer}, InPathCheck{executable: "conntrack", mandatory: true, exec: execer}, InPathCheck{executable: "ip", mandatory: true, exec: execer}, InPathCheck{executable: "iptables", mandatory: true, exec: execer}, diff --git a/cmd/kubeadm/app/preflight/checks_test.go b/cmd/kubeadm/app/preflight/checks_test.go index 8ccf303c08c..d6a21b52b96 100644 --- a/cmd/kubeadm/app/preflight/checks_test.go +++ b/cmd/kubeadm/app/preflight/checks_test.go @@ -32,7 +32,6 @@ import ( "github.com/lithammer/dedent" "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/version" "k8s.io/utils/exec" @@ -40,9 +39,7 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime" ) var ( @@ -851,108 +848,6 @@ func TestSetHasItemOrAll(t *testing.T) { } } -func TestImagePullCheck(t *testing.T) { - fcmd := fakeexec.FakeCmd{ - RunScript: []fakeexec.FakeAction{ - // Test case 1: img1 and img2 exist, img3 doesn't exist - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, - - // Test case 2: images don't exist - func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, - }, - CombinedOutputScript: []fakeexec.FakeAction{ - // Test case1: pull only img3 - func() ([]byte, []byte, error) { return []byte("pause"), nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, nil }, - // Test case 2: fail to pull image2 and image3 - // If the pull fails, it will be retried 5 times (see PullImageRetry in constants/constants.go) - func() ([]byte, []byte, error) { return []byte("pause"), nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - }, - } - - fexec := &fakeexec.FakeExec{ - CommandScript: []fakeexec.FakeCommandAction{ - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - }, - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, - } - - containerRuntime, err := utilruntime.NewContainerRuntime(fexec, constants.DefaultCRISocket) - if err != nil { - t.Errorf("unexpected NewContainerRuntime error: %v", err) - } - - check := ImagePullCheck{ - runtime: containerRuntime, - sandboxImage: "pause", - imageList: []string{"img1", "img2", "img3"}, - imagePullPolicy: corev1.PullIfNotPresent, - imagePullSerial: true, - } - warnings, errors := check.Check() - if len(warnings) != 0 { - t.Fatalf("did not expect any warnings but got %q", warnings) - } - if len(errors) != 0 { - t.Fatalf("expected 1 errors but got %d: %q", len(errors), errors) - } - - warnings, errors = check.Check() - if len(warnings) != 0 { - t.Fatalf("did not expect any warnings but got %q", warnings) - } - if len(errors) != 2 { - t.Fatalf("expected 2 errors but got %d: %q", len(errors), errors) - } - - // Test with unknown policy - check = ImagePullCheck{ - runtime: containerRuntime, - sandboxImage: "pause", - imageList: []string{"img1", "img2", "img3"}, - imagePullPolicy: "", - imagePullSerial: true, - } - _, errors = check.Check() - if len(errors) != 1 { - t.Fatalf("expected 1 error but got %d: %q", len(errors), errors) - } -} - func TestNumCPUCheck(t *testing.T) { var tests = []struct { numCPU int diff --git a/cmd/kubeadm/app/util/runtime/impl.go b/cmd/kubeadm/app/util/runtime/impl.go new file mode 100644 index 00000000000..b71f589d45c --- /dev/null +++ b/cmd/kubeadm/app/util/runtime/impl.go @@ -0,0 +1,71 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package runtime + +import ( + "context" + "time" + + criapi "k8s.io/cri-api/pkg/apis" + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + criclient "k8s.io/cri-client/pkg" +) + +type defaultImpl struct{} + +type impl interface { + NewRemoteRuntimeService(endpoint string, connectionTimeout time.Duration) (criapi.RuntimeService, error) + NewRemoteImageService(endpoint string, connectionTimeout time.Duration) (criapi.ImageManagerService, error) + Status(ctx context.Context, runtimeService criapi.RuntimeService, verbose bool) (*runtimeapi.StatusResponse, error) + ListPodSandbox(ctx context.Context, runtimeService criapi.RuntimeService, filter *runtimeapi.PodSandboxFilter) ([]*runtimeapi.PodSandbox, error) + StopPodSandbox(ctx context.Context, runtimeService criapi.RuntimeService, sandboxID string) error + RemovePodSandbox(ctx context.Context, runtimeService criapi.RuntimeService, podSandboxID string) error + PullImage(ctx context.Context, imageService criapi.ImageManagerService, image *runtimeapi.ImageSpec, auth *runtimeapi.AuthConfig, podSandboxConfig *runtimeapi.PodSandboxConfig) (string, error) + ImageStatus(ctx context.Context, imageService criapi.ImageManagerService, image *runtimeapi.ImageSpec, verbose bool) (*runtimeapi.ImageStatusResponse, error) +} + +func (*defaultImpl) NewRemoteRuntimeService(endpoint string, connectionTimeout time.Duration) (criapi.RuntimeService, error) { + return criclient.NewRemoteRuntimeService(endpoint, defaultTimeout, nil, nil) +} + +func (*defaultImpl) NewRemoteImageService(endpoint string, connectionTimeout time.Duration) (criapi.ImageManagerService, error) { + return criclient.NewRemoteImageService(endpoint, connectionTimeout, nil, nil) +} + +func (*defaultImpl) Status(ctx context.Context, runtimeService criapi.RuntimeService, verbose bool) (*runtimeapi.StatusResponse, error) { + return runtimeService.Status(ctx, verbose) +} + +func (*defaultImpl) ListPodSandbox(ctx context.Context, runtimeService criapi.RuntimeService, filter *runtimeapi.PodSandboxFilter) ([]*runtimeapi.PodSandbox, error) { + return runtimeService.ListPodSandbox(ctx, filter) +} + +func (*defaultImpl) StopPodSandbox(ctx context.Context, runtimeService criapi.RuntimeService, sandboxID string) error { + return runtimeService.StopPodSandbox(ctx, sandboxID) +} + +func (*defaultImpl) RemovePodSandbox(ctx context.Context, runtimeService criapi.RuntimeService, podSandboxID string) error { + return runtimeService.RemovePodSandbox(ctx, podSandboxID) +} + +func (*defaultImpl) PullImage(ctx context.Context, imageService criapi.ImageManagerService, image *runtimeapi.ImageSpec, auth *runtimeapi.AuthConfig, podSandboxConfig *runtimeapi.PodSandboxConfig) (string, error) { + return imageService.PullImage(ctx, image, auth, podSandboxConfig) +} + +func (*defaultImpl) ImageStatus(ctx context.Context, imageService criapi.ImageManagerService, image *runtimeapi.ImageSpec, verbose bool) (*runtimeapi.ImageStatusResponse, error) { + return imageService.ImageStatus(ctx, image, verbose) +} diff --git a/cmd/kubeadm/app/util/runtime/runtime.go b/cmd/kubeadm/app/util/runtime/runtime.go index 1036fa4c60d..28bde230f1b 100644 --- a/cmd/kubeadm/app/util/runtime/runtime.go +++ b/cmd/kubeadm/app/util/runtime/runtime.go @@ -14,17 +14,20 @@ See the License for the specific language governing permissions and limitations under the License. */ +// Package runtime provides the kubeadm container runtime implementation. package runtime import ( - "os" + "context" + "encoding/json" "strings" + "time" "github.com/pkg/errors" - errorsutil "k8s.io/apimachinery/pkg/util/errors" + criapi "k8s.io/cri-api/pkg/apis" + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" "k8s.io/klog/v2" - utilsexec "k8s.io/utils/exec" "k8s.io/kubernetes/cmd/kubeadm/app/constants" ) @@ -38,64 +41,94 @@ var defaultKnownCRISockets = []string{ // ContainerRuntime is an interface for working with container runtimes type ContainerRuntime interface { - Socket() string + Connect() error + SetImpl(impl) IsRunning() error ListKubeContainers() ([]string, error) RemoveContainers(containers []string) error PullImage(image string) error PullImagesInParallel(images []string, ifNotPresent bool) error - ImageExists(image string) (bool, error) + ImageExists(image string) bool SandboxImage() (string, error) } // CRIRuntime is a struct that interfaces with the CRI type CRIRuntime struct { - exec utilsexec.Interface - criSocket string - crictlPath string + impl impl + criSocket string + runtimeService criapi.RuntimeService + imageService criapi.ImageManagerService } +// defaultTimeout is the default timeout inherited by crictl +const defaultTimeout = 2 * time.Second + // NewContainerRuntime sets up and returns a ContainerRuntime struct -func NewContainerRuntime(execer utilsexec.Interface, criSocket string) (ContainerRuntime, error) { - const toolName = "crictl" - crictlPath, err := execer.LookPath(toolName) +func NewContainerRuntime(criSocket string) ContainerRuntime { + return &CRIRuntime{ + impl: &defaultImpl{}, + criSocket: criSocket, + } +} + +// SetImpl can be used to set the internal implementation for testing purposes. +func (runtime *CRIRuntime) SetImpl(impl impl) { + runtime.impl = impl +} + +// Connect establishes a connection with the CRI runtime. +func (runtime *CRIRuntime) Connect() error { + runtimeService, err := runtime.impl.NewRemoteRuntimeService(runtime.criSocket, defaultTimeout) if err != nil { - return nil, errors.Wrapf(err, "%s is required by the container runtime", toolName) + return errors.Wrap(err, "failed to create new CRI runtime service") } - return &CRIRuntime{execer, criSocket, crictlPath}, nil + runtime.runtimeService = runtimeService + + imageService, err := runtime.impl.NewRemoteImageService(runtime.criSocket, defaultTimeout) + if err != nil { + return errors.Wrap(err, "failed to create new CRI image service") + } + runtime.imageService = imageService + + return nil } -// Socket returns the CRI socket endpoint -func (runtime *CRIRuntime) Socket() string { - return runtime.criSocket -} - -// crictl creates a crictl command for the provided args. -func (runtime *CRIRuntime) crictl(args ...string) utilsexec.Cmd { - cmd := runtime.exec.Command(runtime.crictlPath, append([]string{"-r", runtime.Socket(), "-i", runtime.Socket()}, args...)...) - cmd.SetEnv(os.Environ()) - return cmd -} - -// IsRunning checks if runtime is running +// IsRunning checks if runtime is running. func (runtime *CRIRuntime) IsRunning() error { - if out, err := runtime.crictl("info").CombinedOutput(); err != nil { - return errors.Wrapf(err, "container runtime is not running: output: %s, error", string(out)) + ctx, cancel := defaultContext() + defer cancel() + + res, err := runtime.impl.Status(ctx, runtime.runtimeService, false) + if err != nil { + return errors.Wrap(err, "container runtime is not running") } + + for _, condition := range res.GetStatus().GetConditions() { + if !condition.GetStatus() { + return errors.Errorf( + "container runtime condition %q is not true. reason: %s, message: %s", + condition.GetType(), condition.GetReason(), condition.GetMessage(), + ) + } + } + return nil } // ListKubeContainers lists running k8s CRI pods func (runtime *CRIRuntime) ListKubeContainers() ([]string, error) { - // Disable debug mode regardless how the crictl is configured so that the debug info won't be - // iterpreted to the Pod ID. - args := []string{"-D=false", "pods", "-q"} - out, err := runtime.crictl(args...).CombinedOutput() + ctx, cancel := defaultContext() + defer cancel() + + sandboxes, err := runtime.impl.ListPodSandbox(ctx, runtime.runtimeService, nil) if err != nil { - return nil, errors.Wrapf(err, "output: %s, error", string(out)) + return nil, errors.Wrap(err, "failed to list pod sandboxes") } + pods := []string{} - pods = append(pods, strings.Fields(string(out))...) + for _, sandbox := range sandboxes { + pods = append(pods, sandbox.GetId()) + } return pods, nil } @@ -106,16 +139,23 @@ func (runtime *CRIRuntime) RemoveContainers(containers []string) error { var lastErr error for i := 0; i < constants.RemoveContainerRetry; i++ { klog.V(5).Infof("Attempting to remove container %v", container) - out, err := runtime.crictl("stopp", container).CombinedOutput() - if err != nil { - lastErr = errors.Wrapf(err, "failed to stop running pod %s: output: %s", container, string(out)) + + ctx, cancel := defaultContext() + if err := runtime.impl.StopPodSandbox(ctx, runtime.runtimeService, container); err != nil { + lastErr = errors.Wrapf(err, "failed to stop running pod %s", container) + cancel() continue } - out, err = runtime.crictl("rmp", container).CombinedOutput() - if err != nil { - lastErr = errors.Wrapf(err, "failed to remove running container %s: output: %s", container, string(out)) + cancel() + + ctx, cancel = defaultContext() + if err := runtime.impl.RemovePodSandbox(ctx, runtime.runtimeService, container); err != nil { + lastErr = errors.Wrapf(err, "failed to remove pod %s", container) + cancel() continue } + cancel() + lastErr = nil break } @@ -128,16 +168,13 @@ func (runtime *CRIRuntime) RemoveContainers(containers []string) error { } // PullImage pulls the image -func (runtime *CRIRuntime) PullImage(image string) error { - var err error - var out []byte +func (runtime *CRIRuntime) PullImage(image string) (err error) { for i := 0; i < constants.PullImageRetry; i++ { - out, err = runtime.crictl("pull", image).CombinedOutput() - if err == nil { + if _, err = runtime.impl.PullImage(context.Background(), runtime.imageService, &runtimeapi.ImageSpec{Image: image}, nil, nil); err == nil { return nil } } - return errors.Wrapf(err, "output: %s, error", out) + return errors.Wrapf(err, "failed to pull image %s", image) } // PullImagesInParallel pulls a list of images in parallel @@ -146,8 +183,12 @@ func (runtime *CRIRuntime) PullImagesInParallel(images []string, ifNotPresent bo return errorsutil.NewAggregate(errs) } +func defaultContext() (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), defaultTimeout) +} + func pullImagesInParallelImpl(images []string, ifNotPresent bool, - imageExistsFunc func(string) (bool, error), pullImageFunc func(string) error) []error { + imageExistsFunc func(string) bool, pullImageFunc func(string) error) []error { var errs []error errChan := make(chan error, len(images)) @@ -157,11 +198,7 @@ func pullImagesInParallelImpl(images []string, ifNotPresent bool, image := img go func() { if ifNotPresent { - exists, err := imageExistsFunc(image) - if err != nil { - errChan <- errors.WithMessagef(err, "failed to check if image %s exists", image) - return - } + exists := imageExistsFunc(image) if exists { klog.V(1).Infof("image exists: %s", image) errChan <- nil @@ -188,9 +225,11 @@ func pullImagesInParallelImpl(images []string, ifNotPresent bool, } // ImageExists checks to see if the image exists on the system -func (runtime *CRIRuntime) ImageExists(image string) (bool, error) { - err := runtime.crictl("inspecti", image).Run() - return err == nil, nil +func (runtime *CRIRuntime) ImageExists(image string) bool { + ctx, cancel := defaultContext() + defer cancel() + _, err := runtime.impl.ImageStatus(ctx, runtime.imageService, &runtimeapi.ImageSpec{Image: image}, false) + return err == nil } // detectCRISocketImpl is separated out only for test purposes, DON'T call it directly, use DetectCRISocket instead @@ -212,7 +251,7 @@ func detectCRISocketImpl(isSocket func(string) bool, knownCRISockets []string) ( return foundCRISockets[0], nil default: // Multiple CRIs installed? - return "", errors.Errorf("Found multiple CRI endpoints on the host. Please define which one do you wish "+ + return "", errors.Errorf("found multiple CRI endpoints on the host. Please define which one do you wish "+ "to use by setting the 'criSocket' field in the kubeadm configuration file: %s", strings.Join(foundCRISockets, ", ")) } @@ -225,16 +264,26 @@ func DetectCRISocket() (string, error) { // SandboxImage returns the sandbox image used by the container runtime func (runtime *CRIRuntime) SandboxImage() (string, error) { - args := []string{"-D=false", "info", "-o", "go-template", "--template", "{{.config.sandboxImage}}"} - out, err := runtime.crictl(args...).CombinedOutput() + ctx, cancel := defaultContext() + defer cancel() + status, err := runtime.impl.Status(ctx, runtime.runtimeService, true) if err != nil { - return "", errors.Wrapf(err, "output: %s, error", string(out)) + return "", errors.Wrap(err, "failed to get runtime status") } - sandboxImage := strings.TrimSpace(string(out)) - if len(sandboxImage) > 0 { - return sandboxImage, nil + infoConfig, ok := status.GetInfo()["config"] + if !ok { + return "", errors.Errorf("no 'config' field in CRI info: %+v", status) } - return "", errors.Errorf("the detected sandbox image is empty") + type config struct { + SandboxImage string `json:"sandboxImage,omitempty"` + } + c := config{} + + if err := json.Unmarshal([]byte(infoConfig), &c); err != nil { + return "", errors.Wrap(err, "failed to unmarshal CRI info config") + } + + return c.SandboxImage, nil } diff --git a/cmd/kubeadm/app/util/runtime/runtime_fake_test.go b/cmd/kubeadm/app/util/runtime/runtime_fake_test.go new file mode 100644 index 00000000000..e42e239e575 --- /dev/null +++ b/cmd/kubeadm/app/util/runtime/runtime_fake_test.go @@ -0,0 +1,152 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package runtime + +import ( + "context" + "time" + + cri "k8s.io/cri-api/pkg/apis" + v1 "k8s.io/cri-api/pkg/apis/runtime/v1" +) + +type fakeImpl struct { + imageStatusReturns struct { + res *v1.ImageStatusResponse + err error + } + listPodSandboxReturns struct { + res []*v1.PodSandbox + err error + } + newRemoteImageServiceReturns struct { + res cri.ImageManagerService + err error + } + newRemoteRuntimeServiceReturns struct { + res cri.RuntimeService + err error + } + pullImageReturns struct { + res string + err error + } + removePodSandboxReturns struct { + res error + } + statusReturns struct { + res *v1.StatusResponse + err error + } + stopPodSandboxReturns struct { + res error + } +} + +func (fake *fakeImpl) ImageStatus(context.Context, cri.ImageManagerService, *v1.ImageSpec, bool) (*v1.ImageStatusResponse, error) { + fakeReturns := fake.imageStatusReturns + return fakeReturns.res, fakeReturns.err +} + +func (fake *fakeImpl) ImageStatusReturns(res *v1.ImageStatusResponse, err error) { + fake.imageStatusReturns = struct { + res *v1.ImageStatusResponse + err error + }{res, err} +} + +func (fake *fakeImpl) ListPodSandbox(context.Context, cri.RuntimeService, *v1.PodSandboxFilter) ([]*v1.PodSandbox, error) { + fakeReturns := fake.listPodSandboxReturns + return fakeReturns.res, fakeReturns.err +} + +func (fake *fakeImpl) ListPodSandboxReturns(res []*v1.PodSandbox, err error) { + fake.listPodSandboxReturns = struct { + res []*v1.PodSandbox + err error + }{res, err} +} + +func (fake *fakeImpl) NewRemoteImageService(string, time.Duration) (cri.ImageManagerService, error) { + fakeReturns := fake.newRemoteImageServiceReturns + return fakeReturns.res, fakeReturns.err +} + +func (fake *fakeImpl) NewRemoteImageServiceReturns(res cri.ImageManagerService, err error) { + fake.newRemoteImageServiceReturns = struct { + res cri.ImageManagerService + err error + }{res, err} +} + +func (fake *fakeImpl) NewRemoteRuntimeService(string, time.Duration) (cri.RuntimeService, error) { + fakeReturns := fake.newRemoteRuntimeServiceReturns + return fakeReturns.res, fakeReturns.err +} + +func (fake *fakeImpl) NewRemoteRuntimeServiceReturns(res cri.RuntimeService, err error) { + fake.newRemoteRuntimeServiceReturns = struct { + res cri.RuntimeService + err error + }{res, err} +} + +func (fake *fakeImpl) PullImage(context.Context, cri.ImageManagerService, *v1.ImageSpec, *v1.AuthConfig, *v1.PodSandboxConfig) (string, error) { + fakeReturns := fake.pullImageReturns + return fakeReturns.res, fakeReturns.err +} + +func (fake *fakeImpl) PullImageReturns(res string, err error) { + fake.pullImageReturns = struct { + res string + err error + }{res, err} +} + +func (fake *fakeImpl) RemovePodSandbox(context.Context, cri.RuntimeService, string) error { + fakeReturns := fake.removePodSandboxReturns + return fakeReturns.res +} + +func (fake *fakeImpl) RemovePodSandboxReturns(res error) { + fake.removePodSandboxReturns = struct { + res error + }{res} +} + +func (fake *fakeImpl) Status(context.Context, cri.RuntimeService, bool) (*v1.StatusResponse, error) { + fakeReturns := fake.statusReturns + return fakeReturns.res, fakeReturns.err +} + +func (fake *fakeImpl) StatusReturns(res *v1.StatusResponse, err error) { + fake.statusReturns = struct { + res *v1.StatusResponse + err error + }{res, err} +} + +func (fake *fakeImpl) StopPodSandbox(context.Context, cri.RuntimeService, string) error { + fakeReturns := fake.stopPodSandboxReturns + return fakeReturns.res +} + +func (fake *fakeImpl) StopPodSandboxReturns(res error) { + fake.stopPodSandboxReturns = struct { + res error + }{res} +} diff --git a/cmd/kubeadm/app/util/runtime/runtime_test.go b/cmd/kubeadm/app/util/runtime/runtime_test.go index ff2452a654c..caeacb75f73 100644 --- a/cmd/kubeadm/app/util/runtime/runtime_test.go +++ b/cmd/kubeadm/app/util/runtime/runtime_test.go @@ -17,331 +17,314 @@ limitations under the License. package runtime import ( + "errors" "net" "os" - "reflect" "runtime" "testing" - "github.com/pkg/errors" + "github.com/stretchr/testify/assert" - "k8s.io/utils/exec" - fakeexec "k8s.io/utils/exec/testing" + v1 "k8s.io/cri-api/pkg/apis/runtime/v1" "k8s.io/kubernetes/cmd/kubeadm/app/constants" ) +var errTest = errors.New("test") + func TestNewContainerRuntime(t *testing.T) { - execLookPathOK := &fakeexec.FakeExec{ - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, - } - execLookPathErr := &fakeexec.FakeExec{ - LookPathFunc: func(cmd string) (string, error) { return "", errors.Errorf("%s not found", cmd) }, - } - cases := []struct { - name string - execer *fakeexec.FakeExec - isError bool + for _, tc := range []struct { + name string + prepare func(*fakeImpl) + shouldError bool }{ - {"valid: crictl present", execLookPathOK, false}, - {"invalid: no crictl", execLookPathErr, true}, - } - - for _, tc := range cases { + { + name: "valid", + shouldError: false, + }, + { + name: "invalid: new runtime service fails", + prepare: func(mock *fakeImpl) { + mock.NewRemoteRuntimeServiceReturns(nil, errTest) + }, + shouldError: true, + }, + { + name: "invalid: new image service fails", + prepare: func(mock *fakeImpl) { + mock.NewRemoteImageServiceReturns(nil, errTest) + }, + shouldError: true, + }, + } { t.Run(tc.name, func(t *testing.T) { - _, err := NewContainerRuntime(tc.execer, "unix:///some/socket.sock") - if err != nil { - if !tc.isError { - t.Fatalf("unexpected NewContainerRuntime error. error: %v", err) - } - return // expected error occurs, impossible to test runtime further + containerRuntime := NewContainerRuntime("") + mock := &fakeImpl{} + if tc.prepare != nil { + tc.prepare(mock) } - if tc.isError && err == nil { - t.Fatal("unexpected NewContainerRuntime success") - } - }) - } -} + containerRuntime.SetImpl(mock) -func genFakeActions(fcmd *fakeexec.FakeCmd, num int) []fakeexec.FakeCommandAction { - var actions []fakeexec.FakeCommandAction - for i := 0; i < num; i++ { - actions = append(actions, func(cmd string, args ...string) exec.Cmd { - return fakeexec.InitFakeCmd(fcmd, cmd, args...) + err := containerRuntime.Connect() + + assert.Equal(t, tc.shouldError, err != nil) }) } - return actions } func TestIsRunning(t *testing.T) { - fcmd := fakeexec.FakeCmd{ - CombinedOutputScript: []fakeexec.FakeAction{ - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - }, - } - - criExecer := &fakeexec.FakeExec{ - CommandScript: genFakeActions(&fcmd, len(fcmd.CombinedOutputScript)), - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, - } - - cases := []struct { - name string - criSocket string - execer *fakeexec.FakeExec - isError bool + for _, tc := range []struct { + name string + prepare func(*fakeImpl) + shouldError bool }{ - {"valid: CRI-O is running", "unix:///var/run/crio/crio.sock", criExecer, false}, - {"invalid: CRI-O is not running", "unix:///var/run/crio/crio.sock", criExecer, true}, - } - - for _, tc := range cases { + { + name: "valid", + shouldError: false, + }, + { + name: "invalid: runtime status fails", + prepare: func(mock *fakeImpl) { + mock.StatusReturns(nil, errTest) + }, + shouldError: true, + }, + { + name: "invalid: runtime condition status not 'true'", + prepare: func(mock *fakeImpl) { + mock.StatusReturns(&v1.StatusResponse{Status: &v1.RuntimeStatus{ + Conditions: []*v1.RuntimeCondition{ + { + Status: false, + }, + }, + }, + }, nil) + }, + shouldError: true, + }, + } { t.Run(tc.name, func(t *testing.T) { - runtime, err := NewContainerRuntime(tc.execer, tc.criSocket) - if err != nil { - t.Fatalf("unexpected NewContainerRuntime error: %v", err) - } - isRunning := runtime.IsRunning() - if tc.isError && isRunning == nil { - t.Error("unexpected IsRunning() success") - } - if !tc.isError && isRunning != nil { - t.Error("unexpected IsRunning() error") + containerRuntime := NewContainerRuntime("") + mock := &fakeImpl{} + if tc.prepare != nil { + tc.prepare(mock) } + containerRuntime.SetImpl(mock) + + err := containerRuntime.IsRunning() + + assert.Equal(t, tc.shouldError, err != nil) }) } } func TestListKubeContainers(t *testing.T) { - fcmd := fakeexec.FakeCmd{ - CombinedOutputScript: []fakeexec.FakeAction{ - func() ([]byte, []byte, error) { return []byte("k8s_p1\nk8s_p2"), nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("k8s_p1\nk8s_p2"), nil, nil }, - }, - } - execer := &fakeexec.FakeExec{ - CommandScript: genFakeActions(&fcmd, len(fcmd.CombinedOutputScript)), - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, - } - - cases := []struct { - name string - criSocket string - isError bool + for _, tc := range []struct { + name string + expected []string + prepare func(*fakeImpl) + shouldError bool }{ - {"valid: list containers using CRI socket url", "unix:///var/run/crio/crio.sock", false}, - {"invalid: list containers using CRI socket url", "unix:///var/run/crio/crio.sock", true}, - } - - for _, tc := range cases { + { + name: "valid", + prepare: func(mock *fakeImpl) { + mock.ListPodSandboxReturns([]*v1.PodSandbox{ + {Id: "first"}, + {Id: "second"}, + }, nil) + }, + expected: []string{"first", "second"}, + shouldError: false, + }, + { + name: "invalid: list pod sandbox fails", + prepare: func(mock *fakeImpl) { + mock.ListPodSandboxReturns(nil, errTest) + }, + shouldError: true, + }, + } { t.Run(tc.name, func(t *testing.T) { - runtime, err := NewContainerRuntime(execer, tc.criSocket) - if err != nil { - t.Fatalf("unexpected NewContainerRuntime error: %v", err) + containerRuntime := NewContainerRuntime("") + mock := &fakeImpl{} + if tc.prepare != nil { + tc.prepare(mock) } + containerRuntime.SetImpl(mock) - containers, err := runtime.ListKubeContainers() - if tc.isError { - if err == nil { - t.Errorf("unexpected ListKubeContainers success") - } - return - } else if err != nil { - t.Errorf("unexpected ListKubeContainers error: %v", err) - } + containers, err := containerRuntime.ListKubeContainers() - if !reflect.DeepEqual(containers, []string{"k8s_p1", "k8s_p2"}) { - t.Errorf("unexpected ListKubeContainers output: %v", containers) - } + assert.Equal(t, tc.shouldError, err != nil) + assert.Equal(t, tc.expected, containers) }) } } func TestSandboxImage(t *testing.T) { - fcmd := fakeexec.FakeCmd{ - CombinedOutputScript: []fakeexec.FakeAction{ - func() ([]byte, []byte, error) { return []byte("registry.k8s.io/pause:ver"), nil, nil }, - func() ([]byte, []byte, error) { return []byte("registry.k8s.io/pause:ver\n"), nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, - }, - } - - execer := &fakeexec.FakeExec{ - CommandScript: genFakeActions(&fcmd, len(fcmd.CombinedOutputScript)), - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, - } - - cases := []struct { - name string - expected string - isError bool + for _, tc := range []struct { + name, expected string + prepare func(*fakeImpl) + shouldError bool }{ - {"valid: read sandbox image normally", "registry.k8s.io/pause:ver", false}, - {"valid: read sandbox image with leading/trailing white spaces", "registry.k8s.io/pause:ver", false}, - {"invalid: read empty sandbox image", "", true}, - {"invalid: failed to read sandbox image", "", true}, - } - - for _, tc := range cases { + { + name: "valid", + prepare: func(mock *fakeImpl) { + mock.StatusReturns(&v1.StatusResponse{Info: map[string]string{ + "config": `{"sandboxImage": "pause"}`, + }}, nil) + }, + expected: "pause", + shouldError: false, + }, + { + name: "invalid: runtime status fails", + prepare: func(mock *fakeImpl) { + mock.StatusReturns(nil, errTest) + }, + shouldError: true, + }, + { + name: "invalid: no config JSON", + prepare: func(mock *fakeImpl) { + mock.StatusReturns(&v1.StatusResponse{Info: map[string]string{ + "config": "wrong", + }}, nil) + }, + shouldError: true, + }, + { + name: "invalid: no config", + prepare: func(mock *fakeImpl) { + mock.StatusReturns(&v1.StatusResponse{Info: map[string]string{}}, nil) + }, + shouldError: true, + }, + } { t.Run(tc.name, func(t *testing.T) { - runtime, err := NewContainerRuntime(execer, "unix:///some/socket.sock") - if err != nil { - t.Fatalf("unexpected NewContainerRuntime error: %v", err) + containerRuntime := NewContainerRuntime("") + mock := &fakeImpl{} + if tc.prepare != nil { + tc.prepare(mock) } + containerRuntime.SetImpl(mock) - sandboxImage, err := runtime.SandboxImage() - if tc.isError { - if err == nil { - t.Errorf("unexpected SandboxImage success") - } - return - } else if err != nil { - t.Errorf("unexpected SandboxImage error: %v", err) - } + image, err := containerRuntime.SandboxImage() - if sandboxImage != tc.expected { - t.Errorf("expected sandbox image %v, but got %v", tc.expected, sandboxImage) - } + assert.Equal(t, tc.shouldError, err != nil) + assert.Equal(t, tc.expected, image) }) } } func TestRemoveContainers(t *testing.T) { - fakeOK := func() ([]byte, []byte, error) { return nil, nil, nil } - fakeErr := func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} } - fcmd := fakeexec.FakeCmd{ - CombinedOutputScript: []fakeexec.FakeAction{ - fakeOK, fakeOK, fakeOK, fakeOK, fakeOK, fakeOK, // Test case 1 - fakeOK, fakeOK, fakeOK, fakeErr, fakeOK, fakeErr, fakeOK, fakeErr, fakeOK, fakeErr, fakeOK, fakeErr, fakeOK, fakeOK, // Test case 2 - fakeErr, fakeErr, fakeErr, fakeErr, fakeErr, fakeOK, fakeOK, fakeOK, fakeOK, // Test case 3 - }, - } - execer := &fakeexec.FakeExec{ - CommandScript: genFakeActions(&fcmd, len(fcmd.CombinedOutputScript)), - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, - } - - cases := []struct { - name string - criSocket string - containers []string - isError bool + for _, tc := range []struct { + name string + containers []string + prepare func(*fakeImpl) + shouldError bool }{ - {"valid: remove containers using CRI", "unix:///var/run/crio/crio.sock", []string{"k8s_p1", "k8s_p2", "k8s_p3"}, false}, // Test case 1 - {"invalid: CRI rmp failure", "unix:///var/run/crio/crio.sock", []string{"k8s_p1", "k8s_p2", "k8s_p3"}, true}, // Test case 2 - {"invalid: CRI stopp failure", "unix:///var/run/crio/crio.sock", []string{"k8s_p1", "k8s_p2", "k8s_p3"}, true}, // Test case 3 - } - - for _, tc := range cases { + { + name: "valid", + }, + { + name: "valid: two containers", + containers: []string{"1", "2"}, + shouldError: false, + }, + { + name: "invalid: remove pod sandbox fails", + containers: []string{"1"}, + prepare: func(mock *fakeImpl) { + mock.RemovePodSandboxReturns(errTest) + }, + shouldError: true, + }, + { + name: "invalid: stop pod sandbox fails", + containers: []string{"1"}, + prepare: func(mock *fakeImpl) { + mock.StopPodSandboxReturns(errTest) + }, + shouldError: true, + }, + } { t.Run(tc.name, func(t *testing.T) { - runtime, err := NewContainerRuntime(execer, tc.criSocket) - if err != nil { - t.Fatalf("unexpected NewContainerRuntime error: %v, criSocket: %s", err, tc.criSocket) + containerRuntime := NewContainerRuntime("") + mock := &fakeImpl{} + if tc.prepare != nil { + tc.prepare(mock) } + containerRuntime.SetImpl(mock) - err = runtime.RemoveContainers(tc.containers) - if !tc.isError && err != nil { - t.Errorf("unexpected RemoveContainers errors: %v, criSocket: %s, containers: %v", err, tc.criSocket, tc.containers) - } - if tc.isError && err == nil { - t.Errorf("unexpected RemoveContainers success, criSocket: %s, containers: %v", tc.criSocket, tc.containers) - } + err := containerRuntime.RemoveContainers(tc.containers) + + assert.Equal(t, tc.shouldError, err != nil) }) } } func TestPullImage(t *testing.T) { - fcmd := fakeexec.FakeCmd{ - CombinedOutputScript: []fakeexec.FakeAction{ - func() ([]byte, []byte, error) { return nil, nil, nil }, - // If the pull fails, it will be retried 5 times (see PullImageRetry in constants/constants.go) - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return nil, nil, nil }, - // If the pull fails, it will be retried 5 times (see PullImageRetry in constants/constants.go) - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - }, - } - execer := &fakeexec.FakeExec{ - CommandScript: genFakeActions(&fcmd, len(fcmd.CombinedOutputScript)), - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, - } - - cases := []struct { - name string - criSocket string - image string - isError bool + for _, tc := range []struct { + name string + prepare func(*fakeImpl) + shouldError bool }{ - {"valid: pull image using CRI", "unix:///var/run/crio/crio.sock", "image1", false}, - {"invalid: CRI pull error", "unix:///var/run/crio/crio.sock", "image2", true}, - } - - for _, tc := range cases { + { + name: "valid", + }, + { + name: "invalid: pull image fails", + prepare: func(mock *fakeImpl) { + mock.PullImageReturns("", errTest) + }, + shouldError: true, + }, + } { t.Run(tc.name, func(t *testing.T) { - runtime, err := NewContainerRuntime(execer, tc.criSocket) - if err != nil { - t.Fatalf("unexpected NewContainerRuntime error: %v, criSocket: %s", err, tc.criSocket) + containerRuntime := NewContainerRuntime("") + mock := &fakeImpl{} + if tc.prepare != nil { + tc.prepare(mock) } + containerRuntime.SetImpl(mock) - err = runtime.PullImage(tc.image) - if !tc.isError && err != nil { - t.Errorf("unexpected PullImage error: %v, criSocket: %s, image: %s", err, tc.criSocket, tc.image) - } - if tc.isError && err == nil { - t.Errorf("unexpected PullImage success, criSocket: %s, image: %s", tc.criSocket, tc.image) - } + err := containerRuntime.PullImage("") + + assert.Equal(t, tc.shouldError, err != nil) }) } } func TestImageExists(t *testing.T) { - fcmd := fakeexec.FakeCmd{ - RunScript: []fakeexec.FakeAction{ - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, - }, - } - execer := &fakeexec.FakeExec{ - CommandScript: genFakeActions(&fcmd, len(fcmd.RunScript)), - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, - } - - cases := []struct { - name string - criSocket string - image string - result bool + for _, tc := range []struct { + name string + expected bool + prepare func(*fakeImpl) }{ - {"valid: test if image exists using CRI", "unix:///var/run/crio/crio.sock", "image1", false}, - {"invalid: CRI inspect failure", "unix:///var/run/crio/crio.sock", "image2", true}, - } - - for _, tc := range cases { + { + name: "valid", + expected: true, + }, + { + name: "invalid: image status fails", + prepare: func(mock *fakeImpl) { + mock.ImageStatusReturns(nil, errTest) + }, + expected: false, + }, + } { t.Run(tc.name, func(t *testing.T) { - runtime, err := NewContainerRuntime(execer, tc.criSocket) - if err != nil { - t.Fatalf("unexpected NewContainerRuntime error: %v, criSocket: %s", err, tc.criSocket) + containerRuntime := NewContainerRuntime("") + mock := &fakeImpl{} + if tc.prepare != nil { + tc.prepare(mock) } + containerRuntime.SetImpl(mock) - result, err := runtime.ImageExists(tc.image) - if !tc.result != result { - t.Errorf("unexpected ImageExists result: %t, criSocket: %s, image: %s, expected result: %t", err, tc.criSocket, tc.image, tc.result) - } + exists := containerRuntime.ImageExists("") + + assert.Equal(t, tc.expected, exists) }) } } @@ -462,77 +445,39 @@ func TestDetectCRISocketImpl(t *testing.T) { } } -func TestPullImagesInParallelImpl(t *testing.T) { - testError := errors.New("error") - - tests := []struct { - name string - images []string - ifNotPresent bool - imageExistsFunc func(string) (bool, error) - pullImageFunc func(string) error - expectedErrors int +func TestPullImagesInParallel(t *testing.T) { + for _, tc := range []struct { + name string + ifNotPresent bool + prepare func(*fakeImpl) + shouldError bool }{ { - name: "all images exist, no errors", - images: []string{"foo", "bar", "baz"}, - ifNotPresent: true, - imageExistsFunc: func(string) (bool, error) { - return true, nil - }, - pullImageFunc: nil, - expectedErrors: 0, + name: "valid", }, { - name: "cannot check if one image exists due to error", - images: []string{"foo", "bar", "baz"}, + name: "valid: ifNotPresent is true", ifNotPresent: true, - imageExistsFunc: func(image string) (bool, error) { - if image == "baz" { - return false, testError - } - return true, nil - }, - pullImageFunc: nil, - expectedErrors: 1, }, { - name: "cannot pull two images", - images: []string{"foo", "bar", "baz"}, - ifNotPresent: true, - imageExistsFunc: func(string) (bool, error) { - return false, nil + name: "invalid: pull fails", + prepare: func(mock *fakeImpl) { + mock.PullImageReturns("", errTest) }, - pullImageFunc: func(image string) error { - if image == "foo" { - return nil - } - return testError - }, - expectedErrors: 2, + shouldError: true, }, - { - name: "pull all images", - images: []string{"foo", "bar", "baz"}, - ifNotPresent: true, - imageExistsFunc: func(string) (bool, error) { - return false, nil - }, - pullImageFunc: func(string) error { - return nil - }, - expectedErrors: 0, - }, - } - - for _, tc := range tests { + } { t.Run(tc.name, func(t *testing.T) { - actual := pullImagesInParallelImpl(tc.images, tc.ifNotPresent, - tc.imageExistsFunc, tc.pullImageFunc) - if len(actual) != tc.expectedErrors { - t.Fatalf("expected non-nil errors: %v, got: %v, full list of errors: %v", - tc.expectedErrors, len(actual), actual) + containerRuntime := NewContainerRuntime("") + mock := &fakeImpl{} + if tc.prepare != nil { + tc.prepare(mock) } + containerRuntime.SetImpl(mock) + + err := containerRuntime.PullImagesInParallel([]string{"first", "second"}, tc.ifNotPresent) + + assert.Equal(t, tc.shouldError, err != nil) }) } }