From c7234aa17791ef0a06e4a29d32238689e1b1ede6 Mon Sep 17 00:00:00 2001 From: SataQiu <1527062125@qq.com> Date: Sat, 11 Jan 2020 15:10:56 +0800 Subject: [PATCH] kubeadm: support automatic retry after failing to pull image --- cmd/kubeadm/app/constants/constants.go | 2 ++ cmd/kubeadm/app/preflight/checks_test.go | 17 ++++++++++++++ cmd/kubeadm/app/util/runtime/runtime.go | 24 +++++++++++++------- cmd/kubeadm/app/util/runtime/runtime_test.go | 10 ++++++++ 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index 75adf43558a..0c8abaaf2fa 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -181,6 +181,8 @@ const ( PatchNodeTimeout = 2 * time.Minute // TLSBootstrapTimeout specifies how long kubeadm should wait for the kubelet to perform the TLS Bootstrap TLSBootstrapTimeout = 2 * time.Minute + // PullImageRetry specifies how many times ContainerRuntime retries when pulling image failed + PullImageRetry = 5 // PrepullImagesInParallelTimeout specifies how long kubeadm should wait for prepulling images in parallel before timing out PrepullImagesInParallelTimeout = 10 * time.Second diff --git a/cmd/kubeadm/app/preflight/checks_test.go b/cmd/kubeadm/app/preflight/checks_test.go index c600c8d1b47..3a6b2f27ab9 100644 --- a/cmd/kubeadm/app/preflight/checks_test.go +++ b/cmd/kubeadm/app/preflight/checks_test.go @@ -762,9 +762,18 @@ func TestImagePullCheck(t *testing.T) { // Test case1: pull only img3 func() ([]byte, []byte, error) { return nil, nil, nil }, // Test case 2: fail to pull image2 and image3 + // If the pull fails, it will be retried 5 times (see PullImageRetry in constants/constants.go) func() ([]byte, []byte, error) { return nil, nil, nil }, func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, + func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, + func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, + func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, + func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, + func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, + func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, + func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, + func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, }, } @@ -780,6 +789,14 @@ func TestImagePullCheck(t *testing.T) { func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/docker", nil }, } diff --git a/cmd/kubeadm/app/util/runtime/runtime.go b/cmd/kubeadm/app/util/runtime/runtime.go index 71e2c04521a..4dc08805a13 100644 --- a/cmd/kubeadm/app/util/runtime/runtime.go +++ b/cmd/kubeadm/app/util/runtime/runtime.go @@ -151,20 +151,28 @@ func (runtime *DockerRuntime) RemoveContainers(containers []string) error { // PullImage pulls the image func (runtime *CRIRuntime) PullImage(image string) error { - out, err := runtime.exec.Command("crictl", "-r", runtime.criSocket, "pull", image).CombinedOutput() - if err != nil { - return errors.Wrapf(err, "output: %s, error", string(out)) + var err error + var out []byte + for i := 0; i < constants.PullImageRetry; i++ { + out, err = runtime.exec.Command("crictl", "-r", runtime.criSocket, "pull", image).CombinedOutput() + if err == nil { + return nil + } } - return nil + return errors.Wrapf(err, "output: %s, error", out) } // PullImage pulls the image func (runtime *DockerRuntime) PullImage(image string) error { - out, err := runtime.exec.Command("docker", "pull", image).CombinedOutput() - if err != nil { - return errors.Wrapf(err, "output: %s, error", string(out)) + var err error + var out []byte + for i := 0; i < constants.PullImageRetry; i++ { + out, err = runtime.exec.Command("docker", "pull", image).CombinedOutput() + if err == nil { + return nil + } } - return nil + return errors.Wrapf(err, "output: %s, error", out) } // ImageExists checks to see if the image exists on the system diff --git a/cmd/kubeadm/app/util/runtime/runtime_test.go b/cmd/kubeadm/app/util/runtime/runtime_test.go index 49c2d11386d..4352d9fdf78 100644 --- a/cmd/kubeadm/app/util/runtime/runtime_test.go +++ b/cmd/kubeadm/app/util/runtime/runtime_test.go @@ -229,8 +229,18 @@ func TestPullImage(t *testing.T) { fcmd := fakeexec.FakeCmd{ CombinedOutputScript: []fakeexec.FakeAction{ func() ([]byte, []byte, error) { return nil, nil, nil }, + // If the pull fails, it will be retried 5 times (see PullImageRetry in constants/constants.go) + func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, + func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, + func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, + func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, func() ([]byte, []byte, error) { return nil, nil, nil }, + // If the pull fails, it will be retried 5 times (see PullImageRetry in constants/constants.go) + func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, + func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, + func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, + func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, }, }