github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-31 07:20:13 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #86899 from SataQiu/enable-pull-retry-20200107

Browse Source 
kubeadm: support automatic retry after failing to pull image
This commit is contained in:
Kubernetes Prow Robot 2020-01-13 13:58:41 -08:00 committed by GitHub
commit 3e8155ead3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 45 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/kubeadm/app/constants/constants.go
View File

@ -181,6 +181,8 @@ const (
PatchNodeTimeout = 2 * time.Minute
// TLSBootstrapTimeout specifies how long kubeadm should wait for the kubelet to perform the TLS Bootstrap
TLSBootstrapTimeout = 2 * time.Minute
// PullImageRetry specifies how many times ContainerRuntime retries when pulling image failed
PullImageRetry = 5
// PrepullImagesInParallelTimeout specifies how long kubeadm should wait for prepulling images in parallel before timing out
PrepullImagesInParallelTimeout = 10 * time.Second

17
cmd/kubeadm/app/preflight/checks_test.go
View File

@ -762,9 +762,18 @@ func TestImagePullCheck(t *testing.T) {
// Test case1: pull only img3
func() ([]byte, []byte, error) { return nil, nil, nil },
// Test case 2: fail to pull image2 and image3
// If the pull fails, it will be retried 5 times (see PullImageRetry in constants/constants.go)
func() ([]byte, []byte, error) { return nil, nil, nil },
func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} },
func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} },
func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} },
func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} },
func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} },
func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} },
func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} },
func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} },
func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} },
func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} },
},
}
@ -780,6 +789,14 @@ func TestImagePullCheck(t *testing.T) {
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
},
LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/docker", nil },
}

24
cmd/kubeadm/app/util/runtime/runtime.go
View File

@ -151,20 +151,28 @@ func (runtime *DockerRuntime) RemoveContainers(containers []string) error {
// PullImage pulls the image
func (runtime *CRIRuntime) PullImage(image string) error {
out, err := runtime.exec.Command("crictl", "-r", runtime.criSocket, "pull", image).CombinedOutput()
if err != nil {
return errors.Wrapf(err, "output: %s, error", string(out))
var err error
var out []byte
for i := 0; i < constants.PullImageRetry; i++ {
out, err = runtime.exec.Command("crictl", "-r", runtime.criSocket, "pull", image).CombinedOutput()
if err == nil {
return nil
}
}
return nil
return errors.Wrapf(err, "output: %s, error", out)
}
// PullImage pulls the image
func (runtime *DockerRuntime) PullImage(image string) error {
out, err := runtime.exec.Command("docker", "pull", image).CombinedOutput()
if err != nil {
return errors.Wrapf(err, "output: %s, error", string(out))
var err error
var out []byte
for i := 0; i < constants.PullImageRetry; i++ {
out, err = runtime.exec.Command("docker", "pull", image).CombinedOutput()
if err == nil {
return nil
}
}
return nil
return errors.Wrapf(err, "output: %s, error", out)
}
// ImageExists checks to see if the image exists on the system

10
cmd/kubeadm/app/util/runtime/runtime_test.go
View File

@ -229,8 +229,18 @@ func TestPullImage(t *testing.T) {
fcmd := fakeexec.FakeCmd{
CombinedOutputScript: []fakeexec.FakeAction{
func() ([]byte, []byte, error) { return nil, nil, nil },
// If the pull fails, it will be retried 5 times (see PullImageRetry in constants/constants.go)
func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} },
func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} },
func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} },
func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} },
func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} },
func() ([]byte, []byte, error) { return nil, nil, nil },
// If the pull fails, it will be retried 5 times (see PullImageRetry in constants/constants.go)
func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} },
func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} },
func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} },
func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} },
func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} },
},
}