github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-28 05:57:25 +00:00
Code Issues Projects Releases Wiki Activity

kubeadm: support retry mechanism for removing container in reset phase

Browse Source
This commit is contained in:
SataQiu 2022-06-29 17:54:56 +08:00
parent 6269784cd0
commit 3889a6cac8
3 changed files with 23 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/kubeadm/app/constants/constants.go
View File

@ -218,6 +218,8 @@ const (
APICallWithReadTimeout = 15 * time.Second APICallWithReadTimeout = 15 * time.Second
// PullImageRetry specifies how many times ContainerRuntime retries when pulling image failed // PullImageRetry specifies how many times ContainerRuntime retries when pulling image failed
PullImageRetry = 5 PullImageRetry = 5
// RemoveContainerRetry specifies how many times ContainerRuntime retries when removing container failed
RemoveContainerRetry = 5
// DefaultControlPlaneTimeout specifies the default control plane (actually API Server) timeout for use by kubeadm // DefaultControlPlaneTimeout specifies the default control plane (actually API Server) timeout for use by kubeadm
DefaultControlPlaneTimeout = 4 * time.Minute DefaultControlPlaneTimeout = 4 * time.Minute

19
cmd/kubeadm/app/util/runtime/runtime.go
View File

@ -23,6 +23,7 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
errorsutil "k8s.io/apimachinery/pkg/util/errors" errorsutil "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/klog/v2"
utilsexec "k8s.io/utils/exec" utilsexec "k8s.io/utils/exec"
"k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/constants"
@ -97,15 +98,25 @@ func (runtime *CRIRuntime) ListKubeContainers() ([]string, error) {
func (runtime *CRIRuntime) RemoveContainers(containers []string) error { func (runtime *CRIRuntime) RemoveContainers(containers []string) error {
errs := []error{} errs := []error{}
for _, container := range containers { for _, container := range containers {
var lastErr error
for i := 0; i < constants.RemoveContainerRetry; i++ {
klog.V(5).Infof("Attempting to remove container %v", container)
out, err := runtime.crictl("stopp", container).CombinedOutput() out, err := runtime.crictl("stopp", container).CombinedOutput()
if err != nil { if err != nil {
// don't stop on errors, try to remove as many containers as possible lastErr = errors.Wrapf(err, "failed to stop running pod %s: output: %s", container, string(out))
errs = append(errs, errors.Wrapf(err, "failed to stop running pod %s: output: %s, error", container, string(out))) continue
} else { }
out, err = runtime.crictl("rmp", container).CombinedOutput() out, err = runtime.crictl("rmp", container).CombinedOutput()
if err != nil { if err != nil {
errs = append(errs, errors.Wrapf(err, "failed to remove running container %s: output: %s, error", container, string(out))) lastErr = errors.Wrapf(err, "failed to remove running container %s: output: %s", container, string(out))
continue
} }
lastErr = nil
break
}
if lastErr != nil {
errs = append(errs, lastErr)
} }
} }
return errorsutil.NewAggregate(errs) return errorsutil.NewAggregate(errs)

11
cmd/kubeadm/app/util/runtime/runtime_test.go
View File

@ -167,11 +167,8 @@ func TestRemoveContainers(t *testing.T) {
fcmd := fakeexec.FakeCmd{ fcmd := fakeexec.FakeCmd{
CombinedOutputScript: []fakeexec.FakeAction{ CombinedOutputScript: []fakeexec.FakeAction{
fakeOK, fakeOK, fakeOK, fakeOK, fakeOK, fakeOK, // Test case 1 fakeOK, fakeOK, fakeOK, fakeOK, fakeOK, fakeOK, // Test case 1
fakeOK, fakeOK, fakeOK, fakeErr, fakeOK, fakeOK, fakeOK, fakeOK, fakeOK, fakeErr, fakeOK, fakeErr, fakeOK, fakeErr, fakeOK, fakeErr, fakeOK, fakeErr, fakeOK, fakeOK, // Test case 2
fakeErr, fakeOK, fakeOK, fakeErr, fakeOK, fakeErr, fakeErr, fakeErr, fakeErr, fakeErr, fakeOK, fakeOK, fakeOK, fakeOK, // Test case 3
fakeOK, fakeOK, fakeOK, fakeOK, fakeOK, fakeOK,
fakeOK, fakeOK, fakeOK, fakeErr, fakeOK, fakeOK,
fakeErr, fakeOK, fakeOK, fakeErr, fakeOK,
}, },
} }
execer := fakeexec.FakeExec{ execer := fakeexec.FakeExec{
@ -186,8 +183,8 @@ func TestRemoveContainers(t *testing.T) {
isError bool isError bool
}{ }{
{"valid: remove containers using CRI", "unix:///var/run/crio/crio.sock", []string{"k8s_p1", "k8s_p2", "k8s_p3"}, false}, // Test case 1 {"valid: remove containers using CRI", "unix:///var/run/crio/crio.sock", []string{"k8s_p1", "k8s_p2", "k8s_p3"}, false}, // Test case 1
{"invalid: CRI rmp failure", "unix:///var/run/crio/crio.sock", []string{"k8s_p1", "k8s_p2", "k8s_p3"}, true}, {"invalid: CRI rmp failure", "unix:///var/run/crio/crio.sock", []string{"k8s_p1", "k8s_p2", "k8s_p3"}, true}, // Test case 2
{"invalid: CRI stopp failure", "unix:///var/run/crio/crio.sock", []string{"k8s_p1", "k8s_p2", "k8s_p3"}, true}, {"invalid: CRI stopp failure", "unix:///var/run/crio/crio.sock", []string{"k8s_p1", "k8s_p2", "k8s_p3"}, true}, // Test case 3
} }
for _, tc := range cases { for _, tc := range cases {