github/multus-cni
1
0
Fork 1
mirror of https://github.com/k8snetworkplumbingwg/multus-cni.git synced 2025-08-16 07:15:54 +00:00
Code Issues Projects Releases Wiki Activity

Retry pod retrieval on multiple error types

Browse Source 
Also retry on the following error types (cmd ADD/DEL):
- IsInternalError
- IsConnectionReset
- IsConnectionRefused

Signed-off-by: Miguel Duarte Barroso <mdbarroso@redhat.com>
This commit is contained in:
Miguel Duarte Barroso 2021-06-24 13:35:41 +02:00 committed by Tomofumi Hayashi
parent 7f5620db76
commit 1eeab9b589
1 changed files with 19 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
pkg/multus/multus.go
View File

@ -40,6 +40,7 @@ import (
"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types" "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
v1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/errors"
k8snet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/util/wait"
) )
@ -503,6 +504,18 @@ func cmdPluginErr(k8sArgs *types.K8sArgs, confName string, format string, args .
return logging.Errorf(msg+format, args...) return logging.Errorf(msg+format, args...)
} }
func isCriticalRequestRetriable(err error) bool {
logging.Debugf("isCriticalRequestRetriable: %v", err)
errorTypesAllowingRetry := []func(error) bool{
errors.IsServiceUnavailable, errors.IsInternalError, k8snet.IsConnectionReset, k8snet.IsConnectionRefused}
for _, f := range errorTypesAllowingRetry {
if f(err) {
return true
}
}
return false
}
//CmdAdd ... //CmdAdd ...
func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (cnitypes.Result, error) { func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (cnitypes.Result, error) {
n, err := types.LoadNetConf(args.StdinData) n, err := types.LoadNetConf(args.StdinData)
@ -536,8 +549,8 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
pod, err = kubeClient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME)) pod, err = kubeClient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
if err != nil { if err != nil {
var waitErr error var waitErr error
// in case of ServiceUnavailable, retry 10 times with 0.5 sec interval // in case of a retriable error, retry 10 times with 0.5 sec interval
if errors.IsServiceUnavailable(err) { if isCriticalRequestRetriable(err) {
pollDuration := 500 * time.Millisecond pollDuration := 500 * time.Millisecond
pollTimeout := 5 * time.Second pollTimeout := 5 * time.Second
waitErr = wait.PollImmediate(pollDuration, pollTimeout, func() (bool, error) { waitErr = wait.PollImmediate(pollDuration, pollTimeout, func() (bool, error) {
@ -546,7 +559,7 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
}) })
// retry failed, then return error with retry out // retry failed, then return error with retry out
if waitErr != nil { if waitErr != nil {
return nil, cmdErr(k8sArgs, "error getting pod by service unavailable: %v", err) return nil, cmdErr(k8sArgs, "error getting pod with error: %v", err)
} }
} else { } else {
// Other case, return error // Other case, return error
@ -763,8 +776,8 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
pod, err = kubeClient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME)) pod, err = kubeClient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
if err != nil { if err != nil {
var waitErr error var waitErr error
// in case of ServiceUnavailable, retry 10 times with 0.5 sec interval // in case of a retriable error, retry 10 times with 0.5 sec interval
if errors.IsServiceUnavailable(err) { if isCriticalRequestRetriable(err) {
pollDuration := 500 * time.Millisecond pollDuration := 500 * time.Millisecond
pollTimeout := 5 * time.Second pollTimeout := 5 * time.Second
waitErr = wait.PollImmediate(pollDuration, pollTimeout, func() (bool, error) { waitErr = wait.PollImmediate(pollDuration, pollTimeout, func() (bool, error) {
@ -773,7 +786,7 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
}) })
// retry failed, then return error with retry out // retry failed, then return error with retry out
if waitErr != nil { if waitErr != nil {
return cmdErr(k8sArgs, "error getting pod by service unavailable: %v", err) return cmdErr(k8sArgs, "error getting pod with error: %v", err)
} }
} else if errors.IsNotFound(err) { } else if errors.IsNotFound(err) {
// If not found, proceed to remove interface with cache // If not found, proceed to remove interface with cache