diff --git a/docs/configuration.md b/docs/configuration.md index 7d56b3401..f75b9a0ee 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -37,7 +37,8 @@ Following is the example of multus config file, in `/etc/cni/net.d/`. }, { "type": "macvlan", ... (snip) - }] + }], + allowTryDeleteOnErr: false } ``` @@ -62,6 +63,7 @@ User should chose following parameters combination (`clusterNetwork`+`defaultNet * `systemNamespaces` ([]string, optional): list of namespaces for Kubernetes system (namespaces listed here will not have `defaultNetworks` added) * `multusNamespace` (string, optional): namespace for `clusterNetwork`/`defaultNetworks` * `delegates` ([]map,required): number of delegate details in the Multus +* `retryDeleteOnError` (bool, optional): Enable or disable delegate DEL message to next when some missing error. Defaults to false. ### Network selection flow of clusterNetwork/defaultNetworks diff --git a/pkg/multus/multus.go b/pkg/multus/multus.go index 8dd95c89f..d11a2db7c 100644 --- a/pkg/multus/multus.go +++ b/pkg/multus/multus.go @@ -825,6 +825,7 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er // Read the cache to get delegates json for the pod netconfBytes, path, err := consumeScratchNetConf(args.ContainerID, in.CNIDir) + removeCacheConf := false if err != nil { // Fetch delegates again if cache is not exist and pod info can be read if os.IsNotExist(err) && pod != nil { @@ -854,7 +855,7 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er return nil } } else { - defer os.Remove(path) + removeCacheConf = true in.Delegates = []*types.DelegateNetConf{} if err := json.Unmarshal(netconfBytes, &in.Delegates); err != nil { return cmdErr(k8sArgs, "failed to load netconf: %v", err) @@ -896,5 +897,26 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er } } - return delPlugins(exec, pod, args, k8sArgs, in.Delegates, len(in.Delegates)-1, in.RuntimeConfig, in) + e := delPlugins(exec, pod, args, k8sArgs, in.Delegates, len(in.Delegates)-1, in.RuntimeConfig, in) + + // Enable Option only delegate plugin delete success to delete cache file + // CNI Runtime maybe return an error to block sandbox cleanup a while initiative, + // like starting, prepare something, it will be OK when retry later + // put "delete cache file" off later ensure have enough info delegate DEL message when Pod has been fully + // deleted from ETCD before sandbox cleanup success.. + if in.RetryDeleteOnError { + if removeCacheConf { + // Kubelet though this error as has been cleanup success and never retry, clean cache also + // Block sandbox cleanup error message can not contain "no such file or directory", CNI Runtime maybe should adaptor it ! + if e == nil || strings.Contains(e.Error(), "no such file or directory") { + _ = os.Remove(path) // lgtm[go/path-injection] + } + } + } else { + if removeCacheConf { + _ = os.Remove(path) // lgtm[go/path-injection] + } + } + + return e } diff --git a/pkg/types/types.go b/pkg/types/types.go index 489498645..99f8e8a0f 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -61,6 +61,9 @@ type NetConf struct { SystemNamespaces []string `json:"systemNamespaces"` // Option to set the namespace that multus-cni uses (clusterNetwork/defaultNetworks) MultusNamespace string `json:"multusNamespace"` + + // Retry delegate DEL message to next when some error + RetryDeleteOnError bool `json:"retryDeleteOnError"` } // RuntimeConfig specifies CNI RuntimeConfig