delegate plugin delete success, delete cache file (#926)

docs/configuration.md

@@ -37,7 +37,8 @@ Following is the example of multus config file, in `/etc/cni/net.d/`.
     }, {
         "type": "macvlan",
         ... (snip)
-    }]
+    }],
+    allowTryDeleteOnErr: false
 }
 ```
 
@@ -62,6 +63,7 @@ User should chose following parameters combination (`clusterNetwork`+`defaultNet
 * `systemNamespaces` ([]string, optional): list of namespaces for Kubernetes system (namespaces listed here will not have `defaultNetworks` added)
 * `multusNamespace` (string, optional): namespace for `clusterNetwork`/`defaultNetworks`
 * `delegates` ([]map,required): number of delegate details in the Multus
+* `retryDeleteOnError` (bool, optional): Enable or disable delegate DEL message to next when some missing error. Defaults to false.
 
 ### Network selection flow of clusterNetwork/defaultNetworks
 

pkg/multus/multus.go

@@ -825,6 +825,7 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 
 	// Read the cache to get delegates json for the pod
 	netconfBytes, path, err := consumeScratchNetConf(args.ContainerID, in.CNIDir)
+	removeCacheConf := false
 	if err != nil {
 		// Fetch delegates again if cache is not exist and pod info can be read
 		if os.IsNotExist(err) && pod != nil {
@@ -854,7 +855,7 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 			return nil
 		}
 	} else {
-		defer os.Remove(path)
+		removeCacheConf = true
 		in.Delegates = []*types.DelegateNetConf{}
 		if err := json.Unmarshal(netconfBytes, &in.Delegates); err != nil {
 			return cmdErr(k8sArgs, "failed to load netconf: %v", err)
@@ -896,5 +897,26 @@ func CmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) er
 		}
 	}
 
-	return delPlugins(exec, pod, args, k8sArgs, in.Delegates, len(in.Delegates)-1, in.RuntimeConfig, in)
+	e := delPlugins(exec, pod, args, k8sArgs, in.Delegates, len(in.Delegates)-1, in.RuntimeConfig, in)
+
+	// Enable Option only delegate plugin delete success to delete cache file
+	// CNI Runtime maybe return an error to block sandbox cleanup a while initiative,
+	// like starting, prepare something, it will be OK when retry later
+	// put "delete cache file" off later ensure have enough info delegate DEL message when Pod has been fully
+	// deleted from ETCD before sandbox cleanup success..
+	if in.RetryDeleteOnError {
+		if removeCacheConf {
+			// Kubelet though this error as has been cleanup success and never retry, clean cache also
+			// Block sandbox cleanup error message can not contain "no such file or directory", CNI Runtime maybe should adaptor it !
+			if e == nil || strings.Contains(e.Error(), "no such file or directory") {
+				_ = os.Remove(path) // lgtm[go/path-injection]
+			}
+		}
+	} else {
+		if removeCacheConf {
+			_ = os.Remove(path) // lgtm[go/path-injection]
+		}
+	}
+
+	return e
 }

pkg/types/types.go

@@ -61,6 +61,9 @@ type NetConf struct {
 	SystemNamespaces []string `json:"systemNamespaces"`
 	// Option to set the namespace that multus-cni uses (clusterNetwork/defaultNetworks)
 	MultusNamespace string `json:"multusNamespace"`
+
+	// Retry delegate DEL message to next when some error
+	RetryDeleteOnError bool `json:"retryDeleteOnError"`
 }
 
 // RuntimeConfig specifies CNI RuntimeConfig