Replace setenv with runtimeConfig set

setenv refers environment variables, which is unique in process,
not unique to go routine. Hence it may causes some issue in multi
threaded case, hence it is replaced with libcni's runtimeConfig
value set to set these variables at libcni side, after process
fork.

pkg/multus/multus.go

@@ -206,7 +206,7 @@ func confCheck(rt *libcni.RuntimeConf, rawNetconf []byte, multusNetconf *types.N
 }
 
 func confDel(rt *libcni.RuntimeConf, rawNetconf []byte, multusNetconf *types.NetConf, exec invoke.Exec) error {
-	logging.Debugf("conflistDel: %v, %s", rt, string(rawNetconf))
+	logging.Debugf("confDel: %v, %s", rt, string(rawNetconf))
 	// In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go
 	binDirs := filepath.SplitList(os.Getenv("CNI_PATH"))
 	binDirs = append([]string{multusNetconf.BinDir}, binDirs...)
@@ -285,23 +285,15 @@ func conflistDel(rt *libcni.RuntimeConf, rawnetconflist []byte, multusNetconf *t
 	return err
 }
 
-func delegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, ifName string, delegate *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf, cniArgs string) (cnitypes.Result, error) {
+func delegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, delegate *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf) (cnitypes.Result, error) {
-	logging.Debugf("delegateAdd: %v, %s, %v, %v", exec, ifName, delegate, rt)
+	logging.Debugf("delegateAdd: %v, %v, %v", exec, delegate, rt)
-	if os.Setenv("CNI_IFNAME", ifName) != nil {
-		return nil, logging.Errorf("delegateAdd: error setting environment variable CNI_IFNAME")
-	}
 
-	if err := validateIfName(os.Getenv("CNI_NETNS"), ifName); err != nil {
+	if err := validateIfName(rt.NetNS, rt.IfName); err != nil {
-		return nil, logging.Errorf("delegateAdd: cannot set %q interface name to %q: %v", delegate.Conf.Type, ifName, err)
+		return nil, logging.Errorf("delegateAdd: cannot set %q interface name to %q: %v", delegate.Conf.Type, rt.IfName, err)
 	}
 
 	// Deprecated in ver 3.5.
 	if delegate.MacRequest != "" || delegate.IPRequest != nil {
-		if cniArgs != "" {
-			cniArgs = fmt.Sprintf("%s;IgnoreUnknown=true", cniArgs)
-		} else {
-			cniArgs = "IgnoreUnknown=true"
-		}
 		if delegate.MacRequest != "" {
 			// validate Mac address
 			_, err := net.ParseMAC(delegate.MacRequest)
@@ -309,8 +301,7 @@ func delegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, ifNa
 				return nil, logging.Errorf("delegateAdd: failed to parse mac address %q", delegate.MacRequest)
 			}
 
-			cniArgs = fmt.Sprintf("%s;MAC=%s", cniArgs, delegate.MacRequest)
+			logging.Debugf("delegateAdd: set MAC address %q to %q", delegate.MacRequest, rt.IfName)
-			logging.Debugf("delegateAdd: set MAC address %q to %q", delegate.MacRequest, ifName)
 			rt.Args = append(rt.Args, [2]string{"MAC", delegate.MacRequest})
 		}
 
@@ -328,8 +319,7 @@ func delegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, ifNa
 			}
 
 			ips := strings.Join(delegate.IPRequest, ",")
-			cniArgs = fmt.Sprintf("%s;IP=%s", cniArgs, ips)
+			logging.Debugf("delegateAdd: set IP address %q to %q", ips, rt.IfName)
-			logging.Debugf("delegateAdd: set IP address %q to %q", ips, ifName)
 			rt.Args = append(rt.Args, [2]string{"IP", ips})
 		}
 	}
@@ -390,11 +380,8 @@ func delegateAdd(exec invoke.Exec, kubeClient *k8s.ClientInfo, pod *v1.Pod, ifNa
 	return result, nil
 }
 
-func delegateCheck(exec invoke.Exec, ifName string, delegateConf *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf) error {
+func delegateCheck(exec invoke.Exec, delegateConf *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf) error {
-	logging.Debugf("delegateCheck: %v, %s, %v, %v", exec, ifName, delegateConf, rt)
+	logging.Debugf("delegateCheck: %v, %v, %v", exec, delegateConf, rt)
-	if os.Setenv("CNI_IFNAME", ifName) != nil {
-		return logging.Errorf("delegateCheck: error setting environment variable CNI_IFNAME")
-	}
 
 	if logging.GetLoggingLevel() >= logging.VerboseLevel {
 		var cniConfName string
@@ -422,11 +409,8 @@ func delegateCheck(exec invoke.Exec, ifName string, delegateConf *types.Delegate
 	return err
 }
 
-func delegateDel(exec invoke.Exec, pod *v1.Pod, ifName string, delegateConf *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf) error {
+func delegateDel(exec invoke.Exec, pod *v1.Pod, delegateConf *types.DelegateNetConf, rt *libcni.RuntimeConf, multusNetconf *types.NetConf) error {
-	logging.Debugf("delegateDel: %v, %v, %s, %v, %v", exec, pod, ifName, delegateConf, rt)
+	logging.Debugf("delegateDel: %v, %v, %v, %v", exec, pod, delegateConf, rt)
-	if os.Setenv("CNI_IFNAME", ifName) != nil {
-		return logging.Errorf("delegateDel: error setting environment variable CNI_IFNAME")
-	}
 
 	if logging.GetLoggingLevel() >= logging.VerboseLevel {
 		var confName string
@@ -463,16 +447,13 @@ func delegateDel(exec invoke.Exec, pod *v1.Pod, ifName string, delegateConf *typ
 // with each of the delegates' configuration
 func delPlugins(exec invoke.Exec, pod *v1.Pod, args *skel.CmdArgs, k8sArgs *types.K8sArgs, delegates []*types.DelegateNetConf, lastIdx int, netRt *types.RuntimeConfig, multusNetconf *types.NetConf) error {
 	logging.Debugf("delPlugins: %v, %v, %v, %v, %v, %d, %v", exec, pod, args, k8sArgs, delegates, lastIdx, netRt)
-	if os.Setenv("CNI_COMMAND", "DEL") != nil {
-		return logging.Errorf("delPlugins: error setting environment variable CNI_COMMAND to a value of DEL")
-	}
 
 	var errorstrings []string
 	for idx := lastIdx; idx >= 0; idx-- {
 		ifName := getIfname(delegates[idx], args.IfName, idx)
 		rt, cniDeviceInfoPath := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, netRt, delegates[idx])
 		// Attempt to delete all but do not error out, instead, collect all errors.
-		if err := delegateDel(exec, pod, ifName, delegates[idx], rt, multusNetconf); err != nil {
+		if err := delegateDel(exec, pod, delegates[idx], rt, multusNetconf); err != nil {
 			errorstrings = append(errorstrings, err.Error())
 		}
 		if cniDeviceInfoPath != "" {
@@ -623,7 +604,6 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 
 	var result, tmpResult cnitypes.Result
 	var netStatus []nettypes.NetworkStatus
-	cniArgs := os.Getenv("CNI_ARGS")
 	for idx, delegate := range n.Delegates {
 		ifName := getIfname(delegate, args.IfName, idx)
 		rt, cniDeviceInfoPath := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, n.RuntimeConfig, delegate)
@@ -637,7 +617,7 @@ func CmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo) (c
 		}
 
 		netName := ""
-		tmpResult, err = delegateAdd(exec, kubeClient, pod, ifName, delegate, rt, n, cniArgs)
+		tmpResult, err = delegateAdd(exec, kubeClient, pod, delegate, rt, n)
 		if err != nil {
 			// If the add failed, tear down all networks we already added
 			netName = delegate.Conf.Name
@@ -776,7 +756,7 @@ func CmdCheck(args *skel.CmdArgs, exec invoke.Exec, kubeClient *k8s.ClientInfo)
 		ifName := getIfname(delegate, args.IfName, idx)
 
 		rt, _ := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, in.RuntimeConfig, delegate)
-		err = delegateCheck(exec, ifName, delegate, rt, in)
+		err = delegateCheck(exec, delegate, rt, in)
 		if err != nil {
 			return err
 		}

pkg/types/conf.go

@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
+	"os"
 	"strings"
 
 	"github.com/containernetworking/cni/libcni"
@@ -184,7 +185,7 @@ func mergeCNIRuntimeConfig(runtimeConfig *RuntimeConfig, delegate *DelegateNetCo
 // CreateCNIRuntimeConf create CNI RuntimeConf for a delegate. If delegate configuration
 // exists, merge data with the runtime config.
 func CreateCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, rc *RuntimeConfig, delegate *DelegateNetConf) (*libcni.RuntimeConf, string) {
-	logging.Debugf("LoadCNIRuntimeConf: %v, %v, %s, %v %v", args, k8sArgs, ifName, rc, delegate)
+	logging.Debugf("CreateCNIRuntimeConf: %v, %v, %s, %v %v", args, k8sArgs, ifName, rc, delegate)
 	var cniDeviceInfoFile string
 	var delegateRc *RuntimeConfig
 
@@ -208,7 +209,7 @@ func CreateCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, r
 		ContainerID: args.ContainerID,
 		NetNS:       args.Netns,
 		IfName:      ifName,
-		// NOTE: Verbose logging depends on this order, so please keep Args order.
+		// NOTE: Verbose logging (pod namespace/pod name)depends on this order, so please keep Args order.
 		Args: [][2]string{
 			{"IgnoreUnknown", string("true")},
 			{"K8S_POD_NAMESPACE", string(k8sArgs.K8S_POD_NAMESPACE)},
@@ -218,6 +219,32 @@ func CreateCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, r
 		},
 	}
 
+	// get CNI_ARGS and set it if it does not exist in rt.Args
+	cniArgs := os.Getenv("CNI_ARGS")
+	if cniArgs != "" {
+		for _, arg := range strings.Split(cniArgs, ";") {
+			for _, keyval := range strings.Split(arg, "=") {
+				if len(keyval) != 2 {
+					logging.Errorf("CreateCNIRuntimeConf: CNI_ARGS %s %s %d is not recognized as CNI arg, skipped", arg, keyval, len(keyval))
+					continue
+				}
+
+				envKey := string(keyval[0])
+				envVal := string(keyval[1])
+				isExists := false
+				for _, rtArg := range rt.Args {
+					if rtArg[0] == envKey {
+						isExists = true
+					}
+				}
+				if isExists != false {
+					logging.Debugf("CreateCNIRuntimeConf: add new val: %s", arg)
+					rt.Args = append(rt.Args, [2]string{envKey, envVal})
+				}
+			}
+		}
+	}
+
 	if delegateRc != nil {
 		capabilityArgs := map[string]interface{}{}
 		if len(delegateRc.PortMaps) != 0 {

pkg/types/conf_test.go

@@ -47,7 +47,6 @@ var _ = Describe("config operations", func() {
 		var err error
 		testNS, err = testutils.NewNS()
 		Expect(err).NotTo(HaveOccurred())
-		os.Setenv("CNI_NETNS", testNS.Path())
 		os.Setenv("CNI_PATH", "/some/path")
 
 		tmpDir, err = ioutil.TempDir("", "multus_tmp")
@@ -57,7 +56,6 @@ var _ = Describe("config operations", func() {
 	AfterEach(func() {
 		Expect(testNS.Close()).To(Succeed())
 		os.Unsetenv("CNI_PATH")
-		os.Unsetenv("CNI_ARGS")
 		err := os.RemoveAll(tmpDir)
 		Expect(err).NotTo(HaveOccurred())
 	})