Update test.sh with coveralls job inclusion

This change fixes glide.yaml: track on cni's subpackage change and
some error about bytes package. With this change, verified that
'glide up' works without any error.

.travis.yml

@@ -10,12 +10,12 @@ env:
 
 before_install:
   - sudo apt-get update -qq
+  - go get github.com/mattn/goveralls
 
 install:
   # workaround golint install error in https://github.com/golang/lint/issues/288
   - mkdir -p $GOPATH/src/golang.org/x
   - pushd $GOPATH/src/golang.org/x
-  - git clone https://github.com/golang/tools.git
   - git clone https://github.com/golang/lint.git
   - go get github.com/golang/lint/golint
   - popd
@@ -29,6 +29,7 @@ before_script:
 script:
   - ./build
   - sudo ./test.sh
+  - $GOPATH/bin/goveralls  -coverprofile=coverage.out -service=travis-ci
   - mkdir -p ${TRAVIS_BUILD_DIR}/dist
   - tar cvfz ${TRAVIS_BUILD_DIR}/dist/multus-cni_amd64.tar.gz --warning=no-file-changed --exclude="dist" --exclude="vendor" .
   - docker build -t nfvpe/multus -f ./images/Dockerfile .

README.md

@@ -1,5 +1,7 @@
 ![multus-cni Logo](https://github.com/intel/multus-cni/blob/master/doc/images/Multus.png)
 
+[![Travis CI](https://travis-ci.org/intel/multus-cni.svg?branch=master)](https://travis-ci.org/intel/multus-cni/builds)[![Coverage Status](https://coveralls.io/repos/github/intel/multus-cni/badge.svg?branch=master)](https://coveralls.io/github/intel/multus-cni?branch=master)[![Go Report Card](https://goreportcard.com/badge/github.com/intel/multus-cni)](https://goreportcard.com/report/github.com/intel/multus-cni)
+
    * [MULTUS CNI plugin](#multus-cni-plugin)
       * [Quickstart Guide](#quickstart-guide)
       * [Multi-Homed pod](#multi-homed-pod)

examples/npwg-demo-1/04_macvlan1.yml

@@ -1,6 +1,6 @@
 ---
-apiVersion: "kubernetes.cni.cncf.io/v1"
-kind: Network
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
 metadata:
   name: macvlan-conf-1
 spec: 
@@ -17,8 +17,8 @@ spec:
             }
         }'
 ---
-apiVersion: "kubernetes.cni.cncf.io/v1"
-kind: Network
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
 metadata:
   name: macvlan-conf-2
 spec: 
@@ -35,8 +35,8 @@ spec:
             }
         }'
 ---
-apiVersion: "kubernetes.cni.cncf.io/v1"
-kind: Network
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
 metadata:
   name: macvlan-conf-3
 spec: 
@@ -53,8 +53,8 @@ spec:
             }
         }'
 ---
-apiVersion: "kubernetes.cni.cncf.io/v1"
-kind: Network
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
 metadata:
   name: macvlan-conf-4
 spec: 

examples/npwg-demo-1/05_vlan1.yml

@@ -1,6 +1,6 @@
 ---
-apiVersion: "kubernetes.cni.cncf.io/v1"
-kind: Network
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
 metadata:
   name: vlan-conf-1-1
   namespace: testns1

glide.yaml

@@ -1,16 +1,18 @@
 package: github.com/intel/multus-cni
+ignore:
+- bytes
 import:
 - package: github.com/containernetworking/cni
   version: 07c1a6da47b7fbf8b357f4949ecce2113e598491
   subpackages:
-  - pkg/ip
-  - pkg/ipam
   - pkg/skel
   - pkg/types
   - pkg/version
 - package: github.com/containernetworking/plugins
   version: 2b8b1ac0af4568e928d96ccc5f47b075416eeabd
   subpackages:
+  - pkg/ip
+  - pkg/ipam
   - pkg/ns
 - package: github.com/onsi/ginkgo
   version: 7f8ab55aaf3b86885aa55b762e803744d1674700

images/70-multus.conf

@@ -6,7 +6,8 @@
       "type": "flannel",
       "name": "flannel.1",
       "delegate": {
-        "isDefaultGateway": true
+        "isDefaultGateway": true,
+        "hairpinMode": true
       }
     }
   ],

k8sclient/k8sclient.go

@@ -31,6 +31,7 @@ import (
 	"github.com/containernetworking/cni/libcni"
 	"github.com/containernetworking/cni/pkg/skel"
 	cnitypes "github.com/containernetworking/cni/pkg/types"
+	"github.com/intel/multus-cni/logging"
 	"github.com/intel/multus-cni/types"
 )
 
@@ -67,6 +68,7 @@ func (d *defaultKubeClient) UpdatePodStatus(pod *v1.Pod) (*v1.Pod, error) {
 }
 
 func setKubeClientInfo(c *clientInfo, client KubeClient, k8sArgs *types.K8sArgs) {
+	logging.Debugf("setKubeClientInfo: %v, %v, %v", c, client, k8sArgs)
 	c.Client = client
 	c.Podnamespace = string(k8sArgs.K8S_POD_NAMESPACE)
 	c.Podname = string(k8sArgs.K8S_POD_NAME)
@@ -74,9 +76,10 @@ func setKubeClientInfo(c *clientInfo, client KubeClient, k8sArgs *types.K8sArgs)
 
 func SetNetworkStatus(k *clientInfo, netStatus []*types.NetworkStatus) error {
 
+	logging.Debugf("SetNetworkStatus: %v, %v", k, netStatus)
 	pod, err := k.Client.GetPod(k.Podnamespace, k.Podname)
 	if err != nil {
-		return fmt.Errorf("SetNetworkStatus: failed to query the pod %v in out of cluster comm: %v", k.Podname, err)
+		return logging.Errorf("SetNetworkStatus: failed to query the pod %v in out of cluster comm: %v", k.Podname, err)
 	}
 
 	var ns string
@@ -85,7 +88,7 @@ func SetNetworkStatus(k *clientInfo, netStatus []*types.NetworkStatus) error {
 		for _, nets := range netStatus {
 			data, err := json.MarshalIndent(nets, "", "    ")
 			if err != nil {
-				return fmt.Errorf("SetNetworkStatus: error with Marshal Indent: %v", err)
+				return logging.Errorf("SetNetworkStatus: error with Marshal Indent: %v", err)
 			}
 			networkStatus = append(networkStatus, string(data))
 		}
@@ -94,13 +97,14 @@ func SetNetworkStatus(k *clientInfo, netStatus []*types.NetworkStatus) error {
 	}
 	_, err = setPodNetworkAnnotation(k.Client, k.Podnamespace, pod, ns)
 	if err != nil {
-		return fmt.Errorf("SetNetworkStatus: failed to update the pod %v in out of cluster comm: %v", k.Podname, err)
+		return logging.Errorf("SetNetworkStatus: failed to update the pod %v in out of cluster comm: %v", k.Podname, err)
 	}
 
 	return nil
 }
 
 func setPodNetworkAnnotation(client KubeClient, namespace string, pod *v1.Pod, networkstatus string) (*v1.Pod, error) {
+	logging.Debugf("setPodNetworkAnnotation: %v, %s, %v, %s", client, namespace, pod, networkstatus)
 	//if pod annotations is empty, make sure it allocatable
 	if len(pod.Annotations) == 0 {
 		pod.Annotations = make(map[string]string)
@@ -122,7 +126,7 @@ func setPodNetworkAnnotation(client KubeClient, namespace string, pod *v1.Pod, n
 		pod, err = client.UpdatePodStatus(pod)
 		return err
 	}); resultErr != nil {
-		return nil, fmt.Errorf("status update failed for pod %s/%s: %v", pod.Namespace, pod.Name, resultErr)
+		return nil, logging.Errorf("status update failed for pod %s/%s: %v", pod.Namespace, pod.Name, resultErr)
 	}
 	return pod, nil
 }
@@ -130,9 +134,10 @@ func setPodNetworkAnnotation(client KubeClient, namespace string, pod *v1.Pod, n
 func getPodNetworkAnnotation(client KubeClient, k8sArgs *types.K8sArgs) (string, string, error) {
 	var err error
 
+	logging.Debugf("getPodNetworkAnnotation: %v, %v", client, k8sArgs)
 	pod, err := client.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
 	if err != nil {
-		return "", "", fmt.Errorf("getPodNetworkAnnotation: failed to query the pod %v in out of cluster comm: %v", string(k8sArgs.K8S_POD_NAME), err)
+		return "", "", logging.Errorf("getPodNetworkAnnotation: failed to query the pod %v in out of cluster comm: %v", string(k8sArgs.K8S_POD_NAME), err)
 	}
 
 	return pod.Annotations["k8s.v1.cni.cncf.io/networks"], pod.ObjectMeta.Namespace, nil
@@ -143,6 +148,7 @@ func parsePodNetworkObjectName(podnetwork string) (string, string, string, error
 	var netIfName string
 	var networkName string
 
+	logging.Debugf("parsePodNetworkObjectName: %s", podnetwork)
 	slashItems := strings.Split(podnetwork, "/")
 	if len(slashItems) == 2 {
 		netNsName = strings.TrimSpace(slashItems[0])
@@ -150,7 +156,7 @@ func parsePodNetworkObjectName(podnetwork string) (string, string, string, error
 	} else if len(slashItems) == 1 {
 		networkName = slashItems[0]
 	} else {
-		return "", "", "", fmt.Errorf("Invalid network object (failed at '/')")
+		return "", "", "", logging.Errorf("Invalid network object (failed at '/')")
 	}
 
 	atItems := strings.Split(networkName, "@")
@@ -158,7 +164,7 @@ func parsePodNetworkObjectName(podnetwork string) (string, string, string, error
 	if len(atItems) == 2 {
 		netIfName = strings.TrimSpace(atItems[1])
 	} else if len(atItems) != 1 {
-		return "", "", "", fmt.Errorf("Invalid network object (failed at '@')")
+		return "", "", "", logging.Errorf("Invalid network object (failed at '@')")
 	}
 
 	// Check and see if each item matches the specification for valid attachment name.
@@ -170,23 +176,25 @@ func parsePodNetworkObjectName(podnetwork string) (string, string, string, error
 	for i := range allItems {
 		matched, _ := regexp.MatchString("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", allItems[i])
 		if !matched && len([]rune(allItems[i])) > 0 {
-			return "", "", "", fmt.Errorf(fmt.Sprintf("Failed to parse: one or more items did not match comma-delimited format (must consist of lower case alphanumeric characters). Must start and end with an alphanumeric character), mismatch @ '%v'", allItems[i]))
+			return "", "", "", logging.Errorf(fmt.Sprintf("Failed to parse: one or more items did not match comma-delimited format (must consist of lower case alphanumeric characters). Must start and end with an alphanumeric character), mismatch @ '%v'", allItems[i]))
 		}
 	}
 
+	logging.Debugf("parsePodNetworkObjectName: parsed: %s, %s, %s", netNsName, networkName, netIfName)
 	return netNsName, networkName, netIfName, nil
 }
 
 func parsePodNetworkAnnotation(podNetworks, defaultNamespace string) ([]*types.NetworkSelectionElement, error) {
 	var networks []*types.NetworkSelectionElement
 
+	logging.Debugf("parsePodNetworkAnnotation: %s, %s", podNetworks, defaultNamespace)
 	if podNetworks == "" {
-		return nil, fmt.Errorf("parsePodNetworkAnnotation: pod annotation not having \"network\" as key, refer Multus README.md for the usage guide")
+		return nil, logging.Errorf("parsePodNetworkAnnotation: pod annotation not having \"network\" as key, refer Multus README.md for the usage guide")
 	}
 
 	if strings.IndexAny(podNetworks, "[{\"") >= 0 {
 		if err := json.Unmarshal([]byte(podNetworks), &networks); err != nil {
-			return nil, fmt.Errorf("parsePodNetworkAnnotation: failed to parse pod Network Attachment Selection Annotation JSON format: %v", err)
+			return nil, logging.Errorf("parsePodNetworkAnnotation: failed to parse pod Network Attachment Selection Annotation JSON format: %v", err)
 		}
 	} else {
 		// Comma-delimited list of network attachment object names
@@ -197,7 +205,7 @@ func parsePodNetworkAnnotation(podNetworks, defaultNamespace string) ([]*types.N
 			// Parse network name (i.e. <namespace>/<network name>@<ifname>)
 			netNsName, networkName, netIfName, err := parsePodNetworkObjectName(item)
 			if err != nil {
-				return nil, fmt.Errorf("parsePodNetworkAnnotation: %v", err)
+				return nil, logging.Errorf("parsePodNetworkAnnotation: %v", err)
 			}
 
 			networks = append(networks, &types.NetworkSelectionElement{
@@ -218,6 +226,7 @@ func parsePodNetworkAnnotation(podNetworks, defaultNamespace string) ([]*types.N
 }
 
 func getCNIConfigFromFile(name string, confdir string) ([]byte, error) {
+	logging.Debugf("getCNIConfigFromFile: %s, %s", name, confdir)
 
 	// In the absence of valid keys in a Spec, the runtime (or
 	// meta-plugin) should load and execute a CNI .configlist
@@ -228,9 +237,9 @@ func getCNIConfigFromFile(name string, confdir string) ([]byte, error) {
 	files, err := libcni.ConfFiles(confdir, []string{".conf", ".json", ".conflist"})
 	switch {
 	case err != nil:
-		return nil, fmt.Errorf("No networks found in %s", confdir)
+		return nil, logging.Errorf("No networks found in %s", confdir)
 	case len(files) == 0:
-		return nil, fmt.Errorf("No networks found in %s", confdir)
+		return nil, logging.Errorf("No networks found in %s", confdir)
 	}
 
 	for _, confFile := range files {
@@ -238,7 +247,7 @@ func getCNIConfigFromFile(name string, confdir string) ([]byte, error) {
 		if strings.HasSuffix(confFile, ".conflist") {
 			confList, err = libcni.ConfListFromFile(confFile)
 			if err != nil {
-				return nil, fmt.Errorf("Error loading CNI conflist file %s: %v", confFile, err)
+				return nil, logging.Errorf("Error loading CNI conflist file %s: %v", confFile, err)
 			}
 
 			if confList.Name == name {
@@ -248,21 +257,21 @@ func getCNIConfigFromFile(name string, confdir string) ([]byte, error) {
 		} else {
 			conf, err := libcni.ConfFromFile(confFile)
 			if err != nil {
-				return nil, fmt.Errorf("Error loading CNI config file %s: %v", confFile, err)
+				return nil, logging.Errorf("Error loading CNI config file %s: %v", confFile, err)
 			}
 
 			if conf.Network.Name == name {
 				// Ensure the config has a "type" so we know what plugin to run.
 				// Also catches the case where somebody put a conflist into a conf file.
 				if conf.Network.Type == "" {
-					return nil, fmt.Errorf("Error loading CNI config file %s: no 'type'; perhaps this is a .conflist?", confFile)
+					return nil, logging.Errorf("Error loading CNI config file %s: no 'type'; perhaps this is a .conflist?", confFile)
 				}
 				return conf.Bytes, nil
 			}
 		}
 	}
 
-	return nil, fmt.Errorf("no network available in the name %s in cni dir %s", name, confdir)
+	return nil, logging.Errorf("no network available in the name %s in cni dir %s", name, confdir)
 }
 
 // getCNIConfigFromSpec reads a CNI JSON configuration from the NetworkAttachmentDefinition
@@ -271,10 +280,11 @@ func getCNIConfigFromSpec(configData, netName string) ([]byte, error) {
 	var rawConfig map[string]interface{}
 	var err error
 
+	logging.Debugf("getCNIConfigFromSpec: %s, %s", configData, netName)
 	configBytes := []byte(configData)
 	err = json.Unmarshal(configBytes, &rawConfig)
 	if err != nil {
-		return nil, fmt.Errorf("getCNIConfigFromSpec: failed to unmarshal Spec.Config: %v", err)
+		return nil, logging.Errorf("getCNIConfigFromSpec: failed to unmarshal Spec.Config: %v", err)
 	}
 
 	// Inject network name if missing from Config for the thick plugin case
@@ -282,7 +292,7 @@ func getCNIConfigFromSpec(configData, netName string) ([]byte, error) {
 		rawConfig["name"] = netName
 		configBytes, err = json.Marshal(rawConfig)
 		if err != nil {
-			return nil, fmt.Errorf("getCNIConfigFromSpec: failed to re-marshal Spec.Config: %v", err)
+			return nil, logging.Errorf("getCNIConfigFromSpec: failed to re-marshal Spec.Config: %v", err)
 		}
 	}
 
@@ -293,6 +303,7 @@ func cniConfigFromNetworkResource(customResource *types.NetworkAttachmentDefinit
 	var config []byte
 	var err error
 
+	logging.Debugf("cniConfigFromNetworkResource: %v, %s", customResource, confdir)
 	emptySpec := types.NetworkAttachmentDefinitionSpec{}
 	if customResource.Spec == emptySpec {
 		// Network Spec empty; generate delegate from CNI JSON config
@@ -300,7 +311,7 @@ func cniConfigFromNetworkResource(customResource *types.NetworkAttachmentDefinit
 		// name as the custom resource
 		config, err = getCNIConfigFromFile(customResource.Metadata.Name, confdir)
 		if err != nil {
-			return nil, fmt.Errorf("cniConfigFromNetworkResource: err in getCNIConfigFromFile: %v", err)
+			return nil, logging.Errorf("cniConfigFromNetworkResource: err in getCNIConfigFromFile: %v", err)
 		}
 	} else {
 		// Config contains a standard JSON-encoded CNI configuration
@@ -308,7 +319,7 @@ func cniConfigFromNetworkResource(customResource *types.NetworkAttachmentDefinit
 		// execute.
 		config, err = getCNIConfigFromSpec(customResource.Spec.Config, customResource.Metadata.Name)
 		if err != nil {
-			return nil, fmt.Errorf("cniConfigFromNetworkResource: err in getCNIConfigFromSpec: %v", err)
+			return nil, logging.Errorf("cniConfigFromNetworkResource: err in getCNIConfigFromSpec: %v", err)
 		}
 	}
 
@@ -316,15 +327,16 @@ func cniConfigFromNetworkResource(customResource *types.NetworkAttachmentDefinit
 }
 
 func getKubernetesDelegate(client KubeClient, net *types.NetworkSelectionElement, confdir string) (*types.DelegateNetConf, error) {
+	logging.Debugf("getKubernetesDelegate: %v, %v, %s", client, net, confdir)
 	rawPath := fmt.Sprintf("/apis/k8s.cni.cncf.io/v1/namespaces/%s/network-attachment-definitions/%s", net.Namespace, net.Name)
 	netData, err := client.GetRawWithPath(rawPath)
 	if err != nil {
-		return nil, fmt.Errorf("getKubernetesDelegate: failed to get network resource, refer Multus README.md for the usage guide: %v", err)
+		return nil, logging.Errorf("getKubernetesDelegate: failed to get network resource, refer Multus README.md for the usage guide: %v", err)
 	}
 
 	customResource := &types.NetworkAttachmentDefinition{}
 	if err := json.Unmarshal(netData, customResource); err != nil {
-		return nil, fmt.Errorf("getKubernetesDelegate: failed to get the netplugin data: %v", err)
+		return nil, logging.Errorf("getKubernetesDelegate: failed to get the netplugin data: %v", err)
 	}
 
 	configBytes, err := cniConfigFromNetworkResource(customResource, confdir)
@@ -349,6 +361,7 @@ type KubeClient interface {
 func GetK8sArgs(args *skel.CmdArgs) (*types.K8sArgs, error) {
 	k8sArgs := &types.K8sArgs{}
 
+	logging.Debugf("GetK8sNetwork: %v", args)
 	err := cnitypes.LoadArgs(args.Args, k8sArgs)
 	if err != nil {
 		return nil, err
@@ -363,6 +376,7 @@ func TryLoadK8sDelegates(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient
 	var err error
 	clientInfo := &clientInfo{}
 
+	logging.Debugf("TryLoadK8sDelegates: %v, %v, %v", k8sArgs, conf, kubeClient)
 	kubeClient, err = GetK8sClient(conf.Kubeconfig, kubeClient)
 	if err != nil {
 		return 0, nil, err
@@ -371,7 +385,7 @@ func TryLoadK8sDelegates(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient
 	if kubeClient == nil {
 		if len(conf.Delegates) == 0 {
 			// No available kube client and no delegates, we can't do anything
-			return 0, nil, fmt.Errorf("must have either Kubernetes config or delegates, refer Multus README.md for the usage guide")
+			return 0, nil, logging.Errorf("must have either Kubernetes config or delegates, refer Multus README.md for the usage guide")
 		}
 		return 0, nil, nil
 	}
@@ -380,9 +394,9 @@ func TryLoadK8sDelegates(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient
 	delegates, err := GetK8sNetwork(kubeClient, k8sArgs, conf.ConfDir)
 	if err != nil {
 		if _, ok := err.(*NoK8sNetworkError); ok {
-			return 0, nil, nil
+			return 0, clientInfo, nil
 		}
-		return 0, nil, fmt.Errorf("Multus: Err in getting k8s network from pod: %v", err)
+		return 0, nil, logging.Errorf("Multus: Err in getting k8s network from pod: %v", err)
 	}
 
 	if err = conf.AddDelegates(delegates); err != nil {
@@ -393,6 +407,7 @@ func TryLoadK8sDelegates(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient
 }
 
 func GetK8sClient(kubeconfig string, kubeClient KubeClient) (KubeClient, error) {
+	logging.Debugf("GetK8sClient: %s, %v", kubeconfig, kubeClient)
 	// If we get a valid kubeClient (eg from testcases) just return that
 	// one.
 	if kubeClient != nil {
@@ -407,13 +422,13 @@ func GetK8sClient(kubeconfig string, kubeClient KubeClient) (KubeClient, error)
 		// uses the current context in kubeconfig
 		config, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
 		if err != nil {
-			return nil, fmt.Errorf("GetK8sClient: failed to get context for the kubeconfig %v, refer Multus README.md for the usage guide: %v", kubeconfig, err)
+			return nil, logging.Errorf("GetK8sClient: failed to get context for the kubeconfig %v, refer Multus README.md for the usage guide: %v", kubeconfig, err)
 		}
 	} else if os.Getenv("KUBERNETES_SERVICE_HOST") != "" && os.Getenv("KUBERNETES_SERVICE_PORT") != "" {
 		// Try in-cluster config where multus might be running in a kubernetes pod
 		config, err = rest.InClusterConfig()
 		if err != nil {
-			return nil, fmt.Errorf("createK8sClient: failed to get context for in-cluster kube config, refer Multus README.md for the usage guide: %v", err)
+			return nil, logging.Errorf("createK8sClient: failed to get context for in-cluster kube config, refer Multus README.md for the usage guide: %v", err)
 		}
 	} else {
 		// No kubernetes config; assume we shouldn't talk to Kube at all
@@ -430,6 +445,7 @@ func GetK8sClient(kubeconfig string, kubeClient KubeClient) (KubeClient, error)
 }
 
 func GetK8sNetwork(k8sclient KubeClient, k8sArgs *types.K8sArgs, confdir string) ([]*types.DelegateNetConf, error) {
+	logging.Debugf("GetK8sNetwork: %v, %v, %v", k8sclient, k8sArgs, confdir)
 
 	netAnnot, defaultNamespace, err := getPodNetworkAnnotation(k8sclient, k8sArgs)
 	if err != nil {
@@ -450,7 +466,7 @@ func GetK8sNetwork(k8sclient KubeClient, k8sArgs *types.K8sArgs, confdir string)
 	for _, net := range networks {
 		delegate, err := getKubernetesDelegate(k8sclient, net, confdir)
 		if err != nil {
-			return nil, fmt.Errorf("GetK8sNetwork: failed getting the delegate: %v", err)
+			return nil, logging.Errorf("GetK8sNetwork: failed getting the delegate: %v", err)
 		}
 		delegates = append(delegates, delegate)
 	}

logging/logging.go

@@ -37,6 +37,7 @@ const (
 var loggingStderr bool
 var loggingFp *os.File
 var loggingLevel Level
+
 const defaultTimestampFormat = time.RFC3339
 
 func (l Level) String() string {
@@ -75,8 +76,9 @@ func Debugf(format string, a ...interface{}) {
 	Printf(DebugLevel, format, a...)
 }
 
-func Errorf(format string, a ...interface{}) {
+func Errorf(format string, a ...interface{}) error {
 	Printf(ErrorLevel, format, a...)
+	return fmt.Errorf(format, a...)
 }
 
 func Panicf(format string, a ...interface{}) {
@@ -86,7 +88,7 @@ func Panicf(format string, a ...interface{}) {
 	Printf(PanicLevel, "========= Stack trace output end ========")
 }
 
-func GetLoggingLevel (levelStr string) Level {
+func GetLoggingLevel(levelStr string) Level {
 	switch strings.ToLower(levelStr) {
 	case "debug":
 		return DebugLevel
@@ -99,18 +101,18 @@ func GetLoggingLevel (levelStr string) Level {
 	return UnknownLevel
 }
 
-func SetLogLevel (levelStr string) {
+func SetLogLevel(levelStr string) {
 	level := GetLoggingLevel(levelStr)
 	if level < MaxLevel {
 		loggingLevel = level
 	}
 }
 
-func SetLogStderr (enable bool) {
+func SetLogStderr(enable bool) {
 	loggingStderr = enable
 }
 
-func SetLogFile (filename string) {
+func SetLogFile(filename string) {
 	if filename == "" {
 		return
 	}

multus/multus.go

@@ -32,26 +32,29 @@ import (
 	"github.com/containernetworking/cni/pkg/version"
 	"github.com/containernetworking/plugins/pkg/ns"
 	k8s "github.com/intel/multus-cni/k8sclient"
+	"github.com/intel/multus-cni/logging"
 	"github.com/intel/multus-cni/types"
 	"github.com/vishvananda/netlink"
 )
 
 func saveScratchNetConf(containerID, dataDir string, netconf []byte) error {
+	logging.Debugf("saveScratchNetConf: %s, %s, %s", containerID, dataDir, string(netconf))
 	if err := os.MkdirAll(dataDir, 0700); err != nil {
-		return fmt.Errorf("failed to create the multus data directory(%q): %v", dataDir, err)
+		return logging.Errorf("failed to create the multus data directory(%q): %v", dataDir, err)
 	}
 
 	path := filepath.Join(dataDir, containerID)
 
 	err := ioutil.WriteFile(path, netconf, 0600)
 	if err != nil {
-		return fmt.Errorf("failed to write container data in the path(%q): %v", path, err)
+		return logging.Errorf("failed to write container data in the path(%q): %v", path, err)
 	}
 
 	return err
 }
 
 func consumeScratchNetConf(containerID, dataDir string) ([]byte, error) {
+	logging.Debugf("consumeScratchNetConf: %s, %s", containerID, dataDir)
 	path := filepath.Join(dataDir, containerID)
 	defer os.Remove(path)
 
@@ -59,6 +62,7 @@ func consumeScratchNetConf(containerID, dataDir string) ([]byte, error) {
 }
 
 func getIfname(delegate *types.DelegateNetConf, argif string, idx int) string {
+	logging.Debugf("getIfname: %v, %s, %d", delegate, argif, idx)
 	if delegate.IfnameRequest != "" {
 		return delegate.IfnameRequest
 	}
@@ -73,22 +77,24 @@ func getIfname(delegate *types.DelegateNetConf, argif string, idx int) string {
 }
 
 func saveDelegates(containerID, dataDir string, delegates []*types.DelegateNetConf) error {
+	logging.Debugf("saveDelegates: %s, %s, %v", containerID, dataDir, delegates)
 	delegatesBytes, err := json.Marshal(delegates)
 	if err != nil {
-		return fmt.Errorf("error serializing delegate netconf: %v", err)
+		return logging.Errorf("error serializing delegate netconf: %v", err)
 	}
 
 	if err = saveScratchNetConf(containerID, dataDir, delegatesBytes); err != nil {
-		return fmt.Errorf("error in saving the  delegates : %v", err)
+		return logging.Errorf("error in saving the  delegates : %v", err)
 	}
 
 	return err
 }
 
 func validateIfName(nsname string, ifname string) error {
+	logging.Debugf("validateIfName: %s, %s", nsname, ifname)
 	podNs, err := ns.GetNS(nsname)
 	if err != nil {
-		return fmt.Errorf("no netns: %v", err)
+		return logging.Errorf("no netns: %v", err)
 	}
 
 	err = podNs.Do(func(_ ns.NetNS) error {
@@ -99,61 +105,64 @@ func validateIfName(nsname string, ifname string) error {
 			}
 			return err
 		}
-		return fmt.Errorf("ifname %s is already exist", ifname)
+		return logging.Errorf("ifname %s is already exist", ifname)
 	})
 
 	return err
 }
 
 func conflistAdd(rt *libcni.RuntimeConf, rawnetconflist []byte, binDir string) (cnitypes.Result, error) {
+	logging.Debugf("conflistAdd: %v, %s, %s", rt, string(rawnetconflist), binDir)
 	// In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go
 	binDirs := []string{binDir}
 	cniNet := libcni.CNIConfig{Path: binDirs}
 
 	confList, err := libcni.ConfListFromBytes(rawnetconflist)
 	if err != nil {
-		return nil, fmt.Errorf("error in converting the raw bytes to conflist: %v", err)
+		return nil, logging.Errorf("error in converting the raw bytes to conflist: %v", err)
 	}
 
 	result, err := cniNet.AddNetworkList(confList, rt)
 	if err != nil {
-		return nil, fmt.Errorf("error in getting result from AddNetworkList: %v", err)
+		return nil, logging.Errorf("error in getting result from AddNetworkList: %v", err)
 	}
 
 	return result, nil
 }
 
 func conflistDel(rt *libcni.RuntimeConf, rawnetconflist []byte, binDir string) error {
+	logging.Debugf("conflistDel: %v, %s, %s", rt, string(rawnetconflist), binDir)
 	// In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go
 	binDirs := []string{binDir}
 	cniNet := libcni.CNIConfig{Path: binDirs}
 
 	confList, err := libcni.ConfListFromBytes(rawnetconflist)
 	if err != nil {
-		return fmt.Errorf("error in converting the raw bytes to conflist: %v", err)
+		return logging.Errorf("error in converting the raw bytes to conflist: %v", err)
 	}
 
 	err = cniNet.DelNetworkList(confList, rt)
 	if err != nil {
-		return fmt.Errorf("error in getting result from DelNetworkList: %v", err)
+		return logging.Errorf("error in getting result from DelNetworkList: %v", err)
 	}
 
 	return err
 }
 
 func delegateAdd(exec invoke.Exec, ifName string, delegate *types.DelegateNetConf, rt *libcni.RuntimeConf, binDir string) (cnitypes.Result, error) {
+	logging.Debugf("delegateAdd: %v, %s, %v, %v, %s", exec, ifName, delegate, rt, binDir)
 	if os.Setenv("CNI_IFNAME", ifName) != nil {
-		return nil, fmt.Errorf("Multus: error in setting CNI_IFNAME")
+		return nil, logging.Errorf("Multus: error in setting CNI_IFNAME")
 	}
 
 	if err := validateIfName(os.Getenv("CNI_NETNS"), ifName); err != nil {
-		return nil, fmt.Errorf("cannot set %q ifname to %q: %v", delegate.Conf.Type, ifName, err)
+		return nil, logging.Errorf("cannot set %q ifname to %q: %v", delegate.Conf.Type, ifName, err)
 	}
 
 	if delegate.ConfListPlugin != false {
 		result, err := conflistAdd(rt, delegate.Bytes, binDir)
 		if err != nil {
-			return nil, fmt.Errorf("Multus: error in invoke Conflist add - %q: %v", delegate.ConfList.Name, err)
+			return nil, logging.Errorf("Multus: error in invoke Conflist add - %q: %v", delegate.ConfList.Name, err)
 		}
 
 		return result, nil
@@ -161,36 +170,38 @@ func delegateAdd(exec invoke.Exec, ifName string, delegate *types.DelegateNetCon
 
 	result, err := invoke.DelegateAdd(delegate.Conf.Type, delegate.Bytes, exec)
 	if err != nil {
-		return nil, fmt.Errorf("Multus: error in invoke Delegate add - %q: %v", delegate.Conf.Type, err)
+		return nil, logging.Errorf("Multus: error in invoke Delegate add - %q: %v", delegate.Conf.Type, err)
 	}
 
 	return result, nil
 }
 
 func delegateDel(exec invoke.Exec, ifName string, delegateConf *types.DelegateNetConf, rt *libcni.RuntimeConf, binDir string) error {
+	logging.Debugf("delegateDel: %v, %s, %v, %v, %s", exec, ifName, delegateConf, rt, binDir)
 	if os.Setenv("CNI_IFNAME", ifName) != nil {
-		return fmt.Errorf("Multus: error in setting CNI_IFNAME")
+		return logging.Errorf("Multus: error in setting CNI_IFNAME")
 	}
 
 	if delegateConf.ConfListPlugin != false {
 		err := conflistDel(rt, delegateConf.Bytes, binDir)
 		if err != nil {
-			return fmt.Errorf("Multus: error in invoke Conflist Del - %q: %v", delegateConf.ConfList.Name, err)
+			return logging.Errorf("Multus: error in invoke Conflist Del - %q: %v", delegateConf.ConfList.Name, err)
 		}
 
 		return err
 	}
 
 	if err := invoke.DelegateDel(delegateConf.Conf.Type, delegateConf.Bytes, exec); err != nil {
-		return fmt.Errorf("Multus: error in invoke Delegate del - %q: %v", delegateConf.Conf.Type, err)
+		return logging.Errorf("Multus: error in invoke Delegate del - %q: %v", delegateConf.Conf.Type, err)
 	}
 
 	return nil
 }
 
 func delPlugins(exec invoke.Exec, argIfname string, delegates []*types.DelegateNetConf, lastIdx int, rt *libcni.RuntimeConf, binDir string) error {
+	logging.Debugf("delPlugins: %v, %s, %v, %d, %v, %s", exec, argIfname, delegates, lastIdx, rt, binDir)
 	if os.Setenv("CNI_COMMAND", "DEL") != nil {
-		return fmt.Errorf("Multus: error in setting CNI_COMMAND to DEL")
+		return logging.Errorf("Multus: error in setting CNI_COMMAND to DEL")
 	}
 
 	for idx := lastIdx; idx >= 0; idx-- {
@@ -205,25 +216,26 @@ func delPlugins(exec invoke.Exec, argIfname string, delegates []*types.DelegateN
 }
 
 func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cnitypes.Result, error) {
+	logging.Debugf("cmdAdd: %v, %v, %v", args, exec, kubeClient)
 	n, err := types.LoadNetConf(args.StdinData)
 	if err != nil {
-		return nil, fmt.Errorf("err in loading netconf: %v", err)
+		return nil, logging.Errorf("err in loading netconf: %v", err)
 	}
 
 	k8sArgs, err := k8s.GetK8sArgs(args)
 	if err != nil {
-		return nil, fmt.Errorf("Multus: Err in getting k8s args: %v", err)
+		return nil, logging.Errorf("Multus: Err in getting k8s args: %v", err)
 	}
 
 	numK8sDelegates, kc, err := k8s.TryLoadK8sDelegates(k8sArgs, n, kubeClient)
 	if err != nil {
-		return nil, fmt.Errorf("Multus: Err in loading K8s Delegates k8s args: %v", err)
+		return nil, logging.Errorf("Multus: Err in loading K8s Delegates k8s args: %v", err)
 	}
 
 	if numK8sDelegates == 0 {
 		// cache the multus config if we have only Multus delegates
 		if err := saveDelegates(args.ContainerID, n.CNIDir, n.Delegates); err != nil {
-			return nil, fmt.Errorf("Multus: Err in saving the delegates: %v", err)
+			return nil, logging.Errorf("Multus: Err in saving the delegates: %v", err)
 		}
 	}
 
@@ -246,27 +258,31 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn
 		}
 
 		//create the network status, only in case Multus as kubeconfig
-		if n.Kubeconfig != "" && kc.Podnamespace != "kube-system" {
+		if n.Kubeconfig != "" && kc != nil {
+			if kc.Podnamespace != "kube-system" {
 				delegateNetStatus, err := types.LoadNetworkStatus(tmpResult, delegate.Conf.Name, delegate.MasterPlugin)
 				if err != nil {
-				return nil, fmt.Errorf("Multus: Err in setting  networks status: %v", err)
+					return nil, logging.Errorf("Multus: Err in setting  networks status: %v", err)
 				}
 
 				netStatus = append(netStatus, delegateNetStatus)
 			}
 		}
+	}
 
 	if err != nil {
 		// Ignore errors; DEL must be idempotent anyway
 		_ = delPlugins(exec, args.IfName, n.Delegates, lastIdx, rt, n.BinDir)
-		return nil, fmt.Errorf("Multus: Err in tearing down failed plugins: %v", err)
+		return nil, logging.Errorf("Multus: Err in tearing down failed plugins: %v", err)
 	}
 
 	//set the network status annotation in apiserver, only in case Multus as kubeconfig
-	if n.Kubeconfig != "" && kc.Podnamespace != "kube-system" {
+	if n.Kubeconfig != "" && kc != nil {
+		if kc.Podnamespace != "kube-system" {
 			err = k8s.SetNetworkStatus(kc, netStatus)
 			if err != nil {
-			return nil, fmt.Errorf("Multus: Err set the networks status: %v", err)
+				return nil, logging.Errorf("Multus: Err set the networks status: %v", err)
+			}
 		}
 	}
 
@@ -274,6 +290,7 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn
 }
 
 func cmdGet(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cnitypes.Result, error) {
+	logging.Debugf("cmdGet: %v, %v, %v", args, exec, kubeClient)
 	in, err := types.LoadNetConf(args.StdinData)
 	if err != nil {
 		return nil, err
@@ -285,14 +302,31 @@ func cmdGet(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn
 }
 
 func cmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) error {
+	logging.Debugf("cmdDel: %v, %v, %v", args, exec, kubeClient)
 	in, err := types.LoadNetConf(args.StdinData)
 	if err != nil {
 		return err
 	}
 
+	if args.Netns == "" {
+		return nil
+	}
+	netns, err := ns.GetNS(args.Netns)
+	if err != nil {
+		//  if NetNs is passed down by the Cloud Orchestration Engine, or if it called multiple times
+		// so don't return an error if the device is already removed.
+		// https://github.com/kubernetes/kubernetes/issues/43014#issuecomment-287164444
+		_, ok := err.(ns.NSPathNotExistErr)
+		if ok {
+			return nil
+		}
+		return fmt.Errorf("failed to open netns %q: %v", netns, err)
+	}
+	defer netns.Close()
+
 	k8sArgs, err := k8s.GetK8sArgs(args)
 	if err != nil {
-		return fmt.Errorf("Multus: Err in getting k8s args: %v", err)
+		return logging.Errorf("Multus: Err in getting k8s args: %v", err)
 	}
 
 	numK8sDelegates, kc, err := k8s.TryLoadK8sDelegates(k8sArgs, in, kubeClient)
@@ -308,19 +342,21 @@ func cmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) err
 				// Per spec should ignore error if resources are missing / already removed
 				return nil
 			}
-			return fmt.Errorf("Multus: Err in  reading the delegates: %v", err)
+			return logging.Errorf("Multus: Err in  reading the delegates: %v", err)
 		}
 
 		if err := json.Unmarshal(netconfBytes, &in.Delegates); err != nil {
-			return fmt.Errorf("Multus: failed to load netconf: %v", err)
+			return logging.Errorf("Multus: failed to load netconf: %v", err)
 		}
 	}
 
 	//unset the network status annotation in apiserver, only in case Multus as kubeconfig
-	if in.Kubeconfig != "" && kc.Podnamespace != "kube-system" {
+	if in.Kubeconfig != "" && kc != nil {
+		if kc.Podnamespace != "kube-system" {
 			err := k8s.SetNetworkStatus(kc, nil)
 			if err != nil {
-			return fmt.Errorf("Multus: Err unset the networks status: %v", err)
+				return logging.Errorf("Multus: Err unset the networks status: %v", err)
+			}
 		}
 	}
 

test.sh

@@ -13,5 +13,5 @@ export GO15VENDOREXPERIMENT=1
 export GOBIN=${PWD}/bin
 export GOPATH=${PWD}/gopath
 
-bash -c "umask 0; cd ${GOPATH}/src/${REPO_PATH}; PATH=${GOROOT}/bin:$(pwd)/bin:${PATH} go test ./..."
+bash -c "umask 0; cd ${GOPATH}/src/${REPO_PATH}; PATH=${GOROOT}/bin:$(pwd)/bin:${PATH} go test -v -covermode=count -coverprofile=coverage.out ./..."
 

types/conf.go

@@ -17,7 +17,6 @@ package types
 
 import (
 	"encoding/json"
-	"fmt"
 
 	"github.com/containernetworking/cni/libcni"
 	"github.com/containernetworking/cni/pkg/skel"
@@ -35,15 +34,16 @@ const (
 
 func LoadDelegateNetConfList(bytes []byte, delegateConf *DelegateNetConf) error {
 
+	logging.Debugf("LoadDelegateNetConfList: %s, %v", string(bytes), delegateConf)
 	if err := json.Unmarshal(bytes, &delegateConf.ConfList); err != nil {
-		return fmt.Errorf("err in unmarshalling delegate conflist: %v", err)
+		return logging.Errorf("err in unmarshalling delegate conflist: %v", err)
 	}
 
 	if delegateConf.ConfList.Plugins == nil {
-		return fmt.Errorf("delegate must have the 'type'or 'Plugin' field")
+		return logging.Errorf("delegate must have the 'type'or 'Plugin' field")
 	}
 	if delegateConf.ConfList.Plugins[0].Type == "" {
-		return fmt.Errorf("a plugin delegate must have the 'type' field")
+		return logging.Errorf("a plugin delegate must have the 'type' field")
 	}
 	delegateConf.ConfListPlugin = true
 	return nil
@@ -52,14 +52,15 @@ func LoadDelegateNetConfList(bytes []byte, delegateConf *DelegateNetConf) error
 // Convert raw CNI JSON into a DelegateNetConf structure
 func LoadDelegateNetConf(bytes []byte, ifnameRequest string) (*DelegateNetConf, error) {
 	delegateConf := &DelegateNetConf{}
+	logging.Debugf("LoadDelegateNetConf: %s, %s", string(bytes), ifnameRequest)
 	if err := json.Unmarshal(bytes, &delegateConf.Conf); err != nil {
-		return nil, fmt.Errorf("error in LoadDelegateNetConf - unmarshalling delegate config: %v", err)
+		return nil, logging.Errorf("error in LoadDelegateNetConf - unmarshalling delegate config: %v", err)
 	}
 
 	// Do some minimal validation
 	if delegateConf.Conf.Type == "" {
 		if err := LoadDelegateNetConfList(bytes, delegateConf); err != nil {
-			return nil, fmt.Errorf("error in LoadDelegateNetConf: %v", err)
+			return nil, logging.Errorf("error in LoadDelegateNetConf: %v", err)
 		}
 	}
 
@@ -74,6 +75,7 @@ func LoadDelegateNetConf(bytes []byte, ifnameRequest string) (*DelegateNetConf,
 
 func LoadCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string) (*libcni.RuntimeConf, error) {
 
+	logging.Debugf("LoadCNIRuntimeConf: %v, %v, %s", args, k8sArgs, ifName)
 	// In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go#buildCNIRuntimeConf
 	// Todo
 	// ingress, egress and bandwidth capability features as same as kubelet.
@@ -92,10 +94,12 @@ func LoadCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string) (*l
 }
 
 func LoadNetworkStatus(r types.Result, netName string, defaultNet bool) (*NetworkStatus, error) {
+	logging.Debugf("LoadNetworkStatus: %v, %s, %s", r, netName, defaultNet)
+
 	// Convert whatever the IPAM result was into the current Result type
 	result, err := current.NewResultFromResult(r)
 	if err != nil {
-		return nil, fmt.Errorf("error convert the type.Result to current.Result: %v", err)
+		return nil, logging.Errorf("error convert the type.Result to current.Result: %v", err)
 	}
 
 	netstatus := &NetworkStatus{}
@@ -128,8 +132,10 @@ func LoadNetworkStatus(r types.Result, netName string, defaultNet bool) (*Networ
 
 func LoadNetConf(bytes []byte) (*NetConf, error) {
 	netconf := &NetConf{}
+
+	logging.Debugf("LoadNetConf: %s", string(bytes))
 	if err := json.Unmarshal(bytes, netconf); err != nil {
-		return nil, fmt.Errorf("failed to load netconf: %v", err)
+		return nil, logging.Errorf("failed to load netconf: %v", err)
 	}
 
 	// Logging
@@ -144,16 +150,16 @@ func LoadNetConf(bytes []byte) (*NetConf, error) {
 	if netconf.RawPrevResult != nil {
 		resultBytes, err := json.Marshal(netconf.RawPrevResult)
 		if err != nil {
-			return nil, fmt.Errorf("could not serialize prevResult: %v", err)
+			return nil, logging.Errorf("could not serialize prevResult: %v", err)
 		}
 		res, err := version.NewResult(netconf.CNIVersion, resultBytes)
 		if err != nil {
-			return nil, fmt.Errorf("could not parse prevResult: %v", err)
+			return nil, logging.Errorf("could not parse prevResult: %v", err)
 		}
 		netconf.RawPrevResult = nil
 		netconf.PrevResult, err = current.NewResultFromResult(res)
 		if err != nil {
-			return nil, fmt.Errorf("could not convert result to current version: %v", err)
+			return nil, logging.Errorf("could not convert result to current version: %v", err)
 		}
 	}
 
@@ -164,7 +170,7 @@ func LoadNetConf(bytes []byte) (*NetConf, error) {
 	// the existing delegate list and all delegates executed in-order.
 
 	if len(netconf.RawDelegates) == 0 {
-		return nil, fmt.Errorf("at least one delegate must be specified")
+		return nil, logging.Errorf("at least one delegate must be specified")
 	}
 
 	if netconf.CNIDir == "" {
@@ -182,11 +188,11 @@ func LoadNetConf(bytes []byte) (*NetConf, error) {
 	for idx, rawConf := range netconf.RawDelegates {
 		bytes, err := json.Marshal(rawConf)
 		if err != nil {
-			return nil, fmt.Errorf("error marshalling delegate %d config: %v", idx, err)
+			return nil, logging.Errorf("error marshalling delegate %d config: %v", idx, err)
 		}
 		delegateConf, err := LoadDelegateNetConf(bytes, "")
 		if err != nil {
-			return nil, fmt.Errorf("failed to load delegate %d config: %v", idx, err)
+			return nil, logging.Errorf("failed to load delegate %d config: %v", idx, err)
 		}
 		netconf.Delegates = append(netconf.Delegates, delegateConf)
 	}
@@ -200,6 +206,7 @@ func LoadNetConf(bytes []byte) (*NetConf, error) {
 
 // AddDelegates appends the new delegates to the delegates list
 func (n *NetConf) AddDelegates(newDelegates []*DelegateNetConf) error {
+	logging.Debugf("AddDelegates: %v", newDelegates)
 	n.Delegates = append(n.Delegates, newDelegates...)
 	return nil
 }