Add clusterNetwork/defaultNetwork into multus

To support CRD/file/directory, add clusterNetwork/defaultNetwork
in multus.conf file.

README.md

@@ -135,11 +135,54 @@ $ kubectl exec -it samplepod -- ip a
 
 ## Network configuration reference
 
-- name (string, required): the name of the network
+Following is the example of multus config file, in `/etc/cni/net.d/`.
-- type (string, required): "multus"
+```
-- kubeconfig (string, optional): kubeconfig file for the out of cluster communication with kube-apiserver. See the example [kubeconfig](https://github.com/intel/multus-cni/blob/master/doc/node-kubeconfig.yaml)
+{
-- delegates (([]map,required): number of delegate details in the Multus
+    "name": "node-cni-network",
-- capabilities ({}list, optional): [capabilities](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md#dynamic-plugin-specific-fields-capabilities--runtime-configuration) supported by at least one of the delegates. (NOTE: Multus only supports portMappings capability for now). See the [example](https://github.com/intel/multus-cni/blob/master/examples/multus-ptp-portmap.conf).
+    "type": "multus",
+    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+    "confDir": "/etc/cni/multus/net.d",
+    "cniDir": "/var/lib/cni/multus",
+    "binDir": "/opt/cni/bin",
+    "logFile": "/var/log/multus.log",
+    "logLevel": "debug",
+    /* NOTE: you can set clusterNetwork+defaultNetworks OR delegates!! (this is only for manual) */
+    "clusterNetwork": "defaultCRD",
+    "defaultNetwork": ["sidecarCRD", "flannel"],
+    "delegates": [{
+        "type": "weave-net",
+        "hairpinMode": true
+    }, {
+        "type": "macvlan",
+        ... (snip)
+    }]
+}
+```
+
+- `name` (string, required): the name of the network
+- `type` (string, required): "multus"
+- `confDir` (string, optional): directory for CNI config file that multus reads. default `/etc/cni/multus/net.d`
+- `cniDir` (string, optional): Multus CNI data directory, default `/var/lib/cni/multus`
+- `binDir` (string, optional): directory for CNI plugins which multus calls. default `/opt/cni/bin`
+- `kubeconfig` (string, optional): kubeconfig file for the out of cluster communication with kube-apiserver. See the example [kubeconfig](https://github.com/intel/multus-cni/blob/master/doc/node-kubeconfig.yaml). If you would like to use CRD (i.e. network attachment definition), this is required
+- `logFile` (string, optional): file path for log file. multus puts log in given file
+- `logLevel` (string, optional): logging level ("debug", "error" or "panic")
+- `capabilities` ({}list, optional): [capabilities](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md#dynamic-plugin-specific-fields-capabilities--runtime-configuration) supported by at least one of the delegates. (NOTE: Multus only supports portMappings capability for now). See the [example](https://github.com/intel/multus-cni/blob/master/examples/multus-ptp-portmap.conf).
+
+User should chose following parameters combination (`clusterNetwork`+`defaultNetworks` or `delegates`):
+
+- `clusterNetwork` (string, required): default CNI network for pods, used in kubernetes cluster (Pod IP and so on): name of network-attachment-definition, CNI json file name (without extention, .conf/.conflist) or directory for CNI config file
+- `defaultNetworks` ([]string, required): default CNI network attachment: name of network-attachment-definition, CNI json file name (without extention, .conf/.conflist) or directory for CNI config file
+- `delegates` ([]map,required): number of delegate details in the Multus
+
+### Network selection flow of clusterNetwork/defaultNetworks
+
+Multus will find network for clusterNetwork/defaultNetworks as following sequences:
+
+1. CRD object for given network name
+1. CNI json config file in `confDir`. Given name should be without extention, like .conf/.conflist. (e.g. "test" for "test.conf")
+1. Directory for CNI json config file. Multus will find alphabetically first file for the network.
+1. Multus raise error message
 
 ## Usage with Kubernetes CRD based network objects
 
@@ -616,7 +659,6 @@ pod "multus-test" created
   - [Node Feature Discovery](https://github.com/kubernetes-incubator/node-feature-discovery)
   - [CPU Manager for Kubernetes](https://github.com/Intel-Corp/CPU-Manager-for-Kubernetes)
 
-
 ## Need help
 
 - Read [Containers Experience Kits](https://networkbuilders.intel.com/network-technologies/container-experience-kits)

k8sclient/k8sclient.go

@@ -396,7 +396,7 @@ type KubeClient interface {
 func GetK8sArgs(args *skel.CmdArgs) (*types.K8sArgs, error) {
 	k8sArgs := &types.K8sArgs{}
 
-	logging.Debugf("GetK8sNetwork: %v", args)
+	logging.Debugf("GetK8sArgs: %v", args)
 	err := cnitypes.LoadArgs(args.Args, k8sArgs)
 	if err != nil {
 		return nil, err
@@ -407,11 +407,11 @@ func GetK8sArgs(args *skel.CmdArgs) (*types.K8sArgs, error) {
 
 // Attempts to load Kubernetes-defined delegates and add them to the Multus config.
 // Returns the number of Kubernetes-defined delegates added or an error.
-func TryLoadK8sDelegates(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient KubeClient) (int, *clientInfo, error) {
+func TryLoadPodDelegates(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient KubeClient) (int, *clientInfo, error) {
 	var err error
 	clientInfo := &clientInfo{}
 
-	logging.Debugf("TryLoadK8sDelegates: %v, %v, %v", k8sArgs, conf, kubeClient)
+	logging.Debugf("TryLoadPodDelegates: %v, %v, %v", k8sArgs, conf, kubeClient)
 	kubeClient, err = GetK8sClient(conf.Kubeconfig, kubeClient)
 	if err != nil {
 		return 0, nil, err
@@ -426,7 +426,7 @@ func TryLoadK8sDelegates(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient
 	}
 
 	setKubeClientInfo(clientInfo, kubeClient, k8sArgs)
-	delegates, err := GetK8sNetwork(kubeClient, k8sArgs, conf.ConfDir)
+	delegates, err := GetPodNetwork(kubeClient, k8sArgs, conf.ConfDir)
 	if err != nil {
 		if _, ok := err.(*NoK8sNetworkError); ok {
 			return 0, clientInfo, nil
@@ -479,8 +479,8 @@ func GetK8sClient(kubeconfig string, kubeClient KubeClient) (KubeClient, error)
 	return &defaultKubeClient{client: client}, nil
 }
 
-func GetK8sNetwork(k8sclient KubeClient, k8sArgs *types.K8sArgs, confdir string) ([]*types.DelegateNetConf, error) {
+func GetPodNetwork(k8sclient KubeClient, k8sArgs *types.K8sArgs, confdir string) ([]*types.DelegateNetConf, error) {
-	logging.Debugf("GetK8sNetwork: %v, %v, %v", k8sclient, k8sArgs, confdir)
+	logging.Debugf("GetPodNetwork: %v, %v, %v", k8sclient, k8sArgs, confdir)
 
 	netAnnot, defaultNamespace, podID, err := getPodNetworkAnnotation(k8sclient, k8sArgs)
 	if err != nil {
@@ -509,7 +509,7 @@ func GetK8sNetwork(k8sclient KubeClient, k8sArgs *types.K8sArgs, confdir string)
 	for _, net := range networks {
 		delegate, updatedResourceMap, err := getKubernetesDelegate(k8sclient, net, confdir, podID, resourceMap)
 		if err != nil {
-			return nil, logging.Errorf("GetK8sNetwork: failed getting the delegate: %v", err)
+			return nil, logging.Errorf("GetPodNetwork: failed getting the delegate: %v", err)
 		}
 		delegates = append(delegates, delegate)
 		resourceMap = updatedResourceMap
@@ -517,3 +517,115 @@ func GetK8sNetwork(k8sclient KubeClient, k8sArgs *types.K8sArgs, confdir string)
 
 	return delegates, nil
 }
+
+func getDefaultNetDelegateCRD(client KubeClient, net string, confdir string) (*types.DelegateNetConf, error) {
+	logging.Debugf("getDefaultNetDelegate: %v, %v, %s", client, net, confdir)
+	rawPath := fmt.Sprintf("/apis/k8s.cni.cncf.io/v1/namespaces/%s/network-attachment-definitions/%s", "default", net)
+	netData, err := client.GetRawWithPath(rawPath)
+	if err != nil {
+		logging.Debugf("getDefaultNetDelegate: failed to get network resource, refer Multus README.md for the usage guide: %v", err)
+		return nil, nil
+	}
+
+	customResource := &types.NetworkAttachmentDefinition{}
+	if err := json.Unmarshal(netData, customResource); err != nil {
+		return nil, logging.Errorf("getDefaultNetDelegate: failed to get the netplugin data: %v", err)
+	}
+
+	configBytes, err := cniConfigFromNetworkResource(customResource, confdir)
+	if err != nil {
+		return nil, err
+	}
+
+	delegate, err := types.LoadDelegateNetConf(configBytes, "")
+	if err != nil {
+		return nil, err
+	}
+
+	return delegate, nil
+}
+
+func getNetDelegate(client KubeClient, netname string, confdir string) (*types.DelegateNetConf, error) {
+	logging.Debugf("getNetDelegate: %v, %v, %v", client, netname, confdir)
+	// option1) search CRD object for the network
+	delegate, err := getDefaultNetDelegateCRD(client, netname, confdir)
+	if err == nil {
+		return delegate, nil
+	}
+
+	// option2) search CNI json config file
+	var configBytes []byte
+	configBytes, err = getCNIConfigFromFile(netname, confdir)
+	if err == nil {
+		delegate, err := types.LoadDelegateNetConf(configBytes, "")
+		if err != nil {
+			return nil, err
+		}
+		return delegate, nil
+	}
+
+	// option3) search directry
+	fInfo, err := os.Stat(netname)
+	if err == nil {
+		if fInfo.IsDir() {
+			files, err := libcni.ConfFiles(netname, []string{".conf", ".conflist"})
+			if len(files) > 1 {
+				var configBytes []byte
+				configBytes, err = getCNIConfigFromFile(files[0], netname)
+				if err == nil {
+					delegate, err := types.LoadDelegateNetConf(configBytes, "")
+					if err != nil {
+						return nil, err
+					}
+					return delegate, nil
+				}
+			}
+		}
+	}
+	return nil, logging.Errorf("getNetDelegate: cannot find network: %v", netname)
+}
+
+// GetDefaultNetwork parses 'defaultNetwork' config, gets network json and put it into netconf.Delegates.
+func GetDefaultNetworks(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient KubeClient) error {
+	logging.Debugf("GetDefaultNetworks: %v, %v, %v", k8sArgs, conf, kubeClient)
+	var delegates []*types.DelegateNetConf
+
+	kubeClient, err := GetK8sClient(conf.Kubeconfig, kubeClient)
+	if err != nil {
+		return err
+	}
+	if kubeClient == nil {
+		if len(conf.Delegates) == 0 {
+			// No available kube client and no delegates, we can't do anything
+			return logging.Errorf("must have either Kubernetes config or delegates, refer Multus README.md for the usage guide")
+		}
+		return nil
+	}
+
+	//setKubeClientInfo(clientInfo, kubeClient, k8sArgs) XXX
+
+	delegate, err := getNetDelegate(kubeClient, conf.ClusterNetwork, conf.ConfDir)
+	if err != nil {
+		return err
+	}
+	delegate.MasterPlugin = true
+	delegates = append(delegates, delegate)
+
+	// First delegate is always the master plugin
+	conf.Delegates[0].MasterPlugin = true
+
+	//need to revisit
+	for _, netname := range conf.DefaultNetworks {
+		delegate, err := getNetDelegate(kubeClient, netname, conf.ConfDir)
+		if err != nil {
+			return err
+		}
+		delegates = append(delegates, delegate)
+	}
+
+	if err = conf.AddDelegates(delegates); err != nil {
+		return err
+	}
+
+	return nil
+}

k8sclient/k8sclient_test.go

@@ -81,7 +81,7 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(err).NotTo(HaveOccurred())
 		k8sArgs, err := GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
-		delegates, err := GetK8sNetwork(kubeClient, k8sArgs, tmpDir)
+		delegates, err := GetPodNetwork(kubeClient, k8sArgs, tmpDir)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fKubeClient.PodCount).To(Equal(1))
 		Expect(fKubeClient.NetCount).To(Equal(2))
@@ -114,9 +114,9 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(err).NotTo(HaveOccurred())
 		k8sArgs, err := GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
-		delegates, err := GetK8sNetwork(kubeClient, k8sArgs, tmpDir)
+		delegates, err := GetPodNetwork(kubeClient, k8sArgs, tmpDir)
 		Expect(len(delegates)).To(Equal(0))
-		Expect(err).To(MatchError("GetK8sNetwork: failed getting the delegate: getKubernetesDelegate: failed to get network resource, refer Multus README.md for the usage guide: resource not found"))
+		Expect(err).To(MatchError("GetPodNetwork: failed getting the delegate: getKubernetesDelegate: failed to get network resource, refer Multus README.md for the usage guide: resource not found"))
 	})
 
 	It("retrieves delegates from kubernetes using JSON format annotation", func() {
@@ -158,7 +158,7 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(err).NotTo(HaveOccurred())
 		k8sArgs, err := GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
-		delegates, err := GetK8sNetwork(kubeClient, k8sArgs, tmpDir)
+		delegates, err := GetPodNetwork(kubeClient, k8sArgs, tmpDir)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fKubeClient.PodCount).To(Equal(1))
 		Expect(fKubeClient.NetCount).To(Equal(3))
@@ -185,7 +185,7 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(err).NotTo(HaveOccurred())
 		k8sArgs, err := GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
-		delegates, err := GetK8sNetwork(kubeClient, k8sArgs, tmpDir)
+		delegates, err := GetPodNetwork(kubeClient, k8sArgs, tmpDir)
 		Expect(len(delegates)).To(Equal(0))
 		Expect(err).To(MatchError("parsePodNetworkAnnotation: failed to parse pod Network Attachment Selection Annotation JSON format: invalid character 'a' looking for beginning of value"))
 	})
@@ -215,7 +215,7 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(err).NotTo(HaveOccurred())
 		k8sArgs, err := GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
-		delegates, err := GetK8sNetwork(kubeClient, k8sArgs, tmpDir)
+		delegates, err := GetPodNetwork(kubeClient, k8sArgs, tmpDir)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fKubeClient.PodCount).To(Equal(1))
 		Expect(fKubeClient.NetCount).To(Equal(2))
@@ -241,7 +241,7 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(err).NotTo(HaveOccurred())
 		k8sArgs, err := GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
-		delegates, err := GetK8sNetwork(kubeClient, k8sArgs, tmpDir)
+		delegates, err := GetPodNetwork(kubeClient, k8sArgs, tmpDir)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fKubeClient.PodCount).To(Equal(1))
 		Expect(fKubeClient.NetCount).To(Equal(1))
@@ -272,8 +272,8 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(err).NotTo(HaveOccurred())
 		k8sArgs, err := GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
-		delegates, err := GetK8sNetwork(kubeClient, k8sArgs, tmpDir)
+		delegates, err := GetPodNetwork(kubeClient, k8sArgs, tmpDir)
 		Expect(len(delegates)).To(Equal(0))
-		Expect(err).To(MatchError(fmt.Sprintf("GetK8sNetwork: failed getting the delegate: cniConfigFromNetworkResource: err in getCNIConfigFromFile: Error loading CNI config file %s: error parsing configuration: invalid character 'a' looking for beginning of value", net2Name)))
+		Expect(err).To(MatchError(fmt.Sprintf("GetPodNetwork: failed getting the delegate: cniConfigFromNetworkResource: err in getCNIConfigFromFile: Error loading CNI config file %s: error parsing configuration: invalid character 'a' looking for beginning of value", net2Name)))
 	})
 })

multus/multus.go

@@ -248,7 +248,16 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn
 		}
 	})
 
-	numK8sDelegates, kc, err := k8s.TryLoadK8sDelegates(k8sArgs, n, kubeClient)
+	if n.ClusterNetwork != "" {
+		err = k8s.GetDefaultNetworks(k8sArgs, n, kubeClient)
+		if err != nil {
+			return nil, logging.Errorf("XXX")
+		}
+		// First delegate is always the master plugin
+		n.Delegates[0].MasterPlugin = true
+	}
+
+	numK8sDelegates, kc, err := k8s.TryLoadPodDelegates(k8sArgs, n, kubeClient)
 	if err != nil {
 		return nil, logging.Errorf("Multus: Err in loading K8s Delegates k8s args: %v", err)
 	}
@@ -350,7 +359,7 @@ func cmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) err
 		return logging.Errorf("Multus: Err in getting k8s args: %v", err)
 	}
 
-	numK8sDelegates, kc, err := k8s.TryLoadK8sDelegates(k8sArgs, in, kubeClient)
+	numK8sDelegates, kc, err := k8s.TryLoadPodDelegates(k8sArgs, in, kubeClient)
 	if err != nil {
 		return err
 	}

types/conf.go

@@ -110,7 +110,7 @@ func LoadCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, rc
 }
 
 func LoadNetworkStatus(r types.Result, netName string, defaultNet bool) (*NetworkStatus, error) {
-	logging.Debugf("LoadNetworkStatus: %v, %s, %s", r, netName, defaultNet)
+	logging.Debugf("LoadNetworkStatus: %v, %s, %t", r, netName, defaultNet)
 
 	// Convert whatever the IPAM result was into the current Result type
 	result, err := current.NewResultFromResult(r)
@@ -185,8 +185,8 @@ func LoadNetConf(bytes []byte) (*NetConf, error) {
 	// the master plugin. Kubernetes CRD delegates are then appended to
 	// the existing delegate list and all delegates executed in-order.
 
-	if len(netconf.RawDelegates) == 0 {
+	if len(netconf.RawDelegates) == 0 && netconf.ClusterNetwork == "" {
-		return nil, logging.Errorf("at least one delegate must be specified")
+		return nil, logging.Errorf("at least one delegate/defaultNetwork must be specified")
 	}
 
 	if netconf.CNIDir == "" {
@@ -205,12 +205,18 @@ func LoadNetConf(bytes []byte) (*NetConf, error) {
 		netconf.ReadinessIndicatorFile = defaultReadinessIndicatorFile
 	}
 
+	// get RawDelegates and put delegates field
+	if len(netconf.DefaultNetworks) == 0 {
+		// for Delegates
+		if len(netconf.RawDelegates) == 0 {
+			return nil, logging.Errorf("at least one delegate must be specified")
+		}
 		for idx, rawConf := range netconf.RawDelegates {
 			bytes, err := json.Marshal(rawConf)
 			if err != nil {
 				return nil, logging.Errorf("error marshalling delegate %d config: %v", idx, err)
 			}
-		delegateConf, err := LoadDelegateNetConf(bytes, "", "")
+			delegateConf, err := LoadDelegateNetConf(bytes, "")
 			if err != nil {
 				return nil, logging.Errorf("failed to load delegate %d config: %v", idx, err)
 			}
@@ -220,6 +226,7 @@ func LoadNetConf(bytes []byte) (*NetConf, error) {
 
 		// First delegate is always the master plugin
 		netconf.Delegates[0].MasterPlugin = true
+	}
 
 	return netconf, nil
 }

types/types.go

@@ -40,6 +40,8 @@ type NetConf struct {
 	Delegates       []*DelegateNetConf       `json:"-"`
 	NetStatus       []*NetworkStatus         `json:"-"`
 	Kubeconfig      string                   `json:"kubeconfig"`
+	ClusterNetwork  string                   `json:"clusterNetwork"`
+	DefaultNetworks []string                 `json:"defaultNetworks"`
 	LogFile         string                   `json:"logFile"`
 	LogLevel        string                   `json:"logLevel"`
 	RuntimeConfig   *RuntimeConfig           `json:"runtimeConfig,omitempty"`