diff --git a/README.md b/README.md index 80e26395d..0c0e47087 100644 --- a/README.md +++ b/README.md @@ -135,11 +135,54 @@ $ kubectl exec -it samplepod -- ip a ## Network configuration reference -- name (string, required): the name of the network -- type (string, required): "multus" -- kubeconfig (string, optional): kubeconfig file for the out of cluster communication with kube-apiserver. See the example [kubeconfig](https://github.com/intel/multus-cni/blob/master/doc/node-kubeconfig.yaml) -- delegates (([]map,required): number of delegate details in the Multus -- capabilities ({}list, optional): [capabilities](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md#dynamic-plugin-specific-fields-capabilities--runtime-configuration) supported by at least one of the delegates. (NOTE: Multus only supports portMappings capability for now). See the [example](https://github.com/intel/multus-cni/blob/master/examples/multus-ptp-portmap.conf). +Following is the example of multus config file, in `/etc/cni/net.d/`. +``` +{ + "name": "node-cni-network", + "type": "multus", + "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml", + "confDir": "/etc/cni/multus/net.d", + "cniDir": "/var/lib/cni/multus", + "binDir": "/opt/cni/bin", + "logFile": "/var/log/multus.log", + "logLevel": "debug", + /* NOTE: you can set clusterNetwork+defaultNetworks OR delegates!! (this is only for manual) */ + "clusterNetwork": "defaultCRD", + "defaultNetwork": ["sidecarCRD", "flannel"], + "delegates": [{ + "type": "weave-net", + "hairpinMode": true + }, { + "type": "macvlan", + ... (snip) + }] +} +``` + +- `name` (string, required): the name of the network +- `type` (string, required): "multus" +- `confDir` (string, optional): directory for CNI config file that multus reads. default `/etc/cni/multus/net.d` +- `cniDir` (string, optional): Multus CNI data directory, default `/var/lib/cni/multus` +- `binDir` (string, optional): directory for CNI plugins which multus calls. default `/opt/cni/bin` +- `kubeconfig` (string, optional): kubeconfig file for the out of cluster communication with kube-apiserver. See the example [kubeconfig](https://github.com/intel/multus-cni/blob/master/doc/node-kubeconfig.yaml). If you would like to use CRD (i.e. network attachment definition), this is required +- `logFile` (string, optional): file path for log file. multus puts log in given file +- `logLevel` (string, optional): logging level ("debug", "error" or "panic") +- `capabilities` ({}list, optional): [capabilities](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md#dynamic-plugin-specific-fields-capabilities--runtime-configuration) supported by at least one of the delegates. (NOTE: Multus only supports portMappings capability for now). See the [example](https://github.com/intel/multus-cni/blob/master/examples/multus-ptp-portmap.conf). + +User should chose following parameters combination (`clusterNetwork`+`defaultNetworks` or `delegates`): + +- `clusterNetwork` (string, required): default CNI network for pods, used in kubernetes cluster (Pod IP and so on): name of network-attachment-definition, CNI json file name (without extention, .conf/.conflist) or directory for CNI config file +- `defaultNetworks` ([]string, required): default CNI network attachment: name of network-attachment-definition, CNI json file name (without extention, .conf/.conflist) or directory for CNI config file +- `delegates` ([]map,required): number of delegate details in the Multus + +### Network selection flow of clusterNetwork/defaultNetworks + +Multus will find network for clusterNetwork/defaultNetworks as following sequences: + +1. CRD object for given network name +1. CNI json config file in `confDir`. Given name should be without extention, like .conf/.conflist. (e.g. "test" for "test.conf") +1. Directory for CNI json config file. Multus will find alphabetically first file for the network. +1. Multus raise error message ## Usage with Kubernetes CRD based network objects @@ -616,7 +659,6 @@ pod "multus-test" created - [Node Feature Discovery](https://github.com/kubernetes-incubator/node-feature-discovery) - [CPU Manager for Kubernetes](https://github.com/Intel-Corp/CPU-Manager-for-Kubernetes) - ## Need help - Read [Containers Experience Kits](https://networkbuilders.intel.com/network-technologies/container-experience-kits) diff --git a/k8sclient/k8sclient.go b/k8sclient/k8sclient.go index fe5b548af..c5c93c146 100644 --- a/k8sclient/k8sclient.go +++ b/k8sclient/k8sclient.go @@ -396,7 +396,7 @@ type KubeClient interface { func GetK8sArgs(args *skel.CmdArgs) (*types.K8sArgs, error) { k8sArgs := &types.K8sArgs{} - logging.Debugf("GetK8sNetwork: %v", args) + logging.Debugf("GetK8sArgs: %v", args) err := cnitypes.LoadArgs(args.Args, k8sArgs) if err != nil { return nil, err @@ -407,11 +407,11 @@ func GetK8sArgs(args *skel.CmdArgs) (*types.K8sArgs, error) { // Attempts to load Kubernetes-defined delegates and add them to the Multus config. // Returns the number of Kubernetes-defined delegates added or an error. -func TryLoadK8sDelegates(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient KubeClient) (int, *clientInfo, error) { +func TryLoadPodDelegates(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient KubeClient) (int, *clientInfo, error) { var err error clientInfo := &clientInfo{} - logging.Debugf("TryLoadK8sDelegates: %v, %v, %v", k8sArgs, conf, kubeClient) + logging.Debugf("TryLoadPodDelegates: %v, %v, %v", k8sArgs, conf, kubeClient) kubeClient, err = GetK8sClient(conf.Kubeconfig, kubeClient) if err != nil { return 0, nil, err @@ -426,7 +426,7 @@ func TryLoadK8sDelegates(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient } setKubeClientInfo(clientInfo, kubeClient, k8sArgs) - delegates, err := GetK8sNetwork(kubeClient, k8sArgs, conf.ConfDir) + delegates, err := GetPodNetwork(kubeClient, k8sArgs, conf.ConfDir) if err != nil { if _, ok := err.(*NoK8sNetworkError); ok { return 0, clientInfo, nil @@ -479,8 +479,8 @@ func GetK8sClient(kubeconfig string, kubeClient KubeClient) (KubeClient, error) return &defaultKubeClient{client: client}, nil } -func GetK8sNetwork(k8sclient KubeClient, k8sArgs *types.K8sArgs, confdir string) ([]*types.DelegateNetConf, error) { - logging.Debugf("GetK8sNetwork: %v, %v, %v", k8sclient, k8sArgs, confdir) +func GetPodNetwork(k8sclient KubeClient, k8sArgs *types.K8sArgs, confdir string) ([]*types.DelegateNetConf, error) { + logging.Debugf("GetPodNetwork: %v, %v, %v", k8sclient, k8sArgs, confdir) netAnnot, defaultNamespace, podID, err := getPodNetworkAnnotation(k8sclient, k8sArgs) if err != nil { @@ -509,7 +509,7 @@ func GetK8sNetwork(k8sclient KubeClient, k8sArgs *types.K8sArgs, confdir string) for _, net := range networks { delegate, updatedResourceMap, err := getKubernetesDelegate(k8sclient, net, confdir, podID, resourceMap) if err != nil { - return nil, logging.Errorf("GetK8sNetwork: failed getting the delegate: %v", err) + return nil, logging.Errorf("GetPodNetwork: failed getting the delegate: %v", err) } delegates = append(delegates, delegate) resourceMap = updatedResourceMap @@ -517,3 +517,115 @@ func GetK8sNetwork(k8sclient KubeClient, k8sArgs *types.K8sArgs, confdir string) return delegates, nil } + +func getDefaultNetDelegateCRD(client KubeClient, net string, confdir string) (*types.DelegateNetConf, error) { + logging.Debugf("getDefaultNetDelegate: %v, %v, %s", client, net, confdir) + rawPath := fmt.Sprintf("/apis/k8s.cni.cncf.io/v1/namespaces/%s/network-attachment-definitions/%s", "default", net) + netData, err := client.GetRawWithPath(rawPath) + if err != nil { + logging.Debugf("getDefaultNetDelegate: failed to get network resource, refer Multus README.md for the usage guide: %v", err) + return nil, nil + } + + customResource := &types.NetworkAttachmentDefinition{} + if err := json.Unmarshal(netData, customResource); err != nil { + return nil, logging.Errorf("getDefaultNetDelegate: failed to get the netplugin data: %v", err) + } + + configBytes, err := cniConfigFromNetworkResource(customResource, confdir) + if err != nil { + return nil, err + } + + delegate, err := types.LoadDelegateNetConf(configBytes, "") + if err != nil { + return nil, err + } + + return delegate, nil +} + +func getNetDelegate(client KubeClient, netname string, confdir string) (*types.DelegateNetConf, error) { + logging.Debugf("getNetDelegate: %v, %v, %v", client, netname, confdir) + // option1) search CRD object for the network + delegate, err := getDefaultNetDelegateCRD(client, netname, confdir) + if err == nil { + return delegate, nil + } + + // option2) search CNI json config file + var configBytes []byte + configBytes, err = getCNIConfigFromFile(netname, confdir) + if err == nil { + delegate, err := types.LoadDelegateNetConf(configBytes, "") + if err != nil { + return nil, err + } + return delegate, nil + } + + // option3) search directry + fInfo, err := os.Stat(netname) + if err == nil { + if fInfo.IsDir() { + files, err := libcni.ConfFiles(netname, []string{".conf", ".conflist"}) + if len(files) > 1 { + var configBytes []byte + configBytes, err = getCNIConfigFromFile(files[0], netname) + if err == nil { + delegate, err := types.LoadDelegateNetConf(configBytes, "") + if err != nil { + return nil, err + } + return delegate, nil + } + } + } + } + return nil, logging.Errorf("getNetDelegate: cannot find network: %v", netname) +} + +// GetDefaultNetwork parses 'defaultNetwork' config, gets network json and put it into netconf.Delegates. +func GetDefaultNetworks(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient KubeClient) error { + logging.Debugf("GetDefaultNetworks: %v, %v, %v", k8sArgs, conf, kubeClient) + var delegates []*types.DelegateNetConf + + kubeClient, err := GetK8sClient(conf.Kubeconfig, kubeClient) + if err != nil { + return err + } + if kubeClient == nil { + if len(conf.Delegates) == 0 { + // No available kube client and no delegates, we can't do anything + return logging.Errorf("must have either Kubernetes config or delegates, refer Multus README.md for the usage guide") + } + return nil + } + + //setKubeClientInfo(clientInfo, kubeClient, k8sArgs) XXX + + delegate, err := getNetDelegate(kubeClient, conf.ClusterNetwork, conf.ConfDir) + if err != nil { + return err + } + delegate.MasterPlugin = true + delegates = append(delegates, delegate) + + // First delegate is always the master plugin + conf.Delegates[0].MasterPlugin = true + + //need to revisit + for _, netname := range conf.DefaultNetworks { + delegate, err := getNetDelegate(kubeClient, netname, conf.ConfDir) + if err != nil { + return err + } + delegates = append(delegates, delegate) + } + + if err = conf.AddDelegates(delegates); err != nil { + return err + } + + return nil +} diff --git a/k8sclient/k8sclient_test.go b/k8sclient/k8sclient_test.go index bd60e91eb..71c1035cf 100644 --- a/k8sclient/k8sclient_test.go +++ b/k8sclient/k8sclient_test.go @@ -81,7 +81,7 @@ var _ = Describe("k8sclient operations", func() { Expect(err).NotTo(HaveOccurred()) k8sArgs, err := GetK8sArgs(args) Expect(err).NotTo(HaveOccurred()) - delegates, err := GetK8sNetwork(kubeClient, k8sArgs, tmpDir) + delegates, err := GetPodNetwork(kubeClient, k8sArgs, tmpDir) Expect(err).NotTo(HaveOccurred()) Expect(fKubeClient.PodCount).To(Equal(1)) Expect(fKubeClient.NetCount).To(Equal(2)) @@ -114,9 +114,9 @@ var _ = Describe("k8sclient operations", func() { Expect(err).NotTo(HaveOccurred()) k8sArgs, err := GetK8sArgs(args) Expect(err).NotTo(HaveOccurred()) - delegates, err := GetK8sNetwork(kubeClient, k8sArgs, tmpDir) + delegates, err := GetPodNetwork(kubeClient, k8sArgs, tmpDir) Expect(len(delegates)).To(Equal(0)) - Expect(err).To(MatchError("GetK8sNetwork: failed getting the delegate: getKubernetesDelegate: failed to get network resource, refer Multus README.md for the usage guide: resource not found")) + Expect(err).To(MatchError("GetPodNetwork: failed getting the delegate: getKubernetesDelegate: failed to get network resource, refer Multus README.md for the usage guide: resource not found")) }) It("retrieves delegates from kubernetes using JSON format annotation", func() { @@ -158,7 +158,7 @@ var _ = Describe("k8sclient operations", func() { Expect(err).NotTo(HaveOccurred()) k8sArgs, err := GetK8sArgs(args) Expect(err).NotTo(HaveOccurred()) - delegates, err := GetK8sNetwork(kubeClient, k8sArgs, tmpDir) + delegates, err := GetPodNetwork(kubeClient, k8sArgs, tmpDir) Expect(err).NotTo(HaveOccurred()) Expect(fKubeClient.PodCount).To(Equal(1)) Expect(fKubeClient.NetCount).To(Equal(3)) @@ -185,7 +185,7 @@ var _ = Describe("k8sclient operations", func() { Expect(err).NotTo(HaveOccurred()) k8sArgs, err := GetK8sArgs(args) Expect(err).NotTo(HaveOccurred()) - delegates, err := GetK8sNetwork(kubeClient, k8sArgs, tmpDir) + delegates, err := GetPodNetwork(kubeClient, k8sArgs, tmpDir) Expect(len(delegates)).To(Equal(0)) Expect(err).To(MatchError("parsePodNetworkAnnotation: failed to parse pod Network Attachment Selection Annotation JSON format: invalid character 'a' looking for beginning of value")) }) @@ -215,7 +215,7 @@ var _ = Describe("k8sclient operations", func() { Expect(err).NotTo(HaveOccurred()) k8sArgs, err := GetK8sArgs(args) Expect(err).NotTo(HaveOccurred()) - delegates, err := GetK8sNetwork(kubeClient, k8sArgs, tmpDir) + delegates, err := GetPodNetwork(kubeClient, k8sArgs, tmpDir) Expect(err).NotTo(HaveOccurred()) Expect(fKubeClient.PodCount).To(Equal(1)) Expect(fKubeClient.NetCount).To(Equal(2)) @@ -241,7 +241,7 @@ var _ = Describe("k8sclient operations", func() { Expect(err).NotTo(HaveOccurred()) k8sArgs, err := GetK8sArgs(args) Expect(err).NotTo(HaveOccurred()) - delegates, err := GetK8sNetwork(kubeClient, k8sArgs, tmpDir) + delegates, err := GetPodNetwork(kubeClient, k8sArgs, tmpDir) Expect(err).NotTo(HaveOccurred()) Expect(fKubeClient.PodCount).To(Equal(1)) Expect(fKubeClient.NetCount).To(Equal(1)) @@ -272,8 +272,8 @@ var _ = Describe("k8sclient operations", func() { Expect(err).NotTo(HaveOccurred()) k8sArgs, err := GetK8sArgs(args) Expect(err).NotTo(HaveOccurred()) - delegates, err := GetK8sNetwork(kubeClient, k8sArgs, tmpDir) + delegates, err := GetPodNetwork(kubeClient, k8sArgs, tmpDir) Expect(len(delegates)).To(Equal(0)) - Expect(err).To(MatchError(fmt.Sprintf("GetK8sNetwork: failed getting the delegate: cniConfigFromNetworkResource: err in getCNIConfigFromFile: Error loading CNI config file %s: error parsing configuration: invalid character 'a' looking for beginning of value", net2Name))) + Expect(err).To(MatchError(fmt.Sprintf("GetPodNetwork: failed getting the delegate: cniConfigFromNetworkResource: err in getCNIConfigFromFile: Error loading CNI config file %s: error parsing configuration: invalid character 'a' looking for beginning of value", net2Name))) }) }) diff --git a/multus/multus.go b/multus/multus.go index e3e4b84e1..6fd75f146 100644 --- a/multus/multus.go +++ b/multus/multus.go @@ -248,7 +248,16 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn } }) - numK8sDelegates, kc, err := k8s.TryLoadK8sDelegates(k8sArgs, n, kubeClient) + if n.ClusterNetwork != "" { + err = k8s.GetDefaultNetworks(k8sArgs, n, kubeClient) + if err != nil { + return nil, logging.Errorf("XXX") + } + // First delegate is always the master plugin + n.Delegates[0].MasterPlugin = true + } + + numK8sDelegates, kc, err := k8s.TryLoadPodDelegates(k8sArgs, n, kubeClient) if err != nil { return nil, logging.Errorf("Multus: Err in loading K8s Delegates k8s args: %v", err) } @@ -350,7 +359,7 @@ func cmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) err return logging.Errorf("Multus: Err in getting k8s args: %v", err) } - numK8sDelegates, kc, err := k8s.TryLoadK8sDelegates(k8sArgs, in, kubeClient) + numK8sDelegates, kc, err := k8s.TryLoadPodDelegates(k8sArgs, in, kubeClient) if err != nil { return err } diff --git a/types/conf.go b/types/conf.go index 3081ad1d4..7968a9dbf 100644 --- a/types/conf.go +++ b/types/conf.go @@ -110,7 +110,7 @@ func LoadCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, rc } func LoadNetworkStatus(r types.Result, netName string, defaultNet bool) (*NetworkStatus, error) { - logging.Debugf("LoadNetworkStatus: %v, %s, %s", r, netName, defaultNet) + logging.Debugf("LoadNetworkStatus: %v, %s, %t", r, netName, defaultNet) // Convert whatever the IPAM result was into the current Result type result, err := current.NewResultFromResult(r) @@ -185,8 +185,8 @@ func LoadNetConf(bytes []byte) (*NetConf, error) { // the master plugin. Kubernetes CRD delegates are then appended to // the existing delegate list and all delegates executed in-order. - if len(netconf.RawDelegates) == 0 { - return nil, logging.Errorf("at least one delegate must be specified") + if len(netconf.RawDelegates) == 0 && netconf.ClusterNetwork == "" { + return nil, logging.Errorf("at least one delegate/defaultNetwork must be specified") } if netconf.CNIDir == "" { @@ -205,21 +205,28 @@ func LoadNetConf(bytes []byte) (*NetConf, error) { netconf.ReadinessIndicatorFile = defaultReadinessIndicatorFile } - for idx, rawConf := range netconf.RawDelegates { - bytes, err := json.Marshal(rawConf) - if err != nil { - return nil, logging.Errorf("error marshalling delegate %d config: %v", idx, err) + // get RawDelegates and put delegates field + if len(netconf.DefaultNetworks) == 0 { + // for Delegates + if len(netconf.RawDelegates) == 0 { + return nil, logging.Errorf("at least one delegate must be specified") } - delegateConf, err := LoadDelegateNetConf(bytes, "", "") - if err != nil { - return nil, logging.Errorf("failed to load delegate %d config: %v", idx, err) + for idx, rawConf := range netconf.RawDelegates { + bytes, err := json.Marshal(rawConf) + if err != nil { + return nil, logging.Errorf("error marshalling delegate %d config: %v", idx, err) + } + delegateConf, err := LoadDelegateNetConf(bytes, "") + if err != nil { + return nil, logging.Errorf("failed to load delegate %d config: %v", idx, err) + } + netconf.Delegates = append(netconf.Delegates, delegateConf) } - netconf.Delegates = append(netconf.Delegates, delegateConf) - } - netconf.RawDelegates = nil + netconf.RawDelegates = nil - // First delegate is always the master plugin - netconf.Delegates[0].MasterPlugin = true + // First delegate is always the master plugin + netconf.Delegates[0].MasterPlugin = true + } return netconf, nil } diff --git a/types/types.go b/types/types.go index fa5e48ace..60e0ff63e 100644 --- a/types/types.go +++ b/types/types.go @@ -36,13 +36,15 @@ type NetConf struct { CNIDir string `json:"cniDir"` BinDir string `json:"binDir"` // RawDelegates is private to the NetConf class; use Delegates instead - RawDelegates []map[string]interface{} `json:"delegates"` - Delegates []*DelegateNetConf `json:"-"` - NetStatus []*NetworkStatus `json:"-"` - Kubeconfig string `json:"kubeconfig"` - LogFile string `json:"logFile"` - LogLevel string `json:"logLevel"` - RuntimeConfig *RuntimeConfig `json:"runtimeConfig,omitempty"` + RawDelegates []map[string]interface{} `json:"delegates"` + Delegates []*DelegateNetConf `json:"-"` + NetStatus []*NetworkStatus `json:"-"` + Kubeconfig string `json:"kubeconfig"` + ClusterNetwork string `json:"clusterNetwork"` + DefaultNetworks []string `json:"defaultNetworks"` + LogFile string `json:"logFile"` + LogLevel string `json:"logLevel"` + RuntimeConfig *RuntimeConfig `json:"runtimeConfig,omitempty"` // Default network readiness options ReadinessIndicatorFile string `json:readinessindicatorfile` }