diff --git a/README.md b/README.md index 8adf5a1a3..556eee517 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ * [Using with Multus conf file](#using-with-multus-conf-file) * [Logging Options](#logging-options) * [How to use with Network Device plugins?](#cni-running-with-network-device-plugin) + * [Default Network Readiness Checks](#default-network-readiness-checks) * [Testing Multus CNI](#testing-multus-cni) * [Multiple flannel networks](#multiple-flannel-networks) * [Configure Kubernetes with CNI](#configure-kubernetes-with-cni) @@ -499,6 +500,21 @@ Allocation of the Network device(such as SRIOV VFs) are done by Device plugins(E * [Device plugin & CNI, NUMA Manager alignment - technical architecture document](https://docs.google.com/document/d/1Ewe9Of84GkP0b2Q2PC0y9RVZNkN2WeVEagX9m99Nrzc/edit) * Reference implementation : [SRIOV Network device plugin](https://github.com/intel/sriov-network-device-plugin) * Example: [How to make Multus work with device plugin?](https://github.com/intel/multus-cni/tree/master/examples#passing-down-device-information) + +## Default Network Readiness Checks + +You may wish for your "default network" (that is, the CNI plugin & its configuration you specify as your default delegate) to become ready before you attach networks with Multus. This is disabled by default and not used unless you add the readiness check option(s) to your CNI configuration file. + +For example, if you use Flannel as a default network, the recommended method for Flannel to be installed is via a daemonset that also drops a configuration file in `/etc/cni/net.d/`. This may apply to other plugins that place that configuration file upon their readiness, hence, Multus uses their configuration filename as a semaphore and optionally waits to attach networks to pods until that file exists. + +In this manner, you may prevent pods from crash looping, and instead wait for that default network to be ready. + +Only one option is necessary to configure this functionality: + +* `readinessindicatorfile`: The path to a file whose existance denotes that the default network is ready. + +*NOTE*: If `readinessindicatorfile` is unset, or is an empty string, this functionality will be disabled, and is disabled by default. + ## Testing Multus CNI ### Multiple flannel networks diff --git a/multus/multus.go b/multus/multus.go index 73f150159..f461674bf 100644 --- a/multus/multus.go +++ b/multus/multus.go @@ -24,6 +24,7 @@ import ( "io/ioutil" "os" "path/filepath" + "time" "github.com/containernetworking/cni/libcni" "github.com/containernetworking/cni/pkg/invoke" @@ -35,8 +36,16 @@ import ( "github.com/intel/multus-cni/logging" "github.com/intel/multus-cni/types" "github.com/vishvananda/netlink" + "k8s.io/apimachinery/pkg/util/wait" ) +var defaultReadinessBackoff = wait.Backoff{ + Steps: 4, + Duration: 250 * time.Millisecond, + Factor: 4.0, + Jitter: 0.1, +} + func saveScratchNetConf(containerID, dataDir string, netconf []byte) error { logging.Debugf("saveScratchNetConf: %s, %s, %s", containerID, dataDir, string(netconf)) if err := os.MkdirAll(dataDir, 0700); err != nil { @@ -227,6 +236,16 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn return nil, logging.Errorf("Multus: Err in getting k8s args: %v", err) } + wait.ExponentialBackoff(defaultReadinessBackoff, func() (bool, error) { + _, err := os.Stat(n.ReadinessIndicatorFile) + switch { + case err == nil: + return true, nil + default: + return false, nil + } + }) + numK8sDelegates, kc, err := k8s.TryLoadK8sDelegates(k8sArgs, n, kubeClient) if err != nil { return nil, logging.Errorf("Multus: Err in loading K8s Delegates k8s args: %v", err) diff --git a/multus/multus_test.go b/multus/multus_test.go index 35cad0255..d05a0dbb1 100644 --- a/multus/multus_test.go +++ b/multus/multus_test.go @@ -173,6 +173,8 @@ var _ = Describe("multus operations", func() { StdinData: []byte(`{ "name": "node-cni-network", "type": "multus", + "defaultnetworkfile": "/tmp/foo.multus.conf", + "defaultnetworkwaitseconds": 3, "delegates": [{ "name": "weave1", "cniVersion": "0.2.0", @@ -185,6 +187,10 @@ var _ = Describe("multus operations", func() { }`), } + // Touch the default network file. + configPath := "/tmp/foo.multus.conf" + os.OpenFile(configPath, os.O_RDONLY|os.O_CREATE, 0755) + fExec := &fakeExec{} expectedResult1 := &types020.Result{ CNIVersion: "0.2.0", @@ -226,6 +232,13 @@ var _ = Describe("multus operations", func() { err = cmdDel(args, fExec, nil) Expect(err).NotTo(HaveOccurred()) Expect(fExec.delIndex).To(Equal(len(fExec.plugins))) + + // Cleanup default network file. + if _, errStat := os.Stat(configPath); errStat == nil { + errRemove := os.Remove(configPath) + Expect(errRemove).NotTo(HaveOccurred()) + } + }) It("executes delegates and kubernetes networks", func() { diff --git a/types/conf.go b/types/conf.go index bfbc13bce..ffc330ec4 100644 --- a/types/conf.go +++ b/types/conf.go @@ -27,9 +27,10 @@ import ( ) const ( - defaultCNIDir = "/var/lib/cni/multus" - defaultConfDir = "/etc/cni/multus/net.d" - defaultBinDir = "/opt/cni/bin" + defaultCNIDir = "/var/lib/cni/multus" + defaultConfDir = "/etc/cni/multus/net.d" + defaultBinDir = "/opt/cni/bin" + defaultReadinessIndicatorFile = "" ) func LoadDelegateNetConfList(bytes []byte, delegateConf *DelegateNetConf) error { @@ -194,6 +195,10 @@ func LoadNetConf(bytes []byte) (*NetConf, error) { netconf.BinDir = defaultBinDir } + if netconf.ReadinessIndicatorFile == "" { + netconf.ReadinessIndicatorFile = defaultReadinessIndicatorFile + } + for idx, rawConf := range netconf.RawDelegates { bytes, err := json.Marshal(rawConf) if err != nil { diff --git a/types/conf_test.go b/types/conf_test.go index fa64a3ddb..b6775eef1 100644 --- a/types/conf_test.go +++ b/types/conf_test.go @@ -81,4 +81,40 @@ var _ = Describe("config operations", func() { _, err := LoadNetConf([]byte(conf)) Expect(err).To(HaveOccurred()) }) + + It("has defaults set for network readiness", func() { + conf := `{ + "name": "defaultnetwork", + "type": "multus", + "kubeconfig": "/etc/kubernetes/kubelet.conf", + "delegates": [{ + "cniVersion": "0.3.0", + "name": "defaultnetwork", + "type": "flannel", + "isDefaultGateway": true + }] +}` + netConf, err := LoadNetConf([]byte(conf)) + Expect(err).NotTo(HaveOccurred()) + Expect(netConf.ReadinessIndicatorFile).To(Equal("")) + }) + + It("honors overrides for network readiness", func() { + conf := `{ + "name": "defaultnetwork", + "type": "multus", + "readinessindicatorfile": "/etc/cni/net.d/foo", + "kubeconfig": "/etc/kubernetes/kubelet.conf", + "delegates": [{ + "cniVersion": "0.3.0", + "name": "defaultnetwork", + "type": "flannel", + "isDefaultGateway": true + }] +}` + netConf, err := LoadNetConf([]byte(conf)) + Expect(err).NotTo(HaveOccurred()) + Expect(netConf.ReadinessIndicatorFile).To(Equal("/etc/cni/net.d/foo")) + }) + }) diff --git a/types/types.go b/types/types.go index b0760280b..efe7d3cfd 100644 --- a/types/types.go +++ b/types/types.go @@ -36,12 +36,13 @@ type NetConf struct { CNIDir string `json:"cniDir"` BinDir string `json:"binDir"` // RawDelegates is private to the NetConf class; use Delegates instead - RawDelegates []map[string]interface{} `json:"delegates"` - Delegates []*DelegateNetConf `json:"-"` - NetStatus []*NetworkStatus `json:"-"` - Kubeconfig string `json:"kubeconfig"` - LogFile string `json:"logFile"` - LogLevel string `json:"logLevel"` + RawDelegates []map[string]interface{} `json:"delegates"` + Delegates []*DelegateNetConf `json:"-"` + NetStatus []*NetworkStatus `json:"-"` + Kubeconfig string `json:"kubeconfig"` + LogFile string `json:"logFile"` + LogLevel string `json:"logLevel"` + ReadinessIndicatorFile string `json:readinessindicatorfile` } type NetworkStatus struct {