Merge pull request #58714 from dcbw/cni-plugin-dirs

Automatic merge from submit-queue (batch tested with PRs 59740, 59728, 60080, 60086, 58714). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

kubelet: make --cni-bin-dir accept a comma-separated list of CNI plugin directories

Allow CNI-related network plugin drivers (kubenet, cni) to search a list of
directories for plugin binaries instead of just one.  This allows using an
administrator-provided path and fallbacks to others (like the previous default
of /opt/cni/bin) for backwards compatibility.

```release-note
kubelet's --cni-bin-dir option now accepts multiple comma-separated CNI binary directory paths, which are search for CNI plugins in the given order.
```

@kubernetes/rh-networking @kubernetes/sig-network-misc @freehan @pecameron @rajatchopra

cmd/kubelet/app/plugins.go

@@ -114,12 +114,12 @@ func GetDynamicPluginProber(pluginDir string) volume.DynamicPluginProber {
 }
 
 // ProbeNetworkPlugins collects all compiled-in plugins
-func ProbeNetworkPlugins(cniConfDir, cniBinDir string) []network.NetworkPlugin {
+func ProbeNetworkPlugins(cniConfDir string, cniBinDirs []string) []network.NetworkPlugin {
 	allPlugins := []network.NetworkPlugin{}
 
 	// for each existing plugin, add to the list
-	allPlugins = append(allPlugins, cni.ProbeNetworkPlugins(cniConfDir, cniBinDir)...)
+	allPlugins = append(allPlugins, cni.ProbeNetworkPlugins(cniConfDir, cniBinDirs)...)
-	allPlugins = append(allPlugins, kubenet.NewPlugin(cniBinDir))
+	allPlugins = append(allPlugins, kubenet.NewPlugin(cniBinDirs))
 
 	return allPlugins
 }

cmd/kubelet/app/server.go

@@ -80,6 +80,7 @@ import (
 	evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api"
 	dynamickubeletconfig "k8s.io/kubernetes/pkg/kubelet/kubeletconfig"
 	"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/configfiles"
+	"k8s.io/kubernetes/pkg/kubelet/network/cni"
 	"k8s.io/kubernetes/pkg/kubelet/server"
 	"k8s.io/kubernetes/pkg/kubelet/server/streaming"
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
@@ -358,7 +359,7 @@ func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, err
 		ExternalKubeClient:  nil,
 		EventClient:         nil,
 		Mounter:             mounter,
-		NetworkPlugins:      ProbeNetworkPlugins(s.CNIConfDir, s.CNIBinDir),
+		NetworkPlugins:      ProbeNetworkPlugins(s.CNIConfDir, cni.SplitDirs(s.CNIBinDir)),
 		OOMAdjuster:         oom.NewOOMAdjuster(),
 		OSInterface:         kubecontainer.RealOS{},
 		Writer:              writer,
@@ -1117,7 +1118,7 @@ func RunDockershim(f *options.KubeletFlags, c *kubeletconfiginternal.KubeletConf
 		NonMasqueradeCIDR: f.NonMasqueradeCIDR,
 		PluginName:        r.NetworkPluginName,
 		PluginConfDir:     r.CNIConfDir,
-		PluginBinDir:      r.CNIBinDir,
+		PluginBinDirs:     cni.SplitDirs(r.CNIBinDir),
 		MTU:               int(r.NetworkPluginMTU),
 		LegacyRuntimeHost: nh,
 	}

pkg/kubelet/BUILD

@@ -66,6 +66,7 @@ go_library(
         "//pkg/kubelet/metrics/collectors:go_default_library",
         "//pkg/kubelet/mountpod:go_default_library",
         "//pkg/kubelet/network:go_default_library",
+        "//pkg/kubelet/network/cni:go_default_library",
         "//pkg/kubelet/network/dns:go_default_library",
         "//pkg/kubelet/pleg:go_default_library",
         "//pkg/kubelet/pod:go_default_library",

pkg/kubelet/config/flags.go

@@ -98,7 +98,7 @@ func (s *ContainerRuntimeOptions) AddFlags(fs *pflag.FlagSet) {
 	// Network plugin settings. Shared by both docker and rkt.
 	fs.StringVar(&s.NetworkPluginName, "network-plugin", s.NetworkPluginName, "<Warning: Alpha feature> The name of the network plugin to be invoked for various events in kubelet/pod lifecycle")
 	fs.StringVar(&s.CNIConfDir, "cni-conf-dir", s.CNIConfDir, "<Warning: Alpha feature> The full path of the directory in which to search for CNI config files. Default: /etc/cni/net.d")
-	fs.StringVar(&s.CNIBinDir, "cni-bin-dir", s.CNIBinDir, "<Warning: Alpha feature> The full path of the directory in which to search for CNI plugin binaries. Default: /opt/cni/bin")
+	fs.StringVar(&s.CNIBinDir, "cni-bin-dir", s.CNIBinDir, "<Warning: Alpha feature> A comma-separated list of full paths of directories in which to search for CNI plugin binaries. Default: /opt/cni/bin")
 	fs.Int32Var(&s.NetworkPluginMTU, "network-plugin-mtu", s.NetworkPluginMTU, "<Warning: Alpha feature> The MTU to be passed to the network plugin, to override the default. Set to 0 to use the default 1460 MTU.")
 
 	// Rkt-specific settings.

pkg/kubelet/dockershim/docker_service.go

@@ -110,10 +110,10 @@ type NetworkPluginSettings struct {
 	NonMasqueradeCIDR string
 	// PluginName is the name of the plugin, runtime shim probes for
 	PluginName string
-	// PluginBinDir is the directory in which the binaries for the plugin with
+	// PluginBinDirs is an array of directories in which the binaries for
-	// PluginName is kept. The admin is responsible for provisioning these
+	// the plugin with PluginName may be found. The admin is responsible for
-	// binaries before-hand.
+	// provisioning these binaries before-hand.
-	PluginBinDir string
+	PluginBinDirs []string
 	// PluginConfDir is the directory in which the admin places a CNI conf.
 	// Depending on the plugin, this may be an optional field, eg: kubenet
 	// generates its own plugin conf.
@@ -229,8 +229,8 @@ func NewDockerService(config *ClientConfig, podSandboxImage string, streamingCon
 		}
 	}
 	// dockershim currently only supports CNI plugins.
-	cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, pluginSettings.PluginBinDir)
+	cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, pluginSettings.PluginBinDirs)
-	cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDir))
+	cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDirs))
 	netHost := &dockerNetworkHost{
 		pluginSettings.LegacyRuntimeHost,
 		&namespaceGetter{ds},

pkg/kubelet/kubelet.go

@@ -79,6 +79,7 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/metrics"
 	"k8s.io/kubernetes/pkg/kubelet/metrics/collectors"
 	"k8s.io/kubernetes/pkg/kubelet/network"
+	"k8s.io/kubernetes/pkg/kubelet/network/cni"
 	"k8s.io/kubernetes/pkg/kubelet/network/dns"
 	"k8s.io/kubernetes/pkg/kubelet/pleg"
 	kubepod "k8s.io/kubernetes/pkg/kubelet/pod"
@@ -587,7 +588,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 		NonMasqueradeCIDR: nonMasqueradeCIDR,
 		PluginName:        crOptions.NetworkPluginName,
 		PluginConfDir:     crOptions.CNIConfDir,
-		PluginBinDir:      crOptions.CNIBinDir,
+		PluginBinDirs:     cni.SplitDirs(crOptions.CNIBinDir),
 		MTU:               int(crOptions.NetworkPluginMTU),
 	}
 

pkg/kubelet/network/cni/cni.go

@@ -34,9 +34,8 @@ import (
 
 const (
 	CNIPluginName  = "cni"
-	DefaultNetDir        = "/etc/cni/net.d"
+	DefaultConfDir = "/etc/cni/net.d"
-	DefaultCNIDir        = "/opt/cni/bin"
+	DefaultBinDir  = "/opt/cni/bin"
-	VendorCNIDirTemplate = "%s/opt/%s/bin"
 )
 
 type cniNetworkPlugin struct {
@@ -50,9 +49,8 @@ type cniNetworkPlugin struct {
 	host        network.Host
 	execer      utilexec.Interface
 	nsenterPath string
-	pluginDir          string
+	confDir     string
-	binDir             string
+	binDirs     []string
-	vendorCNIDirPrefix string
 }
 
 type cniNetwork struct {
@@ -70,17 +68,33 @@ type cniPortMapping struct {
 	HostIP        string `json:"hostIP"`
 }
 
-func probeNetworkPluginsWithVendorCNIDirPrefix(pluginDir, binDir, vendorCNIDirPrefix string) []network.NetworkPlugin {
+func SplitDirs(dirs string) []string {
-	if binDir == "" {
+	// Use comma rather than colon to work better with Windows too
-		binDir = DefaultCNIDir
+	return strings.Split(dirs, ",")
+}
+
+func ProbeNetworkPlugins(confDir string, binDirs []string) []network.NetworkPlugin {
+	old := binDirs
+	binDirs = make([]string, len(binDirs))
+	for _, dir := range old {
+		if dir != "" {
+			binDirs = append(binDirs, dir)
 		}
+	}
+	if len(binDirs) == 0 {
+		binDirs = []string{DefaultBinDir}
+	}
+
+	if confDir == "" {
+		confDir = DefaultConfDir
+	}
+
 	plugin := &cniNetworkPlugin{
 		defaultNetwork: nil,
-		loNetwork:          getLoNetwork(binDir, vendorCNIDirPrefix),
+		loNetwork:      getLoNetwork(binDirs),
 		execer:         utilexec.New(),
-		pluginDir:          pluginDir,
+		confDir:        confDir,
-		binDir:             binDir,
+		binDirs:        binDirs,
-		vendorCNIDirPrefix: vendorCNIDirPrefix,
 	}
 
 	// sync NetworkConfig in best effort during probing.
@@ -88,20 +102,13 @@ func probeNetworkPluginsWithVendorCNIDirPrefix(pluginDir, binDir, vendorCNIDirPr
 	return []network.NetworkPlugin{plugin}
 }
 
-func ProbeNetworkPlugins(pluginDir, binDir string) []network.NetworkPlugin {
+func getDefaultCNINetwork(confDir string, binDirs []string) (*cniNetwork, error) {
-	return probeNetworkPluginsWithVendorCNIDirPrefix(pluginDir, binDir, "")
+	files, err := libcni.ConfFiles(confDir, []string{".conf", ".conflist", ".json"})
-}
-
-func getDefaultCNINetwork(pluginDir, binDir, vendorCNIDirPrefix string) (*cniNetwork, error) {
-	if pluginDir == "" {
-		pluginDir = DefaultNetDir
-	}
-	files, err := libcni.ConfFiles(pluginDir, []string{".conf", ".conflist", ".json"})
 	switch {
 	case err != nil:
 		return nil, err
 	case len(files) == 0:
-		return nil, fmt.Errorf("No networks found in %s", pluginDir)
+		return nil, fmt.Errorf("No networks found in %s", confDir)
 	}
 
 	sort.Strings(files)
@@ -136,21 +143,15 @@ func getDefaultCNINetwork(pluginDir, binDir, vendorCNIDirPrefix string) (*cniNet
 			glog.Warningf("CNI config list %s has no networks, skipping", confFile)
 			continue
 		}
-		confType := confList.Plugins[0].Network.Type
 
-		// Search for vendor-specific plugins as well as default plugins in the CNI codebase.
+		network := &cniNetwork{
-		vendorDir := vendorCNIDir(vendorCNIDirPrefix, confType)
+			name:          confList.Name,
-		cninet := &libcni.CNIConfig{
+			NetworkConfig: confList,
-			Path: []string{vendorDir, binDir},
+			CNIConfig:     &libcni.CNIConfig{Path: binDirs},
 		}
-		network := &cniNetwork{name: confList.Name, NetworkConfig: confList, CNIConfig: cninet}
 		return network, nil
 	}
-	return nil, fmt.Errorf("No valid networks found in %s", pluginDir)
+	return nil, fmt.Errorf("No valid networks found in %s", confDir)
-}
-
-func vendorCNIDir(prefix, pluginType string) string {
-	return fmt.Sprintf(VendorCNIDirTemplate, prefix, pluginType)
 }
 
 func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode kubeletconfig.HairpinMode, nonMasqueradeCIDR string, mtu int) error {
@@ -166,7 +167,7 @@ func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode kubeletconfi
 }
 
 func (plugin *cniNetworkPlugin) syncNetworkConfig() {
-	network, err := getDefaultCNINetwork(plugin.pluginDir, plugin.binDir, plugin.vendorCNIDirPrefix)
+	network, err := getDefaultCNINetwork(plugin.confDir, plugin.binDirs)
 	if err != nil {
 		glog.Warningf("Unable to update cni config: %s", err)
 		return
@@ -198,7 +199,7 @@ func (plugin *cniNetworkPlugin) Name() string {
 }
 
 func (plugin *cniNetworkPlugin) Status() error {
-	// sync network config from pluginDir periodically to detect network config updates
+	// sync network config from confDir periodically to detect network config updates
 	plugin.syncNetworkConfig()
 
 	// Can't set up pods if we don't have any CNI network configs yet

pkg/kubelet/network/cni/cni_others.go

@@ -26,7 +26,7 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/network"
 )
 
-func getLoNetwork(binDir, vendorDirPrefix string) *cniNetwork {
+func getLoNetwork(binDirs []string) *cniNetwork {
 	loConfig, err := libcni.ConfListFromBytes([]byte(`{
   "cniVersion": "0.2.0",
   "name": "cni-loopback",
@@ -39,13 +39,10 @@ func getLoNetwork(binDir, vendorDirPrefix string) *cniNetwork {
 		// catch this
 		panic(err)
 	}
-	cninet := &libcni.CNIConfig{
-		Path: []string{vendorCNIDir(vendorDirPrefix, "loopback"), binDir},
-	}
 	loNetwork := &cniNetwork{
 		name:          "lo",
 		NetworkConfig: loConfig,
-		CNIConfig:     cninet,
+		CNIConfig:     &libcni.CNIConfig{Path: binDirs},
 	}
 
 	return loNetwork

pkg/kubelet/network/cni/cni_test.go

@@ -47,31 +47,28 @@ import (
 	fakeexec "k8s.io/utils/exec/testing"
 )
 
-func installPluginUnderTest(t *testing.T, testVendorCNIDirPrefix, testNetworkConfigPath, vendorName string, plugName string) {
+// Returns .in file path, .out file path, and .env file path
-	pluginDir := path.Join(testNetworkConfigPath, plugName)
+func installPluginUnderTest(t *testing.T, testBinDir, testConfDir, testDataDir, binName string, confName string) (string, string, string) {
-	err := os.MkdirAll(pluginDir, 0777)
+	for _, dir := range []string{testBinDir, testConfDir, testDataDir} {
+		err := os.MkdirAll(dir, 0777)
 		if err != nil {
-		t.Fatalf("Failed to create plugin config dir: %v", err)
+			t.Fatalf("Failed to create test plugin dir %s: %v", dir, err)
 		}
-	pluginConfig := path.Join(pluginDir, plugName+".conf")
-	f, err := os.Create(pluginConfig)
-	if err != nil {
-		t.Fatalf("Failed to install plugin")
 	}
-	networkConfig := fmt.Sprintf(`{ "name": "%s", "type": "%s", "capabilities": {"portMappings": true}  }`, plugName, vendorName)
 
+	confFile := path.Join(testConfDir, confName+".conf")
+	f, err := os.Create(confFile)
+	if err != nil {
+		t.Fatalf("Failed to install plugin %s: %v", confFile, err)
+	}
+	networkConfig := fmt.Sprintf(`{ "name": "%s", "type": "%s", "capabilities": {"portMappings": true}  }`, confName, binName)
 	_, err = f.WriteString(networkConfig)
 	if err != nil {
 		t.Fatalf("Failed to write network config file (%v)", err)
 	}
 	f.Close()
 
-	vendorCNIDir := fmt.Sprintf(VendorCNIDirTemplate, testVendorCNIDirPrefix, vendorName)
+	pluginExec := path.Join(testBinDir, binName)
-	err = os.MkdirAll(vendorCNIDir, 0777)
-	if err != nil {
-		t.Fatalf("Failed to create plugin dir: %v", err)
-	}
-	pluginExec := path.Join(vendorCNIDir, vendorName)
 	f, err = os.Create(pluginExec)
 
 	const execScriptTempl = `#!/bin/bash
@@ -83,11 +80,14 @@ mkdir -p {{.OutputDir}} &> /dev/null
 echo -n "$CNI_COMMAND $CNI_NETNS $K8S_POD_NAMESPACE $K8S_POD_NAME $K8S_POD_INFRA_CONTAINER_ID" >& {{.OutputFile}}
 echo -n "{ \"ip4\": { \"ip\": \"10.1.0.23/24\" } }"
 `
+	inputFile := path.Join(testDataDir, binName+".in")
+	outputFile := path.Join(testDataDir, binName+".out")
+	envFile := path.Join(testDataDir, binName+".env")
 	execTemplateData := &map[string]interface{}{
-		"InputFile":  path.Join(pluginDir, plugName+".in"),
+		"InputFile":  inputFile,
-		"OutputFile": path.Join(pluginDir, plugName+".out"),
+		"OutputFile": outputFile,
-		"OutputEnv":  path.Join(pluginDir, plugName+".env"),
+		"OutputEnv":  envFile,
-		"OutputDir":  pluginDir,
+		"OutputDir":  testDataDir,
 	}
 
 	tObj := template.Must(template.New("test").Parse(execScriptTempl))
@@ -107,6 +107,8 @@ echo -n "{ \"ip4\": { \"ip\": \"10.1.0.23/24\" } }"
 	}
 
 	f.Close()
+
+	return inputFile, outputFile, envFile
 }
 
 func tearDownPlugin(tmpDir string) {
@@ -155,8 +157,8 @@ func (fnh *fakeNetworkHost) SupportsLegacyFeatures() bool {
 
 func TestCNIPlugin(t *testing.T) {
 	// install some random plugin
-	pluginName := fmt.Sprintf("test%d", rand.Intn(1000))
+	netName := fmt.Sprintf("test%d", rand.Intn(1000))
-	vendorName := fmt.Sprintf("test_vendor%d", rand.Intn(1000))
+	binName := fmt.Sprintf("test_vendor%d", rand.Intn(1000))
 
 	podIP := "10.0.0.2"
 	podIPOutput := fmt.Sprintf("4: eth0    inet %s/24 scope global dynamic eth0\\       valid_lft forever preferred_lft forever", podIP)
@@ -183,10 +185,11 @@ func TestCNIPlugin(t *testing.T) {
 	// TODO mock for the test plugin too
 
 	tmpDir := utiltesting.MkTmpdirOrDie("cni-test")
-	testNetworkConfigPath := path.Join(tmpDir, "plugins", "net", "cni")
+	testConfDir := path.Join(tmpDir, "etc", "cni", "net.d")
-	testVendorCNIDirPrefix := tmpDir
+	testBinDir := path.Join(tmpDir, "opt", "cni", "bin")
+	testDataDir := path.Join(tmpDir, "output")
 	defer tearDownPlugin(tmpDir)
-	installPluginUnderTest(t, testVendorCNIDirPrefix, testNetworkConfigPath, vendorName, pluginName)
+	inputFile, outputFile, outputEnv := installPluginUnderTest(t, testBinDir, testConfDir, testDataDir, binName, netName)
 
 	containerID := kubecontainer.ContainerID{Type: "test", ID: "test_infra_container"}
 	pods := []*containertest.FakePod{{
@@ -198,7 +201,7 @@ func TestCNIPlugin(t *testing.T) {
 		NetnsPath: "/proc/12345/ns/net",
 	}}
 
-	plugins := probeNetworkPluginsWithVendorCNIDirPrefix(path.Join(testNetworkConfigPath, pluginName), "", testVendorCNIDirPrefix)
+	plugins := ProbeNetworkPlugins(testConfDir, []string{testBinDir})
 	if len(plugins) != 1 {
 		t.Fatalf("Expected only one network plugin, got %d", len(plugins))
 	}
@@ -238,9 +241,7 @@ func TestCNIPlugin(t *testing.T) {
 	if err != nil {
 		t.Errorf("Expected nil: %v", err)
 	}
-	outputEnv := path.Join(testNetworkConfigPath, pluginName, pluginName+".env")
 	eo, eerr := ioutil.ReadFile(outputEnv)
-	outputFile := path.Join(testNetworkConfigPath, pluginName, pluginName+".out")
 	output, err := ioutil.ReadFile(outputFile)
 	if err != nil || eerr != nil {
 		t.Errorf("Failed to read output file %s: %v (env %s err %v)", outputFile, err, eo, eerr)
@@ -257,7 +258,6 @@ func TestCNIPlugin(t *testing.T) {
 			PortMappings []map[string]interface{} `json:"portMappings"`
 		} `json:"runtimeConfig"`
 	}{}
-	inputFile := path.Join(testNetworkConfigPath, pluginName, pluginName+".in")
 	inputBytes, inerr := ioutil.ReadFile(inputFile)
 	parseerr := json.Unmarshal(inputBytes, &inputConfig)
 	if inerr != nil || parseerr != nil {
@@ -285,7 +285,7 @@ func TestCNIPlugin(t *testing.T) {
 	if err != nil {
 		t.Errorf("Expected nil: %v", err)
 	}
-	output, err = ioutil.ReadFile(path.Join(testNetworkConfigPath, pluginName, pluginName+".out"))
+	output, err = ioutil.ReadFile(outputFile)
 	expectedOutput = "DEL /proc/12345/ns/net podNamespace podName test_infra_container"
 	if string(output) != expectedOutput {
 		t.Errorf("Mismatch in expected output for setup hook. Expected '%s', got '%s'", expectedOutput, string(output))
@@ -295,7 +295,7 @@ func TestCNIPlugin(t *testing.T) {
 }
 
 func TestLoNetNonNil(t *testing.T) {
-	if conf := getLoNetwork("", ""); conf == nil {
+	if conf := getLoNetwork(nil); conf == nil {
 		t.Error("Expected non-nil lo network")
 	}
 }

pkg/kubelet/network/cni/cni_windows.go

@@ -27,7 +27,7 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/network"
 )
 
-func getLoNetwork(binDir, vendorDirPrefix string) *cniNetwork {
+func getLoNetwork(binDirs []string) *cniNetwork {
 	return nil
 }
 

pkg/kubelet/network/kubenet/kubenet_linux.go

@@ -66,7 +66,7 @@ const (
 	defaultIPAMDir = "/var/lib/cni/networks"
 )
 
-// CNI plugins required by kubenet in /opt/cni/bin or vendor directory
+// CNI plugins required by kubenet in /opt/cni/bin or user-specified directory
 var requiredCNIPlugins = [...]string{"bridge", "host-local", "loopback"}
 
 type kubenetNetworkPlugin struct {
@@ -91,15 +91,15 @@ type kubenetNetworkPlugin struct {
 	iptables        utiliptables.Interface
 	sysctl          utilsysctl.Interface
 	ebtables        utilebtables.Interface
-	// vendorDir is passed by kubelet cni-bin-dir parameter.
+	// binDirs is passed by kubelet cni-bin-dir parameter.
-	// kubenet will search for cni binaries in DefaultCNIDir first, then continue to vendorDir.
+	// kubenet will search for CNI binaries in DefaultCNIDir first, then continue to binDirs.
-	vendorDir         string
+	binDirs           []string
 	nonMasqueradeCIDR string
 	podCidr           string
 	gateway           net.IP
 }
 
-func NewPlugin(networkPluginDir string) network.NetworkPlugin {
+func NewPlugin(networkPluginDirs []string) network.NetworkPlugin {
 	protocol := utiliptables.ProtocolIpv4
 	execer := utilexec.New()
 	dbus := utildbus.New()
@@ -110,7 +110,7 @@ func NewPlugin(networkPluginDir string) network.NetworkPlugin {
 		execer:            utilexec.New(),
 		iptables:          iptInterface,
 		sysctl:            sysctl,
-		vendorDir:         networkPluginDir,
+		binDirs:           append([]string{DefaultCNIDir}, networkPluginDirs...),
 		hostportSyncer:    hostport.NewHostportSyncer(iptInterface),
 		hostportManager:   hostport.NewHostportManager(iptInterface),
 		nonMasqueradeCIDR: "10.0.0.0/8",
@@ -121,9 +121,7 @@ func (plugin *kubenetNetworkPlugin) Init(host network.Host, hairpinMode kubeletc
 	plugin.host = host
 	plugin.hairpinMode = hairpinMode
 	plugin.nonMasqueradeCIDR = nonMasqueradeCIDR
-	plugin.cniConfig = &libcni.CNIConfig{
+	plugin.cniConfig = &libcni.CNIConfig{Path: plugin.binDirs}
-		Path: []string{DefaultCNIDir, plugin.vendorDir},
-	}
 
 	if mtu == network.UseDefaultMTU {
 		if link, err := findMinMTU(); err == nil {
@@ -564,22 +562,24 @@ func (plugin *kubenetNetworkPlugin) Status() error {
 		return fmt.Errorf("Kubenet does not have netConfig. This is most likely due to lack of PodCIDR")
 	}
 
-	if !plugin.checkCNIPlugin() {
+	if !plugin.checkRequiredCNIPlugins() {
-		return fmt.Errorf("could not locate kubenet required CNI plugins %v at %q or %q", requiredCNIPlugins, DefaultCNIDir, plugin.vendorDir)
+		return fmt.Errorf("could not locate kubenet required CNI plugins %v at %q", requiredCNIPlugins, plugin.binDirs)
 	}
 	return nil
 }
 
-// checkCNIPlugin returns if all kubenet required cni plugins can be found at /opt/cni/bin or user specified NetworkPluginDir.
+// checkRequiredCNIPlugins returns if all kubenet required cni plugins can be found at /opt/cni/bin or user specified NetworkPluginDir.
-func (plugin *kubenetNetworkPlugin) checkCNIPlugin() bool {
+func (plugin *kubenetNetworkPlugin) checkRequiredCNIPlugins() bool {
-	if plugin.checkCNIPluginInDir(DefaultCNIDir) || plugin.checkCNIPluginInDir(plugin.vendorDir) {
+	for _, dir := range plugin.binDirs {
+		if plugin.checkRequiredCNIPluginsInOneDir(dir) {
 			return true
 		}
+	}
 	return false
 }
 
-// checkCNIPluginInDir returns if all required cni plugins are placed in dir
+// checkRequiredCNIPluginsInOneDir returns true if all required cni plugins are placed in dir
-func (plugin *kubenetNetworkPlugin) checkCNIPluginInDir(dir string) bool {
+func (plugin *kubenetNetworkPlugin) checkRequiredCNIPluginsInOneDir(dir string) bool {
 	files, err := ioutil.ReadDir(dir)
 	if err != nil {
 		return false

pkg/kubelet/network/kubenet/kubenet_unsupported.go

@@ -30,7 +30,7 @@ type kubenetNetworkPlugin struct {
 	network.NoopNetworkPlugin
 }
 
-func NewPlugin(networkPluginDir string) network.NetworkPlugin {
+func NewPlugin(networkPluginDirs []string) network.NetworkPlugin {
 	return &kubenetNetworkPlugin{}
 }
 

pkg/kubelet/rkt/rkt_test.go

@@ -1215,6 +1215,7 @@ func TestGenerateRunCommand(t *testing.T) {
 	hostName := "test-hostname"
 	boolTrue := true
 	boolFalse := false
+	pluginDirs := []string{"/tmp"}
 
 	tests := []struct {
 		networkPlugin network.NetworkPlugin
@@ -1231,7 +1232,7 @@ func TestGenerateRunCommand(t *testing.T) {
 	}{
 		// Case #0, returns error.
 		{
-			kubenet.NewPlugin("/tmp"),
+			kubenet.NewPlugin(pluginDirs),
 			&v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Name: "pod-name-foo",
@@ -1250,7 +1251,7 @@ func TestGenerateRunCommand(t *testing.T) {
 		},
 		// Case #1, returns no dns, with private-net.
 		{
-			kubenet.NewPlugin("/tmp"),
+			kubenet.NewPlugin(pluginDirs),
 			&v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Name: "pod-name-foo",
@@ -1269,7 +1270,7 @@ func TestGenerateRunCommand(t *testing.T) {
 		},
 		// Case #2, returns no dns, with host-net.
 		{
-			kubenet.NewPlugin("/tmp"),
+			kubenet.NewPlugin(pluginDirs),
 			&v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Name: "pod-name-foo",
@@ -1290,7 +1291,7 @@ func TestGenerateRunCommand(t *testing.T) {
 		},
 		// Case #3, returns dns, dns searches, with private-net.
 		{
-			kubenet.NewPlugin("/tmp"),
+			kubenet.NewPlugin(pluginDirs),
 			&v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Name: "pod-name-foo",
@@ -1311,7 +1312,7 @@ func TestGenerateRunCommand(t *testing.T) {
 		},
 		// Case #4, returns no dns, dns searches, with host-network.
 		{
-			kubenet.NewPlugin("/tmp"),
+			kubenet.NewPlugin(pluginDirs),
 			&v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Name: "pod-name-foo",
@@ -1351,7 +1352,7 @@ func TestGenerateRunCommand(t *testing.T) {
 		},
 		// Case #6, if all containers are privileged, the result should have 'insecure-options=all-run'
 		{
-			kubenet.NewPlugin("/tmp"),
+			kubenet.NewPlugin(pluginDirs),
 			&v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Name: "pod-name-foo",
@@ -1373,7 +1374,7 @@ func TestGenerateRunCommand(t *testing.T) {
 		},
 		// Case #7, if not all containers are privileged, the result should not have 'insecure-options=all-run'
 		{
-			kubenet.NewPlugin("/tmp"),
+			kubenet.NewPlugin(pluginDirs),
 			&v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Name: "pod-name-foo",