From 16eaaaed836ed66fac76f8624d528bc8cd4426bb Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 23 Jan 2018 11:45:20 -0600 Subject: [PATCH 1/4] cni: clarify bin/conf directory variable names pluginDir -> confDir DefaultNetDir -> DefaultConfDir DefaultCNIDir -> DefaultBinDir --- pkg/kubelet/network/cni/cni.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/pkg/kubelet/network/cni/cni.go b/pkg/kubelet/network/cni/cni.go index 15724afc486..40b1a85470c 100644 --- a/pkg/kubelet/network/cni/cni.go +++ b/pkg/kubelet/network/cni/cni.go @@ -34,8 +34,8 @@ import ( const ( CNIPluginName = "cni" - DefaultNetDir = "/etc/cni/net.d" - DefaultCNIDir = "/opt/cni/bin" + DefaultConfDir = "/etc/cni/net.d" + DefaultBinDir = "/opt/cni/bin" VendorCNIDirTemplate = "%s/opt/%s/bin" ) @@ -50,7 +50,7 @@ type cniNetworkPlugin struct { host network.Host execer utilexec.Interface nsenterPath string - pluginDir string + confDir string binDir string vendorCNIDirPrefix string } @@ -70,15 +70,15 @@ type cniPortMapping struct { HostIP string `json:"hostIP"` } -func probeNetworkPluginsWithVendorCNIDirPrefix(pluginDir, binDir, vendorCNIDirPrefix string) []network.NetworkPlugin { +func probeNetworkPluginsWithVendorCNIDirPrefix(confDir, binDir, vendorCNIDirPrefix string) []network.NetworkPlugin { if binDir == "" { - binDir = DefaultCNIDir + binDir = DefaultBinDir } plugin := &cniNetworkPlugin{ defaultNetwork: nil, loNetwork: getLoNetwork(binDir, vendorCNIDirPrefix), execer: utilexec.New(), - pluginDir: pluginDir, + confDir: confDir, binDir: binDir, vendorCNIDirPrefix: vendorCNIDirPrefix, } @@ -88,20 +88,20 @@ func probeNetworkPluginsWithVendorCNIDirPrefix(pluginDir, binDir, vendorCNIDirPr return []network.NetworkPlugin{plugin} } -func ProbeNetworkPlugins(pluginDir, binDir string) []network.NetworkPlugin { - return probeNetworkPluginsWithVendorCNIDirPrefix(pluginDir, binDir, "") +func ProbeNetworkPlugins(confDir, binDir string) []network.NetworkPlugin { + return probeNetworkPluginsWithVendorCNIDirPrefix(confDir, binDir, "") } -func getDefaultCNINetwork(pluginDir, binDir, vendorCNIDirPrefix string) (*cniNetwork, error) { - if pluginDir == "" { - pluginDir = DefaultNetDir +func getDefaultCNINetwork(confDir, binDir, vendorCNIDirPrefix string) (*cniNetwork, error) { + if confDir == "" { + confDir = DefaultConfDir } - files, err := libcni.ConfFiles(pluginDir, []string{".conf", ".conflist", ".json"}) + files, err := libcni.ConfFiles(confDir, []string{".conf", ".conflist", ".json"}) switch { case err != nil: return nil, err case len(files) == 0: - return nil, fmt.Errorf("No networks found in %s", pluginDir) + return nil, fmt.Errorf("No networks found in %s", confDir) } sort.Strings(files) @@ -146,7 +146,7 @@ func getDefaultCNINetwork(pluginDir, binDir, vendorCNIDirPrefix string) (*cniNet network := &cniNetwork{name: confList.Name, NetworkConfig: confList, CNIConfig: cninet} return network, nil } - return nil, fmt.Errorf("No valid networks found in %s", pluginDir) + return nil, fmt.Errorf("No valid networks found in %s", confDir) } func vendorCNIDir(prefix, pluginType string) string { @@ -166,7 +166,7 @@ func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode kubeletconfi } func (plugin *cniNetworkPlugin) syncNetworkConfig() { - network, err := getDefaultCNINetwork(plugin.pluginDir, plugin.binDir, plugin.vendorCNIDirPrefix) + network, err := getDefaultCNINetwork(plugin.confDir, plugin.binDir, plugin.vendorCNIDirPrefix) if err != nil { glog.Warningf("Unable to update cni config: %s", err) return @@ -198,7 +198,7 @@ func (plugin *cniNetworkPlugin) Name() string { } func (plugin *cniNetworkPlugin) Status() error { - // sync network config from pluginDir periodically to detect network config updates + // sync network config from confDir periodically to detect network config updates plugin.syncNetworkConfig() // Can't set up pods if we don't have any CNI network configs yet From 69ac723b78464b9474f9c069bbfd3dbbdbd19239 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 23 Jan 2018 13:27:38 -0600 Subject: [PATCH 2/4] cni: convert "vendor" option to multiple plugin binary search paths It's only used for the test code and after talking with Rajat, the vendor stuff was never really used anyway. So convert the vendor code into a plain array of plugin binary search paths, which is all the vendor code was doing anyway. --- cmd/kubelet/app/plugins.go | 2 +- pkg/kubelet/dockershim/docker_service.go | 2 +- pkg/kubelet/network/cni/cni.go | 72 +++++++++++------------- pkg/kubelet/network/cni/cni_others.go | 7 +-- pkg/kubelet/network/cni/cni_test.go | 64 ++++++++++----------- pkg/kubelet/network/cni/cni_windows.go | 2 +- 6 files changed, 71 insertions(+), 78 deletions(-) diff --git a/cmd/kubelet/app/plugins.go b/cmd/kubelet/app/plugins.go index ef41bb8e909..31947597541 100644 --- a/cmd/kubelet/app/plugins.go +++ b/cmd/kubelet/app/plugins.go @@ -118,7 +118,7 @@ func ProbeNetworkPlugins(cniConfDir, cniBinDir string) []network.NetworkPlugin { allPlugins := []network.NetworkPlugin{} // for each existing plugin, add to the list - allPlugins = append(allPlugins, cni.ProbeNetworkPlugins(cniConfDir, cniBinDir)...) + allPlugins = append(allPlugins, cni.ProbeNetworkPlugins(cniConfDir, []string{cniBinDir})...) allPlugins = append(allPlugins, kubenet.NewPlugin(cniBinDir)) return allPlugins diff --git a/pkg/kubelet/dockershim/docker_service.go b/pkg/kubelet/dockershim/docker_service.go index 3c88a0e4e0c..bdf1dfc78b5 100644 --- a/pkg/kubelet/dockershim/docker_service.go +++ b/pkg/kubelet/dockershim/docker_service.go @@ -229,7 +229,7 @@ func NewDockerService(config *ClientConfig, podSandboxImage string, streamingCon } } // dockershim currently only supports CNI plugins. - cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, pluginSettings.PluginBinDir) + cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, []string{pluginSettings.PluginBinDir}) cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDir)) netHost := &dockerNetworkHost{ pluginSettings.LegacyRuntimeHost, diff --git a/pkg/kubelet/network/cni/cni.go b/pkg/kubelet/network/cni/cni.go index 40b1a85470c..0f6741442b9 100644 --- a/pkg/kubelet/network/cni/cni.go +++ b/pkg/kubelet/network/cni/cni.go @@ -33,10 +33,9 @@ import ( ) const ( - CNIPluginName = "cni" - DefaultConfDir = "/etc/cni/net.d" - DefaultBinDir = "/opt/cni/bin" - VendorCNIDirTemplate = "%s/opt/%s/bin" + CNIPluginName = "cni" + DefaultConfDir = "/etc/cni/net.d" + DefaultBinDir = "/opt/cni/bin" ) type cniNetworkPlugin struct { @@ -47,12 +46,11 @@ type cniNetworkPlugin struct { sync.RWMutex defaultNetwork *cniNetwork - host network.Host - execer utilexec.Interface - nsenterPath string - confDir string - binDir string - vendorCNIDirPrefix string + host network.Host + execer utilexec.Interface + nsenterPath string + confDir string + binDirs []string } type cniNetwork struct { @@ -70,17 +68,28 @@ type cniPortMapping struct { HostIP string `json:"hostIP"` } -func probeNetworkPluginsWithVendorCNIDirPrefix(confDir, binDir, vendorCNIDirPrefix string) []network.NetworkPlugin { - if binDir == "" { - binDir = DefaultBinDir +func ProbeNetworkPlugins(confDir string, binDirs []string) []network.NetworkPlugin { + old := binDirs + binDirs = make([]string, len(binDirs)) + for _, dir := range old { + if dir != "" { + binDirs = append(binDirs, dir) + } } + if len(binDirs) == 0 { + binDirs = []string{DefaultBinDir} + } + + if confDir == "" { + confDir = DefaultConfDir + } + plugin := &cniNetworkPlugin{ - defaultNetwork: nil, - loNetwork: getLoNetwork(binDir, vendorCNIDirPrefix), - execer: utilexec.New(), - confDir: confDir, - binDir: binDir, - vendorCNIDirPrefix: vendorCNIDirPrefix, + defaultNetwork: nil, + loNetwork: getLoNetwork(binDirs), + execer: utilexec.New(), + confDir: confDir, + binDirs: binDirs, } // sync NetworkConfig in best effort during probing. @@ -88,14 +97,7 @@ func probeNetworkPluginsWithVendorCNIDirPrefix(confDir, binDir, vendorCNIDirPref return []network.NetworkPlugin{plugin} } -func ProbeNetworkPlugins(confDir, binDir string) []network.NetworkPlugin { - return probeNetworkPluginsWithVendorCNIDirPrefix(confDir, binDir, "") -} - -func getDefaultCNINetwork(confDir, binDir, vendorCNIDirPrefix string) (*cniNetwork, error) { - if confDir == "" { - confDir = DefaultConfDir - } +func getDefaultCNINetwork(confDir string, binDirs []string) (*cniNetwork, error) { files, err := libcni.ConfFiles(confDir, []string{".conf", ".conflist", ".json"}) switch { case err != nil: @@ -136,23 +138,17 @@ func getDefaultCNINetwork(confDir, binDir, vendorCNIDirPrefix string) (*cniNetwo glog.Warningf("CNI config list %s has no networks, skipping", confFile) continue } - confType := confList.Plugins[0].Network.Type - // Search for vendor-specific plugins as well as default plugins in the CNI codebase. - vendorDir := vendorCNIDir(vendorCNIDirPrefix, confType) - cninet := &libcni.CNIConfig{ - Path: []string{vendorDir, binDir}, + network := &cniNetwork{ + name: confList.Name, + NetworkConfig: confList, + CNIConfig: &libcni.CNIConfig{Path: binDirs}, } - network := &cniNetwork{name: confList.Name, NetworkConfig: confList, CNIConfig: cninet} return network, nil } return nil, fmt.Errorf("No valid networks found in %s", confDir) } -func vendorCNIDir(prefix, pluginType string) string { - return fmt.Sprintf(VendorCNIDirTemplate, prefix, pluginType) -} - func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode kubeletconfig.HairpinMode, nonMasqueradeCIDR string, mtu int) error { err := plugin.platformInit() if err != nil { @@ -166,7 +162,7 @@ func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode kubeletconfi } func (plugin *cniNetworkPlugin) syncNetworkConfig() { - network, err := getDefaultCNINetwork(plugin.confDir, plugin.binDir, plugin.vendorCNIDirPrefix) + network, err := getDefaultCNINetwork(plugin.confDir, plugin.binDirs) if err != nil { glog.Warningf("Unable to update cni config: %s", err) return diff --git a/pkg/kubelet/network/cni/cni_others.go b/pkg/kubelet/network/cni/cni_others.go index 735db9cc69c..cdc0c1a11f0 100644 --- a/pkg/kubelet/network/cni/cni_others.go +++ b/pkg/kubelet/network/cni/cni_others.go @@ -26,7 +26,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/network" ) -func getLoNetwork(binDir, vendorDirPrefix string) *cniNetwork { +func getLoNetwork(binDirs []string) *cniNetwork { loConfig, err := libcni.ConfListFromBytes([]byte(`{ "cniVersion": "0.2.0", "name": "cni-loopback", @@ -39,13 +39,10 @@ func getLoNetwork(binDir, vendorDirPrefix string) *cniNetwork { // catch this panic(err) } - cninet := &libcni.CNIConfig{ - Path: []string{vendorCNIDir(vendorDirPrefix, "loopback"), binDir}, - } loNetwork := &cniNetwork{ name: "lo", NetworkConfig: loConfig, - CNIConfig: cninet, + CNIConfig: &libcni.CNIConfig{Path: binDirs}, } return loNetwork diff --git a/pkg/kubelet/network/cni/cni_test.go b/pkg/kubelet/network/cni/cni_test.go index b34488ed4d3..f1701a22d4b 100644 --- a/pkg/kubelet/network/cni/cni_test.go +++ b/pkg/kubelet/network/cni/cni_test.go @@ -47,31 +47,28 @@ import ( fakeexec "k8s.io/utils/exec/testing" ) -func installPluginUnderTest(t *testing.T, testVendorCNIDirPrefix, testNetworkConfigPath, vendorName string, plugName string) { - pluginDir := path.Join(testNetworkConfigPath, plugName) - err := os.MkdirAll(pluginDir, 0777) - if err != nil { - t.Fatalf("Failed to create plugin config dir: %v", err) +// Returns .in file path, .out file path, and .env file path +func installPluginUnderTest(t *testing.T, testBinDir, testConfDir, testDataDir, binName string, confName string) (string, string, string) { + for _, dir := range []string{testBinDir, testConfDir, testDataDir} { + err := os.MkdirAll(dir, 0777) + if err != nil { + t.Fatalf("Failed to create test plugin dir %s: %v", dir, err) + } } - pluginConfig := path.Join(pluginDir, plugName+".conf") - f, err := os.Create(pluginConfig) - if err != nil { - t.Fatalf("Failed to install plugin") - } - networkConfig := fmt.Sprintf(`{ "name": "%s", "type": "%s", "capabilities": {"portMappings": true} }`, plugName, vendorName) + confFile := path.Join(testConfDir, confName+".conf") + f, err := os.Create(confFile) + if err != nil { + t.Fatalf("Failed to install plugin %s: %v", confFile, err) + } + networkConfig := fmt.Sprintf(`{ "name": "%s", "type": "%s", "capabilities": {"portMappings": true} }`, confName, binName) _, err = f.WriteString(networkConfig) if err != nil { t.Fatalf("Failed to write network config file (%v)", err) } f.Close() - vendorCNIDir := fmt.Sprintf(VendorCNIDirTemplate, testVendorCNIDirPrefix, vendorName) - err = os.MkdirAll(vendorCNIDir, 0777) - if err != nil { - t.Fatalf("Failed to create plugin dir: %v", err) - } - pluginExec := path.Join(vendorCNIDir, vendorName) + pluginExec := path.Join(testBinDir, binName) f, err = os.Create(pluginExec) const execScriptTempl = `#!/bin/bash @@ -83,11 +80,14 @@ mkdir -p {{.OutputDir}} &> /dev/null echo -n "$CNI_COMMAND $CNI_NETNS $K8S_POD_NAMESPACE $K8S_POD_NAME $K8S_POD_INFRA_CONTAINER_ID" >& {{.OutputFile}} echo -n "{ \"ip4\": { \"ip\": \"10.1.0.23/24\" } }" ` + inputFile := path.Join(testDataDir, binName+".in") + outputFile := path.Join(testDataDir, binName+".out") + envFile := path.Join(testDataDir, binName+".env") execTemplateData := &map[string]interface{}{ - "InputFile": path.Join(pluginDir, plugName+".in"), - "OutputFile": path.Join(pluginDir, plugName+".out"), - "OutputEnv": path.Join(pluginDir, plugName+".env"), - "OutputDir": pluginDir, + "InputFile": inputFile, + "OutputFile": outputFile, + "OutputEnv": envFile, + "OutputDir": testDataDir, } tObj := template.Must(template.New("test").Parse(execScriptTempl)) @@ -107,6 +107,8 @@ echo -n "{ \"ip4\": { \"ip\": \"10.1.0.23/24\" } }" } f.Close() + + return inputFile, outputFile, envFile } func tearDownPlugin(tmpDir string) { @@ -155,8 +157,8 @@ func (fnh *fakeNetworkHost) SupportsLegacyFeatures() bool { func TestCNIPlugin(t *testing.T) { // install some random plugin - pluginName := fmt.Sprintf("test%d", rand.Intn(1000)) - vendorName := fmt.Sprintf("test_vendor%d", rand.Intn(1000)) + netName := fmt.Sprintf("test%d", rand.Intn(1000)) + binName := fmt.Sprintf("test_vendor%d", rand.Intn(1000)) podIP := "10.0.0.2" podIPOutput := fmt.Sprintf("4: eth0 inet %s/24 scope global dynamic eth0\\ valid_lft forever preferred_lft forever", podIP) @@ -183,10 +185,11 @@ func TestCNIPlugin(t *testing.T) { // TODO mock for the test plugin too tmpDir := utiltesting.MkTmpdirOrDie("cni-test") - testNetworkConfigPath := path.Join(tmpDir, "plugins", "net", "cni") - testVendorCNIDirPrefix := tmpDir + testConfDir := path.Join(tmpDir, "etc", "cni", "net.d") + testBinDir := path.Join(tmpDir, "opt", "cni", "bin") + testDataDir := path.Join(tmpDir, "output") defer tearDownPlugin(tmpDir) - installPluginUnderTest(t, testVendorCNIDirPrefix, testNetworkConfigPath, vendorName, pluginName) + inputFile, outputFile, outputEnv := installPluginUnderTest(t, testBinDir, testConfDir, testDataDir, binName, netName) containerID := kubecontainer.ContainerID{Type: "test", ID: "test_infra_container"} pods := []*containertest.FakePod{{ @@ -198,7 +201,7 @@ func TestCNIPlugin(t *testing.T) { NetnsPath: "/proc/12345/ns/net", }} - plugins := probeNetworkPluginsWithVendorCNIDirPrefix(path.Join(testNetworkConfigPath, pluginName), "", testVendorCNIDirPrefix) + plugins := ProbeNetworkPlugins(testConfDir, []string{testBinDir}) if len(plugins) != 1 { t.Fatalf("Expected only one network plugin, got %d", len(plugins)) } @@ -238,9 +241,7 @@ func TestCNIPlugin(t *testing.T) { if err != nil { t.Errorf("Expected nil: %v", err) } - outputEnv := path.Join(testNetworkConfigPath, pluginName, pluginName+".env") eo, eerr := ioutil.ReadFile(outputEnv) - outputFile := path.Join(testNetworkConfigPath, pluginName, pluginName+".out") output, err := ioutil.ReadFile(outputFile) if err != nil || eerr != nil { t.Errorf("Failed to read output file %s: %v (env %s err %v)", outputFile, err, eo, eerr) @@ -257,7 +258,6 @@ func TestCNIPlugin(t *testing.T) { PortMappings []map[string]interface{} `json:"portMappings"` } `json:"runtimeConfig"` }{} - inputFile := path.Join(testNetworkConfigPath, pluginName, pluginName+".in") inputBytes, inerr := ioutil.ReadFile(inputFile) parseerr := json.Unmarshal(inputBytes, &inputConfig) if inerr != nil || parseerr != nil { @@ -285,7 +285,7 @@ func TestCNIPlugin(t *testing.T) { if err != nil { t.Errorf("Expected nil: %v", err) } - output, err = ioutil.ReadFile(path.Join(testNetworkConfigPath, pluginName, pluginName+".out")) + output, err = ioutil.ReadFile(outputFile) expectedOutput = "DEL /proc/12345/ns/net podNamespace podName test_infra_container" if string(output) != expectedOutput { t.Errorf("Mismatch in expected output for setup hook. Expected '%s', got '%s'", expectedOutput, string(output)) @@ -295,7 +295,7 @@ func TestCNIPlugin(t *testing.T) { } func TestLoNetNonNil(t *testing.T) { - if conf := getLoNetwork("", ""); conf == nil { + if conf := getLoNetwork(nil); conf == nil { t.Error("Expected non-nil lo network") } } diff --git a/pkg/kubelet/network/cni/cni_windows.go b/pkg/kubelet/network/cni/cni_windows.go index 9a0b17c8f11..81cc5ca1fe1 100644 --- a/pkg/kubelet/network/cni/cni_windows.go +++ b/pkg/kubelet/network/cni/cni_windows.go @@ -27,7 +27,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/network" ) -func getLoNetwork(binDir, vendorDirPrefix string) *cniNetwork { +func getLoNetwork(binDirs []string) *cniNetwork { return nil } From ee2ea223e7b6b684710c0f17ed3b7dd87e10701f Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 23 Jan 2018 14:03:51 -0600 Subject: [PATCH 3/4] kubenet: accept a list of CNI binary plugin paths --- cmd/kubelet/app/plugins.go | 2 +- pkg/kubelet/network/kubenet/kubenet_linux.go | 34 ++++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/cmd/kubelet/app/plugins.go b/cmd/kubelet/app/plugins.go index 31947597541..6fb144f1b67 100644 --- a/cmd/kubelet/app/plugins.go +++ b/cmd/kubelet/app/plugins.go @@ -119,7 +119,7 @@ func ProbeNetworkPlugins(cniConfDir, cniBinDir string) []network.NetworkPlugin { // for each existing plugin, add to the list allPlugins = append(allPlugins, cni.ProbeNetworkPlugins(cniConfDir, []string{cniBinDir})...) - allPlugins = append(allPlugins, kubenet.NewPlugin(cniBinDir)) + allPlugins = append(allPlugins, kubenet.NewPlugin([]string{cniBinDir})) return allPlugins } diff --git a/pkg/kubelet/network/kubenet/kubenet_linux.go b/pkg/kubelet/network/kubenet/kubenet_linux.go index 9fc7e3b279f..eff23ea219e 100644 --- a/pkg/kubelet/network/kubenet/kubenet_linux.go +++ b/pkg/kubelet/network/kubenet/kubenet_linux.go @@ -66,7 +66,7 @@ const ( defaultIPAMDir = "/var/lib/cni/networks" ) -// CNI plugins required by kubenet in /opt/cni/bin or vendor directory +// CNI plugins required by kubenet in /opt/cni/bin or user-specified directory var requiredCNIPlugins = [...]string{"bridge", "host-local", "loopback"} type kubenetNetworkPlugin struct { @@ -91,15 +91,15 @@ type kubenetNetworkPlugin struct { iptables utiliptables.Interface sysctl utilsysctl.Interface ebtables utilebtables.Interface - // vendorDir is passed by kubelet cni-bin-dir parameter. - // kubenet will search for cni binaries in DefaultCNIDir first, then continue to vendorDir. - vendorDir string + // binDirs is passed by kubelet cni-bin-dir parameter. + // kubenet will search for CNI binaries in DefaultCNIDir first, then continue to binDirs. + binDirs []string nonMasqueradeCIDR string podCidr string gateway net.IP } -func NewPlugin(networkPluginDir string) network.NetworkPlugin { +func NewPlugin(networkPluginDirs []string) network.NetworkPlugin { protocol := utiliptables.ProtocolIpv4 execer := utilexec.New() dbus := utildbus.New() @@ -110,7 +110,7 @@ func NewPlugin(networkPluginDir string) network.NetworkPlugin { execer: utilexec.New(), iptables: iptInterface, sysctl: sysctl, - vendorDir: networkPluginDir, + binDirs: append([]string{DefaultCNIDir}, networkPluginDirs...), hostportSyncer: hostport.NewHostportSyncer(iptInterface), hostportManager: hostport.NewHostportManager(iptInterface), nonMasqueradeCIDR: "10.0.0.0/8", @@ -121,9 +121,7 @@ func (plugin *kubenetNetworkPlugin) Init(host network.Host, hairpinMode kubeletc plugin.host = host plugin.hairpinMode = hairpinMode plugin.nonMasqueradeCIDR = nonMasqueradeCIDR - plugin.cniConfig = &libcni.CNIConfig{ - Path: []string{DefaultCNIDir, plugin.vendorDir}, - } + plugin.cniConfig = &libcni.CNIConfig{Path: plugin.binDirs} if mtu == network.UseDefaultMTU { if link, err := findMinMTU(); err == nil { @@ -564,22 +562,24 @@ func (plugin *kubenetNetworkPlugin) Status() error { return fmt.Errorf("Kubenet does not have netConfig. This is most likely due to lack of PodCIDR") } - if !plugin.checkCNIPlugin() { - return fmt.Errorf("could not locate kubenet required CNI plugins %v at %q or %q", requiredCNIPlugins, DefaultCNIDir, plugin.vendorDir) + if !plugin.checkRequiredCNIPlugins() { + return fmt.Errorf("could not locate kubenet required CNI plugins %v at %q or %q", requiredCNIPlugins, plugin.binDirs) } return nil } -// checkCNIPlugin returns if all kubenet required cni plugins can be found at /opt/cni/bin or user specified NetworkPluginDir. -func (plugin *kubenetNetworkPlugin) checkCNIPlugin() bool { - if plugin.checkCNIPluginInDir(DefaultCNIDir) || plugin.checkCNIPluginInDir(plugin.vendorDir) { - return true +// checkRequiredCNIPlugins returns if all kubenet required cni plugins can be found at /opt/cni/bin or user specified NetworkPluginDir. +func (plugin *kubenetNetworkPlugin) checkRequiredCNIPlugins() bool { + for _, dir := range plugin.binDirs { + if plugin.checkRequiredCNIPluginsInOneDir(dir) { + return true + } } return false } -// checkCNIPluginInDir returns if all required cni plugins are placed in dir -func (plugin *kubenetNetworkPlugin) checkCNIPluginInDir(dir string) bool { +// checkRequiredCNIPluginsInOneDir returns true if all required cni plugins are placed in dir +func (plugin *kubenetNetworkPlugin) checkRequiredCNIPluginsInOneDir(dir string) bool { files, err := ioutil.ReadDir(dir) if err != nil { return false From 8778e50083b02c3ba0286221f9c8097593266bee Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 23 Jan 2018 14:04:38 -0600 Subject: [PATCH 4/4] kubelet: make --cni-bin-dir accept a comma-separated list of CNI plugin directories Allow CNI-related network plugin drivers (kubenet, cni) to search a list of directories for plugin binaries instead of just one. This allows using an administrator-provided path and fallbacks to others (like the previous default of /opt/cni/bin) for backwards compatibility. --- cmd/kubelet/app/plugins.go | 6 +++--- cmd/kubelet/app/server.go | 5 +++-- pkg/kubelet/BUILD | 1 + pkg/kubelet/config/flags.go | 2 +- pkg/kubelet/dockershim/docker_service.go | 12 ++++++------ pkg/kubelet/kubelet.go | 3 ++- pkg/kubelet/network/cni/cni.go | 5 +++++ pkg/kubelet/network/kubenet/kubenet_linux.go | 2 +- .../network/kubenet/kubenet_unsupported.go | 2 +- pkg/kubelet/rkt/rkt_test.go | 15 ++++++++------- 10 files changed, 31 insertions(+), 22 deletions(-) diff --git a/cmd/kubelet/app/plugins.go b/cmd/kubelet/app/plugins.go index 6fb144f1b67..cbb20ba93ac 100644 --- a/cmd/kubelet/app/plugins.go +++ b/cmd/kubelet/app/plugins.go @@ -114,12 +114,12 @@ func GetDynamicPluginProber(pluginDir string) volume.DynamicPluginProber { } // ProbeNetworkPlugins collects all compiled-in plugins -func ProbeNetworkPlugins(cniConfDir, cniBinDir string) []network.NetworkPlugin { +func ProbeNetworkPlugins(cniConfDir string, cniBinDirs []string) []network.NetworkPlugin { allPlugins := []network.NetworkPlugin{} // for each existing plugin, add to the list - allPlugins = append(allPlugins, cni.ProbeNetworkPlugins(cniConfDir, []string{cniBinDir})...) - allPlugins = append(allPlugins, kubenet.NewPlugin([]string{cniBinDir})) + allPlugins = append(allPlugins, cni.ProbeNetworkPlugins(cniConfDir, cniBinDirs)...) + allPlugins = append(allPlugins, kubenet.NewPlugin(cniBinDirs)) return allPlugins } diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index dce5c4f2138..501bda8d86a 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -78,6 +78,7 @@ import ( evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api" dynamickubeletconfig "k8s.io/kubernetes/pkg/kubelet/kubeletconfig" "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/configfiles" + "k8s.io/kubernetes/pkg/kubelet/network/cni" "k8s.io/kubernetes/pkg/kubelet/server" "k8s.io/kubernetes/pkg/kubelet/server/streaming" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" @@ -355,7 +356,7 @@ func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, err ExternalKubeClient: nil, EventClient: nil, Mounter: mounter, - NetworkPlugins: ProbeNetworkPlugins(s.CNIConfDir, s.CNIBinDir), + NetworkPlugins: ProbeNetworkPlugins(s.CNIConfDir, cni.SplitDirs(s.CNIBinDir)), OOMAdjuster: oom.NewOOMAdjuster(), OSInterface: kubecontainer.RealOS{}, Writer: writer, @@ -1096,7 +1097,7 @@ func RunDockershim(f *options.KubeletFlags, c *kubeletconfiginternal.KubeletConf NonMasqueradeCIDR: f.NonMasqueradeCIDR, PluginName: r.NetworkPluginName, PluginConfDir: r.CNIConfDir, - PluginBinDir: r.CNIBinDir, + PluginBinDirs: cni.SplitDirs(r.CNIBinDir), MTU: int(r.NetworkPluginMTU), LegacyRuntimeHost: nh, } diff --git a/pkg/kubelet/BUILD b/pkg/kubelet/BUILD index f0230fbb113..6a8b3028001 100644 --- a/pkg/kubelet/BUILD +++ b/pkg/kubelet/BUILD @@ -66,6 +66,7 @@ go_library( "//pkg/kubelet/metrics/collectors:go_default_library", "//pkg/kubelet/mountpod:go_default_library", "//pkg/kubelet/network:go_default_library", + "//pkg/kubelet/network/cni:go_default_library", "//pkg/kubelet/network/dns:go_default_library", "//pkg/kubelet/pleg:go_default_library", "//pkg/kubelet/pod:go_default_library", diff --git a/pkg/kubelet/config/flags.go b/pkg/kubelet/config/flags.go index f9bde6a6f70..9e1c94d59ca 100644 --- a/pkg/kubelet/config/flags.go +++ b/pkg/kubelet/config/flags.go @@ -98,7 +98,7 @@ func (s *ContainerRuntimeOptions) AddFlags(fs *pflag.FlagSet) { // Network plugin settings. Shared by both docker and rkt. fs.StringVar(&s.NetworkPluginName, "network-plugin", s.NetworkPluginName, " The name of the network plugin to be invoked for various events in kubelet/pod lifecycle") fs.StringVar(&s.CNIConfDir, "cni-conf-dir", s.CNIConfDir, " The full path of the directory in which to search for CNI config files. Default: /etc/cni/net.d") - fs.StringVar(&s.CNIBinDir, "cni-bin-dir", s.CNIBinDir, " The full path of the directory in which to search for CNI plugin binaries. Default: /opt/cni/bin") + fs.StringVar(&s.CNIBinDir, "cni-bin-dir", s.CNIBinDir, " A comma-separated list of full paths of directories in which to search for CNI plugin binaries. Default: /opt/cni/bin") fs.Int32Var(&s.NetworkPluginMTU, "network-plugin-mtu", s.NetworkPluginMTU, " The MTU to be passed to the network plugin, to override the default. Set to 0 to use the default 1460 MTU.") // Rkt-specific settings. diff --git a/pkg/kubelet/dockershim/docker_service.go b/pkg/kubelet/dockershim/docker_service.go index bdf1dfc78b5..82828d0df68 100644 --- a/pkg/kubelet/dockershim/docker_service.go +++ b/pkg/kubelet/dockershim/docker_service.go @@ -110,10 +110,10 @@ type NetworkPluginSettings struct { NonMasqueradeCIDR string // PluginName is the name of the plugin, runtime shim probes for PluginName string - // PluginBinDir is the directory in which the binaries for the plugin with - // PluginName is kept. The admin is responsible for provisioning these - // binaries before-hand. - PluginBinDir string + // PluginBinDirs is an array of directories in which the binaries for + // the plugin with PluginName may be found. The admin is responsible for + // provisioning these binaries before-hand. + PluginBinDirs []string // PluginConfDir is the directory in which the admin places a CNI conf. // Depending on the plugin, this may be an optional field, eg: kubenet // generates its own plugin conf. @@ -229,8 +229,8 @@ func NewDockerService(config *ClientConfig, podSandboxImage string, streamingCon } } // dockershim currently only supports CNI plugins. - cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, []string{pluginSettings.PluginBinDir}) - cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDir)) + cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, pluginSettings.PluginBinDirs) + cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDirs)) netHost := &dockerNetworkHost{ pluginSettings.LegacyRuntimeHost, &namespaceGetter{ds}, diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 5a58f208460..a45ef72f679 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -79,6 +79,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/metrics" "k8s.io/kubernetes/pkg/kubelet/metrics/collectors" "k8s.io/kubernetes/pkg/kubelet/network" + "k8s.io/kubernetes/pkg/kubelet/network/cni" "k8s.io/kubernetes/pkg/kubelet/network/dns" "k8s.io/kubernetes/pkg/kubelet/pleg" kubepod "k8s.io/kubernetes/pkg/kubelet/pod" @@ -587,7 +588,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, NonMasqueradeCIDR: nonMasqueradeCIDR, PluginName: crOptions.NetworkPluginName, PluginConfDir: crOptions.CNIConfDir, - PluginBinDir: crOptions.CNIBinDir, + PluginBinDirs: cni.SplitDirs(crOptions.CNIBinDir), MTU: int(crOptions.NetworkPluginMTU), } diff --git a/pkg/kubelet/network/cni/cni.go b/pkg/kubelet/network/cni/cni.go index 0f6741442b9..692e218deec 100644 --- a/pkg/kubelet/network/cni/cni.go +++ b/pkg/kubelet/network/cni/cni.go @@ -68,6 +68,11 @@ type cniPortMapping struct { HostIP string `json:"hostIP"` } +func SplitDirs(dirs string) []string { + // Use comma rather than colon to work better with Windows too + return strings.Split(dirs, ",") +} + func ProbeNetworkPlugins(confDir string, binDirs []string) []network.NetworkPlugin { old := binDirs binDirs = make([]string, len(binDirs)) diff --git a/pkg/kubelet/network/kubenet/kubenet_linux.go b/pkg/kubelet/network/kubenet/kubenet_linux.go index eff23ea219e..8c8904034a2 100644 --- a/pkg/kubelet/network/kubenet/kubenet_linux.go +++ b/pkg/kubelet/network/kubenet/kubenet_linux.go @@ -563,7 +563,7 @@ func (plugin *kubenetNetworkPlugin) Status() error { } if !plugin.checkRequiredCNIPlugins() { - return fmt.Errorf("could not locate kubenet required CNI plugins %v at %q or %q", requiredCNIPlugins, plugin.binDirs) + return fmt.Errorf("could not locate kubenet required CNI plugins %v at %q", requiredCNIPlugins, plugin.binDirs) } return nil } diff --git a/pkg/kubelet/network/kubenet/kubenet_unsupported.go b/pkg/kubelet/network/kubenet/kubenet_unsupported.go index 1cbc7ab7251..ba2ee3f531c 100644 --- a/pkg/kubelet/network/kubenet/kubenet_unsupported.go +++ b/pkg/kubelet/network/kubenet/kubenet_unsupported.go @@ -30,7 +30,7 @@ type kubenetNetworkPlugin struct { network.NoopNetworkPlugin } -func NewPlugin(networkPluginDir string) network.NetworkPlugin { +func NewPlugin(networkPluginDirs []string) network.NetworkPlugin { return &kubenetNetworkPlugin{} } diff --git a/pkg/kubelet/rkt/rkt_test.go b/pkg/kubelet/rkt/rkt_test.go index d235fe289c2..8e5f6cdd812 100644 --- a/pkg/kubelet/rkt/rkt_test.go +++ b/pkg/kubelet/rkt/rkt_test.go @@ -1215,6 +1215,7 @@ func TestGenerateRunCommand(t *testing.T) { hostName := "test-hostname" boolTrue := true boolFalse := false + pluginDirs := []string{"/tmp"} tests := []struct { networkPlugin network.NetworkPlugin @@ -1231,7 +1232,7 @@ func TestGenerateRunCommand(t *testing.T) { }{ // Case #0, returns error. { - kubenet.NewPlugin("/tmp"), + kubenet.NewPlugin(pluginDirs), &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod-name-foo", @@ -1250,7 +1251,7 @@ func TestGenerateRunCommand(t *testing.T) { }, // Case #1, returns no dns, with private-net. { - kubenet.NewPlugin("/tmp"), + kubenet.NewPlugin(pluginDirs), &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod-name-foo", @@ -1269,7 +1270,7 @@ func TestGenerateRunCommand(t *testing.T) { }, // Case #2, returns no dns, with host-net. { - kubenet.NewPlugin("/tmp"), + kubenet.NewPlugin(pluginDirs), &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod-name-foo", @@ -1290,7 +1291,7 @@ func TestGenerateRunCommand(t *testing.T) { }, // Case #3, returns dns, dns searches, with private-net. { - kubenet.NewPlugin("/tmp"), + kubenet.NewPlugin(pluginDirs), &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod-name-foo", @@ -1311,7 +1312,7 @@ func TestGenerateRunCommand(t *testing.T) { }, // Case #4, returns no dns, dns searches, with host-network. { - kubenet.NewPlugin("/tmp"), + kubenet.NewPlugin(pluginDirs), &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod-name-foo", @@ -1351,7 +1352,7 @@ func TestGenerateRunCommand(t *testing.T) { }, // Case #6, if all containers are privileged, the result should have 'insecure-options=all-run' { - kubenet.NewPlugin("/tmp"), + kubenet.NewPlugin(pluginDirs), &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod-name-foo", @@ -1373,7 +1374,7 @@ func TestGenerateRunCommand(t *testing.T) { }, // Case #7, if not all containers are privileged, the result should not have 'insecure-options=all-run' { - kubenet.NewPlugin("/tmp"), + kubenet.NewPlugin(pluginDirs), &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod-name-foo",