diff --git a/cmd/kubelet/app/plugins.go b/cmd/kubelet/app/plugins.go index ef41bb8e909..31947597541 100644 --- a/cmd/kubelet/app/plugins.go +++ b/cmd/kubelet/app/plugins.go @@ -118,7 +118,7 @@ func ProbeNetworkPlugins(cniConfDir, cniBinDir string) []network.NetworkPlugin { allPlugins := []network.NetworkPlugin{} // for each existing plugin, add to the list - allPlugins = append(allPlugins, cni.ProbeNetworkPlugins(cniConfDir, cniBinDir)...) + allPlugins = append(allPlugins, cni.ProbeNetworkPlugins(cniConfDir, []string{cniBinDir})...) allPlugins = append(allPlugins, kubenet.NewPlugin(cniBinDir)) return allPlugins diff --git a/pkg/kubelet/dockershim/docker_service.go b/pkg/kubelet/dockershim/docker_service.go index 3c88a0e4e0c..bdf1dfc78b5 100644 --- a/pkg/kubelet/dockershim/docker_service.go +++ b/pkg/kubelet/dockershim/docker_service.go @@ -229,7 +229,7 @@ func NewDockerService(config *ClientConfig, podSandboxImage string, streamingCon } } // dockershim currently only supports CNI plugins. - cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, pluginSettings.PluginBinDir) + cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, []string{pluginSettings.PluginBinDir}) cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDir)) netHost := &dockerNetworkHost{ pluginSettings.LegacyRuntimeHost, diff --git a/pkg/kubelet/network/cni/cni.go b/pkg/kubelet/network/cni/cni.go index 40b1a85470c..0f6741442b9 100644 --- a/pkg/kubelet/network/cni/cni.go +++ b/pkg/kubelet/network/cni/cni.go @@ -33,10 +33,9 @@ import ( ) const ( - CNIPluginName = "cni" - DefaultConfDir = "/etc/cni/net.d" - DefaultBinDir = "/opt/cni/bin" - VendorCNIDirTemplate = "%s/opt/%s/bin" + CNIPluginName = "cni" + DefaultConfDir = "/etc/cni/net.d" + DefaultBinDir = "/opt/cni/bin" ) type cniNetworkPlugin struct { @@ -47,12 +46,11 @@ type cniNetworkPlugin struct { sync.RWMutex defaultNetwork *cniNetwork - host network.Host - execer utilexec.Interface - nsenterPath string - confDir string - binDir string - vendorCNIDirPrefix string + host network.Host + execer utilexec.Interface + nsenterPath string + confDir string + binDirs []string } type cniNetwork struct { @@ -70,17 +68,28 @@ type cniPortMapping struct { HostIP string `json:"hostIP"` } -func probeNetworkPluginsWithVendorCNIDirPrefix(confDir, binDir, vendorCNIDirPrefix string) []network.NetworkPlugin { - if binDir == "" { - binDir = DefaultBinDir +func ProbeNetworkPlugins(confDir string, binDirs []string) []network.NetworkPlugin { + old := binDirs + binDirs = make([]string, len(binDirs)) + for _, dir := range old { + if dir != "" { + binDirs = append(binDirs, dir) + } } + if len(binDirs) == 0 { + binDirs = []string{DefaultBinDir} + } + + if confDir == "" { + confDir = DefaultConfDir + } + plugin := &cniNetworkPlugin{ - defaultNetwork: nil, - loNetwork: getLoNetwork(binDir, vendorCNIDirPrefix), - execer: utilexec.New(), - confDir: confDir, - binDir: binDir, - vendorCNIDirPrefix: vendorCNIDirPrefix, + defaultNetwork: nil, + loNetwork: getLoNetwork(binDirs), + execer: utilexec.New(), + confDir: confDir, + binDirs: binDirs, } // sync NetworkConfig in best effort during probing. @@ -88,14 +97,7 @@ func probeNetworkPluginsWithVendorCNIDirPrefix(confDir, binDir, vendorCNIDirPref return []network.NetworkPlugin{plugin} } -func ProbeNetworkPlugins(confDir, binDir string) []network.NetworkPlugin { - return probeNetworkPluginsWithVendorCNIDirPrefix(confDir, binDir, "") -} - -func getDefaultCNINetwork(confDir, binDir, vendorCNIDirPrefix string) (*cniNetwork, error) { - if confDir == "" { - confDir = DefaultConfDir - } +func getDefaultCNINetwork(confDir string, binDirs []string) (*cniNetwork, error) { files, err := libcni.ConfFiles(confDir, []string{".conf", ".conflist", ".json"}) switch { case err != nil: @@ -136,23 +138,17 @@ func getDefaultCNINetwork(confDir, binDir, vendorCNIDirPrefix string) (*cniNetwo glog.Warningf("CNI config list %s has no networks, skipping", confFile) continue } - confType := confList.Plugins[0].Network.Type - // Search for vendor-specific plugins as well as default plugins in the CNI codebase. - vendorDir := vendorCNIDir(vendorCNIDirPrefix, confType) - cninet := &libcni.CNIConfig{ - Path: []string{vendorDir, binDir}, + network := &cniNetwork{ + name: confList.Name, + NetworkConfig: confList, + CNIConfig: &libcni.CNIConfig{Path: binDirs}, } - network := &cniNetwork{name: confList.Name, NetworkConfig: confList, CNIConfig: cninet} return network, nil } return nil, fmt.Errorf("No valid networks found in %s", confDir) } -func vendorCNIDir(prefix, pluginType string) string { - return fmt.Sprintf(VendorCNIDirTemplate, prefix, pluginType) -} - func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode kubeletconfig.HairpinMode, nonMasqueradeCIDR string, mtu int) error { err := plugin.platformInit() if err != nil { @@ -166,7 +162,7 @@ func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode kubeletconfi } func (plugin *cniNetworkPlugin) syncNetworkConfig() { - network, err := getDefaultCNINetwork(plugin.confDir, plugin.binDir, plugin.vendorCNIDirPrefix) + network, err := getDefaultCNINetwork(plugin.confDir, plugin.binDirs) if err != nil { glog.Warningf("Unable to update cni config: %s", err) return diff --git a/pkg/kubelet/network/cni/cni_others.go b/pkg/kubelet/network/cni/cni_others.go index 735db9cc69c..cdc0c1a11f0 100644 --- a/pkg/kubelet/network/cni/cni_others.go +++ b/pkg/kubelet/network/cni/cni_others.go @@ -26,7 +26,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/network" ) -func getLoNetwork(binDir, vendorDirPrefix string) *cniNetwork { +func getLoNetwork(binDirs []string) *cniNetwork { loConfig, err := libcni.ConfListFromBytes([]byte(`{ "cniVersion": "0.2.0", "name": "cni-loopback", @@ -39,13 +39,10 @@ func getLoNetwork(binDir, vendorDirPrefix string) *cniNetwork { // catch this panic(err) } - cninet := &libcni.CNIConfig{ - Path: []string{vendorCNIDir(vendorDirPrefix, "loopback"), binDir}, - } loNetwork := &cniNetwork{ name: "lo", NetworkConfig: loConfig, - CNIConfig: cninet, + CNIConfig: &libcni.CNIConfig{Path: binDirs}, } return loNetwork diff --git a/pkg/kubelet/network/cni/cni_test.go b/pkg/kubelet/network/cni/cni_test.go index b34488ed4d3..f1701a22d4b 100644 --- a/pkg/kubelet/network/cni/cni_test.go +++ b/pkg/kubelet/network/cni/cni_test.go @@ -47,31 +47,28 @@ import ( fakeexec "k8s.io/utils/exec/testing" ) -func installPluginUnderTest(t *testing.T, testVendorCNIDirPrefix, testNetworkConfigPath, vendorName string, plugName string) { - pluginDir := path.Join(testNetworkConfigPath, plugName) - err := os.MkdirAll(pluginDir, 0777) - if err != nil { - t.Fatalf("Failed to create plugin config dir: %v", err) +// Returns .in file path, .out file path, and .env file path +func installPluginUnderTest(t *testing.T, testBinDir, testConfDir, testDataDir, binName string, confName string) (string, string, string) { + for _, dir := range []string{testBinDir, testConfDir, testDataDir} { + err := os.MkdirAll(dir, 0777) + if err != nil { + t.Fatalf("Failed to create test plugin dir %s: %v", dir, err) + } } - pluginConfig := path.Join(pluginDir, plugName+".conf") - f, err := os.Create(pluginConfig) - if err != nil { - t.Fatalf("Failed to install plugin") - } - networkConfig := fmt.Sprintf(`{ "name": "%s", "type": "%s", "capabilities": {"portMappings": true} }`, plugName, vendorName) + confFile := path.Join(testConfDir, confName+".conf") + f, err := os.Create(confFile) + if err != nil { + t.Fatalf("Failed to install plugin %s: %v", confFile, err) + } + networkConfig := fmt.Sprintf(`{ "name": "%s", "type": "%s", "capabilities": {"portMappings": true} }`, confName, binName) _, err = f.WriteString(networkConfig) if err != nil { t.Fatalf("Failed to write network config file (%v)", err) } f.Close() - vendorCNIDir := fmt.Sprintf(VendorCNIDirTemplate, testVendorCNIDirPrefix, vendorName) - err = os.MkdirAll(vendorCNIDir, 0777) - if err != nil { - t.Fatalf("Failed to create plugin dir: %v", err) - } - pluginExec := path.Join(vendorCNIDir, vendorName) + pluginExec := path.Join(testBinDir, binName) f, err = os.Create(pluginExec) const execScriptTempl = `#!/bin/bash @@ -83,11 +80,14 @@ mkdir -p {{.OutputDir}} &> /dev/null echo -n "$CNI_COMMAND $CNI_NETNS $K8S_POD_NAMESPACE $K8S_POD_NAME $K8S_POD_INFRA_CONTAINER_ID" >& {{.OutputFile}} echo -n "{ \"ip4\": { \"ip\": \"10.1.0.23/24\" } }" ` + inputFile := path.Join(testDataDir, binName+".in") + outputFile := path.Join(testDataDir, binName+".out") + envFile := path.Join(testDataDir, binName+".env") execTemplateData := &map[string]interface{}{ - "InputFile": path.Join(pluginDir, plugName+".in"), - "OutputFile": path.Join(pluginDir, plugName+".out"), - "OutputEnv": path.Join(pluginDir, plugName+".env"), - "OutputDir": pluginDir, + "InputFile": inputFile, + "OutputFile": outputFile, + "OutputEnv": envFile, + "OutputDir": testDataDir, } tObj := template.Must(template.New("test").Parse(execScriptTempl)) @@ -107,6 +107,8 @@ echo -n "{ \"ip4\": { \"ip\": \"10.1.0.23/24\" } }" } f.Close() + + return inputFile, outputFile, envFile } func tearDownPlugin(tmpDir string) { @@ -155,8 +157,8 @@ func (fnh *fakeNetworkHost) SupportsLegacyFeatures() bool { func TestCNIPlugin(t *testing.T) { // install some random plugin - pluginName := fmt.Sprintf("test%d", rand.Intn(1000)) - vendorName := fmt.Sprintf("test_vendor%d", rand.Intn(1000)) + netName := fmt.Sprintf("test%d", rand.Intn(1000)) + binName := fmt.Sprintf("test_vendor%d", rand.Intn(1000)) podIP := "10.0.0.2" podIPOutput := fmt.Sprintf("4: eth0 inet %s/24 scope global dynamic eth0\\ valid_lft forever preferred_lft forever", podIP) @@ -183,10 +185,11 @@ func TestCNIPlugin(t *testing.T) { // TODO mock for the test plugin too tmpDir := utiltesting.MkTmpdirOrDie("cni-test") - testNetworkConfigPath := path.Join(tmpDir, "plugins", "net", "cni") - testVendorCNIDirPrefix := tmpDir + testConfDir := path.Join(tmpDir, "etc", "cni", "net.d") + testBinDir := path.Join(tmpDir, "opt", "cni", "bin") + testDataDir := path.Join(tmpDir, "output") defer tearDownPlugin(tmpDir) - installPluginUnderTest(t, testVendorCNIDirPrefix, testNetworkConfigPath, vendorName, pluginName) + inputFile, outputFile, outputEnv := installPluginUnderTest(t, testBinDir, testConfDir, testDataDir, binName, netName) containerID := kubecontainer.ContainerID{Type: "test", ID: "test_infra_container"} pods := []*containertest.FakePod{{ @@ -198,7 +201,7 @@ func TestCNIPlugin(t *testing.T) { NetnsPath: "/proc/12345/ns/net", }} - plugins := probeNetworkPluginsWithVendorCNIDirPrefix(path.Join(testNetworkConfigPath, pluginName), "", testVendorCNIDirPrefix) + plugins := ProbeNetworkPlugins(testConfDir, []string{testBinDir}) if len(plugins) != 1 { t.Fatalf("Expected only one network plugin, got %d", len(plugins)) } @@ -238,9 +241,7 @@ func TestCNIPlugin(t *testing.T) { if err != nil { t.Errorf("Expected nil: %v", err) } - outputEnv := path.Join(testNetworkConfigPath, pluginName, pluginName+".env") eo, eerr := ioutil.ReadFile(outputEnv) - outputFile := path.Join(testNetworkConfigPath, pluginName, pluginName+".out") output, err := ioutil.ReadFile(outputFile) if err != nil || eerr != nil { t.Errorf("Failed to read output file %s: %v (env %s err %v)", outputFile, err, eo, eerr) @@ -257,7 +258,6 @@ func TestCNIPlugin(t *testing.T) { PortMappings []map[string]interface{} `json:"portMappings"` } `json:"runtimeConfig"` }{} - inputFile := path.Join(testNetworkConfigPath, pluginName, pluginName+".in") inputBytes, inerr := ioutil.ReadFile(inputFile) parseerr := json.Unmarshal(inputBytes, &inputConfig) if inerr != nil || parseerr != nil { @@ -285,7 +285,7 @@ func TestCNIPlugin(t *testing.T) { if err != nil { t.Errorf("Expected nil: %v", err) } - output, err = ioutil.ReadFile(path.Join(testNetworkConfigPath, pluginName, pluginName+".out")) + output, err = ioutil.ReadFile(outputFile) expectedOutput = "DEL /proc/12345/ns/net podNamespace podName test_infra_container" if string(output) != expectedOutput { t.Errorf("Mismatch in expected output for setup hook. Expected '%s', got '%s'", expectedOutput, string(output)) @@ -295,7 +295,7 @@ func TestCNIPlugin(t *testing.T) { } func TestLoNetNonNil(t *testing.T) { - if conf := getLoNetwork("", ""); conf == nil { + if conf := getLoNetwork(nil); conf == nil { t.Error("Expected non-nil lo network") } } diff --git a/pkg/kubelet/network/cni/cni_windows.go b/pkg/kubelet/network/cni/cni_windows.go index 9a0b17c8f11..81cc5ca1fe1 100644 --- a/pkg/kubelet/network/cni/cni_windows.go +++ b/pkg/kubelet/network/cni/cni_windows.go @@ -27,7 +27,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/network" ) -func getLoNetwork(binDir, vendorDirPrefix string) *cniNetwork { +func getLoNetwork(binDirs []string) *cniNetwork { return nil }