From 8739ade3fada54317e86d5022340ca4f002ac829 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 11 Jun 2019 09:21:31 -0500 Subject: [PATCH] kubelet: add CNI cache dir option and plumb through to CNI and kubenet libcni 0.7.0 caches ADD operation results and allows the runtime to retrieve these from the cache. In case the user wants a different cache directory than the defaul, plumb that through like we do for --cni-bin-dir and --cni-conf-dir. --- cmd/kubeadm/app/cmd/phases/reset/cleanupnode.go | 2 +- cmd/kubelet/app/options/container_runtime.go | 5 +++-- cmd/kubelet/app/server.go | 1 + pkg/kubelet/config/flags.go | 4 ++++ pkg/kubelet/dockershim/docker_service.go | 6 ++++-- pkg/kubelet/dockershim/network/cni/cni.go | 5 ++++- pkg/kubelet/dockershim/network/cni/cni_test.go | 3 ++- .../dockershim/network/kubenet/kubenet_linux.go | 5 ++++- .../network/kubenet/kubenet_unsupported.go | 2 +- pkg/kubelet/kubelet.go | 1 + test/e2e_node/services/kubelet.go | 17 ++++++++++++++++- 11 files changed, 41 insertions(+), 10 deletions(-) diff --git a/cmd/kubeadm/app/cmd/phases/reset/cleanupnode.go b/cmd/kubeadm/app/cmd/phases/reset/cleanupnode.go index 3c880da1f3b..410cc5639fb 100644 --- a/cmd/kubeadm/app/cmd/phases/reset/cleanupnode.go +++ b/cmd/kubeadm/app/cmd/phases/reset/cleanupnode.go @@ -82,7 +82,7 @@ func runCleanupNode(c workflow.RunData) error { klog.Errorf("[reset] Failed to remove containers: %v", err) } - r.AddDirsToClean("/etc/cni/net.d", "/var/lib/dockershim", "/var/run/kubernetes") + r.AddDirsToClean("/etc/cni/net.d", "/var/lib/dockershim", "/var/run/kubernetes", "/var/lib/cni") // Remove contents from the config and pki directories klog.V(1).Infoln("[reset] Removing contents from the config and pki directories") diff --git a/cmd/kubelet/app/options/container_runtime.go b/cmd/kubelet/app/options/container_runtime.go index a811577d958..fba438a6196 100644 --- a/cmd/kubelet/app/options/container_runtime.go +++ b/cmd/kubelet/app/options/container_runtime.go @@ -54,7 +54,8 @@ func NewContainerRuntimeOptions() *config.ContainerRuntimeOptions { ExperimentalDockershim: false, //Alpha feature - CNIBinDir: "/opt/cni/bin", - CNIConfDir: "/etc/cni/net.d", + CNIBinDir: "/opt/cni/bin", + CNIConfDir: "/etc/cni/net.d", + CNICacheDir: "/var/lib/cni/cache", } } diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 131d4310160..807d742cb9d 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -1251,6 +1251,7 @@ func RunDockershim(f *options.KubeletFlags, c *kubeletconfiginternal.KubeletConf PluginName: r.NetworkPluginName, PluginConfDir: r.CNIConfDir, PluginBinDirString: r.CNIBinDir, + PluginCacheDir: r.CNICacheDir, MTU: int(r.NetworkPluginMTU), } diff --git a/pkg/kubelet/config/flags.go b/pkg/kubelet/config/flags.go index 4b5494e7866..bb5a97122f0 100644 --- a/pkg/kubelet/config/flags.go +++ b/pkg/kubelet/config/flags.go @@ -73,6 +73,9 @@ type ContainerRuntimeOptions struct { // CNIBinDir is the full path of the directory in which to search for // CNI plugin binaries CNIBinDir string + // CNICacheDir is the full path of the directory in which CNI should store + // cache files + CNICacheDir string } func (s *ContainerRuntimeOptions) AddFlags(fs *pflag.FlagSet) { @@ -96,5 +99,6 @@ func (s *ContainerRuntimeOptions) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.NetworkPluginName, "network-plugin", s.NetworkPluginName, fmt.Sprintf(" The name of the network plugin to be invoked for various events in kubelet/pod lifecycle. %s", dockerOnlyWarning)) fs.StringVar(&s.CNIConfDir, "cni-conf-dir", s.CNIConfDir, fmt.Sprintf(" The full path of the directory in which to search for CNI config files. %s", dockerOnlyWarning)) fs.StringVar(&s.CNIBinDir, "cni-bin-dir", s.CNIBinDir, fmt.Sprintf(" A comma-separated list of full paths of directories in which to search for CNI plugin binaries. %s", dockerOnlyWarning)) + fs.StringVar(&s.CNICacheDir, "cni-cache-dir", s.CNICacheDir, fmt.Sprintf(" The full path of the directory in which CNI should store cache files. %s", dockerOnlyWarning)) fs.Int32Var(&s.NetworkPluginMTU, "network-plugin-mtu", s.NetworkPluginMTU, fmt.Sprintf(" The MTU to be passed to the network plugin, to override the default. Set to 0 to use the default 1460 MTU. %s", dockerOnlyWarning)) } diff --git a/pkg/kubelet/dockershim/docker_service.go b/pkg/kubelet/dockershim/docker_service.go index ba1d17a89ac..16cd9fda9b8 100644 --- a/pkg/kubelet/dockershim/docker_service.go +++ b/pkg/kubelet/dockershim/docker_service.go @@ -123,6 +123,8 @@ type NetworkPluginSettings struct { // Depending on the plugin, this may be an optional field, eg: kubenet // generates its own plugin conf. PluginConfDir string + // PluginCacheDir is the directory in which CNI should store cache files. + PluginCacheDir string // MTU is the desired MTU for network devices created by the plugin. MTU int } @@ -239,8 +241,8 @@ func NewDockerService(config *ClientConfig, podSandboxImage string, streamingCon // dockershim currently only supports CNI plugins. pluginSettings.PluginBinDirs = cni.SplitDirs(pluginSettings.PluginBinDirString) - cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, pluginSettings.PluginBinDirs) - cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDirs)) + cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, pluginSettings.PluginCacheDir, pluginSettings.PluginBinDirs) + cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDirs, pluginSettings.PluginCacheDir)) netHost := &dockerNetworkHost{ &namespaceGetter{ds}, &portMappingGetter{ds}, diff --git a/pkg/kubelet/dockershim/network/cni/cni.go b/pkg/kubelet/dockershim/network/cni/cni.go index f391ae68181..3e79c16c93f 100644 --- a/pkg/kubelet/dockershim/network/cni/cni.go +++ b/pkg/kubelet/dockershim/network/cni/cni.go @@ -60,6 +60,7 @@ type cniNetworkPlugin struct { nsenterPath string confDir string binDirs []string + cacheDir string podCidr string } @@ -116,7 +117,7 @@ func SplitDirs(dirs string) []string { return strings.Split(dirs, ",") } -func ProbeNetworkPlugins(confDir string, binDirs []string) []network.NetworkPlugin { +func ProbeNetworkPlugins(confDir, cacheDir string, binDirs []string) []network.NetworkPlugin { old := binDirs binDirs = make([]string, 0, len(binDirs)) for _, dir := range old { @@ -131,6 +132,7 @@ func ProbeNetworkPlugins(confDir string, binDirs []string) []network.NetworkPlug execer: utilexec.New(), confDir: confDir, binDirs: binDirs, + cacheDir: cacheDir, } // sync NetworkConfig in best effort during probing. @@ -362,6 +364,7 @@ func (plugin *cniNetworkPlugin) buildCNIRuntimeConf(podName string, podNs string ContainerID: podSandboxID.ID, NetNS: podNetnsPath, IfName: network.DefaultInterfaceName, + CacheDir: plugin.cacheDir, Args: [][2]string{ {"IgnoreUnknown", "1"}, {"K8S_POD_NAMESPACE", podNs}, diff --git a/pkg/kubelet/dockershim/network/cni/cni_test.go b/pkg/kubelet/dockershim/network/cni/cni_test.go index 18593a3abe7..aa8fe117d21 100644 --- a/pkg/kubelet/dockershim/network/cni/cni_test.go +++ b/pkg/kubelet/dockershim/network/cni/cni_test.go @@ -194,6 +194,7 @@ func TestCNIPlugin(t *testing.T) { testConfDir := path.Join(tmpDir, "etc", "cni", "net.d") testBinDir := path.Join(tmpDir, "opt", "cni", "bin") testDataDir := path.Join(tmpDir, "output") + testCacheDir := path.Join(tmpDir, "var", "lib", "cni", "cache") defer tearDownPlugin(tmpDir) inputFile, outputFile, outputEnv := installPluginUnderTest(t, testBinDir, testConfDir, testDataDir, binName, netName, podIP) @@ -207,7 +208,7 @@ func TestCNIPlugin(t *testing.T) { NetnsPath: "/proc/12345/ns/net", }} - plugins := ProbeNetworkPlugins(testConfDir, []string{testBinDir}) + plugins := ProbeNetworkPlugins(testConfDir, testCacheDir, []string{testBinDir}) if len(plugins) != 1 { t.Fatalf("Expected only one network plugin, got %d", len(plugins)) } diff --git a/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go b/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go index d1b7a871532..a22719cf45d 100644 --- a/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go +++ b/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go @@ -96,9 +96,10 @@ type kubenetNetworkPlugin struct { nonMasqueradeCIDR string podCidr string gateway net.IP + cacheDir string } -func NewPlugin(networkPluginDirs []string) network.NetworkPlugin { +func NewPlugin(networkPluginDirs []string, cacheDir string) network.NetworkPlugin { protocol := utiliptables.ProtocolIpv4 execer := utilexec.New() dbus := utildbus.New() @@ -113,6 +114,7 @@ func NewPlugin(networkPluginDirs []string) network.NetworkPlugin { hostportSyncer: hostport.NewHostportSyncer(iptInterface), hostportManager: hostport.NewHostportManager(iptInterface), nonMasqueradeCIDR: "10.0.0.0/8", + cacheDir: cacheDir, } } @@ -558,6 +560,7 @@ func (plugin *kubenetNetworkPlugin) buildCNIRuntimeConf(ifName string, id kubeco ContainerID: id.ID, NetNS: netnsPath, IfName: ifName, + CacheDir: plugin.cacheDir, }, nil } diff --git a/pkg/kubelet/dockershim/network/kubenet/kubenet_unsupported.go b/pkg/kubelet/dockershim/network/kubenet/kubenet_unsupported.go index d2a59126b3e..5b7ef00cb26 100644 --- a/pkg/kubelet/dockershim/network/kubenet/kubenet_unsupported.go +++ b/pkg/kubelet/dockershim/network/kubenet/kubenet_unsupported.go @@ -30,7 +30,7 @@ type kubenetNetworkPlugin struct { network.NoopNetworkPlugin } -func NewPlugin(networkPluginDirs []string) network.NetworkPlugin { +func NewPlugin(networkPluginDirs []string, cacheDir string) network.NetworkPlugin { return &kubenetNetworkPlugin{} } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index fad4d0f6af2..acdbf364451 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -607,6 +607,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, PluginName: crOptions.NetworkPluginName, PluginConfDir: crOptions.CNIConfDir, PluginBinDirString: crOptions.CNIBinDir, + PluginCacheDir: crOptions.CNICacheDir, MTU: int(crOptions.NetworkPluginMTU), } diff --git a/test/e2e_node/services/kubelet.go b/test/e2e_node/services/kubelet.go index 5392c8b657b..f3188202ea4 100644 --- a/test/e2e_node/services/kubelet.go +++ b/test/e2e_node/services/kubelet.go @@ -289,10 +289,16 @@ func (e *E2EServices) startKubelet() (*server, error) { return nil, err } + cniCacheDir, err := getCNICacheDirectory() + if err != nil { + return nil, err + } + cmdArgs = append(cmdArgs, "--network-plugin=kubenet", "--cni-bin-dir", cniBinDir, - "--cni-conf-dir", cniConfDir) + "--cni-conf-dir", cniConfDir, + "--cni-cache-dir", cniCacheDir) // Keep hostname override for convenience. if framework.TestContext.NodeName != "" { // If node name is specified, set hostname override. @@ -467,6 +473,15 @@ func getCNIConfDirectory() (string, error) { return filepath.Join(cwd, "cni", "net.d"), nil } +// getCNICacheDirectory returns CNI Cache directory. +func getCNICacheDirectory() (string, error) { + cwd, err := os.Getwd() + if err != nil { + return "", err + } + return filepath.Join(cwd, "cni", "cache"), nil +} + // getDynamicConfigDir returns the directory for dynamic Kubelet configuration func getDynamicConfigDir() (string, error) { cwd, err := os.Getwd()