diff --git a/cmd/kubeadm/app/cmd/phases/reset/cleanupnode.go b/cmd/kubeadm/app/cmd/phases/reset/cleanupnode.go index 3c880da1f3b..410cc5639fb 100644 --- a/cmd/kubeadm/app/cmd/phases/reset/cleanupnode.go +++ b/cmd/kubeadm/app/cmd/phases/reset/cleanupnode.go @@ -82,7 +82,7 @@ func runCleanupNode(c workflow.RunData) error { klog.Errorf("[reset] Failed to remove containers: %v", err) } - r.AddDirsToClean("/etc/cni/net.d", "/var/lib/dockershim", "/var/run/kubernetes") + r.AddDirsToClean("/etc/cni/net.d", "/var/lib/dockershim", "/var/run/kubernetes", "/var/lib/cni") // Remove contents from the config and pki directories klog.V(1).Infoln("[reset] Removing contents from the config and pki directories") diff --git a/cmd/kubelet/app/options/container_runtime.go b/cmd/kubelet/app/options/container_runtime.go index a811577d958..fba438a6196 100644 --- a/cmd/kubelet/app/options/container_runtime.go +++ b/cmd/kubelet/app/options/container_runtime.go @@ -54,7 +54,8 @@ func NewContainerRuntimeOptions() *config.ContainerRuntimeOptions { ExperimentalDockershim: false, //Alpha feature - CNIBinDir: "/opt/cni/bin", - CNIConfDir: "/etc/cni/net.d", + CNIBinDir: "/opt/cni/bin", + CNIConfDir: "/etc/cni/net.d", + CNICacheDir: "/var/lib/cni/cache", } } diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 96edb62e7a3..614d9035aef 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -1248,6 +1248,7 @@ func RunDockershim(f *options.KubeletFlags, c *kubeletconfiginternal.KubeletConf PluginName: r.NetworkPluginName, PluginConfDir: r.CNIConfDir, PluginBinDirString: r.CNIBinDir, + PluginCacheDir: r.CNICacheDir, MTU: int(r.NetworkPluginMTU), } diff --git a/go.mod b/go.mod index 5b509b83c84..5278f9c78b6 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/containerd/console v0.0.0-20170925154832-84eeaae905fa // indirect github.com/containerd/containerd v1.0.2 // indirect github.com/containerd/typeurl v0.0.0-20190228175220-2a93cfde8c20 // indirect - github.com/containernetworking/cni v0.6.0 + github.com/containernetworking/cni v0.7.1 github.com/coreos/etcd v3.3.13+incompatible github.com/coreos/go-semver v0.3.0 github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 @@ -225,7 +225,7 @@ replace ( github.com/containerd/console => github.com/containerd/console v0.0.0-20170925154832-84eeaae905fa github.com/containerd/containerd => github.com/containerd/containerd v1.0.2 github.com/containerd/typeurl => github.com/containerd/typeurl v0.0.0-20190228175220-2a93cfde8c20 - github.com/containernetworking/cni => github.com/containernetworking/cni v0.6.0 + github.com/containernetworking/cni => github.com/containernetworking/cni v0.7.1 github.com/coreos/bbolt => github.com/coreos/bbolt v1.3.1-coreos.6 github.com/coreos/etcd => github.com/coreos/etcd v3.3.13+incompatible github.com/coreos/go-etcd => github.com/coreos/go-etcd v2.0.0+incompatible diff --git a/go.sum b/go.sum index 52424b1e150..33957eae6cf 100644 --- a/go.sum +++ b/go.sum @@ -75,8 +75,8 @@ github.com/containerd/containerd v1.0.2 h1:AcqeeOunmUuo2CvPPtHMhWn7mi54clu+j9yqX github.com/containerd/containerd v1.0.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/typeurl v0.0.0-20190228175220-2a93cfde8c20 h1:14r0i3IeJj6zkNLigAJiv/TWSR8EY+pxIjv5tFiT+n8= github.com/containerd/typeurl v0.0.0-20190228175220-2a93cfde8c20/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= -github.com/containernetworking/cni v0.6.0 h1:FXICGBZNMtdHlW65trpoHviHctQD3seWhRRcqp2hMOU= -github.com/containernetworking/cni v0.6.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v0.7.1 h1:fE3r16wpSEyaqY4Z4oFrLMmIGfBYIKpPrHK31EJ9FzE= +github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/coreos/bbolt v1.3.1-coreos.6 h1:uTXKg9gY70s9jMAKdfljFQcuh4e/BXOM+V+d00KFj3A= github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ= diff --git a/pkg/kubelet/config/flags.go b/pkg/kubelet/config/flags.go index 4b5494e7866..bb5a97122f0 100644 --- a/pkg/kubelet/config/flags.go +++ b/pkg/kubelet/config/flags.go @@ -73,6 +73,9 @@ type ContainerRuntimeOptions struct { // CNIBinDir is the full path of the directory in which to search for // CNI plugin binaries CNIBinDir string + // CNICacheDir is the full path of the directory in which CNI should store + // cache files + CNICacheDir string } func (s *ContainerRuntimeOptions) AddFlags(fs *pflag.FlagSet) { @@ -96,5 +99,6 @@ func (s *ContainerRuntimeOptions) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.NetworkPluginName, "network-plugin", s.NetworkPluginName, fmt.Sprintf(" The name of the network plugin to be invoked for various events in kubelet/pod lifecycle. %s", dockerOnlyWarning)) fs.StringVar(&s.CNIConfDir, "cni-conf-dir", s.CNIConfDir, fmt.Sprintf(" The full path of the directory in which to search for CNI config files. %s", dockerOnlyWarning)) fs.StringVar(&s.CNIBinDir, "cni-bin-dir", s.CNIBinDir, fmt.Sprintf(" A comma-separated list of full paths of directories in which to search for CNI plugin binaries. %s", dockerOnlyWarning)) + fs.StringVar(&s.CNICacheDir, "cni-cache-dir", s.CNICacheDir, fmt.Sprintf(" The full path of the directory in which CNI should store cache files. %s", dockerOnlyWarning)) fs.Int32Var(&s.NetworkPluginMTU, "network-plugin-mtu", s.NetworkPluginMTU, fmt.Sprintf(" The MTU to be passed to the network plugin, to override the default. Set to 0 to use the default 1460 MTU. %s", dockerOnlyWarning)) } diff --git a/pkg/kubelet/dockershim/docker_service.go b/pkg/kubelet/dockershim/docker_service.go index ba1d17a89ac..16cd9fda9b8 100644 --- a/pkg/kubelet/dockershim/docker_service.go +++ b/pkg/kubelet/dockershim/docker_service.go @@ -123,6 +123,8 @@ type NetworkPluginSettings struct { // Depending on the plugin, this may be an optional field, eg: kubenet // generates its own plugin conf. PluginConfDir string + // PluginCacheDir is the directory in which CNI should store cache files. + PluginCacheDir string // MTU is the desired MTU for network devices created by the plugin. MTU int } @@ -239,8 +241,8 @@ func NewDockerService(config *ClientConfig, podSandboxImage string, streamingCon // dockershim currently only supports CNI plugins. pluginSettings.PluginBinDirs = cni.SplitDirs(pluginSettings.PluginBinDirString) - cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, pluginSettings.PluginBinDirs) - cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDirs)) + cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, pluginSettings.PluginCacheDir, pluginSettings.PluginBinDirs) + cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDirs, pluginSettings.PluginCacheDir)) netHost := &dockerNetworkHost{ &namespaceGetter{ds}, &portMappingGetter{ds}, diff --git a/pkg/kubelet/dockershim/network/cni/cni.go b/pkg/kubelet/dockershim/network/cni/cni.go index c2f5b257346..3e79c16c93f 100644 --- a/pkg/kubelet/dockershim/network/cni/cni.go +++ b/pkg/kubelet/dockershim/network/cni/cni.go @@ -17,6 +17,7 @@ limitations under the License. package cni import ( + "context" "encoding/json" "errors" "fmt" @@ -59,6 +60,7 @@ type cniNetworkPlugin struct { nsenterPath string confDir string binDirs []string + cacheDir string podCidr string } @@ -115,7 +117,7 @@ func SplitDirs(dirs string) []string { return strings.Split(dirs, ",") } -func ProbeNetworkPlugins(confDir string, binDirs []string) []network.NetworkPlugin { +func ProbeNetworkPlugins(confDir, cacheDir string, binDirs []string) []network.NetworkPlugin { old := binDirs binDirs = make([]string, 0, len(binDirs)) for _, dir := range old { @@ -130,6 +132,7 @@ func ProbeNetworkPlugins(confDir string, binDirs []string) []network.NetworkPlug execer: utilexec.New(), confDir: confDir, binDirs: binDirs, + cacheDir: cacheDir, } // sync NetworkConfig in best effort during probing. @@ -326,7 +329,7 @@ func (plugin *cniNetworkPlugin) addToNetwork(network *cniNetwork, podName string pdesc := podDesc(podNamespace, podName, podSandboxID) netConf, cniNet := network.NetworkConfig, network.CNIConfig klog.V(4).Infof("Adding %s to network %s/%s netns %q", pdesc, netConf.Plugins[0].Network.Type, netConf.Name, podNetnsPath) - res, err := cniNet.AddNetworkList(netConf, rt) + res, err := cniNet.AddNetworkList(context.TODO(), netConf, rt) if err != nil { klog.Errorf("Error adding %s to network %s/%s: %v", pdesc, netConf.Plugins[0].Network.Type, netConf.Name, err) return nil, err @@ -345,7 +348,7 @@ func (plugin *cniNetworkPlugin) deleteFromNetwork(network *cniNetwork, podName s pdesc := podDesc(podNamespace, podName, podSandboxID) netConf, cniNet := network.NetworkConfig, network.CNIConfig klog.V(4).Infof("Deleting %s from network %s/%s netns %q", pdesc, netConf.Plugins[0].Network.Type, netConf.Name, podNetnsPath) - err = cniNet.DelNetworkList(netConf, rt) + err = cniNet.DelNetworkList(context.TODO(), netConf, rt) // The pod may not get deleted successfully at the first time. // Ignore "no such file or directory" error in case the network has already been deleted in previous attempts. if err != nil && !strings.Contains(err.Error(), "no such file or directory") { @@ -361,6 +364,7 @@ func (plugin *cniNetworkPlugin) buildCNIRuntimeConf(podName string, podNs string ContainerID: podSandboxID.ID, NetNS: podNetnsPath, IfName: network.DefaultInterfaceName, + CacheDir: plugin.cacheDir, Args: [][2]string{ {"IgnoreUnknown", "1"}, {"K8S_POD_NAMESPACE", podNs}, diff --git a/pkg/kubelet/dockershim/network/cni/cni_test.go b/pkg/kubelet/dockershim/network/cni/cni_test.go index 997bf973b7f..aa8fe117d21 100644 --- a/pkg/kubelet/dockershim/network/cni/cni_test.go +++ b/pkg/kubelet/dockershim/network/cni/cni_test.go @@ -20,6 +20,7 @@ package cni import ( "bytes" + "context" "encoding/json" "fmt" "io/ioutil" @@ -48,7 +49,7 @@ import ( ) // Returns .in file path, .out file path, and .env file path -func installPluginUnderTest(t *testing.T, testBinDir, testConfDir, testDataDir, binName string, confName string) (string, string, string) { +func installPluginUnderTest(t *testing.T, testBinDir, testConfDir, testDataDir, binName string, confName, podIP string) (string, string, string) { for _, dir := range []string{testBinDir, testConfDir, testDataDir} { err := os.MkdirAll(dir, 0777) if err != nil { @@ -56,12 +57,14 @@ func installPluginUnderTest(t *testing.T, testBinDir, testConfDir, testDataDir, } } + const cniVersion = "0.2.0" + confFile := path.Join(testConfDir, confName+".conf") f, err := os.Create(confFile) if err != nil { t.Fatalf("Failed to install plugin %s: %v", confFile, err) } - networkConfig := fmt.Sprintf(`{ "name": "%s", "type": "%s", "capabilities": {"portMappings": true, "bandwidth": true, "ipRanges": true} }`, confName, binName) + networkConfig := fmt.Sprintf(`{ "cniVersion": "%s", "name": "%s", "type": "%s", "capabilities": {"portMappings": true, "bandwidth": true, "ipRanges": true} }`, cniVersion, confName, binName) _, err = f.WriteString(networkConfig) if err != nil { t.Fatalf("Failed to write network config file (%v)", err) @@ -78,8 +81,8 @@ echo "%@" >> {{.OutputEnv}} export $(echo ${CNI_ARGS} | sed 's/;/ /g') &> /dev/null mkdir -p {{.OutputDir}} &> /dev/null echo -n "$CNI_COMMAND $CNI_NETNS $K8S_POD_NAMESPACE $K8S_POD_NAME $K8S_POD_INFRA_CONTAINER_ID" >& {{.OutputFile}} -echo -n "{ \"ip4\": { \"ip\": \"10.1.0.23/24\" } }" -` +echo -n "{ \"cniVersion\": \"{{.CNIVersion}}\", \"ip4\": { \"ip\": \"{{.PodIP}}/24\" } }"` + inputFile := path.Join(testDataDir, binName+".in") outputFile := path.Join(testDataDir, binName+".out") envFile := path.Join(testDataDir, binName+".env") @@ -88,6 +91,8 @@ echo -n "{ \"ip4\": { \"ip\": \"10.1.0.23/24\" } }" "OutputFile": outputFile, "OutputEnv": envFile, "OutputDir": testDataDir, + "CNIVersion": cniVersion, + "PodIP": podIP, } tObj := template.Must(template.New("test").Parse(execScriptTempl)) @@ -189,8 +194,9 @@ func TestCNIPlugin(t *testing.T) { testConfDir := path.Join(tmpDir, "etc", "cni", "net.d") testBinDir := path.Join(tmpDir, "opt", "cni", "bin") testDataDir := path.Join(tmpDir, "output") + testCacheDir := path.Join(tmpDir, "var", "lib", "cni", "cache") defer tearDownPlugin(tmpDir) - inputFile, outputFile, outputEnv := installPluginUnderTest(t, testBinDir, testConfDir, testDataDir, binName, netName) + inputFile, outputFile, outputEnv := installPluginUnderTest(t, testBinDir, testConfDir, testDataDir, binName, netName, podIP) containerID := kubecontainer.ContainerID{Type: "test", ID: "test_infra_container"} pods := []*containertest.FakePod{{ @@ -202,7 +208,7 @@ func TestCNIPlugin(t *testing.T) { NetnsPath: "/proc/12345/ns/net", }} - plugins := ProbeNetworkPlugins(testConfDir, []string{testBinDir}) + plugins := ProbeNetworkPlugins(testConfDir, testCacheDir, []string{testBinDir}) if len(plugins) != 1 { t.Fatalf("Expected only one network plugin, got %d", len(plugins)) } @@ -217,7 +223,7 @@ func TestCNIPlugin(t *testing.T) { cniPlugin.execer = fexec cniPlugin.loNetwork.CNIConfig = mockLoCNI - mockLoCNI.On("AddNetworkList", cniPlugin.loNetwork.NetworkConfig, mock.AnythingOfType("*libcni.RuntimeConf")).Return(&types020.Result{IP4: &types020.IPConfig{IP: net.IPNet{IP: []byte{127, 0, 0, 1}}}}, nil) + mockLoCNI.On("AddNetworkList", context.TODO(), cniPlugin.loNetwork.NetworkConfig, mock.AnythingOfType("*libcni.RuntimeConf")).Return(&types020.Result{IP4: &types020.IPConfig{IP: net.IPNet{IP: []byte{127, 0, 0, 1}}}}, nil) // Check that status returns an error if err := cniPlugin.Status(); err == nil { diff --git a/pkg/kubelet/dockershim/network/cni/testing/mock_cni.go b/pkg/kubelet/dockershim/network/cni/testing/mock_cni.go index 1e5c0d00ee2..afc3a510f09 100644 --- a/pkg/kubelet/dockershim/network/cni/testing/mock_cni.go +++ b/pkg/kubelet/dockershim/network/cni/testing/mock_cni.go @@ -19,6 +19,7 @@ limitations under the License. package mock_cni import ( + "context" "github.com/containernetworking/cni/libcni" "github.com/containernetworking/cni/pkg/types" "github.com/stretchr/testify/mock" @@ -28,22 +29,52 @@ type MockCNI struct { mock.Mock } -func (m *MockCNI) AddNetwork(net *libcni.NetworkConfig, rt *libcni.RuntimeConf) (types.Result, error) { +func (m *MockCNI) AddNetwork(ctx context.Context, net *libcni.NetworkConfig, rt *libcni.RuntimeConf) (types.Result, error) { + args := m.Called(ctx, net, rt) + return args.Get(0).(types.Result), args.Error(1) +} + +func (m *MockCNI) DelNetwork(ctx context.Context, net *libcni.NetworkConfig, rt *libcni.RuntimeConf) error { + args := m.Called(ctx, net, rt) + return args.Error(0) +} + +func (m *MockCNI) DelNetworkList(ctx context.Context, net *libcni.NetworkConfigList, rt *libcni.RuntimeConf) error { + args := m.Called(ctx, net, rt) + return args.Error(0) +} + +func (m *MockCNI) GetNetworkListCachedResult(net *libcni.NetworkConfigList, rt *libcni.RuntimeConf) (types.Result, error) { args := m.Called(net, rt) return args.Get(0).(types.Result), args.Error(1) } -func (m *MockCNI) DelNetwork(net *libcni.NetworkConfig, rt *libcni.RuntimeConf) error { - args := m.Called(net, rt) - return args.Error(0) -} - -func (m *MockCNI) DelNetworkList(net *libcni.NetworkConfigList, rt *libcni.RuntimeConf) error { - args := m.Called(net, rt) - return args.Error(0) -} - -func (m *MockCNI) AddNetworkList(net *libcni.NetworkConfigList, rt *libcni.RuntimeConf) (types.Result, error) { - args := m.Called(net, rt) +func (m *MockCNI) AddNetworkList(ctx context.Context, net *libcni.NetworkConfigList, rt *libcni.RuntimeConf) (types.Result, error) { + args := m.Called(ctx, net, rt) return args.Get(0).(types.Result), args.Error(1) } + +func (m *MockCNI) CheckNetworkList(ctx context.Context, net *libcni.NetworkConfigList, rt *libcni.RuntimeConf) error { + args := m.Called(ctx, net, rt) + return args.Error(0) +} + +func (m *MockCNI) CheckNetwork(ctx context.Context, net *libcni.NetworkConfig, rt *libcni.RuntimeConf) error { + args := m.Called(ctx, net, rt) + return args.Error(0) +} + +func (m *MockCNI) GetNetworkCachedResult(net *libcni.NetworkConfig, rt *libcni.RuntimeConf) (types.Result, error) { + args := m.Called(net, rt) + return args.Get(0).(types.Result), args.Error(0) +} + +func (m *MockCNI) ValidateNetworkList(ctx context.Context, net *libcni.NetworkConfigList) ([]string, error) { + args := m.Called(ctx, net) + return args.Get(0).([]string), args.Error(0) +} + +func (m *MockCNI) ValidateNetwork(ctx context.Context, net *libcni.NetworkConfig) ([]string, error) { + args := m.Called(ctx, net) + return args.Get(0).([]string), args.Error(0) +} diff --git a/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go b/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go index acd47ee2fbd..a22719cf45d 100644 --- a/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go +++ b/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go @@ -19,6 +19,7 @@ limitations under the License. package kubenet import ( + "context" "fmt" "io/ioutil" "net" @@ -95,9 +96,10 @@ type kubenetNetworkPlugin struct { nonMasqueradeCIDR string podCidr string gateway net.IP + cacheDir string } -func NewPlugin(networkPluginDirs []string) network.NetworkPlugin { +func NewPlugin(networkPluginDirs []string, cacheDir string) network.NetworkPlugin { protocol := utiliptables.ProtocolIpv4 execer := utilexec.New() dbus := utildbus.New() @@ -112,6 +114,7 @@ func NewPlugin(networkPluginDirs []string) network.NetworkPlugin { hostportSyncer: hostport.NewHostportSyncer(iptInterface), hostportManager: hostport.NewHostportManager(iptInterface), nonMasqueradeCIDR: "10.0.0.0/8", + cacheDir: cacheDir, } } @@ -557,6 +560,7 @@ func (plugin *kubenetNetworkPlugin) buildCNIRuntimeConf(ifName string, id kubeco ContainerID: id.ID, NetNS: netnsPath, IfName: ifName, + CacheDir: plugin.cacheDir, }, nil } @@ -570,7 +574,7 @@ func (plugin *kubenetNetworkPlugin) addContainerToNetwork(config *libcni.Network // The network plugin can take up to 3 seconds to execute, // so yield the lock while it runs. plugin.mu.Unlock() - res, err := plugin.cniConfig.AddNetwork(config, rt) + res, err := plugin.cniConfig.AddNetwork(context.TODO(), config, rt) plugin.mu.Lock() if err != nil { return nil, fmt.Errorf("Error adding container to network: %v", err) @@ -585,7 +589,7 @@ func (plugin *kubenetNetworkPlugin) delContainerFromNetwork(config *libcni.Netwo } klog.V(3).Infof("Removing %s/%s from '%s' with CNI '%s' plugin and runtime: %+v", namespace, name, config.Network.Name, config.Network.Type, rt) - err = plugin.cniConfig.DelNetwork(config, rt) + err = plugin.cniConfig.DelNetwork(context.TODO(), config, rt) // The pod may not get deleted successfully at the first time. // Ignore "no such file or directory" error in case the network has already been deleted in previous attempts. if err != nil && !strings.Contains(err.Error(), "no such file or directory") { diff --git a/pkg/kubelet/dockershim/network/kubenet/kubenet_linux_test.go b/pkg/kubelet/dockershim/network/kubenet/kubenet_linux_test.go index 0c50149a98c..cc91b74f624 100644 --- a/pkg/kubelet/dockershim/network/kubenet/kubenet_linux_test.go +++ b/pkg/kubelet/dockershim/network/kubenet/kubenet_linux_test.go @@ -143,7 +143,7 @@ func TestTeardownCallsShaper(t *testing.T) { kubenet.bandwidthShaper = fshaper kubenet.hostportSyncer = hostporttest.NewFakeHostportSyncer() - mockcni.On("DelNetwork", mock.AnythingOfType("*libcni.NetworkConfig"), mock.AnythingOfType("*libcni.RuntimeConf")).Return(nil) + mockcni.On("DelNetwork", mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("*libcni.NetworkConfig"), mock.AnythingOfType("*libcni.RuntimeConf")).Return(nil) details := make(map[string]interface{}) details[network.NET_PLUGIN_EVENT_POD_CIDR_CHANGE_DETAIL_CIDR] = "10.0.0.1/24" @@ -247,7 +247,7 @@ func TestTearDownWithoutRuntime(t *testing.T) { existingContainerID := kubecontainer.BuildContainerID("docker", "123") kubenet.podIPs[existingContainerID] = tc.ip - mockcni.On("DelNetwork", mock.AnythingOfType("*libcni.NetworkConfig"), mock.AnythingOfType("*libcni.RuntimeConf")).Return(nil) + mockcni.On("DelNetwork", mock.AnythingOfType("*context.emptyCtx"), mock.AnythingOfType("*libcni.NetworkConfig"), mock.AnythingOfType("*libcni.RuntimeConf")).Return(nil) if err := kubenet.TearDownPod("namespace", "name", existingContainerID); err != nil { t.Fatalf("Unexpected error in TearDownPod: %v", err) diff --git a/pkg/kubelet/dockershim/network/kubenet/kubenet_unsupported.go b/pkg/kubelet/dockershim/network/kubenet/kubenet_unsupported.go index d2a59126b3e..5b7ef00cb26 100644 --- a/pkg/kubelet/dockershim/network/kubenet/kubenet_unsupported.go +++ b/pkg/kubelet/dockershim/network/kubenet/kubenet_unsupported.go @@ -30,7 +30,7 @@ type kubenetNetworkPlugin struct { network.NoopNetworkPlugin } -func NewPlugin(networkPluginDirs []string) network.NetworkPlugin { +func NewPlugin(networkPluginDirs []string, cacheDir string) network.NetworkPlugin { return &kubenetNetworkPlugin{} } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 0cbb08b77ae..832763f67ef 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -608,6 +608,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, PluginName: crOptions.NetworkPluginName, PluginConfDir: crOptions.CNIConfDir, PluginBinDirString: crOptions.CNIBinDir, + PluginCacheDir: crOptions.CNICacheDir, MTU: int(crOptions.NetworkPluginMTU), } diff --git a/test/e2e_node/services/kubelet.go b/test/e2e_node/services/kubelet.go index 5392c8b657b..f3188202ea4 100644 --- a/test/e2e_node/services/kubelet.go +++ b/test/e2e_node/services/kubelet.go @@ -289,10 +289,16 @@ func (e *E2EServices) startKubelet() (*server, error) { return nil, err } + cniCacheDir, err := getCNICacheDirectory() + if err != nil { + return nil, err + } + cmdArgs = append(cmdArgs, "--network-plugin=kubenet", "--cni-bin-dir", cniBinDir, - "--cni-conf-dir", cniConfDir) + "--cni-conf-dir", cniConfDir, + "--cni-cache-dir", cniCacheDir) // Keep hostname override for convenience. if framework.TestContext.NodeName != "" { // If node name is specified, set hostname override. @@ -467,6 +473,15 @@ func getCNIConfDirectory() (string, error) { return filepath.Join(cwd, "cni", "net.d"), nil } +// getCNICacheDirectory returns CNI Cache directory. +func getCNICacheDirectory() (string, error) { + cwd, err := os.Getwd() + if err != nil { + return "", err + } + return filepath.Join(cwd, "cni", "cache"), nil +} + // getDynamicConfigDir returns the directory for dynamic Kubelet configuration func getDynamicConfigDir() (string, error) { cwd, err := os.Getwd() diff --git a/vendor/github.com/containernetworking/cni/libcni/api.go b/vendor/github.com/containernetworking/cni/libcni/api.go index a23cbb2c57b..0f14d3427e9 100644 --- a/vendor/github.com/containernetworking/cni/libcni/api.go +++ b/vendor/github.com/containernetworking/cni/libcni/api.go @@ -15,7 +15,12 @@ package libcni import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" "os" + "path/filepath" "strings" "github.com/containernetworking/cni/pkg/invoke" @@ -23,6 +28,14 @@ import ( "github.com/containernetworking/cni/pkg/version" ) +var ( + CacheDir = "/var/lib/cni" +) + +// A RuntimeConf holds the arguments to one invocation of a CNI plugin +// excepting the network configuration, with the nested exception that +// the `runtimeConfig` from the network configuration is included +// here. type RuntimeConf struct { ContainerID string NetNS string @@ -34,6 +47,9 @@ type RuntimeConf struct { // in this map which match the capabilities of the plugin are passed // to the plugin CapabilityArgs map[string]interface{} + + // A cache directory in which to library data. Defaults to CacheDir + CacheDir string } type NetworkConfig struct { @@ -42,33 +58,52 @@ type NetworkConfig struct { } type NetworkConfigList struct { - Name string - CNIVersion string - Plugins []*NetworkConfig - Bytes []byte + Name string + CNIVersion string + DisableCheck bool + Plugins []*NetworkConfig + Bytes []byte } type CNI interface { - AddNetworkList(net *NetworkConfigList, rt *RuntimeConf) (types.Result, error) - DelNetworkList(net *NetworkConfigList, rt *RuntimeConf) error + AddNetworkList(ctx context.Context, net *NetworkConfigList, rt *RuntimeConf) (types.Result, error) + CheckNetworkList(ctx context.Context, net *NetworkConfigList, rt *RuntimeConf) error + DelNetworkList(ctx context.Context, net *NetworkConfigList, rt *RuntimeConf) error + GetNetworkListCachedResult(net *NetworkConfigList, rt *RuntimeConf) (types.Result, error) - AddNetwork(net *NetworkConfig, rt *RuntimeConf) (types.Result, error) - DelNetwork(net *NetworkConfig, rt *RuntimeConf) error + AddNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) (types.Result, error) + CheckNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) error + DelNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) error + GetNetworkCachedResult(net *NetworkConfig, rt *RuntimeConf) (types.Result, error) + + ValidateNetworkList(ctx context.Context, net *NetworkConfigList) ([]string, error) + ValidateNetwork(ctx context.Context, net *NetworkConfig) ([]string, error) } type CNIConfig struct { Path []string + exec invoke.Exec } // CNIConfig implements the CNI interface var _ CNI = &CNIConfig{} -func buildOneConfig(list *NetworkConfigList, orig *NetworkConfig, prevResult types.Result, rt *RuntimeConf) (*NetworkConfig, error) { +// NewCNIConfig returns a new CNIConfig object that will search for plugins +// in the given paths and use the given exec interface to run those plugins, +// or if the exec interface is not given, will use a default exec handler. +func NewCNIConfig(path []string, exec invoke.Exec) *CNIConfig { + return &CNIConfig{ + Path: path, + exec: exec, + } +} + +func buildOneConfig(name, cniVersion string, orig *NetworkConfig, prevResult types.Result, rt *RuntimeConf) (*NetworkConfig, error) { var err error inject := map[string]interface{}{ - "name": list.Name, - "cniVersion": list.CNIVersion, + "name": name, + "cniVersion": cniVersion, } // Add previous plugin result if prevResult != nil { @@ -92,7 +127,7 @@ func buildOneConfig(list *NetworkConfigList, orig *NetworkConfig, prevResult typ // These capabilities arguments are filtered through the plugin's advertised // capabilities from its config JSON, and any keys in the CapabilityArgs // matching plugin capabilities are added to the "runtimeConfig" dictionary -// sent to the plugin via JSON on stdin. For exmaple, if the plugin's +// sent to the plugin via JSON on stdin. For example, if the plugin's // capabilities include "portMappings", and the CapabilityArgs map includes a // "portMappings" key, that key and its value are added to the "runtimeConfig" // dictionary to be passed to the plugin's stdin. @@ -119,45 +154,154 @@ func injectRuntimeConfig(orig *NetworkConfig, rt *RuntimeConf) (*NetworkConfig, return orig, nil } +// ensure we have a usable exec if the CNIConfig was not given one +func (c *CNIConfig) ensureExec() invoke.Exec { + if c.exec == nil { + c.exec = &invoke.DefaultExec{ + RawExec: &invoke.RawExec{Stderr: os.Stderr}, + PluginDecoder: version.PluginDecoder{}, + } + } + return c.exec +} + +func getResultCacheFilePath(netName string, rt *RuntimeConf) string { + cacheDir := rt.CacheDir + if cacheDir == "" { + cacheDir = CacheDir + } + return filepath.Join(cacheDir, "results", fmt.Sprintf("%s-%s-%s", netName, rt.ContainerID, rt.IfName)) +} + +func setCachedResult(result types.Result, netName string, rt *RuntimeConf) error { + data, err := json.Marshal(result) + if err != nil { + return err + } + fname := getResultCacheFilePath(netName, rt) + if err := os.MkdirAll(filepath.Dir(fname), 0700); err != nil { + return err + } + return ioutil.WriteFile(fname, data, 0600) +} + +func delCachedResult(netName string, rt *RuntimeConf) error { + fname := getResultCacheFilePath(netName, rt) + return os.Remove(fname) +} + +func getCachedResult(netName, cniVersion string, rt *RuntimeConf) (types.Result, error) { + fname := getResultCacheFilePath(netName, rt) + data, err := ioutil.ReadFile(fname) + if err != nil { + // Ignore read errors; the cached result may not exist on-disk + return nil, nil + } + + // Read the version of the cached result + decoder := version.ConfigDecoder{} + resultCniVersion, err := decoder.Decode(data) + if err != nil { + return nil, err + } + + // Ensure we can understand the result + result, err := version.NewResult(resultCniVersion, data) + if err != nil { + return nil, err + } + + // Convert to the config version to ensure plugins get prevResult + // in the same version as the config. The cached result version + // should match the config version unless the config was changed + // while the container was running. + result, err = result.GetAsVersion(cniVersion) + if err != nil && resultCniVersion != cniVersion { + return nil, fmt.Errorf("failed to convert cached result version %q to config version %q: %v", resultCniVersion, cniVersion, err) + } + return result, err +} + +// GetNetworkListCachedResult returns the cached Result of the previous +// previous AddNetworkList() operation for a network list, or an error. +func (c *CNIConfig) GetNetworkListCachedResult(list *NetworkConfigList, rt *RuntimeConf) (types.Result, error) { + return getCachedResult(list.Name, list.CNIVersion, rt) +} + +// GetNetworkCachedResult returns the cached Result of the previous +// previous AddNetwork() operation for a network, or an error. +func (c *CNIConfig) GetNetworkCachedResult(net *NetworkConfig, rt *RuntimeConf) (types.Result, error) { + return getCachedResult(net.Network.Name, net.Network.CNIVersion, rt) +} + +func (c *CNIConfig) addNetwork(ctx context.Context, name, cniVersion string, net *NetworkConfig, prevResult types.Result, rt *RuntimeConf) (types.Result, error) { + c.ensureExec() + pluginPath, err := c.exec.FindInPath(net.Network.Type, c.Path) + if err != nil { + return nil, err + } + + newConf, err := buildOneConfig(name, cniVersion, net, prevResult, rt) + if err != nil { + return nil, err + } + + return invoke.ExecPluginWithResult(ctx, pluginPath, newConf.Bytes, c.args("ADD", rt), c.exec) +} + // AddNetworkList executes a sequence of plugins with the ADD command -func (c *CNIConfig) AddNetworkList(list *NetworkConfigList, rt *RuntimeConf) (types.Result, error) { - var prevResult types.Result +func (c *CNIConfig) AddNetworkList(ctx context.Context, list *NetworkConfigList, rt *RuntimeConf) (types.Result, error) { + var err error + var result types.Result for _, net := range list.Plugins { - pluginPath, err := invoke.FindInPath(net.Network.Type, c.Path) - if err != nil { - return nil, err - } - - newConf, err := buildOneConfig(list, net, prevResult, rt) - if err != nil { - return nil, err - } - - prevResult, err = invoke.ExecPluginWithResult(pluginPath, newConf.Bytes, c.args("ADD", rt)) + result, err = c.addNetwork(ctx, list.Name, list.CNIVersion, net, result, rt) if err != nil { return nil, err } } - return prevResult, nil + if err = setCachedResult(result, list.Name, rt); err != nil { + return nil, fmt.Errorf("failed to set network %q cached result: %v", list.Name, err) + } + + return result, nil } -// DelNetworkList executes a sequence of plugins with the DEL command -func (c *CNIConfig) DelNetworkList(list *NetworkConfigList, rt *RuntimeConf) error { - for i := len(list.Plugins) - 1; i >= 0; i-- { - net := list.Plugins[i] +func (c *CNIConfig) checkNetwork(ctx context.Context, name, cniVersion string, net *NetworkConfig, prevResult types.Result, rt *RuntimeConf) error { + c.ensureExec() + pluginPath, err := c.exec.FindInPath(net.Network.Type, c.Path) + if err != nil { + return err + } - pluginPath, err := invoke.FindInPath(net.Network.Type, c.Path) - if err != nil { - return err - } + newConf, err := buildOneConfig(name, cniVersion, net, prevResult, rt) + if err != nil { + return err + } - newConf, err := buildOneConfig(list, net, nil, rt) - if err != nil { - return err - } + return invoke.ExecPluginWithoutResult(ctx, pluginPath, newConf.Bytes, c.args("CHECK", rt), c.exec) +} - if err := invoke.ExecPluginWithoutResult(pluginPath, newConf.Bytes, c.args("DEL", rt)); err != nil { +// CheckNetworkList executes a sequence of plugins with the CHECK command +func (c *CNIConfig) CheckNetworkList(ctx context.Context, list *NetworkConfigList, rt *RuntimeConf) error { + // CHECK was added in CNI spec version 0.4.0 and higher + if gtet, err := version.GreaterThanOrEqualTo(list.CNIVersion, "0.4.0"); err != nil { + return err + } else if !gtet { + return fmt.Errorf("configuration version %q does not support the CHECK command", list.CNIVersion) + } + + if list.DisableCheck { + return nil + } + + cachedResult, err := getCachedResult(list.Name, list.CNIVersion, rt) + if err != nil { + return fmt.Errorf("failed to get network %q cached result: %v", list.Name, err) + } + + for _, net := range list.Plugins { + if err := c.checkNetwork(ctx, list.Name, list.CNIVersion, net, cachedResult, rt); err != nil { return err } } @@ -165,45 +309,179 @@ func (c *CNIConfig) DelNetworkList(list *NetworkConfigList, rt *RuntimeConf) err return nil } +func (c *CNIConfig) delNetwork(ctx context.Context, name, cniVersion string, net *NetworkConfig, prevResult types.Result, rt *RuntimeConf) error { + c.ensureExec() + pluginPath, err := c.exec.FindInPath(net.Network.Type, c.Path) + if err != nil { + return err + } + + newConf, err := buildOneConfig(name, cniVersion, net, prevResult, rt) + if err != nil { + return err + } + + return invoke.ExecPluginWithoutResult(ctx, pluginPath, newConf.Bytes, c.args("DEL", rt), c.exec) +} + +// DelNetworkList executes a sequence of plugins with the DEL command +func (c *CNIConfig) DelNetworkList(ctx context.Context, list *NetworkConfigList, rt *RuntimeConf) error { + var cachedResult types.Result + + // Cached result on DEL was added in CNI spec version 0.4.0 and higher + if gtet, err := version.GreaterThanOrEqualTo(list.CNIVersion, "0.4.0"); err != nil { + return err + } else if gtet { + cachedResult, err = getCachedResult(list.Name, list.CNIVersion, rt) + if err != nil { + return fmt.Errorf("failed to get network %q cached result: %v", list.Name, err) + } + } + + for i := len(list.Plugins) - 1; i >= 0; i-- { + net := list.Plugins[i] + if err := c.delNetwork(ctx, list.Name, list.CNIVersion, net, cachedResult, rt); err != nil { + return err + } + } + _ = delCachedResult(list.Name, rt) + + return nil +} + // AddNetwork executes the plugin with the ADD command -func (c *CNIConfig) AddNetwork(net *NetworkConfig, rt *RuntimeConf) (types.Result, error) { - pluginPath, err := invoke.FindInPath(net.Network.Type, c.Path) +func (c *CNIConfig) AddNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) (types.Result, error) { + result, err := c.addNetwork(ctx, net.Network.Name, net.Network.CNIVersion, net, nil, rt) if err != nil { return nil, err } - net, err = injectRuntimeConfig(net, rt) - if err != nil { - return nil, err + if err = setCachedResult(result, net.Network.Name, rt); err != nil { + return nil, fmt.Errorf("failed to set network %q cached result: %v", net.Network.Name, err) } - return invoke.ExecPluginWithResult(pluginPath, net.Bytes, c.args("ADD", rt)) + return result, nil +} + +// CheckNetwork executes the plugin with the CHECK command +func (c *CNIConfig) CheckNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) error { + // CHECK was added in CNI spec version 0.4.0 and higher + if gtet, err := version.GreaterThanOrEqualTo(net.Network.CNIVersion, "0.4.0"); err != nil { + return err + } else if !gtet { + return fmt.Errorf("configuration version %q does not support the CHECK command", net.Network.CNIVersion) + } + + cachedResult, err := getCachedResult(net.Network.Name, net.Network.CNIVersion, rt) + if err != nil { + return fmt.Errorf("failed to get network %q cached result: %v", net.Network.Name, err) + } + return c.checkNetwork(ctx, net.Network.Name, net.Network.CNIVersion, net, cachedResult, rt) } // DelNetwork executes the plugin with the DEL command -func (c *CNIConfig) DelNetwork(net *NetworkConfig, rt *RuntimeConf) error { - pluginPath, err := invoke.FindInPath(net.Network.Type, c.Path) +func (c *CNIConfig) DelNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) error { + var cachedResult types.Result + + // Cached result on DEL was added in CNI spec version 0.4.0 and higher + if gtet, err := version.GreaterThanOrEqualTo(net.Network.CNIVersion, "0.4.0"); err != nil { + return err + } else if gtet { + cachedResult, err = getCachedResult(net.Network.Name, net.Network.CNIVersion, rt) + if err != nil { + return fmt.Errorf("failed to get network %q cached result: %v", net.Network.Name, err) + } + } + + if err := c.delNetwork(ctx, net.Network.Name, net.Network.CNIVersion, net, cachedResult, rt); err != nil { + return err + } + _ = delCachedResult(net.Network.Name, rt) + return nil +} + +// ValidateNetworkList checks that a configuration is reasonably valid. +// - all the specified plugins exist on disk +// - every plugin supports the desired version. +// +// Returns a list of all capabilities supported by the configuration, or error +func (c *CNIConfig) ValidateNetworkList(ctx context.Context, list *NetworkConfigList) ([]string, error) { + version := list.CNIVersion + + // holding map for seen caps (in case of duplicates) + caps := map[string]interface{}{} + + errs := []error{} + for _, net := range list.Plugins { + if err := c.validatePlugin(ctx, net.Network.Type, version); err != nil { + errs = append(errs, err) + } + for c, enabled := range net.Network.Capabilities { + if !enabled { + continue + } + caps[c] = struct{}{} + } + } + + if len(errs) > 0 { + return nil, fmt.Errorf("%v", errs) + } + + // make caps list + cc := make([]string, 0, len(caps)) + for c := range caps { + cc = append(cc, c) + } + + return cc, nil +} + +// ValidateNetwork checks that a configuration is reasonably valid. +// It uses the same logic as ValidateNetworkList) +// Returns a list of capabilities +func (c *CNIConfig) ValidateNetwork(ctx context.Context, net *NetworkConfig) ([]string, error) { + caps := []string{} + for c, ok := range net.Network.Capabilities { + if ok { + caps = append(caps, c) + } + } + if err := c.validatePlugin(ctx, net.Network.Type, net.Network.CNIVersion); err != nil { + return nil, err + } + return caps, nil +} + +// validatePlugin checks that an individual plugin's configuration is sane +func (c *CNIConfig) validatePlugin(ctx context.Context, pluginName, expectedVersion string) error { + pluginPath, err := invoke.FindInPath(pluginName, c.Path) if err != nil { return err } - net, err = injectRuntimeConfig(net, rt) + vi, err := invoke.GetVersionInfo(ctx, pluginPath, c.exec) if err != nil { return err } - - return invoke.ExecPluginWithoutResult(pluginPath, net.Bytes, c.args("DEL", rt)) + for _, vers := range vi.SupportedVersions() { + if vers == expectedVersion { + return nil + } + } + return fmt.Errorf("plugin %s does not support config version %q", pluginName, expectedVersion) } // GetVersionInfo reports which versions of the CNI spec are supported by // the given plugin. -func (c *CNIConfig) GetVersionInfo(pluginType string) (version.PluginInfo, error) { - pluginPath, err := invoke.FindInPath(pluginType, c.Path) +func (c *CNIConfig) GetVersionInfo(ctx context.Context, pluginType string) (version.PluginInfo, error) { + c.ensureExec() + pluginPath, err := c.exec.FindInPath(pluginType, c.Path) if err != nil { return nil, err } - return invoke.GetVersionInfo(pluginPath) + return invoke.GetVersionInfo(ctx, pluginPath, c.exec) } // ===== diff --git a/vendor/github.com/containernetworking/cni/libcni/conf.go b/vendor/github.com/containernetworking/cni/libcni/conf.go index c7738c66503..ea56c509d01 100644 --- a/vendor/github.com/containernetworking/cni/libcni/conf.go +++ b/vendor/github.com/containernetworking/cni/libcni/conf.go @@ -45,6 +45,9 @@ func ConfFromBytes(bytes []byte) (*NetworkConfig, error) { if err := json.Unmarshal(bytes, &conf.Network); err != nil { return nil, fmt.Errorf("error parsing configuration: %s", err) } + if conf.Network.Type == "" { + return nil, fmt.Errorf("error parsing configuration: missing 'type'") + } return conf, nil } @@ -80,10 +83,19 @@ func ConfListFromBytes(bytes []byte) (*NetworkConfigList, error) { } } + disableCheck := false + if rawDisableCheck, ok := rawList["disableCheck"]; ok { + disableCheck, ok = rawDisableCheck.(bool) + if !ok { + return nil, fmt.Errorf("error parsing configuration list: invalid disableCheck type %T", rawDisableCheck) + } + } + list := &NetworkConfigList{ - Name: name, - CNIVersion: cniVersion, - Bytes: bytes, + Name: name, + DisableCheck: disableCheck, + CNIVersion: cniVersion, + Bytes: bytes, } var plugins []interface{} diff --git a/vendor/github.com/containernetworking/cni/pkg/invoke/args.go b/vendor/github.com/containernetworking/cni/pkg/invoke/args.go index 39b63972305..913528c1d59 100644 --- a/vendor/github.com/containernetworking/cni/pkg/invoke/args.go +++ b/vendor/github.com/containernetworking/cni/pkg/invoke/args.go @@ -15,6 +15,7 @@ package invoke import ( + "fmt" "os" "strings" ) @@ -22,6 +23,8 @@ import ( type CNIArgs interface { // For use with os/exec; i.e., return nil to inherit the // environment from this process + // For use in delegation; inherit the environment from this + // process and allow overrides AsEnv() []string } @@ -57,17 +60,17 @@ func (args *Args) AsEnv() []string { pluginArgsStr = stringify(args.PluginArgs) } - // Ensure that the custom values are first, so any value present in - // the process environment won't override them. - env = append([]string{ - "CNI_COMMAND=" + args.Command, - "CNI_CONTAINERID=" + args.ContainerID, - "CNI_NETNS=" + args.NetNS, - "CNI_ARGS=" + pluginArgsStr, - "CNI_IFNAME=" + args.IfName, - "CNI_PATH=" + args.Path, - }, env...) - return env + // Duplicated values which come first will be overrided, so we must put the + // custom values in the end to avoid being overrided by the process environments. + env = append(env, + "CNI_COMMAND="+args.Command, + "CNI_CONTAINERID="+args.ContainerID, + "CNI_NETNS="+args.NetNS, + "CNI_ARGS="+pluginArgsStr, + "CNI_IFNAME="+args.IfName, + "CNI_PATH="+args.Path, + ) + return dedupEnv(env) } // taken from rkt/networking/net_plugin.go @@ -80,3 +83,46 @@ func stringify(pluginArgs [][2]string) string { return strings.Join(entries, ";") } + +// DelegateArgs implements the CNIArgs interface +// used for delegation to inherit from environments +// and allow some overrides like CNI_COMMAND +var _ CNIArgs = &DelegateArgs{} + +type DelegateArgs struct { + Command string +} + +func (d *DelegateArgs) AsEnv() []string { + env := os.Environ() + + // The custom values should come in the end to override the existing + // process environment of the same key. + env = append(env, + "CNI_COMMAND="+d.Command, + ) + return dedupEnv(env) +} + +// dedupEnv returns a copy of env with any duplicates removed, in favor of later values. +// Items not of the normal environment "key=value" form are preserved unchanged. +func dedupEnv(env []string) []string { + out := make([]string, 0, len(env)) + envMap := map[string]string{} + + for _, kv := range env { + // find the first "=" in environment, if not, just keep it + eq := strings.Index(kv, "=") + if eq < 0 { + out = append(out, kv) + continue + } + envMap[kv[:eq]] = kv[eq+1:] + } + + for k, v := range envMap { + out = append(out, fmt.Sprintf("%s=%s", k, v)) + } + + return out +} diff --git a/vendor/github.com/containernetworking/cni/pkg/invoke/delegate.go b/vendor/github.com/containernetworking/cni/pkg/invoke/delegate.go index c78a69eebc8..8defe4dd398 100644 --- a/vendor/github.com/containernetworking/cni/pkg/invoke/delegate.go +++ b/vendor/github.com/containernetworking/cni/pkg/invoke/delegate.go @@ -15,39 +15,66 @@ package invoke import ( - "fmt" + "context" "os" "path/filepath" "github.com/containernetworking/cni/pkg/types" ) -func DelegateAdd(delegatePlugin string, netconf []byte) (types.Result, error) { - if os.Getenv("CNI_COMMAND") != "ADD" { - return nil, fmt.Errorf("CNI_COMMAND is not ADD") +func delegateCommon(delegatePlugin string, exec Exec) (string, Exec, error) { + if exec == nil { + exec = defaultExec } paths := filepath.SplitList(os.Getenv("CNI_PATH")) + pluginPath, err := exec.FindInPath(delegatePlugin, paths) + if err != nil { + return "", nil, err + } - pluginPath, err := FindInPath(delegatePlugin, paths) + return pluginPath, exec, nil +} + +// DelegateAdd calls the given delegate plugin with the CNI ADD action and +// JSON configuration +func DelegateAdd(ctx context.Context, delegatePlugin string, netconf []byte, exec Exec) (types.Result, error) { + pluginPath, realExec, err := delegateCommon(delegatePlugin, exec) if err != nil { return nil, err } - return ExecPluginWithResult(pluginPath, netconf, ArgsFromEnv()) + // DelegateAdd will override the original "CNI_COMMAND" env from process with ADD + return ExecPluginWithResult(ctx, pluginPath, netconf, delegateArgs("ADD"), realExec) } -func DelegateDel(delegatePlugin string, netconf []byte) error { - if os.Getenv("CNI_COMMAND") != "DEL" { - return fmt.Errorf("CNI_COMMAND is not DEL") - } - - paths := filepath.SplitList(os.Getenv("CNI_PATH")) - - pluginPath, err := FindInPath(delegatePlugin, paths) +// DelegateCheck calls the given delegate plugin with the CNI CHECK action and +// JSON configuration +func DelegateCheck(ctx context.Context, delegatePlugin string, netconf []byte, exec Exec) error { + pluginPath, realExec, err := delegateCommon(delegatePlugin, exec) if err != nil { return err } - return ExecPluginWithoutResult(pluginPath, netconf, ArgsFromEnv()) + // DelegateCheck will override the original CNI_COMMAND env from process with CHECK + return ExecPluginWithoutResult(ctx, pluginPath, netconf, delegateArgs("CHECK"), realExec) +} + +// DelegateDel calls the given delegate plugin with the CNI DEL action and +// JSON configuration +func DelegateDel(ctx context.Context, delegatePlugin string, netconf []byte, exec Exec) error { + pluginPath, realExec, err := delegateCommon(delegatePlugin, exec) + if err != nil { + return err + } + + // DelegateDel will override the original CNI_COMMAND env from process with DEL + return ExecPluginWithoutResult(ctx, pluginPath, netconf, delegateArgs("DEL"), realExec) +} + +// return CNIArgs used by delegation +func delegateArgs(action string) *DelegateArgs { + return &DelegateArgs{ + Command: action, + } } diff --git a/vendor/github.com/containernetworking/cni/pkg/invoke/exec.go b/vendor/github.com/containernetworking/cni/pkg/invoke/exec.go index fc47e7c8253..8e6d30b8290 100644 --- a/vendor/github.com/containernetworking/cni/pkg/invoke/exec.go +++ b/vendor/github.com/containernetworking/cni/pkg/invoke/exec.go @@ -15,6 +15,7 @@ package invoke import ( + "context" "fmt" "os" @@ -22,34 +23,62 @@ import ( "github.com/containernetworking/cni/pkg/version" ) -func ExecPluginWithResult(pluginPath string, netconf []byte, args CNIArgs) (types.Result, error) { - return defaultPluginExec.WithResult(pluginPath, netconf, args) +// Exec is an interface encapsulates all operations that deal with finding +// and executing a CNI plugin. Tests may provide a fake implementation +// to avoid writing fake plugins to temporary directories during the test. +type Exec interface { + ExecPlugin(ctx context.Context, pluginPath string, stdinData []byte, environ []string) ([]byte, error) + FindInPath(plugin string, paths []string) (string, error) + Decode(jsonBytes []byte) (version.PluginInfo, error) } -func ExecPluginWithoutResult(pluginPath string, netconf []byte, args CNIArgs) error { - return defaultPluginExec.WithoutResult(pluginPath, netconf, args) -} +// For example, a testcase could pass an instance of the following fakeExec +// object to ExecPluginWithResult() to verify the incoming stdin and environment +// and provide a tailored response: +// +//import ( +// "encoding/json" +// "path" +// "strings" +//) +// +//type fakeExec struct { +// version.PluginDecoder +//} +// +//func (f *fakeExec) ExecPlugin(pluginPath string, stdinData []byte, environ []string) ([]byte, error) { +// net := &types.NetConf{} +// err := json.Unmarshal(stdinData, net) +// if err != nil { +// return nil, fmt.Errorf("failed to unmarshal configuration: %v", err) +// } +// pluginName := path.Base(pluginPath) +// if pluginName != net.Type { +// return nil, fmt.Errorf("plugin name %q did not match config type %q", pluginName, net.Type) +// } +// for _, e := range environ { +// // Check environment for forced failure request +// parts := strings.Split(e, "=") +// if len(parts) > 0 && parts[0] == "FAIL" { +// return nil, fmt.Errorf("failed to execute plugin %s", pluginName) +// } +// } +// return []byte("{\"CNIVersion\":\"0.4.0\"}"), nil +//} +// +//func (f *fakeExec) FindInPath(plugin string, paths []string) (string, error) { +// if len(paths) > 0 { +// return path.Join(paths[0], plugin), nil +// } +// return "", fmt.Errorf("failed to find plugin %s in paths %v", plugin, paths) +//} -func GetVersionInfo(pluginPath string) (version.PluginInfo, error) { - return defaultPluginExec.GetVersionInfo(pluginPath) -} - -var defaultPluginExec = &PluginExec{ - RawExec: &RawExec{Stderr: os.Stderr}, - VersionDecoder: &version.PluginDecoder{}, -} - -type PluginExec struct { - RawExec interface { - ExecPlugin(pluginPath string, stdinData []byte, environ []string) ([]byte, error) +func ExecPluginWithResult(ctx context.Context, pluginPath string, netconf []byte, args CNIArgs, exec Exec) (types.Result, error) { + if exec == nil { + exec = defaultExec } - VersionDecoder interface { - Decode(jsonBytes []byte) (version.PluginInfo, error) - } -} -func (e *PluginExec) WithResult(pluginPath string, netconf []byte, args CNIArgs) (types.Result, error) { - stdoutBytes, err := e.RawExec.ExecPlugin(pluginPath, netconf, args.AsEnv()) + stdoutBytes, err := exec.ExecPlugin(ctx, pluginPath, netconf, args.AsEnv()) if err != nil { return nil, err } @@ -64,8 +93,11 @@ func (e *PluginExec) WithResult(pluginPath string, netconf []byte, args CNIArgs) return version.NewResult(confVersion, stdoutBytes) } -func (e *PluginExec) WithoutResult(pluginPath string, netconf []byte, args CNIArgs) error { - _, err := e.RawExec.ExecPlugin(pluginPath, netconf, args.AsEnv()) +func ExecPluginWithoutResult(ctx context.Context, pluginPath string, netconf []byte, args CNIArgs, exec Exec) error { + if exec == nil { + exec = defaultExec + } + _, err := exec.ExecPlugin(ctx, pluginPath, netconf, args.AsEnv()) return err } @@ -73,7 +105,10 @@ func (e *PluginExec) WithoutResult(pluginPath string, netconf []byte, args CNIAr // For recent-enough plugins, it uses the information returned by the VERSION // command. For older plugins which do not recognize that command, it reports // version 0.1.0 -func (e *PluginExec) GetVersionInfo(pluginPath string) (version.PluginInfo, error) { +func GetVersionInfo(ctx context.Context, pluginPath string, exec Exec) (version.PluginInfo, error) { + if exec == nil { + exec = defaultExec + } args := &Args{ Command: "VERSION", @@ -83,7 +118,7 @@ func (e *PluginExec) GetVersionInfo(pluginPath string) (version.PluginInfo, erro Path: "dummy", } stdin := []byte(fmt.Sprintf(`{"cniVersion":%q}`, version.Current())) - stdoutBytes, err := e.RawExec.ExecPlugin(pluginPath, stdin, args.AsEnv()) + stdoutBytes, err := exec.ExecPlugin(ctx, pluginPath, stdin, args.AsEnv()) if err != nil { if err.Error() == "unknown CNI_COMMAND: VERSION" { return version.PluginSupports("0.1.0"), nil @@ -91,5 +126,19 @@ func (e *PluginExec) GetVersionInfo(pluginPath string) (version.PluginInfo, erro return nil, err } - return e.VersionDecoder.Decode(stdoutBytes) + return exec.Decode(stdoutBytes) +} + +// DefaultExec is an object that implements the Exec interface which looks +// for and executes plugins from disk. +type DefaultExec struct { + *RawExec + version.PluginDecoder +} + +// DefaultExec implements the Exec interface +var _ Exec = &DefaultExec{} + +var defaultExec = &DefaultExec{ + RawExec: &RawExec{Stderr: os.Stderr}, } diff --git a/vendor/github.com/containernetworking/cni/pkg/invoke/os_unix.go b/vendor/github.com/containernetworking/cni/pkg/invoke/os_unix.go index bab5737a99d..9bcfb455367 100644 --- a/vendor/github.com/containernetworking/cni/pkg/invoke/os_unix.go +++ b/vendor/github.com/containernetworking/cni/pkg/invoke/os_unix.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build darwin dragonfly freebsd linux netbsd opensbd solaris +// +build darwin dragonfly freebsd linux netbsd openbsd solaris package invoke diff --git a/vendor/github.com/containernetworking/cni/pkg/invoke/raw_exec.go b/vendor/github.com/containernetworking/cni/pkg/invoke/raw_exec.go index 93f1e75d9fa..ad8498ba27d 100644 --- a/vendor/github.com/containernetworking/cni/pkg/invoke/raw_exec.go +++ b/vendor/github.com/containernetworking/cni/pkg/invoke/raw_exec.go @@ -16,6 +16,7 @@ package invoke import ( "bytes" + "context" "encoding/json" "fmt" "io" @@ -28,17 +29,13 @@ type RawExec struct { Stderr io.Writer } -func (e *RawExec) ExecPlugin(pluginPath string, stdinData []byte, environ []string) ([]byte, error) { +func (e *RawExec) ExecPlugin(ctx context.Context, pluginPath string, stdinData []byte, environ []string) ([]byte, error) { stdout := &bytes.Buffer{} - - c := exec.Cmd{ - Env: environ, - Path: pluginPath, - Args: []string{pluginPath}, - Stdin: bytes.NewBuffer(stdinData), - Stdout: stdout, - Stderr: e.Stderr, - } + c := exec.CommandContext(ctx, pluginPath) + c.Env = environ + c.Stdin = bytes.NewBuffer(stdinData) + c.Stdout = stdout + c.Stderr = e.Stderr if err := c.Run(); err != nil { return nil, pluginErr(err, stdout.Bytes()) } @@ -49,7 +46,9 @@ func (e *RawExec) ExecPlugin(pluginPath string, stdinData []byte, environ []stri func pluginErr(err error, output []byte) error { if _, ok := err.(*exec.ExitError); ok { emsg := types.Error{} - if perr := json.Unmarshal(output, &emsg); perr != nil { + if len(output) == 0 { + emsg.Msg = "netplugin failed with no error message" + } else if perr := json.Unmarshal(output, &emsg); perr != nil { emsg.Msg = fmt.Sprintf("netplugin failed but error parsing its diagnostic message %q: %v", string(output), perr) } return &emsg @@ -57,3 +56,7 @@ func pluginErr(err error, output []byte) error { return err } + +func (e *RawExec) FindInPath(plugin string, paths []string) (string, error) { + return FindInPath(plugin, paths) +} diff --git a/vendor/github.com/containernetworking/cni/pkg/types/020/types.go b/vendor/github.com/containernetworking/cni/pkg/types/020/types.go index 2833aba787f..53256167fad 100644 --- a/vendor/github.com/containernetworking/cni/pkg/types/020/types.go +++ b/vendor/github.com/containernetworking/cni/pkg/types/020/types.go @@ -17,6 +17,7 @@ package types020 import ( "encoding/json" "fmt" + "io" "net" "os" @@ -73,11 +74,15 @@ func (r *Result) GetAsVersion(version string) (types.Result, error) { } func (r *Result) Print() error { + return r.PrintTo(os.Stdout) +} + +func (r *Result) PrintTo(writer io.Writer) error { data, err := json.MarshalIndent(r, "", " ") if err != nil { return err } - _, err = os.Stdout.Write(data) + _, err = writer.Write(data) return err } diff --git a/vendor/github.com/containernetworking/cni/pkg/types/current/types.go b/vendor/github.com/containernetworking/cni/pkg/types/current/types.go index caac92ba77c..7267a2e6d1f 100644 --- a/vendor/github.com/containernetworking/cni/pkg/types/current/types.go +++ b/vendor/github.com/containernetworking/cni/pkg/types/current/types.go @@ -17,6 +17,7 @@ package current import ( "encoding/json" "fmt" + "io" "net" "os" @@ -24,9 +25,9 @@ import ( "github.com/containernetworking/cni/pkg/types/020" ) -const ImplementedSpecVersion string = "0.3.1" +const ImplementedSpecVersion string = "0.4.0" -var SupportedVersions = []string{"0.3.0", ImplementedSpecVersion} +var SupportedVersions = []string{"0.3.0", "0.3.1", ImplementedSpecVersion} func NewResult(data []byte) (types.Result, error) { result := &Result{} @@ -75,13 +76,9 @@ func convertFrom020(result types.Result) (*Result, error) { Gateway: oldResult.IP4.Gateway, }) for _, route := range oldResult.IP4.Routes { - gw := route.GW - if gw == nil { - gw = oldResult.IP4.Gateway - } newResult.Routes = append(newResult.Routes, &types.Route{ Dst: route.Dst, - GW: gw, + GW: route.GW, }) } } @@ -93,21 +90,13 @@ func convertFrom020(result types.Result) (*Result, error) { Gateway: oldResult.IP6.Gateway, }) for _, route := range oldResult.IP6.Routes { - gw := route.GW - if gw == nil { - gw = oldResult.IP6.Gateway - } newResult.Routes = append(newResult.Routes, &types.Route{ Dst: route.Dst, - GW: gw, + GW: route.GW, }) } } - if len(newResult.IPs) == 0 { - return nil, fmt.Errorf("cannot convert: no valid IP addresses") - } - return newResult, nil } @@ -196,7 +185,7 @@ func (r *Result) Version() string { func (r *Result) GetAsVersion(version string) (types.Result, error) { switch version { - case "0.3.0", ImplementedSpecVersion: + case "0.3.0", "0.3.1", ImplementedSpecVersion: r.CNIVersion = version return r, nil case types020.SupportedVersions[0], types020.SupportedVersions[1], types020.SupportedVersions[2]: @@ -206,11 +195,15 @@ func (r *Result) GetAsVersion(version string) (types.Result, error) { } func (r *Result) Print() error { + return r.PrintTo(os.Stdout) +} + +func (r *Result) PrintTo(writer io.Writer) error { data, err := json.MarshalIndent(r, "", " ") if err != nil { return err } - _, err = os.Stdout.Write(data) + _, err = writer.Write(data) return err } diff --git a/vendor/github.com/containernetworking/cni/pkg/types/types.go b/vendor/github.com/containernetworking/cni/pkg/types/types.go index 6412756007e..d0d11006a05 100644 --- a/vendor/github.com/containernetworking/cni/pkg/types/types.go +++ b/vendor/github.com/containernetworking/cni/pkg/types/types.go @@ -18,6 +18,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "net" "os" ) @@ -63,25 +64,31 @@ type NetConf struct { Name string `json:"name,omitempty"` Type string `json:"type,omitempty"` Capabilities map[string]bool `json:"capabilities,omitempty"` - IPAM struct { - Type string `json:"type,omitempty"` - } `json:"ipam,omitempty"` - DNS DNS `json:"dns"` + IPAM IPAM `json:"ipam,omitempty"` + DNS DNS `json:"dns"` + + RawPrevResult map[string]interface{} `json:"prevResult,omitempty"` + PrevResult Result `json:"-"` +} + +type IPAM struct { + Type string `json:"type,omitempty"` } // NetConfList describes an ordered list of networks. type NetConfList struct { CNIVersion string `json:"cniVersion,omitempty"` - Name string `json:"name,omitempty"` - Plugins []*NetConf `json:"plugins,omitempty"` + Name string `json:"name,omitempty"` + DisableCheck bool `json:"disableCheck,omitempty"` + Plugins []*NetConf `json:"plugins,omitempty"` } type ResultFactoryFunc func([]byte) (Result, error) // Result is an interface that provides the result of plugin execution type Result interface { - // The highest CNI specification result verison the result supports + // The highest CNI specification result version the result supports // without having to convert Version() string @@ -92,6 +99,9 @@ type Result interface { // Prints the result in JSON format to stdout Print() error + // Prints the result in JSON format to provided writer + PrintTo(writer io.Writer) error + // Returns a JSON string representation of the result String() string } @@ -167,7 +177,7 @@ func (r *Route) UnmarshalJSON(data []byte) error { return nil } -func (r *Route) MarshalJSON() ([]byte, error) { +func (r Route) MarshalJSON() ([]byte, error) { rt := route{ Dst: IPNet(r.Dst), GW: r.GW, diff --git a/vendor/github.com/containernetworking/cni/pkg/version/plugin.go b/vendor/github.com/containernetworking/cni/pkg/version/plugin.go index 8a46728105c..1df427243f3 100644 --- a/vendor/github.com/containernetworking/cni/pkg/version/plugin.go +++ b/vendor/github.com/containernetworking/cni/pkg/version/plugin.go @@ -18,6 +18,8 @@ import ( "encoding/json" "fmt" "io" + "strconv" + "strings" ) // PluginInfo reports information about CNI versioning @@ -79,3 +81,64 @@ func (*PluginDecoder) Decode(jsonBytes []byte) (PluginInfo, error) { } return &info, nil } + +// ParseVersion parses a version string like "3.0.1" or "0.4.5" into major, +// minor, and micro numbers or returns an error +func ParseVersion(version string) (int, int, int, error) { + var major, minor, micro int + if version == "" { + return -1, -1, -1, fmt.Errorf("invalid version %q: the version is empty", version) + } + + parts := strings.Split(version, ".") + if len(parts) >= 4 { + return -1, -1, -1, fmt.Errorf("invalid version %q: too many parts", version) + } + + major, err := strconv.Atoi(parts[0]) + if err != nil { + return -1, -1, -1, fmt.Errorf("failed to convert major version part %q: %v", parts[0], err) + } + + if len(parts) >= 2 { + minor, err = strconv.Atoi(parts[1]) + if err != nil { + return -1, -1, -1, fmt.Errorf("failed to convert minor version part %q: %v", parts[1], err) + } + } + + if len(parts) >= 3 { + micro, err = strconv.Atoi(parts[2]) + if err != nil { + return -1, -1, -1, fmt.Errorf("failed to convert micro version part %q: %v", parts[2], err) + } + } + + return major, minor, micro, nil +} + +// GreaterThanOrEqualTo takes two string versions, parses them into major/minor/micro +// numbers, and compares them to determine whether the first version is greater +// than or equal to the second +func GreaterThanOrEqualTo(version, otherVersion string) (bool, error) { + firstMajor, firstMinor, firstMicro, err := ParseVersion(version) + if err != nil { + return false, err + } + + secondMajor, secondMinor, secondMicro, err := ParseVersion(otherVersion) + if err != nil { + return false, err + } + + if firstMajor > secondMajor { + return true, nil + } else if firstMajor == secondMajor { + if firstMinor > secondMinor { + return true, nil + } else if firstMinor == secondMinor && firstMicro >= secondMicro { + return true, nil + } + } + return false, nil +} diff --git a/vendor/github.com/containernetworking/cni/pkg/version/version.go b/vendor/github.com/containernetworking/cni/pkg/version/version.go index efe8ea8716d..8f3508e61f3 100644 --- a/vendor/github.com/containernetworking/cni/pkg/version/version.go +++ b/vendor/github.com/containernetworking/cni/pkg/version/version.go @@ -15,6 +15,7 @@ package version import ( + "encoding/json" "fmt" "github.com/containernetworking/cni/pkg/types" @@ -24,7 +25,7 @@ import ( // Current reports the version of the CNI spec implemented by this library func Current() string { - return "0.3.1" + return "0.4.0" } // Legacy PluginInfo describes a plugin that is backwards compatible with the @@ -35,7 +36,7 @@ func Current() string { // Any future CNI spec versions which meet this definition should be added to // this list. var Legacy = PluginSupports("0.1.0", "0.2.0") -var All = PluginSupports("0.1.0", "0.2.0", "0.3.0", "0.3.1") +var All = PluginSupports("0.1.0", "0.2.0", "0.3.0", "0.3.1", "0.4.0") var resultFactories = []struct { supportedVersions []string @@ -59,3 +60,24 @@ func NewResult(version string, resultBytes []byte) (types.Result, error) { return nil, fmt.Errorf("unsupported CNI result version %q", version) } + +// ParsePrevResult parses a prevResult in a NetConf structure and sets +// the NetConf's PrevResult member to the parsed Result object. +func ParsePrevResult(conf *types.NetConf) error { + if conf.RawPrevResult == nil { + return nil + } + + resultBytes, err := json.Marshal(conf.RawPrevResult) + if err != nil { + return fmt.Errorf("could not serialize prevResult: %v", err) + } + + conf.RawPrevResult = nil + conf.PrevResult, err = NewResult(conf.CNIVersion, resultBytes) + if err != nil { + return fmt.Errorf("could not parse prevResult: %v", err) + } + + return nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index fa8ed13d59a..f5b5d29dff3 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -176,7 +176,7 @@ github.com/containerd/containerd/containers github.com/containerd/containerd/dialer github.com/containerd/containerd/errdefs github.com/containerd/containerd/namespaces -# github.com/containernetworking/cni v0.6.0 => github.com/containernetworking/cni v0.6.0 +# github.com/containernetworking/cni v0.7.1 => github.com/containernetworking/cni v0.7.1 github.com/containernetworking/cni/libcni github.com/containernetworking/cni/pkg/invoke github.com/containernetworking/cni/pkg/types