From adec211ae101e2cf70dfacf3b75809cff36fd8c4 Mon Sep 17 00:00:00 2001 From: Tomofumi Hayashi Date: Fri, 16 Aug 2019 18:55:12 +0900 Subject: [PATCH] Support CNI RuntimeConfig for portmap/bandwidth/ip/mac This changes introduce CNI RuntimeConfig for portmap, bandwidth, ip and mac for latest specification. IP and Mac is previously applied through CNI_ARGS, but it is changed to RuntimeConfig for now. --- k8sclient/k8sclient.go | 33 ++++++- multus/multus.go | 89 +++++++++++++----- multus/multus_test.go | 203 +++++++++++++++++++++++++---------------- types/conf.go | 52 ++++++++++- types/conf_test.go | 142 ++++++++++++++++++++++++++-- types/types.go | 40 ++++++-- 6 files changed, 427 insertions(+), 132 deletions(-) diff --git a/k8sclient/k8sclient.go b/k8sclient/k8sclient.go index 4f48e4415..1c9a0264a 100644 --- a/k8sclient/k8sclient.go +++ b/k8sclient/k8sclient.go @@ -17,6 +17,7 @@ package k8sclient import ( "encoding/json" "fmt" + "net" "os" "regexp" "strings" @@ -228,13 +229,35 @@ func parsePodNetworkAnnotation(podNetworks, defaultNamespace string) ([]*types.N } } - for _, net := range networks { - if net.Namespace == "" { - net.Namespace = defaultNamespace + for _, n := range networks { + if n.Namespace == "" { + n.Namespace = defaultNamespace } // compatibility pre v3.2, will be removed in v4.0 - if net.ObsolatedInterfaceRequest != "" && net.InterfaceRequest == "" { - net.InterfaceRequest = net.ObsolatedInterfaceRequest + if n.DeprecatedInterfaceRequest != "" && n.InterfaceRequest == "" { + n.InterfaceRequest = n.DeprecatedInterfaceRequest + } + if n.MacRequest != "" { + // validate MAC address + if _, err := net.ParseMAC(n.MacRequest); err != nil { + return nil, logging.Errorf("parsePodNetworkAnnotation: failed to mac: %v", err) + } + } + if n.IPRequest != nil { + for _, ip := range n.IPRequest { + // validate IP address + if strings.Contains(ip, "/") { + if _, _, err := net.ParseCIDR(ip); err != nil { + return nil, logging.Errorf("failed to parse CIDR %q: %v", ip, err) + } + } else if net.ParseIP(ip) == nil { + return nil, logging.Errorf("failed to parse IP address %q", ip) + } + } + } + // compatibility pre v3.2, will be removed in v4.0 + if n.DeprecatedInterfaceRequest != "" && n.InterfaceRequest == "" { + n.InterfaceRequest = n.DeprecatedInterfaceRequest } } diff --git a/multus/multus.go b/multus/multus.go index 873231543..f60b955e7 100644 --- a/multus/multus.go +++ b/multus/multus.go @@ -144,11 +144,51 @@ func validateIfName(nsname string, ifname string) error { return err } +func confAdd(rt *libcni.RuntimeConf, rawNetconf []byte, binDir string, exec invoke.Exec) (cnitypes.Result, error) { + logging.Debugf("conflistAdd: %v, %s, %s", rt, string(rawNetconf), binDir) + // In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go + binDirs := filepath.SplitList(os.Getenv("CNI_PATH")) + binDirs = append([]string{binDir}, binDirs...) + cniNet := libcni.NewCNIConfig(binDirs, exec) + + conf, err := libcni.ConfFromBytes(rawNetconf) + if err != nil { + return nil, logging.Errorf("error in converting the raw bytes to conf: %v", err) + } + + result, err := cniNet.AddNetwork(context.Background(), conf, rt) + if err != nil { + return nil, logging.Errorf("error in getting result from AddNetwork: %v", err) + } + + return result, nil +} + +func confDel(rt *libcni.RuntimeConf, rawNetconf []byte, binDir string, exec invoke.Exec) error { + logging.Debugf("conflistDel: %v, %s, %s", rt, string(rawNetconf), binDir) + // In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go + binDirs := filepath.SplitList(os.Getenv("CNI_PATH")) + binDirs = append([]string{binDir}, binDirs...) + cniNet := libcni.NewCNIConfig(binDirs, exec) + + conf, err := libcni.ConfFromBytes(rawNetconf) + if err != nil { + return logging.Errorf("error in converting the raw bytes to conf: %v", err) + } + + err = cniNet.DelNetwork(context.Background(), conf, rt) + if err != nil { + return logging.Errorf("error in getting result from DelNetwork: %v", err) + } + + return err +} + func conflistAdd(rt *libcni.RuntimeConf, rawnetconflist []byte, binDir string, exec invoke.Exec) (cnitypes.Result, error) { logging.Debugf("conflistAdd: %v, %s, %s", rt, string(rawnetconflist), binDir) // In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go binDirs := filepath.SplitList(os.Getenv("CNI_PATH")) - binDirs = append(binDirs, binDir) + binDirs = append([]string{binDir}, binDirs...) cniNet := libcni.NewCNIConfig(binDirs, exec) confList, err := libcni.ConfListFromBytes(rawnetconflist) @@ -168,7 +208,7 @@ func conflistDel(rt *libcni.RuntimeConf, rawnetconflist []byte, binDir string, e logging.Debugf("conflistDel: %v, %s, %s", rt, string(rawnetconflist), binDir) // In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go binDirs := filepath.SplitList(os.Getenv("CNI_PATH")) - binDirs = append(binDirs, binDir) + binDirs = append([]string{binDir}, binDirs...) cniNet := libcni.NewCNIConfig(binDirs, exec) confList, err := libcni.ConfListFromBytes(rawnetconflist) @@ -194,7 +234,8 @@ func delegateAdd(exec invoke.Exec, ifName string, delegate *types.DelegateNetCon return nil, logging.Errorf("delegateAdd: cannot set %q interface name to %q: %v", delegate.Conf.Type, ifName, err) } - if delegate.MacRequest != "" || delegate.IPRequest != "" { + // Deprecated in ver 3.5. + if delegate.MacRequest != "" || delegate.IPRequest != nil { if cniArgs != "" { cniArgs = fmt.Sprintf("%s;IgnoreUnknown=true", cniArgs) } else { @@ -209,24 +250,26 @@ func delegateAdd(exec invoke.Exec, ifName string, delegate *types.DelegateNetCon cniArgs = fmt.Sprintf("%s;MAC=%s", cniArgs, delegate.MacRequest) logging.Debugf("delegateAdd: set MAC address %q to %q", delegate.MacRequest, ifName) + rt.Args = append(rt.Args, [2]string{ "MAC", delegate.MacRequest }) } - if delegate.IPRequest != "" { + if delegate.IPRequest != nil { // validate IP address - if strings.Contains(delegate.IPRequest, "/") { - _, _, err := net.ParseCIDR(delegate.IPRequest) - if err != nil { - return nil, logging.Errorf("delegateAdd: failed to parse CIDR %q", delegate.MacRequest) + for _, ip := range delegate.IPRequest { + if strings.Contains(ip, "/") { + _, _, err := net.ParseCIDR(ip) + if err != nil { + return nil, logging.Errorf("delegateAdd: failed to parse IP address %q", ip) + } + } else if net.ParseIP(ip) == nil { + return nil, logging.Errorf("delegateAdd: failed to parse IP address %q", ip) } - } else if net.ParseIP(delegate.IPRequest) == nil { - return nil, logging.Errorf("delegateAdd: failed to parse IP address %q", delegate.IPRequest) } - cniArgs = fmt.Sprintf("%s;IP=%s", cniArgs, delegate.IPRequest) - logging.Debugf("delegateAdd: set IP address %q to %q", delegate.IPRequest, ifName) - } - if os.Setenv("CNI_ARGS", cniArgs) != nil { - return nil, logging.Errorf("delegateAdd: cannot set %q mac to %q and ip to %q", delegate.Conf.Type, delegate.MacRequest, delegate.IPRequest) + ips := strings.Join(delegate.IPRequest, ",") + cniArgs = fmt.Sprintf("%s;IP=%s", cniArgs, ips) + logging.Debugf("delegateAdd: set IP address %q to %q", ips, ifName) + rt.Args = append(rt.Args, [2]string{ "IP", ips }) } } @@ -238,10 +281,7 @@ func delegateAdd(exec invoke.Exec, ifName string, delegate *types.DelegateNetCon return nil, logging.Errorf("delegateAdd: error invoking conflistAdd - %q: %v", delegate.ConfList.Name, err) } } else { - origpath := os.Getenv("CNI_PATH") - os.Setenv("CNI_PATH", origpath+":"+binDir) - result, err = invoke.DelegateAdd(context.Background(), delegate.Conf.Type, delegate.Bytes, exec) - os.Setenv("CNI_PATH", origpath) + result, err = confAdd(rt, delegate.Bytes, binDir, exec) if err != nil { return nil, logging.Errorf("delegateAdd: error invoking DelegateAdd - %q: %v", delegate.Conf.Type, err) } @@ -285,13 +325,10 @@ func delegateDel(exec invoke.Exec, ifName string, delegateConf *types.DelegateNe return logging.Errorf("delegateDel: error invoking ConflistDel - %q: %v", delegateConf.ConfList.Name, err) } } else { - origpath := os.Getenv("CNI_PATH") - os.Setenv("CNI_PATH", origpath+":"+binDir) - if err = invoke.DelegateDel(context.Background(), delegateConf.Conf.Type, delegateConf.Bytes, exec); err != nil { - os.Setenv("CNI_PATH", origpath) + err = confDel(rt, delegateConf.Bytes, binDir, exec) + if err != nil { return logging.Errorf("delegateDel: error invoking DelegateDel - %q: %v", delegateConf.Conf.Type, err) } - os.Setenv("CNI_PATH", origpath) } return err @@ -367,7 +404,9 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn cniArgs := os.Getenv("CNI_ARGS") for idx, delegate := range n.Delegates { ifName := getIfname(delegate, args.IfName, idx) - rt := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, n.RuntimeConfig) + + runtimeConfig := types.MergeCNIRuntimeConfig(n.RuntimeConfig, delegate) + rt := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, runtimeConfig) tmpResult, err = delegateAdd(exec, ifName, delegate, rt, n.BinDir, cniArgs) if err != nil { // If the add failed, tear down all networks we already added diff --git a/multus/multus_test.go b/multus/multus_test.go index 44840c2f0..fb0ff8e50 100644 --- a/multus/multus_test.go +++ b/multus/multus_test.go @@ -29,6 +29,7 @@ import ( "github.com/containernetworking/cni/pkg/skel" cnitypes "github.com/containernetworking/cni/pkg/types" types020 "github.com/containernetworking/cni/pkg/types/020" + current "github.com/containernetworking/cni/pkg/types/current" cniversion "github.com/containernetworking/cni/pkg/version" "github.com/containernetworking/plugins/pkg/ns" "github.com/containernetworking/plugins/pkg/testutils" @@ -62,7 +63,17 @@ type fakeExec struct { plugins []*fakePlugin } -func (f *fakeExec) addPlugin(expectedEnv []string, expectedIfname, expectedConf string, result *types020.Result, err error) { +func (f *fakeExec) addPlugin(expectedEnv []string, expectedIfname, expectedConf string, result *current.Result, err error) { + f.plugins = append(f.plugins, &fakePlugin{ + expectedEnv: expectedEnv, + expectedConf: expectedConf, + expectedIfname: expectedIfname, + result: result, + err: err, + }) +} + +func (f *fakeExec) addPlugin020(expectedEnv []string, expectedIfname, expectedConf string, result *types020.Result, err error) { f.plugins = append(f.plugins, &fakePlugin{ expectedEnv: expectedEnv, expectedConf: expectedConf, @@ -151,6 +162,7 @@ func (f *fakeExec) FindInPath(plugin string, paths []string) (string, error) { var _ = Describe("multus operations", func() { var testNS ns.NetNS var tmpDir string + resultCNIVersion := "0.4.0" BeforeEach(func() { // Create a new NetNS so we don't modify the host @@ -212,7 +224,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "weave-net" }` - fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil) + fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil) expectedResult2 := &types020.Result{ CNIVersion: "0.2.0", @@ -225,7 +237,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "other-plugin" }` - fExec.addPlugin(nil, "net1", expectedConf2, expectedResult2, nil) + fExec.addPlugin020(nil, "net1", expectedConf2, expectedResult2, nil) os.Setenv("CNI_COMMAND", "ADD") os.Setenv("CNI_IFNAME", "eth0") @@ -288,7 +300,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "weave-net" }` - fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil) + fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil) expectedResult2 := &types020.Result{ CNIVersion: "0.2.0", @@ -301,7 +313,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "other-plugin" }` - fExec.addPlugin(nil, "net1", expectedConf2, expectedResult2, nil) + fExec.addPlugin020(nil, "net1", expectedConf2, expectedResult2, nil) os.Setenv("CNI_COMMAND", "ADD") os.Setenv("CNI_IFNAME", "eth0") @@ -365,7 +377,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "weave-net" }` - fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil) + fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil) expectedResult2 := &types020.Result{ CNIVersion: "0.2.0", @@ -378,7 +390,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "other-plugin" }` - fExec.addPlugin(nil, "net1", expectedConf2, expectedResult2, nil) + fExec.addPlugin020(nil, "net1", expectedConf2, expectedResult2, nil) os.Setenv("CNI_COMMAND", "ADD") os.Setenv("CNI_IFNAME", "eth0") @@ -444,7 +456,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "weave-net" }` - fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil) + fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil) expectedResult2 := &types020.Result{ CNIVersion: "0.2.0", @@ -457,7 +469,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "other-plugin" }` - fExec.addPlugin(nil, "net1", expectedConf2, expectedResult2, nil) + fExec.addPlugin020(nil, "net1", expectedConf2, expectedResult2, nil) os.Setenv("CNI_COMMAND", "ADD") os.Setenv("CNI_IFNAME", "eth0") @@ -521,7 +533,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "weave-net" }` - fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil) + fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil) expectedResult2 := &types020.Result{ CNIVersion: "0.2.0", @@ -534,7 +546,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "other-plugin" }` - fExec.addPlugin(nil, "net1", expectedConf2, expectedResult2, nil) + fExec.addPlugin020(nil, "net1", expectedConf2, expectedResult2, nil) os.Setenv("CNI_COMMAND", "ADD") os.Setenv("CNI_IFNAME", "eth0") @@ -601,7 +613,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "weave-net" }` - fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil) + fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil) expectedResult2 := &types020.Result{ CNIVersion: "0.2.0", @@ -614,7 +626,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "other-plugin" }` - fExec.addPlugin(nil, "net1", expectedConf2, expectedResult2, nil) + fExec.addPlugin020(nil, "net1", expectedConf2, expectedResult2, nil) os.Setenv("CNI_COMMAND", "ADD") os.Setenv("CNI_IFNAME", "eth0") @@ -678,7 +690,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "weave-net" }` - fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil) + fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil) expectedResult2 := &types020.Result{ CNIVersion: "0.2.0", @@ -691,7 +703,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "other-plugin" }` - fExec.addPlugin(nil, "net1", expectedConf2, expectedResult2, nil) + fExec.addPlugin020(nil, "net1", expectedConf2, expectedResult2, nil) os.Setenv("CNI_COMMAND", "ADD") os.Setenv("CNI_IFNAME", "eth0") @@ -757,7 +769,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "weave-net" }` - fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil) + fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil) expectedResult2 := &types020.Result{ CNIVersion: "0.2.0", @@ -770,7 +782,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "other-plugin" }` - fExec.addPlugin(nil, "net1", expectedConf2, expectedResult2, nil) + fExec.addPlugin020(nil, "net1", expectedConf2, expectedResult2, nil) os.Setenv("CNI_COMMAND", "ADD") os.Setenv("CNI_IFNAME", "eth0") @@ -834,7 +846,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "weave-net" }` - fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil) + fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil) expectedResult2 := &types020.Result{ CNIVersion: "0.2.0", @@ -847,7 +859,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "other-plugin" }` - fExec.addPlugin(nil, "net1", expectedConf2, expectedResult2, nil) + fExec.addPlugin020(nil, "net1", expectedConf2, expectedResult2, nil) os.Setenv("CNI_COMMAND", "ADD") os.Setenv("CNI_IFNAME", "eth0") @@ -913,7 +925,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "weave-net" }` - fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil) + fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil) expectedResult2 := &types020.Result{ CNIVersion: "0.2.0", @@ -926,7 +938,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "other-plugin" }` - fExec.addPlugin(nil, "net1", expectedConf2, expectedResult2, nil) + fExec.addPlugin020(nil, "net1", expectedConf2, expectedResult2, nil) os.Setenv("CNI_COMMAND", "ADD") os.Setenv("CNI_IFNAME", "eth0") @@ -988,7 +1000,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "weave-net" }` - fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil) + fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil) expectedResult2 := &types020.Result{ CNIVersion: "0.2.0", @@ -1001,7 +1013,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "other-plugin" }` - fExec.addPlugin(nil, "net1", expectedConf2, expectedResult2, nil) + fExec.addPlugin020(nil, "net1", expectedConf2, expectedResult2, nil) os.Setenv("CNI_COMMAND", "ADD") os.Setenv("CNI_IFNAME", "eth0") @@ -1067,7 +1079,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "weave-net" }` - fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil) + fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil) expectedResult2 := &types020.Result{ CNIVersion: "0.2.0", @@ -1080,7 +1092,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "other-plugin" }` - fExec.addPlugin(nil, "net1", expectedConf2, expectedResult2, nil) + fExec.addPlugin020(nil, "net1", expectedConf2, expectedResult2, nil) os.Setenv("CNI_COMMAND", "ADD") os.Setenv("CNI_IFNAME", "eth0") @@ -1146,11 +1158,11 @@ var _ = Describe("multus operations", func() { IP: *testhelpers.EnsureCIDR("1.1.1.2/24"), }, } - fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil) + fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil) // This plugin invocation should fail err := fmt.Errorf("expected plugin failure") - fExec.addPlugin(nil, "net1", expectedConf2, nil, err) + fExec.addPlugin020(nil, "net1", expectedConf2, nil, err) os.Setenv("CNI_COMMAND", "ADD") os.Setenv("CNI_IFNAME", "eth0") @@ -1204,11 +1216,11 @@ var _ = Describe("multus operations", func() { IP: *testhelpers.EnsureCIDR("1.1.1.2/24"), }, } - fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil) + fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil) // This plugin invocation should fail err := fmt.Errorf("expected plugin failure") - fExec.addPlugin(nil, "net1", expectedConf2, nil, err) + fExec.addPlugin020(nil, "net1", expectedConf2, nil, err) os.Setenv("CNI_COMMAND", "ADD") os.Setenv("CNI_IFNAME", "eth0") @@ -1225,24 +1237,31 @@ var _ = Describe("multus operations", func() { }) - It("executes delegates with interface name and MAC and IP addr", func() { + It("executes delegates with runtimeConfigs", func() { podNet := `[{"name":"net1", - "interface": "test1", - "ips":"1.2.3.4/24"}, - {"name":"net2", - "mac": "c2:11:22:33:44:66", - "ips": "10.0.0.1"} + "mac": "c2:11:22:33:44:66", + "ips": [ "10.0.0.1" ], + "bandwidth": { + "ingressRate": 2048, + "ingressBurst": 1600, + "egressRate": 4096, + "egressBurst": 1600 + }, + "portMappings": [ + { + "hostPort": 8080, "containerPort": 80, "protocol": "tcp" + }, + { + "hostPort": 8000, "containerPort": 8001, "protocol": "udp" + }] + } ]` fakePod := testhelpers.NewFakePod("testpod", podNet, "") net1 := `{ "name": "net1", "type": "mynet", - "cniVersion": "0.2.0" - }` - net2 := `{ - "name": "net2", - "type": "mynet2", - "cniVersion": "0.2.0" + "capabilities": {"mac": true, "ips": true, "bandwidth": true, "portMappings": true}, + "cniVersion": "0.3.1" }` args := &skel.CmdArgs{ ContainerID: "123456789", @@ -1255,42 +1274,69 @@ var _ = Describe("multus operations", func() { "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml", "delegates": [{ "name": "weave1", - "cniVersion": "0.2.0", + "cniVersion": "0.3.1", "type": "weave-net" }] }`), } fExec := &fakeExec{} - expectedResult1 := &types020.Result{ - CNIVersion: "0.2.0", - IP4: &types020.IPConfig{ - IP: *testhelpers.EnsureCIDR("1.1.1.2/24"), + expectedResult1 := ¤t.Result{ + CNIVersion: resultCNIVersion, + IPs: []*current.IPConfig{¤t.IPConfig{ + Address: *testhelpers.EnsureCIDR("1.1.1.2/24"), + }, }, } expectedConf1 := `{ "name": "weave1", - "cniVersion": "0.2.0", + "cniVersion": "0.3.1", "type": "weave-net" }` - fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil) - fExec.addPlugin([]string{"CNI_ARGS=IgnoreUnknown=true;IP=1.2.3.4/24"}, "test1", net1, &types020.Result{ - CNIVersion: "0.2.0", - IP4: &types020.IPConfig{ - IP: *testhelpers.EnsureCIDR("1.1.1.3/24"), + expectedNet1 := `{ + "name": "net1", + "type": "mynet", + "capabilities": { + "mac": true, + "ips": true, + "bandwidth": true, + "portMappings": true + }, + "runtimeConfig": { + "ips": [ "10.0.0.1" ], + "mac": "c2:11:22:33:44:66", + "bandwidth": { + "ingressRate": 2048, + "ingressBurst": 1600, + "egressRate": 4096, + "egressBurst": 1600 + }, + "portMappings": [ + { + "hostPort": 8080, + "containerPort": 80, + "protocol": "tcp" + }, + { + "hostPort": 8000, + "containerPort": 8001, + "protocol": "udp" + }] + }, + "cniVersion": "0.3.1" + }` + fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil) + fExec.addPlugin(nil, "net1", expectedNet1, ¤t.Result{ + CNIVersion: "0.3.1", + IPs: []*current.IPConfig{¤t.IPConfig{ + Address: *testhelpers.EnsureCIDR("1.1.1.3/24"), }, - }, nil) - fExec.addPlugin([]string{"CNI_ARGS=IgnoreUnknown=true;MAC=c2:11:22:33:44:66;IP=10.0.0.1"}, "net2", net2, &types020.Result{ - CNIVersion: "0.2.0", - IP4: &types020.IPConfig{ - IP: *testhelpers.EnsureCIDR("1.1.1.4/24"), }, }, nil) fKubeClient := testhelpers.NewFakeKubeClient() fKubeClient.AddPod(fakePod) fKubeClient.AddNetConfig(fakePod.ObjectMeta.Namespace, "net1", net1) - fKubeClient.AddNetConfig(fakePod.ObjectMeta.Namespace, "net2", net2) os.Setenv("CNI_COMMAND", "ADD") os.Setenv("CNI_IFNAME", "eth0") @@ -1298,10 +1344,11 @@ var _ = Describe("multus operations", func() { Expect(err).NotTo(HaveOccurred()) Expect(fExec.addIndex).To(Equal(len(fExec.plugins))) Expect(fKubeClient.PodCount).To(Equal(2)) - Expect(fKubeClient.NetCount).To(Equal(2)) - r := result.(*types020.Result) + Expect(fKubeClient.NetCount).To(Equal(1)) + r := result.(*current.Result) // plugin 1 is the masterplugin Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue()) + }) It("executes delegates and kubernetes networks", func() { @@ -1350,14 +1397,14 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "weave-net" }` - fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil) - fExec.addPlugin(nil, "net1", net1, &types020.Result{ + fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil) + fExec.addPlugin020(nil, "net1", net1, &types020.Result{ CNIVersion: "0.2.0", IP4: &types020.IPConfig{ IP: *testhelpers.EnsureCIDR("1.1.1.3/24"), }, }, nil) - fExec.addPlugin(nil, "net2", net2, &types020.Result{ + fExec.addPlugin020(nil, "net2", net2, &types020.Result{ CNIVersion: "0.2.0", IP4: &types020.IPConfig{ IP: *testhelpers.EnsureCIDR("1.1.1.4/24"), @@ -1419,8 +1466,8 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "weave-net" }` - fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil) - fExec.addPlugin(nil, "net1", net1, &types020.Result{ + fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil) + fExec.addPlugin020(nil, "net1", net1, &types020.Result{ CNIVersion: "0.2.0", IP4: &types020.IPConfig{ IP: *testhelpers.EnsureCIDR("1.1.1.3/24"), @@ -1487,8 +1534,8 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "weave-net" }` - fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil) - fExec.addPlugin(nil, "net1", net1, &types020.Result{ + fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil) + fExec.addPlugin020(nil, "net1", net1, &types020.Result{ CNIVersion: "0.2.0", IP4: &types020.IPConfig{ IP: *testhelpers.EnsureCIDR("1.1.1.3/24"), @@ -1555,8 +1602,8 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "weave-net" }` - fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil) - fExec.addPlugin(nil, "net1", net1, &types020.Result{ + fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil) + fExec.addPlugin020(nil, "net1", net1, &types020.Result{ CNIVersion: "0.2.0", IP4: &types020.IPConfig{ IP: *testhelpers.EnsureCIDR("1.1.1.3/24"), @@ -1625,7 +1672,7 @@ var _ = Describe("multus operations", func() { ] } }` - fExec.addPlugin(nil, "eth0", expectedConf1, nil, nil) + fExec.addPlugin020(nil, "eth0", expectedConf1, nil, nil) os.Setenv("CNI_COMMAND", "ADD") os.Setenv("CNI_IFNAME", "eth0") _, err := cmdAdd(args, fExec, nil) @@ -1661,7 +1708,7 @@ var _ = Describe("multus operations", func() { } fExec := &fakeExec{} - fExec.addPlugin(nil, "eth0", net1, expectedResult1, nil) + fExec.addPlugin020(nil, "eth0", net1, expectedResult1, nil) fKubeClient := testhelpers.NewFakeKubeClient() fKubeClient.AddPod(fakePod) @@ -1725,8 +1772,8 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "weave-net" }` - fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil) - fExec.addPlugin(nil, "net1", net1, &types020.Result{ + fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil) + fExec.addPlugin020(nil, "net1", net1, &types020.Result{ CNIVersion: "0.2.0", IP4: &types020.IPConfig{ IP: *testhelpers.EnsureCIDR("1.1.1.3/24"), @@ -1803,8 +1850,8 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "weave-net" }` - fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil) - fExec.addPlugin(nil, "net1", net1, &types020.Result{ + fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil) + fExec.addPlugin020(nil, "net1", net1, &types020.Result{ CNIVersion: "0.2.0", IP4: &types020.IPConfig{ IP: *testhelpers.EnsureCIDR("1.1.1.3/24"), @@ -1881,7 +1928,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "weave-net" }` - fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil) + fExec.addPlugin020(nil, "eth0", expectedConf1, expectedResult1, nil) expectedResult2 := &types020.Result{ CNIVersion: "0.2.0", @@ -1894,7 +1941,7 @@ var _ = Describe("multus operations", func() { "cniVersion": "0.2.0", "type": "other-plugin" }` - fExec.addPlugin(nil, "net1", expectedConf2, expectedResult2, nil) + fExec.addPlugin020(nil, "net1", expectedConf2, expectedResult2, nil) os.Setenv("CNI_COMMAND", "ADD") os.Setenv("CNI_IFNAME", "eth0") @@ -1952,7 +1999,7 @@ var _ = Describe("multus operations", func() { ] } }` - fExec.addPlugin(nil, "eth0", expectedConf1, nil, nil) + fExec.addPlugin020(nil, "eth0", expectedConf1, nil, nil) os.Setenv("CNI_COMMAND", "ADD") os.Setenv("CNI_IFNAME", "eth0") diff --git a/types/conf.go b/types/conf.go index 91a9aeb83..2944f7fec 100644 --- a/types/conf.go +++ b/types/conf.go @@ -90,9 +90,15 @@ func LoadDelegateNetConf(bytes []byte, net *NetworkSelectionElement, deviceID st if net.MacRequest != "" { delegateConf.MacRequest = net.MacRequest } - if net.IPRequest != "" { + if net.IPRequest != nil { delegateConf.IPRequest = net.IPRequest } + if net.BandwidthRequest != nil { + delegateConf.BandwidthRequest = net.BandwidthRequest + } + if net.PortMappingsRequest != nil { + delegateConf.PortMappingsRequest = net.PortMappingsRequest + } } delegateConf.Bytes = bytes @@ -100,6 +106,33 @@ func LoadDelegateNetConf(bytes []byte, net *NetworkSelectionElement, deviceID st return delegateConf, nil } +// MergeCNIRuntimeConfig creates CNI runtimeconfig from delegate +func MergeCNIRuntimeConfig(runtimeConfig *RuntimeConfig, delegate *DelegateNetConf) *RuntimeConfig { + logging.Debugf("MergeCNIRuntimeConfig: %v %v", runtimeConfig, delegate) + if runtimeConfig == nil { + runtimeConfig = &RuntimeConfig{} + } + + // multus inject RuntimeConfig only in case of non MasterPlugin. + if delegate.MasterPlugin != true { + logging.Debugf("MergeCNIRuntimeConfig: add runtimeConfig for net-attach-def: %v", runtimeConfig) + if delegate.PortMappingsRequest != nil { + runtimeConfig.PortMaps = delegate.PortMappingsRequest + } + if delegate.BandwidthRequest != nil { + runtimeConfig.Bandwidth = delegate.BandwidthRequest + } + if delegate.IPRequest != nil { + runtimeConfig.IPs = delegate.IPRequest + } + if delegate.MacRequest != "" { + runtimeConfig.Mac = delegate.MacRequest + } + } + + return runtimeConfig +} + // CreateCNIRuntimeConf create CNI RuntimeConf func CreateCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, rc *RuntimeConfig) *libcni.RuntimeConf { logging.Debugf("LoadCNIRuntimeConf: %v, %v, %s, %v", args, k8sArgs, ifName, rc) @@ -112,7 +145,7 @@ func CreateCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, r NetNS: args.Netns, IfName: ifName, Args: [][2]string{ - {"IgnoreUnknown", "1"}, + {"IgnoreUnknown", string("true")}, {"K8S_POD_NAMESPACE", string(k8sArgs.K8S_POD_NAMESPACE)}, {"K8S_POD_NAME", string(k8sArgs.K8S_POD_NAME)}, {"K8S_POD_INFRA_CONTAINER_ID", string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID)}, @@ -120,9 +153,20 @@ func CreateCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, r } if rc != nil { - rt.CapabilityArgs = map[string]interface{}{ - "portMappings": rc.PortMaps, + capabilityArgs := map[string]interface{}{} + if len(rc.PortMaps) != 0 { + capabilityArgs["portMappings"] = rc.PortMaps } + if rc.Bandwidth != nil { + capabilityArgs["bandwidth"] = rc.Bandwidth + } + if len(rc.IPs) != 0 { + capabilityArgs["ips"] = rc.IPs + } + if len(rc.Mac) != 0 { + capabilityArgs["mac"] = rc.Mac + } + rt.CapabilityArgs = capabilityArgs } return rt } diff --git a/types/conf_test.go b/types/conf_test.go index 07885b3af..3111e6112 100644 --- a/types/conf_test.go +++ b/types/conf_test.go @@ -383,17 +383,20 @@ var _ = Describe("config operations", func() { k8sArgs := &K8sArgs{K8S_POD_NAME: "dummy", K8S_POD_NAMESPACE: "namespacedummy", K8S_POD_INFRA_CONTAINER_ID: "123456789"} rc := &RuntimeConfig{} - tempportmap := make([]PortMapEntry, 2) - rc.PortMaps = tempportmap + rc.PortMaps = make([]*PortMapEntry, 2) - rc.PortMaps[0].HostPort = 0 - rc.PortMaps[0].ContainerPort = 1 - rc.PortMaps[0].Protocol = "sampleProtocol" - rc.PortMaps[0].HostIP = "sampleHostIP" - rc.PortMaps[1].HostPort = 1 - rc.PortMaps[1].ContainerPort = 2 - rc.PortMaps[1].Protocol = "anotherSampleProtocol" - rc.PortMaps[1].HostIP = "anotherSampleHostIP" + rc.PortMaps[0] = &PortMapEntry{ + HostPort: 0, + ContainerPort: 1, + Protocol: "sampleProtocol", + HostIP: "sampleHostIP", + } + rc.PortMaps[1] = &PortMapEntry{ + HostPort: 1, + ContainerPort: 2, + Protocol: "anotherSampleProtocol", + HostIP: "anotherSampleHostIP", + } rt := CreateCNIRuntimeConf(args, k8sArgs, "", rc) fmt.Println("rt.ContainerID: ", rt.ContainerID) @@ -467,4 +470,123 @@ var _ = Describe("config operations", func() { Expect(err).To(HaveOccurred()) }) + + It("verify the network selection elements goes into delegateconf", func() { + cniConfig := `{ + "name": "weave1", + "cniVersion": "0.2.0", + "type": "weave-net" + }` + bandwidthEntry1 := &BandwidthEntry{ + IngressRate: 100, + IngressBurst: 200, + EgressRate: 100, + EgressBurst: 200, + } + + portMapEntry1 := &PortMapEntry{ + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + HostIP: "10.0.0.1", + } + + networkSelection := &NetworkSelectionElement{ + Name: "testname", + InterfaceRequest: "testIF1", + MacRequest: "c2:11:22:33:44:66", + IPRequest: []string{"10.0.0.1/24"}, + BandwidthRequest: bandwidthEntry1, + PortMappingsRequest: []*PortMapEntry{portMapEntry1}, + } + + delegateConf, err := LoadDelegateNetConf([]byte(cniConfig), networkSelection, "") + Expect(err).NotTo(HaveOccurred()) + Expect(delegateConf.IfnameRequest).To(Equal(networkSelection.InterfaceRequest)) + Expect(delegateConf.MacRequest).To(Equal(networkSelection.MacRequest)) + Expect(delegateConf.IPRequest).To(Equal(networkSelection.IPRequest)) + Expect(delegateConf.BandwidthRequest).To(Equal(networkSelection.BandwidthRequest)) + Expect(delegateConf.PortMappingsRequest).To(Equal(networkSelection.PortMappingsRequest)) + }) + + It("test MergeCNIRuntimeConfig with masterPlugin", func() { + conf := `{ + "name": "node-cni-network", + "type": "multus", + "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml", + "delegates": [{ + "type": "weave-net" + }], + "runtimeConfig": { + "portMappings": [ + {"hostPort": 8080, "containerPort": 80, "protocol": "tcp"} + ] + } + }` + bandwidthEntry1 := &BandwidthEntry{ + IngressRate: 100, + IngressBurst: 200, + EgressRate: 100, + EgressBurst: 200, + } + portMapEntry1 := &PortMapEntry{ + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + HostIP: "10.0.0.1", + } + + networkSelection := &NetworkSelectionElement{ + Name: "testname", + InterfaceRequest: "testIF1", + MacRequest: "c2:11:22:33:44:66", + IPRequest: []string{"10.0.0.1/24"}, + BandwidthRequest: bandwidthEntry1, + PortMappingsRequest: []*PortMapEntry{portMapEntry1}, + } + delegate, err := LoadDelegateNetConf([]byte(conf), networkSelection, "") + delegate.MasterPlugin = true + Expect(err).NotTo(HaveOccurred()) + runtimeConf := MergeCNIRuntimeConfig(&RuntimeConfig{}, delegate) + Expect(runtimeConf.PortMaps).To(BeNil()) + Expect(runtimeConf.Bandwidth).To(BeNil()) + }) + + It("test MergeCNIRuntimeConfig with delegate plugin", func() { + conf := `{ + "name": "weave1", + "cniVersion": "0.2.0", + "type": "weave-net" + }` + bandwidthEntry1 := &BandwidthEntry{ + IngressRate: 100, + IngressBurst: 200, + EgressRate: 100, + EgressBurst: 200, + } + portMapEntry1 := &PortMapEntry{ + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + HostIP: "10.0.0.1", + } + + networkSelection := &NetworkSelectionElement{ + Name: "testname", + InterfaceRequest: "testIF1", + MacRequest: "c2:11:22:33:44:66", + IPRequest: []string{"10.0.0.1/24"}, + BandwidthRequest: bandwidthEntry1, + PortMappingsRequest: []*PortMapEntry{portMapEntry1}, + } + delegate, err := LoadDelegateNetConf([]byte(conf), networkSelection, "") + Expect(err).NotTo(HaveOccurred()) + runtimeConf := MergeCNIRuntimeConfig(&RuntimeConfig{}, delegate) + Expect(runtimeConf.PortMaps).NotTo(BeNil()) + Expect(len(runtimeConf.PortMaps)).To(BeEquivalentTo(1)) + Expect(runtimeConf.PortMaps[0]).To(Equal(portMapEntry1)) + Expect(runtimeConf.Bandwidth).To(Equal(bandwidthEntry1)) + Expect(len(runtimeConf.IPs)).To(BeEquivalentTo(1)) + Expect(runtimeConf.Mac).To(Equal("c2:11:22:33:44:66")) + }) }) diff --git a/types/types.go b/types/types.go index 02321d82e..693b98084 100644 --- a/types/types.go +++ b/types/types.go @@ -58,17 +58,29 @@ type NetConf struct { // RuntimeConfig specifies CNI RuntimeConfig type RuntimeConfig struct { - PortMaps []PortMapEntry `json:"portMappings,omitempty"` + PortMaps []*PortMapEntry `json:"portMappings,omitempty"` + Bandwidth *BandwidthEntry `json:"bandwidth,omitempty"` + IPs []string `json:"ips,omitempty"` + Mac string `json:"mac,omitempty"` } // PortMapEntry for CNI PortMapEntry type PortMapEntry struct { HostPort int `json:"hostPort"` ContainerPort int `json:"containerPort"` - Protocol string `json:"protocol"` + Protocol string `json:"protocol,omitempty"` HostIP string `json:"hostIP,omitempty"` } +// BandwidthEntry for CNI BandwidthEntry +type BandwidthEntry struct { + IngressRate int `json:"ingressRate"` + IngressBurst int `json:"ingressBurst"` + + EgressRate int `json:"egressRate"` + EgressBurst int `json:"egressBurst"` +} + // NetworkStatus is for network status annotation for pod type NetworkStatus struct { Name string `json:"name"` @@ -81,11 +93,13 @@ type NetworkStatus struct { // DelegateNetConf for net-attach-def for pod type DelegateNetConf struct { - Conf types.NetConf - ConfList types.NetConfList - IfnameRequest string `json:"ifnameRequest,omitempty"` - MacRequest string `json:"macRequest,omitempty"` - IPRequest string `json:"ipRequest,omitempty"` + Conf types.NetConf + ConfList types.NetConfList + IfnameRequest string `json:"ifnameRequest,omitempty"` + MacRequest string `json:"macRequest,omitempty"` + IPRequest []string `json:"ipRequest,omitempty"` + PortMappingsRequest []*PortMapEntry `json:"-"` + BandwidthRequest *BandwidthEntry `json:"-"` // MasterPlugin is only used internal housekeeping MasterPlugin bool `json:"-"` // Conflist plugin is only used internal housekeeping @@ -133,16 +147,22 @@ type NetworkSelectionElement struct { Namespace string `json:"namespace,omitempty"` // IPRequest contains an optional requested IP address for this network // attachment - IPRequest string `json:"ips,omitempty"` + IPRequest []string `json:"ips,omitempty"` // MacRequest contains an optional requested MAC address for this // network attachment MacRequest string `json:"mac,omitempty"` // InterfaceRequest contains an optional requested name for the // network interface this attachment will create in the container InterfaceRequest string `json:"interface,omitempty"` - // ObsoateInterfaceRequest is obsolated parameter at pre 3.2. + // DeprecatedInterfaceRequest is obsolated parameter at pre 3.2. // This will be removed in 4.0 release. - ObsolatedInterfaceRequest string `json:"interfaceRequest,omitempty"` + DeprecatedInterfaceRequest string `json:"interfaceRequest,omitempty"` + // PortMappingsRequest contains an optional requested port mapping + // for the network + PortMappingsRequest []*PortMapEntry `json:"portMappings,omitempty"` + // BandwidthRequest contains an optional requested bandwidth for + // the network + BandwidthRequest *BandwidthEntry `json:"bandwidth,omitempty"` } // K8sArgs is the valid CNI_ARGS used for Kubernetes