From 555a734e2193b38f1da85c231d95750b74755045 Mon Sep 17 00:00:00 2001 From: Peng Liu Date: Fri, 28 Sep 2018 15:02:05 +0800 Subject: [PATCH] Specify Pod default network in Annotations Signed-off-by: Peng Liu --- k8sclient/k8sclient.go | 54 +++++++++++++++++++ k8sclient/k8sclient_test.go | 100 ++++++++++++++++++++++++++++++++---- multus/multus_test.go | 8 +-- testing/testing.go | 14 +++-- 4 files changed, 157 insertions(+), 19 deletions(-) diff --git a/k8sclient/k8sclient.go b/k8sclient/k8sclient.go index 1bbfe9516..5e7b9a390 100644 --- a/k8sclient/k8sclient.go +++ b/k8sclient/k8sclient.go @@ -426,6 +426,16 @@ func TryLoadPodDelegates(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient } setKubeClientInfo(clientInfo, kubeClient, k8sArgs) + + delegate, err := tryLoadK8sPodDefaultNetwork(k8sArgs, conf, kubeClient) + logging.Debugf("TryLoadK8sDelegates: load cluster network %v from pod annotations", delegate) + if err != nil { + return 0, nil, logging.Errorf("TryLoadK8sDelegates: Err in loading K8s cluster default network from pod annotation: %v", err) + }else if delegate != nil{ + // Overwrite the cluster default network. + conf.Delegates[0] = delegate + } + delegates, err := GetPodNetwork(kubeClient, k8sArgs, conf.ConfDir) if err != nil { if _, ok := err.(*NoK8sNetworkError); ok { @@ -624,3 +634,47 @@ func GetDefaultNetworks(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient return nil } + +func getPodDefaultNetworkAnnotation(client KubeClient, k8sArgs *types.K8sArgs) (string, error) { + logging.Debugf("getPodDefaultNetworkAnnotation: %v, %v", client, k8sArgs) + pod, err := client.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME)) + if err != nil { + return "", logging.Errorf("getPodDefaultNetworkAnnotation: failed to query the pod %v in out of cluster comm: %v", string(k8sArgs.K8S_POD_NAME), err) + } + + if v, ok := pod.Annotations["multus-cni.io/default-network"]; ok { + return v, nil + } else { + return "", nil + } +} + +// tryLoadK8sPodDefaultNetwork get pod default network from annotations +func tryLoadK8sPodDefaultNetwork(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient KubeClient) (*types.DelegateNetConf, error) { + logging.Debugf("tryLoadK8sPodDefaultNetwork: %v, %v", kubeClient, k8sArgs) + + netAnnot, err := getPodDefaultNetworkAnnotation(kubeClient, k8sArgs) + if err != nil { + return nil, err + } + if netAnnot == "" { + return nil, nil + } + + // The CRD object of default network should only be defined in default namespace + networks, err := parsePodNetworkAnnotation(netAnnot, "default") + if err != nil { + return nil, err + } + if len(networks) > 1 { + return nil, logging.Errorf("tryLoadK8sPodDefaultNetwork: more than one default network is specified: %s", netAnnot) + } + + delegate, _, err := getKubernetesDelegate(kubeClient, networks[0], conf.ConfDir, "", nil) + if err != nil { + return nil, logging.Errorf("tryLoadK8sPodDefaultNetwork: failed getting the delegate: %v", err) + } + delegate.MasterPlugin = true + + return delegate, nil +} diff --git a/k8sclient/k8sclient_test.go b/k8sclient/k8sclient_test.go index 1adf10803..24e20a673 100644 --- a/k8sclient/k8sclient_test.go +++ b/k8sclient/k8sclient_test.go @@ -51,7 +51,7 @@ var _ = Describe("k8sclient operations", func() { }) It("retrieves delegates from kubernetes using simple format annotation", func() { - fakePod := testutils.NewFakePod("testpod", "net1,net2") + fakePod := testutils.NewFakePod("testpod", "net1,net2", "") net1 := `{ "name": "net1", "type": "mynet", @@ -97,7 +97,7 @@ var _ = Describe("k8sclient operations", func() { }) It("fails when the network does not exist", func() { - fakePod := testutils.NewFakePod("testpod", "net1,net2") + fakePod := testutils.NewFakePod("testpod", "net1,net2", "") net3 := `{ "name": "net3", "type": "mynet3", @@ -132,7 +132,7 @@ var _ = Describe("k8sclient operations", func() { "name":"net3", "namespace":"other-ns" } -]`) +]`, "") args := &skel.CmdArgs{ Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace), } @@ -174,7 +174,7 @@ var _ = Describe("k8sclient operations", func() { }) It("fails when the JSON format annotation is invalid", func() { - fakePod := testutils.NewFakePod("testpod", "[adsfasdfasdfasf]") + fakePod := testutils.NewFakePod("testpod", "[adsfasdfasdfasf]", "") args := &skel.CmdArgs{ Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace), } @@ -192,7 +192,7 @@ var _ = Describe("k8sclient operations", func() { }) It("retrieves delegates from kubernetes using on-disk config files", func() { - fakePod := testutils.NewFakePod("testpod", "net1,net2") + fakePod := testutils.NewFakePod("testpod", "net1,net2", "") args := &skel.CmdArgs{ Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace), } @@ -229,7 +229,7 @@ var _ = Describe("k8sclient operations", func() { }) It("injects network name into minimal thick plugin CNI config", func() { - fakePod := testutils.NewFakePod("testpod", "net1") + fakePod := testutils.NewFakePod("testpod", "net1", "") args := &skel.CmdArgs{ Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace), } @@ -253,7 +253,7 @@ var _ = Describe("k8sclient operations", func() { }) It("fails when on-disk config file is not valid", func() { - fakePod := testutils.NewFakePod("testpod", "net1,net2") + fakePod := testutils.NewFakePod("testpod", "net1,net2", "") args := &skel.CmdArgs{ Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace), } @@ -279,7 +279,7 @@ var _ = Describe("k8sclient operations", func() { }) It("retrieves cluster network from CRD", func() { - fakePod := testutils.NewFakePod("testpod", "") + fakePod := testutils.NewFakePod("testpod", "", "") conf := `{ "name":"node-cni-network", "type":"multus", @@ -294,6 +294,7 @@ var _ = Describe("k8sclient operations", func() { } fKubeClient := testutils.NewFakeKubeClient() + fKubeClient.AddPod(fakePod) fKubeClient.AddNetConfig("default", "myCRD1", "{\"type\": \"mynet\"}") kubeClient, err := GetK8sClient("", fKubeClient) Expect(err).NotTo(HaveOccurred()) @@ -308,7 +309,7 @@ var _ = Describe("k8sclient operations", func() { }) It("retrieves cluster network from file", func() { - fakePod := testutils.NewFakePod("testpod", "") + fakePod := testutils.NewFakePod("testpod", "", "") conf := `{ "name":"node-cni-network", "type":"multus", @@ -324,6 +325,7 @@ var _ = Describe("k8sclient operations", func() { } fKubeClient := testutils.NewFakeKubeClient() + fKubeClient.AddPod(fakePod) net1Name := filepath.Join(tmpDir, "10-net1.conf") fKubeClient.AddNetFile(fakePod.ObjectMeta.Namespace, "net1", net1Name, `{ "name": "myFile1", @@ -343,7 +345,7 @@ var _ = Describe("k8sclient operations", func() { }) It("retrieves cluster network from path", func() { - fakePod := testutils.NewFakePod("testpod", "") + fakePod := testutils.NewFakePod("testpod", "", "") conf := fmt.Sprintf(`{ "name":"node-cni-network", "type":"multus", @@ -358,6 +360,7 @@ var _ = Describe("k8sclient operations", func() { } fKubeClient := testutils.NewFakeKubeClient() + fKubeClient.AddPod(fakePod) net1Name := filepath.Join(tmpDir, "10-net1.conf") fKubeClient.AddNetFile(fakePod.ObjectMeta.Namespace, "10-net1", net1Name, `{ "name": "net1", @@ -377,7 +380,7 @@ var _ = Describe("k8sclient operations", func() { }) It("Error in case of CRD not found", func() { - fakePod := testutils.NewFakePod("testpod", "") + fakePod := testutils.NewFakePod("testpod", "", "") conf := `{ "name":"node-cni-network", "type":"multus", @@ -392,6 +395,7 @@ var _ = Describe("k8sclient operations", func() { } fKubeClient := testutils.NewFakeKubeClient() + fKubeClient.AddPod(fakePod) kubeClient, err := GetK8sClient("", fKubeClient) Expect(err).NotTo(HaveOccurred()) k8sArgs, err := GetK8sArgs(args) @@ -400,4 +404,78 @@ var _ = Describe("k8sclient operations", func() { err = GetDefaultNetworks(k8sArgs, netConf, kubeClient) Expect(err).To(HaveOccurred()) }) + + It("overwrite cluster network when Pod annotation is set", func() { + + fakePod := testutils.NewFakePod("testpod", "", "net1") + conf := `{ + "name":"node-cni-network", + "type":"multus", + "clusterNetwork": "net2", + "kubeconfig":"/etc/kubernetes/node-kubeconfig.yaml" + }` + netConf, err := types.LoadNetConf([]byte(conf)) + Expect(err).NotTo(HaveOccurred()) + + args := &skel.CmdArgs{ + Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace), + } + + fKubeClient := testutils.NewFakeKubeClient() + fKubeClient.AddPod(fakePod) + fKubeClient.AddNetConfig("default", "net1", "{\"type\": \"mynet1\"}") + fKubeClient.AddNetConfig("default", "net2", "{\"type\": \"mynet2\"}") + kubeClient, err := GetK8sClient("", fKubeClient) + Expect(err).NotTo(HaveOccurred()) + k8sArgs, err := GetK8sArgs(args) + Expect(err).NotTo(HaveOccurred()) + + err = GetDefaultNetworks(k8sArgs, netConf, kubeClient) + Expect(err).NotTo(HaveOccurred()) + Expect(len(netConf.Delegates)).To(Equal(1)) + Expect(netConf.Delegates[0].Conf.Name).To(Equal("net2")) + Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet2")) + + numK8sDelegates, _, err := TryLoadPodDelegates(k8sArgs, netConf, kubeClient) + Expect(err).NotTo(HaveOccurred()) + Expect(numK8sDelegates).To(Equal(0)) + Expect(netConf.Delegates[0].Conf.Name).To(Equal("net1")) + Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet1")) + }) + + It("overwrite multus config when Pod annotation is set", func() { + + fakePod := testutils.NewFakePod("testpod", "", "net1") + conf := `{ + "name":"node-cni-network", + "type":"multus", + "kubeconfig":"/etc/kubernetes/node-kubeconfig.yaml", + "delegates": [{ + "type": "mynet2", + "name": "net2" + }] + }` + netConf, err := types.LoadNetConf([]byte(conf)) + Expect(netConf.Delegates[0].Conf.Name).To(Equal("net2")) + Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet2")) + Expect(err).NotTo(HaveOccurred()) + + args := &skel.CmdArgs{ + Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace), + } + + fKubeClient := testutils.NewFakeKubeClient() + fKubeClient.AddPod(fakePod) + fKubeClient.AddNetConfig("default", "net1", "{\"type\": \"mynet1\"}") + kubeClient, err := GetK8sClient("", fKubeClient) + Expect(err).NotTo(HaveOccurred()) + k8sArgs, err := GetK8sArgs(args) + Expect(err).NotTo(HaveOccurred()) + + numK8sDelegates, _, err := TryLoadPodDelegates(k8sArgs, netConf, kubeClient) + Expect(err).NotTo(HaveOccurred()) + Expect(numK8sDelegates).To(Equal(0)) + Expect(netConf.Delegates[0].Conf.Name).To(Equal("net1")) + Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet1")) + }) }) diff --git a/multus/multus_test.go b/multus/multus_test.go index 2b3e1833d..2547247c5 100644 --- a/multus/multus_test.go +++ b/multus/multus_test.go @@ -253,7 +253,7 @@ var _ = Describe("multus operations", func() { "mac": "c2:11:22:33:44:66", "ips": "10.0.0.1"} ]` - fakePod := testhelpers.NewFakePod("testpod", podNet) + fakePod := testhelpers.NewFakePod("testpod", podNet, "") net1 := `{ "name": "net1", "type": "mynet", @@ -317,7 +317,7 @@ var _ = Describe("multus operations", func() { result, err := cmdAdd(args, fExec, fKubeClient) Expect(err).NotTo(HaveOccurred()) Expect(fExec.addIndex).To(Equal(len(fExec.plugins))) - Expect(fKubeClient.PodCount).To(Equal(2)) + Expect(fKubeClient.PodCount).To(Equal(3)) Expect(fKubeClient.NetCount).To(Equal(2)) r := result.(*types020.Result) // plugin 1 is the masterplugin @@ -325,7 +325,7 @@ var _ = Describe("multus operations", func() { }) It("executes delegates and kubernetes networks", func() { - fakePod := testhelpers.NewFakePod("testpod", "net1,net2") + fakePod := testhelpers.NewFakePod("testpod", "net1,net2", "") net1 := `{ "name": "net1", "type": "mynet", @@ -396,7 +396,7 @@ var _ = Describe("multus operations", func() { result, err := cmdAdd(args, fExec, fKubeClient) Expect(err).NotTo(HaveOccurred()) Expect(fExec.addIndex).To(Equal(len(fExec.plugins))) - Expect(fKubeClient.PodCount).To(Equal(2)) + Expect(fKubeClient.PodCount).To(Equal(3)) Expect(fKubeClient.NetCount).To(Equal(2)) r := result.(*types020.Result) // plugin 1 is the masterplugin diff --git a/testing/testing.go b/testing/testing.go index ceadd95fa..0e3d8453f 100644 --- a/testing/testing.go +++ b/testing/testing.go @@ -103,7 +103,7 @@ func (f *FakeKubeClient) AddPod(pod *v1.Pod) { f.pods[key] = pod } -func NewFakePod(name string, netAnnotation string) *v1.Pod { +func NewFakePod(name string, netAnnotation string, defaultNetAnnotation string) *v1.Pod { pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -115,13 +115,19 @@ func NewFakePod(name string, netAnnotation string) *v1.Pod { }, }, } + annotations := make(map[string]string) + if netAnnotation != "" { netAnnotation = strings.Replace(netAnnotation, "\n", "", -1) netAnnotation = strings.Replace(netAnnotation, "\t", "", -1) - pod.ObjectMeta.Annotations = map[string]string{ - "k8s.v1.cni.cncf.io/networks": netAnnotation, - } + annotations["k8s.v1.cni.cncf.io/networks"] = netAnnotation } + + if defaultNetAnnotation != "" { + annotations["multus-cni.io/default-network"] = defaultNetAnnotation + } + + pod.ObjectMeta.Annotations = annotations return pod }