Specify Pod default network in Annotations

Signed-off-by: Peng Liu <pliu@redhat.com>

k8sclient/k8sclient.go

@@ -426,6 +426,16 @@ func TryLoadPodDelegates(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient
 	}
 
 	setKubeClientInfo(clientInfo, kubeClient, k8sArgs)
+
+	delegate, err := tryLoadK8sPodDefaultNetwork(k8sArgs, conf, kubeClient)
+	logging.Debugf("TryLoadK8sDelegates: load cluster network %v from pod annotations", delegate)
+	if err != nil {
+		return 0, nil, logging.Errorf("TryLoadK8sDelegates: Err in loading K8s cluster default network from pod annotation: %v", err)
+	}else if delegate != nil{
+		// Overwrite the cluster default network.
+		conf.Delegates[0] = delegate
+	}
+
 	delegates, err := GetPodNetwork(kubeClient, k8sArgs, conf.ConfDir)
 	if err != nil {
 		if _, ok := err.(*NoK8sNetworkError); ok {
@@ -624,3 +634,47 @@ func GetDefaultNetworks(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient
 
 	return nil
 }
+
+func getPodDefaultNetworkAnnotation(client KubeClient, k8sArgs *types.K8sArgs) (string, error) {
+	logging.Debugf("getPodDefaultNetworkAnnotation: %v, %v", client, k8sArgs)
+	pod, err := client.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
+	if err != nil {
+		return "", logging.Errorf("getPodDefaultNetworkAnnotation: failed to query the pod %v in out of cluster comm: %v", string(k8sArgs.K8S_POD_NAME), err)
+	}
+
+	if v, ok := pod.Annotations["multus-cni.io/default-network"]; ok {
+		return v, nil
+	} else {
+		return "", nil
+	}
+}
+
+// tryLoadK8sPodDefaultNetwork get pod default network from annotations
+func tryLoadK8sPodDefaultNetwork(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient KubeClient) (*types.DelegateNetConf, error) {
+	logging.Debugf("tryLoadK8sPodDefaultNetwork: %v, %v", kubeClient, k8sArgs)
+
+	netAnnot, err := getPodDefaultNetworkAnnotation(kubeClient, k8sArgs)
+	if err != nil {
+		return nil, err
+	}
+	if netAnnot == "" {
+		return nil, nil
+	}
+
+	// The CRD object of default network should only be defined in default namespace
+	networks, err := parsePodNetworkAnnotation(netAnnot, "default")
+	if err != nil {
+		return nil, err
+	}
+	if len(networks) > 1 {
+		return nil, logging.Errorf("tryLoadK8sPodDefaultNetwork: more than one default network is specified: %s", netAnnot)
+	}
+
+	delegate, _, err := getKubernetesDelegate(kubeClient, networks[0], conf.ConfDir, "", nil)
+	if err != nil {
+		return nil, logging.Errorf("tryLoadK8sPodDefaultNetwork: failed getting the delegate: %v", err)
+	}
+	delegate.MasterPlugin = true
+
+	return delegate, nil
+}

k8sclient/k8sclient_test.go

@@ -51,7 +51,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("retrieves delegates from kubernetes using simple format annotation", func() {
-		fakePod := testutils.NewFakePod("testpod", "net1,net2")
+		fakePod := testutils.NewFakePod("testpod", "net1,net2", "")
 		net1 := `{
 	"name": "net1",
 	"type": "mynet",
@@ -97,7 +97,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("fails when the network does not exist", func() {
-		fakePod := testutils.NewFakePod("testpod", "net1,net2")
+		fakePod := testutils.NewFakePod("testpod", "net1,net2", "")
 		net3 := `{
 	"name": "net3",
 	"type": "mynet3",
@@ -132,7 +132,7 @@ var _ = Describe("k8sclient operations", func() {
   "name":"net3",
   "namespace":"other-ns"
 }
-]`)
+]`, "")
 		args := &skel.CmdArgs{
 			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
 		}
@@ -174,7 +174,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("fails when the JSON format annotation is invalid", func() {
-		fakePod := testutils.NewFakePod("testpod", "[adsfasdfasdfasf]")
+		fakePod := testutils.NewFakePod("testpod", "[adsfasdfasdfasf]", "")
 		args := &skel.CmdArgs{
 			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
 		}
@@ -192,7 +192,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("retrieves delegates from kubernetes using on-disk config files", func() {
-		fakePod := testutils.NewFakePod("testpod", "net1,net2")
+		fakePod := testutils.NewFakePod("testpod", "net1,net2", "")
 		args := &skel.CmdArgs{
 			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
 		}
@@ -229,7 +229,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("injects network name into minimal thick plugin CNI config", func() {
-		fakePod := testutils.NewFakePod("testpod", "net1")
+		fakePod := testutils.NewFakePod("testpod", "net1", "")
 		args := &skel.CmdArgs{
 			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
 		}
@@ -253,7 +253,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("fails when on-disk config file is not valid", func() {
-		fakePod := testutils.NewFakePod("testpod", "net1,net2")
+		fakePod := testutils.NewFakePod("testpod", "net1,net2", "")
 		args := &skel.CmdArgs{
 			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
 		}
@@ -279,7 +279,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("retrieves cluster network from CRD", func() {
-		fakePod := testutils.NewFakePod("testpod", "")
+		fakePod := testutils.NewFakePod("testpod", "", "")
 		conf := `{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -294,6 +294,7 @@ var _ = Describe("k8sclient operations", func() {
 		}
 
 		fKubeClient := testutils.NewFakeKubeClient()
+		fKubeClient.AddPod(fakePod)
 		fKubeClient.AddNetConfig("default", "myCRD1", "{\"type\": \"mynet\"}")
 		kubeClient, err := GetK8sClient("", fKubeClient)
 		Expect(err).NotTo(HaveOccurred())
@@ -308,7 +309,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("retrieves cluster network from file", func() {
-		fakePod := testutils.NewFakePod("testpod", "")
+		fakePod := testutils.NewFakePod("testpod", "", "")
 		conf := `{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -324,6 +325,7 @@ var _ = Describe("k8sclient operations", func() {
 		}
 
 		fKubeClient := testutils.NewFakeKubeClient()
+		fKubeClient.AddPod(fakePod)
 		net1Name := filepath.Join(tmpDir, "10-net1.conf")
 		fKubeClient.AddNetFile(fakePod.ObjectMeta.Namespace, "net1", net1Name, `{
 	"name": "myFile1",
@@ -343,7 +345,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("retrieves cluster network from path", func() {
-		fakePod := testutils.NewFakePod("testpod", "")
+		fakePod := testutils.NewFakePod("testpod", "", "")
 		conf := fmt.Sprintf(`{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -358,6 +360,7 @@ var _ = Describe("k8sclient operations", func() {
 		}
 
 		fKubeClient := testutils.NewFakeKubeClient()
+		fKubeClient.AddPod(fakePod)
 		net1Name := filepath.Join(tmpDir, "10-net1.conf")
 		fKubeClient.AddNetFile(fakePod.ObjectMeta.Namespace, "10-net1", net1Name, `{
 	"name": "net1",
@@ -377,7 +380,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("Error in case of CRD not found", func() {
-		fakePod := testutils.NewFakePod("testpod", "")
+		fakePod := testutils.NewFakePod("testpod", "", "")
 		conf := `{
 			"name":"node-cni-network",
 			"type":"multus",
@@ -392,6 +395,7 @@ var _ = Describe("k8sclient operations", func() {
 		}
 
 		fKubeClient := testutils.NewFakeKubeClient()
+		fKubeClient.AddPod(fakePod)
 		kubeClient, err := GetK8sClient("", fKubeClient)
 		Expect(err).NotTo(HaveOccurred())
 		k8sArgs, err := GetK8sArgs(args)
@@ -400,4 +404,78 @@ var _ = Describe("k8sclient operations", func() {
 		err = GetDefaultNetworks(k8sArgs, netConf, kubeClient)
 		Expect(err).To(HaveOccurred())
 	})
+
+	It("overwrite cluster network when Pod annotation is set", func() {
+
+		fakePod := testutils.NewFakePod("testpod", "", "net1")
+		conf := `{
+			"name":"node-cni-network",
+			"type":"multus",
+			"clusterNetwork": "net2",
+			"kubeconfig":"/etc/kubernetes/node-kubeconfig.yaml"
+		}`
+		netConf, err := types.LoadNetConf([]byte(conf))
+		Expect(err).NotTo(HaveOccurred())
+
+		args := &skel.CmdArgs{
+			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+		}
+
+		fKubeClient := testutils.NewFakeKubeClient()
+		fKubeClient.AddPod(fakePod)
+		fKubeClient.AddNetConfig("default", "net1", "{\"type\": \"mynet1\"}")
+		fKubeClient.AddNetConfig("default", "net2", "{\"type\": \"mynet2\"}")
+		kubeClient, err := GetK8sClient("", fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		k8sArgs, err := GetK8sArgs(args)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = GetDefaultNetworks(k8sArgs, netConf, kubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(len(netConf.Delegates)).To(Equal(1))
+		Expect(netConf.Delegates[0].Conf.Name).To(Equal("net2"))
+		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet2"))
+
+		numK8sDelegates, _, err := TryLoadPodDelegates(k8sArgs, netConf, kubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(numK8sDelegates).To(Equal(0))
+		Expect(netConf.Delegates[0].Conf.Name).To(Equal("net1"))
+		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet1"))
+	})
+
+	It("overwrite multus config when Pod annotation is set", func() {
+
+		fakePod := testutils.NewFakePod("testpod", "", "net1")
+		conf := `{
+			"name":"node-cni-network",
+			"type":"multus",
+			"kubeconfig":"/etc/kubernetes/node-kubeconfig.yaml",
+			"delegates": [{
+				"type": "mynet2",
+				"name": "net2"
+			}]
+		}`
+		netConf, err := types.LoadNetConf([]byte(conf))
+		Expect(netConf.Delegates[0].Conf.Name).To(Equal("net2"))
+		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet2"))
+		Expect(err).NotTo(HaveOccurred())
+
+		args := &skel.CmdArgs{
+			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+		}
+
+		fKubeClient := testutils.NewFakeKubeClient()
+		fKubeClient.AddPod(fakePod)
+		fKubeClient.AddNetConfig("default", "net1", "{\"type\": \"mynet1\"}")
+		kubeClient, err := GetK8sClient("", fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		k8sArgs, err := GetK8sArgs(args)
+		Expect(err).NotTo(HaveOccurred())
+
+		numK8sDelegates, _, err := TryLoadPodDelegates(k8sArgs, netConf, kubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(numK8sDelegates).To(Equal(0))
+		Expect(netConf.Delegates[0].Conf.Name).To(Equal("net1"))
+		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet1"))
+	})
 })

multus/multus_test.go

@@ -253,7 +253,7 @@ var _ = Describe("multus operations", func() {
 		 "mac": "c2:11:22:33:44:66",
 		 "ips": "10.0.0.1"}
 ]`
-		fakePod := testhelpers.NewFakePod("testpod", podNet)
+		fakePod := testhelpers.NewFakePod("testpod", podNet, "")
 		net1 := `{
 	"name": "net1",
 	"type": "mynet",
@@ -317,7 +317,7 @@ var _ = Describe("multus operations", func() {
 		result, err := cmdAdd(args, fExec, fKubeClient)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
-		Expect(fKubeClient.PodCount).To(Equal(2))
+		Expect(fKubeClient.PodCount).To(Equal(3))
 		Expect(fKubeClient.NetCount).To(Equal(2))
 		r := result.(*types020.Result)
 		// plugin 1 is the masterplugin
@@ -325,7 +325,7 @@ var _ = Describe("multus operations", func() {
 	})
 
 	It("executes delegates and kubernetes networks", func() {
-		fakePod := testhelpers.NewFakePod("testpod", "net1,net2")
+		fakePod := testhelpers.NewFakePod("testpod", "net1,net2", "")
 		net1 := `{
 	"name": "net1",
 	"type": "mynet",
@@ -396,7 +396,7 @@ var _ = Describe("multus operations", func() {
 		result, err := cmdAdd(args, fExec, fKubeClient)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
-		Expect(fKubeClient.PodCount).To(Equal(2))
+		Expect(fKubeClient.PodCount).To(Equal(3))
 		Expect(fKubeClient.NetCount).To(Equal(2))
 		r := result.(*types020.Result)
 		// plugin 1 is the masterplugin

testing/testing.go

@@ -103,7 +103,7 @@ func (f *FakeKubeClient) AddPod(pod *v1.Pod) {
 	f.pods[key] = pod
 }
 
-func NewFakePod(name string, netAnnotation string) *v1.Pod {
+func NewFakePod(name string, netAnnotation string, defaultNetAnnotation string) *v1.Pod {
 	pod := &v1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
@@ -115,13 +115,19 @@ func NewFakePod(name string, netAnnotation string) *v1.Pod {
 			},
 		},
 	}
+	annotations := make(map[string]string)
+
 	if netAnnotation != "" {
 		netAnnotation = strings.Replace(netAnnotation, "\n", "", -1)
 		netAnnotation = strings.Replace(netAnnotation, "\t", "", -1)
-		pod.ObjectMeta.Annotations = map[string]string{
+		annotations["k8s.v1.cni.cncf.io/networks"] = netAnnotation
-			"k8s.v1.cni.cncf.io/networks": netAnnotation,
-		}
 	}
+
+	if defaultNetAnnotation != "" {
+		annotations["multus-cni.io/default-network"] = defaultNetAnnotation
+	}
+
+	pod.ObjectMeta.Annotations = annotations
 	return pod
 }