From 165e23b72c04643c39fb318af38822a54b59c8dd Mon Sep 17 00:00:00 2001
From: Tomofumi Hayashi <tohayash@redhat.com>
Date: Tue, 27 Aug 2019 16:52:58 +0900
Subject: [PATCH] Support gateway in NetworkSelectionElement

Changes config JSON from gateway to default-route, adds Readme, fixes lint

Co-authored-by: dougbtv <dosmith@redhat.com>
---
 doc/how-to-use.md      | 66 ++++++++++++++++++++++++++++++++++++++++++
 k8sclient/k8sclient.go | 16 ++++++++++
 multus/multus.go       |  9 ++++++
 netutils/netutils.go   | 62 +++++++++++++++++++++++++++++++++++++++
 types/conf.go          | 28 ++++++++++++++++++
 types/types.go         |  7 ++++-
 6 files changed, 187 insertions(+), 1 deletion(-)
 create mode 100644 netutils/netutils.go

diff --git a/doc/how-to-use.md b/doc/how-to-use.md
index a67c3522d..f9edbc2ca 100644
--- a/doc/how-to-use.md
+++ b/doc/how-to-use.md
@@ -469,6 +469,72 @@ $ kubectl exec -it pod-case-06 -- ip -d address
 | macvlan1 | macvlan interface (macvlan-conf-1) |
 | net2 | macvlan interface (macvlan-conf-2) |
 
+## Specifying a default route for a specific attachment
+
+Typically, the default route for a pod will route traffic over the `eth0` and therefore over the cluster-wide default network. You may wish to specify that a different network attachment will have the default route.
+
+You can achieve this by using the JSON formatted annotation and specifying a `default-route` key.
+
+*NOTE*: It's important that you consider that this may impact some functionality of getting traffic to route over the cluster-wide default network.
+
+For example, we have a this configuration for macvlan:
+
+```
+cat <<EOF | kubectl create -f -
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: macvlan-conf
+spec:
+  config: '{
+      "cniVersion": "0.3.0",
+      "type": "macvlan",
+      "master": "eth0",
+      "mode": "bridge",
+      "ipam": {
+        "type": "host-local",
+        "subnet": "192.168.2.0/24",
+        "rangeStart": "192.168.2.200",
+        "rangeEnd": "192.168.2.216",
+        "routes": [
+          { "dst": "0.0.0.0/0" }
+        ],
+        "gateway": "192.168.2.1"
+      }
+    }'
+EOF
+```
+
+We can then create a pod which uses the `default-route` key in the JSON formatted `k8s.v1.cni.cncf.io/networks` annotation. 
+
+```
+cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: samplepod
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[{
+      "name": "macvlan-conf",
+      "default-route": ["192.168.2.1"]
+    }]'
+spec:
+  containers:
+  - name: samplepod
+    command: ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait"]
+    image: dougbtv/centos-network
+EOF
+```
+
+This will set `192.168.2.1` as the default route over the `net1` interface, such as:
+
+```
+$ kubectl exec -it samplepod -- ip route
+default via 192.168.2.1 dev net1 
+10.244.0.0/24 dev eth0  proto kernel  scope link  src 10.244.0.169 
+10.244.0.0/16 via 10.244.0.1 dev eth0 
+```
+
 ## Entrypoint Parameters
 
 Multus CNI, when installed using the daemonset-style installation uses an entrypoint script which copies the Multus binary into place, places CNI configurations. This entrypoint takes a variety of parameters for customization.
diff --git a/k8sclient/k8sclient.go b/k8sclient/k8sclient.go
index 89bd022d6..81549bef7 100644
--- a/k8sclient/k8sclient.go
+++ b/k8sclient/k8sclient.go
@@ -490,8 +490,24 @@ func TryLoadPodDelegates(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient
 		if err = conf.AddDelegates(delegates); err != nil {
 			return 0, nil, err
 		}
+
+		// Check gatewayRequest is configured in delegates
+		// and mark its config if gateway filter is required
+		isGatewayConfigured := false
+		for _, delegate := range conf.Delegates {
+			if delegate.GatewayRequest != nil {
+				isGatewayConfigured = true
+				break
+			}
+		}
+
+		if isGatewayConfigured == true {
+			types.CheckGatewayConfig(conf.Delegates)
+		}
+
 		return len(delegates), clientInfo, nil
 	}
+
 	return 0, clientInfo, nil
 }
 
diff --git a/multus/multus.go b/multus/multus.go
index f60b955e7..2cbd172b2 100644
--- a/multus/multus.go
+++ b/multus/multus.go
@@ -38,6 +38,7 @@ import (
 	"github.com/containernetworking/plugins/pkg/ns"
 	k8s "github.com/intel/multus-cni/k8sclient"
 	"github.com/intel/multus-cni/logging"
+	"github.com/intel/multus-cni/netutils"
 	"github.com/intel/multus-cni/types"
 	"github.com/vishvananda/netlink"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -419,6 +420,14 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn
 			return nil, logging.Errorf("Multus: error adding pod to network %q: %v", netName, err)
 		}
 
+		// Remove gateway from routing table if the gateway is not used
+		if delegate.IsFilterGateway {
+			tmpResult, err = netutils.DeleteDefaultGW(args, ifName, &tmpResult)
+			if err != nil {
+				return nil, logging.Errorf("Multus: Err in deleting gateway: %v", err)
+			}
+		}
+
 		// Master plugin result is always used if present
 		if delegate.MasterPlugin || result == nil {
 			result = tmpResult
diff --git a/netutils/netutils.go b/netutils/netutils.go
new file mode 100644
index 000000000..a42eb3efe
--- /dev/null
+++ b/netutils/netutils.go
@@ -0,0 +1,62 @@
+// Copyright (c) 2019 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package netutils
+
+import (
+	"github.com/containernetworking/cni/pkg/skel"
+	cnitypes "github.com/containernetworking/cni/pkg/types"
+	"github.com/containernetworking/cni/pkg/types/current"
+	"github.com/containernetworking/plugins/pkg/ns"
+
+	"github.com/intel/multus-cni/logging"
+
+	"github.com/vishvananda/netlink"
+)
+
+// DeleteDefaultGW removes the default gateway from marked interfaces.
+func DeleteDefaultGW(args *skel.CmdArgs, ifName string, res *cnitypes.Result) (*current.Result, error) {
+	logging.Debugf("XXX: DeleteDefaultGW: %s", args.Netns)
+	result, err := current.NewResultFromResult(*res)
+	if err != nil {
+		return nil, logging.Errorf("XXX: %v", err)
+	}
+
+	netns, err := ns.GetNS(args.Netns)
+	if err != nil {
+		return nil, logging.Errorf("XXX: %v", err)
+	}
+	defer netns.Close()
+
+	err = netns.Do(func(_ ns.NetNS) error {
+		var err error
+		link, _ := netlink.LinkByName(ifName)
+		routes, _ := netlink.RouteList(link, netlink.FAMILY_ALL)
+		for _, nlroute := range routes {
+			if nlroute.Dst == nil {
+				err = netlink.RouteDel(&nlroute)
+			}
+		}
+		return err
+	})
+	var newRoutes []*cnitypes.Route
+	for _, route := range result.Routes {
+		if mask, _ := route.Dst.Mask.Size(); mask != 0 {
+			newRoutes = append(newRoutes, route)
+		}
+	}
+	result.Routes = newRoutes
+	return result, err
+}
diff --git a/types/conf.go b/types/conf.go
index a44c419a7..35174631f 100644
--- a/types/conf.go
+++ b/types/conf.go
@@ -17,6 +17,7 @@ package types
 
 import (
 	"encoding/json"
+	"net"
 
 	"github.com/containernetworking/cni/libcni"
 	"github.com/containernetworking/cni/pkg/skel"
@@ -111,6 +112,9 @@ func LoadDelegateNetConf(bytes []byte, net *NetworkSelectionElement, deviceID st
 		if net.PortMappingsRequest != nil {
 			delegateConf.PortMappingsRequest = net.PortMappingsRequest
 		}
+		if net.GatewayRequest != nil {
+			delegateConf.GatewayRequest = append(delegateConf.GatewayRequest, net.GatewayRequest...)
+		}
 	}
 
 	delegateConf.Bytes = bytes
@@ -183,6 +187,18 @@ func CreateCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, r
 	return rt
 }
 
+// GetGatewayFromResult retrieves gateway IP addresses from CNI result
+func GetGatewayFromResult(result *current.Result) []net.IP {
+	var gateways []net.IP
+
+	for _, route := range result.Routes {
+		if mask, _ := route.Dst.Mask.Size(); mask == 0 {
+			gateways = append(gateways, route.GW)
+		}
+	}
+	return gateways
+}
+
 // LoadNetworkStatus create network status from CNI result
 func LoadNetworkStatus(r types.Result, netName string, defaultNet bool) (*NetworkStatus, error) {
 	logging.Debugf("LoadNetworkStatus: %v, %s, %t", r, netName, defaultNet)
@@ -215,6 +231,7 @@ func LoadNetworkStatus(r types.Result, netName string, defaultNet bool) (*Networ
 	}
 
 	netstatus.DNS = result.DNS
+	netstatus.Gateway = GetGatewayFromResult(result)
 
 	return netstatus, nil
 
@@ -449,6 +466,17 @@ func addCNIArgsInConfList(inBytes []byte, cniArgs *map[string]interface{}) ([]by
 	return configBytes, nil
 }
 
+// CheckGatewayConfig check gatewayRequest and mark IsFilterGateway flag if
+// gw filtering is required
+func CheckGatewayConfig(delegates []*DelegateNetConf) {
+	// Check the Gateway
+	for i, delegate := range delegates {
+		if delegate.GatewayRequest == nil {
+			delegates[i].IsFilterGateway = true
+		}
+	}
+}
+
 // CheckSystemNamespaces checks whether given namespace is in systemNamespaces or not.
 func CheckSystemNamespaces(namespace string, systemNamespaces []string) bool {
 	for _, nsname := range systemNamespaces {
diff --git a/types/types.go b/types/types.go
index 2a0a3102f..c5370099d 100644
--- a/types/types.go
+++ b/types/types.go
@@ -88,6 +88,7 @@ type NetworkStatus struct {
 	Mac       string    `json:"mac,omitempty"`
 	Default   bool      `json:"default,omitempty"`
 	DNS       types.DNS `json:"dns,omitempty"`
+	Gateway   []net.IP  `json:"default-route,omitempty"`
 }
 
 // DelegateNetConf for net-attach-def for pod
@@ -99,6 +100,8 @@ type DelegateNetConf struct {
 	IPRequest           []string        `json:"ipRequest,omitempty"`
 	PortMappingsRequest []*PortMapEntry `json:"-"`
 	BandwidthRequest    *BandwidthEntry `json:"-"`
+	GatewayRequest      []net.IP        `json:"default-route,omitempty"`
+	IsFilterGateway     bool
 	// MasterPlugin is only used internal housekeeping
 	MasterPlugin bool `json:"-"`
 	// Conflist plugin is only used internal housekeeping
@@ -136,7 +139,9 @@ type NetworkSelectionElement struct {
 	// the network
 	BandwidthRequest *BandwidthEntry `json:"bandwidth,omitempty"`
 	// CNIArgs contains additional CNI arguments for the network interface
-	CNIArgs	*map[string]interface{} `json:"cni-args"`
+	CNIArgs *map[string]interface{} `json:"cni-args"`
+	// GatewayRequest contains default route IP address for the pod
+	GatewayRequest []net.IP `json:"default-route,omitempty"`
 }
 
 // K8sArgs is the valid CNI_ARGS used for Kubernetes