network: add new NetInterworkingModel "none" and endpoint type TapEndpoint

This model is for not creating a new net ns for VM and directly
creating taps in the host net ns.

Signed-off-by: Ruidong Cao <caoruidong@huawei.com>

cli/config/configuration.toml.in

@@ -230,6 +230,10 @@ path = "@NETMONPATH@"
 #   - macvtap
 #     Used when the Container network interface can be bridged using
 #     macvtap.
+#
+#   - none
+#     Used when customize network. Only creates a tap device. No veth pair.
+#
 internetworking_model="@DEFNETWORKMODEL@"
 
 # If enabled, the runtime will create opentracing.io traces and spans.

virtcontainers/endpoint.go

@@ -44,6 +44,9 @@ const (
 
 	// MacvtapEndpointType is macvtap network interface.
 	MacvtapEndpointType EndpointType = "macvtap"
+
+	// TapEndpointType is tap network interface.
+	TapEndpointType EndpointType = "tap"
 )
 
 // Set sets an endpoint type based on the input string.
@@ -64,6 +67,9 @@ func (endpointType *EndpointType) Set(value string) error {
 	case "macvtap":
 		*endpointType = MacvtapEndpointType
 		return nil
+	case "tap":
+		*endpointType = TapEndpointType
+		return nil
 	default:
 		return fmt.Errorf("Unknown endpoint type %s", value)
 	}
@@ -82,6 +88,8 @@ func (endpointType *EndpointType) String() string {
 		return string(BridgedMacvlanEndpointType)
 	case MacvtapEndpointType:
 		return string(MacvtapEndpointType)
+	case TapEndpointType:
+		return string(TapEndpointType)
 	default:
 		return ""
 	}

virtcontainers/network.go

@@ -51,6 +51,9 @@ const (
 	// This will be used for vethtap, macvtap, ipvtap
 	NetXConnectEnlightenedModel
 
+	// NetXConnectNoneModel can be used when the VM is in the host network namespace
+	NetXConnectNoneModel
+
 	// NetXConnectInvalidModel is the last item to check valid values by IsValid()
 	NetXConnectInvalidModel
 )
@@ -75,6 +78,9 @@ func (n *NetInterworkingModel) SetModel(modelName string) error {
 	case "enlightened":
 		*n = NetXConnectEnlightenedModel
 		return nil
+	case "none":
+		*n = NetXConnectNoneModel
+		return nil
 	}
 	return fmt.Errorf("Unknown type %s", modelName)
 }
@@ -122,7 +128,7 @@ type NetworkInterface struct {
 	Addrs    []netlink.Addr
 }
 
-// TapInterface defines a tap nic.
+// TapInterface defines a tap interface
 type TapInterface struct {
 	ID       string
 	Name     string
@@ -199,6 +205,94 @@ func (n NetworkNamespace) MarshalJSON() ([]byte, error) {
 	return b, err
 }
 
+func generateEndpoints(typedEndpoints []TypedJSONEndpoint) ([]Endpoint, error) {
+	var endpoints []Endpoint
+
+	for _, e := range typedEndpoints {
+		switch e.Type {
+		case PhysicalEndpointType:
+			var endpoint PhysicalEndpoint
+			err := json.Unmarshal(e.Data, &endpoint)
+			if err != nil {
+				return nil, err
+			}
+
+			endpoints = append(endpoints, &endpoint)
+			networkLogger().WithFields(logrus.Fields{
+				"endpoint":      endpoint,
+				"endpoint-type": "physical",
+			}).Info("endpoint unmarshalled")
+
+		case VethEndpointType:
+			var endpoint VethEndpoint
+			err := json.Unmarshal(e.Data, &endpoint)
+			if err != nil {
+				return nil, err
+			}
+
+			endpoints = append(endpoints, &endpoint)
+			networkLogger().WithFields(logrus.Fields{
+				"endpoint":      endpoint,
+				"endpoint-type": "virtual",
+			}).Info("endpoint unmarshalled")
+
+		case VhostUserEndpointType:
+			var endpoint VhostUserEndpoint
+			err := json.Unmarshal(e.Data, &endpoint)
+			if err != nil {
+				return nil, err
+			}
+
+			endpoints = append(endpoints, &endpoint)
+			networkLogger().WithFields(logrus.Fields{
+				"endpoint":      endpoint,
+				"endpoint-type": "vhostuser",
+			}).Info("endpoint unmarshalled")
+
+		case BridgedMacvlanEndpointType:
+			var endpoint BridgedMacvlanEndpoint
+			err := json.Unmarshal(e.Data, &endpoint)
+			if err != nil {
+				return nil, err
+			}
+
+			networkLogger().WithFields(logrus.Fields{
+				"endpoint":      endpoint,
+				"endpoint-type": "macvlan",
+			}).Info("endpoint unmarshalled")
+
+		case MacvtapEndpointType:
+			var endpoint MacvtapEndpoint
+			err := json.Unmarshal(e.Data, &endpoint)
+			if err != nil {
+				return nil, err
+			}
+
+			networkLogger().WithFields(logrus.Fields{
+				"endpoint":      endpoint,
+				"endpoint-type": "macvtap",
+			}).Info("endpoint unmarshalled")
+
+		case TapEndpointType:
+			var endpoint TapEndpoint
+			err := json.Unmarshal(e.Data, &endpoint)
+			if err != nil {
+				return nil, err
+			}
+
+			endpoints = append(endpoints, &endpoint)
+			networkLogger().WithFields(logrus.Fields{
+				"endpoint":      endpoint,
+				"endpoint-type": "tap",
+			}).Info("endpoint unmarshalled")
+
+		default:
+			networkLogger().WithField("endpoint-type", e.Type).Error("Ignoring unknown endpoint type")
+		}
+	}
+	return endpoints, nil
+}
+
 // UnmarshalJSON is the custom NetworkNamespace unmarshalling routine.
 // This is needed for unmarshalling the Endpoints interfaces array.
 func (n *NetworkNamespace) UnmarshalJSON(b []byte) error {
@@ -219,79 +313,11 @@ func (n *NetworkNamespace) UnmarshalJSON(b []byte) error {
 	if err := json.Unmarshal([]byte(string(s.Endpoints)), &typedEndpoints); err != nil {
 		return err
 	}
-
-	var endpoints []Endpoint
-
-	for _, e := range typedEndpoints {
-		switch e.Type {
-		case PhysicalEndpointType:
-			var endpoint PhysicalEndpoint
-			err := json.Unmarshal(e.Data, &endpoint)
+	endpoints, err := generateEndpoints(typedEndpoints)
 	if err != nil {
 		return err
 	}
 
-			endpoints = append(endpoints, &endpoint)
-			networkLogger().WithFields(logrus.Fields{
-				"endpoint":      endpoint,
-				"endpoint-type": "physical",
-			}).Info("endpoint unmarshalled")
-
-		case VethEndpointType:
-			var endpoint VethEndpoint
-			err := json.Unmarshal(e.Data, &endpoint)
-			if err != nil {
-				return err
-			}
-
-			endpoints = append(endpoints, &endpoint)
-			networkLogger().WithFields(logrus.Fields{
-				"endpoint":      endpoint,
-				"endpoint-type": "virtual",
-			}).Info("endpoint unmarshalled")
-
-		case VhostUserEndpointType:
-			var endpoint VhostUserEndpoint
-			err := json.Unmarshal(e.Data, &endpoint)
-			if err != nil {
-				return err
-			}
-
-			endpoints = append(endpoints, &endpoint)
-			networkLogger().WithFields(logrus.Fields{
-				"endpoint":      endpoint,
-				"endpoint-type": "vhostuser",
-			}).Info("endpoint unmarshalled")
-
-		case BridgedMacvlanEndpointType:
-			var endpoint BridgedMacvlanEndpoint
-			err := json.Unmarshal(e.Data, &endpoint)
-			if err != nil {
-				return err
-			}
-
-			networkLogger().WithFields(logrus.Fields{
-				"endpoint":      endpoint,
-				"endpoint-type": "macvlan",
-			}).Info("endpoint unmarshalled")
-
-		case MacvtapEndpointType:
-			var endpoint MacvtapEndpoint
-			err := json.Unmarshal(e.Data, &endpoint)
-			if err != nil {
-				return err
-			}
-
-			networkLogger().WithFields(logrus.Fields{
-				"endpoint":      endpoint,
-				"endpoint-type": "macvtap",
-			}).Info("endpoint unmarshalled")
-
-		default:
-			networkLogger().WithField("endpoint-type", e.Type).Error("Ignoring unknown endpoint type")
-		}
-	}
-
 	(*n).Endpoints = endpoints
 	return nil
 }
@@ -828,6 +854,13 @@ func createNetNS() (string, error) {
 // into runtime.LockOSThread(), meaning it won't be executed in a
 // different thread than the one expected by the caller.
 func doNetNS(netNSPath string, cb func(ns.NetNS) error) error {
+	// if netNSPath is empty, the callback function will be run in the current network namespace.
+	// So skip the whole function, just call cb(). cb() needs a NetNS as arg but ignored, give it a fake one.
+	if netNSPath == "" {
+		var netNs ns.NetNS
+		return cb(netNs)
+	}
+
 	runtime.LockOSThread()
 	defer runtime.UnlockOSThread()
 
@@ -1123,6 +1156,9 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel) (E
 		} else if netInfo.Iface.Type == "macvtap" {
 			networkLogger().Infof("macvtap interface found")
 			endpoint, err = createMacvtapNetworkEndpoint(netInfo)
+		} else if netInfo.Iface.Type == "tap" {
+			networkLogger().Info("tap interface found")
+			endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name)
 		} else if netInfo.Iface.Type == "veth" {
 			endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model)
 		} else {

virtcontainers/qemu.go

@@ -872,6 +872,9 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) error {
 		case VethEndpointType:
 			drive := endpoint.(*VethEndpoint)
 			tap = drive.NetPair.TapInterface
+		case TapEndpointType:
+			drive := endpoint.(*TapEndpoint)
+			tap = drive.TapInterface
 		default:
 			return fmt.Errorf("this endpoint is not supported")
 		}
@@ -896,6 +899,9 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) error {
 		case VethEndpointType:
 			drive := endpoint.(*VethEndpoint)
 			tap = drive.NetPair.TapInterface
+		case TapEndpointType:
+			drive := endpoint.(*TapEndpoint)
+			tap = drive.TapInterface
 		default:
 			return fmt.Errorf("this endpoint is not supported")
 		}

virtcontainers/sandbox.go

@@ -1089,6 +1089,13 @@ func (s *Sandbox) generateNetInfo(inf *grpc.Interface) (NetworkInfo, error) {
 		addrs = append(addrs, *netlinkAddr)
 	}
 
+	var ifaceType string
+	if s.config.NetworkConfig.InterworkingModel == NetXConnectNoneModel {
+		ifaceType = "tap"
+	} else {
+		ifaceType = "veth"
+	}
+
 	return NetworkInfo{
 		Iface: NetlinkIface{
 			LinkAttrs: netlink.LinkAttrs{
@@ -1096,7 +1103,7 @@ func (s *Sandbox) generateNetInfo(inf *grpc.Interface) (NetworkInfo, error) {
 				HardwareAddr: hw,
 				MTU:          int(inf.Mtu),
 			},
-			Type: "",
+			Type: ifaceType,
 		},
 		Addrs: addrs,
 	}, nil

virtcontainers/tap_endpoint.go

@@ -0,0 +1,189 @@
+// Copyright (c) 2018 Huawei Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package virtcontainers
+
+import (
+	"fmt"
+
+	"github.com/containernetworking/plugins/pkg/ns"
+	"github.com/vishvananda/netlink"
+
+	"github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
+)
+
+// TapEndpoint represents just a tap endpoint
+type TapEndpoint struct {
+	TapInterface       TapInterface
+	EndpointProperties NetworkInfo
+	EndpointType       EndpointType
+	PCIAddr            string
+}
+
+// Properties returns the properties of the tap interface.
+func (endpoint *TapEndpoint) Properties() NetworkInfo {
+	return endpoint.EndpointProperties
+}
+
+// Name returns name of the tap interface in the network pair.
+func (endpoint *TapEndpoint) Name() string {
+	return endpoint.TapInterface.Name
+}
+
+// HardwareAddr returns the mac address that is assigned to the tap interface
+func (endpoint *TapEndpoint) HardwareAddr() string {
+	return endpoint.TapInterface.TAPIface.HardAddr
+}
+
+// Type identifies the endpoint as a tap endpoint.
+func (endpoint *TapEndpoint) Type() EndpointType {
+	return endpoint.EndpointType
+}
+
+// PciAddr returns the PCI address of the endpoint.
+func (endpoint *TapEndpoint) PciAddr() string {
+	return endpoint.PCIAddr
+}
+
+// SetPciAddr sets the PCI address of the endpoint.
+func (endpoint *TapEndpoint) SetPciAddr(pciAddr string) {
+	endpoint.PCIAddr = pciAddr
+}
+
+// NetworkPair returns the network pair of the endpoint.
+func (endpoint *TapEndpoint) NetworkPair() *NetworkInterfacePair {
+	return nil
+}
+
+// SetProperties sets the properties for the endpoint.
+func (endpoint *TapEndpoint) SetProperties(properties NetworkInfo) {
+	endpoint.EndpointProperties = properties
+}
+
+// Attach for tap endpoint adds the tap interface to the hypervisor.
+func (endpoint *TapEndpoint) Attach(h hypervisor) error {
+	return fmt.Errorf("TapEndpoint does not support Attach, if you're using docker please use --net none")
+}
+
+// Detach for the tap endpoint tears down the tap
+func (endpoint *TapEndpoint) Detach(netNsCreated bool, netNsPath string) error {
+	if !netNsCreated && netNsPath != "" {
+		return nil
+	}
+
+	networkLogger().WithField("endpoint-type", TapEndpointType).Info("Detaching endpoint")
+	return doNetNS(netNsPath, func(_ ns.NetNS) error {
+		return unTapNetwork(endpoint.TapInterface.TAPIface.Name)
+	})
+}
+
+// HotAttach for the tap endpoint uses hot plug device
+func (endpoint *TapEndpoint) HotAttach(h hypervisor) error {
+	networkLogger().Info("Hot attaching tap endpoint")
+	if err := tapNetwork(endpoint, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil {
+		networkLogger().WithError(err).Error("Error bridging tap ep")
+		return err
+	}
+
+	if _, err := h.hotplugAddDevice(endpoint, netDev); err != nil {
+		networkLogger().WithError(err).Error("Error attach tap ep")
+		return err
+	}
+	return nil
+}
+
+// HotDetach for the tap endpoint uses hot pull device
+func (endpoint *TapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
+	networkLogger().Info("Hot detaching tap endpoint")
+	if err := doNetNS(netNsPath, func(_ ns.NetNS) error {
+		return unTapNetwork(endpoint.TapInterface.TAPIface.Name)
+	}); err != nil {
+		networkLogger().WithError(err).Warn("Error un-bridging tap ep")
+	}
+
+	if _, err := h.hotplugRemoveDevice(endpoint, netDev); err != nil {
+		networkLogger().WithError(err).Error("Error detach tap ep")
+		return err
+	}
+	return nil
+}
+
+func createTapNetworkEndpoint(idx int, ifName string) (*TapEndpoint, error) {
+	if idx < 0 {
+		return &TapEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx)
+	}
+	uniqueID := uuid.Generate().String()
+
+	endpoint := &TapEndpoint{
+		TapInterface: TapInterface{
+			ID:   uniqueID,
+			Name: fmt.Sprintf("eth%d", idx),
+			TAPIface: NetworkInterface{
+				Name: fmt.Sprintf("tap%d_kata", idx),
+			},
+		},
+		EndpointType: TapEndpointType,
+	}
+	if ifName != "" {
+		endpoint.TapInterface.Name = ifName
+	}
+
+	return endpoint, nil
+}
+
+func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) error {
+	netHandle, err := netlink.NewHandle()
+	if err != nil {
+		return err
+	}
+	defer netHandle.Delete()
+
+	tapLink, fds, err := createLink(netHandle, endpoint.TapInterface.TAPIface.Name, &netlink.Tuntap{}, int(numCPUs))
+	if err != nil {
+		return fmt.Errorf("Could not create TAP interface: %s", err)
+	}
+	endpoint.TapInterface.VMFds = fds
+	if !disableVhostNet {
+		vhostFds, err := createVhostFds(int(numCPUs))
+		if err != nil {
+			return fmt.Errorf("Could not setup vhost fds %s : %s", endpoint.TapInterface.Name, err)
+		}
+		endpoint.TapInterface.VhostFds = vhostFds
+	}
+	linkAttrs := endpoint.Properties().Iface.LinkAttrs
+
+	// Save the MAC address to the TAP so that it can later be used
+	// to build the QMP command line. This MAC address has to be
+	// the one inside the VM in order to avoid any firewall issues. The
+	// bridge created by the network plugin on the host actually expects
+	// to see traffic from this MAC address and not another one.
+	endpoint.TapInterface.TAPIface.HardAddr = linkAttrs.HardwareAddr.String()
+	if err := netHandle.LinkSetMTU(tapLink, linkAttrs.MTU); err != nil {
+		return fmt.Errorf("Could not set TAP MTU %d: %s", linkAttrs.MTU, err)
+	}
+	if err := netHandle.LinkSetUp(tapLink); err != nil {
+		return fmt.Errorf("Could not enable TAP %s: %s", endpoint.TapInterface.Name, err)
+	}
+	return nil
+}
+
+func unTapNetwork(name string) error {
+	netHandle, err := netlink.NewHandle()
+	if err != nil {
+		return err
+	}
+	defer netHandle.Delete()
+	tapLink, err := getLinkByName(netHandle, name, &netlink.Tuntap{})
+	if err != nil {
+		return fmt.Errorf("Could not get TAP interface: %s", err)
+	}
+	if err := netHandle.LinkSetDown(tapLink); err != nil {
+		return fmt.Errorf("Could not disable TAP %s: %s", name, err)
+	}
+	if err := netHandle.LinkDel(tapLink); err != nil {
+		return fmt.Errorf("Could not remove TAP %s: %s", name, err)
+	}
+	return nil
+}