cli: add configuration option to enable/disable vhost_net

Add `disable_vhost_net` option to enable or disable the use of
vhost_net. Vhost_net can improve network performance.

Signed-off-by: Ruidong Cao <caoruidong@huawei.com>

cli/config.go

@@ -94,6 +94,7 @@ type hypervisor struct {
 	EnableIOThreads       bool   `toml:"enable_iothreads"`
 	UseVSock              bool   `toml:"use_vsock"`
 	HotplugVFIOOnRootBus  bool   `toml:"hotplug_vfio_on_root_bus"`
+	DisableVhostNet       bool   `toml:"disable_vhost_net"`
 }
 
 type proxy struct {
@@ -375,6 +376,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		Msize9p:               h.msize9p(),
 		UseVSock:              useVSock,
 		HotplugVFIOOnRootBus:  h.HotplugVFIOOnRootBus,
+		DisableVhostNet:       h.DisableVhostNet,
 	}, nil
 }
 

cli/config/configuration.toml.in

@@ -147,6 +147,10 @@ enable_iothreads = @DEFENABLEIOTHREADS@
 # Default false
 #hotplug_vfio_on_root_bus = true
 
+# If host doesn't support vhost_net, set to true. Thus we won't create vhost fds for nics.
+# Default false
+#disable_vhost_net = true
+
 [factory]
 # VM templating support. Once enabled, new VMs are created from template
 # using vm cloning. They will share the same initial kernel, initramfs and

virtcontainers/hypervisor.go

@@ -240,6 +240,9 @@ type HypervisorConfig struct {
 	// DevicesStatePath is the VM device state file path. Used when either BootToBeTemplate or
 	// BootFromTemplate is true.
 	DevicesStatePath string
+
+	// DisableVhostNet is used to indicate if host supports vhost_net
+	DisableVhostNet bool
 }
 
 func (conf *HypervisorConfig) checkTemplateConfig() error {

virtcontainers/network.go

@@ -225,7 +225,7 @@ func networkLogger() *logrus.Entry {
 func (endpoint *VirtualEndpoint) Attach(h hypervisor) error {
 	networkLogger().WithField("endpoint-type", "virtual").Info("Attaching endpoint")
 
-	if err := xconnectVMNetwork(&(endpoint.NetPair), true, h.hypervisorConfig().NumVCPUs); err != nil {
+	if err := xconnectVMNetwork(&(endpoint.NetPair), true, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil {
 		networkLogger().WithError(err).Error("Error bridging virtual endpoint")
 		return err
 	}
@@ -245,14 +245,14 @@ func (endpoint *VirtualEndpoint) Detach(netNsCreated bool, netNsPath string) err
 	networkLogger().WithField("endpoint-type", "virtual").Info("Detaching endpoint")
 
 	return doNetNS(netNsPath, func(_ ns.NetNS) error {
-		return xconnectVMNetwork(&(endpoint.NetPair), false, 0)
+		return xconnectVMNetwork(&(endpoint.NetPair), false, 0, false)
 	})
 }
 
 // HotAttach for the virtual endpoint uses hot plug device
 func (endpoint *VirtualEndpoint) HotAttach(h hypervisor) error {
 	networkLogger().Info("Hot attaching virtual endpoint")
-	if err := xconnectVMNetwork(&(endpoint.NetPair), true, h.hypervisorConfig().NumVCPUs); err != nil {
+	if err := xconnectVMNetwork(&(endpoint.NetPair), true, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil {
 		networkLogger().WithError(err).Error("Error bridging virtual ep")
 		return err
 	}
@@ -271,7 +271,7 @@ func (endpoint *VirtualEndpoint) HotDetach(h hypervisor, netNsCreated bool, netN
 	}
 	networkLogger().Info("Hot detaching virtual endpoint")
 	if err := doNetNS(netNsPath, func(_ ns.NetNS) error {
-		return xconnectVMNetwork(&(endpoint.NetPair), false, 0)
+		return xconnectVMNetwork(&(endpoint.NetPair), false, 0, h.hypervisorConfig().DisableVhostNet)
 	}); err != nil {
 		networkLogger().WithError(err).Error("Error abridging virtual ep")
 		return err
@@ -721,7 +721,7 @@ func getLinkByName(netHandle *netlink.Handle, name string, expectedLink netlink.
 }
 
 // The endpoint type should dictate how the connection needs to be made
-func xconnectVMNetwork(netPair *NetworkInterfacePair, connect bool, numCPUs uint32) error {
+func xconnectVMNetwork(netPair *NetworkInterfacePair, connect bool, numCPUs uint32, disableVhostNet bool) error {
 	if netPair.NetInterworkingModel == NetXConnectDefaultModel {
 		netPair.NetInterworkingModel = DefaultNetInterworkingModel
 	}
@@ -729,13 +729,13 @@ func xconnectVMNetwork(netPair *NetworkInterfacePair, connect bool, numCPUs uint
 	case NetXConnectBridgedModel:
 		netPair.NetInterworkingModel = NetXConnectBridgedModel
 		if connect {
-			return bridgeNetworkPair(netPair, numCPUs)
+			return bridgeNetworkPair(netPair, numCPUs, disableVhostNet)
 		}
 		return unBridgeNetworkPair(*netPair)
 	case NetXConnectMacVtapModel:
 		netPair.NetInterworkingModel = NetXConnectMacVtapModel
 		if connect {
-			return tapNetworkPair(netPair, numCPUs)
+			return tapNetworkPair(netPair, numCPUs, disableVhostNet)
 		}
 		return untapNetworkPair(*netPair)
 	case NetXConnectEnlightenedModel:
@@ -824,7 +824,7 @@ func setIPs(link netlink.Link, addrs []netlink.Addr) error {
 	return nil
 }
 
-func tapNetworkPair(netPair *NetworkInterfacePair, numCPUs uint32) error {
+func tapNetworkPair(netPair *NetworkInterfacePair, numCPUs uint32, disableVhostNet bool) error {
 	netHandle, err := netlink.NewHandle()
 	if err != nil {
 		return err
@@ -904,16 +904,18 @@ func tapNetworkPair(netPair *NetworkInterfacePair, numCPUs uint32) error {
 		return fmt.Errorf("Could not setup macvtap fds %s: %s", netPair.TAPIface, err)
 	}
 
-	vhostFds, err := createVhostFds(int(numCPUs))
+	if !disableVhostNet {
-	if err != nil {
+		vhostFds, err := createVhostFds(int(numCPUs))
-		return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err)
+		if err != nil {
+			return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err)
+		}
+		netPair.VhostFds = vhostFds
 	}
-	netPair.VhostFds = vhostFds
 
 	return nil
 }
 
-func bridgeNetworkPair(netPair *NetworkInterfacePair, numCPUs uint32) error {
+func bridgeNetworkPair(netPair *NetworkInterfacePair, numCPUs uint32, disableVhostNet bool) error {
 	netHandle, err := netlink.NewHandle()
 	if err != nil {
 		return err
@@ -926,11 +928,13 @@ func bridgeNetworkPair(netPair *NetworkInterfacePair, numCPUs uint32) error {
 	}
 	netPair.VMFds = fds
 
-	vhostFds, err := createVhostFds(int(numCPUs))
+	if !disableVhostNet {
-	if err != nil {
+		vhostFds, err := createVhostFds(int(numCPUs))
-		return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err)
+		if err != nil {
+			return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err)
+		}
+		netPair.VhostFds = vhostFds
 	}
-	netPair.VhostFds = vhostFds
 
 	vethLink, err := getLinkByName(netHandle, netPair.VirtIface.Name, &netlink.Veth{})
 	if err != nil {

virtcontainers/qemu.go

@@ -249,6 +249,13 @@ func (q *qemu) init(ctx context.Context, id string, hypervisorConfig *Hypervisor
 		q.arch.disableNestingChecks()
 	}
 
+	if !q.config.DisableVhostNet {
+		q.arch.enableVhostNet()
+	} else {
+		q.Logger().Debug("Disable vhost_net")
+		q.arch.disableVhostNet()
+	}
+
 	return nil
 }
 

virtcontainers/qemu_arch_base.go

@@ -24,6 +24,12 @@ type qemuArch interface {
 	// disableNestingChecks nesting checks will be ignored
 	disableNestingChecks()
 
+	// enableVhostNet vhost will be enabled
+	enableVhostNet()
+
+	// disableVhostNet vhost will be disabled
+	disableVhostNet()
+
 	// machine returns the machine type
 	machine() (govmmQemu.Machine, error)
 
@@ -92,6 +98,7 @@ type qemuArch interface {
 type qemuArchBase struct {
 	machineType           string
 	nestedRun             bool
+	vhost                 bool
 	networkIndex          int
 	qemuPaths             map[string]string
 	supportedQemuMachines []govmmQemu.Machine
@@ -176,6 +183,14 @@ func (q *qemuArchBase) disableNestingChecks() {
 	q.nestedRun = false
 }
 
+func (q *qemuArchBase) enableVhostNet() {
+	q.vhost = true
+}
+
+func (q *qemuArchBase) disableVhostNet() {
+	q.vhost = false
+}
+
 func (q *qemuArchBase) machine() (govmmQemu.Machine, error) {
 	for _, m := range q.supportedQemuMachines {
 		if m.Type == q.machineType {
@@ -437,7 +452,7 @@ func (q *qemuArchBase) appendNetwork(devices []govmmQemu.Device, endpoint Endpoi
 				MACAddress:    ep.NetPair.TAPIface.HardAddr,
 				DownScript:    "no",
 				Script:        "no",
-				VHost:         true,
+				VHost:         q.vhost,
 				DisableModern: q.nestedRun,
 				FDs:           ep.NetPair.VMFds,
 				VhostFDs:      ep.NetPair.VhostFds,