From 5ebb7cf6f58d831a18e65cd96f46740b2f199519 Mon Sep 17 00:00:00 2001 From: Ruidong Date: Wed, 12 Sep 2018 06:50:53 +0800 Subject: [PATCH 1/2] vendor: Update govmm to disable vhost when host doesn't support vhost-net If the length of vhostfds is zero, it means host doesn't support vhost. So do not pass vhost="on" in QMP. Full list: 1a1fee7 qemu/qmp: nic can works without vhost Fixes #169 Signed-off-by: Ruidong Cao --- Gopkg.lock | 4 ++-- Gopkg.toml | 2 +- vendor/github.com/intel/govmm/qemu/qmp.go | 14 ++++++++------ 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index dad0bb45d..a7a9c88e9 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -123,11 +123,11 @@ revision = "3520598351bb3500a49ae9563f5539666ae0a27c" [[projects]] - digest = "1:7434a85a1d6c2bf64f322087ec7a7f84ab9e971179bef7b95deabaf0e3f7c126" + digest = "1:2b062c3645f86b71bdb0487b4d69746704695fc09158fe4638c213d38c6cc5dc" name = "github.com/intel/govmm" packages = ["qemu"] pruneopts = "NUT" - revision = "e2c716433e444017507e3587ce071868fd164380" + revision = "032705ba6aae05a9bf41e296cf89c8529cffb822" [[projects]] digest = "1:01c37fcb6e2a1fe1321a97faaef74c66ac531ea292ca3f929b7189cc400b1d47" diff --git a/Gopkg.toml b/Gopkg.toml index 5491364d2..c5ea29b91 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -52,7 +52,7 @@ [[constraint]] name = "github.com/intel/govmm" - revision = "e2c716433e444017507e3587ce071868fd164380" + revision = "032705ba6aae05a9bf41e296cf89c8529cffb822" [[constraint]] name = "github.com/kata-containers/agent" diff --git a/vendor/github.com/intel/govmm/qemu/qmp.go b/vendor/github.com/intel/govmm/qemu/qmp.go index addf4b2fa..e9fa65e52 100644 --- a/vendor/github.com/intel/govmm/qemu/qmp.go +++ b/vendor/github.com/intel/govmm/qemu/qmp.go @@ -812,13 +812,15 @@ func (q *QMP) ExecuteNetdevChardevAdd(ctx context.Context, netdevType, netdevID, // Must be valid QMP identifier. func (q *QMP) ExecuteNetdevAddByFds(ctx context.Context, netdevType, netdevID string, fdNames, vhostFdNames []string) error { fdNameStr := strings.Join(fdNames, ":") - vhostFdNameStr := strings.Join(vhostFdNames, ":") args := map[string]interface{}{ - "type": netdevType, - "id": netdevID, - "fds": fdNameStr, - "vhost": "on", - "vhostfds": vhostFdNameStr, + "type": netdevType, + "id": netdevID, + "fds": fdNameStr, + } + if len(vhostFdNames) > 0 { + vhostFdNameStr := strings.Join(vhostFdNames, ":") + args["vhost"] = "on" + args["vhostfds"] = vhostFdNameStr } return q.executeCommand(ctx, "netdev_add", args, nil) From 225e10cfc4bb99722b6f5734a1e840138bcea8a0 Mon Sep 17 00:00:00 2001 From: Ruidong Date: Fri, 14 Sep 2018 02:17:06 +0800 Subject: [PATCH 2/2] cli: add configuration option to enable/disable vhost_net Add `disable_vhost_net` option to enable or disable the use of vhost_net. Vhost_net can improve network performance. Signed-off-by: Ruidong Cao --- cli/config.go | 2 ++ cli/config/configuration.toml.in | 4 ++++ virtcontainers/hypervisor.go | 3 +++ virtcontainers/network.go | 38 ++++++++++++++++++-------------- virtcontainers/qemu.go | 7 ++++++ virtcontainers/qemu_arch_base.go | 17 +++++++++++++- 6 files changed, 53 insertions(+), 18 deletions(-) diff --git a/cli/config.go b/cli/config.go index 890dc8fa3..b57755a36 100644 --- a/cli/config.go +++ b/cli/config.go @@ -94,6 +94,7 @@ type hypervisor struct { EnableIOThreads bool `toml:"enable_iothreads"` UseVSock bool `toml:"use_vsock"` HotplugVFIOOnRootBus bool `toml:"hotplug_vfio_on_root_bus"` + DisableVhostNet bool `toml:"disable_vhost_net"` } type proxy struct { @@ -375,6 +376,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { Msize9p: h.msize9p(), UseVSock: useVSock, HotplugVFIOOnRootBus: h.HotplugVFIOOnRootBus, + DisableVhostNet: h.DisableVhostNet, }, nil } diff --git a/cli/config/configuration.toml.in b/cli/config/configuration.toml.in index c93bc8259..842ff16b0 100644 --- a/cli/config/configuration.toml.in +++ b/cli/config/configuration.toml.in @@ -147,6 +147,10 @@ enable_iothreads = @DEFENABLEIOTHREADS@ # Default false #hotplug_vfio_on_root_bus = true +# If host doesn't support vhost_net, set to true. Thus we won't create vhost fds for nics. +# Default false +#disable_vhost_net = true + [factory] # VM templating support. Once enabled, new VMs are created from template # using vm cloning. They will share the same initial kernel, initramfs and diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go index 6d0a850f8..8d2a8a1fb 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor.go @@ -240,6 +240,9 @@ type HypervisorConfig struct { // DevicesStatePath is the VM device state file path. Used when either BootToBeTemplate or // BootFromTemplate is true. DevicesStatePath string + + // DisableVhostNet is used to indicate if host supports vhost_net + DisableVhostNet bool } func (conf *HypervisorConfig) checkTemplateConfig() error { diff --git a/virtcontainers/network.go b/virtcontainers/network.go index 080bf00eb..cd35c6c0f 100644 --- a/virtcontainers/network.go +++ b/virtcontainers/network.go @@ -225,7 +225,7 @@ func networkLogger() *logrus.Entry { func (endpoint *VirtualEndpoint) Attach(h hypervisor) error { networkLogger().WithField("endpoint-type", "virtual").Info("Attaching endpoint") - if err := xconnectVMNetwork(&(endpoint.NetPair), true, h.hypervisorConfig().NumVCPUs); err != nil { + if err := xconnectVMNetwork(&(endpoint.NetPair), true, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil { networkLogger().WithError(err).Error("Error bridging virtual endpoint") return err } @@ -245,14 +245,14 @@ func (endpoint *VirtualEndpoint) Detach(netNsCreated bool, netNsPath string) err networkLogger().WithField("endpoint-type", "virtual").Info("Detaching endpoint") return doNetNS(netNsPath, func(_ ns.NetNS) error { - return xconnectVMNetwork(&(endpoint.NetPair), false, 0) + return xconnectVMNetwork(&(endpoint.NetPair), false, 0, false) }) } // HotAttach for the virtual endpoint uses hot plug device func (endpoint *VirtualEndpoint) HotAttach(h hypervisor) error { networkLogger().Info("Hot attaching virtual endpoint") - if err := xconnectVMNetwork(&(endpoint.NetPair), true, h.hypervisorConfig().NumVCPUs); err != nil { + if err := xconnectVMNetwork(&(endpoint.NetPair), true, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil { networkLogger().WithError(err).Error("Error bridging virtual ep") return err } @@ -271,7 +271,7 @@ func (endpoint *VirtualEndpoint) HotDetach(h hypervisor, netNsCreated bool, netN } networkLogger().Info("Hot detaching virtual endpoint") if err := doNetNS(netNsPath, func(_ ns.NetNS) error { - return xconnectVMNetwork(&(endpoint.NetPair), false, 0) + return xconnectVMNetwork(&(endpoint.NetPair), false, 0, h.hypervisorConfig().DisableVhostNet) }); err != nil { networkLogger().WithError(err).Error("Error abridging virtual ep") return err @@ -721,7 +721,7 @@ func getLinkByName(netHandle *netlink.Handle, name string, expectedLink netlink. } // The endpoint type should dictate how the connection needs to be made -func xconnectVMNetwork(netPair *NetworkInterfacePair, connect bool, numCPUs uint32) error { +func xconnectVMNetwork(netPair *NetworkInterfacePair, connect bool, numCPUs uint32, disableVhostNet bool) error { if netPair.NetInterworkingModel == NetXConnectDefaultModel { netPair.NetInterworkingModel = DefaultNetInterworkingModel } @@ -729,13 +729,13 @@ func xconnectVMNetwork(netPair *NetworkInterfacePair, connect bool, numCPUs uint case NetXConnectBridgedModel: netPair.NetInterworkingModel = NetXConnectBridgedModel if connect { - return bridgeNetworkPair(netPair, numCPUs) + return bridgeNetworkPair(netPair, numCPUs, disableVhostNet) } return unBridgeNetworkPair(*netPair) case NetXConnectMacVtapModel: netPair.NetInterworkingModel = NetXConnectMacVtapModel if connect { - return tapNetworkPair(netPair, numCPUs) + return tapNetworkPair(netPair, numCPUs, disableVhostNet) } return untapNetworkPair(*netPair) case NetXConnectEnlightenedModel: @@ -824,7 +824,7 @@ func setIPs(link netlink.Link, addrs []netlink.Addr) error { return nil } -func tapNetworkPair(netPair *NetworkInterfacePair, numCPUs uint32) error { +func tapNetworkPair(netPair *NetworkInterfacePair, numCPUs uint32, disableVhostNet bool) error { netHandle, err := netlink.NewHandle() if err != nil { return err @@ -904,16 +904,18 @@ func tapNetworkPair(netPair *NetworkInterfacePair, numCPUs uint32) error { return fmt.Errorf("Could not setup macvtap fds %s: %s", netPair.TAPIface, err) } - vhostFds, err := createVhostFds(int(numCPUs)) - if err != nil { - return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err) + if !disableVhostNet { + vhostFds, err := createVhostFds(int(numCPUs)) + if err != nil { + return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err) + } + netPair.VhostFds = vhostFds } - netPair.VhostFds = vhostFds return nil } -func bridgeNetworkPair(netPair *NetworkInterfacePair, numCPUs uint32) error { +func bridgeNetworkPair(netPair *NetworkInterfacePair, numCPUs uint32, disableVhostNet bool) error { netHandle, err := netlink.NewHandle() if err != nil { return err @@ -926,11 +928,13 @@ func bridgeNetworkPair(netPair *NetworkInterfacePair, numCPUs uint32) error { } netPair.VMFds = fds - vhostFds, err := createVhostFds(int(numCPUs)) - if err != nil { - return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err) + if !disableVhostNet { + vhostFds, err := createVhostFds(int(numCPUs)) + if err != nil { + return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err) + } + netPair.VhostFds = vhostFds } - netPair.VhostFds = vhostFds vethLink, err := getLinkByName(netHandle, netPair.VirtIface.Name, &netlink.Veth{}) if err != nil { diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index 09fb8f096..4aa2bca70 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -249,6 +249,13 @@ func (q *qemu) init(ctx context.Context, id string, hypervisorConfig *Hypervisor q.arch.disableNestingChecks() } + if !q.config.DisableVhostNet { + q.arch.enableVhostNet() + } else { + q.Logger().Debug("Disable vhost_net") + q.arch.disableVhostNet() + } + return nil } diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go index b40ebfcca..8dd607fa5 100644 --- a/virtcontainers/qemu_arch_base.go +++ b/virtcontainers/qemu_arch_base.go @@ -24,6 +24,12 @@ type qemuArch interface { // disableNestingChecks nesting checks will be ignored disableNestingChecks() + // enableVhostNet vhost will be enabled + enableVhostNet() + + // disableVhostNet vhost will be disabled + disableVhostNet() + // machine returns the machine type machine() (govmmQemu.Machine, error) @@ -92,6 +98,7 @@ type qemuArch interface { type qemuArchBase struct { machineType string nestedRun bool + vhost bool networkIndex int qemuPaths map[string]string supportedQemuMachines []govmmQemu.Machine @@ -176,6 +183,14 @@ func (q *qemuArchBase) disableNestingChecks() { q.nestedRun = false } +func (q *qemuArchBase) enableVhostNet() { + q.vhost = true +} + +func (q *qemuArchBase) disableVhostNet() { + q.vhost = false +} + func (q *qemuArchBase) machine() (govmmQemu.Machine, error) { for _, m := range q.supportedQemuMachines { if m.Type == q.machineType { @@ -437,7 +452,7 @@ func (q *qemuArchBase) appendNetwork(devices []govmmQemu.Device, endpoint Endpoi MACAddress: ep.NetPair.TAPIface.HardAddr, DownScript: "no", Script: "no", - VHost: true, + VHost: q.vhost, DisableModern: q.nestedRun, FDs: ep.NetPair.VMFds, VhostFDs: ep.NetPair.VhostFds,