diff --git a/virtcontainers/bridgedmacvlan_endpoint.go b/virtcontainers/bridgedmacvlan_endpoint.go index 9ea0a30663..d2efd56f53 100644 --- a/virtcontainers/bridgedmacvlan_endpoint.go +++ b/virtcontainers/bridgedmacvlan_endpoint.go @@ -84,7 +84,7 @@ func (endpoint *BridgedMacvlanEndpoint) NetworkPair() *NetworkInterfacePair { // Attach for virtual endpoint bridges the network pair and adds the // tap interface of the network pair to the hypervisor. func (endpoint *BridgedMacvlanEndpoint) Attach(h hypervisor) error { - if err := xconnectVMNetwork(endpoint, true, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil { + if err := xConnectVMNetwork(endpoint, h); err != nil { networkLogger().WithError(err).Error("Error bridging virtual ep") return err } @@ -102,7 +102,7 @@ func (endpoint *BridgedMacvlanEndpoint) Detach(netNsCreated bool, netNsPath stri } return doNetNS(netNsPath, func(_ ns.NetNS) error { - return xconnectVMNetwork(endpoint, false, 0, false) + return xDisconnectVMNetwork(endpoint) }) } diff --git a/virtcontainers/capabilities.go b/virtcontainers/capabilities.go index 8760c99691..6eb34c3b81 100644 --- a/virtcontainers/capabilities.go +++ b/virtcontainers/capabilities.go @@ -8,6 +8,7 @@ package virtcontainers const ( blockDeviceSupport = 1 << iota blockDeviceHotplugSupport + multiQueueSupport ) type capabilities struct { @@ -35,3 +36,14 @@ func (caps *capabilities) isBlockDeviceHotplugSupported() bool { func (caps *capabilities) setBlockDeviceHotplugSupport() { caps.flags |= blockDeviceHotplugSupport } + +func (caps *capabilities) isMultiQueueSupported() bool { + if caps.flags&multiQueueSupport != 0 { + return true + } + return false +} + +func (caps *capabilities) setMultiQueueSupport() { + caps.flags |= multiQueueSupport +} diff --git a/virtcontainers/ipvlan_endpoint.go b/virtcontainers/ipvlan_endpoint.go index beab38d867..b71bbd0fbe 100644 --- a/virtcontainers/ipvlan_endpoint.go +++ b/virtcontainers/ipvlan_endpoint.go @@ -87,7 +87,7 @@ func (endpoint *IPVlanEndpoint) NetworkPair() *NetworkInterfacePair { // Attach for virtual endpoint bridges the network pair and adds the // tap interface of the network pair to the hypervisor. func (endpoint *IPVlanEndpoint) Attach(h hypervisor) error { - if err := xconnectVMNetwork(endpoint, true, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil { + if err := xConnectVMNetwork(endpoint, h); err != nil { networkLogger().WithError(err).Error("Error bridging virtual ep") return err } @@ -105,7 +105,7 @@ func (endpoint *IPVlanEndpoint) Detach(netNsCreated bool, netNsPath string) erro } return doNetNS(netNsPath, func(_ ns.NetNS) error { - return xconnectVMNetwork(endpoint, false, 0, false) + return xDisconnectVMNetwork(endpoint) }) } diff --git a/virtcontainers/network.go b/virtcontainers/network.go index a0846e638b..e1d1d25e3c 100644 --- a/virtcontainers/network.go +++ b/virtcontainers/network.go @@ -396,7 +396,7 @@ func newNetwork(networkType NetworkModel) network { } } -func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Link, numCPUs int) (netlink.Link, []*os.File, error) { +func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Link, queues int) (netlink.Link, []*os.File, error) { var newLink netlink.Link var fds []*os.File @@ -410,7 +410,7 @@ func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Lin newLink = &netlink.Tuntap{ LinkAttrs: netlink.LinkAttrs{Name: name}, Mode: netlink.TUNTAP_MODE_TAP, - Queues: numCPUs, + Queues: queues, Flags: netlink.TUNTAP_MULTI_QUEUE_DEFAULTS | netlink.TUNTAP_VNET_HDR, } case (&netlink.Macvtap{}).Type(): @@ -501,30 +501,50 @@ func getLinkByName(netHandle *netlink.Handle, name string, expectedLink netlink. return nil, fmt.Errorf("Incorrect link type %s, expecting %s", link.Type(), expectedLink.Type()) } -// The endpoint type should dictate how the connection needs to be made -func xconnectVMNetwork(endpoint Endpoint, connect bool, numCPUs uint32, disableVhostNet bool) error { +// The endpoint type should dictate how the connection needs to happen. +func xConnectVMNetwork(endpoint Endpoint, h hypervisor) error { + netPair := endpoint.NetworkPair() + + queues := 0 + caps := h.capabilities() + if caps.isMultiQueueSupported() { + queues = int(h.hypervisorConfig().NumVCPUs) + } + + disableVhostNet := h.hypervisorConfig().DisableVhostNet + + if netPair.NetInterworkingModel == NetXConnectDefaultModel { + netPair.NetInterworkingModel = DefaultNetInterworkingModel + } + + switch netPair.NetInterworkingModel { + case NetXConnectBridgedModel: + return bridgeNetworkPair(endpoint, queues, disableVhostNet) + case NetXConnectMacVtapModel: + return tapNetworkPair(endpoint, queues, disableVhostNet) + case NetXConnectTCFilterModel: + return setupTCFiltering(endpoint, queues, disableVhostNet) + case NetXConnectEnlightenedModel: + return fmt.Errorf("Unsupported networking model") + default: + return fmt.Errorf("Invalid internetworking model") + } +} + +// The endpoint type should dictate how the disconnection needs to happen. +func xDisconnectVMNetwork(endpoint Endpoint) error { netPair := endpoint.NetworkPair() if netPair.NetInterworkingModel == NetXConnectDefaultModel { netPair.NetInterworkingModel = DefaultNetInterworkingModel } + switch netPair.NetInterworkingModel { case NetXConnectBridgedModel: - netPair.NetInterworkingModel = NetXConnectBridgedModel - if connect { - return bridgeNetworkPair(endpoint, numCPUs, disableVhostNet) - } return unBridgeNetworkPair(endpoint) case NetXConnectMacVtapModel: - netPair.NetInterworkingModel = NetXConnectMacVtapModel - if connect { - return tapNetworkPair(endpoint, numCPUs, disableVhostNet) - } return untapNetworkPair(endpoint) case NetXConnectTCFilterModel: - if connect { - return setupTCFiltering(endpoint, numCPUs, disableVhostNet) - } return removeTCFiltering(endpoint) case NetXConnectEnlightenedModel: return fmt.Errorf("Unsupported networking model") @@ -573,10 +593,10 @@ const linkRange = 0xFFFF // This will allow upto 2^16 containers const linkRetries = 128 // The numbers of time we try to find a non conflicting index const macvtapWorkaround = true -func createMacVtap(netHandle *netlink.Handle, name string, link netlink.Link, numCPUs int) (taplink netlink.Link, err error) { +func createMacVtap(netHandle *netlink.Handle, name string, link netlink.Link, queues int) (taplink netlink.Link, err error) { if !macvtapWorkaround { - taplink, _, err = createLink(netHandle, name, link, numCPUs) + taplink, _, err = createLink(netHandle, name, link, queues) return } @@ -585,7 +605,7 @@ func createMacVtap(netHandle *netlink.Handle, name string, link netlink.Link, nu for i := 0; i < linkRetries; i++ { index := hostLinkOffset + (r.Int() & linkRange) link.Attrs().Index = index - taplink, _, err = createLink(netHandle, name, link, numCPUs) + taplink, _, err = createLink(netHandle, name, link, queues) if err == nil { break } @@ -612,7 +632,7 @@ func setIPs(link netlink.Link, addrs []netlink.Addr) error { return nil } -func tapNetworkPair(endpoint Endpoint, numCPUs uint32, disableVhostNet bool) error { +func tapNetworkPair(endpoint Endpoint, queues int, disableVhostNet bool) error { netHandle, err := netlink.NewHandle() if err != nil { return err @@ -638,7 +658,7 @@ func tapNetworkPair(endpoint Endpoint, numCPUs uint32, disableVhostNet bool) err ParentIndex: attrs.Index, }, }, - }, int(numCPUs)) + }, queues) if err != nil { return fmt.Errorf("Could not create TAP interface: %s", err) @@ -690,13 +710,13 @@ func tapNetworkPair(endpoint Endpoint, numCPUs uint32, disableVhostNet bool) err // Note: The underlying interfaces need to be up prior to fd creation. - netPair.VMFds, err = createMacvtapFds(tapLink.Attrs().Index, int(numCPUs)) + netPair.VMFds, err = createMacvtapFds(tapLink.Attrs().Index, queues) if err != nil { return fmt.Errorf("Could not setup macvtap fds %s: %s", netPair.TAPIface, err) } if !disableVhostNet { - vhostFds, err := createVhostFds(int(numCPUs)) + vhostFds, err := createVhostFds(queues) if err != nil { return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err) } @@ -706,7 +726,7 @@ func tapNetworkPair(endpoint Endpoint, numCPUs uint32, disableVhostNet bool) err return nil } -func bridgeNetworkPair(endpoint Endpoint, numCPUs uint32, disableVhostNet bool) error { +func bridgeNetworkPair(endpoint Endpoint, queues int, disableVhostNet bool) error { netHandle, err := netlink.NewHandle() if err != nil { return err @@ -715,14 +735,14 @@ func bridgeNetworkPair(endpoint Endpoint, numCPUs uint32, disableVhostNet bool) netPair := endpoint.NetworkPair() - tapLink, fds, err := createLink(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}, int(numCPUs)) + tapLink, fds, err := createLink(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}, queues) if err != nil { return fmt.Errorf("Could not create TAP interface: %s", err) } netPair.VMFds = fds if !disableVhostNet { - vhostFds, err := createVhostFds(int(numCPUs)) + vhostFds, err := createVhostFds(queues) if err != nil { return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err) } @@ -760,7 +780,7 @@ func bridgeNetworkPair(endpoint Endpoint, numCPUs uint32, disableVhostNet bool) } mcastSnoop := false - bridgeLink, _, err := createLink(netHandle, netPair.Name, &netlink.Bridge{MulticastSnooping: &mcastSnoop}, int(numCPUs)) + bridgeLink, _, err := createLink(netHandle, netPair.Name, &netlink.Bridge{MulticastSnooping: &mcastSnoop}, queues) if err != nil { return fmt.Errorf("Could not create bridge: %s", err) } @@ -790,7 +810,7 @@ func bridgeNetworkPair(endpoint Endpoint, numCPUs uint32, disableVhostNet bool) return nil } -func setupTCFiltering(endpoint Endpoint, numCPUs uint32, disableVhostNet bool) error { +func setupTCFiltering(endpoint Endpoint, queues int, disableVhostNet bool) error { netHandle, err := netlink.NewHandle() if err != nil { return err @@ -799,14 +819,14 @@ func setupTCFiltering(endpoint Endpoint, numCPUs uint32, disableVhostNet bool) e netPair := endpoint.NetworkPair() - tapLink, fds, err := createLink(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}, int(numCPUs)) + tapLink, fds, err := createLink(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}, queues) if err != nil { return fmt.Errorf("Could not create TAP interface: %s", err) } netPair.VMFds = fds if !disableVhostNet { - vhostFds, err := createVhostFds(int(numCPUs)) + vhostFds, err := createVhostFds(queues) if err != nil { return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err) } diff --git a/virtcontainers/qemu_amd64.go b/virtcontainers/qemu_amd64.go index 5eb002ff26..8dfddde932 100644 --- a/virtcontainers/qemu_amd64.go +++ b/virtcontainers/qemu_amd64.go @@ -108,6 +108,8 @@ func (q *qemuAmd64) capabilities() capabilities { caps.setBlockDeviceHotplugSupport() } + caps.setMultiQueueSupport() + return caps } diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go index 63f5065538..864edfe376 100644 --- a/virtcontainers/qemu_arch_base.go +++ b/virtcontainers/qemu_arch_base.go @@ -237,6 +237,7 @@ func (q *qemuArchBase) kernelParameters(debug bool) []Param { func (q *qemuArchBase) capabilities() capabilities { var caps capabilities caps.setBlockDeviceHotplugSupport() + caps.setMultiQueueSupport() return caps } diff --git a/virtcontainers/qemu_ppc64le.go b/virtcontainers/qemu_ppc64le.go index 6b5d8b654a..05d37ce755 100644 --- a/virtcontainers/qemu_ppc64le.go +++ b/virtcontainers/qemu_ppc64le.go @@ -94,6 +94,8 @@ func (q *qemuPPC64le) capabilities() capabilities { caps.setBlockDeviceHotplugSupport() } + caps.setMultiQueueSupport() + return caps } diff --git a/virtcontainers/veth_endpoint.go b/virtcontainers/veth_endpoint.go index 3246151542..66994a1497 100644 --- a/virtcontainers/veth_endpoint.go +++ b/virtcontainers/veth_endpoint.go @@ -88,7 +88,7 @@ func (endpoint *VethEndpoint) SetProperties(properties NetworkInfo) { // Attach for veth endpoint bridges the network pair and adds the // tap interface of the network pair to the hypervisor. func (endpoint *VethEndpoint) Attach(h hypervisor) error { - if err := xconnectVMNetwork(endpoint, true, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil { + if err := xConnectVMNetwork(endpoint, h); err != nil { networkLogger().WithError(err).Error("Error bridging virtual endpoint") return err } @@ -106,13 +106,13 @@ func (endpoint *VethEndpoint) Detach(netNsCreated bool, netNsPath string) error } return doNetNS(netNsPath, func(_ ns.NetNS) error { - return xconnectVMNetwork(endpoint, false, 0, false) + return xDisconnectVMNetwork(endpoint) }) } // HotAttach for the veth endpoint uses hot plug device func (endpoint *VethEndpoint) HotAttach(h hypervisor) error { - if err := xconnectVMNetwork(endpoint, true, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil { + if err := xConnectVMNetwork(endpoint, h); err != nil { networkLogger().WithError(err).Error("Error bridging virtual ep") return err } @@ -131,7 +131,7 @@ func (endpoint *VethEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPa } if err := doNetNS(netNsPath, func(_ ns.NetNS) error { - return xconnectVMNetwork(endpoint, false, 0, h.hypervisorConfig().DisableVhostNet) + return xDisconnectVMNetwork(endpoint) }); err != nil { networkLogger().WithError(err).Warn("Error un-bridging virtual ep") }