github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-08-20 08:54:03 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #719 from amshinde/net-queue-size

Browse Source 
Set the queue size for Multiqueue virtio-net as the number of vCPUs on the guest.
This commit is contained in:
Peng Tao 2018-09-13 22:40:06 +08:00 committed by GitHub
commit 5404aaba6f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 31 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
virtcontainers/hypervisor.go
View File

@ -560,4 +560,5 @@ type hypervisor interface {
getSandboxConsole(sandboxID string) (string, error) getSandboxConsole(sandboxID string) (string, error)
disconnect() disconnect()
capabilities() capabilities capabilities() capabilities
hypervisorConfig() HypervisorConfig
} }

4
virtcontainers/mock_hypervisor.go
View File

@ -24,6 +24,10 @@ func (m *mockHypervisor) capabilities() capabilities {
return capabilities{} return capabilities{}
} }
func (m *mockHypervisor) hypervisorConfig() HypervisorConfig {
return HypervisorConfig{}
}
func (m *mockHypervisor) createSandbox() error { func (m *mockHypervisor) createSandbox() error {
return nil return nil
} }

50
virtcontainers/network.go
View File

@ -95,7 +95,6 @@ const (
defaultRouteLabel = "default" defaultRouteLabel = "default"
defaultFilePerms = 0600 defaultFilePerms = 0600
defaultQlen = 1500 defaultQlen = 1500
defaultQueues = 8
) )
// DNSInfo describes the DNS setup related to a network interface. // DNSInfo describes the DNS setup related to a network interface.
@ -226,7 +225,7 @@ func networkLogger() *logrus.Entry {
func (endpoint *VirtualEndpoint) Attach(h hypervisor) error { func (endpoint *VirtualEndpoint) Attach(h hypervisor) error {
networkLogger().WithField("endpoint-type", "virtual").Info("Attaching endpoint") networkLogger().WithField("endpoint-type", "virtual").Info("Attaching endpoint")
if err := xconnectVMNetwork(&(endpoint.NetPair), true); err != nil { if err := xconnectVMNetwork(&(endpoint.NetPair), true, h.hypervisorConfig().NumVCPUs); err != nil {
networkLogger().WithError(err).Error("Error bridging virtual endpoint") networkLogger().WithError(err).Error("Error bridging virtual endpoint")
return err return err
} }
@ -246,14 +245,14 @@ func (endpoint *VirtualEndpoint) Detach(netNsCreated bool, netNsPath string) err
networkLogger().WithField("endpoint-type", "virtual").Info("Detaching endpoint") networkLogger().WithField("endpoint-type", "virtual").Info("Detaching endpoint")
return doNetNS(netNsPath, func(_ ns.NetNS) error { return doNetNS(netNsPath, func(_ ns.NetNS) error {
return xconnectVMNetwork(&(endpoint.NetPair), false) return xconnectVMNetwork(&(endpoint.NetPair), false, 0)
}) })
} }
// HotAttach for the virtual endpoint uses hot plug device // HotAttach for the virtual endpoint uses hot plug device
func (endpoint *VirtualEndpoint) HotAttach(h hypervisor) error { func (endpoint *VirtualEndpoint) HotAttach(h hypervisor) error {
networkLogger().Info("Hot attaching virtual endpoint") networkLogger().Info("Hot attaching virtual endpoint")
if err := xconnectVMNetwork(&(endpoint.NetPair), true); err != nil { if err := xconnectVMNetwork(&(endpoint.NetPair), true, h.hypervisorConfig().NumVCPUs); err != nil {
networkLogger().WithError(err).Error("Error bridging virtual ep") networkLogger().WithError(err).Error("Error bridging virtual ep")
return err return err
} }
@ -272,7 +271,7 @@ func (endpoint *VirtualEndpoint) HotDetach(h hypervisor, netNsCreated bool, netN
} }
networkLogger().Info("Hot detaching virtual endpoint") networkLogger().Info("Hot detaching virtual endpoint")
if err := doNetNS(netNsPath, func(_ ns.NetNS) error { if err := doNetNS(netNsPath, func(_ ns.NetNS) error {
return xconnectVMNetwork(&(endpoint.NetPair), false) return xconnectVMNetwork(&(endpoint.NetPair), false, 0)
}); err != nil { }); err != nil {
networkLogger().WithError(err).Error("Error abridging virtual ep") networkLogger().WithError(err).Error("Error abridging virtual ep")
return err return err
@ -641,7 +640,7 @@ func newNetwork(networkType NetworkModel) network {
} }
} }
func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Link) (netlink.Link, []*os.File, error) { func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Link, numCPUs int) (netlink.Link, []*os.File, error) {
var newLink netlink.Link var newLink netlink.Link
var fds []*os.File var fds []*os.File
@ -655,7 +654,7 @@ func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Lin
newLink = &netlink.Tuntap{ newLink = &netlink.Tuntap{
LinkAttrs: netlink.LinkAttrs{Name: name}, LinkAttrs: netlink.LinkAttrs{Name: name},
Mode: netlink.TUNTAP_MODE_TAP, Mode: netlink.TUNTAP_MODE_TAP,
Queues: defaultQueues, Queues: numCPUs,
Flags: netlink.TUNTAP_MULTI_QUEUE_DEFAULTS | netlink.TUNTAP_VNET_HDR, Flags: netlink.TUNTAP_MULTI_QUEUE_DEFAULTS | netlink.TUNTAP_VNET_HDR,
} }
case (&netlink.Macvtap{}).Type(): case (&netlink.Macvtap{}).Type():
@ -722,7 +721,7 @@ func getLinkByName(netHandle *netlink.Handle, name string, expectedLink netlink.
} }
// The endpoint type should dictate how the connection needs to be made // The endpoint type should dictate how the connection needs to be made
func xconnectVMNetwork(netPair *NetworkInterfacePair, connect bool) error { func xconnectVMNetwork(netPair *NetworkInterfacePair, connect bool, numCPUs uint32) error {
if netPair.NetInterworkingModel == NetXConnectDefaultModel { if netPair.NetInterworkingModel == NetXConnectDefaultModel {
netPair.NetInterworkingModel = DefaultNetInterworkingModel netPair.NetInterworkingModel = DefaultNetInterworkingModel
} }
@ -730,13 +729,13 @@ func xconnectVMNetwork(netPair *NetworkInterfacePair, connect bool) error {
case NetXConnectBridgedModel: case NetXConnectBridgedModel:
netPair.NetInterworkingModel = NetXConnectBridgedModel netPair.NetInterworkingModel = NetXConnectBridgedModel
if connect { if connect {
return bridgeNetworkPair(netPair) return bridgeNetworkPair(netPair, numCPUs)
} }
return unBridgeNetworkPair(*netPair) return unBridgeNetworkPair(*netPair)
case NetXConnectMacVtapModel: case NetXConnectMacVtapModel:
netPair.NetInterworkingModel = NetXConnectMacVtapModel netPair.NetInterworkingModel = NetXConnectMacVtapModel
if connect { if connect {
return tapNetworkPair(netPair) return tapNetworkPair(netPair, numCPUs)
} }
return untapNetworkPair(*netPair) return untapNetworkPair(*netPair)
case NetXConnectEnlightenedModel: case NetXConnectEnlightenedModel:
@ -786,10 +785,10 @@ const linkRange = 0xFFFF // This will allow upto 2^16 containers
const linkRetries = 128 // The numbers of time we try to find a non conflicting index const linkRetries = 128 // The numbers of time we try to find a non conflicting index
const macvtapWorkaround = true const macvtapWorkaround = true
func createMacVtap(netHandle *netlink.Handle, name string, link netlink.Link) (taplink netlink.Link, err error) { func createMacVtap(netHandle *netlink.Handle, name string, link netlink.Link, numCPUs int) (taplink netlink.Link, err error) {
if !macvtapWorkaround { if !macvtapWorkaround {
taplink, _, err = createLink(netHandle, name, link) taplink, _, err = createLink(netHandle, name, link, numCPUs)
return return
} }
@ -798,7 +797,7 @@ func createMacVtap(netHandle *netlink.Handle, name string, link netlink.Link) (t
for i := 0; i < linkRetries; i++ { for i := 0; i < linkRetries; i++ {
index := hostLinkOffset + (r.Int() & linkRange) index := hostLinkOffset + (r.Int() & linkRange)
link.Attrs().Index = index link.Attrs().Index = index
taplink, _, err = createLink(netHandle, name, link) taplink, _, err = createLink(netHandle, name, link, numCPUs)
if err == nil { if err == nil {
break break
} }
@ -825,7 +824,7 @@ func setIPs(link netlink.Link, addrs []netlink.Addr) error {
return nil return nil
} }
func tapNetworkPair(netPair *NetworkInterfacePair) error { func tapNetworkPair(netPair *NetworkInterfacePair, numCPUs uint32) error {
netHandle, err := netlink.NewHandle() netHandle, err := netlink.NewHandle()
if err != nil { if err != nil {
return err return err
@ -848,7 +847,7 @@ func tapNetworkPair(netPair *NetworkInterfacePair) error {
ParentIndex: vethLinkAttrs.Index, ParentIndex: vethLinkAttrs.Index,
}, },
}, },
}) }, int(numCPUs))
if err != nil { if err != nil {
return fmt.Errorf("Could not create TAP interface: %s", err) return fmt.Errorf("Could not create TAP interface: %s", err)
@ -900,21 +899,12 @@ func tapNetworkPair(netPair *NetworkInterfacePair) error {
// Note: The underlying interfaces need to be up prior to fd creation. // Note: The underlying interfaces need to be up prior to fd creation.
// Setup the multiqueue fds to be consumed by QEMU as macvtap cannot netPair.VMFds, err = createMacvtapFds(tapLink.Attrs().Index, int(numCPUs))
// be directly connected.
// Ideally we want
// netdev.FDs, err = createMacvtapFds(netdev.ID, int(config.SMP.CPUs))
// We do not have global context here, hence a manifest constant
// that matches our minimum vCPU configuration
// Another option is to defer this to ciao qemu library which does have
// global context but cannot handle errors when setting up the network
netPair.VMFds, err = createMacvtapFds(tapLink.Attrs().Index, defaultQueues)
if err != nil { if err != nil {
return fmt.Errorf("Could not setup macvtap fds %s: %s", netPair.TAPIface, err) return fmt.Errorf("Could not setup macvtap fds %s: %s", netPair.TAPIface, err)
} }
vhostFds, err := createVhostFds(defaultQueues) vhostFds, err := createVhostFds(int(numCPUs))
if err != nil { if err != nil {
return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err) return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err)
} }
@ -923,20 +913,20 @@ func tapNetworkPair(netPair *NetworkInterfacePair) error {
return nil return nil
} }
func bridgeNetworkPair(netPair *NetworkInterfacePair) error { func bridgeNetworkPair(netPair *NetworkInterfacePair, numCPUs uint32) error {
netHandle, err := netlink.NewHandle() netHandle, err := netlink.NewHandle()
if err != nil { if err != nil {
return err return err
} }
defer netHandle.Delete() defer netHandle.Delete()
tapLink, fds, err := createLink(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}) tapLink, fds, err := createLink(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}, int(numCPUs))
if err != nil { if err != nil {
return fmt.Errorf("Could not create TAP interface: %s", err) return fmt.Errorf("Could not create TAP interface: %s", err)
} }
netPair.VMFds = fds netPair.VMFds = fds
vhostFds, err := createVhostFds(defaultQueues) vhostFds, err := createVhostFds(int(numCPUs))
if err != nil { if err != nil {
return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err) return fmt.Errorf("Could not setup vhost fds %s : %s", netPair.VirtIface.Name, err)
} }
@ -970,7 +960,7 @@ func bridgeNetworkPair(netPair *NetworkInterfacePair) error {
} }
mcastSnoop := false mcastSnoop := false
bridgeLink, _, err := createLink(netHandle, netPair.Name, &netlink.Bridge{MulticastSnooping: &mcastSnoop}) bridgeLink, _, err := createLink(netHandle, netPair.Name, &netlink.Bridge{MulticastSnooping: &mcastSnoop}, int(numCPUs))
if err != nil { if err != nil {
return fmt.Errorf("Could not create bridge: %s", err) return fmt.Errorf("Could not create bridge: %s", err)
} }

8
virtcontainers/qemu.go
View File

@ -160,6 +160,10 @@ func (q *qemu) capabilities() capabilities {
return q.arch.capabilities() return q.arch.capabilities()
} }
func (q *qemu) hypervisorConfig() HypervisorConfig {
return q.config
}
// get the QEMU binary path // get the QEMU binary path
func (q *qemu) qemuPath() (string, error) { func (q *qemu) qemuPath() (string, error) {
p, err := q.config.HypervisorAssetPath() p, err := q.config.HypervisorAssetPath()
@ -844,7 +848,7 @@ func (q *qemu) hotplugNetDevice(drive VirtualEndpoint, op operation) error {
if op == addDevice { if op == addDevice {
switch drive.NetPair.NetInterworkingModel { switch drive.NetPair.NetInterworkingModel {
case NetXConnectBridgedModel: case NetXConnectBridgedModel:
if err := q.qmpMonitorCh.qmp.ExecuteNetdevAdd(q.qmpMonitorCh.ctx, "tap", drive.NetPair.Name, drive.NetPair.TAPIface.Name, "no", "no", defaultQueues); err != nil { if err := q.qmpMonitorCh.qmp.ExecuteNetdevAdd(q.qmpMonitorCh.ctx, "tap", drive.NetPair.Name, drive.NetPair.TAPIface.Name, "no", "no", int(q.config.NumVCPUs)); err != nil {
return err return err
} }
case NetXConnectMacVtapModel: case NetXConnectMacVtapModel:
@ -859,7 +863,7 @@ func (q *qemu) hotplugNetDevice(drive VirtualEndpoint, op operation) error {
return err return err
} }
drive.PCIAddr = fmt.Sprintf("%02x/%s", bridge.Addr, addr) drive.PCIAddr = fmt.Sprintf("%02x/%s", bridge.Addr, addr)
if err = q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, drive.NetPair.Name, devID, drive.NetPair.TAPIface.HardAddr, addr, bridge.ID, defaultQueues); err != nil { if err = q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, drive.NetPair.Name, devID, drive.NetPair.TAPIface.HardAddr, addr, bridge.ID, int(q.config.NumVCPUs)); err != nil {
return err return err
} }
} else { } else {