diff --git a/virtcontainers/network.go b/virtcontainers/network.go index b9e9bf577b..ab4389e488 100644 --- a/virtcontainers/network.go +++ b/virtcontainers/network.go @@ -6,6 +6,7 @@ package virtcontainers import ( + cryptoRand "crypto/rand" "encoding/hex" "encoding/json" "fmt" @@ -153,6 +154,7 @@ type Endpoint interface { HardwareAddr() string Type() EndpointType PciAddr() string + NetworkPair() *NetworkInterfacePair SetProperties(NetworkInfo) Attach(hypervisor) error @@ -194,6 +196,23 @@ type VhostUserEndpoint struct { PCIAddr string } +// BridgedMacvlanEndpoint represents a macvlan endpoint that is bridged to the VM +type BridgedMacvlanEndpoint struct { + NetPair NetworkInterfacePair + EndpointProperties NetworkInfo + EndpointType EndpointType + PCIAddr string +} + +// MacvtapEndpoint represents a macvtap endpoint +type MacvtapEndpoint struct { + EndpointProperties NetworkInfo + EndpointType EndpointType + VMFds []*os.File + VhostFds []*os.File + PCIAddr string +} + // Properties returns properties for the veth interface in the network pair. func (endpoint *VirtualEndpoint) Properties() NetworkInfo { return endpoint.EndpointProperties @@ -220,6 +239,11 @@ func (endpoint *VirtualEndpoint) PciAddr() string { return endpoint.PCIAddr } +// NetworkPair returns the network pair of the endpoint. +func (endpoint *VirtualEndpoint) NetworkPair() *NetworkInterfacePair { + return &endpoint.NetPair +} + // SetProperties sets the properties for the endpoint. func (endpoint *VirtualEndpoint) SetProperties(properties NetworkInfo) { endpoint.EndpointProperties = properties @@ -234,7 +258,7 @@ func networkLogger() *logrus.Entry { func (endpoint *VirtualEndpoint) Attach(h hypervisor) error { networkLogger().WithField("endpoint-type", "virtual").Info("Attaching endpoint") - if err := xconnectVMNetwork(&(endpoint.NetPair), true, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil { + if err := xconnectVMNetwork(endpoint, true, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil { networkLogger().WithError(err).Error("Error bridging virtual endpoint") return err } @@ -254,14 +278,14 @@ func (endpoint *VirtualEndpoint) Detach(netNsCreated bool, netNsPath string) err networkLogger().WithField("endpoint-type", "virtual").Info("Detaching endpoint") return doNetNS(netNsPath, func(_ ns.NetNS) error { - return xconnectVMNetwork(&(endpoint.NetPair), false, 0, false) + return xconnectVMNetwork(endpoint, false, 0, false) }) } // HotAttach for the virtual endpoint uses hot plug device func (endpoint *VirtualEndpoint) HotAttach(h hypervisor) error { networkLogger().Info("Hot attaching virtual endpoint") - if err := xconnectVMNetwork(&(endpoint.NetPair), true, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil { + if err := xconnectVMNetwork(endpoint, true, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil { networkLogger().WithError(err).Error("Error bridging virtual ep") return err } @@ -280,7 +304,7 @@ func (endpoint *VirtualEndpoint) HotDetach(h hypervisor, netNsCreated bool, netN } networkLogger().Info("Hot detaching virtual endpoint") if err := doNetNS(netNsPath, func(_ ns.NetNS) error { - return xconnectVMNetwork(&(endpoint.NetPair), false, 0, h.hypervisorConfig().DisableVhostNet) + return xconnectVMNetwork(endpoint, false, 0, h.hypervisorConfig().DisableVhostNet) }); err != nil { networkLogger().WithError(err).Warn("Error un-bridging virtual ep") } @@ -322,6 +346,11 @@ func (endpoint *VhostUserEndpoint) PciAddr() string { return endpoint.PCIAddr } +// NetworkPair returns the network pair of the endpoint. +func (endpoint *VhostUserEndpoint) NetworkPair() *NetworkInterfacePair { + return nil +} + // Attach for vhostuser endpoint func (endpoint *VhostUserEndpoint) Attach(h hypervisor) error { networkLogger().WithField("endpoint-type", "vhostuser").Info("Attaching endpoint") @@ -401,6 +430,11 @@ func (endpoint *PhysicalEndpoint) SetProperties(properties NetworkInfo) { endpoint.EndpointProperties = properties } +// NetworkPair returns the network pair of the endpoint. +func (endpoint *PhysicalEndpoint) NetworkPair() *NetworkInterfacePair { + return nil +} + // Attach for physical endpoint binds the physical network interface to // vfio-pci and adds device to the hypervisor with vfio-passthrough. func (endpoint *PhysicalEndpoint) Attach(h hypervisor) error { @@ -443,6 +477,157 @@ func (endpoint *PhysicalEndpoint) HotDetach(h hypervisor, netNsCreated bool, net return fmt.Errorf("PhysicalEndpoint does not support Hot detach") } +// Macvlan + +// Properties returns properties of the interface. +func (endpoint *BridgedMacvlanEndpoint) Properties() NetworkInfo { + return endpoint.EndpointProperties +} + +// Name returns name of the veth interface in the network pair. +func (endpoint *BridgedMacvlanEndpoint) Name() string { + return endpoint.NetPair.VirtIface.Name +} + +// HardwareAddr returns the mac address that is assigned to the tap interface +// in th network pair. +func (endpoint *BridgedMacvlanEndpoint) HardwareAddr() string { + return endpoint.NetPair.TAPIface.HardAddr +} + +// Type identifies the endpoint as a virtual endpoint. +func (endpoint *BridgedMacvlanEndpoint) Type() EndpointType { + return endpoint.EndpointType +} + +// SetProperties sets the properties for the endpoint. +func (endpoint *BridgedMacvlanEndpoint) SetProperties(properties NetworkInfo) { + endpoint.EndpointProperties = properties +} + +// PciAddr returns the PCI address of the endpoint. +func (endpoint *BridgedMacvlanEndpoint) PciAddr() string { + return endpoint.PCIAddr +} + +// NetworkPair returns the network pair of the endpoint. +func (endpoint *BridgedMacvlanEndpoint) NetworkPair() *NetworkInterfacePair { + return &endpoint.NetPair +} + +// Attach for virtual endpoint bridges the network pair and adds the +// tap interface of the network pair to the hypervisor. +func (endpoint *BridgedMacvlanEndpoint) Attach(h hypervisor) error { + networkLogger().Info("Attaching macvlan endpoint") + if err := xconnectVMNetwork(endpoint, true, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil { + networkLogger().WithError(err).Error("Error bridging virtual ep") + return err + } + + return h.addDevice(endpoint, netDev) +} + +// Detach for the virtual endpoint tears down the tap and bridge +// created for the veth interface. +func (endpoint *BridgedMacvlanEndpoint) Detach(netNsCreated bool, netNsPath string) error { + // The network namespace would have been deleted at this point + // if it has not been created by virtcontainers. + if !netNsCreated { + return nil + } + + networkLogger().Info("Detaching virtual endpoint") + + return doNetNS(netNsPath, func(_ ns.NetNS) error { + //return xconnectVMNetwork(&(endpoint.NetPair), false, 0, false, endpoint.EndpointType) + return xconnectVMNetwork(endpoint, false, 0, false) + }) +} + +// HotAttach for physical endpoint not supported yet +func (endpoint *BridgedMacvlanEndpoint) HotAttach(h hypervisor) error { + return fmt.Errorf("BridgedMacvlanEndpoint does not support Hot attach") +} + +// HotDetach for physical endpoint not supported yet +func (endpoint *BridgedMacvlanEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { + return fmt.Errorf("BridgedMacvlanEndpoint does not support Hot detach") +} + +//Macvtap + +// Properties returns the properties of the macvtap interface. +func (endpoint *MacvtapEndpoint) Properties() NetworkInfo { + return endpoint.EndpointProperties +} + +// HardwareAddr returns the mac address of the macvtap network interface. +func (endpoint *MacvtapEndpoint) HardwareAddr() string { + return endpoint.EndpointProperties.Iface.HardwareAddr.String() +} + +// Name returns name of the macvtap interface. +func (endpoint *MacvtapEndpoint) Name() string { + return endpoint.EndpointProperties.Iface.Name +} + +// Type indentifies the endpoint as a macvtap endpoint. +func (endpoint *MacvtapEndpoint) Type() EndpointType { + return endpoint.EndpointType +} + +// SetProperties sets the properties of the macvtap endpoint. +func (endpoint *MacvtapEndpoint) SetProperties(properties NetworkInfo) { + endpoint.EndpointProperties = properties +} + +// Attach for macvtap endpoint passes macvtap device to the hypervisor. +func (endpoint *MacvtapEndpoint) Attach(h hypervisor) error { + networkLogger().WithField("endpoint-type", "macvtap").Info("Attaching endpoint") + var err error + + endpoint.VMFds, err = createMacvtapFds(endpoint.EndpointProperties.Iface.Index, int(h.hypervisorConfig().NumVCPUs)) + if err != nil { + return fmt.Errorf("Could not setup macvtap fds %s: %s", endpoint.EndpointProperties.Iface.Name, err) + } + + if !h.hypervisorConfig().DisableVhostNet { + vhostFds, err := createVhostFds(int(h.hypervisorConfig().NumVCPUs)) + if err != nil { + return fmt.Errorf("Could not setup vhost fds %s : %s", endpoint.EndpointProperties.Iface.Name, err) + } + endpoint.VhostFds = vhostFds + } + + return h.addDevice(endpoint, netDev) +} + +// Detach for macvtap endpoint does nothing. +func (endpoint *MacvtapEndpoint) Detach(netNsCreated bool, netNsPath string) error { + networkLogger().WithField("endpoint-type", "macvtap").Info("Detaching endpoint") + return nil +} + +// HotAttach for macvtap endpoint not supported yet +func (endpoint *MacvtapEndpoint) HotAttach(h hypervisor) error { + return fmt.Errorf("MacvtapEndpoint does not support Hot attach") +} + +// HotDetach for macvtap endpoint not supported yet +func (endpoint *MacvtapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { + return fmt.Errorf("MacvtapEndpoint does not support Hot detach") +} + +// PciAddr returns the PCI address of the endpoint. +func (endpoint *MacvtapEndpoint) PciAddr() string { + return endpoint.PCIAddr +} + +// NetworkPair returns the network pair of the endpoint. +func (endpoint *MacvtapEndpoint) NetworkPair() *NetworkInterfacePair { + return nil +} + // EndpointType identifies the type of the network endpoint. type EndpointType string @@ -455,6 +640,12 @@ const ( // VhostUserEndpointType is the vhostuser network interface. VhostUserEndpointType EndpointType = "vhost-user" + + // BridgedMacvlanEndpointType is macvlan network interface. + BridgedMacvlanEndpointType EndpointType = "macvlan" + + // MacvtapEndpointType is macvtap network interface. + MacvtapEndpointType EndpointType = "macvtap" ) // Set sets an endpoint type based on the input string. @@ -469,6 +660,12 @@ func (endpointType *EndpointType) Set(value string) error { case "vhost-user": *endpointType = VhostUserEndpointType return nil + case "macvlan": + *endpointType = BridgedMacvlanEndpointType + return nil + case "macvtap": + *endpointType = MacvtapEndpointType + return nil default: return fmt.Errorf("Unknown endpoint type %s", value) } @@ -483,6 +680,10 @@ func (endpointType *EndpointType) String() string { return string(VirtualEndpointType) case VhostUserEndpointType: return string(VhostUserEndpointType) + case BridgedMacvlanEndpointType: + return string(BridgedMacvlanEndpointType) + case MacvtapEndpointType: + return string(MacvtapEndpointType) default: return "" } @@ -601,6 +802,30 @@ func (n *NetworkNamespace) UnmarshalJSON(b []byte) error { "endpoint-type": "vhostuser", }).Info("endpoint unmarshalled") + case BridgedMacvlanEndpointType: + var endpoint BridgedMacvlanEndpoint + err := json.Unmarshal(e.Data, &endpoint) + if err != nil { + return err + } + + networkLogger().WithFields(logrus.Fields{ + "endpoint": endpoint, + "endpoint-type": "macvlan", + }).Info("endpoint unmarshalled") + + case MacvtapEndpointType: + var endpoint MacvtapEndpoint + err := json.Unmarshal(e.Data, &endpoint) + if err != nil { + return err + } + + networkLogger().WithFields(logrus.Fields{ + "endpoint": endpoint, + "endpoint-type": "macvtap", + }).Info("endpoint unmarshalled") + default: networkLogger().WithField("endpoint-type", e.Type).Error("Ignoring unknown endpoint type") } @@ -709,6 +934,21 @@ func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Lin return newLink, fds, err } +func getLinkForEndpoint(endpoint Endpoint, netHandle *netlink.Handle) (netlink.Link, error) { + var link netlink.Link + + switch ep := endpoint.(type) { + case *VirtualEndpoint: + link = &netlink.Veth{} + case *BridgedMacvlanEndpoint: + link = &netlink.Macvlan{} + default: + return nil, fmt.Errorf("Unexpected endpointType %s", ep.Type()) + } + + return getLinkByName(netHandle, endpoint.NetworkPair().VirtIface.Name, link) +} + func getLinkByName(netHandle *netlink.Handle, name string, expectedLink netlink.Link) (netlink.Link, error) { link, err := netHandle.LinkByName(name) if err != nil { @@ -732,6 +972,10 @@ func getLinkByName(netHandle *netlink.Handle, name string, expectedLink netlink. if l, ok := link.(*netlink.Macvtap); ok { return l, nil } + case (&netlink.Macvlan{}).Type(): + if l, ok := link.(*netlink.Macvlan); ok { + return l, nil + } default: return nil, fmt.Errorf("Unsupported link type %s", expectedLink.Type()) } @@ -740,7 +984,9 @@ func getLinkByName(netHandle *netlink.Handle, name string, expectedLink netlink. } // The endpoint type should dictate how the connection needs to be made -func xconnectVMNetwork(netPair *NetworkInterfacePair, connect bool, numCPUs uint32, disableVhostNet bool) error { +func xconnectVMNetwork(endpoint Endpoint, connect bool, numCPUs uint32, disableVhostNet bool) error { + netPair := endpoint.NetworkPair() + if netPair.NetInterworkingModel == NetXConnectDefaultModel { netPair.NetInterworkingModel = DefaultNetInterworkingModel } @@ -748,15 +994,15 @@ func xconnectVMNetwork(netPair *NetworkInterfacePair, connect bool, numCPUs uint case NetXConnectBridgedModel: netPair.NetInterworkingModel = NetXConnectBridgedModel if connect { - return bridgeNetworkPair(netPair, numCPUs, disableVhostNet) + return bridgeNetworkPair(endpoint, numCPUs, disableVhostNet) } - return unBridgeNetworkPair(*netPair) + return unBridgeNetworkPair(endpoint) case NetXConnectMacVtapModel: netPair.NetInterworkingModel = NetXConnectMacVtapModel if connect { - return tapNetworkPair(netPair, numCPUs, disableVhostNet) + return tapNetworkPair(endpoint, numCPUs, disableVhostNet) } - return untapNetworkPair(*netPair) + return untapNetworkPair(endpoint) case NetXConnectEnlightenedModel: return fmt.Errorf("Unsupported networking model") default: @@ -843,18 +1089,21 @@ func setIPs(link netlink.Link, addrs []netlink.Addr) error { return nil } -func tapNetworkPair(netPair *NetworkInterfacePair, numCPUs uint32, disableVhostNet bool) error { +func tapNetworkPair(endpoint Endpoint, numCPUs uint32, disableVhostNet bool) error { netHandle, err := netlink.NewHandle() if err != nil { return err } defer netHandle.Delete() - vethLink, err := getLinkByName(netHandle, netPair.VirtIface.Name, &netlink.Veth{}) + netPair := endpoint.NetworkPair() + + link, err := getLinkForEndpoint(endpoint, netHandle) if err != nil { - return fmt.Errorf("Could not get veth interface: %s: %s", netPair.VirtIface.Name, err) + return err } - vethLinkAttrs := vethLink.Attrs() + + attrs := link.Attrs() // Attach the macvtap interface to the underlying container // interface. Also picks relevant attributes from the parent @@ -862,8 +1111,8 @@ func tapNetworkPair(netPair *NetworkInterfacePair, numCPUs uint32, disableVhostN &netlink.Macvtap{ Macvlan: netlink.Macvlan{ LinkAttrs: netlink.LinkAttrs{ - TxQLen: vethLinkAttrs.TxQLen, - ParentIndex: vethLinkAttrs.Index, + TxQLen: attrs.TxQLen, + ParentIndex: attrs.Index, }, }, }, int(numCPUs)) @@ -877,18 +1126,18 @@ func tapNetworkPair(netPair *NetworkInterfacePair, numCPUs uint32, disableVhostN // the one inside the VM in order to avoid any firewall issues. The // bridge created by the network plugin on the host actually expects // to see traffic from this MAC address and not another one. - tapHardAddr := vethLinkAttrs.HardwareAddr - netPair.TAPIface.HardAddr = vethLinkAttrs.HardwareAddr.String() + tapHardAddr := attrs.HardwareAddr + netPair.TAPIface.HardAddr = attrs.HardwareAddr.String() - if err := netHandle.LinkSetMTU(tapLink, vethLinkAttrs.MTU); err != nil { - return fmt.Errorf("Could not set TAP MTU %d: %s", vethLinkAttrs.MTU, err) + if err := netHandle.LinkSetMTU(tapLink, attrs.MTU); err != nil { + return fmt.Errorf("Could not set TAP MTU %d: %s", attrs.MTU, err) } hardAddr, err := net.ParseMAC(netPair.VirtIface.HardAddr) if err != nil { return err } - if err := netHandle.LinkSetHardwareAddr(vethLink, hardAddr); err != nil { + if err := netHandle.LinkSetHardwareAddr(link, hardAddr); err != nil { return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) } @@ -903,16 +1152,16 @@ func tapNetworkPair(netPair *NetworkInterfacePair, numCPUs uint32, disableVhostN } // Clear the IP addresses from the veth interface to prevent ARP conflict - netPair.VirtIface.Addrs, err = netlink.AddrList(vethLink, netlink.FAMILY_V4) + netPair.VirtIface.Addrs, err = netlink.AddrList(link, netlink.FAMILY_V4) if err != nil { return fmt.Errorf("Unable to obtain veth IP addresses: %s", err) } - if err := clearIPs(vethLink, netPair.VirtIface.Addrs); err != nil { + if err := clearIPs(link, netPair.VirtIface.Addrs); err != nil { return fmt.Errorf("Unable to clear veth IP addresses: %s", err) } - if err := netHandle.LinkSetUp(vethLink); err != nil { + if err := netHandle.LinkSetUp(link); err != nil { return fmt.Errorf("Could not enable veth %s: %s", netPair.VirtIface.Name, err) } @@ -934,13 +1183,15 @@ func tapNetworkPair(netPair *NetworkInterfacePair, numCPUs uint32, disableVhostN return nil } -func bridgeNetworkPair(netPair *NetworkInterfacePair, numCPUs uint32, disableVhostNet bool) error { +func bridgeNetworkPair(endpoint Endpoint, numCPUs uint32, disableVhostNet bool) error { netHandle, err := netlink.NewHandle() if err != nil { return err } defer netHandle.Delete() + netPair := endpoint.NetworkPair() + tapLink, fds, err := createLink(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}, int(numCPUs)) if err != nil { return fmt.Errorf("Could not create TAP interface: %s", err) @@ -955,29 +1206,32 @@ func bridgeNetworkPair(netPair *NetworkInterfacePair, numCPUs uint32, disableVho netPair.VhostFds = vhostFds } - vethLink, err := getLinkByName(netHandle, netPair.VirtIface.Name, &netlink.Veth{}) + var attrs *netlink.LinkAttrs + var link netlink.Link + + link, err = getLinkForEndpoint(endpoint, netHandle) if err != nil { - return fmt.Errorf("Could not get veth interface %s : %s", netPair.VirtIface.Name, err) + return err } - vethLinkAttrs := vethLink.Attrs() + attrs = link.Attrs() // Save the veth MAC address to the TAP so that it can later be used // to build the hypervisor command line. This MAC address has to be // the one inside the VM in order to avoid any firewall issues. The // bridge created by the network plugin on the host actually expects // to see traffic from this MAC address and not another one. - netPair.TAPIface.HardAddr = vethLinkAttrs.HardwareAddr.String() + netPair.TAPIface.HardAddr = attrs.HardwareAddr.String() - if err := netHandle.LinkSetMTU(tapLink, vethLinkAttrs.MTU); err != nil { - return fmt.Errorf("Could not set TAP MTU %d: %s", vethLinkAttrs.MTU, err) + if err := netHandle.LinkSetMTU(tapLink, attrs.MTU); err != nil { + return fmt.Errorf("Could not set TAP MTU %d: %s", attrs.MTU, err) } hardAddr, err := net.ParseMAC(netPair.VirtIface.HardAddr) if err != nil { return err } - if err := netHandle.LinkSetHardwareAddr(vethLink, hardAddr); err != nil { + if err := netHandle.LinkSetHardwareAddr(link, hardAddr); err != nil { return fmt.Errorf("Could not set MAC address %s for veth interface %s: %s", netPair.VirtIface.HardAddr, netPair.VirtIface.Name, err) } @@ -997,12 +1251,12 @@ func bridgeNetworkPair(netPair *NetworkInterfacePair, numCPUs uint32, disableVho return fmt.Errorf("Could not enable TAP %s: %s", netPair.TAPIface.Name, err) } - if err := netHandle.LinkSetMaster(vethLink, bridgeLink.(*netlink.Bridge)); err != nil { + if err := netHandle.LinkSetMaster(link, bridgeLink.(*netlink.Bridge)); err != nil { return fmt.Errorf("Could not attach veth %s to the bridge %s: %s", netPair.VirtIface.Name, netPair.Name, err) } - if err := netHandle.LinkSetUp(vethLink); err != nil { + if err := netHandle.LinkSetUp(link); err != nil { return fmt.Errorf("Could not enable veth %s: %s", netPair.VirtIface.Name, err) } @@ -1013,13 +1267,15 @@ func bridgeNetworkPair(netPair *NetworkInterfacePair, numCPUs uint32, disableVho return nil } -func untapNetworkPair(netPair NetworkInterfacePair) error { +func untapNetworkPair(endpoint Endpoint) error { netHandle, err := netlink.NewHandle() if err != nil { return err } defer netHandle.Delete() + netPair := endpoint.NetworkPair() + tapLink, err := getLinkByName(netHandle, netPair.TAPIface.Name, &netlink.Macvtap{}) if err != nil { return fmt.Errorf("Could not get TAP interface %s: %s", netPair.TAPIface.Name, err) @@ -1029,28 +1285,29 @@ func untapNetworkPair(netPair NetworkInterfacePair) error { return fmt.Errorf("Could not remove TAP %s: %s", netPair.TAPIface.Name, err) } - vethLink, err := getLinkByName(netHandle, netPair.VirtIface.Name, &netlink.Veth{}) + link, err := getLinkForEndpoint(endpoint, netHandle) if err != nil { - // The veth pair is not totally managed by virtcontainers - networkLogger().WithError(err).WithField("veth-name", netPair.VirtIface.Name).Warn("Could not get veth interface") - } else { - if err := netHandle.LinkSetDown(vethLink); err != nil { - return fmt.Errorf("Could not disable veth %s: %s", netPair.VirtIface.Name, err) - } + return err + } + + if err := netHandle.LinkSetDown(link); err != nil { + return fmt.Errorf("Could not disable veth %s: %s", netPair.VirtIface.Name, err) } // Restore the IPs that were cleared - err = setIPs(vethLink, netPair.VirtIface.Addrs) + err = setIPs(link, netPair.VirtIface.Addrs) return err } -func unBridgeNetworkPair(netPair NetworkInterfacePair) error { +func unBridgeNetworkPair(endpoint Endpoint) error { netHandle, err := netlink.NewHandle() if err != nil { return err } defer netHandle.Delete() + netPair := endpoint.NetworkPair() + tapLink, err := getLinkByName(netHandle, netPair.TAPIface.Name, &netlink.Tuntap{}) if err != nil { return fmt.Errorf("Could not get TAP interface: %s", err) @@ -1081,19 +1338,17 @@ func unBridgeNetworkPair(netPair NetworkInterfacePair) error { return fmt.Errorf("Could not remove TAP %s: %s", netPair.TAPIface.Name, err) } - vethLink, err := getLinkByName(netHandle, netPair.VirtIface.Name, &netlink.Veth{}) + link, err := getLinkForEndpoint(endpoint, netHandle) if err != nil { - // The veth pair is not totally managed by virtcontainers - networkLogger().WithError(err).Warn("Could not get veth interface") - } else { - if err := netHandle.LinkSetDown(vethLink); err != nil { - return fmt.Errorf("Could not disable veth %s: %s", netPair.VirtIface.Name, err) - } + return err + } - if err := netHandle.LinkSetNoMaster(vethLink); err != nil { - return fmt.Errorf("Could not detach veth %s: %s", netPair.VirtIface.Name, err) - } + if err := netHandle.LinkSetDown(link); err != nil { + return fmt.Errorf("Could not disable veth %s: %s", netPair.VirtIface.Name, err) + } + if err := netHandle.LinkSetNoMaster(link); err != nil { + return fmt.Errorf("Could not detach veth %s: %s", netPair.VirtIface.Name, err) } return nil @@ -1160,29 +1415,48 @@ func createVirtualNetworkEndpoint(idx int, ifName string, interworkingModel NetI return &VirtualEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx) } - uniqueID := uuid.Generate().String() - - hardAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, byte(idx >> 8), byte(idx)} + netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel) + if err != nil { + return nil, err + } endpoint := &VirtualEndpoint{ // TODO This is too specific. We may need to create multiple // end point types here and then decide how to connect them // at the time of hypervisor attach and not here - NetPair: NetworkInterfacePair{ - ID: uniqueID, - Name: fmt.Sprintf("br%d_kata", idx), - VirtIface: NetworkInterface{ - Name: fmt.Sprintf("eth%d", idx), - HardAddr: hardAddr.String(), - }, - TAPIface: NetworkInterface{ - Name: fmt.Sprintf("tap%d_kata", idx), - }, - NetInterworkingModel: interworkingModel, - }, + NetPair: netPair, EndpointType: VirtualEndpointType, } + if ifName != "" { + endpoint.NetPair.VirtIface.Name = ifName + } + return endpoint, nil +} + +func createMacvtapNetworkEndpoint(netInfo NetworkInfo) (*MacvtapEndpoint, error) { + endpoint := &MacvtapEndpoint{ + EndpointType: MacvtapEndpointType, + EndpointProperties: netInfo, + } + + return endpoint, nil +} + +func createBridgedMacvlanNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel) (*BridgedMacvlanEndpoint, error) { + if idx < 0 { + return &BridgedMacvlanEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx) + } + + netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel) + if err != nil { + return nil, err + } + + endpoint := &BridgedMacvlanEndpoint{ + NetPair: netPair, + EndpointType: BridgedMacvlanEndpointType, + } if ifName != "" { endpoint.NetPair.VirtIface.Name = ifName } @@ -1277,6 +1551,46 @@ func generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*grpc.Interface, return ifaces, routes, nil } +func createNetworkInterfacePair(idx int, ifName string, interworkingModel NetInterworkingModel) (NetworkInterfacePair, error) { + uniqueID := uuid.Generate().String() + + randomMacAddr, err := generateRandomPrivateMacAddr() + if err != nil { + return NetworkInterfacePair{}, fmt.Errorf("Could not generate random mac address: %s", err) + } + + netPair := NetworkInterfacePair{ + ID: uniqueID, + Name: fmt.Sprintf("br%d_kata", idx), + VirtIface: NetworkInterface{ + Name: fmt.Sprintf("eth%d", idx), + HardAddr: randomMacAddr, + }, + TAPIface: NetworkInterface{ + Name: fmt.Sprintf("tap%d_kata", idx), + }, + NetInterworkingModel: interworkingModel, + } + + return netPair, nil +} + +func generateRandomPrivateMacAddr() (string, error) { + buf := make([]byte, 6) + _, err := cryptoRand.Read(buf) + if err != nil { + return "", err + } + + // Set the local bit for local addresses + // Addresses in this range are local mac addresses: + // x2-xx-xx-xx-xx-xx , x6-xx-xx-xx-xx-xx , xA-xx-xx-xx-xx-xx , xE-xx-xx-xx-xx-xx + buf[0] = (buf[0] | 2) & 0xfe + + hardAddr := net.HardwareAddr(buf) + return hardAddr.String(), nil +} + func networkInfoFromLink(handle *netlink.Handle, link netlink.Link) (NetworkInfo, error) { addrs, err := handle.AddrList(link, netlink.FAMILY_ALL) if err != nil { @@ -1388,6 +1702,12 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel) (E if socketPath != "" { networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found") endpoint, err = createVhostUserEndpoint(netInfo, socketPath) + } else if netInfo.Iface.Type == "macvlan" { + networkLogger().Infof("macvlan interface found") + endpoint, err = createBridgedMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, model) + } else if netInfo.Iface.Type == "macvtap" { + networkLogger().Infof("macvtap interface found") + endpoint, err = createMacvtapNetworkEndpoint(netInfo) } else { endpoint, err = createVirtualNetworkEndpoint(idx, netInfo.Iface.Name, model) } diff --git a/virtcontainers/network_test.go b/virtcontainers/network_test.go index 6a82e8a591..a0f276b3c5 100644 --- a/virtcontainers/network_test.go +++ b/virtcontainers/network_test.go @@ -140,6 +140,18 @@ func TestVirtualEndpointTypeSet(t *testing.T) { testEndpointTypeSet(t, "virtual", VirtualEndpointType) } +func TestVhostUserEndpointTypeSet(t *testing.T) { + testEndpointTypeSet(t, "vhost-user", VhostUserEndpointType) +} + +func TestBridgedMacvlanEndpointTypeSet(t *testing.T) { + testEndpointTypeSet(t, "macvlan", BridgedMacvlanEndpointType) +} + +func TestMacvtapEndpointTypeSet(t *testing.T) { + testEndpointTypeSet(t, "macvtap", MacvtapEndpointType) +} + func TestEndpointTypeSetFailure(t *testing.T) { var endpointType EndpointType @@ -167,6 +179,21 @@ func TestVirtualEndpointTypeString(t *testing.T) { testEndpointTypeString(t, &endpointType, string(VirtualEndpointType)) } +func TestVhostUserEndpointTypeString(t *testing.T) { + endpointType := VhostUserEndpointType + testEndpointTypeString(t, &endpointType, string(VhostUserEndpointType)) +} + +func TestBridgedMacvlanEndpointTypeString(t *testing.T) { + endpointType := BridgedMacvlanEndpointType + testEndpointTypeString(t, &endpointType, string(BridgedMacvlanEndpointType)) +} + +func TestMacvtapEndpointTypeString(t *testing.T) { + endpointType := MacvtapEndpointType + testEndpointTypeString(t, &endpointType, string(MacvtapEndpointType)) +} + func TestIncorrectEndpointTypeString(t *testing.T) { var endpointType EndpointType testEndpointTypeString(t, &endpointType, "") @@ -230,8 +257,11 @@ func TestCreateVirtualNetworkEndpoint(t *testing.T) { // the resulting ID will be random - so let's overwrite to test the rest of the flow result.NetPair.ID = "uniqueTestID-4" + // the resulting mac address will be random - so lets overwrite it + result.NetPair.VirtIface.HardAddr = macAddr.String() + if reflect.DeepEqual(result, expected) == false { - t.Fatal() + t.Fatalf("\nGot: %+v, \n\nExpected: %+v", result, expected) } } @@ -262,8 +292,11 @@ func TestCreateVirtualNetworkEndpointChooseIfaceName(t *testing.T) { // the resulting ID will be random - so let's overwrite to test the rest of the flow result.NetPair.ID = "uniqueTestID-4" + // the resulting mac address will be random - so lets overwrite it + result.NetPair.VirtIface.HardAddr = macAddr.String() + if reflect.DeepEqual(result, expected) == false { - t.Fatal() + t.Fatalf("\nGot: %+v, \n\nExpected: %+v", result, expected) } } @@ -287,6 +320,62 @@ func TestCreateVirtualNetworkEndpointInvalidArgs(t *testing.T) { } } +func TestCreateBridgedMacvlanEndpoint(t *testing.T) { + macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x04} + + expected := &BridgedMacvlanEndpoint{ + NetPair: NetworkInterfacePair{ + ID: "uniqueTestID-4", + Name: "br4_kata", + VirtIface: NetworkInterface{ + Name: "eth4", + HardAddr: macAddr.String(), + }, + TAPIface: NetworkInterface{ + Name: "tap4_kata", + }, + NetInterworkingModel: DefaultNetInterworkingModel, + }, + EndpointType: BridgedMacvlanEndpointType, + } + + result, err := createBridgedMacvlanNetworkEndpoint(4, "", DefaultNetInterworkingModel) + if err != nil { + t.Fatal(err) + } + + // the resulting ID will be random - so let's overwrite to test the rest of the flow + result.NetPair.ID = "uniqueTestID-4" + + // the resulting mac address will be random - so lets overwrite it + result.NetPair.VirtIface.HardAddr = macAddr.String() + + if reflect.DeepEqual(result, expected) == false { + t.Fatalf("\nGot: %+v, \n\nExpected: %+v", result, expected) + } +} + +func TestCreateMacvtapEndpoint(t *testing.T) { + netInfo := NetworkInfo{ + Iface: NetlinkIface{ + Type: "macvtap", + }, + } + expected := &MacvtapEndpoint{ + EndpointType: MacvtapEndpointType, + EndpointProperties: netInfo, + } + + result, err := createMacvtapNetworkEndpoint(netInfo) + if err != nil { + t.Fatal(err) + } + + if reflect.DeepEqual(result, expected) == false { + t.Fatalf("\nGot: %+v, \n\nExpected: %+v", result, expected) + } +} + func TestIsPhysicalIface(t *testing.T) { if os.Geteuid() != 0 { t.Skip(testDisabledAsNonRoot) @@ -599,3 +688,114 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) { "Routes returned didn't match: got %+v, expecting %+v", resRoutes, expectedRoutes) } + +func TestGenerateRandomPrivateMacAdd(t *testing.T) { + assert := assert.New(t) + + addr1, err := generateRandomPrivateMacAddr() + assert.NoError(err) + + _, err = net.ParseMAC(addr1) + assert.NoError(err) + + addr2, err := generateRandomPrivateMacAddr() + assert.NoError(err) + + _, err = net.ParseMAC(addr2) + assert.NoError(err) + + assert.NotEqual(addr1, addr2) +} + +func TestCreateGetBridgeLink(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip(testDisabledAsNonRoot) + } + + assert := assert.New(t) + + netHandle, err := netlink.NewHandle() + defer netHandle.Delete() + + assert.NoError(err) + + brName := "testbr0" + brLink, _, err := createLink(netHandle, brName, &netlink.Bridge{}, 1) + assert.NoError(err) + assert.NotNil(brLink) + + brLink, err = getLinkByName(netHandle, brName, &netlink.Bridge{}) + assert.NoError(err) + + err = netHandle.LinkDel(brLink) + assert.NoError(err) +} + +func TestCreateGetTunTapLink(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip(testDisabledAsNonRoot) + } + + assert := assert.New(t) + + netHandle, err := netlink.NewHandle() + defer netHandle.Delete() + + assert.NoError(err) + + tapName := "testtap0" + tapLink, fds, err := createLink(netHandle, tapName, &netlink.Tuntap{}, 1) + assert.NoError(err) + assert.NotNil(tapLink) + assert.NotZero(len(fds)) + + tapLink, err = getLinkByName(netHandle, tapName, &netlink.Tuntap{}) + assert.NoError(err) + + err = netHandle.LinkDel(tapLink) + assert.NoError(err) +} + +func TestCreateMacVtap(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip(testDisabledAsNonRoot) + } + + assert := assert.New(t) + + netHandle, err := netlink.NewHandle() + defer netHandle.Delete() + + assert.NoError(err) + + brName := "testbr0" + brLink, _, err := createLink(netHandle, brName, &netlink.Bridge{}, 1) + assert.NoError(err) + + attrs := brLink.Attrs() + + mcLink := &netlink.Macvtap{ + Macvlan: netlink.Macvlan{ + LinkAttrs: netlink.LinkAttrs{ + TxQLen: attrs.TxQLen, + ParentIndex: attrs.Index, + }, + }, + } + + macvtapName := "testmc0" + _, err = createMacVtap(netHandle, macvtapName, mcLink, 1) + assert.NoError(err) + + macvtapLink, err := getLinkByName(netHandle, macvtapName, &netlink.Macvtap{}) + assert.NoError(err) + + err = netHandle.LinkDel(macvtapLink) + assert.NoError(err) + + brLink, err = getLinkByName(netHandle, brName, &netlink.Bridge{}) + assert.NoError(err) + + err = netHandle.LinkDel(brLink) + assert.NoError(err) +} diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go index d0a2cc18da..f0d3cb7146 100644 --- a/virtcontainers/qemu_arch_base.go +++ b/virtcontainers/qemu_arch_base.go @@ -441,23 +441,42 @@ func networkModelToQemuType(model NetInterworkingModel) govmmQemu.NetDeviceType func (q *qemuArchBase) appendNetwork(devices []govmmQemu.Device, endpoint Endpoint) []govmmQemu.Device { switch ep := endpoint.(type) { - case *VirtualEndpoint: + case *VirtualEndpoint, *BridgedMacvlanEndpoint: + netPair := ep.NetworkPair() devices = append(devices, govmmQemu.NetDevice{ - Type: networkModelToQemuType(ep.NetPair.NetInterworkingModel), + Type: networkModelToQemuType(netPair.NetInterworkingModel), Driver: govmmQemu.VirtioNetPCI, ID: fmt.Sprintf("network-%d", q.networkIndex), - IFName: ep.NetPair.TAPIface.Name, - MACAddress: ep.NetPair.TAPIface.HardAddr, + IFName: netPair.TAPIface.Name, + MACAddress: netPair.TAPIface.HardAddr, DownScript: "no", Script: "no", VHost: q.vhost, DisableModern: q.nestedRun, - FDs: ep.NetPair.VMFds, - VhostFDs: ep.NetPair.VhostFds, + FDs: netPair.VMFds, + VhostFDs: netPair.VhostFds, }, ) q.networkIndex++ + case *MacvtapEndpoint: + devices = append(devices, + govmmQemu.NetDevice{ + Type: govmmQemu.MACVTAP, + Driver: govmmQemu.VirtioNetPCI, + ID: fmt.Sprintf("network-%d", q.networkIndex), + IFName: ep.Name(), + MACAddress: ep.HardwareAddr(), + DownScript: "no", + Script: "no", + VHost: q.vhost, + DisableModern: q.nestedRun, + FDs: ep.VMFds, + VhostFDs: ep.VhostFds, + }, + ) + q.networkIndex++ + } return devices diff --git a/virtcontainers/qemu_arch_base_test.go b/virtcontainers/qemu_arch_base_test.go index 7da056647b..02784c8bac 100644 --- a/virtcontainers/qemu_arch_base_test.go +++ b/virtcontainers/qemu_arch_base_test.go @@ -8,6 +8,7 @@ package virtcontainers import ( "fmt" "io/ioutil" + "net" "path/filepath" "testing" @@ -432,3 +433,65 @@ func TestQemuArchBaseAppendSCSIController(t *testing.T) { _, ioThread = qemuArchBase.appendSCSIController(devices, true) assert.NotNil(ioThread) } + +func TestQemuArchBaseAppendNetwork(t *testing.T) { + var devices []govmmQemu.Device + assert := assert.New(t) + qemuArchBase := newQemuArchBase() + + macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x04} + + macvlanEp := &BridgedMacvlanEndpoint{ + NetPair: NetworkInterfacePair{ + ID: "uniqueTestID-4", + Name: "br4_kata", + VirtIface: NetworkInterface{ + Name: "eth4", + HardAddr: macAddr.String(), + }, + TAPIface: NetworkInterface{ + Name: "tap4_kata", + }, + NetInterworkingModel: DefaultNetInterworkingModel, + }, + EndpointType: BridgedMacvlanEndpointType, + } + + macvtapEp := &MacvtapEndpoint{ + EndpointType: MacvtapEndpointType, + EndpointProperties: NetworkInfo{ + Iface: NetlinkIface{ + Type: "macvtap", + }, + }, + } + + expectedOut := []govmmQemu.Device{ + govmmQemu.NetDevice{ + Type: networkModelToQemuType(macvlanEp.NetPair.NetInterworkingModel), + Driver: govmmQemu.VirtioNetPCI, + ID: fmt.Sprintf("network-%d", 0), + IFName: macvlanEp.NetPair.TAPIface.Name, + MACAddress: macvlanEp.NetPair.TAPIface.HardAddr, + DownScript: "no", + Script: "no", + FDs: macvlanEp.NetPair.VMFds, + VhostFDs: macvlanEp.NetPair.VhostFds, + }, + govmmQemu.NetDevice{ + Type: govmmQemu.MACVTAP, + Driver: govmmQemu.VirtioNetPCI, + ID: fmt.Sprintf("network-%d", 1), + IFName: macvtapEp.Name(), + MACAddress: macvtapEp.HardwareAddr(), + DownScript: "no", + Script: "no", + FDs: macvtapEp.VMFds, + VhostFDs: macvtapEp.VhostFds, + }, + } + + devices = qemuArchBase.appendNetwork(devices, macvlanEp) + devices = qemuArchBase.appendNetwork(devices, macvtapEp) + assert.Equal(expectedOut, devices) +}