diff --git a/virtcontainers/endpoint.go b/virtcontainers/endpoint.go index ff39e6861d..01b5e77fb2 100644 --- a/virtcontainers/endpoint.go +++ b/virtcontainers/endpoint.go @@ -53,6 +53,9 @@ const ( // TapEndpointType is tap network interface. TapEndpointType EndpointType = "tap" + // TuntapEndpointType is a tap network interface. + TuntapEndpointType EndpointType = "tuntap" + // IPVlanEndpointType is ipvlan network interface. IPVlanEndpointType EndpointType = "ipvlan" ) @@ -78,6 +81,9 @@ func (endpointType *EndpointType) Set(value string) error { case "tap": *endpointType = TapEndpointType return nil + case "tuntap": + *endpointType = TuntapEndpointType + return nil case "ipvlan": *endpointType = IPVlanEndpointType return nil @@ -101,6 +107,8 @@ func (endpointType *EndpointType) String() string { return string(MacvtapEndpointType) case TapEndpointType: return string(TapEndpointType) + case TuntapEndpointType: + return string(TuntapEndpointType) case IPVlanEndpointType: return string(IPVlanEndpointType) default: @@ -183,3 +191,33 @@ func loadNetIfPair(pair *persistapi.NetworkInterfacePair) *NetworkInterfacePair NetInterworkingModel: NetInterworkingModel(pair.NetInterworkingModel), } } + +func saveTuntapIf(tuntapif *TuntapInterface) *persistapi.TuntapInterface { + if tuntapif == nil { + return nil + } + + return &persistapi.TuntapInterface{ + Name: tuntapif.Name, + TAPIface: persistapi.NetworkInterface{ + Name: tuntapif.TAPIface.Name, + HardAddr: tuntapif.TAPIface.HardAddr, + Addrs: tuntapif.TAPIface.Addrs, + }, + } +} + +func loadTuntapIf(tuntapif *persistapi.TuntapInterface) *TuntapInterface { + if tuntapif == nil { + return nil + } + + return &TuntapInterface{ + Name: tuntapif.Name, + TAPIface: NetworkInterface{ + Name: tuntapif.TAPIface.Name, + HardAddr: tuntapif.TAPIface.HardAddr, + Addrs: tuntapif.TAPIface.Addrs, + }, + } +} diff --git a/virtcontainers/network.go b/virtcontainers/network.go index 36a520694c..bf53537577 100644 --- a/virtcontainers/network.go +++ b/virtcontainers/network.go @@ -159,6 +159,12 @@ type TapInterface struct { VhostFds []*os.File } +// TuntapInterface defines a tap interface +type TuntapInterface struct { + Name string + TAPIface NetworkInterface +} + // NetworkInterfacePair defines a pair between VM and virtual network interfaces. type NetworkInterfacePair struct { TapInterface @@ -262,6 +268,10 @@ func generateEndpoints(typedEndpoints []TypedJSONEndpoint) ([]Endpoint, error) { var endpoint IPVlanEndpoint endpointInf = &endpoint + case TuntapEndpointType: + var endpoint TuntapEndpoint + endpointInf = &endpoint + default: networkLogger().WithField("endpoint-type", e.Type).Error("Ignoring unknown endpoint type") } @@ -373,6 +383,8 @@ func getLinkForEndpoint(endpoint Endpoint, netHandle *netlink.Handle) (netlink.L link = &netlink.Macvlan{} case *IPVlanEndpoint: link = &netlink.IPVlan{} + case *TuntapEndpoint: + link = &netlink.Tuntap{} default: return nil, fmt.Errorf("Unexpected endpointType %s", ep.Type()) } @@ -392,7 +404,7 @@ func getLinkByName(netHandle *netlink.Handle, name string, expectedLink netlink. return l, nil } case (&netlink.Tuntap{}).Type(): - if l, ok := link.(*netlink.GenericLink); ok { + if l, ok := link.(*netlink.Tuntap); ok { return l, nil } case (&netlink.Veth{}).Type(): @@ -1237,6 +1249,10 @@ func createNetworkInterfacePair(idx int, ifName string, interworkingModel NetInt NetInterworkingModel: interworkingModel, } + if ifName != "" { + netPair.VirtIface.Name = ifName + } + return netPair, nil } @@ -1323,7 +1339,7 @@ func createEndpointsFromScan(networkNSPath string, config *NetworkConfig) ([]End } if err := doNetNS(networkNSPath, func(_ ns.NetNS) error { - endpoint, errCreate = createEndpoint(netInfo, idx, config.InterworkingModel) + endpoint, errCreate = createEndpoint(netInfo, idx, config.InterworkingModel, link) return errCreate }); err != nil { return []Endpoint{}, err @@ -1344,7 +1360,7 @@ func createEndpointsFromScan(networkNSPath string, config *NetworkConfig) ([]End return endpoints, nil } -func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel) (Endpoint, error) { +func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, link netlink.Link) (Endpoint, error) { var endpoint Endpoint // TODO: This is the incoming interface // based on the incoming interface we should create @@ -1382,12 +1398,27 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel) (E } else if netInfo.Iface.Type == "tap" { networkLogger().Info("tap interface found") endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name) + } else if netInfo.Iface.Type == "tuntap" { + if link != nil { + switch link.(*netlink.Tuntap).Mode { + case 0: + // mount /sys/class/net to get links + return nil, fmt.Errorf("Network device mode not determined correctly. Mount sysfs in caller") + case 1: + return nil, fmt.Errorf("tun networking device not yet supported") + case 2: + networkLogger().Info("tuntap tap interface found") + endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model) + default: + return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode) + } + } } else if netInfo.Iface.Type == "veth" { endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model) } else if netInfo.Iface.Type == "ipvlan" { endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name) } else { - return nil, fmt.Errorf("Unsupported network interface") + return nil, fmt.Errorf("Unsupported network interface: %s", netInfo.Iface.Type) } } diff --git a/virtcontainers/persist/api/network.go b/virtcontainers/persist/api/network.go index 2efebf978b..69610c6706 100644 --- a/virtcontainers/persist/api/network.go +++ b/virtcontainers/persist/api/network.go @@ -26,6 +26,12 @@ type TapInterface struct { // remove VMFds and VhostFds } +// TuntapInterface defines a tap interface +type TuntapInterface struct { + Name string + TAPIface NetworkInterface +} + // NetworkInterfacePair defines a pair between VM and virtual network interfaces. type NetworkInterfacePair struct { TapInterface @@ -49,6 +55,10 @@ type TapEndpoint struct { TapInterface TapInterface } +type TuntapEndpoint struct { + TuntapInterface TuntapInterface +} + type BridgedMacvlanEndpoint struct { NetPair NetworkInterfacePair } @@ -80,6 +90,7 @@ type NetworkEndpoint struct { Macvtap *MacvtapEndpoint `json:",omitempty"` Tap *TapEndpoint `json:",omitempty"` IPVlan *IPVlanEndpoint `json:",omitempty"` + Tuntap *TuntapEndpoint `json:",omitempty"` } // NetworkInfo contains network information of sandbox diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go index 8889593f04..9d6d68cf33 100644 --- a/virtcontainers/qemu_arch_base.go +++ b/virtcontainers/qemu_arch_base.go @@ -514,6 +514,21 @@ func genericNetwork(endpoint Endpoint, vhost, nestedRun bool, index int) (govmmQ FDs: ep.VMFds, VhostFDs: ep.VhostFds, } + case *TuntapEndpoint: + netPair := ep.NetworkPair() + d = govmmQemu.NetDevice{ + Type: govmmQemu.NetDeviceType("tap"), + Driver: govmmQemu.VirtioNet, + ID: fmt.Sprintf("network-%d", index), + IFName: netPair.TAPIface.Name, + MACAddress: netPair.TAPIface.HardAddr, + DownScript: "no", + Script: "no", + VHost: vhost, + DisableModern: nestedRun, + FDs: netPair.VMFds, + VhostFDs: netPair.VhostFds, + } default: return govmmQemu.NetDevice{}, fmt.Errorf("Unknown type for endpoint") } diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 214e3cb555..0d88e90115 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -914,7 +914,7 @@ func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, erro return nil, err } - endpoint, err := createEndpoint(netInfo, len(s.networkNS.Endpoints), s.config.NetworkConfig.InterworkingModel) + endpoint, err := createEndpoint(netInfo, len(s.networkNS.Endpoints), s.config.NetworkConfig.InterworkingModel, nil) if err != nil { return nil, err } diff --git a/virtcontainers/tuntap_endpoint.go b/virtcontainers/tuntap_endpoint.go new file mode 100644 index 0000000000..b076d69494 --- /dev/null +++ b/virtcontainers/tuntap_endpoint.go @@ -0,0 +1,214 @@ +// Copyright (c) 2018 Huawei Corporation +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "fmt" + "net" + + "github.com/containernetworking/plugins/pkg/ns" + "github.com/vishvananda/netlink" + + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" +) + +// TuntapEndpoint represents just a tap endpoint +type TuntapEndpoint struct { + NetPair NetworkInterfacePair + TuntapInterface TuntapInterface + EndpointProperties NetworkInfo + EndpointType EndpointType + PCIAddr string +} + +// Properties returns the properties of the tap interface. +func (endpoint *TuntapEndpoint) Properties() NetworkInfo { + return endpoint.EndpointProperties +} + +// Name returns name of the tap interface in the network pair. +func (endpoint *TuntapEndpoint) Name() string { + return endpoint.TuntapInterface.Name +} + +// HardwareAddr returns the mac address that is assigned to the tap interface +func (endpoint *TuntapEndpoint) HardwareAddr() string { + return endpoint.TuntapInterface.TAPIface.HardAddr +} + +// Type identifies the endpoint as a tap endpoint. +func (endpoint *TuntapEndpoint) Type() EndpointType { + return endpoint.EndpointType +} + +// PciAddr returns the PCI address of the endpoint. +func (endpoint *TuntapEndpoint) PciAddr() string { + return endpoint.PCIAddr +} + +// SetPciAddr sets the PCI address of the endpoint. +func (endpoint *TuntapEndpoint) SetPciAddr(pciAddr string) { + endpoint.PCIAddr = pciAddr +} + +// NetworkPair returns the network pair of the endpoint. +func (endpoint *TuntapEndpoint) NetworkPair() *NetworkInterfacePair { + return &endpoint.NetPair +} + +// SetProperties sets the properties for the endpoint. +func (endpoint *TuntapEndpoint) SetProperties(properties NetworkInfo) { + endpoint.EndpointProperties = properties +} + +// Attach for tap endpoint adds the tap interface to the hypervisor. +func (endpoint *TuntapEndpoint) Attach(h hypervisor) error { + if err := xConnectVMNetwork(endpoint, h); err != nil { + networkLogger().WithError(err).Error("Error bridging virtual endpoint") + return err + } + return h.addDevice(endpoint, netDev) +} + +// Detach for the tap endpoint tears down the tap +func (endpoint *TuntapEndpoint) Detach(netNsCreated bool, netNsPath string) error { + if !netNsCreated && netNsPath != "" { + return nil + } + + networkLogger().WithField("endpoint-type", TuntapEndpointType).Info("Detaching endpoint") + return doNetNS(netNsPath, func(_ ns.NetNS) error { + return unTuntapNetwork(endpoint.TuntapInterface.TAPIface.Name) + }) +} + +// HotAttach for the tap endpoint uses hot plug device +func (endpoint *TuntapEndpoint) HotAttach(h hypervisor) error { + networkLogger().Info("Hot attaching tap endpoint") + if err := tuntapNetwork(endpoint, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil { + networkLogger().WithError(err).Error("Error bridging tap ep") + return err + } + + if _, err := h.hotplugAddDevice(endpoint, netDev); err != nil { + networkLogger().WithError(err).Error("Error attach tap ep") + return err + } + return nil +} + +// HotDetach for the tap endpoint uses hot pull device +func (endpoint *TuntapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { + networkLogger().Info("Hot detaching tap endpoint") + if err := doNetNS(netNsPath, func(_ ns.NetNS) error { + return unTuntapNetwork(endpoint.TuntapInterface.TAPIface.Name) + }); err != nil { + networkLogger().WithError(err).Warn("Error un-bridging tap ep") + } + + if _, err := h.hotplugRemoveDevice(endpoint, netDev); err != nil { + networkLogger().WithError(err).Error("Error detach tap ep") + return err + } + return nil +} + +func createTuntapNetworkEndpoint(idx int, ifName string, hwName net.HardwareAddr, internetworkingModel NetInterworkingModel) (*TuntapEndpoint, error) { + if idx < 0 { + return &TuntapEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx) + } + + netPair, err := createNetworkInterfacePair(idx, ifName, internetworkingModel) + if err != nil { + return nil, err + } + + endpoint := &TuntapEndpoint{ + NetPair: netPair, + TuntapInterface: TuntapInterface{ + Name: fmt.Sprintf("eth%d", idx), + TAPIface: NetworkInterface{ + Name: fmt.Sprintf("tap%d_kata", idx), + HardAddr: fmt.Sprintf("%s", hwName), //nolint:gosimple + }, + }, + EndpointType: TuntapEndpointType, + } + + if ifName != "" { + endpoint.TuntapInterface.Name = ifName + } + + return endpoint, nil +} + +func tuntapNetwork(endpoint *TuntapEndpoint, numCPUs uint32, disableVhostNet bool) error { + netHandle, err := netlink.NewHandle() + if err != nil { + return err + } + defer netHandle.Delete() + + tapLink, _, err := createLink(netHandle, endpoint.TuntapInterface.TAPIface.Name, &netlink.Tuntap{}, int(numCPUs)) + if err != nil { + return fmt.Errorf("Could not create TAP interface: %s", err) + } + linkAttrs := endpoint.Properties().Iface.LinkAttrs + + // Save the MAC address to the TAP so that it can later be used + // to build the QMP command line. This MAC address has to be + // the one inside the VM in order to avoid any firewall issues. The + // bridge created by the network plugin on the host actually expects + // to see traffic from this MAC address and not another one. + endpoint.TuntapInterface.TAPIface.HardAddr = linkAttrs.HardwareAddr.String() + if err := netHandle.LinkSetMTU(tapLink, linkAttrs.MTU); err != nil { + return fmt.Errorf("Could not set TAP MTU %d: %s", linkAttrs.MTU, err) + } + if err := netHandle.LinkSetUp(tapLink); err != nil { + return fmt.Errorf("Could not enable TAP %s: %s", endpoint.TuntapInterface.Name, err) + } + return nil +} + +func unTuntapNetwork(name string) error { + netHandle, err := netlink.NewHandle() + if err != nil { + return err + } + defer netHandle.Delete() + tapLink, err := getLinkByName(netHandle, name, &netlink.Tuntap{}) + if err != nil { + return fmt.Errorf("Could not get TAP interface: %s", err) + } + if err := netHandle.LinkSetDown(tapLink); err != nil { + return fmt.Errorf("Could not disable TAP %s: %s", name, err) + } + if err := netHandle.LinkDel(tapLink); err != nil { + return fmt.Errorf("Could not remove TAP %s: %s", name, err) + } + return nil + +} +func (endpoint *TuntapEndpoint) save() persistapi.NetworkEndpoint { + tuntapif := saveTuntapIf(&endpoint.TuntapInterface) + + return persistapi.NetworkEndpoint{ + Type: string(endpoint.Type()), + Tuntap: &persistapi.TuntapEndpoint{ + TuntapInterface: *tuntapif, + }, + } +} + +func (endpoint *TuntapEndpoint) load(s persistapi.NetworkEndpoint) { + endpoint.EndpointType = TuntapEndpointType + + if s.Tuntap != nil { + tuntapif := loadTuntapIf(&s.Tuntap.TuntapInterface) + endpoint.TuntapInterface = *tuntapif + } +}