mirror of
https://github.com/kata-containers/kata-containers.git
synced 2025-07-04 02:56:18 +00:00
network: Refactor network.go
Split endpoint implementations into their own file. Fixes #799 Signed-off-by: Archana Shinde <archana.m.shinde@intel.com>
This commit is contained in:
Archana Shinde
parent
7fb2706667
commit
adcd9106f9
116
virtcontainers/bridgedmacvlan_endpoint.go
Normal file
116
virtcontainers/bridgedmacvlan_endpoint.go
Normal file
@ -0,0 +1,116 @@
|
||||
// Copyright (c) 2018 Intel Corporation
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
package virtcontainers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/containernetworking/plugins/pkg/ns"
|
||||
)
|
||||
|
||||
// BridgedMacvlanEndpoint represents a macvlan endpoint that is bridged to the VM
|
||||
type BridgedMacvlanEndpoint struct {
|
||||
NetPair NetworkInterfacePair
|
||||
EndpointProperties NetworkInfo
|
||||
EndpointType EndpointType
|
||||
PCIAddr string
|
||||
}
|
||||
|
||||
func createBridgedMacvlanNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel) (*BridgedMacvlanEndpoint, error) {
|
||||
if idx < 0 {
|
||||
return &BridgedMacvlanEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx)
|
||||
}
|
||||
|
||||
netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
endpoint := &BridgedMacvlanEndpoint{
|
||||
NetPair: netPair,
|
||||
EndpointType: BridgedMacvlanEndpointType,
|
||||
}
|
||||
if ifName != "" {
|
||||
endpoint.NetPair.VirtIface.Name = ifName
|
||||
}
|
||||
|
||||
return endpoint, nil
|
||||
}
|
||||
|
||||
// Properties returns properties of the interface.
|
||||
func (endpoint *BridgedMacvlanEndpoint) Properties() NetworkInfo {
|
||||
return endpoint.EndpointProperties
|
||||
}
|
||||
|
||||
// Name returns name of the veth interface in the network pair.
|
||||
func (endpoint *BridgedMacvlanEndpoint) Name() string {
|
||||
return endpoint.NetPair.VirtIface.Name
|
||||
}
|
||||
|
||||
// HardwareAddr returns the mac address that is assigned to the tap interface
|
||||
// in th network pair.
|
||||
func (endpoint *BridgedMacvlanEndpoint) HardwareAddr() string {
|
||||
return endpoint.NetPair.TAPIface.HardAddr
|
||||
}
|
||||
|
||||
// Type identifies the endpoint as a virtual endpoint.
|
||||
func (endpoint *BridgedMacvlanEndpoint) Type() EndpointType {
|
||||
return endpoint.EndpointType
|
||||
}
|
||||
|
||||
// SetProperties sets the properties for the endpoint.
|
||||
func (endpoint *BridgedMacvlanEndpoint) SetProperties(properties NetworkInfo) {
|
||||
endpoint.EndpointProperties = properties
|
||||
}
|
||||
|
||||
// PciAddr returns the PCI address of the endpoint.
|
||||
func (endpoint *BridgedMacvlanEndpoint) PciAddr() string {
|
||||
return endpoint.PCIAddr
|
||||
}
|
||||
|
||||
// NetworkPair returns the network pair of the endpoint.
|
||||
func (endpoint *BridgedMacvlanEndpoint) NetworkPair() *NetworkInterfacePair {
|
||||
return &endpoint.NetPair
|
||||
}
|
||||
|
||||
// Attach for virtual endpoint bridges the network pair and adds the
|
||||
// tap interface of the network pair to the hypervisor.
|
||||
func (endpoint *BridgedMacvlanEndpoint) Attach(h hypervisor) error {
|
||||
networkLogger().Info("Attaching macvlan endpoint")
|
||||
if err := xconnectVMNetwork(endpoint, true, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil {
|
||||
networkLogger().WithError(err).Error("Error bridging virtual ep")
|
||||
return err
|
||||
}
|
||||
|
||||
return h.addDevice(endpoint, netDev)
|
||||
}
|
||||
|
||||
// Detach for the virtual endpoint tears down the tap and bridge
|
||||
// created for the veth interface.
|
||||
func (endpoint *BridgedMacvlanEndpoint) Detach(netNsCreated bool, netNsPath string) error {
|
||||
// The network namespace would have been deleted at this point
|
||||
// if it has not been created by virtcontainers.
|
||||
if !netNsCreated {
|
||||
return nil
|
||||
}
|
||||
|
||||
networkLogger().Info("Detaching virtual endpoint")
|
||||
|
||||
return doNetNS(netNsPath, func(_ ns.NetNS) error {
|
||||
//return xconnectVMNetwork(&(endpoint.NetPair), false, 0, false, endpoint.EndpointType)
|
||||
return xconnectVMNetwork(endpoint, false, 0, false)
|
||||
})
|
||||
}
|
||||
|
||||
// HotAttach for physical endpoint not supported yet
|
||||
func (endpoint *BridgedMacvlanEndpoint) HotAttach(h hypervisor) error {
|
||||
return fmt.Errorf("BridgedMacvlanEndpoint does not support Hot attach")
|
||||
}
|
||||
|
||||
// HotDetach for physical endpoint not supported yet
|
||||
func (endpoint *BridgedMacvlanEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
|
||||
return fmt.Errorf("BridgedMacvlanEndpoint does not support Hot detach")
|
||||
}
|
||||
87
virtcontainers/endpoint.go
Normal file
87
virtcontainers/endpoint.go
Normal file
@ -0,0 +1,87 @@
|
||||
// Copyright (c) 2018 Intel Corporation
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
package virtcontainers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Endpoint represents a physical or virtual network interface.
|
||||
type Endpoint interface {
|
||||
Properties() NetworkInfo
|
||||
Name() string
|
||||
HardwareAddr() string
|
||||
Type() EndpointType
|
||||
PciAddr() string
|
||||
NetworkPair() *NetworkInterfacePair
|
||||
|
||||
SetProperties(NetworkInfo)
|
||||
Attach(hypervisor) error
|
||||
Detach(netNsCreated bool, netNsPath string) error
|
||||
HotAttach(h hypervisor) error
|
||||
HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error
|
||||
}
|
||||
|
||||
// EndpointType identifies the type of the network endpoint.
|
||||
type EndpointType string
|
||||
|
||||
const (
|
||||
// PhysicalEndpointType is the physical network interface.
|
||||
PhysicalEndpointType EndpointType = "physical"
|
||||
|
||||
// VirtualEndpointType is the virtual network interface.
|
||||
VirtualEndpointType EndpointType = "virtual"
|
||||
|
||||
// VhostUserEndpointType is the vhostuser network interface.
|
||||
VhostUserEndpointType EndpointType = "vhost-user"
|
||||
|
||||
// BridgedMacvlanEndpointType is macvlan network interface.
|
||||
BridgedMacvlanEndpointType EndpointType = "macvlan"
|
||||
|
||||
// MacvtapEndpointType is macvtap network interface.
|
||||
MacvtapEndpointType EndpointType = "macvtap"
|
||||
)
|
||||
|
||||
// Set sets an endpoint type based on the input string.
|
||||
func (endpointType *EndpointType) Set(value string) error {
|
||||
switch value {
|
||||
case "physical":
|
||||
*endpointType = PhysicalEndpointType
|
||||
return nil
|
||||
case "virtual":
|
||||
*endpointType = VirtualEndpointType
|
||||
return nil
|
||||
case "vhost-user":
|
||||
*endpointType = VhostUserEndpointType
|
||||
return nil
|
||||
case "macvlan":
|
||||
*endpointType = BridgedMacvlanEndpointType
|
||||
return nil
|
||||
case "macvtap":
|
||||
*endpointType = MacvtapEndpointType
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("Unknown endpoint type %s", value)
|
||||
}
|
||||
}
|
||||
|
||||
// String converts an endpoint type to a string.
|
||||
func (endpointType *EndpointType) String() string {
|
||||
switch *endpointType {
|
||||
case PhysicalEndpointType:
|
||||
return string(PhysicalEndpointType)
|
||||
case VirtualEndpointType:
|
||||
return string(VirtualEndpointType)
|
||||
case VhostUserEndpointType:
|
||||
return string(VhostUserEndpointType)
|
||||
case BridgedMacvlanEndpointType:
|
||||
return string(BridgedMacvlanEndpointType)
|
||||
case MacvtapEndpointType:
|
||||
return string(MacvtapEndpointType)
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
101
virtcontainers/macvtap_endpoint.go
Normal file
101
virtcontainers/macvtap_endpoint.go
Normal file
@ -0,0 +1,101 @@
|
||||
// Copyright (c) 2018 Intel Corporation
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
package virtcontainers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
// MacvtapEndpoint represents a macvtap endpoint
|
||||
type MacvtapEndpoint struct {
|
||||
EndpointProperties NetworkInfo
|
||||
EndpointType EndpointType
|
||||
VMFds []*os.File
|
||||
VhostFds []*os.File
|
||||
PCIAddr string
|
||||
}
|
||||
|
||||
func createMacvtapNetworkEndpoint(netInfo NetworkInfo) (*MacvtapEndpoint, error) {
|
||||
endpoint := &MacvtapEndpoint{
|
||||
EndpointType: MacvtapEndpointType,
|
||||
EndpointProperties: netInfo,
|
||||
}
|
||||
|
||||
return endpoint, nil
|
||||
}
|
||||
|
||||
// Properties returns the properties of the macvtap interface.
|
||||
func (endpoint *MacvtapEndpoint) Properties() NetworkInfo {
|
||||
return endpoint.EndpointProperties
|
||||
}
|
||||
|
||||
// HardwareAddr returns the mac address of the macvtap network interface.
|
||||
func (endpoint *MacvtapEndpoint) HardwareAddr() string {
|
||||
return endpoint.EndpointProperties.Iface.HardwareAddr.String()
|
||||
}
|
||||
|
||||
// Name returns name of the macvtap interface.
|
||||
func (endpoint *MacvtapEndpoint) Name() string {
|
||||
return endpoint.EndpointProperties.Iface.Name
|
||||
}
|
||||
|
||||
// Type indentifies the endpoint as a macvtap endpoint.
|
||||
func (endpoint *MacvtapEndpoint) Type() EndpointType {
|
||||
return endpoint.EndpointType
|
||||
}
|
||||
|
||||
// SetProperties sets the properties of the macvtap endpoint.
|
||||
func (endpoint *MacvtapEndpoint) SetProperties(properties NetworkInfo) {
|
||||
endpoint.EndpointProperties = properties
|
||||
}
|
||||
|
||||
// Attach for macvtap endpoint passes macvtap device to the hypervisor.
|
||||
func (endpoint *MacvtapEndpoint) Attach(h hypervisor) error {
|
||||
networkLogger().WithField("endpoint-type", "macvtap").Info("Attaching endpoint")
|
||||
var err error
|
||||
|
||||
endpoint.VMFds, err = createMacvtapFds(endpoint.EndpointProperties.Iface.Index, int(h.hypervisorConfig().NumVCPUs))
|
||||
if err != nil {
|
||||
return fmt.Errorf("Could not setup macvtap fds %s: %s", endpoint.EndpointProperties.Iface.Name, err)
|
||||
}
|
||||
|
||||
if !h.hypervisorConfig().DisableVhostNet {
|
||||
vhostFds, err := createVhostFds(int(h.hypervisorConfig().NumVCPUs))
|
||||
if err != nil {
|
||||
return fmt.Errorf("Could not setup vhost fds %s : %s", endpoint.EndpointProperties.Iface.Name, err)
|
||||
}
|
||||
endpoint.VhostFds = vhostFds
|
||||
}
|
||||
|
||||
return h.addDevice(endpoint, netDev)
|
||||
}
|
||||
|
||||
// Detach for macvtap endpoint does nothing.
|
||||
func (endpoint *MacvtapEndpoint) Detach(netNsCreated bool, netNsPath string) error {
|
||||
networkLogger().WithField("endpoint-type", "macvtap").Info("Detaching endpoint")
|
||||
return nil
|
||||
}
|
||||
|
||||
// HotAttach for macvtap endpoint not supported yet
|
||||
func (endpoint *MacvtapEndpoint) HotAttach(h hypervisor) error {
|
||||
return fmt.Errorf("MacvtapEndpoint does not support Hot attach")
|
||||
}
|
||||
|
||||
// HotDetach for macvtap endpoint not supported yet
|
||||
func (endpoint *MacvtapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
|
||||
return fmt.Errorf("MacvtapEndpoint does not support Hot detach")
|
||||
}
|
||||
|
||||
// PciAddr returns the PCI address of the endpoint.
|
||||
func (endpoint *MacvtapEndpoint) PciAddr() string {
|
||||
return endpoint.PCIAddr
|
||||
}
|
||||
|
||||
// NetworkPair returns the network pair of the endpoint.
|
||||
func (endpoint *MacvtapEndpoint) NetworkPair() *NetworkInterfacePair {
|
||||
return nil
|
||||
}
|
||||
@ -7,29 +7,22 @@ package virtcontainers
|
||||
|
||||
import (
|
||||
cryptoRand "crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/containernetworking/plugins/pkg/ns"
|
||||
"github.com/safchain/ethtool"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/vishvananda/netlink"
|
||||
"github.com/vishvananda/netns"
|
||||
"golang.org/x/sys/unix"
|
||||
|
||||
"github.com/kata-containers/agent/protocols/grpc"
|
||||
"github.com/kata-containers/runtime/virtcontainers/device/config"
|
||||
"github.com/kata-containers/runtime/virtcontainers/device/drivers"
|
||||
"github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
|
||||
"github.com/kata-containers/runtime/virtcontainers/utils"
|
||||
)
|
||||
@ -148,548 +141,10 @@ type NetworkConfig struct {
|
||||
InterworkingModel NetInterworkingModel
|
||||
}
|
||||
|
||||
// Endpoint represents a physical or virtual network interface.
|
||||
type Endpoint interface {
|
||||
Properties() NetworkInfo
|
||||
Name() string
|
||||
HardwareAddr() string
|
||||
Type() EndpointType
|
||||
PciAddr() string
|
||||
NetworkPair() *NetworkInterfacePair
|
||||
|
||||
SetProperties(NetworkInfo)
|
||||
Attach(hypervisor) error
|
||||
Detach(netNsCreated bool, netNsPath string) error
|
||||
HotAttach(h hypervisor) error
|
||||
HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error
|
||||
}
|
||||
|
||||
// VirtualEndpoint gathers a network pair and its properties.
|
||||
type VirtualEndpoint struct {
|
||||
NetPair NetworkInterfacePair
|
||||
EndpointProperties NetworkInfo
|
||||
Physical bool
|
||||
EndpointType EndpointType
|
||||
PCIAddr string
|
||||
}
|
||||
|
||||
// PhysicalEndpoint gathers a physical network interface and its properties
|
||||
type PhysicalEndpoint struct {
|
||||
IfaceName string
|
||||
HardAddr string
|
||||
EndpointProperties NetworkInfo
|
||||
EndpointType EndpointType
|
||||
BDF string
|
||||
Driver string
|
||||
VendorDeviceID string
|
||||
PCIAddr string
|
||||
}
|
||||
|
||||
// VhostUserEndpoint represents a vhost-user socket based network interface
|
||||
type VhostUserEndpoint struct {
|
||||
// Path to the vhost-user socket on the host system
|
||||
SocketPath string
|
||||
// MAC address of the interface
|
||||
HardAddr string
|
||||
IfaceName string
|
||||
EndpointProperties NetworkInfo
|
||||
EndpointType EndpointType
|
||||
PCIAddr string
|
||||
}
|
||||
|
||||
// BridgedMacvlanEndpoint represents a macvlan endpoint that is bridged to the VM
|
||||
type BridgedMacvlanEndpoint struct {
|
||||
NetPair NetworkInterfacePair
|
||||
EndpointProperties NetworkInfo
|
||||
EndpointType EndpointType
|
||||
PCIAddr string
|
||||
}
|
||||
|
||||
// MacvtapEndpoint represents a macvtap endpoint
|
||||
type MacvtapEndpoint struct {
|
||||
EndpointProperties NetworkInfo
|
||||
EndpointType EndpointType
|
||||
VMFds []*os.File
|
||||
VhostFds []*os.File
|
||||
PCIAddr string
|
||||
}
|
||||
|
||||
// Properties returns properties for the veth interface in the network pair.
|
||||
func (endpoint *VirtualEndpoint) Properties() NetworkInfo {
|
||||
return endpoint.EndpointProperties
|
||||
}
|
||||
|
||||
// Name returns name of the veth interface in the network pair.
|
||||
func (endpoint *VirtualEndpoint) Name() string {
|
||||
return endpoint.NetPair.VirtIface.Name
|
||||
}
|
||||
|
||||
// HardwareAddr returns the mac address that is assigned to the tap interface
|
||||
// in th network pair.
|
||||
func (endpoint *VirtualEndpoint) HardwareAddr() string {
|
||||
return endpoint.NetPair.TAPIface.HardAddr
|
||||
}
|
||||
|
||||
// Type identifies the endpoint as a virtual endpoint.
|
||||
func (endpoint *VirtualEndpoint) Type() EndpointType {
|
||||
return endpoint.EndpointType
|
||||
}
|
||||
|
||||
// PciAddr returns the PCI address of the endpoint.
|
||||
func (endpoint *VirtualEndpoint) PciAddr() string {
|
||||
return endpoint.PCIAddr
|
||||
}
|
||||
|
||||
// NetworkPair returns the network pair of the endpoint.
|
||||
func (endpoint *VirtualEndpoint) NetworkPair() *NetworkInterfacePair {
|
||||
return &endpoint.NetPair
|
||||
}
|
||||
|
||||
// SetProperties sets the properties for the endpoint.
|
||||
func (endpoint *VirtualEndpoint) SetProperties(properties NetworkInfo) {
|
||||
endpoint.EndpointProperties = properties
|
||||
}
|
||||
|
||||
func networkLogger() *logrus.Entry {
|
||||
return virtLog.WithField("subsystem", "network")
|
||||
}
|
||||
|
||||
// Attach for virtual endpoint bridges the network pair and adds the
|
||||
// tap interface of the network pair to the hypervisor.
|
||||
func (endpoint *VirtualEndpoint) Attach(h hypervisor) error {
|
||||
networkLogger().WithField("endpoint-type", "virtual").Info("Attaching endpoint")
|
||||
|
||||
if err := xconnectVMNetwork(endpoint, true, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil {
|
||||
networkLogger().WithError(err).Error("Error bridging virtual endpoint")
|
||||
return err
|
||||
}
|
||||
|
||||
return h.addDevice(endpoint, netDev)
|
||||
}
|
||||
|
||||
// Detach for the virtual endpoint tears down the tap and bridge
|
||||
// created for the veth interface.
|
||||
func (endpoint *VirtualEndpoint) Detach(netNsCreated bool, netNsPath string) error {
|
||||
// The network namespace would have been deleted at this point
|
||||
// if it has not been created by virtcontainers.
|
||||
if !netNsCreated {
|
||||
return nil
|
||||
}
|
||||
|
||||
networkLogger().WithField("endpoint-type", "virtual").Info("Detaching endpoint")
|
||||
|
||||
return doNetNS(netNsPath, func(_ ns.NetNS) error {
|
||||
return xconnectVMNetwork(endpoint, false, 0, false)
|
||||
})
|
||||
}
|
||||
|
||||
// HotAttach for the virtual endpoint uses hot plug device
|
||||
func (endpoint *VirtualEndpoint) HotAttach(h hypervisor) error {
|
||||
networkLogger().Info("Hot attaching virtual endpoint")
|
||||
if err := xconnectVMNetwork(endpoint, true, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil {
|
||||
networkLogger().WithError(err).Error("Error bridging virtual ep")
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := h.hotplugAddDevice(endpoint, netDev); err != nil {
|
||||
networkLogger().WithError(err).Error("Error attach virtual ep")
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HotDetach for the virtual endpoint uses hot pull device
|
||||
func (endpoint *VirtualEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
|
||||
if !netNsCreated {
|
||||
return nil
|
||||
}
|
||||
networkLogger().Info("Hot detaching virtual endpoint")
|
||||
if err := doNetNS(netNsPath, func(_ ns.NetNS) error {
|
||||
return xconnectVMNetwork(endpoint, false, 0, h.hypervisorConfig().DisableVhostNet)
|
||||
}); err != nil {
|
||||
networkLogger().WithError(err).Warn("Error un-bridging virtual ep")
|
||||
}
|
||||
|
||||
if _, err := h.hotplugRemoveDevice(endpoint, netDev); err != nil {
|
||||
networkLogger().WithError(err).Error("Error detach virtual ep")
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Properties returns the properties of the interface.
|
||||
func (endpoint *VhostUserEndpoint) Properties() NetworkInfo {
|
||||
return endpoint.EndpointProperties
|
||||
}
|
||||
|
||||
// Name returns name of the interface.
|
||||
func (endpoint *VhostUserEndpoint) Name() string {
|
||||
return endpoint.IfaceName
|
||||
}
|
||||
|
||||
// HardwareAddr returns the mac address of the vhostuser network interface
|
||||
func (endpoint *VhostUserEndpoint) HardwareAddr() string {
|
||||
return endpoint.HardAddr
|
||||
}
|
||||
|
||||
// Type indentifies the endpoint as a vhostuser endpoint.
|
||||
func (endpoint *VhostUserEndpoint) Type() EndpointType {
|
||||
return endpoint.EndpointType
|
||||
}
|
||||
|
||||
// SetProperties sets the properties of the endpoint.
|
||||
func (endpoint *VhostUserEndpoint) SetProperties(properties NetworkInfo) {
|
||||
endpoint.EndpointProperties = properties
|
||||
}
|
||||
|
||||
// PciAddr returns the PCI address of the endpoint.
|
||||
func (endpoint *VhostUserEndpoint) PciAddr() string {
|
||||
return endpoint.PCIAddr
|
||||
}
|
||||
|
||||
// NetworkPair returns the network pair of the endpoint.
|
||||
func (endpoint *VhostUserEndpoint) NetworkPair() *NetworkInterfacePair {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Attach for vhostuser endpoint
|
||||
func (endpoint *VhostUserEndpoint) Attach(h hypervisor) error {
|
||||
networkLogger().WithField("endpoint-type", "vhostuser").Info("Attaching endpoint")
|
||||
|
||||
// Generate a unique ID to be used for hypervisor commandline fields
|
||||
randBytes, err := utils.GenerateRandomBytes(8)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
id := hex.EncodeToString(randBytes)
|
||||
|
||||
d := config.VhostUserDeviceAttrs{
|
||||
DevID: id,
|
||||
SocketPath: endpoint.SocketPath,
|
||||
MacAddress: endpoint.HardAddr,
|
||||
Type: config.VhostUserNet,
|
||||
}
|
||||
|
||||
return h.addDevice(d, vhostuserDev)
|
||||
}
|
||||
|
||||
// Detach for vhostuser endpoint
|
||||
func (endpoint *VhostUserEndpoint) Detach(netNsCreated bool, netNsPath string) error {
|
||||
networkLogger().WithField("endpoint-type", "vhostuser").Info("Detaching endpoint")
|
||||
return nil
|
||||
}
|
||||
|
||||
// HotAttach for vhostuser endpoint not supported yet
|
||||
func (endpoint *VhostUserEndpoint) HotAttach(h hypervisor) error {
|
||||
return fmt.Errorf("VhostUserEndpoint does not support Hot attach")
|
||||
}
|
||||
|
||||
// HotDetach for vhostuser endpoint not supported yet
|
||||
func (endpoint *VhostUserEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
|
||||
return fmt.Errorf("VhostUserEndpoint does not support Hot detach")
|
||||
}
|
||||
|
||||
// Create a vhostuser endpoint
|
||||
func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndpoint, error) {
|
||||
|
||||
vhostUserEndpoint := &VhostUserEndpoint{
|
||||
SocketPath: socket,
|
||||
HardAddr: netInfo.Iface.HardwareAddr.String(),
|
||||
IfaceName: netInfo.Iface.Name,
|
||||
EndpointType: VhostUserEndpointType,
|
||||
}
|
||||
return vhostUserEndpoint, nil
|
||||
}
|
||||
|
||||
// Properties returns the properties of the physical interface.
|
||||
func (endpoint *PhysicalEndpoint) Properties() NetworkInfo {
|
||||
return endpoint.EndpointProperties
|
||||
}
|
||||
|
||||
// HardwareAddr returns the mac address of the physical network interface.
|
||||
func (endpoint *PhysicalEndpoint) HardwareAddr() string {
|
||||
return endpoint.HardAddr
|
||||
}
|
||||
|
||||
// Name returns name of the physical interface.
|
||||
func (endpoint *PhysicalEndpoint) Name() string {
|
||||
return endpoint.IfaceName
|
||||
}
|
||||
|
||||
// Type indentifies the endpoint as a physical endpoint.
|
||||
func (endpoint *PhysicalEndpoint) Type() EndpointType {
|
||||
return endpoint.EndpointType
|
||||
}
|
||||
|
||||
// PciAddr returns the PCI address of the endpoint.
|
||||
func (endpoint *PhysicalEndpoint) PciAddr() string {
|
||||
return endpoint.PCIAddr
|
||||
}
|
||||
|
||||
// SetProperties sets the properties of the physical endpoint.
|
||||
func (endpoint *PhysicalEndpoint) SetProperties(properties NetworkInfo) {
|
||||
endpoint.EndpointProperties = properties
|
||||
}
|
||||
|
||||
// NetworkPair returns the network pair of the endpoint.
|
||||
func (endpoint *PhysicalEndpoint) NetworkPair() *NetworkInterfacePair {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Attach for physical endpoint binds the physical network interface to
|
||||
// vfio-pci and adds device to the hypervisor with vfio-passthrough.
|
||||
func (endpoint *PhysicalEndpoint) Attach(h hypervisor) error {
|
||||
networkLogger().WithField("endpoint-type", "physical").Info("Attaching endpoint")
|
||||
|
||||
// Unbind physical interface from host driver and bind to vfio
|
||||
// so that it can be passed to qemu.
|
||||
if err := bindNICToVFIO(endpoint); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// TODO: use device manager as general device management entrance
|
||||
d := config.VFIODev{
|
||||
BDF: endpoint.BDF,
|
||||
}
|
||||
|
||||
return h.addDevice(d, vfioDev)
|
||||
}
|
||||
|
||||
// Detach for physical endpoint unbinds the physical network interface from vfio-pci
|
||||
// and binds it back to the saved host driver.
|
||||
func (endpoint *PhysicalEndpoint) Detach(netNsCreated bool, netNsPath string) error {
|
||||
// Bind back the physical network interface to host.
|
||||
// We need to do this even if a new network namespace has not
|
||||
// been created by virtcontainers.
|
||||
networkLogger().WithField("endpoint-type", "physical").Info("Detaching endpoint")
|
||||
|
||||
// We do not need to enter the network namespace to bind back the
|
||||
// physical interface to host driver.
|
||||
return bindNICToHost(endpoint)
|
||||
}
|
||||
|
||||
// HotAttach for physical endpoint not supported yet
|
||||
func (endpoint *PhysicalEndpoint) HotAttach(h hypervisor) error {
|
||||
return fmt.Errorf("PhysicalEndpoint does not support Hot attach")
|
||||
}
|
||||
|
||||
// HotDetach for physical endpoint not supported yet
|
||||
func (endpoint *PhysicalEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
|
||||
return fmt.Errorf("PhysicalEndpoint does not support Hot detach")
|
||||
}
|
||||
|
||||
// Macvlan
|
||||
|
||||
// Properties returns properties of the interface.
|
||||
func (endpoint *BridgedMacvlanEndpoint) Properties() NetworkInfo {
|
||||
return endpoint.EndpointProperties
|
||||
}
|
||||
|
||||
// Name returns name of the veth interface in the network pair.
|
||||
func (endpoint *BridgedMacvlanEndpoint) Name() string {
|
||||
return endpoint.NetPair.VirtIface.Name
|
||||
}
|
||||
|
||||
// HardwareAddr returns the mac address that is assigned to the tap interface
|
||||
// in th network pair.
|
||||
func (endpoint *BridgedMacvlanEndpoint) HardwareAddr() string {
|
||||
return endpoint.NetPair.TAPIface.HardAddr
|
||||
}
|
||||
|
||||
// Type identifies the endpoint as a virtual endpoint.
|
||||
func (endpoint *BridgedMacvlanEndpoint) Type() EndpointType {
|
||||
return endpoint.EndpointType
|
||||
}
|
||||
|
||||
// SetProperties sets the properties for the endpoint.
|
||||
func (endpoint *BridgedMacvlanEndpoint) SetProperties(properties NetworkInfo) {
|
||||
endpoint.EndpointProperties = properties
|
||||
}
|
||||
|
||||
// PciAddr returns the PCI address of the endpoint.
|
||||
func (endpoint *BridgedMacvlanEndpoint) PciAddr() string {
|
||||
return endpoint.PCIAddr
|
||||
}
|
||||
|
||||
// NetworkPair returns the network pair of the endpoint.
|
||||
func (endpoint *BridgedMacvlanEndpoint) NetworkPair() *NetworkInterfacePair {
|
||||
return &endpoint.NetPair
|
||||
}
|
||||
|
||||
// Attach for virtual endpoint bridges the network pair and adds the
|
||||
// tap interface of the network pair to the hypervisor.
|
||||
func (endpoint *BridgedMacvlanEndpoint) Attach(h hypervisor) error {
|
||||
networkLogger().Info("Attaching macvlan endpoint")
|
||||
if err := xconnectVMNetwork(endpoint, true, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil {
|
||||
networkLogger().WithError(err).Error("Error bridging virtual ep")
|
||||
return err
|
||||
}
|
||||
|
||||
return h.addDevice(endpoint, netDev)
|
||||
}
|
||||
|
||||
// Detach for the virtual endpoint tears down the tap and bridge
|
||||
// created for the veth interface.
|
||||
func (endpoint *BridgedMacvlanEndpoint) Detach(netNsCreated bool, netNsPath string) error {
|
||||
// The network namespace would have been deleted at this point
|
||||
// if it has not been created by virtcontainers.
|
||||
if !netNsCreated {
|
||||
return nil
|
||||
}
|
||||
|
||||
networkLogger().Info("Detaching virtual endpoint")
|
||||
|
||||
return doNetNS(netNsPath, func(_ ns.NetNS) error {
|
||||
//return xconnectVMNetwork(&(endpoint.NetPair), false, 0, false, endpoint.EndpointType)
|
||||
return xconnectVMNetwork(endpoint, false, 0, false)
|
||||
})
|
||||
}
|
||||
|
||||
// HotAttach for physical endpoint not supported yet
|
||||
func (endpoint *BridgedMacvlanEndpoint) HotAttach(h hypervisor) error {
|
||||
return fmt.Errorf("BridgedMacvlanEndpoint does not support Hot attach")
|
||||
}
|
||||
|
||||
// HotDetach for physical endpoint not supported yet
|
||||
func (endpoint *BridgedMacvlanEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
|
||||
return fmt.Errorf("BridgedMacvlanEndpoint does not support Hot detach")
|
||||
}
|
||||
|
||||
//Macvtap
|
||||
|
||||
// Properties returns the properties of the macvtap interface.
|
||||
func (endpoint *MacvtapEndpoint) Properties() NetworkInfo {
|
||||
return endpoint.EndpointProperties
|
||||
}
|
||||
|
||||
// HardwareAddr returns the mac address of the macvtap network interface.
|
||||
func (endpoint *MacvtapEndpoint) HardwareAddr() string {
|
||||
return endpoint.EndpointProperties.Iface.HardwareAddr.String()
|
||||
}
|
||||
|
||||
// Name returns name of the macvtap interface.
|
||||
func (endpoint *MacvtapEndpoint) Name() string {
|
||||
return endpoint.EndpointProperties.Iface.Name
|
||||
}
|
||||
|
||||
// Type indentifies the endpoint as a macvtap endpoint.
|
||||
func (endpoint *MacvtapEndpoint) Type() EndpointType {
|
||||
return endpoint.EndpointType
|
||||
}
|
||||
|
||||
// SetProperties sets the properties of the macvtap endpoint.
|
||||
func (endpoint *MacvtapEndpoint) SetProperties(properties NetworkInfo) {
|
||||
endpoint.EndpointProperties = properties
|
||||
}
|
||||
|
||||
// Attach for macvtap endpoint passes macvtap device to the hypervisor.
|
||||
func (endpoint *MacvtapEndpoint) Attach(h hypervisor) error {
|
||||
networkLogger().WithField("endpoint-type", "macvtap").Info("Attaching endpoint")
|
||||
var err error
|
||||
|
||||
endpoint.VMFds, err = createMacvtapFds(endpoint.EndpointProperties.Iface.Index, int(h.hypervisorConfig().NumVCPUs))
|
||||
if err != nil {
|
||||
return fmt.Errorf("Could not setup macvtap fds %s: %s", endpoint.EndpointProperties.Iface.Name, err)
|
||||
}
|
||||
|
||||
if !h.hypervisorConfig().DisableVhostNet {
|
||||
vhostFds, err := createVhostFds(int(h.hypervisorConfig().NumVCPUs))
|
||||
if err != nil {
|
||||
return fmt.Errorf("Could not setup vhost fds %s : %s", endpoint.EndpointProperties.Iface.Name, err)
|
||||
}
|
||||
endpoint.VhostFds = vhostFds
|
||||
}
|
||||
|
||||
return h.addDevice(endpoint, netDev)
|
||||
}
|
||||
|
||||
// Detach for macvtap endpoint does nothing.
|
||||
func (endpoint *MacvtapEndpoint) Detach(netNsCreated bool, netNsPath string) error {
|
||||
networkLogger().WithField("endpoint-type", "macvtap").Info("Detaching endpoint")
|
||||
return nil
|
||||
}
|
||||
|
||||
// HotAttach for macvtap endpoint not supported yet
|
||||
func (endpoint *MacvtapEndpoint) HotAttach(h hypervisor) error {
|
||||
return fmt.Errorf("MacvtapEndpoint does not support Hot attach")
|
||||
}
|
||||
|
||||
// HotDetach for macvtap endpoint not supported yet
|
||||
func (endpoint *MacvtapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
|
||||
return fmt.Errorf("MacvtapEndpoint does not support Hot detach")
|
||||
}
|
||||
|
||||
// PciAddr returns the PCI address of the endpoint.
|
||||
func (endpoint *MacvtapEndpoint) PciAddr() string {
|
||||
return endpoint.PCIAddr
|
||||
}
|
||||
|
||||
// NetworkPair returns the network pair of the endpoint.
|
||||
func (endpoint *MacvtapEndpoint) NetworkPair() *NetworkInterfacePair {
|
||||
return nil
|
||||
}
|
||||
|
||||
// EndpointType identifies the type of the network endpoint.
|
||||
type EndpointType string
|
||||
|
||||
const (
|
||||
// PhysicalEndpointType is the physical network interface.
|
||||
PhysicalEndpointType EndpointType = "physical"
|
||||
|
||||
// VirtualEndpointType is the virtual network interface.
|
||||
VirtualEndpointType EndpointType = "virtual"
|
||||
|
||||
// VhostUserEndpointType is the vhostuser network interface.
|
||||
VhostUserEndpointType EndpointType = "vhost-user"
|
||||
|
||||
// BridgedMacvlanEndpointType is macvlan network interface.
|
||||
BridgedMacvlanEndpointType EndpointType = "macvlan"
|
||||
|
||||
// MacvtapEndpointType is macvtap network interface.
|
||||
MacvtapEndpointType EndpointType = "macvtap"
|
||||
)
|
||||
|
||||
// Set sets an endpoint type based on the input string.
|
||||
func (endpointType *EndpointType) Set(value string) error {
|
||||
switch value {
|
||||
case "physical":
|
||||
*endpointType = PhysicalEndpointType
|
||||
return nil
|
||||
case "virtual":
|
||||
*endpointType = VirtualEndpointType
|
||||
return nil
|
||||
case "vhost-user":
|
||||
*endpointType = VhostUserEndpointType
|
||||
return nil
|
||||
case "macvlan":
|
||||
*endpointType = BridgedMacvlanEndpointType
|
||||
return nil
|
||||
case "macvtap":
|
||||
*endpointType = MacvtapEndpointType
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("Unknown endpoint type %s", value)
|
||||
}
|
||||
}
|
||||
|
||||
// String converts an endpoint type to a string.
|
||||
func (endpointType *EndpointType) String() string {
|
||||
switch *endpointType {
|
||||
case PhysicalEndpointType:
|
||||
return string(PhysicalEndpointType)
|
||||
case VirtualEndpointType:
|
||||
return string(VirtualEndpointType)
|
||||
case VhostUserEndpointType:
|
||||
return string(VhostUserEndpointType)
|
||||
case BridgedMacvlanEndpointType:
|
||||
return string(BridgedMacvlanEndpointType)
|
||||
case MacvtapEndpointType:
|
||||
return string(MacvtapEndpointType)
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// NetworkNamespace contains all data related to its network namespace.
|
||||
type NetworkNamespace struct {
|
||||
NetNsPath string
|
||||
@ -1411,60 +866,6 @@ func deleteNetNS(netNSPath string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func createVirtualNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel) (*VirtualEndpoint, error) {
|
||||
if idx < 0 {
|
||||
return &VirtualEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx)
|
||||
}
|
||||
|
||||
netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
endpoint := &VirtualEndpoint{
|
||||
// TODO This is too specific. We may need to create multiple
|
||||
// end point types here and then decide how to connect them
|
||||
// at the time of hypervisor attach and not here
|
||||
NetPair: netPair,
|
||||
EndpointType: VirtualEndpointType,
|
||||
}
|
||||
if ifName != "" {
|
||||
endpoint.NetPair.VirtIface.Name = ifName
|
||||
}
|
||||
|
||||
return endpoint, nil
|
||||
}
|
||||
|
||||
func createMacvtapNetworkEndpoint(netInfo NetworkInfo) (*MacvtapEndpoint, error) {
|
||||
endpoint := &MacvtapEndpoint{
|
||||
EndpointType: MacvtapEndpointType,
|
||||
EndpointProperties: netInfo,
|
||||
}
|
||||
|
||||
return endpoint, nil
|
||||
}
|
||||
|
||||
func createBridgedMacvlanNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel) (*BridgedMacvlanEndpoint, error) {
|
||||
if idx < 0 {
|
||||
return &BridgedMacvlanEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx)
|
||||
}
|
||||
|
||||
netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
endpoint := &BridgedMacvlanEndpoint{
|
||||
NetPair: netPair,
|
||||
EndpointType: BridgedMacvlanEndpointType,
|
||||
}
|
||||
if ifName != "" {
|
||||
endpoint.NetPair.VirtIface.Name = ifName
|
||||
}
|
||||
|
||||
return endpoint, nil
|
||||
}
|
||||
|
||||
func generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*grpc.Interface, []*grpc.Route, error) {
|
||||
|
||||
if networkNS.NetNsPath == "" {
|
||||
@ -1723,132 +1124,6 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel) (E
|
||||
return endpoint, err
|
||||
}
|
||||
|
||||
// isPhysicalIface checks if an interface is a physical device.
|
||||
// We use ethtool here to not rely on device sysfs inside the network namespace.
|
||||
func isPhysicalIface(ifaceName string) (bool, error) {
|
||||
if ifaceName == "lo" {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
ethHandle, err := ethtool.NewEthtool()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
bus, err := ethHandle.BusInfo(ifaceName)
|
||||
if err != nil {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Check for a pci bus format
|
||||
tokens := strings.Split(bus, ":")
|
||||
if len(tokens) != 3 {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
var sysPCIDevicesPath = "/sys/bus/pci/devices"
|
||||
|
||||
func createPhysicalEndpoint(netInfo NetworkInfo) (*PhysicalEndpoint, error) {
|
||||
// Get ethtool handle to derive driver and bus
|
||||
ethHandle, err := ethtool.NewEthtool()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Get BDF
|
||||
bdf, err := ethHandle.BusInfo(netInfo.Iface.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Get Driver
|
||||
driver, err := ethHandle.DriverName(netInfo.Iface.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Get vendor and device id from pci space (sys/bus/pci/devices/$bdf)
|
||||
|
||||
ifaceDevicePath := filepath.Join(sysPCIDevicesPath, bdf, "device")
|
||||
contents, err := ioutil.ReadFile(ifaceDevicePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
deviceID := strings.TrimSpace(string(contents))
|
||||
|
||||
// Vendor id
|
||||
ifaceVendorPath := filepath.Join(sysPCIDevicesPath, bdf, "vendor")
|
||||
contents, err = ioutil.ReadFile(ifaceVendorPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
vendorID := strings.TrimSpace(string(contents))
|
||||
vendorDeviceID := fmt.Sprintf("%s %s", vendorID, deviceID)
|
||||
vendorDeviceID = strings.TrimSpace(vendorDeviceID)
|
||||
|
||||
physicalEndpoint := &PhysicalEndpoint{
|
||||
IfaceName: netInfo.Iface.Name,
|
||||
HardAddr: netInfo.Iface.HardwareAddr.String(),
|
||||
VendorDeviceID: vendorDeviceID,
|
||||
EndpointType: PhysicalEndpointType,
|
||||
Driver: driver,
|
||||
BDF: bdf,
|
||||
}
|
||||
|
||||
return physicalEndpoint, nil
|
||||
}
|
||||
|
||||
func bindNICToVFIO(endpoint *PhysicalEndpoint) error {
|
||||
return drivers.BindDevicetoVFIO(endpoint.BDF, endpoint.Driver, endpoint.VendorDeviceID)
|
||||
}
|
||||
|
||||
func bindNICToHost(endpoint *PhysicalEndpoint) error {
|
||||
return drivers.BindDevicetoHost(endpoint.BDF, endpoint.Driver, endpoint.VendorDeviceID)
|
||||
}
|
||||
|
||||
// Long term, this should be made more configurable. For now matching path
|
||||
// provided by CNM VPP and OVS-DPDK plugins, available at github.com/clearcontainers/vpp and
|
||||
// github.com/clearcontainers/ovsdpdk. The plugins create the socket on the host system
|
||||
// using this path.
|
||||
const hostSocketSearchPath = "/tmp/vhostuser_%s/vhu.sock"
|
||||
|
||||
// findVhostUserNetSocketPath checks if an interface is a dummy placeholder
|
||||
// for a vhost-user socket, and if it is it returns the path to the socket
|
||||
func findVhostUserNetSocketPath(netInfo NetworkInfo) (string, error) {
|
||||
if netInfo.Iface.Name == "lo" {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// check for socket file existence at known location.
|
||||
for _, addr := range netInfo.Addrs {
|
||||
socketPath := fmt.Sprintf(hostSocketSearchPath, addr.IPNet.IP)
|
||||
if _, err := os.Stat(socketPath); err == nil {
|
||||
return socketPath, nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// vhostUserSocketPath returns the path of the socket discovered. This discovery
|
||||
// will vary depending on the type of vhost-user socket.
|
||||
// Today only VhostUserNetDevice is supported.
|
||||
func vhostUserSocketPath(info interface{}) (string, error) {
|
||||
|
||||
switch v := info.(type) {
|
||||
case NetworkInfo:
|
||||
return findVhostUserNetSocketPath(v)
|
||||
default:
|
||||
return "", nil
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// network is the virtcontainers network interface.
|
||||
// Container network plugins are used to setup virtual network
|
||||
// between VM netns and the host network physical interface.
|
||||
|
||||
194
virtcontainers/physical_endpoint.go
Normal file
194
virtcontainers/physical_endpoint.go
Normal file
@ -0,0 +1,194 @@
|
||||
// Copyright (c) 2018 Intel Corporation
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
package virtcontainers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/kata-containers/runtime/virtcontainers/device/config"
|
||||
"github.com/kata-containers/runtime/virtcontainers/device/drivers"
|
||||
"github.com/safchain/ethtool"
|
||||
)
|
||||
|
||||
// PhysicalEndpoint gathers a physical network interface and its properties
|
||||
type PhysicalEndpoint struct {
|
||||
IfaceName string
|
||||
HardAddr string
|
||||
EndpointProperties NetworkInfo
|
||||
EndpointType EndpointType
|
||||
BDF string
|
||||
Driver string
|
||||
VendorDeviceID string
|
||||
PCIAddr string
|
||||
}
|
||||
|
||||
// Properties returns the properties of the physical interface.
|
||||
func (endpoint *PhysicalEndpoint) Properties() NetworkInfo {
|
||||
return endpoint.EndpointProperties
|
||||
}
|
||||
|
||||
// HardwareAddr returns the mac address of the physical network interface.
|
||||
func (endpoint *PhysicalEndpoint) HardwareAddr() string {
|
||||
return endpoint.HardAddr
|
||||
}
|
||||
|
||||
// Name returns name of the physical interface.
|
||||
func (endpoint *PhysicalEndpoint) Name() string {
|
||||
return endpoint.IfaceName
|
||||
}
|
||||
|
||||
// Type indentifies the endpoint as a physical endpoint.
|
||||
func (endpoint *PhysicalEndpoint) Type() EndpointType {
|
||||
return endpoint.EndpointType
|
||||
}
|
||||
|
||||
// PciAddr returns the PCI address of the endpoint.
|
||||
func (endpoint *PhysicalEndpoint) PciAddr() string {
|
||||
return endpoint.PCIAddr
|
||||
}
|
||||
|
||||
// SetProperties sets the properties of the physical endpoint.
|
||||
func (endpoint *PhysicalEndpoint) SetProperties(properties NetworkInfo) {
|
||||
endpoint.EndpointProperties = properties
|
||||
}
|
||||
|
||||
// NetworkPair returns the network pair of the endpoint.
|
||||
func (endpoint *PhysicalEndpoint) NetworkPair() *NetworkInterfacePair {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Attach for physical endpoint binds the physical network interface to
|
||||
// vfio-pci and adds device to the hypervisor with vfio-passthrough.
|
||||
func (endpoint *PhysicalEndpoint) Attach(h hypervisor) error {
|
||||
networkLogger().WithField("endpoint-type", "physical").Info("Attaching endpoint")
|
||||
|
||||
// Unbind physical interface from host driver and bind to vfio
|
||||
// so that it can be passed to qemu.
|
||||
if err := bindNICToVFIO(endpoint); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// TODO: use device manager as general device management entrance
|
||||
d := config.VFIODev{
|
||||
BDF: endpoint.BDF,
|
||||
}
|
||||
|
||||
return h.addDevice(d, vfioDev)
|
||||
}
|
||||
|
||||
// Detach for physical endpoint unbinds the physical network interface from vfio-pci
|
||||
// and binds it back to the saved host driver.
|
||||
func (endpoint *PhysicalEndpoint) Detach(netNsCreated bool, netNsPath string) error {
|
||||
// Bind back the physical network interface to host.
|
||||
// We need to do this even if a new network namespace has not
|
||||
// been created by virtcontainers.
|
||||
networkLogger().WithField("endpoint-type", "physical").Info("Detaching endpoint")
|
||||
|
||||
// We do not need to enter the network namespace to bind back the
|
||||
// physical interface to host driver.
|
||||
return bindNICToHost(endpoint)
|
||||
}
|
||||
|
||||
// HotAttach for physical endpoint not supported yet
|
||||
func (endpoint *PhysicalEndpoint) HotAttach(h hypervisor) error {
|
||||
return fmt.Errorf("PhysicalEndpoint does not support Hot attach")
|
||||
}
|
||||
|
||||
// HotDetach for physical endpoint not supported yet
|
||||
func (endpoint *PhysicalEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
|
||||
return fmt.Errorf("PhysicalEndpoint does not support Hot detach")
|
||||
}
|
||||
|
||||
// isPhysicalIface checks if an interface is a physical device.
|
||||
// We use ethtool here to not rely on device sysfs inside the network namespace.
|
||||
func isPhysicalIface(ifaceName string) (bool, error) {
|
||||
if ifaceName == "lo" {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
ethHandle, err := ethtool.NewEthtool()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
bus, err := ethHandle.BusInfo(ifaceName)
|
||||
if err != nil {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Check for a pci bus format
|
||||
tokens := strings.Split(bus, ":")
|
||||
if len(tokens) != 3 {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
var sysPCIDevicesPath = "/sys/bus/pci/devices"
|
||||
|
||||
func createPhysicalEndpoint(netInfo NetworkInfo) (*PhysicalEndpoint, error) {
|
||||
// Get ethtool handle to derive driver and bus
|
||||
ethHandle, err := ethtool.NewEthtool()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Get BDF
|
||||
bdf, err := ethHandle.BusInfo(netInfo.Iface.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Get Driver
|
||||
driver, err := ethHandle.DriverName(netInfo.Iface.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Get vendor and device id from pci space (sys/bus/pci/devices/$bdf)
|
||||
|
||||
ifaceDevicePath := filepath.Join(sysPCIDevicesPath, bdf, "device")
|
||||
contents, err := ioutil.ReadFile(ifaceDevicePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
deviceID := strings.TrimSpace(string(contents))
|
||||
|
||||
// Vendor id
|
||||
ifaceVendorPath := filepath.Join(sysPCIDevicesPath, bdf, "vendor")
|
||||
contents, err = ioutil.ReadFile(ifaceVendorPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
vendorID := strings.TrimSpace(string(contents))
|
||||
vendorDeviceID := fmt.Sprintf("%s %s", vendorID, deviceID)
|
||||
vendorDeviceID = strings.TrimSpace(vendorDeviceID)
|
||||
|
||||
physicalEndpoint := &PhysicalEndpoint{
|
||||
IfaceName: netInfo.Iface.Name,
|
||||
HardAddr: netInfo.Iface.HardwareAddr.String(),
|
||||
VendorDeviceID: vendorDeviceID,
|
||||
EndpointType: PhysicalEndpointType,
|
||||
Driver: driver,
|
||||
BDF: bdf,
|
||||
}
|
||||
|
||||
return physicalEndpoint, nil
|
||||
}
|
||||
|
||||
func bindNICToVFIO(endpoint *PhysicalEndpoint) error {
|
||||
return drivers.BindDevicetoVFIO(endpoint.BDF, endpoint.Driver, endpoint.VendorDeviceID)
|
||||
}
|
||||
|
||||
func bindNICToHost(endpoint *PhysicalEndpoint) error {
|
||||
return drivers.BindDevicetoHost(endpoint.BDF, endpoint.Driver, endpoint.VendorDeviceID)
|
||||
}
|
||||
149
virtcontainers/vhostuser_endpoint.go
Normal file
149
virtcontainers/vhostuser_endpoint.go
Normal file
@ -0,0 +1,149 @@
|
||||
// Copyright (c) 2018 Intel Corporation
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
package virtcontainers
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/kata-containers/runtime/virtcontainers/device/config"
|
||||
"github.com/kata-containers/runtime/virtcontainers/utils"
|
||||
)
|
||||
|
||||
// Long term, this should be made more configurable. For now matching path
|
||||
// provided by CNM VPP and OVS-DPDK plugins, available at github.com/clearcontainers/vpp and
|
||||
// github.com/clearcontainers/ovsdpdk. The plugins create the socket on the host system
|
||||
// using this path.
|
||||
const hostSocketSearchPath = "/tmp/vhostuser_%s/vhu.sock"
|
||||
|
||||
// VhostUserEndpoint represents a vhost-user socket based network interface
|
||||
type VhostUserEndpoint struct {
|
||||
// Path to the vhost-user socket on the host system
|
||||
SocketPath string
|
||||
// MAC address of the interface
|
||||
HardAddr string
|
||||
IfaceName string
|
||||
EndpointProperties NetworkInfo
|
||||
EndpointType EndpointType
|
||||
PCIAddr string
|
||||
}
|
||||
|
||||
// Properties returns the properties of the interface.
|
||||
func (endpoint *VhostUserEndpoint) Properties() NetworkInfo {
|
||||
return endpoint.EndpointProperties
|
||||
}
|
||||
|
||||
// Name returns name of the interface.
|
||||
func (endpoint *VhostUserEndpoint) Name() string {
|
||||
return endpoint.IfaceName
|
||||
}
|
||||
|
||||
// HardwareAddr returns the mac address of the vhostuser network interface
|
||||
func (endpoint *VhostUserEndpoint) HardwareAddr() string {
|
||||
return endpoint.HardAddr
|
||||
}
|
||||
|
||||
// Type indentifies the endpoint as a vhostuser endpoint.
|
||||
func (endpoint *VhostUserEndpoint) Type() EndpointType {
|
||||
return endpoint.EndpointType
|
||||
}
|
||||
|
||||
// SetProperties sets the properties of the endpoint.
|
||||
func (endpoint *VhostUserEndpoint) SetProperties(properties NetworkInfo) {
|
||||
endpoint.EndpointProperties = properties
|
||||
}
|
||||
|
||||
// PciAddr returns the PCI address of the endpoint.
|
||||
func (endpoint *VhostUserEndpoint) PciAddr() string {
|
||||
return endpoint.PCIAddr
|
||||
}
|
||||
|
||||
// NetworkPair returns the network pair of the endpoint.
|
||||
func (endpoint *VhostUserEndpoint) NetworkPair() *NetworkInterfacePair {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Attach for vhostuser endpoint
|
||||
func (endpoint *VhostUserEndpoint) Attach(h hypervisor) error {
|
||||
networkLogger().WithField("endpoint-type", "vhostuser").Info("Attaching endpoint")
|
||||
|
||||
// Generate a unique ID to be used for hypervisor commandline fields
|
||||
randBytes, err := utils.GenerateRandomBytes(8)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
id := hex.EncodeToString(randBytes)
|
||||
|
||||
d := config.VhostUserDeviceAttrs{
|
||||
DevID: id,
|
||||
SocketPath: endpoint.SocketPath,
|
||||
MacAddress: endpoint.HardAddr,
|
||||
Type: config.VhostUserNet,
|
||||
}
|
||||
|
||||
return h.addDevice(d, vhostuserDev)
|
||||
}
|
||||
|
||||
// Detach for vhostuser endpoint
|
||||
func (endpoint *VhostUserEndpoint) Detach(netNsCreated bool, netNsPath string) error {
|
||||
networkLogger().WithField("endpoint-type", "vhostuser").Info("Detaching endpoint")
|
||||
return nil
|
||||
}
|
||||
|
||||
// HotAttach for vhostuser endpoint not supported yet
|
||||
func (endpoint *VhostUserEndpoint) HotAttach(h hypervisor) error {
|
||||
return fmt.Errorf("VhostUserEndpoint does not support Hot attach")
|
||||
}
|
||||
|
||||
// HotDetach for vhostuser endpoint not supported yet
|
||||
func (endpoint *VhostUserEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
|
||||
return fmt.Errorf("VhostUserEndpoint does not support Hot detach")
|
||||
}
|
||||
|
||||
// Create a vhostuser endpoint
|
||||
func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndpoint, error) {
|
||||
|
||||
vhostUserEndpoint := &VhostUserEndpoint{
|
||||
SocketPath: socket,
|
||||
HardAddr: netInfo.Iface.HardwareAddr.String(),
|
||||
IfaceName: netInfo.Iface.Name,
|
||||
EndpointType: VhostUserEndpointType,
|
||||
}
|
||||
return vhostUserEndpoint, nil
|
||||
}
|
||||
|
||||
// findVhostUserNetSocketPath checks if an interface is a dummy placeholder
|
||||
// for a vhost-user socket, and if it is it returns the path to the socket
|
||||
func findVhostUserNetSocketPath(netInfo NetworkInfo) (string, error) {
|
||||
if netInfo.Iface.Name == "lo" {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// check for socket file existence at known location.
|
||||
for _, addr := range netInfo.Addrs {
|
||||
socketPath := fmt.Sprintf(hostSocketSearchPath, addr.IPNet.IP)
|
||||
if _, err := os.Stat(socketPath); err == nil {
|
||||
return socketPath, nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// vhostUserSocketPath returns the path of the socket discovered. This discovery
|
||||
// will vary depending on the type of vhost-user socket.
|
||||
// Today only VhostUserNetDevice is supported.
|
||||
func vhostUserSocketPath(info interface{}) (string, error) {
|
||||
|
||||
switch v := info.(type) {
|
||||
case NetworkInfo:
|
||||
return findVhostUserNetSocketPath(v)
|
||||
default:
|
||||
return "", nil
|
||||
}
|
||||
|
||||
}
|
||||
144
virtcontainers/virtual_endpoint.go
Normal file
144
virtcontainers/virtual_endpoint.go
Normal file
@ -0,0 +1,144 @@
|
||||
// Copyright (c) 2018 Intel Corporation
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
package virtcontainers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/containernetworking/plugins/pkg/ns"
|
||||
)
|
||||
|
||||
// VirtualEndpoint gathers a network pair and its properties.
|
||||
type VirtualEndpoint struct {
|
||||
NetPair NetworkInterfacePair
|
||||
EndpointProperties NetworkInfo
|
||||
Physical bool
|
||||
EndpointType EndpointType
|
||||
PCIAddr string
|
||||
}
|
||||
|
||||
func createVirtualNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel) (*VirtualEndpoint, error) {
|
||||
if idx < 0 {
|
||||
return &VirtualEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx)
|
||||
}
|
||||
|
||||
netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
endpoint := &VirtualEndpoint{
|
||||
// TODO This is too specific. We may need to create multiple
|
||||
// end point types here and then decide how to connect them
|
||||
// at the time of hypervisor attach and not here
|
||||
NetPair: netPair,
|
||||
EndpointType: VirtualEndpointType,
|
||||
}
|
||||
if ifName != "" {
|
||||
endpoint.NetPair.VirtIface.Name = ifName
|
||||
}
|
||||
|
||||
return endpoint, nil
|
||||
}
|
||||
|
||||
// Properties returns properties for the veth interface in the network pair.
|
||||
func (endpoint *VirtualEndpoint) Properties() NetworkInfo {
|
||||
return endpoint.EndpointProperties
|
||||
}
|
||||
|
||||
// Name returns name of the veth interface in the network pair.
|
||||
func (endpoint *VirtualEndpoint) Name() string {
|
||||
return endpoint.NetPair.VirtIface.Name
|
||||
}
|
||||
|
||||
// HardwareAddr returns the mac address that is assigned to the tap interface
|
||||
// in th network pair.
|
||||
func (endpoint *VirtualEndpoint) HardwareAddr() string {
|
||||
return endpoint.NetPair.TAPIface.HardAddr
|
||||
}
|
||||
|
||||
// Type identifies the endpoint as a virtual endpoint.
|
||||
func (endpoint *VirtualEndpoint) Type() EndpointType {
|
||||
return endpoint.EndpointType
|
||||
}
|
||||
|
||||
// PciAddr returns the PCI address of the endpoint.
|
||||
func (endpoint *VirtualEndpoint) PciAddr() string {
|
||||
return endpoint.PCIAddr
|
||||
}
|
||||
|
||||
// NetworkPair returns the network pair of the endpoint.
|
||||
func (endpoint *VirtualEndpoint) NetworkPair() *NetworkInterfacePair {
|
||||
return &endpoint.NetPair
|
||||
}
|
||||
|
||||
// SetProperties sets the properties for the endpoint.
|
||||
func (endpoint *VirtualEndpoint) SetProperties(properties NetworkInfo) {
|
||||
endpoint.EndpointProperties = properties
|
||||
}
|
||||
|
||||
// Attach for virtual endpoint bridges the network pair and adds the
|
||||
// tap interface of the network pair to the hypervisor.
|
||||
func (endpoint *VirtualEndpoint) Attach(h hypervisor) error {
|
||||
networkLogger().WithField("endpoint-type", "virtual").Info("Attaching endpoint")
|
||||
|
||||
if err := xconnectVMNetwork(endpoint, true, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil {
|
||||
networkLogger().WithError(err).Error("Error bridging virtual endpoint")
|
||||
return err
|
||||
}
|
||||
|
||||
return h.addDevice(endpoint, netDev)
|
||||
}
|
||||
|
||||
// Detach for the virtual endpoint tears down the tap and bridge
|
||||
// created for the veth interface.
|
||||
func (endpoint *VirtualEndpoint) Detach(netNsCreated bool, netNsPath string) error {
|
||||
// The network namespace would have been deleted at this point
|
||||
// if it has not been created by virtcontainers.
|
||||
if !netNsCreated {
|
||||
return nil
|
||||
}
|
||||
|
||||
networkLogger().WithField("endpoint-type", "virtual").Info("Detaching endpoint")
|
||||
|
||||
return doNetNS(netNsPath, func(_ ns.NetNS) error {
|
||||
return xconnectVMNetwork(endpoint, false, 0, false)
|
||||
})
|
||||
}
|
||||
|
||||
// HotAttach for the virtual endpoint uses hot plug device
|
||||
func (endpoint *VirtualEndpoint) HotAttach(h hypervisor) error {
|
||||
networkLogger().Info("Hot attaching virtual endpoint")
|
||||
if err := xconnectVMNetwork(endpoint, true, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil {
|
||||
networkLogger().WithError(err).Error("Error bridging virtual ep")
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := h.hotplugAddDevice(endpoint, netDev); err != nil {
|
||||
networkLogger().WithError(err).Error("Error attach virtual ep")
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HotDetach for the virtual endpoint uses hot pull device
|
||||
func (endpoint *VirtualEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
|
||||
if !netNsCreated {
|
||||
return nil
|
||||
}
|
||||
networkLogger().Info("Hot detaching virtual endpoint")
|
||||
if err := doNetNS(netNsPath, func(_ ns.NetNS) error {
|
||||
return xconnectVMNetwork(endpoint, false, 0, h.hypervisorConfig().DisableVhostNet)
|
||||
}); err != nil {
|
||||
networkLogger().WithError(err).Warn("Error un-bridging virtual ep")
|
||||
}
|
||||
|
||||
if _, err := h.hotplugRemoveDevice(endpoint, netDev); err != nil {
|
||||
networkLogger().WithError(err).Error("Error detach virtual ep")
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user