github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-06-26 07:22:20 +00:00
Code Issues Projects Releases Wiki Activity

runtime: add more traces for network

Browse Source 
Add traces for all the endpoinnt types
and the main interface functions.
Record errors for some traces.

Fixes: #1956

Signed-off-by: bin <bin@hyper.sh>
This commit is contained in:
bin 2021-06-02 17:53:30 +08:00
parent a57118d03a
commit 784025bb08
12 changed files with 158 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/runtime/virtcontainers/bridgedmacvlan_endpoint.go
View File

@ -14,6 +14,8 @@ import (
vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
)
var macvlanTrace = getNetworkTrace(BridgedMacvlanEndpointType)
// BridgedMacvlanEndpoint represents a macvlan endpoint that is bridged to the VM
type BridgedMacvlanEndpoint struct {
NetPair NetworkInterfacePair
@ -89,6 +91,9 @@ func (endpoint *BridgedMacvlanEndpoint) NetworkPair() *NetworkInterfacePair {
// Attach for virtual endpoint bridges the network pair and adds the
// tap interface of the network pair to the hypervisor.
func (endpoint *BridgedMacvlanEndpoint) Attach(ctx context.Context, s *Sandbox) error {
span, ctx := macvlanTrace(ctx, "Attach", endpoint)
defer span.End()
h := s.hypervisor
if err := xConnectVMNetwork(ctx, endpoint, h); err != nil {
networkLogger().WithError(err).Error("Error bridging virtual ep")
@ -107,8 +112,11 @@ func (endpoint *BridgedMacvlanEndpoint) Detach(ctx context.Context, netNsCreated
return nil
}
span, ctx := macvlanTrace(ctx, "Detach", endpoint)
defer span.End()
return doNetNS(netNsPath, func(_ ns.NetNS) error {
return xDisconnectVMNetwork(endpoint)
return xDisconnectVMNetwork(ctx, endpoint)
})
}

10
src/runtime/virtcontainers/ipvlan_endpoint.go
View File

@ -14,6 +14,8 @@ import (
vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
)
var ipvlanTrace = getNetworkTrace(IPVlanEndpointType)
// IPVlanEndpoint represents a ipvlan endpoint that is bridged to the VM
type IPVlanEndpoint struct {
NetPair NetworkInterfacePair
@ -92,6 +94,9 @@ func (endpoint *IPVlanEndpoint) NetworkPair() *NetworkInterfacePair {
// Attach for virtual endpoint bridges the network pair and adds the
// tap interface of the network pair to the hypervisor.
func (endpoint *IPVlanEndpoint) Attach(ctx context.Context, s *Sandbox) error {
span, ctx := ipvlanTrace(ctx, "Attach", endpoint)
defer span.End()
h := s.hypervisor
if err := xConnectVMNetwork(ctx, endpoint, h); err != nil {
networkLogger().WithError(err).Error("Error bridging virtual ep")
@ -110,8 +115,11 @@ func (endpoint *IPVlanEndpoint) Detach(ctx context.Context, netNsCreated bool, n
return nil
}
span, ctx := ipvlanTrace(ctx, "Detach", endpoint)
defer span.End()
return doNetNS(netNsPath, func(_ ns.NetNS) error {
return xDisconnectVMNetwork(endpoint)
return xDisconnectVMNetwork(ctx, endpoint)
})
}

2
src/runtime/virtcontainers/kata_agent.go
View File

@ -812,7 +812,7 @@ func (k *kataAgent) startSandbox(ctx context.Context, sandbox *Sandbox) error {
}
// Setup network interfaces and routes
interfaces, routes, neighs, err := generateVCNetworkStructures(sandbox.networkNS)
interfaces, routes, neighs, err := generateVCNetworkStructures(ctx, sandbox.networkNS)
if err != nil {
return err
}

5
src/runtime/virtcontainers/macvtap_endpoint.go
View File

@ -14,6 +14,8 @@ import (
vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
)
var macvtapTrace = getNetworkTrace(MacvtapEndpointType)
// MacvtapEndpoint represents a macvtap endpoint
type MacvtapEndpoint struct {
EndpointProperties NetworkInfo
@ -62,6 +64,9 @@ func (endpoint *MacvtapEndpoint) SetProperties(properties NetworkInfo) {
// Attach for macvtap endpoint passes macvtap device to the hypervisor.
func (endpoint *MacvtapEndpoint) Attach(ctx context.Context, s *Sandbox) error {
var err error
span, ctx := macvtapTrace(ctx, "Attach", endpoint)
defer span.End()
h := s.hypervisor
endpoint.VMFds, err = createMacvtapFds(endpoint.EndpointProperties.Iface.Index, int(h.hypervisorConfig().NumVCPUs))

80
src/runtime/virtcontainers/network.go
View File

@ -421,6 +421,11 @@ func getLinkByName(netHandle *netlink.Handle, name string, expectedLink netlink.
// The endpoint type should dictate how the connection needs to happen.
func xConnectVMNetwork(ctx context.Context, endpoint Endpoint, h hypervisor) error {
var err error
span, ctx := networkTrace(ctx, "xConnectVMNetwork", endpoint)
defer closeSpan(span, err)
netPair := endpoint.NetworkPair()
queues := 0
@ -443,17 +448,23 @@ func xConnectVMNetwork(ctx context.Context, endpoint Endpoint, h hypervisor) err
switch netPair.NetInterworkingModel {
case NetXConnectMacVtapModel:
networkLogger().Info("connect macvtap to VM network")
return tapNetworkPair(endpoint, queues, disableVhostNet)
err = tapNetworkPair(ctx, endpoint, queues, disableVhostNet)
case NetXConnectTCFilterModel:
networkLogger().Info("connect TCFilter to VM network")
return setupTCFiltering(endpoint, queues, disableVhostNet)
err = setupTCFiltering(ctx, endpoint, queues, disableVhostNet)
default:
return fmt.Errorf("Invalid internetworking model")
err = fmt.Errorf("Invalid internetworking model")
}
return err
}
// The endpoint type should dictate how the disconnection needs to happen.
func xDisconnectVMNetwork(endpoint Endpoint) error {
func xDisconnectVMNetwork(ctx context.Context, endpoint Endpoint) error {
var err error
span, ctx := networkTrace(ctx, "xDisconnectVMNetwork", endpoint)
defer closeSpan(span, err)
netPair := endpoint.NetworkPair()
if netPair.NetInterworkingModel == NetXConnectDefaultModel {
@ -462,12 +473,13 @@ func xDisconnectVMNetwork(endpoint Endpoint) error {
switch netPair.NetInterworkingModel {
case NetXConnectMacVtapModel:
return untapNetworkPair(endpoint)
err = untapNetworkPair(ctx, endpoint)
case NetXConnectTCFilterModel:
return removeTCFiltering(endpoint)
err = removeTCFiltering(ctx, endpoint)
default:
return fmt.Errorf("Invalid internetworking model")
err = fmt.Errorf("Invalid internetworking model")
}
return err
}
func createMacvtapFds(linkIndex int, queues int) ([]*os.File, error) {
@ -549,7 +561,10 @@ func setIPs(link netlink.Link, addrs []netlink.Addr) error {
return nil
}
func tapNetworkPair(endpoint Endpoint, queues int, disableVhostNet bool) error {
func tapNetworkPair(ctx context.Context, endpoint Endpoint, queues int, disableVhostNet bool) error {
span, _ := networkTrace(ctx, "tapNetworkPair", endpoint)
defer span.End()
netHandle, err := netlink.NewHandle()
if err != nil {
return err
@ -643,7 +658,10 @@ func tapNetworkPair(endpoint Endpoint, queues int, disableVhostNet bool) error {
return nil
}
func setupTCFiltering(endpoint Endpoint, queues int, disableVhostNet bool) error {
func setupTCFiltering(ctx context.Context, endpoint Endpoint, queues int, disableVhostNet bool) error {
span, _ := networkTrace(ctx, "setupTCFiltering", endpoint)
defer span.End()
netHandle, err := netlink.NewHandle()
if err != nil {
return err
@ -815,7 +833,10 @@ func removeQdiscIngress(link netlink.Link) error {
return nil
}
func untapNetworkPair(endpoint Endpoint) error {
func untapNetworkPair(ctx context.Context, endpoint Endpoint) error {
span, _ := networkTrace(ctx, "untapNetworkPair", endpoint)
defer span.End()
netHandle, err := netlink.NewHandle()
if err != nil {
return err
@ -856,7 +877,10 @@ func untapNetworkPair(endpoint Endpoint) error {
return err
}
func removeTCFiltering(endpoint Endpoint) error {
func removeTCFiltering(ctx context.Context, endpoint Endpoint) error {
span, _ := networkTrace(ctx, "removeTCFiltering", endpoint)
defer span.End()
netHandle, err := netlink.NewHandle()
if err != nil {
return err
@ -952,10 +976,12 @@ func deleteNetNS(netNSPath string) error {
return nil
}
func generateVCNetworkStructures(networkNS NetworkNamespace) ([]*pbTypes.Interface, []*pbTypes.Route, []*pbTypes.ARPNeighbor, error) {
func generateVCNetworkStructures(ctx context.Context, networkNS NetworkNamespace) ([]*pbTypes.Interface, []*pbTypes.Route, []*pbTypes.ARPNeighbor, error) {
if networkNS.NetNsPath == "" {
return nil, nil, nil, nil
}
span, _ := networkTrace(ctx, "generateVCNetworkStructures", nil)
defer span.End()
var routes []*pbTypes.Route
var ifaces []*pbTypes.Interface
@ -1261,11 +1287,31 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
type Network struct {
}
func (n *Network) trace(ctx context.Context, name string) (otelTrace.Span, context.Context) {
tracer := otel.Tracer("kata")
ctx, span := tracer.Start(ctx, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "network")))
var networkTrace = getNetworkTrace("")
return span, ctx
func (n *Network) trace(ctx context.Context, name string) (otelTrace.Span, context.Context) {
return networkTrace(ctx, name, nil)
}
func getNetworkTrace(networkType EndpointType) func(ctx context.Context, name string, endpoint interface{}) (otelTrace.Span, context.Context) {
return func(ctx context.Context, name string, endpoint interface{}) (otelTrace.Span, context.Context) {
tracer := otel.Tracer("kata")
ctx, span := tracer.Start(ctx, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "network")))
if networkType != "" {
span.SetAttributes(otelLabel.Any("type", string(networkType)))
}
if endpoint != nil {
span.SetAttributes(otelLabel.Any("endpoint", endpoint))
}
return span, ctx
}
}
func closeSpan(span otelTrace.Span, err error) {
if err != nil {
span.SetAttributes(otelLabel.Any("error", err))
}
span.End()
}
// Run runs a callback in the specified network namespace.
@ -1288,6 +1334,8 @@ func (n *Network) Add(ctx context.Context, config *NetworkConfig, s *Sandbox, ho
if err != nil {
return endpoints, err
}
span.SetAttributes(otelLabel.Any("endpoints", endpoints))
span.SetAttributes(otelLabel.Bool("hotplug", hotplug))
err = doNetNS(config.NetNSPath, func(_ ns.NetNS) error {
for _, endpoint := range endpoints {

15
src/runtime/virtcontainers/network_test.go
View File

@ -6,6 +6,7 @@
package virtcontainers
import (
"context"
"fmt"
"net"
"reflect"
@ -74,7 +75,7 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) {
nns := NetworkNamespace{NetNsPath: "foobar", NetNsCreated: true, Endpoints: endpoints}
resInterfaces, resRoutes, resNeighs, err := generateVCNetworkStructures(nns)
resInterfaces, resRoutes, resNeighs, err := generateVCNetworkStructures(context.Background(), nns)
//
// Build expected results:
@ -275,10 +276,10 @@ func TestTcRedirectNetwork(t *testing.T) {
err = netHandle.LinkSetUp(link)
assert.NoError(err)
err = setupTCFiltering(endpoint, 1, true)
err = setupTCFiltering(context.Background(), endpoint, 1, true)
assert.NoError(err)
err = removeTCFiltering(endpoint)
err = removeTCFiltering(context.Background(), endpoint)
assert.NoError(err)
// Remove the veth created for testing.
@ -313,7 +314,7 @@ func TestRxRateLimiter(t *testing.T) {
err = netHandle.LinkSetUp(link)
assert.NoError(err)
err = setupTCFiltering(endpoint, 1, true)
err = setupTCFiltering(context.Background(), endpoint, 1, true)
assert.NoError(err)
// 10Mb
@ -327,7 +328,7 @@ func TestRxRateLimiter(t *testing.T) {
err = removeRxRateLimiter(endpoint, currentNS.Path())
assert.NoError(err)
err = removeTCFiltering(endpoint)
err = removeTCFiltering(context.Background(), endpoint)
assert.NoError(err)
// Remove the veth created for testing.
@ -362,7 +363,7 @@ func TestTxRateLimiter(t *testing.T) {
err = netHandle.LinkSetUp(link)
assert.NoError(err)
err = setupTCFiltering(endpoint, 1, true)
err = setupTCFiltering(context.Background(), endpoint, 1, true)
assert.NoError(err)
// 10Mb
@ -376,7 +377,7 @@ func TestTxRateLimiter(t *testing.T) {
err = removeTxRateLimiter(endpoint, currentNS.Path())
assert.NoError(err)
err = removeTCFiltering(endpoint)
err = removeTCFiltering(context.Background(), endpoint)
assert.NoError(err)
// Remove the veth created for testing.

8
src/runtime/virtcontainers/physical_endpoint.go
View File

@ -21,6 +21,8 @@ import (
"github.com/safchain/ethtool"
)
var physicalTrace = getNetworkTrace(PhysicalEndpointType)
// PhysicalEndpoint gathers a physical network interface and its properties
type PhysicalEndpoint struct {
IfaceName string
@ -76,6 +78,9 @@ func (endpoint *PhysicalEndpoint) NetworkPair() *NetworkInterfacePair {
// Attach for physical endpoint binds the physical network interface to
// vfio-pci and adds device to the hypervisor with vfio-passthrough.
func (endpoint *PhysicalEndpoint) Attach(ctx context.Context, s *Sandbox) error {
span, ctx := physicalTrace(ctx, "Attach", endpoint)
defer span.End()
// Unbind physical interface from host driver and bind to vfio
// so that it can be passed to qemu.
vfioPath, err := bindNICToVFIO(endpoint)
@ -103,6 +108,9 @@ func (endpoint *PhysicalEndpoint) Attach(ctx context.Context, s *Sandbox) error
// Detach for physical endpoint unbinds the physical network interface from vfio-pci
// and binds it back to the saved host driver.
func (endpoint *PhysicalEndpoint) Detach(ctx context.Context, netNsCreated bool, netNsPath string) error {
span, _ := physicalTrace(ctx, "Detach", endpoint)
defer span.End()
// Bind back the physical network interface to host.
// We need to do this even if a new network namespace has not
// been created by virtcontainers.

3
src/runtime/virtcontainers/sandbox.go
View File

@ -760,6 +760,9 @@ func (s *Sandbox) createNetwork(ctx context.Context) error {
NetNsCreated: s.config.NetworkConfig.NetNsCreated,
}
span.SetAttributes(otelLabel.Any("networkNS", s.networkNS))
span.SetAttributes(otelLabel.Any("NetworkConfig", s.config.NetworkConfig))
// In case there is a factory, network interfaces are hotplugged
// after vm is started.
if s.factory == nil {

13
src/runtime/virtcontainers/tap_endpoint.go
View File

@ -17,6 +17,8 @@ import (
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/uuid"
)
var tapTrace = getNetworkTrace(TapEndpointType)
// TapEndpoint represents just a tap endpoint
type TapEndpoint struct {
TapInterface TapInterface
@ -78,6 +80,9 @@ func (endpoint *TapEndpoint) Detach(ctx context.Context, netNsCreated bool, netN
return nil
}
span, _ := tapTrace(ctx, "Detach", endpoint)
defer span.End()
networkLogger().WithField("endpoint-type", TapEndpointType).Info("Detaching endpoint")
return doNetNS(netNsPath, func(_ ns.NetNS) error {
return unTapNetwork(endpoint.TapInterface.TAPIface.Name)
@ -87,6 +92,10 @@ func (endpoint *TapEndpoint) Detach(ctx context.Context, netNsCreated bool, netN
// HotAttach for the tap endpoint uses hot plug device
func (endpoint *TapEndpoint) HotAttach(ctx context.Context, h hypervisor) error {
networkLogger().Info("Hot attaching tap endpoint")
span, ctx := tapTrace(ctx, "HotAttach", endpoint)
defer span.End()
if err := tapNetwork(endpoint, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil {
networkLogger().WithError(err).Error("Error bridging tap ep")
return err
@ -102,6 +111,10 @@ func (endpoint *TapEndpoint) HotAttach(ctx context.Context, h hypervisor) error
// HotDetach for the tap endpoint uses hot pull device
func (endpoint *TapEndpoint) HotDetach(ctx context.Context, h hypervisor, netNsCreated bool, netNsPath string) error {
networkLogger().Info("Hot detaching tap endpoint")
span, ctx := tapTrace(ctx, "HotDetach", endpoint)
defer span.End()
if err := doNetNS(netNsPath, func(_ ns.NetNS) error {
return unTapNetwork(endpoint.TapInterface.TAPIface.Name)
}); err != nil {

17
src/runtime/virtcontainers/tuntap_endpoint.go
View File

@ -18,6 +18,8 @@ import (
vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
)
var tuntapTrace = getNetworkTrace(TuntapEndpointType)
// TuntapEndpoint represents just a tap endpoint
type TuntapEndpoint struct {
NetPair NetworkInterfacePair
@ -71,11 +73,15 @@ func (endpoint *TuntapEndpoint) SetProperties(properties NetworkInfo) {
// Attach for tap endpoint adds the tap interface to the hypervisor.
func (endpoint *TuntapEndpoint) Attach(ctx context.Context, s *Sandbox) error {
span, ctx := tuntapTrace(ctx, "Attach", endpoint)
defer span.End()
h := s.hypervisor
if err := xConnectVMNetwork(ctx, endpoint, h); err != nil {
networkLogger().WithError(err).Error("Error bridging virtual endpoint")
return err
}
return h.addDevice(ctx, endpoint, netDev)
}
@ -85,6 +91,9 @@ func (endpoint *TuntapEndpoint) Detach(ctx context.Context, netNsCreated bool, n
return nil
}
span, _ := tuntapTrace(ctx, "Detach", endpoint)
defer span.End()
networkLogger().WithField("endpoint-type", TuntapEndpointType).Info("Detaching endpoint")
return doNetNS(netNsPath, func(_ ns.NetNS) error {
return unTuntapNetwork(endpoint.TuntapInterface.TAPIface.Name)
@ -94,6 +103,10 @@ func (endpoint *TuntapEndpoint) Detach(ctx context.Context, netNsCreated bool, n
// HotAttach for the tap endpoint uses hot plug device
func (endpoint *TuntapEndpoint) HotAttach(ctx context.Context, h hypervisor) error {
networkLogger().Info("Hot attaching tap endpoint")
span, ctx := tuntapTrace(ctx, "HotAttach", endpoint)
defer span.End()
if err := tuntapNetwork(endpoint, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil {
networkLogger().WithError(err).Error("Error bridging tap ep")
return err
@ -109,6 +122,10 @@ func (endpoint *TuntapEndpoint) HotAttach(ctx context.Context, h hypervisor) err
// HotDetach for the tap endpoint uses hot pull device
func (endpoint *TuntapEndpoint) HotDetach(ctx context.Context, h hypervisor, netNsCreated bool, netNsPath string) error {
networkLogger().Info("Hot detaching tap endpoint")
span, ctx := tuntapTrace(ctx, "HotDetach", endpoint)
defer span.End()
if err := doNetNS(netNsPath, func(_ ns.NetNS) error {
return unTuntapNetwork(endpoint.TuntapInterface.TAPIface.Name)
}); err != nil {

18
src/runtime/virtcontainers/veth_endpoint.go
View File

@ -14,6 +14,8 @@ import (
vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
)
var vethTrace = getNetworkTrace(VethEndpointType)
// VethEndpoint gathers a network pair and its properties.
type VethEndpoint struct {
NetPair NetworkInterfacePair
@ -92,6 +94,9 @@ func (endpoint *VethEndpoint) SetProperties(properties NetworkInfo) {
// Attach for veth endpoint bridges the network pair and adds the
// tap interface of the network pair to the hypervisor.
func (endpoint *VethEndpoint) Attach(ctx context.Context, s *Sandbox) error {
span, ctx := vethTrace(ctx, "Attach", endpoint)
defer span.End()
h := s.hypervisor
if err := xConnectVMNetwork(ctx, endpoint, h); err != nil {
networkLogger().WithError(err).Error("Error bridging virtual endpoint")
@ -110,13 +115,19 @@ func (endpoint *VethEndpoint) Detach(ctx context.Context, netNsCreated bool, net
return nil
}
span, ctx := vethTrace(ctx, "Detach", endpoint)
defer span.End()
return doNetNS(netNsPath, func(_ ns.NetNS) error {
return xDisconnectVMNetwork(endpoint)
return xDisconnectVMNetwork(ctx, endpoint)
})
}
// HotAttach for the veth endpoint uses hot plug device
func (endpoint *VethEndpoint) HotAttach(ctx context.Context, h hypervisor) error {
span, ctx := vethTrace(ctx, "HotAttach", endpoint)
defer span.End()
if err := xConnectVMNetwork(ctx, endpoint, h); err != nil {
networkLogger().WithError(err).Error("Error bridging virtual ep")
return err
@ -135,8 +146,11 @@ func (endpoint *VethEndpoint) HotDetach(ctx context.Context, h hypervisor, netNs
return nil
}
span, ctx := vethTrace(ctx, "HotDetach", endpoint)
defer span.End()
if err := doNetNS(netNsPath, func(_ ns.NetNS) error {
return xDisconnectVMNetwork(endpoint)
return xDisconnectVMNetwork(ctx, endpoint)
}); err != nil {
networkLogger().WithError(err).Warn("Error un-bridging virtual ep")
}

5
src/runtime/virtcontainers/vhostuser_endpoint.go
View File

@ -23,6 +23,8 @@ import (
// using this path.
const hostSocketSearchPath = "/tmp/vhostuser_%s/vhu.sock"
var vhostuserTrace = getNetworkTrace(VhostUserEndpointType)
// VhostUserEndpoint represents a vhost-user socket based network interface
type VhostUserEndpoint struct {
// Path to the vhost-user socket on the host system
@ -77,6 +79,9 @@ func (endpoint *VhostUserEndpoint) NetworkPair() *NetworkInterfacePair {
// Attach for vhostuser endpoint
func (endpoint *VhostUserEndpoint) Attach(ctx context.Context, s *Sandbox) error {
span, ctx := vhostuserTrace(ctx, "Attach", endpoint)
defer span.End()
// Generate a unique ID to be used for hypervisor commandline fields
randBytes, err := utils.GenerateRandomBytes(8)
if err != nil {