virtcontainers: add sandbox hotplug network API

Add sandbox hotplug network API to meet design

Signed-off-by: Ruidong Cao <caoruidong@huawei.com>

virtcontainers/agent.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"syscall"
 
+	"github.com/kata-containers/agent/protocols/grpc"
 	"github.com/mitchellh/mapstructure"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 )
@@ -221,4 +222,16 @@ type agent interface {
 
 	// reseedRNG will reseed the guest random number generator
 	reseedRNG(data []byte) error
+
+	// updateInterface will tell the agent to update a nic for an existed Sandbox.
+	updateInterface(inf *grpc.Interface) (*grpc.Interface, error)
+
+	// listInterfaces will tell the agent to list interfaces of an existed Sandbox
+	listInterfaces() ([]*grpc.Interface, error)
+
+	// updateRoutes will tell the agent to update route table for an existed Sandbox.
+	updateRoutes(routes []*grpc.Route) ([]*grpc.Route, error)
+
+	// listRoutes will tell the agent to list routes of an existed Sandbox
+	listRoutes() ([]*grpc.Route, error)
 }

virtcontainers/api.go

@@ -10,6 +10,7 @@ import (
 	"runtime"
 	"syscall"
 
+	"github.com/kata-containers/agent/protocols/grpc"
 	deviceApi "github.com/kata-containers/runtime/virtcontainers/device/api"
 	deviceConfig "github.com/kata-containers/runtime/virtcontainers/device/config"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
@@ -782,3 +783,93 @@ func AddDevice(sandboxID string, info deviceConfig.DeviceInfo) (deviceApi.Device
 
 	return s.AddDevice(info)
 }
+
+func toggleInterface(sandboxID string, inf *grpc.Interface, add bool) (*grpc.Interface, error) {
+	if sandboxID == "" {
+		return nil, errNeedSandboxID
+	}
+
+	lockFile, err := rwLockSandbox(sandboxID)
+	if err != nil {
+		return nil, err
+	}
+	defer unlockSandbox(lockFile)
+
+	s, err := fetchSandbox(sandboxID)
+	if err != nil {
+		return nil, err
+	}
+	if add {
+		return s.AddInterface(inf)
+	}
+	return s.RemoveInterface(inf)
+}
+
+// AddInterface is the virtcontainers add interface entry point.
+func AddInterface(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error) {
+	return toggleInterface(sandboxID, inf, true)
+}
+
+// RemoveInterface is the virtcontainers remove interface entry point.
+func RemoveInterface(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error) {
+	return toggleInterface(sandboxID, inf, false)
+}
+
+// ListInterfaces is the virtcontainers list interfaces entry point.
+func ListInterfaces(sandboxID string) ([]*grpc.Interface, error) {
+	if sandboxID == "" {
+		return nil, errNeedSandboxID
+	}
+
+	lockFile, err := rLockSandbox(sandboxID)
+	if err != nil {
+		return nil, err
+	}
+	defer unlockSandbox(lockFile)
+
+	s, err := fetchSandbox(sandboxID)
+	if err != nil {
+		return nil, err
+	}
+
+	return s.ListInterfaces()
+}
+
+// UpdateRoutes is the virtcontainers update routes entry point.
+func UpdateRoutes(sandboxID string, routes []*grpc.Route) ([]*grpc.Route, error) {
+	if sandboxID == "" {
+		return nil, errNeedSandboxID
+	}
+
+	lockFile, err := rwLockSandbox(sandboxID)
+	if err != nil {
+		return nil, err
+	}
+	defer unlockSandbox(lockFile)
+
+	s, err := fetchSandbox(sandboxID)
+	if err != nil {
+		return nil, err
+	}
+	return s.UpdateRoutes(routes)
+}
+
+// ListRoutes is the virtcontainers list routes entry point.
+func ListRoutes(sandboxID string) ([]*grpc.Route, error) {
+	if sandboxID == "" {
+		return nil, errNeedSandboxID
+	}
+
+	lockFile, err := rLockSandbox(sandboxID)
+	if err != nil {
+		return nil, err
+	}
+	defer unlockSandbox(lockFile)
+
+	s, err := fetchSandbox(sandboxID)
+	if err != nil {
+		return nil, err
+	}
+
+	return s.ListRoutes()
+}

virtcontainers/hyperstart_agent.go

@@ -18,6 +18,7 @@ import (
 	"github.com/vishvananda/netlink"
 
 	proxyClient "github.com/clearcontainers/proxy/client"
+	"github.com/kata-containers/agent/protocols/grpc"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/hyperstart"
 	ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter"
@@ -852,6 +853,26 @@ func (h *hyper) onlineCPUMem(cpus uint32) error {
 	return nil
 }
 
+func (h *hyper) updateInterface(inf *grpc.Interface) (*grpc.Interface, error) {
+	// hyperstart-agent does not support update interface
+	return nil, nil
+}
+
+func (h *hyper) listInterfaces() ([]*grpc.Interface, error) {
+	// hyperstart-agent does not support list interfaces
+	return nil, nil
+}
+
+func (h *hyper) updateRoutes(routes []*grpc.Route) ([]*grpc.Route, error) {
+	// hyperstart-agent does not support update routes
+	return nil, nil
+}
+
+func (h *hyper) listRoutes() ([]*grpc.Route, error) {
+	// hyperstart-agent does not support list routes
+	return nil, nil
+}
+
 func (h *hyper) check() error {
 	// hyperstart-agent does not support check
 	return nil

virtcontainers/implementation.go

@@ -12,6 +12,7 @@ package virtcontainers
 import (
 	"syscall"
 
+	"github.com/kata-containers/agent/protocols/grpc"
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
@@ -148,3 +149,28 @@ func (impl *VCImpl) ResumeContainer(sandboxID, containerID string) error {
 func (impl *VCImpl) AddDevice(sandboxID string, info config.DeviceInfo) (api.Device, error) {
 	return AddDevice(sandboxID, info)
 }
+
+// AddInterface implements the VC function of the same name.
+func (impl *VCImpl) AddInterface(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error) {
+	return AddInterface(sandboxID, inf)
+}
+
+// RemoveInterface implements the VC function of the same name.
+func (impl *VCImpl) RemoveInterface(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error) {
+	return RemoveInterface(sandboxID, inf)
+}
+
+// ListInterfaces implements the VC function of the same name.
+func (impl *VCImpl) ListInterfaces(sandboxID string) ([]*grpc.Interface, error) {
+	return ListInterfaces(sandboxID)
+}
+
+// UpdateRoutes implements the VC function of the same name.
+func (impl *VCImpl) UpdateRoutes(sandboxID string, routes []*grpc.Route) ([]*grpc.Route, error) {
+	return UpdateRoutes(sandboxID, routes)
+}
+
+// ListRoutes implements the VC function of the same name.
+func (impl *VCImpl) ListRoutes(sandboxID string) ([]*grpc.Route, error) {
+	return ListRoutes(sandboxID)
+}

virtcontainers/interfaces.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"syscall"
 
+	"github.com/kata-containers/agent/protocols/grpc"
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
@@ -45,6 +46,12 @@ type VC interface {
 	ResumeContainer(sandboxID, containerID string) error
 
 	AddDevice(sandboxID string, info config.DeviceInfo) (api.Device, error)
+
+	AddInterface(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error)
+	RemoveInterface(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error)
+	ListInterfaces(sandboxID string) ([]*grpc.Interface, error)
+	UpdateRoutes(sandboxID string, routes []*grpc.Route) ([]*grpc.Route, error)
+	ListRoutes(sandboxID string) ([]*grpc.Route, error)
 }
 
 // VCSandbox is the Sandbox interface
@@ -76,6 +83,12 @@ type VCSandbox interface {
 	IOStream(containerID, processID string) (io.WriteCloser, io.Reader, io.Reader, error)
 
 	AddDevice(info config.DeviceInfo) (api.Device, error)
+
+	AddInterface(inf *grpc.Interface) (*grpc.Interface, error)
+	RemoveInterface(inf *grpc.Interface) (*grpc.Interface, error)
+	ListInterfaces() ([]*grpc.Interface, error)
+	UpdateRoutes(routes []*grpc.Route) ([]*grpc.Route, error)
+	ListRoutes() ([]*grpc.Route, error)
 }
 
 // VCContainer is the Container interface

virtcontainers/kata_agent.go

@@ -440,6 +440,82 @@ func (k *kataAgent) generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*
 	return ifaces, routes, nil
 }
 
+func (k *kataAgent) updateInterface(ifc *grpc.Interface) (*grpc.Interface, error) {
+	// send update interface request
+	ifcReq := &grpc.UpdateInterfaceRequest{
+		Interface: ifc,
+	}
+	resultingInterface, err := k.sendReq(ifcReq)
+	if err != nil {
+		k.Logger().WithFields(logrus.Fields{
+			"interface-requested": fmt.Sprintf("%+v", ifc),
+			"resulting-interface": fmt.Sprintf("%+v", resultingInterface),
+		}).WithError(err).Error("update interface request failed")
+	}
+	if resultInterface, ok := resultingInterface.(*grpc.Interface); ok {
+		return resultInterface, err
+	}
+	return nil, err
+}
+
+func (k *kataAgent) updateInterfaces(interfaces []*grpc.Interface) error {
+	for _, ifc := range interfaces {
+		if _, err := k.updateInterface(ifc); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (k *kataAgent) updateRoutes(routes []*grpc.Route) ([]*grpc.Route, error) {
+	if routes != nil {
+		routesReq := &grpc.UpdateRoutesRequest{
+			Routes: &grpc.Routes{
+				Routes: routes,
+			},
+		}
+		resultingRoutes, err := k.sendReq(routesReq)
+		if err != nil {
+			k.Logger().WithFields(logrus.Fields{
+				"routes-requested": fmt.Sprintf("%+v", routes),
+				"resulting-routes": fmt.Sprintf("%+v", resultingRoutes),
+			}).WithError(err).Error("update routes request failed")
+		}
+		resultRoutes, ok := resultingRoutes.(*grpc.Routes)
+		if ok && resultRoutes != nil {
+			return resultRoutes.Routes, err
+		}
+		return nil, err
+	}
+	return nil, nil
+}
+
+func (k *kataAgent) listInterfaces() ([]*grpc.Interface, error) {
+	req := &grpc.ListInterfacesRequest{}
+	resultingInterfaces, err := k.sendReq(req)
+	if err != nil {
+		return nil, err
+	}
+	resultInterfaces, ok := resultingInterfaces.(*grpc.Interfaces)
+	if ok {
+		return resultInterfaces.Interfaces, err
+	}
+	return nil, err
+}
+
+func (k *kataAgent) listRoutes() ([]*grpc.Route, error) {
+	req := &grpc.ListRoutesRequest{}
+	resultingRoutes, err := k.sendReq(req)
+	if err != nil {
+		return nil, err
+	}
+	resultRoutes, ok := resultingRoutes.(*grpc.Routes)
+	if ok {
+		return resultRoutes.Routes, err
+	}
+	return nil, err
+}
+
 func (k *kataAgent) startProxy(sandbox *Sandbox) error {
 	var err error
 
@@ -515,37 +591,12 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error {
 	if err != nil {
 		return err
 	}
-	for _, ifc := range interfaces {
-		// send update interface request
-		ifcReq := &grpc.UpdateInterfaceRequest{
-			Interface: ifc,
-		}
-		resultingInterface, err := k.sendReq(ifcReq)
-		if err != nil {
-			k.Logger().WithFields(logrus.Fields{
-				"interface-requested": fmt.Sprintf("%+v", ifc),
-				"resulting-interface": fmt.Sprintf("%+v", resultingInterface),
-			}).WithError(err).Error("update interface request failed")
+	if err := k.updateInterfaces(interfaces); err != nil {
 		return err
 	}
-	}
-
-	if routes != nil {
-		routesReq := &grpc.UpdateRoutesRequest{
-			Routes: &grpc.Routes{
-				Routes: routes,
-			},
-		}
-
-		resultingRoutes, err := k.sendReq(routesReq)
-		if err != nil {
-			k.Logger().WithFields(logrus.Fields{
-				"routes-requested": fmt.Sprintf("%+v", routes),
-				"resulting-routes": fmt.Sprintf("%+v", resultingRoutes),
-			}).WithError(err).Error("update routes request failed")
+	if _, err := k.updateRoutes(routes); err != nil {
 		return err
 	}
-	}
 
 	sharedDir9pOptions = append(sharedDir9pOptions, fmt.Sprintf("msize=%d", sandbox.config.HypervisorConfig.Msize9p))
 
@@ -1335,6 +1386,12 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) {
 	k.reqHandlers["grpc.UpdateInterfaceRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) {
 		return k.client.UpdateInterface(ctx, req.(*grpc.UpdateInterfaceRequest), opts...)
 	}
+	k.reqHandlers["grpc.ListInterfacesRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) {
+		return k.client.ListInterfaces(ctx, req.(*grpc.ListInterfacesRequest), opts...)
+	}
+	k.reqHandlers["grpc.ListRoutesRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) {
+		return k.client.ListRoutes(ctx, req.(*grpc.ListRoutesRequest), opts...)
+	}
 	k.reqHandlers["grpc.OnlineCPUMemRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) {
 		return k.client.OnlineCPUMem(ctx, req.(*grpc.OnlineCPUMemRequest), opts...)
 	}

virtcontainers/network.go

@@ -156,6 +156,8 @@ type Endpoint interface {
 	SetProperties(NetworkInfo)
 	Attach(hypervisor) error
 	Detach(netNsCreated bool, netNsPath string) error
+	HotAttach(h hypervisor) error
+	HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error
 }
 
 // VirtualEndpoint gathers a network pair and its properties.
@@ -164,6 +166,7 @@ type VirtualEndpoint struct {
 	EndpointProperties NetworkInfo
 	Physical           bool
 	EndpointType       EndpointType
+	PCIAddr            string
 }
 
 // PhysicalEndpoint gathers a physical network interface and its properties
@@ -247,6 +250,41 @@ func (endpoint *VirtualEndpoint) Detach(netNsCreated bool, netNsPath string) err
 	})
 }
 
+// HotAttach for the virtual endpoint uses hot plug device
+func (endpoint *VirtualEndpoint) HotAttach(h hypervisor) error {
+	networkLogger().Info("Hot attaching virtual endpoint")
+	if err := xconnectVMNetwork(&(endpoint.NetPair), true); err != nil {
+		networkLogger().WithError(err).Error("Error bridging virtual ep")
+		return err
+	}
+
+	if _, err := h.hotplugAddDevice(*endpoint, netDev); err != nil {
+		networkLogger().WithError(err).Error("Error attach virtual ep")
+		return err
+	}
+	return nil
+}
+
+// HotDetach for the virtual endpoint uses hot pull device
+func (endpoint *VirtualEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
+	if !netNsCreated {
+		return nil
+	}
+	networkLogger().Info("Hot detaching virtual endpoint")
+	if err := doNetNS(netNsPath, func(_ ns.NetNS) error {
+		return xconnectVMNetwork(&(endpoint.NetPair), false)
+	}); err != nil {
+		networkLogger().WithError(err).Error("Error abridging virtual ep")
+		return err
+	}
+
+	if _, err := h.hotplugRemoveDevice(*endpoint, netDev); err != nil {
+		networkLogger().WithError(err).Error("Error detach virtual ep")
+		return err
+	}
+	return nil
+}
+
 // Properties returns the properties of the interface.
 func (endpoint *VhostUserEndpoint) Properties() NetworkInfo {
 	return endpoint.EndpointProperties
@@ -298,6 +336,16 @@ func (endpoint *VhostUserEndpoint) Detach(netNsCreated bool, netNsPath string) e
 	return nil
 }
 
+// HotAttach for vhostuser endpoint not supported yet
+func (endpoint *VhostUserEndpoint) HotAttach(h hypervisor) error {
+	return fmt.Errorf("VhostUserEndpoint does not support Hot attach")
+}
+
+// HotDetach for vhostuser endpoint not supported yet
+func (endpoint *VhostUserEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
+	return fmt.Errorf("VhostUserEndpoint does not support Hot detach")
+}
+
 // Create a vhostuser endpoint
 func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndpoint, error) {
 
@@ -367,6 +415,16 @@ func (endpoint *PhysicalEndpoint) Detach(netNsCreated bool, netNsPath string) er
 	return bindNICToHost(endpoint)
 }
 
+// HotAttach for physical endpoint not supported yet
+func (endpoint *PhysicalEndpoint) HotAttach(h hypervisor) error {
+	return fmt.Errorf("PhysicalEndpoint does not support Hot attach")
+}
+
+// HotDetach for physical endpoint not supported yet
+func (endpoint *PhysicalEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
+	return fmt.Errorf("PhysicalEndpoint does not support Hot detach")
+}
+
 // EndpointType identifies the type of the network endpoint.
 type EndpointType string
 

virtcontainers/noop_agent.go

@@ -8,6 +8,7 @@ package virtcontainers
 import (
 	"syscall"
 
+	"github.com/kata-containers/agent/protocols/grpc"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 )
 
@@ -96,6 +97,26 @@ func (n *noopAgent) onlineCPUMem(cpus uint32) error {
 	return nil
 }
 
+// updateInterface is the Noop agent Interface update implementation. It does nothing.
+func (n *noopAgent) updateInterface(inf *grpc.Interface) (*grpc.Interface, error) {
+	return nil, nil
+}
+
+// listInterfaces is the Noop agent Interfaces list implementation. It does nothing.
+func (n *noopAgent) listInterfaces() ([]*grpc.Interface, error) {
+	return nil, nil
+}
+
+// updateRoutes is the Noop agent Routes update implementation. It does nothing.
+func (n *noopAgent) updateRoutes(routes []*grpc.Route) ([]*grpc.Route, error) {
+	return nil, nil
+}
+
+// listRoutes is the Noop agent Routes list implementation. It does nothing.
+func (n *noopAgent) listRoutes() ([]*grpc.Route, error) {
+	return nil, nil
+}
+
 // check is the Noop agent health checker. It does nothing.
 func (n *noopAgent) check() error {
 	return nil

virtcontainers/pkg/vcmock/mock.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"syscall"
 
+	"github.com/kata-containers/agent/protocols/grpc"
 	vc "github.com/kata-containers/runtime/virtcontainers"
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
@@ -250,3 +251,48 @@ func (m *VCMock) AddDevice(sandboxID string, info config.DeviceInfo) (api.Device
 
 	return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID)
 }
+
+// AddInterface implements the VC function of the same name.
+func (m *VCMock) AddInterface(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error) {
+	if m.AddInterfaceFunc != nil {
+		return m.AddInterfaceFunc(sandboxID, inf)
+	}
+
+	return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID)
+}
+
+// RemoveInterface implements the VC function of the same name.
+func (m *VCMock) RemoveInterface(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error) {
+	if m.RemoveInterfaceFunc != nil {
+		return m.RemoveInterfaceFunc(sandboxID, inf)
+	}
+
+	return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID)
+}
+
+// ListInterfaces implements the VC function of the same name.
+func (m *VCMock) ListInterfaces(sandboxID string) ([]*grpc.Interface, error) {
+	if m.ListInterfacesFunc != nil {
+		return m.ListInterfacesFunc(sandboxID)
+	}
+
+	return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID)
+}
+
+// UpdateRoutes implements the VC function of the same name.
+func (m *VCMock) UpdateRoutes(sandboxID string, routes []*grpc.Route) ([]*grpc.Route, error) {
+	if m.UpdateRoutesFunc != nil {
+		return m.UpdateRoutesFunc(sandboxID, routes)
+	}
+
+	return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID)
+}
+
+// ListRoutes implements the VC function of the same name.
+func (m *VCMock) ListRoutes(sandboxID string) ([]*grpc.Route, error) {
+	if m.ListRoutesFunc != nil {
+		return m.ListRoutesFunc(sandboxID)
+	}
+
+	return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID)
+}

virtcontainers/pkg/vcmock/sandbox.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"syscall"
 
+	"github.com/kata-containers/agent/protocols/grpc"
 	vc "github.com/kata-containers/runtime/virtcontainers"
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
@@ -145,3 +146,28 @@ func (s *Sandbox) IOStream(containerID, processID string) (io.WriteCloser, io.Re
 func (s *Sandbox) AddDevice(info config.DeviceInfo) (api.Device, error) {
 	return nil, nil
 }
+
+// AddInterface implements the VCSandbox function of the same name.
+func (s *Sandbox) AddInterface(inf *grpc.Interface) (*grpc.Interface, error) {
+	return nil, nil
+}
+
+// RemoveInterface implements the VCSandbox function of the same name.
+func (s *Sandbox) RemoveInterface(inf *grpc.Interface) (*grpc.Interface, error) {
+	return nil, nil
+}
+
+// ListInterfaces implements the VCSandbox function of the same name.
+func (s *Sandbox) ListInterfaces() ([]*grpc.Interface, error) {
+	return nil, nil
+}
+
+// UpdateRoutes implements the VCSandbox function of the same name.
+func (s *Sandbox) UpdateRoutes(routes []*grpc.Route) ([]*grpc.Route, error) {
+	return nil, nil
+}
+
+// ListRoutes implements the VCSandbox function of the same name.
+func (s *Sandbox) ListRoutes() ([]*grpc.Route, error) {
+	return nil, nil
+}

virtcontainers/pkg/vcmock/types.go

@@ -8,6 +8,7 @@ package vcmock
 import (
 	"syscall"
 
+	"github.com/kata-containers/agent/protocols/grpc"
 	vc "github.com/kata-containers/runtime/virtcontainers"
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
@@ -65,4 +66,10 @@ type VCMock struct {
 	ResumeContainerFunc      func(sandboxID, containerID string) error
 
 	AddDeviceFunc func(sandboxID string, info config.DeviceInfo) (api.Device, error)
+
+	AddInterfaceFunc    func(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error)
+	RemoveInterfaceFunc func(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error)
+	ListInterfacesFunc  func(sandboxID string) ([]*grpc.Interface, error)
+	UpdateRoutesFunc    func(sandboxID string, routes []*grpc.Route) ([]*grpc.Route, error)
+	ListRoutesFunc      func(sandboxID string) ([]*grpc.Route, error)
 }

virtcontainers/qemu.go

@@ -757,6 +757,78 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) error {
 	return nil
 }
 
+func (q *qemu) hotplugMacvtap(drive VirtualEndpoint) error {
+	var (
+		VMFdNames    []string
+		VhostFdNames []string
+	)
+	for i, VMFd := range drive.NetPair.VMFds {
+		fdName := fmt.Sprintf("fd%d", i)
+		err := q.qmpMonitorCh.qmp.ExecuteGetFD(q.qmpMonitorCh.ctx, fdName, VMFd)
+		if err != nil {
+			return err
+		}
+		VMFdNames = append(VMFdNames, fdName)
+	}
+	for i, VhostFd := range drive.NetPair.VhostFds {
+		fdName := fmt.Sprintf("vhostfd%d", i)
+		err := q.qmpMonitorCh.qmp.ExecuteGetFD(q.qmpMonitorCh.ctx, fdName, VhostFd)
+		if err != nil {
+			return err
+		}
+		VhostFdNames = append(VhostFdNames, fdName)
+	}
+	return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", drive.NetPair.Name, VMFdNames, VhostFdNames)
+}
+
+func (q *qemu) hotplugNetDevice(drive VirtualEndpoint, op operation) error {
+	defer func(qemu *qemu) {
+		if q.qmpMonitorCh.qmp != nil {
+			q.qmpMonitorCh.qmp.Shutdown()
+		}
+	}(q)
+
+	err := q.qmpSetup()
+	if err != nil {
+		return err
+	}
+	devID := "virtio-" + drive.NetPair.ID
+
+	if op == addDevice {
+		switch drive.NetPair.NetInterworkingModel {
+		case NetXConnectBridgedModel:
+			if err := q.qmpMonitorCh.qmp.ExecuteNetdevAdd(q.qmpMonitorCh.ctx, "tap", drive.NetPair.Name, drive.NetPair.TAPIface.Name, "no", "no", defaultQueues); err != nil {
+				return err
+			}
+		case NetXConnectMacVtapModel:
+			if err := q.hotplugMacvtap(drive); err != nil {
+				return err
+			}
+		default:
+			return fmt.Errorf("this net interworking model is not supported")
+		}
+		addr, bridge, err := q.addDeviceToBridge(drive.NetPair.ID)
+		if err != nil {
+			return err
+		}
+		drive.PCIAddr = fmt.Sprintf("%02x/%s", bridge.Addr, addr)
+		if err = q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, drive.NetPair.Name, devID, drive.NetPair.TAPIface.HardAddr, addr, bridge.ID); err != nil {
+			return err
+		}
+	} else {
+		if err := q.removeDeviceFromBridge(drive.NetPair.ID); err != nil {
+			return err
+		}
+		if err := q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, devID); err != nil {
+			return err
+		}
+		if err := q.qmpMonitorCh.qmp.ExecuteNetdevDel(q.qmpMonitorCh.ctx, drive.NetPair.Name); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operation) (interface{}, error) {
 	switch devType {
 	case blockDev:
@@ -771,6 +843,9 @@ func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operati
 	case memoryDev:
 		memdev := devInfo.(*memoryDevice)
 		return nil, q.hotplugMemory(memdev, op)
+	case netDev:
+		device := devInfo.(VirtualEndpoint)
+		return nil, q.hotplugNetDevice(device, op)
 	default:
 		return nil, fmt.Errorf("cannot hotplug device: unsupported device type '%v'", devType)
 	}
@@ -1038,7 +1113,7 @@ func genericAppendBridges(devices []govmmQemu.Device, bridges []Bridge, machineT
 				Bus:  bus,
 				ID:   b.ID,
 				// Each bridge is required to be assigned a unique chassis id > 0
-				Chassis: (idx + 1),
+				Chassis: idx + 1,
 				SHPC:    true,
 				Addr:    strconv.FormatInt(int64(bridges[idx].Addr), 10),
 			},

virtcontainers/sandbox.go

@@ -8,19 +8,23 @@ package virtcontainers
 import (
 	"fmt"
 	"io"
+	"net"
 	"os"
 	"path/filepath"
 	"strings"
 	"sync"
 	"syscall"
 
+	"github.com/containernetworking/plugins/pkg/ns"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
 
+	"github.com/kata-containers/agent/protocols/grpc"
 	"github.com/kata-containers/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/runtime/virtcontainers/device/drivers"
 	deviceManager "github.com/kata-containers/runtime/virtcontainers/device/manager"
+	"github.com/vishvananda/netlink"
 )
 
 // vmStartTimeout represents the time in seconds a sandbox can wait before
@@ -976,6 +980,93 @@ func (s *Sandbox) removeNetwork() error {
 	return s.network.remove(s, s.networkNS, s.networkNS.NetNsCreated)
 }
 
+func (s *Sandbox) generateNetInfo(inf *grpc.Interface) (NetworkInfo, error) {
+	hw, err := net.ParseMAC(inf.HwAddr)
+	if err != nil {
+		return NetworkInfo{}, err
+	}
+
+	var addrs []netlink.Addr
+	for _, addr := range inf.IPAddresses {
+		netlinkAddrStr := fmt.Sprintf("%s/%s", addr.Address, addr.Mask)
+		netlinkAddr, err := netlink.ParseAddr(netlinkAddrStr)
+		if err != nil {
+			return NetworkInfo{}, fmt.Errorf("could not parse %q: %v", netlinkAddrStr, err)
+		}
+
+		addrs = append(addrs, *netlinkAddr)
+	}
+
+	return NetworkInfo{
+		Iface: NetlinkIface{
+			LinkAttrs: netlink.LinkAttrs{
+				Name:         inf.Name,
+				HardwareAddr: hw,
+				MTU:          int(inf.Mtu),
+			},
+			Type: "",
+		},
+		Addrs: addrs,
+	}, nil
+}
+
+// AddInterface adds new nic to the sandbox.
+func (s *Sandbox) AddInterface(inf *grpc.Interface) (*grpc.Interface, error) {
+	netInfo, err := s.generateNetInfo(inf)
+	if err != nil {
+		return nil, err
+	}
+
+	endpoint, err := createVirtualNetworkEndpoint(len(s.networkNS.Endpoints), inf.Name, s.config.NetworkConfig.InterworkingModel)
+	if err != nil {
+		return nil, err
+	}
+	endpoint.SetProperties(netInfo)
+	if err := doNetNS(s.networkNS.NetNsPath, func(_ ns.NetNS) error {
+		return endpoint.HotAttach(s.hypervisor)
+	}); err != nil {
+		return nil, err
+	}
+
+	// Update the sandbox storage
+	s.networkNS.Endpoints = append(s.networkNS.Endpoints, endpoint)
+	if err := s.storage.storeSandboxNetwork(s.id, s.networkNS); err != nil {
+		return nil, err
+	}
+
+	// Add network for vm
+	return s.agent.updateInterface(inf)
+}
+
+// RemoveInterface removes a nic of the sandbox.
+func (s *Sandbox) RemoveInterface(inf *grpc.Interface) (*grpc.Interface, error) {
+	for i, endpoint := range s.networkNS.Endpoints {
+		if endpoint.HardwareAddr() == inf.HwAddr {
+			if err := endpoint.HotDetach(s.hypervisor, s.networkNS.NetNsCreated, s.networkNS.NetNsPath); err != nil {
+				return inf, err
+			}
+			s.networkNS.Endpoints = append(s.networkNS.Endpoints[:i], s.networkNS.Endpoints[i+1:]...)
+			break
+		}
+	}
+	return nil, nil
+}
+
+// ListInterfaces lists all nics and their configurations in the sandbox.
+func (s *Sandbox) ListInterfaces() ([]*grpc.Interface, error) {
+	return s.agent.listInterfaces()
+}
+
+// UpdateRoutes updates the sandbox route table (e.g. for portmapping support).
+func (s *Sandbox) UpdateRoutes(routes []*grpc.Route) ([]*grpc.Route, error) {
+	return s.agent.updateRoutes(routes)
+}
+
+// ListRoutes lists all routes and their configurations in the sandbox.
+func (s *Sandbox) ListRoutes() ([]*grpc.Route, error) {
+	return s.agent.listRoutes()
+}
+
 // startVM starts the VM.
 func (s *Sandbox) startVM() error {
 	s.Logger().Info("Starting VM")