diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go index f1d6f5b7c0..bf8706d308 100644 --- a/virtcontainers/agent.go +++ b/virtcontainers/agent.go @@ -9,6 +9,7 @@ import ( "fmt" "syscall" + "github.com/kata-containers/agent/protocols/grpc" "github.com/mitchellh/mapstructure" specs "github.com/opencontainers/runtime-spec/specs-go" ) @@ -221,4 +222,16 @@ type agent interface { // reseedRNG will reseed the guest random number generator reseedRNG(data []byte) error + + // updateInterface will tell the agent to update a nic for an existed Sandbox. + updateInterface(inf *grpc.Interface) (*grpc.Interface, error) + + // listInterfaces will tell the agent to list interfaces of an existed Sandbox + listInterfaces() ([]*grpc.Interface, error) + + // updateRoutes will tell the agent to update route table for an existed Sandbox. + updateRoutes(routes []*grpc.Route) ([]*grpc.Route, error) + + // listRoutes will tell the agent to list routes of an existed Sandbox + listRoutes() ([]*grpc.Route, error) } diff --git a/virtcontainers/api.go b/virtcontainers/api.go index c8e8d8bbc9..95c6ff042c 100644 --- a/virtcontainers/api.go +++ b/virtcontainers/api.go @@ -10,6 +10,7 @@ import ( "runtime" "syscall" + "github.com/kata-containers/agent/protocols/grpc" deviceApi "github.com/kata-containers/runtime/virtcontainers/device/api" deviceConfig "github.com/kata-containers/runtime/virtcontainers/device/config" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -782,3 +783,93 @@ func AddDevice(sandboxID string, info deviceConfig.DeviceInfo) (deviceApi.Device return s.AddDevice(info) } + +func toggleInterface(sandboxID string, inf *grpc.Interface, add bool) (*grpc.Interface, error) { + if sandboxID == "" { + return nil, errNeedSandboxID + } + + lockFile, err := rwLockSandbox(sandboxID) + if err != nil { + return nil, err + } + defer unlockSandbox(lockFile) + + s, err := fetchSandbox(sandboxID) + if err != nil { + return nil, err + } + if add { + return s.AddInterface(inf) + } + return s.RemoveInterface(inf) +} + +// AddInterface is the virtcontainers add interface entry point. +func AddInterface(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error) { + return toggleInterface(sandboxID, inf, true) +} + +// RemoveInterface is the virtcontainers remove interface entry point. +func RemoveInterface(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error) { + return toggleInterface(sandboxID, inf, false) +} + +// ListInterfaces is the virtcontainers list interfaces entry point. +func ListInterfaces(sandboxID string) ([]*grpc.Interface, error) { + if sandboxID == "" { + return nil, errNeedSandboxID + } + + lockFile, err := rLockSandbox(sandboxID) + if err != nil { + return nil, err + } + defer unlockSandbox(lockFile) + + s, err := fetchSandbox(sandboxID) + if err != nil { + return nil, err + } + + return s.ListInterfaces() +} + +// UpdateRoutes is the virtcontainers update routes entry point. +func UpdateRoutes(sandboxID string, routes []*grpc.Route) ([]*grpc.Route, error) { + if sandboxID == "" { + return nil, errNeedSandboxID + } + + lockFile, err := rwLockSandbox(sandboxID) + if err != nil { + return nil, err + } + defer unlockSandbox(lockFile) + + s, err := fetchSandbox(sandboxID) + if err != nil { + return nil, err + } + return s.UpdateRoutes(routes) +} + +// ListRoutes is the virtcontainers list routes entry point. +func ListRoutes(sandboxID string) ([]*grpc.Route, error) { + if sandboxID == "" { + return nil, errNeedSandboxID + } + + lockFile, err := rLockSandbox(sandboxID) + if err != nil { + return nil, err + } + defer unlockSandbox(lockFile) + + s, err := fetchSandbox(sandboxID) + if err != nil { + return nil, err + } + + return s.ListRoutes() +} diff --git a/virtcontainers/hyperstart_agent.go b/virtcontainers/hyperstart_agent.go index 936865df5c..5c1181887a 100644 --- a/virtcontainers/hyperstart_agent.go +++ b/virtcontainers/hyperstart_agent.go @@ -18,6 +18,7 @@ import ( "github.com/vishvananda/netlink" proxyClient "github.com/clearcontainers/proxy/client" + "github.com/kata-containers/agent/protocols/grpc" "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/pkg/hyperstart" ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter" @@ -852,6 +853,26 @@ func (h *hyper) onlineCPUMem(cpus uint32) error { return nil } +func (h *hyper) updateInterface(inf *grpc.Interface) (*grpc.Interface, error) { + // hyperstart-agent does not support update interface + return nil, nil +} + +func (h *hyper) listInterfaces() ([]*grpc.Interface, error) { + // hyperstart-agent does not support list interfaces + return nil, nil +} + +func (h *hyper) updateRoutes(routes []*grpc.Route) ([]*grpc.Route, error) { + // hyperstart-agent does not support update routes + return nil, nil +} + +func (h *hyper) listRoutes() ([]*grpc.Route, error) { + // hyperstart-agent does not support list routes + return nil, nil +} + func (h *hyper) check() error { // hyperstart-agent does not support check return nil diff --git a/virtcontainers/implementation.go b/virtcontainers/implementation.go index e9006fbc18..81727dd5a2 100644 --- a/virtcontainers/implementation.go +++ b/virtcontainers/implementation.go @@ -12,6 +12,7 @@ package virtcontainers import ( "syscall" + "github.com/kata-containers/agent/protocols/grpc" "github.com/kata-containers/runtime/virtcontainers/device/api" "github.com/kata-containers/runtime/virtcontainers/device/config" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -148,3 +149,28 @@ func (impl *VCImpl) ResumeContainer(sandboxID, containerID string) error { func (impl *VCImpl) AddDevice(sandboxID string, info config.DeviceInfo) (api.Device, error) { return AddDevice(sandboxID, info) } + +// AddInterface implements the VC function of the same name. +func (impl *VCImpl) AddInterface(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error) { + return AddInterface(sandboxID, inf) +} + +// RemoveInterface implements the VC function of the same name. +func (impl *VCImpl) RemoveInterface(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error) { + return RemoveInterface(sandboxID, inf) +} + +// ListInterfaces implements the VC function of the same name. +func (impl *VCImpl) ListInterfaces(sandboxID string) ([]*grpc.Interface, error) { + return ListInterfaces(sandboxID) +} + +// UpdateRoutes implements the VC function of the same name. +func (impl *VCImpl) UpdateRoutes(sandboxID string, routes []*grpc.Route) ([]*grpc.Route, error) { + return UpdateRoutes(sandboxID, routes) +} + +// ListRoutes implements the VC function of the same name. +func (impl *VCImpl) ListRoutes(sandboxID string) ([]*grpc.Route, error) { + return ListRoutes(sandboxID) +} diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go index d54843e49e..e83ee37447 100644 --- a/virtcontainers/interfaces.go +++ b/virtcontainers/interfaces.go @@ -9,6 +9,7 @@ import ( "io" "syscall" + "github.com/kata-containers/agent/protocols/grpc" "github.com/kata-containers/runtime/virtcontainers/device/api" "github.com/kata-containers/runtime/virtcontainers/device/config" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -45,6 +46,12 @@ type VC interface { ResumeContainer(sandboxID, containerID string) error AddDevice(sandboxID string, info config.DeviceInfo) (api.Device, error) + + AddInterface(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error) + RemoveInterface(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error) + ListInterfaces(sandboxID string) ([]*grpc.Interface, error) + UpdateRoutes(sandboxID string, routes []*grpc.Route) ([]*grpc.Route, error) + ListRoutes(sandboxID string) ([]*grpc.Route, error) } // VCSandbox is the Sandbox interface @@ -76,6 +83,12 @@ type VCSandbox interface { IOStream(containerID, processID string) (io.WriteCloser, io.Reader, io.Reader, error) AddDevice(info config.DeviceInfo) (api.Device, error) + + AddInterface(inf *grpc.Interface) (*grpc.Interface, error) + RemoveInterface(inf *grpc.Interface) (*grpc.Interface, error) + ListInterfaces() ([]*grpc.Interface, error) + UpdateRoutes(routes []*grpc.Route) ([]*grpc.Route, error) + ListRoutes() ([]*grpc.Route, error) } // VCContainer is the Container interface diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index a9372897bc..141b6eef86 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -440,6 +440,82 @@ func (k *kataAgent) generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]* return ifaces, routes, nil } +func (k *kataAgent) updateInterface(ifc *grpc.Interface) (*grpc.Interface, error) { + // send update interface request + ifcReq := &grpc.UpdateInterfaceRequest{ + Interface: ifc, + } + resultingInterface, err := k.sendReq(ifcReq) + if err != nil { + k.Logger().WithFields(logrus.Fields{ + "interface-requested": fmt.Sprintf("%+v", ifc), + "resulting-interface": fmt.Sprintf("%+v", resultingInterface), + }).WithError(err).Error("update interface request failed") + } + if resultInterface, ok := resultingInterface.(*grpc.Interface); ok { + return resultInterface, err + } + return nil, err +} + +func (k *kataAgent) updateInterfaces(interfaces []*grpc.Interface) error { + for _, ifc := range interfaces { + if _, err := k.updateInterface(ifc); err != nil { + return err + } + } + return nil +} + +func (k *kataAgent) updateRoutes(routes []*grpc.Route) ([]*grpc.Route, error) { + if routes != nil { + routesReq := &grpc.UpdateRoutesRequest{ + Routes: &grpc.Routes{ + Routes: routes, + }, + } + resultingRoutes, err := k.sendReq(routesReq) + if err != nil { + k.Logger().WithFields(logrus.Fields{ + "routes-requested": fmt.Sprintf("%+v", routes), + "resulting-routes": fmt.Sprintf("%+v", resultingRoutes), + }).WithError(err).Error("update routes request failed") + } + resultRoutes, ok := resultingRoutes.(*grpc.Routes) + if ok && resultRoutes != nil { + return resultRoutes.Routes, err + } + return nil, err + } + return nil, nil +} + +func (k *kataAgent) listInterfaces() ([]*grpc.Interface, error) { + req := &grpc.ListInterfacesRequest{} + resultingInterfaces, err := k.sendReq(req) + if err != nil { + return nil, err + } + resultInterfaces, ok := resultingInterfaces.(*grpc.Interfaces) + if ok { + return resultInterfaces.Interfaces, err + } + return nil, err +} + +func (k *kataAgent) listRoutes() ([]*grpc.Route, error) { + req := &grpc.ListRoutesRequest{} + resultingRoutes, err := k.sendReq(req) + if err != nil { + return nil, err + } + resultRoutes, ok := resultingRoutes.(*grpc.Routes) + if ok { + return resultRoutes.Routes, err + } + return nil, err +} + func (k *kataAgent) startProxy(sandbox *Sandbox) error { var err error @@ -515,36 +591,11 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error { if err != nil { return err } - for _, ifc := range interfaces { - // send update interface request - ifcReq := &grpc.UpdateInterfaceRequest{ - Interface: ifc, - } - resultingInterface, err := k.sendReq(ifcReq) - if err != nil { - k.Logger().WithFields(logrus.Fields{ - "interface-requested": fmt.Sprintf("%+v", ifc), - "resulting-interface": fmt.Sprintf("%+v", resultingInterface), - }).WithError(err).Error("update interface request failed") - return err - } + if err := k.updateInterfaces(interfaces); err != nil { + return err } - - if routes != nil { - routesReq := &grpc.UpdateRoutesRequest{ - Routes: &grpc.Routes{ - Routes: routes, - }, - } - - resultingRoutes, err := k.sendReq(routesReq) - if err != nil { - k.Logger().WithFields(logrus.Fields{ - "routes-requested": fmt.Sprintf("%+v", routes), - "resulting-routes": fmt.Sprintf("%+v", resultingRoutes), - }).WithError(err).Error("update routes request failed") - return err - } + if _, err := k.updateRoutes(routes); err != nil { + return err } sharedDir9pOptions = append(sharedDir9pOptions, fmt.Sprintf("msize=%d", sandbox.config.HypervisorConfig.Msize9p)) @@ -1335,6 +1386,12 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) { k.reqHandlers["grpc.UpdateInterfaceRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) { return k.client.UpdateInterface(ctx, req.(*grpc.UpdateInterfaceRequest), opts...) } + k.reqHandlers["grpc.ListInterfacesRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) { + return k.client.ListInterfaces(ctx, req.(*grpc.ListInterfacesRequest), opts...) + } + k.reqHandlers["grpc.ListRoutesRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) { + return k.client.ListRoutes(ctx, req.(*grpc.ListRoutesRequest), opts...) + } k.reqHandlers["grpc.OnlineCPUMemRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) { return k.client.OnlineCPUMem(ctx, req.(*grpc.OnlineCPUMemRequest), opts...) } diff --git a/virtcontainers/network.go b/virtcontainers/network.go index baeb43b9a6..bfbac528ba 100644 --- a/virtcontainers/network.go +++ b/virtcontainers/network.go @@ -156,6 +156,8 @@ type Endpoint interface { SetProperties(NetworkInfo) Attach(hypervisor) error Detach(netNsCreated bool, netNsPath string) error + HotAttach(h hypervisor) error + HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error } // VirtualEndpoint gathers a network pair and its properties. @@ -164,6 +166,7 @@ type VirtualEndpoint struct { EndpointProperties NetworkInfo Physical bool EndpointType EndpointType + PCIAddr string } // PhysicalEndpoint gathers a physical network interface and its properties @@ -247,6 +250,41 @@ func (endpoint *VirtualEndpoint) Detach(netNsCreated bool, netNsPath string) err }) } +// HotAttach for the virtual endpoint uses hot plug device +func (endpoint *VirtualEndpoint) HotAttach(h hypervisor) error { + networkLogger().Info("Hot attaching virtual endpoint") + if err := xconnectVMNetwork(&(endpoint.NetPair), true); err != nil { + networkLogger().WithError(err).Error("Error bridging virtual ep") + return err + } + + if _, err := h.hotplugAddDevice(*endpoint, netDev); err != nil { + networkLogger().WithError(err).Error("Error attach virtual ep") + return err + } + return nil +} + +// HotDetach for the virtual endpoint uses hot pull device +func (endpoint *VirtualEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { + if !netNsCreated { + return nil + } + networkLogger().Info("Hot detaching virtual endpoint") + if err := doNetNS(netNsPath, func(_ ns.NetNS) error { + return xconnectVMNetwork(&(endpoint.NetPair), false) + }); err != nil { + networkLogger().WithError(err).Error("Error abridging virtual ep") + return err + } + + if _, err := h.hotplugRemoveDevice(*endpoint, netDev); err != nil { + networkLogger().WithError(err).Error("Error detach virtual ep") + return err + } + return nil +} + // Properties returns the properties of the interface. func (endpoint *VhostUserEndpoint) Properties() NetworkInfo { return endpoint.EndpointProperties @@ -298,6 +336,16 @@ func (endpoint *VhostUserEndpoint) Detach(netNsCreated bool, netNsPath string) e return nil } +// HotAttach for vhostuser endpoint not supported yet +func (endpoint *VhostUserEndpoint) HotAttach(h hypervisor) error { + return fmt.Errorf("VhostUserEndpoint does not support Hot attach") +} + +// HotDetach for vhostuser endpoint not supported yet +func (endpoint *VhostUserEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { + return fmt.Errorf("VhostUserEndpoint does not support Hot detach") +} + // Create a vhostuser endpoint func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndpoint, error) { @@ -367,6 +415,16 @@ func (endpoint *PhysicalEndpoint) Detach(netNsCreated bool, netNsPath string) er return bindNICToHost(endpoint) } +// HotAttach for physical endpoint not supported yet +func (endpoint *PhysicalEndpoint) HotAttach(h hypervisor) error { + return fmt.Errorf("PhysicalEndpoint does not support Hot attach") +} + +// HotDetach for physical endpoint not supported yet +func (endpoint *PhysicalEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { + return fmt.Errorf("PhysicalEndpoint does not support Hot detach") +} + // EndpointType identifies the type of the network endpoint. type EndpointType string diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go index fe919ec587..d5073166e5 100644 --- a/virtcontainers/noop_agent.go +++ b/virtcontainers/noop_agent.go @@ -8,6 +8,7 @@ package virtcontainers import ( "syscall" + "github.com/kata-containers/agent/protocols/grpc" specs "github.com/opencontainers/runtime-spec/specs-go" ) @@ -96,6 +97,26 @@ func (n *noopAgent) onlineCPUMem(cpus uint32) error { return nil } +// updateInterface is the Noop agent Interface update implementation. It does nothing. +func (n *noopAgent) updateInterface(inf *grpc.Interface) (*grpc.Interface, error) { + return nil, nil +} + +// listInterfaces is the Noop agent Interfaces list implementation. It does nothing. +func (n *noopAgent) listInterfaces() ([]*grpc.Interface, error) { + return nil, nil +} + +// updateRoutes is the Noop agent Routes update implementation. It does nothing. +func (n *noopAgent) updateRoutes(routes []*grpc.Route) ([]*grpc.Route, error) { + return nil, nil +} + +// listRoutes is the Noop agent Routes list implementation. It does nothing. +func (n *noopAgent) listRoutes() ([]*grpc.Route, error) { + return nil, nil +} + // check is the Noop agent health checker. It does nothing. func (n *noopAgent) check() error { return nil diff --git a/virtcontainers/pkg/vcmock/mock.go b/virtcontainers/pkg/vcmock/mock.go index bf4cb27724..4c74b0065e 100644 --- a/virtcontainers/pkg/vcmock/mock.go +++ b/virtcontainers/pkg/vcmock/mock.go @@ -19,6 +19,7 @@ import ( "fmt" "syscall" + "github.com/kata-containers/agent/protocols/grpc" vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/device/api" "github.com/kata-containers/runtime/virtcontainers/device/config" @@ -250,3 +251,48 @@ func (m *VCMock) AddDevice(sandboxID string, info config.DeviceInfo) (api.Device return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) } + +// AddInterface implements the VC function of the same name. +func (m *VCMock) AddInterface(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error) { + if m.AddInterfaceFunc != nil { + return m.AddInterfaceFunc(sandboxID, inf) + } + + return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) +} + +// RemoveInterface implements the VC function of the same name. +func (m *VCMock) RemoveInterface(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error) { + if m.RemoveInterfaceFunc != nil { + return m.RemoveInterfaceFunc(sandboxID, inf) + } + + return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) +} + +// ListInterfaces implements the VC function of the same name. +func (m *VCMock) ListInterfaces(sandboxID string) ([]*grpc.Interface, error) { + if m.ListInterfacesFunc != nil { + return m.ListInterfacesFunc(sandboxID) + } + + return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) +} + +// UpdateRoutes implements the VC function of the same name. +func (m *VCMock) UpdateRoutes(sandboxID string, routes []*grpc.Route) ([]*grpc.Route, error) { + if m.UpdateRoutesFunc != nil { + return m.UpdateRoutesFunc(sandboxID, routes) + } + + return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) +} + +// ListRoutes implements the VC function of the same name. +func (m *VCMock) ListRoutes(sandboxID string) ([]*grpc.Route, error) { + if m.ListRoutesFunc != nil { + return m.ListRoutesFunc(sandboxID) + } + + return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) +} diff --git a/virtcontainers/pkg/vcmock/sandbox.go b/virtcontainers/pkg/vcmock/sandbox.go index 7f27888dd4..f3a3c0cf82 100644 --- a/virtcontainers/pkg/vcmock/sandbox.go +++ b/virtcontainers/pkg/vcmock/sandbox.go @@ -9,6 +9,7 @@ import ( "io" "syscall" + "github.com/kata-containers/agent/protocols/grpc" vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/device/api" "github.com/kata-containers/runtime/virtcontainers/device/config" @@ -145,3 +146,28 @@ func (s *Sandbox) IOStream(containerID, processID string) (io.WriteCloser, io.Re func (s *Sandbox) AddDevice(info config.DeviceInfo) (api.Device, error) { return nil, nil } + +// AddInterface implements the VCSandbox function of the same name. +func (s *Sandbox) AddInterface(inf *grpc.Interface) (*grpc.Interface, error) { + return nil, nil +} + +// RemoveInterface implements the VCSandbox function of the same name. +func (s *Sandbox) RemoveInterface(inf *grpc.Interface) (*grpc.Interface, error) { + return nil, nil +} + +// ListInterfaces implements the VCSandbox function of the same name. +func (s *Sandbox) ListInterfaces() ([]*grpc.Interface, error) { + return nil, nil +} + +// UpdateRoutes implements the VCSandbox function of the same name. +func (s *Sandbox) UpdateRoutes(routes []*grpc.Route) ([]*grpc.Route, error) { + return nil, nil +} + +// ListRoutes implements the VCSandbox function of the same name. +func (s *Sandbox) ListRoutes() ([]*grpc.Route, error) { + return nil, nil +} diff --git a/virtcontainers/pkg/vcmock/types.go b/virtcontainers/pkg/vcmock/types.go index c04f2a7cd0..1c7237f13f 100644 --- a/virtcontainers/pkg/vcmock/types.go +++ b/virtcontainers/pkg/vcmock/types.go @@ -8,6 +8,7 @@ package vcmock import ( "syscall" + "github.com/kata-containers/agent/protocols/grpc" vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/device/api" "github.com/kata-containers/runtime/virtcontainers/device/config" @@ -65,4 +66,10 @@ type VCMock struct { ResumeContainerFunc func(sandboxID, containerID string) error AddDeviceFunc func(sandboxID string, info config.DeviceInfo) (api.Device, error) + + AddInterfaceFunc func(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error) + RemoveInterfaceFunc func(sandboxID string, inf *grpc.Interface) (*grpc.Interface, error) + ListInterfacesFunc func(sandboxID string) ([]*grpc.Interface, error) + UpdateRoutesFunc func(sandboxID string, routes []*grpc.Route) ([]*grpc.Route, error) + ListRoutesFunc func(sandboxID string) ([]*grpc.Route, error) } diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index e45b31f8ed..ade8a5ed6b 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -757,6 +757,78 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) error { return nil } +func (q *qemu) hotplugMacvtap(drive VirtualEndpoint) error { + var ( + VMFdNames []string + VhostFdNames []string + ) + for i, VMFd := range drive.NetPair.VMFds { + fdName := fmt.Sprintf("fd%d", i) + err := q.qmpMonitorCh.qmp.ExecuteGetFD(q.qmpMonitorCh.ctx, fdName, VMFd) + if err != nil { + return err + } + VMFdNames = append(VMFdNames, fdName) + } + for i, VhostFd := range drive.NetPair.VhostFds { + fdName := fmt.Sprintf("vhostfd%d", i) + err := q.qmpMonitorCh.qmp.ExecuteGetFD(q.qmpMonitorCh.ctx, fdName, VhostFd) + if err != nil { + return err + } + VhostFdNames = append(VhostFdNames, fdName) + } + return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", drive.NetPair.Name, VMFdNames, VhostFdNames) +} + +func (q *qemu) hotplugNetDevice(drive VirtualEndpoint, op operation) error { + defer func(qemu *qemu) { + if q.qmpMonitorCh.qmp != nil { + q.qmpMonitorCh.qmp.Shutdown() + } + }(q) + + err := q.qmpSetup() + if err != nil { + return err + } + devID := "virtio-" + drive.NetPair.ID + + if op == addDevice { + switch drive.NetPair.NetInterworkingModel { + case NetXConnectBridgedModel: + if err := q.qmpMonitorCh.qmp.ExecuteNetdevAdd(q.qmpMonitorCh.ctx, "tap", drive.NetPair.Name, drive.NetPair.TAPIface.Name, "no", "no", defaultQueues); err != nil { + return err + } + case NetXConnectMacVtapModel: + if err := q.hotplugMacvtap(drive); err != nil { + return err + } + default: + return fmt.Errorf("this net interworking model is not supported") + } + addr, bridge, err := q.addDeviceToBridge(drive.NetPair.ID) + if err != nil { + return err + } + drive.PCIAddr = fmt.Sprintf("%02x/%s", bridge.Addr, addr) + if err = q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, drive.NetPair.Name, devID, drive.NetPair.TAPIface.HardAddr, addr, bridge.ID); err != nil { + return err + } + } else { + if err := q.removeDeviceFromBridge(drive.NetPair.ID); err != nil { + return err + } + if err := q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, devID); err != nil { + return err + } + if err := q.qmpMonitorCh.qmp.ExecuteNetdevDel(q.qmpMonitorCh.ctx, drive.NetPair.Name); err != nil { + return err + } + } + return nil +} + func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operation) (interface{}, error) { switch devType { case blockDev: @@ -771,6 +843,9 @@ func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operati case memoryDev: memdev := devInfo.(*memoryDevice) return nil, q.hotplugMemory(memdev, op) + case netDev: + device := devInfo.(VirtualEndpoint) + return nil, q.hotplugNetDevice(device, op) default: return nil, fmt.Errorf("cannot hotplug device: unsupported device type '%v'", devType) } @@ -1038,7 +1113,7 @@ func genericAppendBridges(devices []govmmQemu.Device, bridges []Bridge, machineT Bus: bus, ID: b.ID, // Each bridge is required to be assigned a unique chassis id > 0 - Chassis: (idx + 1), + Chassis: idx + 1, SHPC: true, Addr: strconv.FormatInt(int64(bridges[idx].Addr), 10), }, diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index c2b17448a2..1ac8e65a4c 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -8,19 +8,23 @@ package virtcontainers import ( "fmt" "io" + "net" "os" "path/filepath" "strings" "sync" "syscall" + "github.com/containernetworking/plugins/pkg/ns" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" + "github.com/kata-containers/agent/protocols/grpc" "github.com/kata-containers/runtime/virtcontainers/device/api" "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/device/drivers" deviceManager "github.com/kata-containers/runtime/virtcontainers/device/manager" + "github.com/vishvananda/netlink" ) // vmStartTimeout represents the time in seconds a sandbox can wait before @@ -976,6 +980,93 @@ func (s *Sandbox) removeNetwork() error { return s.network.remove(s, s.networkNS, s.networkNS.NetNsCreated) } +func (s *Sandbox) generateNetInfo(inf *grpc.Interface) (NetworkInfo, error) { + hw, err := net.ParseMAC(inf.HwAddr) + if err != nil { + return NetworkInfo{}, err + } + + var addrs []netlink.Addr + for _, addr := range inf.IPAddresses { + netlinkAddrStr := fmt.Sprintf("%s/%s", addr.Address, addr.Mask) + netlinkAddr, err := netlink.ParseAddr(netlinkAddrStr) + if err != nil { + return NetworkInfo{}, fmt.Errorf("could not parse %q: %v", netlinkAddrStr, err) + } + + addrs = append(addrs, *netlinkAddr) + } + + return NetworkInfo{ + Iface: NetlinkIface{ + LinkAttrs: netlink.LinkAttrs{ + Name: inf.Name, + HardwareAddr: hw, + MTU: int(inf.Mtu), + }, + Type: "", + }, + Addrs: addrs, + }, nil +} + +// AddInterface adds new nic to the sandbox. +func (s *Sandbox) AddInterface(inf *grpc.Interface) (*grpc.Interface, error) { + netInfo, err := s.generateNetInfo(inf) + if err != nil { + return nil, err + } + + endpoint, err := createVirtualNetworkEndpoint(len(s.networkNS.Endpoints), inf.Name, s.config.NetworkConfig.InterworkingModel) + if err != nil { + return nil, err + } + endpoint.SetProperties(netInfo) + if err := doNetNS(s.networkNS.NetNsPath, func(_ ns.NetNS) error { + return endpoint.HotAttach(s.hypervisor) + }); err != nil { + return nil, err + } + + // Update the sandbox storage + s.networkNS.Endpoints = append(s.networkNS.Endpoints, endpoint) + if err := s.storage.storeSandboxNetwork(s.id, s.networkNS); err != nil { + return nil, err + } + + // Add network for vm + return s.agent.updateInterface(inf) +} + +// RemoveInterface removes a nic of the sandbox. +func (s *Sandbox) RemoveInterface(inf *grpc.Interface) (*grpc.Interface, error) { + for i, endpoint := range s.networkNS.Endpoints { + if endpoint.HardwareAddr() == inf.HwAddr { + if err := endpoint.HotDetach(s.hypervisor, s.networkNS.NetNsCreated, s.networkNS.NetNsPath); err != nil { + return inf, err + } + s.networkNS.Endpoints = append(s.networkNS.Endpoints[:i], s.networkNS.Endpoints[i+1:]...) + break + } + } + return nil, nil +} + +// ListInterfaces lists all nics and their configurations in the sandbox. +func (s *Sandbox) ListInterfaces() ([]*grpc.Interface, error) { + return s.agent.listInterfaces() +} + +// UpdateRoutes updates the sandbox route table (e.g. for portmapping support). +func (s *Sandbox) UpdateRoutes(routes []*grpc.Route) ([]*grpc.Route, error) { + return s.agent.updateRoutes(routes) +} + +// ListRoutes lists all routes and their configurations in the sandbox. +func (s *Sandbox) ListRoutes() ([]*grpc.Route, error) { + return s.agent.listRoutes() +} + // startVM starts the VM. func (s *Sandbox) startVM() error { s.Logger().Info("Starting VM")