network: Implement network hotunplug for physical endpoints

Similar to HotAttach, the HotDetach method signature for network
endoints needs to be changed as well to allow for the method to make
use of device manager to manage the hot unplug of physical network
devices.

Signed-off-by: Archana Shinde <archana.m.shinde@intel.com>

src/runtime/pkg/device/api/interface.go

@@ -94,4 +94,5 @@ type DeviceManager interface {
 	GetDeviceByID(string) Device
 	GetAllDevices() []Device
 	LoadDevices([]config.DeviceState)
+	FindDevice(*config.DeviceInfo) Device
 }

src/runtime/pkg/device/manager/manager.go

@@ -82,7 +82,7 @@ func NewDeviceManager(blockDriver string, vhostUserStoreEnabled bool, vhostUserS
 	return dm
 }
 
-func (dm *deviceManager) findDevice(devInfo *config.DeviceInfo) api.Device {
+func (dm *deviceManager) FindDevice(devInfo *config.DeviceInfo) api.Device {
 	// For devices with a major of -1, we use the host path to find existing instances.
 	if devInfo.Major == -1 {
 		for _, dev := range dm.devices {
@@ -121,7 +121,7 @@ func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (dev api.Device
 		}
 	}()
 
-	if existingDev := dm.findDevice(&devInfo); existingDev != nil {
+	if existingDev := dm.FindDevice(&devInfo); existingDev != nil {
 		return existingDev, nil
 	}
 

src/runtime/virtcontainers/endpoint.go

@@ -27,7 +27,7 @@ type Endpoint interface {
 	Attach(context.Context, *Sandbox) error
 	Detach(ctx context.Context, netNsCreated bool, netNsPath string) error
 	HotAttach(context.Context, *Sandbox) error
-	HotDetach(ctx context.Context, h Hypervisor, netNsCreated bool, netNsPath string) error
+	HotDetach(ctx context.Context, s *Sandbox, netNsCreated bool, netNsPath string) error
 
 	save() persistapi.NetworkEndpoint
 	load(persistapi.NetworkEndpoint)

src/runtime/virtcontainers/ipvlan_endpoint.go

@@ -143,7 +143,7 @@ func (endpoint *IPVlanEndpoint) HotAttach(ctx context.Context, s *Sandbox) error
 	return nil
 }
 
-func (endpoint *IPVlanEndpoint) HotDetach(ctx context.Context, h Hypervisor, netNsCreated bool, netNsPath string) error {
+func (endpoint *IPVlanEndpoint) HotDetach(ctx context.Context, s *Sandbox, netNsCreated bool, netNsPath string) error {
 	if !netNsCreated {
 		return nil
 	}
@@ -157,6 +157,7 @@ func (endpoint *IPVlanEndpoint) HotDetach(ctx context.Context, h Hypervisor, net
 		networkLogger().WithError(err).Warn("Error un-bridging ipvlan ep")
 	}
 
+	h := s.hypervisor
 	if _, err := h.HotplugRemoveDevice(ctx, endpoint, NetDev); err != nil {
 		networkLogger().WithError(err).Error("Error detach ipvlan ep")
 		return err

src/runtime/virtcontainers/macvlan_endpoint.go

@@ -140,7 +140,7 @@ func (endpoint *MacvlanEndpoint) HotAttach(ctx context.Context, s *Sandbox) erro
 	return nil
 }
 
-func (endpoint *MacvlanEndpoint) HotDetach(ctx context.Context, h Hypervisor, netNsCreated bool, netNsPath string) error {
+func (endpoint *MacvlanEndpoint) HotDetach(ctx context.Context, s *Sandbox, netNsCreated bool, netNsPath string) error {
 	if !netNsCreated {
 		return nil
 	}
@@ -154,6 +154,7 @@ func (endpoint *MacvlanEndpoint) HotDetach(ctx context.Context, h Hypervisor, ne
 		networkLogger().WithError(err).Warn("Error un-bridging macvlan ep")
 	}
 
+	h := s.hypervisor
 	if _, err := h.HotplugRemoveDevice(ctx, endpoint, NetDev); err != nil {
 		networkLogger().WithError(err).Error("Error detach macvlan ep")
 		return err

src/runtime/virtcontainers/macvtap_endpoint.go

@@ -98,7 +98,7 @@ func (endpoint *MacvtapEndpoint) HotAttach(ctx context.Context, s *Sandbox) erro
 }
 
 // HotDetach for macvtap endpoint not supported yet
-func (endpoint *MacvtapEndpoint) HotDetach(ctx context.Context, h Hypervisor, netNsCreated bool, netNsPath string) error {
+func (endpoint *MacvtapEndpoint) HotDetach(ctx context.Context, s *Sandbox, netNsCreated bool, netNsPath string) error {
 	return fmt.Errorf("MacvtapEndpoint does not support Hot detach")
 }
 

src/runtime/virtcontainers/network_linux.go

@@ -265,7 +265,7 @@ func (n *LinuxNetwork) removeSingleEndpoint(ctx context.Context, s *Sandbox, end
 	// if required.
 	networkLogger().WithField("endpoint-type", endpoint.Type()).Info("Detaching endpoint")
 	if hotplug && s != nil {
-		if err := endpoint.HotDetach(ctx, s.hypervisor, n.netNSCreated, n.netNSPath); err != nil {
+		if err := endpoint.HotDetach(ctx, s, n.netNSCreated, n.netNSPath); err != nil {
 			return err
 		}
 	} else {

src/runtime/virtcontainers/physical_endpoint.go

@@ -152,8 +152,36 @@ func (endpoint *PhysicalEndpoint) HotAttach(ctx context.Context, s *Sandbox) err
 }
 
 // HotDetach for physical endpoint not supported yet
-func (endpoint *PhysicalEndpoint) HotDetach(ctx context.Context, h Hypervisor, netNsCreated bool, netNsPath string) error {
-	return fmt.Errorf("PhysicalEndpoint does not support Hot detach")
+func (endpoint *PhysicalEndpoint) HotDetach(ctx context.Context, s *Sandbox, netNsCreated bool, netNsPath string) error {
+	span, _ := physicalTrace(ctx, "HotDetach", endpoint)
+	defer span.End()
+
+	var vfioPath string
+	var err error
+
+	if vfioPath, err = drivers.GetVFIODevPath(endpoint.BDF); err != nil {
+		return err
+	}
+
+	c, err := resCtrl.DeviceToCgroupDeviceRule(vfioPath)
+	if err != nil {
+		return err
+	}
+
+	d := config.DeviceInfo{
+		ContainerPath: vfioPath,
+		DevType:       string(c.Type),
+		Major:         c.Major,
+		Minor:         c.Minor,
+		ColdPlug:      false,
+	}
+
+	device := s.devManager.FindDevice(&d)
+	s.devManager.RemoveDevice(device.DeviceID())
+
+	// We do not need to enter the network namespace to bind back the
+	// physical interface to host driver.
+	return bindNICToHost(endpoint)
 }
 
 // isPhysicalIface checks if an interface is a physical device.

src/runtime/virtcontainers/physical_endpoint_test.go

@@ -42,9 +42,11 @@ func TestPhysicalEndpoint_HotDetach(t *testing.T) {
 		HardAddr:  net.HardwareAddr{0x02, 0x00, 0xca, 0xfe, 0x00, 0x04}.String(),
 	}
 
-	h := &mockHypervisor{}
+	s := &Sandbox{
+		hypervisor: &mockHypervisor{},
+	}
 
-	err := v.HotDetach(context.Background(), h, true, "")
+	err := v.HotDetach(context.Background(), s, true, "")
 	assert.Error(err)
 }
 

src/runtime/virtcontainers/tap_endpoint.go

@@ -112,7 +112,7 @@ func (endpoint *TapEndpoint) HotAttach(ctx context.Context, s *Sandbox) error {
 }
 
 // HotDetach for the tap endpoint uses hot pull device
-func (endpoint *TapEndpoint) HotDetach(ctx context.Context, h Hypervisor, netNsCreated bool, netNsPath string) error {
+func (endpoint *TapEndpoint) HotDetach(ctx context.Context, s *Sandbox, netNsCreated bool, netNsPath string) error {
 	networkLogger().Info("Hot detaching tap endpoint")
 
 	span, ctx := tapTrace(ctx, "HotDetach", endpoint)
@@ -124,6 +124,7 @@ func (endpoint *TapEndpoint) HotDetach(ctx context.Context, h Hypervisor, netNsC
 		networkLogger().WithError(err).Warn("Error un-bridging tap ep")
 	}
 
+	h := s.hypervisor
 	if _, err := h.HotplugRemoveDevice(ctx, endpoint, NetDev); err != nil {
 		networkLogger().WithError(err).Error("Error detach tap ep")
 		return err

src/runtime/virtcontainers/tuntap_endpoint.go

@@ -123,7 +123,7 @@ func (endpoint *TuntapEndpoint) HotAttach(ctx context.Context, s *Sandbox) error
 }
 
 // HotDetach for the tun/tap endpoint uses hot pull device
-func (endpoint *TuntapEndpoint) HotDetach(ctx context.Context, h Hypervisor, netNsCreated bool, netNsPath string) error {
+func (endpoint *TuntapEndpoint) HotDetach(ctx context.Context, s *Sandbox, netNsCreated bool, netNsPath string) error {
 	networkLogger().Info("Hot detaching tun/tap endpoint")
 
 	span, ctx := tuntapTrace(ctx, "HotDetach", endpoint)
@@ -135,6 +135,7 @@ func (endpoint *TuntapEndpoint) HotDetach(ctx context.Context, h Hypervisor, net
 		networkLogger().WithError(err).Warn("Error un-bridging tun/tap ep")
 	}
 
+	h := s.hypervisor
 	if _, err := h.HotplugRemoveDevice(ctx, endpoint, NetDev); err != nil {
 		networkLogger().WithError(err).Error("Error detach tun/tap ep")
 		return err

src/runtime/virtcontainers/veth_endpoint.go

@@ -144,7 +144,7 @@ func (endpoint *VethEndpoint) HotAttach(ctx context.Context, s *Sandbox) error {
 }
 
 // HotDetach for the veth endpoint uses hot pull device
-func (endpoint *VethEndpoint) HotDetach(ctx context.Context, h Hypervisor, netNsCreated bool, netNsPath string) error {
+func (endpoint *VethEndpoint) HotDetach(ctx context.Context, s *Sandbox, netNsCreated bool, netNsPath string) error {
 	if !netNsCreated {
 		return nil
 	}
@@ -158,6 +158,7 @@ func (endpoint *VethEndpoint) HotDetach(ctx context.Context, h Hypervisor, netNs
 		networkLogger().WithError(err).Warn("Error un-bridging virtual ep")
 	}
 
+	h := s.hypervisor
 	if _, err := h.HotplugRemoveDevice(ctx, endpoint, NetDev); err != nil {
 		networkLogger().WithError(err).Error("Error detach virtual ep")
 		return err

src/runtime/virtcontainers/vhostuser_endpoint.go

@@ -112,7 +112,7 @@ func (endpoint *VhostUserEndpoint) HotAttach(ctx context.Context, s *Sandbox) er
 }
 
 // HotDetach for vhostuser endpoint not supported yet
-func (endpoint *VhostUserEndpoint) HotDetach(ctx context.Context, h Hypervisor, netNsCreated bool, netNsPath string) error {
+func (endpoint *VhostUserEndpoint) HotDetach(ctx context.Context, s *Sandbox, netNsCreated bool, netNsPath string) error {
 	return fmt.Errorf("VhostUserEndpoint does not support Hot detach")
 }
 

src/runtime/virtcontainers/vhostuser_endpoint_test.go

@@ -113,9 +113,11 @@ func TestVhostUserEndpoint_HotDetach(t *testing.T) {
 		EndpointType: VhostUserEndpointType,
 	}
 
-	h := &mockHypervisor{}
+	s := &Sandbox{
+		hypervisor: &mockHypervisor{},
+	}
 
-	err := v.HotDetach(context.Background(), h, true, "")
+	err := v.HotDetach(context.Background(), s, true, "")
 	assert.Error(err)
 }