From 1636c201f488eeb76a91ef25f8c8bbe2d945dd39 Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Wed, 6 Dec 2023 21:40:15 -0800 Subject: [PATCH] network: Implement network hotunplug for physical endpoints Similar to HotAttach, the HotDetach method signature for network endoints needs to be changed as well to allow for the method to make use of device manager to manage the hot unplug of physical network devices. Signed-off-by: Archana Shinde --- src/runtime/pkg/device/api/interface.go | 1 + src/runtime/pkg/device/manager/manager.go | 4 +-- src/runtime/virtcontainers/endpoint.go | 2 +- src/runtime/virtcontainers/ipvlan_endpoint.go | 3 +- .../virtcontainers/macvlan_endpoint.go | 3 +- .../virtcontainers/macvtap_endpoint.go | 2 +- src/runtime/virtcontainers/network_linux.go | 2 +- .../virtcontainers/physical_endpoint.go | 32 +++++++++++++++++-- .../virtcontainers/physical_endpoint_test.go | 6 ++-- src/runtime/virtcontainers/tap_endpoint.go | 3 +- src/runtime/virtcontainers/tuntap_endpoint.go | 3 +- src/runtime/virtcontainers/veth_endpoint.go | 3 +- .../virtcontainers/vhostuser_endpoint.go | 2 +- .../virtcontainers/vhostuser_endpoint_test.go | 6 ++-- 14 files changed, 55 insertions(+), 17 deletions(-) diff --git a/src/runtime/pkg/device/api/interface.go b/src/runtime/pkg/device/api/interface.go index 074980f44f..6ec7b2885e 100644 --- a/src/runtime/pkg/device/api/interface.go +++ b/src/runtime/pkg/device/api/interface.go @@ -94,4 +94,5 @@ type DeviceManager interface { GetDeviceByID(string) Device GetAllDevices() []Device LoadDevices([]config.DeviceState) + FindDevice(*config.DeviceInfo) Device } diff --git a/src/runtime/pkg/device/manager/manager.go b/src/runtime/pkg/device/manager/manager.go index 3e687cde37..667122f288 100644 --- a/src/runtime/pkg/device/manager/manager.go +++ b/src/runtime/pkg/device/manager/manager.go @@ -82,7 +82,7 @@ func NewDeviceManager(blockDriver string, vhostUserStoreEnabled bool, vhostUserS return dm } -func (dm *deviceManager) findDevice(devInfo *config.DeviceInfo) api.Device { +func (dm *deviceManager) FindDevice(devInfo *config.DeviceInfo) api.Device { // For devices with a major of -1, we use the host path to find existing instances. if devInfo.Major == -1 { for _, dev := range dm.devices { @@ -121,7 +121,7 @@ func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (dev api.Device } }() - if existingDev := dm.findDevice(&devInfo); existingDev != nil { + if existingDev := dm.FindDevice(&devInfo); existingDev != nil { return existingDev, nil } diff --git a/src/runtime/virtcontainers/endpoint.go b/src/runtime/virtcontainers/endpoint.go index cbf384f008..ef50a8eb7e 100644 --- a/src/runtime/virtcontainers/endpoint.go +++ b/src/runtime/virtcontainers/endpoint.go @@ -27,7 +27,7 @@ type Endpoint interface { Attach(context.Context, *Sandbox) error Detach(ctx context.Context, netNsCreated bool, netNsPath string) error HotAttach(context.Context, *Sandbox) error - HotDetach(ctx context.Context, h Hypervisor, netNsCreated bool, netNsPath string) error + HotDetach(ctx context.Context, s *Sandbox, netNsCreated bool, netNsPath string) error save() persistapi.NetworkEndpoint load(persistapi.NetworkEndpoint) diff --git a/src/runtime/virtcontainers/ipvlan_endpoint.go b/src/runtime/virtcontainers/ipvlan_endpoint.go index 74c41576a8..9ea66af9d1 100644 --- a/src/runtime/virtcontainers/ipvlan_endpoint.go +++ b/src/runtime/virtcontainers/ipvlan_endpoint.go @@ -143,7 +143,7 @@ func (endpoint *IPVlanEndpoint) HotAttach(ctx context.Context, s *Sandbox) error return nil } -func (endpoint *IPVlanEndpoint) HotDetach(ctx context.Context, h Hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *IPVlanEndpoint) HotDetach(ctx context.Context, s *Sandbox, netNsCreated bool, netNsPath string) error { if !netNsCreated { return nil } @@ -157,6 +157,7 @@ func (endpoint *IPVlanEndpoint) HotDetach(ctx context.Context, h Hypervisor, net networkLogger().WithError(err).Warn("Error un-bridging ipvlan ep") } + h := s.hypervisor if _, err := h.HotplugRemoveDevice(ctx, endpoint, NetDev); err != nil { networkLogger().WithError(err).Error("Error detach ipvlan ep") return err diff --git a/src/runtime/virtcontainers/macvlan_endpoint.go b/src/runtime/virtcontainers/macvlan_endpoint.go index 7ffcc582c8..71a223e257 100644 --- a/src/runtime/virtcontainers/macvlan_endpoint.go +++ b/src/runtime/virtcontainers/macvlan_endpoint.go @@ -140,7 +140,7 @@ func (endpoint *MacvlanEndpoint) HotAttach(ctx context.Context, s *Sandbox) erro return nil } -func (endpoint *MacvlanEndpoint) HotDetach(ctx context.Context, h Hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *MacvlanEndpoint) HotDetach(ctx context.Context, s *Sandbox, netNsCreated bool, netNsPath string) error { if !netNsCreated { return nil } @@ -154,6 +154,7 @@ func (endpoint *MacvlanEndpoint) HotDetach(ctx context.Context, h Hypervisor, ne networkLogger().WithError(err).Warn("Error un-bridging macvlan ep") } + h := s.hypervisor if _, err := h.HotplugRemoveDevice(ctx, endpoint, NetDev); err != nil { networkLogger().WithError(err).Error("Error detach macvlan ep") return err diff --git a/src/runtime/virtcontainers/macvtap_endpoint.go b/src/runtime/virtcontainers/macvtap_endpoint.go index c53d2603df..27fee59f0b 100644 --- a/src/runtime/virtcontainers/macvtap_endpoint.go +++ b/src/runtime/virtcontainers/macvtap_endpoint.go @@ -98,7 +98,7 @@ func (endpoint *MacvtapEndpoint) HotAttach(ctx context.Context, s *Sandbox) erro } // HotDetach for macvtap endpoint not supported yet -func (endpoint *MacvtapEndpoint) HotDetach(ctx context.Context, h Hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *MacvtapEndpoint) HotDetach(ctx context.Context, s *Sandbox, netNsCreated bool, netNsPath string) error { return fmt.Errorf("MacvtapEndpoint does not support Hot detach") } diff --git a/src/runtime/virtcontainers/network_linux.go b/src/runtime/virtcontainers/network_linux.go index 7160fbf678..8346542e8f 100644 --- a/src/runtime/virtcontainers/network_linux.go +++ b/src/runtime/virtcontainers/network_linux.go @@ -265,7 +265,7 @@ func (n *LinuxNetwork) removeSingleEndpoint(ctx context.Context, s *Sandbox, end // if required. networkLogger().WithField("endpoint-type", endpoint.Type()).Info("Detaching endpoint") if hotplug && s != nil { - if err := endpoint.HotDetach(ctx, s.hypervisor, n.netNSCreated, n.netNSPath); err != nil { + if err := endpoint.HotDetach(ctx, s, n.netNSCreated, n.netNSPath); err != nil { return err } } else { diff --git a/src/runtime/virtcontainers/physical_endpoint.go b/src/runtime/virtcontainers/physical_endpoint.go index db098492b6..de6c3cc87a 100644 --- a/src/runtime/virtcontainers/physical_endpoint.go +++ b/src/runtime/virtcontainers/physical_endpoint.go @@ -152,8 +152,36 @@ func (endpoint *PhysicalEndpoint) HotAttach(ctx context.Context, s *Sandbox) err } // HotDetach for physical endpoint not supported yet -func (endpoint *PhysicalEndpoint) HotDetach(ctx context.Context, h Hypervisor, netNsCreated bool, netNsPath string) error { - return fmt.Errorf("PhysicalEndpoint does not support Hot detach") +func (endpoint *PhysicalEndpoint) HotDetach(ctx context.Context, s *Sandbox, netNsCreated bool, netNsPath string) error { + span, _ := physicalTrace(ctx, "HotDetach", endpoint) + defer span.End() + + var vfioPath string + var err error + + if vfioPath, err = drivers.GetVFIODevPath(endpoint.BDF); err != nil { + return err + } + + c, err := resCtrl.DeviceToCgroupDeviceRule(vfioPath) + if err != nil { + return err + } + + d := config.DeviceInfo{ + ContainerPath: vfioPath, + DevType: string(c.Type), + Major: c.Major, + Minor: c.Minor, + ColdPlug: false, + } + + device := s.devManager.FindDevice(&d) + s.devManager.RemoveDevice(device.DeviceID()) + + // We do not need to enter the network namespace to bind back the + // physical interface to host driver. + return bindNICToHost(endpoint) } // isPhysicalIface checks if an interface is a physical device. diff --git a/src/runtime/virtcontainers/physical_endpoint_test.go b/src/runtime/virtcontainers/physical_endpoint_test.go index 5aed350e8e..d3a0966079 100644 --- a/src/runtime/virtcontainers/physical_endpoint_test.go +++ b/src/runtime/virtcontainers/physical_endpoint_test.go @@ -42,9 +42,11 @@ func TestPhysicalEndpoint_HotDetach(t *testing.T) { HardAddr: net.HardwareAddr{0x02, 0x00, 0xca, 0xfe, 0x00, 0x04}.String(), } - h := &mockHypervisor{} + s := &Sandbox{ + hypervisor: &mockHypervisor{}, + } - err := v.HotDetach(context.Background(), h, true, "") + err := v.HotDetach(context.Background(), s, true, "") assert.Error(err) } diff --git a/src/runtime/virtcontainers/tap_endpoint.go b/src/runtime/virtcontainers/tap_endpoint.go index ff7dbe3af0..14ebebfece 100644 --- a/src/runtime/virtcontainers/tap_endpoint.go +++ b/src/runtime/virtcontainers/tap_endpoint.go @@ -112,7 +112,7 @@ func (endpoint *TapEndpoint) HotAttach(ctx context.Context, s *Sandbox) error { } // HotDetach for the tap endpoint uses hot pull device -func (endpoint *TapEndpoint) HotDetach(ctx context.Context, h Hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *TapEndpoint) HotDetach(ctx context.Context, s *Sandbox, netNsCreated bool, netNsPath string) error { networkLogger().Info("Hot detaching tap endpoint") span, ctx := tapTrace(ctx, "HotDetach", endpoint) @@ -124,6 +124,7 @@ func (endpoint *TapEndpoint) HotDetach(ctx context.Context, h Hypervisor, netNsC networkLogger().WithError(err).Warn("Error un-bridging tap ep") } + h := s.hypervisor if _, err := h.HotplugRemoveDevice(ctx, endpoint, NetDev); err != nil { networkLogger().WithError(err).Error("Error detach tap ep") return err diff --git a/src/runtime/virtcontainers/tuntap_endpoint.go b/src/runtime/virtcontainers/tuntap_endpoint.go index 3631d3d762..6c67b02360 100644 --- a/src/runtime/virtcontainers/tuntap_endpoint.go +++ b/src/runtime/virtcontainers/tuntap_endpoint.go @@ -123,7 +123,7 @@ func (endpoint *TuntapEndpoint) HotAttach(ctx context.Context, s *Sandbox) error } // HotDetach for the tun/tap endpoint uses hot pull device -func (endpoint *TuntapEndpoint) HotDetach(ctx context.Context, h Hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *TuntapEndpoint) HotDetach(ctx context.Context, s *Sandbox, netNsCreated bool, netNsPath string) error { networkLogger().Info("Hot detaching tun/tap endpoint") span, ctx := tuntapTrace(ctx, "HotDetach", endpoint) @@ -135,6 +135,7 @@ func (endpoint *TuntapEndpoint) HotDetach(ctx context.Context, h Hypervisor, net networkLogger().WithError(err).Warn("Error un-bridging tun/tap ep") } + h := s.hypervisor if _, err := h.HotplugRemoveDevice(ctx, endpoint, NetDev); err != nil { networkLogger().WithError(err).Error("Error detach tun/tap ep") return err diff --git a/src/runtime/virtcontainers/veth_endpoint.go b/src/runtime/virtcontainers/veth_endpoint.go index 7102df8c5b..566786d4cb 100644 --- a/src/runtime/virtcontainers/veth_endpoint.go +++ b/src/runtime/virtcontainers/veth_endpoint.go @@ -144,7 +144,7 @@ func (endpoint *VethEndpoint) HotAttach(ctx context.Context, s *Sandbox) error { } // HotDetach for the veth endpoint uses hot pull device -func (endpoint *VethEndpoint) HotDetach(ctx context.Context, h Hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *VethEndpoint) HotDetach(ctx context.Context, s *Sandbox, netNsCreated bool, netNsPath string) error { if !netNsCreated { return nil } @@ -158,6 +158,7 @@ func (endpoint *VethEndpoint) HotDetach(ctx context.Context, h Hypervisor, netNs networkLogger().WithError(err).Warn("Error un-bridging virtual ep") } + h := s.hypervisor if _, err := h.HotplugRemoveDevice(ctx, endpoint, NetDev); err != nil { networkLogger().WithError(err).Error("Error detach virtual ep") return err diff --git a/src/runtime/virtcontainers/vhostuser_endpoint.go b/src/runtime/virtcontainers/vhostuser_endpoint.go index ce884fe428..5b079629db 100644 --- a/src/runtime/virtcontainers/vhostuser_endpoint.go +++ b/src/runtime/virtcontainers/vhostuser_endpoint.go @@ -112,7 +112,7 @@ func (endpoint *VhostUserEndpoint) HotAttach(ctx context.Context, s *Sandbox) er } // HotDetach for vhostuser endpoint not supported yet -func (endpoint *VhostUserEndpoint) HotDetach(ctx context.Context, h Hypervisor, netNsCreated bool, netNsPath string) error { +func (endpoint *VhostUserEndpoint) HotDetach(ctx context.Context, s *Sandbox, netNsCreated bool, netNsPath string) error { return fmt.Errorf("VhostUserEndpoint does not support Hot detach") } diff --git a/src/runtime/virtcontainers/vhostuser_endpoint_test.go b/src/runtime/virtcontainers/vhostuser_endpoint_test.go index 0a8448c32b..fe8c171884 100644 --- a/src/runtime/virtcontainers/vhostuser_endpoint_test.go +++ b/src/runtime/virtcontainers/vhostuser_endpoint_test.go @@ -113,9 +113,11 @@ func TestVhostUserEndpoint_HotDetach(t *testing.T) { EndpointType: VhostUserEndpointType, } - h := &mockHypervisor{} + s := &Sandbox{ + hypervisor: &mockHypervisor{}, + } - err := v.HotDetach(context.Background(), h, true, "") + err := v.HotDetach(context.Background(), s, true, "") assert.Error(err) }