From 1e304e6307a7b45884f823b678d5659569efda09 Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Wed, 6 Dec 2023 12:09:56 -0800 Subject: [PATCH] network: Implement hotplug for physical endpoints Enable physical network interfaces to be hotplugged. For this, we need to change the signature of the HotAttach method to make use of Sandbox instead of Hypervisor. Similar approach was followed for Attach method, but this change was overlooked for HotAttach. The signature change is required in order to make use of device manager and receiver for physical network enpoints. Fixes: #8405 Signed-off-by: Archana Shinde --- src/runtime/virtcontainers/endpoint.go | 2 +- src/runtime/virtcontainers/ipvlan_endpoint.go | 3 +- .../virtcontainers/macvlan_endpoint.go | 3 +- .../virtcontainers/macvtap_endpoint.go | 2 +- src/runtime/virtcontainers/network_linux.go | 2 +- .../virtcontainers/physical_endpoint.go | 28 +++++++++++++++++-- .../virtcontainers/physical_endpoint_test.go | 6 ++-- src/runtime/virtcontainers/tap_endpoint.go | 3 +- src/runtime/virtcontainers/tuntap_endpoint.go | 3 +- src/runtime/virtcontainers/veth_endpoint.go | 3 +- .../virtcontainers/vhostuser_endpoint.go | 2 +- .../virtcontainers/vhostuser_endpoint_test.go | 6 ++-- 12 files changed, 48 insertions(+), 15 deletions(-) diff --git a/src/runtime/virtcontainers/endpoint.go b/src/runtime/virtcontainers/endpoint.go index 6c24b79ac4..cbf384f008 100644 --- a/src/runtime/virtcontainers/endpoint.go +++ b/src/runtime/virtcontainers/endpoint.go @@ -26,7 +26,7 @@ type Endpoint interface { SetPciPath(vcTypes.PciPath) Attach(context.Context, *Sandbox) error Detach(ctx context.Context, netNsCreated bool, netNsPath string) error - HotAttach(ctx context.Context, h Hypervisor) error + HotAttach(context.Context, *Sandbox) error HotDetach(ctx context.Context, h Hypervisor, netNsCreated bool, netNsPath string) error save() persistapi.NetworkEndpoint diff --git a/src/runtime/virtcontainers/ipvlan_endpoint.go b/src/runtime/virtcontainers/ipvlan_endpoint.go index 4c495ba67a..74c41576a8 100644 --- a/src/runtime/virtcontainers/ipvlan_endpoint.go +++ b/src/runtime/virtcontainers/ipvlan_endpoint.go @@ -125,10 +125,11 @@ func (endpoint *IPVlanEndpoint) Detach(ctx context.Context, netNsCreated bool, n }) } -func (endpoint *IPVlanEndpoint) HotAttach(ctx context.Context, h Hypervisor) error { +func (endpoint *IPVlanEndpoint) HotAttach(ctx context.Context, s *Sandbox) error { span, ctx := ipvlanTrace(ctx, "HotAttach", endpoint) defer span.End() + h := s.hypervisor if err := xConnectVMNetwork(ctx, endpoint, h); err != nil { networkLogger().WithError(err).Error("Error bridging ipvlan ep") return err diff --git a/src/runtime/virtcontainers/macvlan_endpoint.go b/src/runtime/virtcontainers/macvlan_endpoint.go index 974019fbb6..7ffcc582c8 100644 --- a/src/runtime/virtcontainers/macvlan_endpoint.go +++ b/src/runtime/virtcontainers/macvlan_endpoint.go @@ -122,10 +122,11 @@ func (endpoint *MacvlanEndpoint) Detach(ctx context.Context, netNsCreated bool, }) } -func (endpoint *MacvlanEndpoint) HotAttach(ctx context.Context, h Hypervisor) error { +func (endpoint *MacvlanEndpoint) HotAttach(ctx context.Context, s *Sandbox) error { span, ctx := macvlanTrace(ctx, "HotAttach", endpoint) defer span.End() + h := s.hypervisor if err := xConnectVMNetwork(ctx, endpoint, h); err != nil { networkLogger().WithError(err).Error("Error bridging macvlan ep") return err diff --git a/src/runtime/virtcontainers/macvtap_endpoint.go b/src/runtime/virtcontainers/macvtap_endpoint.go index b3fd9de7a1..c53d2603df 100644 --- a/src/runtime/virtcontainers/macvtap_endpoint.go +++ b/src/runtime/virtcontainers/macvtap_endpoint.go @@ -93,7 +93,7 @@ func (endpoint *MacvtapEndpoint) Detach(ctx context.Context, netNsCreated bool, } // HotAttach for macvtap endpoint not supported yet -func (endpoint *MacvtapEndpoint) HotAttach(ctx context.Context, h Hypervisor) error { +func (endpoint *MacvtapEndpoint) HotAttach(ctx context.Context, s *Sandbox) error { return fmt.Errorf("MacvtapEndpoint does not support Hot attach") } diff --git a/src/runtime/virtcontainers/network_linux.go b/src/runtime/virtcontainers/network_linux.go index f37f6f3de1..7160fbf678 100644 --- a/src/runtime/virtcontainers/network_linux.go +++ b/src/runtime/virtcontainers/network_linux.go @@ -202,7 +202,7 @@ func (n *LinuxNetwork) addSingleEndpoint(ctx context.Context, s *Sandbox, netInf networkLogger().WithField("endpoint-type", endpoint.Type()).WithField("hotplug", hotplug).Info("Attaching endpoint") if hotplug { - if err := endpoint.HotAttach(ctx, s.hypervisor); err != nil { + if err := endpoint.HotAttach(ctx, s); err != nil { return nil, err } } else { diff --git a/src/runtime/virtcontainers/physical_endpoint.go b/src/runtime/virtcontainers/physical_endpoint.go index 7891e21ff4..db098492b6 100644 --- a/src/runtime/virtcontainers/physical_endpoint.go +++ b/src/runtime/virtcontainers/physical_endpoint.go @@ -123,8 +123,32 @@ func (endpoint *PhysicalEndpoint) Detach(ctx context.Context, netNsCreated bool, } // HotAttach for physical endpoint not supported yet -func (endpoint *PhysicalEndpoint) HotAttach(ctx context.Context, h Hypervisor) error { - return fmt.Errorf("PhysicalEndpoint does not support Hot attach") +func (endpoint *PhysicalEndpoint) HotAttach(ctx context.Context, s *Sandbox) error { + span, ctx := physicalTrace(ctx, "HotAttach", endpoint) + defer span.End() + + // Unbind physical interface from host driver and bind to vfio + // so that it can be passed to the hypervisor. + vfioPath, err := bindNICToVFIO(endpoint) + if err != nil { + return err + } + + c, err := resCtrl.DeviceToCgroupDeviceRule(vfioPath) + if err != nil { + return err + } + + d := config.DeviceInfo{ + ContainerPath: vfioPath, + DevType: string(c.Type), + Major: c.Major, + Minor: c.Minor, + ColdPlug: false, + } + + _, err = s.AddDevice(ctx, d) + return err } // HotDetach for physical endpoint not supported yet diff --git a/src/runtime/virtcontainers/physical_endpoint_test.go b/src/runtime/virtcontainers/physical_endpoint_test.go index bba7f7d1c7..5aed350e8e 100644 --- a/src/runtime/virtcontainers/physical_endpoint_test.go +++ b/src/runtime/virtcontainers/physical_endpoint_test.go @@ -27,9 +27,11 @@ func TestPhysicalEndpoint_HotAttach(t *testing.T) { HardAddr: net.HardwareAddr{0x02, 0x00, 0xca, 0xfe, 0x00, 0x04}.String(), } - h := &mockHypervisor{} + s := &Sandbox{ + hypervisor: &mockHypervisor{}, + } - err := v.HotAttach(context.Background(), h) + err := v.HotAttach(context.Background(), s) assert.Error(err) } diff --git a/src/runtime/virtcontainers/tap_endpoint.go b/src/runtime/virtcontainers/tap_endpoint.go index 2f8ec33532..ff7dbe3af0 100644 --- a/src/runtime/virtcontainers/tap_endpoint.go +++ b/src/runtime/virtcontainers/tap_endpoint.go @@ -92,12 +92,13 @@ func (endpoint *TapEndpoint) Detach(ctx context.Context, netNsCreated bool, netN } // HotAttach for the tap endpoint uses hot plug device -func (endpoint *TapEndpoint) HotAttach(ctx context.Context, h Hypervisor) error { +func (endpoint *TapEndpoint) HotAttach(ctx context.Context, s *Sandbox) error { networkLogger().Info("Hot attaching tap endpoint") span, ctx := tapTrace(ctx, "HotAttach", endpoint) defer span.End() + h := s.hypervisor if err := tapNetwork(endpoint, h.HypervisorConfig().NumVCPUs(), h.HypervisorConfig().DisableVhostNet); err != nil { networkLogger().WithError(err).Error("Error bridging tap ep") return err diff --git a/src/runtime/virtcontainers/tuntap_endpoint.go b/src/runtime/virtcontainers/tuntap_endpoint.go index ddd371e9ea..3631d3d762 100644 --- a/src/runtime/virtcontainers/tuntap_endpoint.go +++ b/src/runtime/virtcontainers/tuntap_endpoint.go @@ -103,12 +103,13 @@ func (endpoint *TuntapEndpoint) Detach(ctx context.Context, netNsCreated bool, n } // HotAttach for the tun/tap endpoint uses hot plug device -func (endpoint *TuntapEndpoint) HotAttach(ctx context.Context, h Hypervisor) error { +func (endpoint *TuntapEndpoint) HotAttach(ctx context.Context, s *Sandbox) error { networkLogger().Info("Hot attaching tun/tap endpoint") span, ctx := tuntapTrace(ctx, "HotAttach", endpoint) defer span.End() + h := s.hypervisor if err := tuntapNetwork(endpoint, h.HypervisorConfig().NumVCPUs(), h.HypervisorConfig().DisableVhostNet); err != nil { networkLogger().WithError(err).Error("Error bridging tun/tap ep") return err diff --git a/src/runtime/virtcontainers/veth_endpoint.go b/src/runtime/virtcontainers/veth_endpoint.go index e3ed63d707..7102df8c5b 100644 --- a/src/runtime/virtcontainers/veth_endpoint.go +++ b/src/runtime/virtcontainers/veth_endpoint.go @@ -126,10 +126,11 @@ func (endpoint *VethEndpoint) Detach(ctx context.Context, netNsCreated bool, net } // HotAttach for the veth endpoint uses hot plug device -func (endpoint *VethEndpoint) HotAttach(ctx context.Context, h Hypervisor) error { +func (endpoint *VethEndpoint) HotAttach(ctx context.Context, s *Sandbox) error { span, ctx := vethTrace(ctx, "HotAttach", endpoint) defer span.End() + h := s.hypervisor if err := xConnectVMNetwork(ctx, endpoint, h); err != nil { networkLogger().WithError(err).Error("Error bridging virtual ep") return err diff --git a/src/runtime/virtcontainers/vhostuser_endpoint.go b/src/runtime/virtcontainers/vhostuser_endpoint.go index 6656f33dc2..ce884fe428 100644 --- a/src/runtime/virtcontainers/vhostuser_endpoint.go +++ b/src/runtime/virtcontainers/vhostuser_endpoint.go @@ -107,7 +107,7 @@ func (endpoint *VhostUserEndpoint) Detach(ctx context.Context, netNsCreated bool } // HotAttach for vhostuser endpoint not supported yet -func (endpoint *VhostUserEndpoint) HotAttach(ctx context.Context, h Hypervisor) error { +func (endpoint *VhostUserEndpoint) HotAttach(ctx context.Context, s *Sandbox) error { return fmt.Errorf("VhostUserEndpoint does not support Hot attach") } diff --git a/src/runtime/virtcontainers/vhostuser_endpoint_test.go b/src/runtime/virtcontainers/vhostuser_endpoint_test.go index a49a3e61e0..0a8448c32b 100644 --- a/src/runtime/virtcontainers/vhostuser_endpoint_test.go +++ b/src/runtime/virtcontainers/vhostuser_endpoint_test.go @@ -97,9 +97,11 @@ func TestVhostUserEndpoint_HotAttach(t *testing.T) { EndpointType: VhostUserEndpointType, } - h := &mockHypervisor{} + s := &Sandbox{ + hypervisor: &mockHypervisor{}, + } - err := v.HotAttach(context.Background(), h) + err := v.HotAttach(context.Background(), s) assert.Error(err) }