virtcontainers: Unify Network endpoints management interface

And only have AddEndpoints/RemoveEndpoints for all cases (single
endpoint vs all of them, hotplug or not).

Signed-off-by: Samuel Ortiz <s.ortiz@apple.com>

src/runtime/virtcontainers/endpoint.go

@@ -228,3 +228,13 @@ func loadTuntapIf(tuntapif *persistapi.TuntapInterface) *TuntapInterface {
 		},
 	}
 }
+
+func findEndpoint(e Endpoint, endpoints []Endpoint) (Endpoint, int) {
+	for idx, ep := range endpoints {
+		if ep.HardwareAddr() == e.HardwareAddr() {
+			return ep, idx
+		}
+	}
+
+	return nil, 0
+}

src/runtime/virtcontainers/network.go

@@ -97,6 +97,7 @@ type NetlinkIface struct {
 type NetworkInfo struct {
 	Iface     NetlinkIface
 	DNS       DNSInfo
+	Link      netlink.Link
 	Addrs     []netlink.Addr
 	Routes    []netlink.Route
 	Neighbors []netlink.Neigh
@@ -197,19 +198,16 @@ type NetworkConfig struct {
 }
 
 type Network interface {
-	// Add adds all needed networking endpoints to a sandbox's network
-	Add(context.Context, *Sandbox, bool) error
+	// AddEndpoint adds endpoints to a sandbox's network.
+	// If the NetworkInfo slice is empty, implementations are expected to scan
+	// the sandbox's network for all existing endpoints.
+	AddEndpoints(context.Context, *Sandbox, []NetworkInfo, bool) ([]Endpoint, error)
 
-	// AddEndpoint adds one single endpoint to a sandbox's network.
-	AddEndpoint(context.Context, *Sandbox, NetworkInfo, netlink.Link, bool) (Endpoint, error)
-
-	// Remove removes all the networking endpoints from a sandbox's network.
+	// RemoveEndpoints removes endpoints from the sandbox's network.
+	// If the the endpoint slice is empty, all endpoints will be removed.
 	// If the network has been created by virtcontainers, Remove also deletes
 	// the network.
-	Remove(context.Context) error
-
-	// RemoveEndpoint removes one single endpoint from the sandbox's network.
-	RemoveEndpoint(context.Context, *Sandbox, int, bool) error
+	RemoveEndpoints(context.Context, *Sandbox, []Endpoint, bool) error
 
 	// Run runs a callback in a sandbox's network.
 	Run(context.Context, func() error) error

src/runtime/virtcontainers/network_linux.go

@@ -107,7 +107,7 @@ func (n *LinuxNetwork) trace(ctx context.Context, name string) (otelTrace.Span,
 	return networkTrace(ctx, name, nil)
 }
 
-func (n *LinuxNetwork) AddEndpoint(ctx context.Context, s *Sandbox, netInfo NetworkInfo, link netlink.Link, hotplug bool) (Endpoint, error) {
+func (n *LinuxNetwork) addSingleEndpoint(ctx context.Context, s *Sandbox, netInfo NetworkInfo, hotplug bool) (Endpoint, error) {
 	var endpoint Endpoint
 	// TODO: This is the incoming interface
 	// based on the incoming interface we should create
@@ -147,8 +147,8 @@ func (n *LinuxNetwork) AddEndpoint(ctx context.Context, s *Sandbox, netInfo Netw
 			networkLogger().Info("tap interface found")
 			endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name)
 		} else if netInfo.Iface.Type == "tuntap" {
-			if link != nil {
-				switch link.(*netlink.Tuntap).Mode {
+			if netInfo.Link != nil {
+				switch netInfo.Link.(*netlink.Tuntap).Mode {
 				case 0:
 					// mount /sys/class/net to get links
 					return nil, fmt.Errorf("Network device mode not determined correctly. Mount sysfs in caller")
@@ -158,7 +158,7 @@ func (n *LinuxNetwork) AddEndpoint(ctx context.Context, s *Sandbox, netInfo Netw
 					networkLogger().Info("tuntap tap interface found")
 					endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, n.interworkingModel)
 				default:
-					return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode)
+					return nil, fmt.Errorf("tuntap network %v mode unsupported", netInfo.Link.(*netlink.Tuntap).Mode)
 				}
 			}
 		} else if netInfo.Iface.Type == "veth" {
@@ -217,7 +217,7 @@ func (n *LinuxNetwork) AddEndpoint(ctx context.Context, s *Sandbox, netInfo Netw
 	return endpoint, nil
 }
 
-func (n *LinuxNetwork) RemoveEndpoint(ctx context.Context, s *Sandbox, idx int, hotplug bool) error {
+func (n *LinuxNetwork) removeSingleEndpoint(ctx context.Context, s *Sandbox, idx int, hotplug bool) error {
 	if idx > len(n.eps)-1 {
 		return fmt.Errorf("Endpoint index overflow")
 	}
@@ -261,7 +261,7 @@ func (n *LinuxNetwork) RemoveEndpoint(ctx context.Context, s *Sandbox, idx int,
 // Scan the networking namespace through netlink and then:
 // 1. Create the endpoints for the relevant interfaces found there.
 // 2. Attach them to the VM.
-func (n *LinuxNetwork) attachEndpoints(ctx context.Context, s *Sandbox, hotplug bool) error {
+func (n *LinuxNetwork) addAllEndpoints(ctx context.Context, s *Sandbox, hotplug bool) error {
 	netnsHandle, err := netns.GetFromPath(n.netNSPath)
 	if err != nil {
 		return err
@@ -298,7 +298,7 @@ func (n *LinuxNetwork) attachEndpoints(ctx context.Context, s *Sandbox, hotplug
 			continue
 		}
 
-		_, err = n.AddEndpoint(ctx, s, netInfo, link, hotplug)
+		_, err = n.addSingleEndpoint(ctx, s, netInfo, hotplug)
 		if err != nil {
 			return err
 		}
@@ -324,36 +324,57 @@ func (n *LinuxNetwork) Run(ctx context.Context, cb func() error) error {
 }
 
 // Add adds all needed interfaces inside the network namespace.
-func (n *LinuxNetwork) Add(ctx context.Context, s *Sandbox, hotplug bool) error {
-	span, ctx := n.trace(ctx, "Add")
+func (n *LinuxNetwork) AddEndpoints(ctx context.Context, s *Sandbox, endpointsInfo []NetworkInfo, hotplug bool) ([]Endpoint, error) {
+	span, ctx := n.trace(ctx, "AddEndpoints")
 	katatrace.AddTags(span, "type", n.interworkingModel.GetModel())
 	defer span.End()
 
-	if err := n.attachEndpoints(ctx, s, hotplug); err != nil {
-		return err
+	if endpointsInfo == nil {
+		if err := n.addAllEndpoints(ctx, s, hotplug); err != nil {
+			return nil, err
+		}
+	} else {
+		for _, ep := range endpointsInfo {
+			if _, err := n.addSingleEndpoint(ctx, s, ep, hotplug); err != nil {
+				n.eps = nil
+				return nil, err
+			}
+		}
 	}
 
 	katatrace.AddTags(span, "endpoints", n.eps, "hotplug", hotplug)
-	networkLogger().Debug("Network added")
+	networkLogger().Debug("Endpoints added")
 
-	return nil
+	return n.eps, nil
 }
 
 // Remove network endpoints in the network namespace. It also deletes the network
 // namespace in case the namespace has been created by us.
-func (n *LinuxNetwork) Remove(ctx context.Context) error {
-	span, ctx := n.trace(ctx, "Remove")
+func (n *LinuxNetwork) RemoveEndpoints(ctx context.Context, s *Sandbox, endpoints []Endpoint, hotplug bool) error {
+	span, ctx := n.trace(ctx, "RemoveEndpoints")
 	defer span.End()
 
-	for i := range n.eps {
-		if err := n.RemoveEndpoint(ctx, nil, i, false); err != nil {
+	eps := n.eps
+	if endpoints != nil {
+		eps = endpoints
+	}
+
+	for idx, ep := range eps {
+		if endpoints != nil {
+			new_ep, _ := findEndpoint(ep, n.eps)
+			if new_ep == nil {
+				continue
+			}
+		}
+
+		if err := n.removeSingleEndpoint(ctx, s, idx, hotplug); err != nil {
 			return err
 		}
 	}
 
-	networkLogger().Debug("Network removed")
+	networkLogger().Debug("Endpoints removed")
 
-	if n.netNSCreated {
+	if n.netNSCreated && endpoints == nil {
 		networkLogger().Infof("Network namespace %q deleted", n.netNSPath)
 		return deleteNetNS(n.netNSPath)
 	}
@@ -1056,6 +1077,7 @@ func networkInfoFromLink(handle *netlink.Handle, link netlink.Link) (NetworkInfo
 		Addrs:     addrs,
 		Routes:    routes,
 		Neighbors: neighbors,
+		Link:      link,
 	}, nil
 }
 

src/runtime/virtcontainers/sandbox.go

@@ -818,7 +818,7 @@ func (s *Sandbox) createNetwork(ctx context.Context) error {
 	// after vm is started.
 	if s.factory == nil {
 		// Add the network
-		if err := s.network.Add(ctx, s, false); err != nil {
+		if _, err := s.network.AddEndpoints(ctx, s, nil, false); err != nil {
 			return err
 		}
 	}
@@ -853,7 +853,7 @@ func (s *Sandbox) removeNetwork(ctx context.Context) error {
 	span, ctx := katatrace.Trace(ctx, s.Logger(), "removeNetwork", sandboxTracingTags, map[string]string{"sandbox_id": s.id})
 	defer span.End()
 
-	return s.network.Remove(ctx)
+	return s.network.RemoveEndpoints(ctx, s, nil, false)
 }
 
 func (s *Sandbox) generateNetInfo(inf *pbTypes.Interface) (NetworkInfo, error) {
@@ -893,21 +893,24 @@ func (s *Sandbox) AddInterface(ctx context.Context, inf *pbTypes.Interface) (*pb
 		return nil, err
 	}
 
-	endpoint, err := s.network.AddEndpoint(ctx, s, netInfo, nil, true)
+	endpoints, err := s.network.AddEndpoints(ctx, s, []NetworkInfo{netInfo}, true)
 	if err != nil {
 		return nil, err
 	}
 
 	defer func() {
 		if err != nil {
-			if errDetach := s.network.RemoveEndpoint(ctx, s, len(s.network.Endpoints())-1, true); err != nil {
-				s.Logger().WithField("endpoint-type", endpoint.Type()).WithError(errDetach).Error("rollback hot attaching endpoint failed")
+			eps := s.network.Endpoints()
+			// The newly added endpoint is last.
+			added_ep := eps[len(eps)-1]
+			if errDetach := s.network.RemoveEndpoints(ctx, s, []Endpoint{added_ep}, true); err != nil {
+				s.Logger().WithField("endpoint-type", added_ep.Type()).WithError(errDetach).Error("rollback hot attaching endpoint failed")
 			}
 		}
 	}()
 
 	// Add network for vm
-	inf.PciPath = endpoint.PciPath().String()
+	inf.PciPath = endpoints[0].PciPath().String()
 	result, err := s.agent.updateInterface(ctx, inf)
 	if err != nil {
 		return nil, err
@@ -923,10 +926,10 @@ func (s *Sandbox) AddInterface(ctx context.Context, inf *pbTypes.Interface) (*pb
 
 // RemoveInterface removes a nic of the sandbox.
 func (s *Sandbox) RemoveInterface(ctx context.Context, inf *pbTypes.Interface) (*pbTypes.Interface, error) {
-	for i, endpoint := range s.network.Endpoints() {
+	for _, endpoint := range s.network.Endpoints() {
 		if endpoint.HardwareAddr() == inf.HwAddr {
 			s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Hot detaching endpoint")
-			if err := s.network.RemoveEndpoint(ctx, s, i, true); err != nil {
+			if err := s.network.RemoveEndpoints(ctx, s, []Endpoint{endpoint}, true); err != nil {
 				return inf, err
 			}
 
@@ -1205,7 +1208,7 @@ func (s *Sandbox) startVM(ctx context.Context) (err error) {
 	// In case of vm factory, network interfaces are hotplugged
 	// after vm is started.
 	if s.factory != nil {
-		if err := s.network.Add(ctx, s, true); err != nil {
+		if _, err := s.network.AddEndpoints(ctx, s, nil, true); err != nil {
 			return err
 		}
 	}