diff --git a/src/runtime/virtcontainers/endpoint.go b/src/runtime/virtcontainers/endpoint.go index 7786bb3e1f..6c24b79ac4 100644 --- a/src/runtime/virtcontainers/endpoint.go +++ b/src/runtime/virtcontainers/endpoint.go @@ -228,3 +228,13 @@ func loadTuntapIf(tuntapif *persistapi.TuntapInterface) *TuntapInterface { }, } } + +func findEndpoint(e Endpoint, endpoints []Endpoint) (Endpoint, int) { + for idx, ep := range endpoints { + if ep.HardwareAddr() == e.HardwareAddr() { + return ep, idx + } + } + + return nil, 0 +} diff --git a/src/runtime/virtcontainers/network.go b/src/runtime/virtcontainers/network.go index 81f5985e30..ee1e927d9a 100644 --- a/src/runtime/virtcontainers/network.go +++ b/src/runtime/virtcontainers/network.go @@ -97,6 +97,7 @@ type NetlinkIface struct { type NetworkInfo struct { Iface NetlinkIface DNS DNSInfo + Link netlink.Link Addrs []netlink.Addr Routes []netlink.Route Neighbors []netlink.Neigh @@ -197,19 +198,16 @@ type NetworkConfig struct { } type Network interface { - // Add adds all needed networking endpoints to a sandbox's network - Add(context.Context, *Sandbox, bool) error + // AddEndpoint adds endpoints to a sandbox's network. + // If the NetworkInfo slice is empty, implementations are expected to scan + // the sandbox's network for all existing endpoints. + AddEndpoints(context.Context, *Sandbox, []NetworkInfo, bool) ([]Endpoint, error) - // AddEndpoint adds one single endpoint to a sandbox's network. - AddEndpoint(context.Context, *Sandbox, NetworkInfo, netlink.Link, bool) (Endpoint, error) - - // Remove removes all the networking endpoints from a sandbox's network. + // RemoveEndpoints removes endpoints from the sandbox's network. + // If the the endpoint slice is empty, all endpoints will be removed. // If the network has been created by virtcontainers, Remove also deletes // the network. - Remove(context.Context) error - - // RemoveEndpoint removes one single endpoint from the sandbox's network. - RemoveEndpoint(context.Context, *Sandbox, int, bool) error + RemoveEndpoints(context.Context, *Sandbox, []Endpoint, bool) error // Run runs a callback in a sandbox's network. Run(context.Context, func() error) error diff --git a/src/runtime/virtcontainers/network_linux.go b/src/runtime/virtcontainers/network_linux.go index 8a9e72cddd..0e8429a2e3 100644 --- a/src/runtime/virtcontainers/network_linux.go +++ b/src/runtime/virtcontainers/network_linux.go @@ -107,7 +107,7 @@ func (n *LinuxNetwork) trace(ctx context.Context, name string) (otelTrace.Span, return networkTrace(ctx, name, nil) } -func (n *LinuxNetwork) AddEndpoint(ctx context.Context, s *Sandbox, netInfo NetworkInfo, link netlink.Link, hotplug bool) (Endpoint, error) { +func (n *LinuxNetwork) addSingleEndpoint(ctx context.Context, s *Sandbox, netInfo NetworkInfo, hotplug bool) (Endpoint, error) { var endpoint Endpoint // TODO: This is the incoming interface // based on the incoming interface we should create @@ -147,8 +147,8 @@ func (n *LinuxNetwork) AddEndpoint(ctx context.Context, s *Sandbox, netInfo Netw networkLogger().Info("tap interface found") endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name) } else if netInfo.Iface.Type == "tuntap" { - if link != nil { - switch link.(*netlink.Tuntap).Mode { + if netInfo.Link != nil { + switch netInfo.Link.(*netlink.Tuntap).Mode { case 0: // mount /sys/class/net to get links return nil, fmt.Errorf("Network device mode not determined correctly. Mount sysfs in caller") @@ -158,7 +158,7 @@ func (n *LinuxNetwork) AddEndpoint(ctx context.Context, s *Sandbox, netInfo Netw networkLogger().Info("tuntap tap interface found") endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, n.interworkingModel) default: - return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode) + return nil, fmt.Errorf("tuntap network %v mode unsupported", netInfo.Link.(*netlink.Tuntap).Mode) } } } else if netInfo.Iface.Type == "veth" { @@ -217,7 +217,7 @@ func (n *LinuxNetwork) AddEndpoint(ctx context.Context, s *Sandbox, netInfo Netw return endpoint, nil } -func (n *LinuxNetwork) RemoveEndpoint(ctx context.Context, s *Sandbox, idx int, hotplug bool) error { +func (n *LinuxNetwork) removeSingleEndpoint(ctx context.Context, s *Sandbox, idx int, hotplug bool) error { if idx > len(n.eps)-1 { return fmt.Errorf("Endpoint index overflow") } @@ -261,7 +261,7 @@ func (n *LinuxNetwork) RemoveEndpoint(ctx context.Context, s *Sandbox, idx int, // Scan the networking namespace through netlink and then: // 1. Create the endpoints for the relevant interfaces found there. // 2. Attach them to the VM. -func (n *LinuxNetwork) attachEndpoints(ctx context.Context, s *Sandbox, hotplug bool) error { +func (n *LinuxNetwork) addAllEndpoints(ctx context.Context, s *Sandbox, hotplug bool) error { netnsHandle, err := netns.GetFromPath(n.netNSPath) if err != nil { return err @@ -298,7 +298,7 @@ func (n *LinuxNetwork) attachEndpoints(ctx context.Context, s *Sandbox, hotplug continue } - _, err = n.AddEndpoint(ctx, s, netInfo, link, hotplug) + _, err = n.addSingleEndpoint(ctx, s, netInfo, hotplug) if err != nil { return err } @@ -324,36 +324,57 @@ func (n *LinuxNetwork) Run(ctx context.Context, cb func() error) error { } // Add adds all needed interfaces inside the network namespace. -func (n *LinuxNetwork) Add(ctx context.Context, s *Sandbox, hotplug bool) error { - span, ctx := n.trace(ctx, "Add") +func (n *LinuxNetwork) AddEndpoints(ctx context.Context, s *Sandbox, endpointsInfo []NetworkInfo, hotplug bool) ([]Endpoint, error) { + span, ctx := n.trace(ctx, "AddEndpoints") katatrace.AddTags(span, "type", n.interworkingModel.GetModel()) defer span.End() - if err := n.attachEndpoints(ctx, s, hotplug); err != nil { - return err + if endpointsInfo == nil { + if err := n.addAllEndpoints(ctx, s, hotplug); err != nil { + return nil, err + } + } else { + for _, ep := range endpointsInfo { + if _, err := n.addSingleEndpoint(ctx, s, ep, hotplug); err != nil { + n.eps = nil + return nil, err + } + } } katatrace.AddTags(span, "endpoints", n.eps, "hotplug", hotplug) - networkLogger().Debug("Network added") + networkLogger().Debug("Endpoints added") - return nil + return n.eps, nil } // Remove network endpoints in the network namespace. It also deletes the network // namespace in case the namespace has been created by us. -func (n *LinuxNetwork) Remove(ctx context.Context) error { - span, ctx := n.trace(ctx, "Remove") +func (n *LinuxNetwork) RemoveEndpoints(ctx context.Context, s *Sandbox, endpoints []Endpoint, hotplug bool) error { + span, ctx := n.trace(ctx, "RemoveEndpoints") defer span.End() - for i := range n.eps { - if err := n.RemoveEndpoint(ctx, nil, i, false); err != nil { + eps := n.eps + if endpoints != nil { + eps = endpoints + } + + for idx, ep := range eps { + if endpoints != nil { + new_ep, _ := findEndpoint(ep, n.eps) + if new_ep == nil { + continue + } + } + + if err := n.removeSingleEndpoint(ctx, s, idx, hotplug); err != nil { return err } } - networkLogger().Debug("Network removed") + networkLogger().Debug("Endpoints removed") - if n.netNSCreated { + if n.netNSCreated && endpoints == nil { networkLogger().Infof("Network namespace %q deleted", n.netNSPath) return deleteNetNS(n.netNSPath) } @@ -1056,6 +1077,7 @@ func networkInfoFromLink(handle *netlink.Handle, link netlink.Link) (NetworkInfo Addrs: addrs, Routes: routes, Neighbors: neighbors, + Link: link, }, nil } diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index 5bccf1b07a..ec6a3d7808 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -818,7 +818,7 @@ func (s *Sandbox) createNetwork(ctx context.Context) error { // after vm is started. if s.factory == nil { // Add the network - if err := s.network.Add(ctx, s, false); err != nil { + if _, err := s.network.AddEndpoints(ctx, s, nil, false); err != nil { return err } } @@ -853,7 +853,7 @@ func (s *Sandbox) removeNetwork(ctx context.Context) error { span, ctx := katatrace.Trace(ctx, s.Logger(), "removeNetwork", sandboxTracingTags, map[string]string{"sandbox_id": s.id}) defer span.End() - return s.network.Remove(ctx) + return s.network.RemoveEndpoints(ctx, s, nil, false) } func (s *Sandbox) generateNetInfo(inf *pbTypes.Interface) (NetworkInfo, error) { @@ -893,21 +893,24 @@ func (s *Sandbox) AddInterface(ctx context.Context, inf *pbTypes.Interface) (*pb return nil, err } - endpoint, err := s.network.AddEndpoint(ctx, s, netInfo, nil, true) + endpoints, err := s.network.AddEndpoints(ctx, s, []NetworkInfo{netInfo}, true) if err != nil { return nil, err } defer func() { if err != nil { - if errDetach := s.network.RemoveEndpoint(ctx, s, len(s.network.Endpoints())-1, true); err != nil { - s.Logger().WithField("endpoint-type", endpoint.Type()).WithError(errDetach).Error("rollback hot attaching endpoint failed") + eps := s.network.Endpoints() + // The newly added endpoint is last. + added_ep := eps[len(eps)-1] + if errDetach := s.network.RemoveEndpoints(ctx, s, []Endpoint{added_ep}, true); err != nil { + s.Logger().WithField("endpoint-type", added_ep.Type()).WithError(errDetach).Error("rollback hot attaching endpoint failed") } } }() // Add network for vm - inf.PciPath = endpoint.PciPath().String() + inf.PciPath = endpoints[0].PciPath().String() result, err := s.agent.updateInterface(ctx, inf) if err != nil { return nil, err @@ -923,10 +926,10 @@ func (s *Sandbox) AddInterface(ctx context.Context, inf *pbTypes.Interface) (*pb // RemoveInterface removes a nic of the sandbox. func (s *Sandbox) RemoveInterface(ctx context.Context, inf *pbTypes.Interface) (*pbTypes.Interface, error) { - for i, endpoint := range s.network.Endpoints() { + for _, endpoint := range s.network.Endpoints() { if endpoint.HardwareAddr() == inf.HwAddr { s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Hot detaching endpoint") - if err := s.network.RemoveEndpoint(ctx, s, i, true); err != nil { + if err := s.network.RemoveEndpoints(ctx, s, []Endpoint{endpoint}, true); err != nil { return inf, err } @@ -1205,7 +1208,7 @@ func (s *Sandbox) startVM(ctx context.Context) (err error) { // In case of vm factory, network interfaces are hotplugged // after vm is started. if s.factory != nil { - if err := s.network.Add(ctx, s, true); err != nil { + if _, err := s.network.AddEndpoints(ctx, s, nil, true); err != nil { return err } }