From 520295b938fed6b75b5a0ee0628735d973c474c5 Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Mon, 29 Jun 2020 20:06:15 -0700 Subject: [PATCH] network: Detect and add static ARP entries [ port from runtime commit 67d3e2c5c5d11738c0c0ff46b1228909a6c81ab0 ] Some network plugins add static arp entries in the network namespace. Scan namespace for static entries and pass these on to the agent to be added within the guest. If the grpc api is not implemented by the agent due to a older running agent, check for this and do not error out to maintain backward compatibility. Signed-off-by: Archana Shinde Signed-off-by: Peng Tao --- src/runtime/virtcontainers/kata_agent.go | 73 +++++++++++++++++-- src/runtime/virtcontainers/network.go | 52 ++++++++++--- src/runtime/virtcontainers/network_test.go | 25 ++++++- src/runtime/virtcontainers/pkg/types/types.go | 8 ++ 4 files changed, 138 insertions(+), 20 deletions(-) diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 1726bba0da..6ce30c391c 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -110,6 +110,7 @@ const ( grpcUpdateInterfaceRequest = "grpc.UpdateInterfaceRequest" grpcListInterfacesRequest = "grpc.ListInterfacesRequest" grpcListRoutesRequest = "grpc.ListRoutesRequest" + grpcAddARPNeighborsRequest = "grpc.AddARPNeighborsRequest" grpcOnlineCPUMemRequest = "grpc.OnlineCPUMemRequest" grpcListProcessesRequest = "grpc.ListProcessesRequest" grpcUpdateContainerRequest = "grpc.UpdateContainerRequest" @@ -638,6 +639,30 @@ func (k *kataAgent) updateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, err return nil, nil } +func (k *kataAgent) addARPNeighbors(neighs []*vcTypes.ARPNeighbor) error { + if neighs != nil { + neighsReq := &grpc.AddARPNeighborsRequest{ + Neighbors: &grpc.ARPNeighbors{ + ARPNeighbors: k.convertToKataAgentNeighbors(neighs), + }, + } + _, err := k.sendReq(neighsReq) + if err != nil { + if grpcStatus.Convert(err).Code() == codes.Unimplemented { + k.Logger().WithFields(logrus.Fields{ + "arpneighbors-requested": fmt.Sprintf("%+v", neighs), + }).Warn("add ARP neighbors request failed due to old agent, please upgrade Kata Containers image version") + return nil + } + k.Logger().WithFields(logrus.Fields{ + "arpneighbors-requested": fmt.Sprintf("%+v", neighs), + }).WithError(err).Error("add ARP neighbors request failed") + } + return err + } + return nil +} + func (k *kataAgent) listInterfaces() ([]*vcTypes.Interface, error) { req := &grpc.ListInterfacesRequest{} resultingInterfaces, err := k.sendReq(req) @@ -843,7 +868,7 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error { // // Setup network interfaces and routes // - interfaces, routes, err := generateInterfacesAndRoutes(sandbox.networkNS) + interfaces, routes, neighs, err := generateVCNetworkStructures(sandbox.networkNS) if err != nil { return err } @@ -853,6 +878,9 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error { if _, err = k.updateRoutes(routes); err != nil { return err } + if err = k.addARPNeighbors(neighs); err != nil { + return err + } storages := setupStorages(sandbox) @@ -1999,6 +2027,9 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) { k.reqHandlers[grpcListRoutesRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { return k.client.AgentServiceClient.ListRoutes(ctx, req.(*grpc.ListRoutesRequest)) } + k.reqHandlers[grpcAddARPNeighborsRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { + return k.client.AgentServiceClient.AddARPNeighbors(ctx, req.(*grpc.AddARPNeighborsRequest)) + } k.reqHandlers[grpcOnlineCPUMemRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { return k.client.AgentServiceClient.OnlineCPUMem(ctx, req.(*grpc.OnlineCPUMemRequest)) } @@ -2175,18 +2206,27 @@ func (k *kataAgent) convertToIPFamily(ipFamily aTypes.IPFamily) int { return netlink.FAMILY_V4 } +func (k *kataAgent) convertToKataAgentIPAddress(ipAddr *vcTypes.IPAddress) (aIPAddr *aTypes.IPAddress) { + if ipAddr == nil { + return nil + } + + aIPAddr = &aTypes.IPAddress{ + Family: k.convertToKataAgentIPFamily(ipAddr.Family), + Address: ipAddr.Address, + Mask: ipAddr.Mask, + } + + return aIPAddr +} + func (k *kataAgent) convertToKataAgentIPAddresses(ipAddrs []*vcTypes.IPAddress) (aIPAddrs []*aTypes.IPAddress) { for _, ipAddr := range ipAddrs { if ipAddr == nil { continue } - aIPAddr := &aTypes.IPAddress{ - Family: k.convertToKataAgentIPFamily(ipAddr.Family), - Address: ipAddr.Address, - Mask: ipAddr.Mask, - } - + aIPAddr := k.convertToKataAgentIPAddress(ipAddr) aIPAddrs = append(aIPAddrs, aIPAddr) } @@ -2268,6 +2308,25 @@ func (k *kataAgent) convertToKataAgentRoutes(routes []*vcTypes.Route) (aRoutes [ return aRoutes } +func (k *kataAgent) convertToKataAgentNeighbors(neighs []*vcTypes.ARPNeighbor) (aNeighs []*aTypes.ARPNeighbor) { + for _, neigh := range neighs { + if neigh == nil { + continue + } + + aNeigh := &aTypes.ARPNeighbor{ + ToIPAddress: k.convertToKataAgentIPAddress(neigh.ToIPAddress), + Device: neigh.Device, + State: int32(neigh.State), + Lladdr: neigh.LLAddr, + } + + aNeighs = append(aNeighs, aNeigh) + } + + return aNeighs +} + func (k *kataAgent) convertToRoutes(aRoutes []*aTypes.Route) (routes []*vcTypes.Route) { for _, aRoute := range aRoutes { if aRoute == nil { diff --git a/src/runtime/virtcontainers/network.go b/src/runtime/virtcontainers/network.go index 3114e3366e..baeee9bb99 100644 --- a/src/runtime/virtcontainers/network.go +++ b/src/runtime/virtcontainers/network.go @@ -118,10 +118,11 @@ type NetlinkIface struct { // NetworkInfo gathers all information related to a network interface. // It can be used to store the description of the underlying network. type NetworkInfo struct { - Iface NetlinkIface - Addrs []netlink.Addr - Routes []netlink.Route - DNS DNSInfo + Iface NetlinkIface + Addrs []netlink.Addr + Routes []netlink.Route + DNS DNSInfo + Neighbors []netlink.Neigh } // NetworkInterface defines a network interface. @@ -942,14 +943,15 @@ func deleteNetNS(netNSPath string) error { return nil } -func generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*vcTypes.Interface, []*vcTypes.Route, error) { +func generateVCNetworkStructures(networkNS NetworkNamespace) ([]*vcTypes.Interface, []*vcTypes.Route, []*vcTypes.ARPNeighbor, error) { if networkNS.NetNsPath == "" { - return nil, nil, nil + return nil, nil, nil, nil } var routes []*vcTypes.Route var ifaces []*vcTypes.Interface + var neighs []*vcTypes.ARPNeighbor for _, endpoint := range networkNS.Endpoints { @@ -1008,10 +1010,36 @@ func generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*vcTypes.Interfa r.Device = endpoint.Name() r.Scope = uint32(route.Scope) routes = append(routes, &r) + } + for _, neigh := range endpoint.Properties().Neighbors { + var n vcTypes.ARPNeighbor + + // We add only static ARP entries + if neigh.State != netlink.NUD_PERMANENT { + continue + } + + n.Device = endpoint.Name() + n.State = neigh.State + n.Flags = neigh.Flags + + if neigh.HardwareAddr != nil { + n.LLAddr = neigh.HardwareAddr.String() + } + + n.ToIPAddress = &vcTypes.IPAddress{ + Family: netlink.FAMILY_V4, + Address: neigh.IP.String(), + } + if neigh.IP.To4() == nil { + n.ToIPAddress.Family = netlink.FAMILY_V6 + } + + neighs = append(neighs, &n) } } - return ifaces, routes, nil + return ifaces, routes, neighs, nil } func createNetworkInterfacePair(idx int, ifName string, interworkingModel NetInterworkingModel) (NetworkInterfacePair, error) { @@ -1071,13 +1099,19 @@ func networkInfoFromLink(handle *netlink.Handle, link netlink.Link) (NetworkInfo return NetworkInfo{}, err } + neighbors, err := handle.NeighList(link.Attrs().Index, netlink.FAMILY_ALL) + if err != nil { + return NetworkInfo{}, err + } + return NetworkInfo{ Iface: NetlinkIface{ LinkAttrs: *(link.Attrs()), Type: link.Type(), }, - Addrs: addrs, - Routes: routes, + Addrs: addrs, + Routes: routes, + Neighbors: neighbors, }, nil } diff --git a/src/runtime/virtcontainers/network_test.go b/src/runtime/virtcontainers/network_test.go index cbb303274f..fdf0c402f5 100644 --- a/src/runtime/virtcontainers/network_test.go +++ b/src/runtime/virtcontainers/network_test.go @@ -65,13 +65,20 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) { {LinkIndex: 329, Dst: nil, Src: nil, Gw: gatewayV6}, } + arpMAC, _ := net.ParseMAC("6a:92:3a:59:70:aa") + + neighs := []netlink.Neigh{ + {LinkIndex: 329, IP: net.IPv4(192, 168, 0, 101), State: netlink.NUD_PERMANENT, HardwareAddr: arpMAC}, + } + networkInfo := NetworkInfo{ Iface: NetlinkIface{ LinkAttrs: netlink.LinkAttrs{MTU: 1500}, Type: "", }, - Addrs: addrs, - Routes: routes, + Addrs: addrs, + Routes: routes, + Neighbors: neighs, } ep0 := &PhysicalEndpoint{ @@ -84,7 +91,7 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) { nns := NetworkNamespace{NetNsPath: "foobar", NetNsCreated: true, Endpoints: endpoints} - resInterfaces, resRoutes, err := generateInterfacesAndRoutes(nns) + resInterfaces, resRoutes, resNeighs, err := generateVCNetworkStructures(nns) // // Build expected results: @@ -106,6 +113,15 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) { {Dest: "", Gateway: "2001:db8:1::1", Device: "eth0", Source: ""}, } + expectedNeighs := []*vcTypes.ARPNeighbor{ + { + Device: "eth0", + State: netlink.NUD_PERMANENT, + LLAddr: "6a:92:3a:59:70:aa", + ToIPAddress: &vcTypes.IPAddress{Address: "192.168.0.101", Family: netlink.FAMILY_V4}, + }, + } + for _, r := range resRoutes { fmt.Printf("resRoute: %+v\n", r) } @@ -115,7 +131,8 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) { "Interfaces returned didn't match: got %+v, expecting %+v", resInterfaces, expectedInterfaces) assert.True(t, reflect.DeepEqual(resRoutes, expectedRoutes), "Routes returned didn't match: got %+v, expecting %+v", resRoutes, expectedRoutes) - + assert.True(t, reflect.DeepEqual(resNeighs, expectedNeighs), + "ARP Neighbors returned didn't match: got %+v, expecting %+v", resNeighs, expectedNeighs) } func TestNetInterworkingModelIsValid(t *testing.T) { diff --git a/src/runtime/virtcontainers/pkg/types/types.go b/src/runtime/virtcontainers/pkg/types/types.go index 0d4a9cfa13..5abb4922ca 100644 --- a/src/runtime/virtcontainers/pkg/types/types.go +++ b/src/runtime/virtcontainers/pkg/types/types.go @@ -39,3 +39,11 @@ type Route struct { Source string Scope uint32 } + +type ARPNeighbor struct { + ToIPAddress *IPAddress + Device string + LLAddr string + State int + Flags int +}