Merge pull request #25185 from freehan/kubenetgetpodstatus

Automatic merge from submit-queue

kubenet try to retrieve ip inside pod net namespace

Kubenet currently stores the ips of pods inside a map. Kubelet gets pod ip from kubenet during syncpod. If Kubelet restarts, all pods on the node lost their ips in podStatus. This PR adds logic to retrieve pod IP from pod netns. 

cc: @yujuhong

pkg/kubelet/network/kubenet/kubenet_linux.go

@@ -21,7 +21,7 @@ package kubenet
 import (
 	"fmt"
 	"net"
-	"strings"
+
 	"sync"
 	"syscall"
 
@@ -37,6 +37,7 @@ import (
 	utilexec "k8s.io/kubernetes/pkg/util/exec"
 	utilsets "k8s.io/kubernetes/pkg/util/sets"
 	utilsysctl "k8s.io/kubernetes/pkg/util/sysctl"
+	"strings"
 )
 
 const (
@@ -50,20 +51,22 @@ const (
 type kubenetNetworkPlugin struct {
 	network.NoopNetworkPlugin
 
-	host      network.Host
-	netConfig *libcni.NetworkConfig
-	cniConfig *libcni.CNIConfig
-	shaper    bandwidth.BandwidthShaper
-
-	podCIDRs map[kubecontainer.ContainerID]string
-	MTU      int
-	mu       sync.Mutex //Mutex for protecting podCIDRs map and netConfig
+	host        network.Host
+	netConfig   *libcni.NetworkConfig
+	cniConfig   *libcni.CNIConfig
+	shaper      bandwidth.BandwidthShaper
+	podCIDRs    map[kubecontainer.ContainerID]string
+	MTU         int
+	mu          sync.Mutex //Mutex for protecting podCIDRs map and netConfig
+	execer      utilexec.Interface
+	nsenterPath string
 }
 
 func NewPlugin() network.NetworkPlugin {
 	return &kubenetNetworkPlugin{
 		podCIDRs: make(map[kubecontainer.ContainerID]string),
 		MTU:      1460,
+		execer:   utilexec.New(),
 	}
 }
 
@@ -86,7 +89,7 @@ func (plugin *kubenetNetworkPlugin) Init(host network.Host) error {
 	// This will return an error on older kernel version (< 3.18) as the module
 	// was built-in, we simply ignore the error here. A better thing to do is
 	// to check the kernel version in the future.
-	utilexec.New().Command("modprobe", "br-netfilter").CombinedOutput()
+	plugin.execer.Command("modprobe", "br-netfilter").CombinedOutput()
 	if err := utilsysctl.SetSysctl(sysctlBridgeCallIptables, 1); err != nil {
 		glog.Warningf("can't set sysctl %s: %v", sysctlBridgeCallIptables, err)
 	}
@@ -330,15 +333,41 @@ func (plugin *kubenetNetworkPlugin) TearDownPod(namespace string, name string, i
 func (plugin *kubenetNetworkPlugin) GetPodNetworkStatus(namespace string, name string, id kubecontainer.ContainerID) (*network.PodNetworkStatus, error) {
 	plugin.mu.Lock()
 	defer plugin.mu.Unlock()
-	cidr, ok := plugin.podCIDRs[id]
-	if !ok {
-		return nil, fmt.Errorf("No IP address found for pod %v", id)
+	// Assuming the ip of pod does not change. Try to retrieve ip from kubenet map first.
+	if cidr, ok := plugin.podCIDRs[id]; ok {
+		ip, _, err := net.ParseCIDR(strings.Trim(cidr, "\n"))
+		if err == nil {
+			return &network.PodNetworkStatus{IP: ip}, nil
+		}
 	}
-
-	ip, _, err := net.ParseCIDR(strings.Trim(cidr, "\n"))
+	// TODO: remove type conversion once kubenet supports multiple runtime
+	runtime, ok := plugin.host.GetRuntime().(*dockertools.DockerManager)
+	if !ok {
+		return nil, fmt.Errorf("Kubenet execution called on non-docker runtime")
+	}
+	netnsPath, err := runtime.GetNetNS(id)
+	if err != nil {
+		return nil, fmt.Errorf("Kubenet failed to retrieve network namespace path: %v", err)
+	}
+	nsenterPath, err := plugin.getNsenterPath()
 	if err != nil {
 		return nil, err
 	}
+	// Try to retrieve ip inside container netwrok namespace
+	output, err := plugin.execer.Command(nsenterPath, fmt.Sprintf("--net=%s", netnsPath), "-F", "--",
+		"ip", "-o", "-4", "addr", "show", "dev", network.DefaultInterfaceName).CombinedOutput()
+	if err != nil {
+		return nil, fmt.Errorf("Unexpected command output %s with error: %v", output, err)
+	}
+	fields := strings.Fields(string(output))
+	if len(fields) < 4 {
+		return nil, fmt.Errorf("Unexpected command output %s ", output)
+	}
+	ip, _, err := net.ParseCIDR(fields[3])
+	if err != nil {
+		return nil, fmt.Errorf("Kubenet failed to parse ip from output %s due to %v", output, err)
+	}
+	plugin.podCIDRs[id] = ip.String()
 	return &network.PodNetworkStatus{IP: ip}, nil
 }
 
@@ -387,3 +416,14 @@ func (plugin *kubenetNetworkPlugin) delContainerFromNetwork(id kubecontainer.Con
 	delete(plugin.podCIDRs, id)
 	return nil
 }
+
+func (plugin *kubenetNetworkPlugin) getNsenterPath() (string, error) {
+	if plugin.nsenterPath == "" {
+		nsenterPath, err := plugin.execer.LookPath("nsenter")
+		if err != nil {
+			return "", err
+		}
+		plugin.nsenterPath = nsenterPath
+	}
+	return plugin.nsenterPath, nil
+}

pkg/kubelet/network/kubenet/kubenet_linux_test.go

@@ -0,0 +1,87 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubenet
+
+import (
+	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	"k8s.io/kubernetes/pkg/kubelet/network"
+	nettest "k8s.io/kubernetes/pkg/kubelet/network/testing"
+	"k8s.io/kubernetes/pkg/util/exec"
+	"testing"
+)
+
+func newFakeKubenetPlugin(initMap map[kubecontainer.ContainerID]string, execer exec.Interface, host network.Host) network.NetworkPlugin {
+	return &kubenetNetworkPlugin{
+		podCIDRs: initMap,
+		execer:   execer,
+		MTU:      1460,
+		host:     host,
+	}
+}
+
+func TestGetPodNetworkStatus(t *testing.T) {
+	podIPMap := make(map[kubecontainer.ContainerID]string)
+	podIPMap[kubecontainer.ContainerID{ID: "1"}] = "10.245.0.2/32"
+	podIPMap[kubecontainer.ContainerID{ID: "2"}] = "10.245.0.3/32"
+
+	fhost := nettest.NewFakeHost(nil)
+	fakeKubenet := newFakeKubenetPlugin(podIPMap, nil, fhost)
+
+	testCases := []struct {
+		id          string
+		expectError bool
+		expectIP    string
+	}{
+		//in podCIDR map
+		{
+			"1",
+			false,
+			"10.245.0.2",
+		},
+		{
+			"2",
+			false,
+			"10.245.0.3",
+		},
+		//not in podCIDR map
+		{
+			"3",
+			true,
+			"",
+		},
+		//TODO: add test cases for retrieving ip inside container network namespace
+	}
+
+	for i, tc := range testCases {
+		out, err := fakeKubenet.GetPodNetworkStatus("", "", kubecontainer.ContainerID{ID: tc.id})
+		if tc.expectError {
+			if err == nil {
+				t.Errorf("Test case %d expects error but got none", i)
+			}
+			continue
+		} else {
+			if err != nil {
+				t.Errorf("Test case %d expects error but got error: %v", i, err)
+			}
+		}
+		if tc.expectIP != out.IP.String() {
+			t.Errorf("Test case %d expects ip %s but got %s", i, tc.expectIP, out.IP.String())
+		}
+	}
+}
+
+//TODO: add unit test for each implementation of network plugin interface