From 8a2895d88f38c85d458d6713ed5821ba4bb6dd29 Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Fri, 6 May 2016 16:01:42 -0700 Subject: [PATCH] rkt: Pass through podIP This is needed for the /etc/hosts mount and the downward API to work. Furthermore, this is required for the reported `PodStatus` to be correct. The `Status` bit mostly worked prior to #25062, and this restores that functionality in addition to the new functionality. --- pkg/kubelet/rkt/rkt.go | 48 +++++++++++++++++++------------------ pkg/kubelet/rkt/rkt_test.go | 38 ++++++++++++++++++----------- 2 files changed, 49 insertions(+), 37 deletions(-) diff --git a/pkg/kubelet/rkt/rkt.go b/pkg/kubelet/rkt/rkt.go index 9e4036e4c6b..9ce2ccc3930 100644 --- a/pkg/kubelet/rkt/rkt.go +++ b/pkg/kubelet/rkt/rkt.go @@ -107,7 +107,6 @@ const ( dockerAuthTemplate = `{"rktKind":"dockerAuth","rktVersion":"v1","registries":[%q],"credentials":{"user":%q,"password":%q}}` defaultRktAPIServiceAddr = "localhost:15441" - defaultNetworkName = "rkt.kubernetes.io" // ndots specifies the minimum number of dots that a domain name must contain for the resolver to consider it as FQDN (fully-qualified) // we want to able to consider SRV lookup names like _dns._udp.kube-dns.default.svc to be considered relative. @@ -574,7 +573,7 @@ func setApp(imgManifest *appcschema.ImageManifest, c *api.Container, opts *kubec } // makePodManifest transforms a kubelet pod spec to the rkt pod manifest. -func (r *Runtime) makePodManifest(pod *api.Pod, pullSecrets []api.Secret) (*appcschema.PodManifest, error) { +func (r *Runtime) makePodManifest(pod *api.Pod, podIP string, pullSecrets []api.Secret) (*appcschema.PodManifest, error) { manifest := appcschema.BlankPodManifest() listResp, err := r.apisvc.ListPods(context.Background(), &rktapi.ListPodsRequest{ @@ -618,7 +617,7 @@ func (r *Runtime) makePodManifest(pod *api.Pod, pullSecrets []api.Secret) (*appc } for _, c := range pod.Spec.Containers { - err := r.newAppcRuntimeApp(pod, c, requiresPrivileged, pullSecrets, manifest) + err := r.newAppcRuntimeApp(pod, podIP, c, requiresPrivileged, pullSecrets, manifest) if err != nil { return nil, err } @@ -723,11 +722,10 @@ func (r *Runtime) makeContainerLogMount(opts *kubecontainer.RunContainerOptions, return &mnt, nil } -func (r *Runtime) newAppcRuntimeApp(pod *api.Pod, c api.Container, requiresPrivileged bool, pullSecrets []api.Secret, manifest *appcschema.PodManifest) error { +func (r *Runtime) newAppcRuntimeApp(pod *api.Pod, podIP string, c api.Container, requiresPrivileged bool, pullSecrets []api.Secret, manifest *appcschema.PodManifest) error { if requiresPrivileged && !capabilities.Get().AllowPrivileged { return fmt.Errorf("cannot make %q: running a custom stage1 requires a privileged security context", format.Pod(pod)) } - if err, _ := r.imagePuller.PullImage(pod, &c, pullSecrets); err != nil { return nil } @@ -750,7 +748,7 @@ func (r *Runtime) newAppcRuntimeApp(pod *api.Pod, c api.Container, requiresPrivi } // TODO: determine how this should be handled for rkt - opts, err := r.runtimeHelper.GenerateRunContainerOptions(pod, &c, "") + opts, err := r.runtimeHelper.GenerateRunContainerOptions(pod, &c, podIP) if err != nil { return err } @@ -998,9 +996,9 @@ func (r *Runtime) preparePodArgs(manifest *appcschema.PodManifest, manifestFileN // // On success, it will return a string that represents name of the unit file // and the runtime pod. -func (r *Runtime) preparePod(pod *api.Pod, pullSecrets []api.Secret, netnsName string) (string, *kubecontainer.Pod, error) { +func (r *Runtime) preparePod(pod *api.Pod, podIP string, pullSecrets []api.Secret, netnsName string) (string, *kubecontainer.Pod, error) { // Generate the appc pod manifest from the k8s pod spec. - manifest, err := r.makePodManifest(pod, pullSecrets) + manifest, err := r.makePodManifest(pod, podIP, pullSecrets) if err != nil { return "", nil, err } @@ -1118,14 +1116,18 @@ func netnsPathFromName(netnsName string) string { return fmt.Sprintf("/var/run/netns/%s", netnsName) } -func (r *Runtime) setupPodNetwork(pod *api.Pod) (string, error) { +// setupPodNetwork creates a network namespace for the given pod and calls +// configured NetworkPlugin's setup function on it. +// It returns the namespace name, configured IP (if available), and an error if +// one occured. +func (r *Runtime) setupPodNetwork(pod *api.Pod) (string, string, error) { netnsName := makePodNetnsName(pod.UID) // Create a new network namespace for the pod r.execer.Command("ip", "netns", "del", netnsName).Output() _, err := r.execer.Command("ip", "netns", "add", netnsName).Output() if err != nil { - return "", fmt.Errorf("failed to create pod network namespace: %v", err) + return "", "", fmt.Errorf("failed to create pod network namespace: %v", err) } // Set up networking with the network plugin @@ -1133,7 +1135,11 @@ func (r *Runtime) setupPodNetwork(pod *api.Pod) (string, error) { containerID := kubecontainer.ContainerID{ID: string(pod.UID)} err = r.networkPlugin.SetUpPod(pod.Namespace, pod.Name, containerID) if err != nil { - return "", fmt.Errorf("failed to set up pod network: %v", err) + return "", "", fmt.Errorf("failed to set up pod network: %v", err) + } + status, err := r.networkPlugin.GetPodNetworkStatus(pod.Namespace, pod.Name, containerID) + if err != nil { + return "", "", fmt.Errorf("failed to get status of pod network: %v", err) } if r.configureHairpinMode { @@ -1142,7 +1148,7 @@ func (r *Runtime) setupPodNetwork(pod *api.Pod) (string, error) { } } - return netnsName, nil + return netnsName, status.IP.String(), nil } // RunPod first creates the unit file for a pod, and then @@ -1152,15 +1158,16 @@ func (r *Runtime) RunPod(pod *api.Pod, pullSecrets []api.Secret) error { var err error var netnsName string + var podIP string if !kubecontainer.IsHostNetworkPod(pod) { - netnsName, err = r.setupPodNetwork(pod) + netnsName, podIP, err = r.setupPodNetwork(pod) if err != nil { r.cleanupPodNetwork(pod) return err } } - name, runtimePod, prepareErr := r.preparePod(pod, pullSecrets, netnsName) + name, runtimePod, prepareErr := r.preparePod(pod, podIP, pullSecrets, netnsName) // Set container references and generate events. // If preparedPod fails, then send out 'failed' events for each container. @@ -2062,7 +2069,6 @@ func (r *Runtime) GetPodStatus(uid types.UID, name, namespace string) (*kubecont return nil, fmt.Errorf("couldn't list pods: %v", err) } - var latestPod *rktapi.Pod var latestRestartCount int = -1 // In this loop, we group all containers from all pods together, @@ -2075,7 +2081,6 @@ func (r *Runtime) GetPodStatus(uid types.UID, name, namespace string) (*kubecont } if restartCount > latestRestartCount { - latestPod = pod latestRestartCount = restartCount } @@ -2091,13 +2096,10 @@ func (r *Runtime) GetPodStatus(uid types.UID, name, namespace string) (*kubecont } } - if latestPod != nil { - // Try to fill the IP info. - for _, n := range latestPod.Networks { - if n.Name == defaultNetworkName { - podStatus.IP = n.Ipv4 - } - } + // TODO(euank): this will not work in host networking mode + containerID := kubecontainer.ContainerID{ID: string(uid)} + if status, err := r.networkPlugin.GetPodNetworkStatus(namespace, name, containerID); err == nil { + podStatus.IP = status.IP.String() } return podStatus, nil diff --git a/pkg/kubelet/rkt/rkt_test.go b/pkg/kubelet/rkt/rkt_test.go index d4d4f6cc7ca..fa09163b76b 100644 --- a/pkg/kubelet/rkt/rkt_test.go +++ b/pkg/kubelet/rkt/rkt_test.go @@ -19,6 +19,7 @@ package rkt import ( "encoding/json" "fmt" + "net" "os" "sort" "testing" @@ -35,6 +36,8 @@ import ( containertesting "k8s.io/kubernetes/pkg/kubelet/container/testing" kubetesting "k8s.io/kubernetes/pkg/kubelet/container/testing" "k8s.io/kubernetes/pkg/kubelet/lifecycle" + "k8s.io/kubernetes/pkg/kubelet/network" + "k8s.io/kubernetes/pkg/kubelet/network/mock_network" "k8s.io/kubernetes/pkg/kubelet/rkt/mock_os" "k8s.io/kubernetes/pkg/kubelet/rkt/mock_rkt" "k8s.io/kubernetes/pkg/types" @@ -68,8 +71,7 @@ func mustRktHash(hash string) *appctypes.Hash { } func makeRktPod(rktPodState rktapi.PodState, - rktPodID, podUID, podName, podNamespace, - podIP string, podCreatedAt, podStartedAt int64, + rktPodID, podUID, podName, podNamespace string, podCreatedAt, podStartedAt int64, podRestartCount string, appNames, imgIDs, imgNames, containerHashes []string, appStates []rktapi.AppState, exitcodes []int32) *rktapi.Pod { @@ -149,7 +151,6 @@ func makeRktPod(rktPodState rktapi.PodState, return &rktapi.Pod{ Id: rktPodID, State: rktPodState, - Networks: []*rktapi.Network{{Name: defaultNetworkName, Ipv4: podIP}}, Apps: apps, Manifest: mustMarshalPodManifest(podManifest), StartedAt: podStartedAt, @@ -367,7 +368,7 @@ func TestGetPods(t *testing.T) { []*rktapi.Pod{ makeRktPod(rktapi.PodState_POD_STATE_RUNNING, "uuid-4002", "42", "guestbook", "default", - "10.10.10.42", ns(10), ns(10), "7", + ns(10), ns(10), "7", []string{"app-1", "app-2"}, []string{"img-id-1", "img-id-2"}, []string{"img-name-1", "img-name-2"}, @@ -405,7 +406,7 @@ func TestGetPods(t *testing.T) { []*rktapi.Pod{ makeRktPod(rktapi.PodState_POD_STATE_RUNNING, "uuid-4002", "42", "guestbook", "default", - "10.10.10.42", ns(10), ns(20), "7", + ns(10), ns(20), "7", []string{"app-1", "app-2"}, []string{"img-id-1", "img-id-2"}, []string{"img-name-1", "img-name-2"}, @@ -415,7 +416,7 @@ func TestGetPods(t *testing.T) { ), makeRktPod(rktapi.PodState_POD_STATE_EXITED, "uuid-4003", "43", "guestbook", "default", - "10.10.10.43", ns(30), ns(40), "7", + ns(30), ns(40), "7", []string{"app-11", "app-22"}, []string{"img-id-11", "img-id-22"}, []string{"img-name-11", "img-name-22"}, @@ -425,7 +426,7 @@ func TestGetPods(t *testing.T) { ), makeRktPod(rktapi.PodState_POD_STATE_EXITED, "uuid-4004", "43", "guestbook", "default", - "10.10.10.44", ns(50), ns(60), "8", + ns(50), ns(60), "8", []string{"app-11", "app-22"}, []string{"img-id-11", "img-id-22"}, []string{"img-name-11", "img-name-22"}, @@ -557,8 +558,11 @@ func TestGetPodsFilters(t *testing.T) { } func TestGetPodStatus(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() fr := newFakeRktInterface() fs := newFakeSystemd() + fnp := mock_network.NewMockNetworkPlugin(ctrl) fos := &containertesting.FakeOS{} frh := &fakeRuntimeHelper{} r := &Runtime{ @@ -566,6 +570,7 @@ func TestGetPodStatus(t *testing.T) { systemd: fs, runtimeHelper: frh, os: fos, + networkPlugin: fnp, } ns := func(seconds int64) int64 { @@ -586,7 +591,7 @@ func TestGetPodStatus(t *testing.T) { []*rktapi.Pod{ makeRktPod(rktapi.PodState_POD_STATE_RUNNING, "uuid-4002", "42", "guestbook", "default", - "10.10.10.42", ns(10), ns(20), "7", + ns(10), ns(20), "7", []string{"app-1", "app-2"}, []string{"img-id-1", "img-id-2"}, []string{"img-name-1", "img-name-2"}, @@ -634,7 +639,7 @@ func TestGetPodStatus(t *testing.T) { []*rktapi.Pod{ makeRktPod(rktapi.PodState_POD_STATE_EXITED, "uuid-4002", "42", "guestbook", "default", - "10.10.10.42", ns(10), ns(20), "7", + ns(10), ns(20), "7", []string{"app-1", "app-2"}, []string{"img-id-1", "img-id-2"}, []string{"img-name-1", "img-name-2"}, @@ -644,7 +649,7 @@ func TestGetPodStatus(t *testing.T) { ), makeRktPod(rktapi.PodState_POD_STATE_RUNNING, // The latest pod is running. "uuid-4003", "42", "guestbook", "default", - "10.10.10.42", ns(10), ns(20), "10", + ns(10), ns(20), "10", []string{"app-1", "app-2"}, []string{"img-id-1", "img-id-2"}, []string{"img-name-1", "img-name-2"}, @@ -716,9 +721,6 @@ func TestGetPodStatus(t *testing.T) { }, } - ctrl := gomock.NewController(t) - defer ctrl.Finish() - for i, tt := range tests { testCaseHint := fmt.Sprintf("test case #%d", i) fr.pods = tt.pods @@ -738,6 +740,14 @@ func TestGetPodStatus(t *testing.T) { return mockFI, nil } + if tt.result.IP != "" { + fnp.EXPECT().GetPodNetworkStatus("default", "guestbook", kubecontainer.ContainerID{ID: "42"}). + Return(&network.PodNetworkStatus{IP: net.ParseIP(tt.result.IP)}, nil) + } else { + fnp.EXPECT().GetPodNetworkStatus("default", "guestbook", kubecontainer.ContainerID{ID: "42"}). + Return(nil, fmt.Errorf("no such network")) + } + status, err := r.GetPodStatus("42", "guestbook", "default") if err != nil { t.Errorf("test case #%d: unexpected error: %v", i, err) @@ -1692,7 +1702,7 @@ func TestMakePodManifestAnnotations(t *testing.T) { hint := fmt.Sprintf("case #%d", i) mockVolumeGetter.EXPECT().GetVolumes(gomock.Any()).Return(kubecontainer.VolumeMap{}, true) - result, err := r.makePodManifest(testCase.in, []api.Secret{}) + result, err := r.makePodManifest(testCase.in, "", []api.Secret{}) assert.Equal(t, err, testCase.outerr, hint) if err == nil { sort.Sort(annotationsByName(result.Annotations))