From a49d28710a33a25e310c9264ff9fc58fd447a96f Mon Sep 17 00:00:00 2001 From: Yu-Ju Hong Date: Wed, 24 Aug 2016 17:08:37 -0700 Subject: [PATCH] Extend PLEG to handle pod sandboxes PLEG will treat them as if they are regular containers and detect changes the same manner. Note that this makes an assumption that container IDs will not collide with the podsandbox IDs. --- pkg/kubelet/container/runtime.go | 9 +++ .../kuberuntime/kuberuntime_manager.go | 19 ++++-- pkg/kubelet/pleg/generic.go | 26 +++++-- pkg/kubelet/pleg/generic_test.go | 68 ++++++++++++++++++- 4 files changed, 113 insertions(+), 9 deletions(-) diff --git a/pkg/kubelet/container/runtime.go b/pkg/kubelet/container/runtime.go index 98d58b3aac4..8a3a412157a 100644 --- a/pkg/kubelet/container/runtime.go +++ b/pkg/kubelet/container/runtime.go @@ -465,6 +465,15 @@ func (p *Pod) FindContainerByID(id ContainerID) *Container { return nil } +func (p *Pod) FindSandboxByID(id ContainerID) *Container { + for _, c := range p.Sandboxes { + if c.ID == id { + return c + } + } + return nil +} + // ToAPIPod converts Pod to api.Pod. Note that if a field in api.Pod has no // corresponding field in Pod, the field would not be populated. func (p *Pod) ToAPIPod() *api.Pod { diff --git a/pkg/kubelet/kuberuntime/kuberuntime_manager.go b/pkg/kubelet/kuberuntime/kuberuntime_manager.go index ca5d322cb3f..afa8283f139 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_manager.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_manager.go @@ -229,7 +229,12 @@ func (m *kubeGenericRuntimeManager) GetPods(all bool) ([]*kubecontainer.Pod, err if err != nil { return nil, err } - for _, s := range sandboxes { + for i := range sandboxes { + s := sandboxes[i] + if s.Metadata == nil { + glog.V(4).Infof("Sandbox does not have metadata: %+v", s) + continue + } podUID := kubetypes.UID(s.Metadata.GetUid()) if _, ok := pods[podUID]; !ok { pods[podUID] = &kubecontainer.Pod{ @@ -241,7 +246,7 @@ func (m *kubeGenericRuntimeManager) GetPods(all bool) ([]*kubecontainer.Pod, err p := pods[podUID] converted, err := m.sandboxToKubeContainer(s) if err != nil { - glog.Warningf("Convert %q sandbox %v of pod %q failed: %v", m.runtimeName, s, podUID, err) + glog.V(4).Infof("Convert %q sandbox %v of pod %q failed: %v", m.runtimeName, s, podUID, err) continue } p.Sandboxes = append(p.Sandboxes, converted) @@ -251,7 +256,13 @@ func (m *kubeGenericRuntimeManager) GetPods(all bool) ([]*kubecontainer.Pod, err if err != nil { return nil, err } - for _, c := range containers { + for i := range containers { + c := containers[i] + if c.Metadata == nil { + glog.V(4).Infof("Container does not have metadata: %+v", c) + continue + } + labelledInfo := getContainerInfoFromLabels(c.Labels) pod, found := pods[labelledInfo.PodUID] if !found { @@ -265,7 +276,7 @@ func (m *kubeGenericRuntimeManager) GetPods(all bool) ([]*kubecontainer.Pod, err converted, err := m.toKubeContainer(c) if err != nil { - glog.Warningf("Convert %s container %v of pod %q failed: %v", m.runtimeName, c, labelledInfo.PodUID, err) + glog.V(4).Infof("Convert %s container %v of pod %q failed: %v", m.runtimeName, c, labelledInfo.PodUID, err) continue } diff --git a/pkg/kubelet/pleg/generic.go b/pkg/kubelet/pleg/generic.go index 375de16cf1b..64ebade8bb3 100644 --- a/pkg/kubelet/pleg/generic.go +++ b/pkg/kubelet/pleg/generic.go @@ -138,6 +138,7 @@ func generateEvents(podID types.UID, cid string, oldState, newState plegContaine if newState == oldState { return nil } + glog.V(4).Infof("GenericPLEG: %v/%v: %v -> %v", podID, cid, oldState, newState) switch newState { case plegContainerRunning: @@ -291,6 +292,17 @@ func getContainersFromPods(pods ...*kubecontainer.Pod) []*kubecontainer.Containe cidSet.Insert(cid) containers = append(containers, c) } + // Update sandboxes as containers + // TODO: keep track of sandboxes explicitly. + for _, c := range p.Sandboxes { + cid := string(c.ID.ID) + if cidSet.Has(cid) { + continue + } + cidSet.Insert(cid) + containers = append(containers, c) + } + } return containers } @@ -342,11 +354,17 @@ func getContainerState(pod *kubecontainer.Pod, cid *kubecontainer.ContainerID) p if pod == nil { return state } - container := pod.FindContainerByID(*cid) - if container == nil { - return state + c := pod.FindContainerByID(*cid) + if c != nil { + return convertState(c.State) } - return convertState(container.State) + // Search through sandboxes too. + c = pod.FindSandboxByID(*cid) + if c != nil { + return convertState(c.State) + } + + return state } func (pr podRecords) getOld(id types.UID) *kubecontainer.Pod { diff --git a/pkg/kubelet/pleg/generic_test.go b/pkg/kubelet/pleg/generic_test.go index 6b5cce5110c..3536e9f48cc 100644 --- a/pkg/kubelet/pleg/generic_test.go +++ b/pkg/kubelet/pleg/generic_test.go @@ -123,7 +123,7 @@ func TestRelisting(t *testing.T) { actual := getEventsFromChannel(ch) verifyEvents(t, expected, actual) - // The second relist should not send out any event because no container + // The second relist should not send out any event because no container has // changed. pleg.relist() verifyEvents(t, expected, actual) @@ -430,3 +430,69 @@ func TestRelistWithReinspection(t *testing.T) { // containers was the same as relist #1, nothing "changed", so there are no new events. assert.Exactly(t, []*PodLifecycleEvent{}, actualEvents) } + +// Test detecting sandbox state changes. +func TestRelistingWithSandboxes(t *testing.T) { + testPleg := newTestGenericPLEG() + pleg, runtime := testPleg.pleg, testPleg.runtime + ch := pleg.Watch() + // The first relist should send a PodSync event to each pod. + runtime.AllPodList = []*containertest.FakePod{ + {Pod: &kubecontainer.Pod{ + ID: "1234", + Sandboxes: []*kubecontainer.Container{ + createTestContainer("c1", kubecontainer.ContainerStateExited), + createTestContainer("c2", kubecontainer.ContainerStateRunning), + createTestContainer("c3", kubecontainer.ContainerStateUnknown), + }, + }}, + {Pod: &kubecontainer.Pod{ + ID: "4567", + Sandboxes: []*kubecontainer.Container{ + createTestContainer("c1", kubecontainer.ContainerStateExited), + }, + }}, + } + pleg.relist() + // Report every running/exited container if we see them for the first time. + expected := []*PodLifecycleEvent{ + {ID: "1234", Type: ContainerStarted, Data: "c2"}, + {ID: "4567", Type: ContainerDied, Data: "c1"}, + {ID: "1234", Type: ContainerDied, Data: "c1"}, + } + actual := getEventsFromChannel(ch) + verifyEvents(t, expected, actual) + + // The second relist should not send out any event because no container has + // changed. + pleg.relist() + verifyEvents(t, expected, actual) + + runtime.AllPodList = []*containertest.FakePod{ + {Pod: &kubecontainer.Pod{ + ID: "1234", + Sandboxes: []*kubecontainer.Container{ + createTestContainer("c2", kubecontainer.ContainerStateExited), + createTestContainer("c3", kubecontainer.ContainerStateRunning), + }, + }}, + {Pod: &kubecontainer.Pod{ + ID: "4567", + Sandboxes: []*kubecontainer.Container{ + createTestContainer("c4", kubecontainer.ContainerStateRunning), + }, + }}, + } + pleg.relist() + // Only report containers that transitioned to running or exited status. + expected = []*PodLifecycleEvent{ + {ID: "1234", Type: ContainerRemoved, Data: "c1"}, + {ID: "1234", Type: ContainerDied, Data: "c2"}, + {ID: "1234", Type: ContainerStarted, Data: "c3"}, + {ID: "4567", Type: ContainerRemoved, Data: "c1"}, + {ID: "4567", Type: ContainerStarted, Data: "c4"}, + } + + actual = getEventsFromChannel(ch) + verifyEvents(t, expected, actual) +}