diff --git a/pkg/kubelet/container/runtime.go b/pkg/kubelet/container/runtime.go index 98d58b3aac4..8a3a412157a 100644 --- a/pkg/kubelet/container/runtime.go +++ b/pkg/kubelet/container/runtime.go @@ -465,6 +465,15 @@ func (p *Pod) FindContainerByID(id ContainerID) *Container { return nil } +func (p *Pod) FindSandboxByID(id ContainerID) *Container { + for _, c := range p.Sandboxes { + if c.ID == id { + return c + } + } + return nil +} + // ToAPIPod converts Pod to api.Pod. Note that if a field in api.Pod has no // corresponding field in Pod, the field would not be populated. func (p *Pod) ToAPIPod() *api.Pod { diff --git a/pkg/kubelet/kuberuntime/kuberuntime_manager.go b/pkg/kubelet/kuberuntime/kuberuntime_manager.go index ca5d322cb3f..afa8283f139 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_manager.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_manager.go @@ -229,7 +229,12 @@ func (m *kubeGenericRuntimeManager) GetPods(all bool) ([]*kubecontainer.Pod, err if err != nil { return nil, err } - for _, s := range sandboxes { + for i := range sandboxes { + s := sandboxes[i] + if s.Metadata == nil { + glog.V(4).Infof("Sandbox does not have metadata: %+v", s) + continue + } podUID := kubetypes.UID(s.Metadata.GetUid()) if _, ok := pods[podUID]; !ok { pods[podUID] = &kubecontainer.Pod{ @@ -241,7 +246,7 @@ func (m *kubeGenericRuntimeManager) GetPods(all bool) ([]*kubecontainer.Pod, err p := pods[podUID] converted, err := m.sandboxToKubeContainer(s) if err != nil { - glog.Warningf("Convert %q sandbox %v of pod %q failed: %v", m.runtimeName, s, podUID, err) + glog.V(4).Infof("Convert %q sandbox %v of pod %q failed: %v", m.runtimeName, s, podUID, err) continue } p.Sandboxes = append(p.Sandboxes, converted) @@ -251,7 +256,13 @@ func (m *kubeGenericRuntimeManager) GetPods(all bool) ([]*kubecontainer.Pod, err if err != nil { return nil, err } - for _, c := range containers { + for i := range containers { + c := containers[i] + if c.Metadata == nil { + glog.V(4).Infof("Container does not have metadata: %+v", c) + continue + } + labelledInfo := getContainerInfoFromLabels(c.Labels) pod, found := pods[labelledInfo.PodUID] if !found { @@ -265,7 +276,7 @@ func (m *kubeGenericRuntimeManager) GetPods(all bool) ([]*kubecontainer.Pod, err converted, err := m.toKubeContainer(c) if err != nil { - glog.Warningf("Convert %s container %v of pod %q failed: %v", m.runtimeName, c, labelledInfo.PodUID, err) + glog.V(4).Infof("Convert %s container %v of pod %q failed: %v", m.runtimeName, c, labelledInfo.PodUID, err) continue } diff --git a/pkg/kubelet/pleg/generic.go b/pkg/kubelet/pleg/generic.go index 375de16cf1b..64ebade8bb3 100644 --- a/pkg/kubelet/pleg/generic.go +++ b/pkg/kubelet/pleg/generic.go @@ -138,6 +138,7 @@ func generateEvents(podID types.UID, cid string, oldState, newState plegContaine if newState == oldState { return nil } + glog.V(4).Infof("GenericPLEG: %v/%v: %v -> %v", podID, cid, oldState, newState) switch newState { case plegContainerRunning: @@ -291,6 +292,17 @@ func getContainersFromPods(pods ...*kubecontainer.Pod) []*kubecontainer.Containe cidSet.Insert(cid) containers = append(containers, c) } + // Update sandboxes as containers + // TODO: keep track of sandboxes explicitly. + for _, c := range p.Sandboxes { + cid := string(c.ID.ID) + if cidSet.Has(cid) { + continue + } + cidSet.Insert(cid) + containers = append(containers, c) + } + } return containers } @@ -342,11 +354,17 @@ func getContainerState(pod *kubecontainer.Pod, cid *kubecontainer.ContainerID) p if pod == nil { return state } - container := pod.FindContainerByID(*cid) - if container == nil { - return state + c := pod.FindContainerByID(*cid) + if c != nil { + return convertState(c.State) } - return convertState(container.State) + // Search through sandboxes too. + c = pod.FindSandboxByID(*cid) + if c != nil { + return convertState(c.State) + } + + return state } func (pr podRecords) getOld(id types.UID) *kubecontainer.Pod { diff --git a/pkg/kubelet/pleg/generic_test.go b/pkg/kubelet/pleg/generic_test.go index 6b5cce5110c..3536e9f48cc 100644 --- a/pkg/kubelet/pleg/generic_test.go +++ b/pkg/kubelet/pleg/generic_test.go @@ -123,7 +123,7 @@ func TestRelisting(t *testing.T) { actual := getEventsFromChannel(ch) verifyEvents(t, expected, actual) - // The second relist should not send out any event because no container + // The second relist should not send out any event because no container has // changed. pleg.relist() verifyEvents(t, expected, actual) @@ -430,3 +430,69 @@ func TestRelistWithReinspection(t *testing.T) { // containers was the same as relist #1, nothing "changed", so there are no new events. assert.Exactly(t, []*PodLifecycleEvent{}, actualEvents) } + +// Test detecting sandbox state changes. +func TestRelistingWithSandboxes(t *testing.T) { + testPleg := newTestGenericPLEG() + pleg, runtime := testPleg.pleg, testPleg.runtime + ch := pleg.Watch() + // The first relist should send a PodSync event to each pod. + runtime.AllPodList = []*containertest.FakePod{ + {Pod: &kubecontainer.Pod{ + ID: "1234", + Sandboxes: []*kubecontainer.Container{ + createTestContainer("c1", kubecontainer.ContainerStateExited), + createTestContainer("c2", kubecontainer.ContainerStateRunning), + createTestContainer("c3", kubecontainer.ContainerStateUnknown), + }, + }}, + {Pod: &kubecontainer.Pod{ + ID: "4567", + Sandboxes: []*kubecontainer.Container{ + createTestContainer("c1", kubecontainer.ContainerStateExited), + }, + }}, + } + pleg.relist() + // Report every running/exited container if we see them for the first time. + expected := []*PodLifecycleEvent{ + {ID: "1234", Type: ContainerStarted, Data: "c2"}, + {ID: "4567", Type: ContainerDied, Data: "c1"}, + {ID: "1234", Type: ContainerDied, Data: "c1"}, + } + actual := getEventsFromChannel(ch) + verifyEvents(t, expected, actual) + + // The second relist should not send out any event because no container has + // changed. + pleg.relist() + verifyEvents(t, expected, actual) + + runtime.AllPodList = []*containertest.FakePod{ + {Pod: &kubecontainer.Pod{ + ID: "1234", + Sandboxes: []*kubecontainer.Container{ + createTestContainer("c2", kubecontainer.ContainerStateExited), + createTestContainer("c3", kubecontainer.ContainerStateRunning), + }, + }}, + {Pod: &kubecontainer.Pod{ + ID: "4567", + Sandboxes: []*kubecontainer.Container{ + createTestContainer("c4", kubecontainer.ContainerStateRunning), + }, + }}, + } + pleg.relist() + // Only report containers that transitioned to running or exited status. + expected = []*PodLifecycleEvent{ + {ID: "1234", Type: ContainerRemoved, Data: "c1"}, + {ID: "1234", Type: ContainerDied, Data: "c2"}, + {ID: "1234", Type: ContainerStarted, Data: "c3"}, + {ID: "4567", Type: ContainerRemoved, Data: "c1"}, + {ID: "4567", Type: ContainerStarted, Data: "c4"}, + } + + actual = getEventsFromChannel(ch) + verifyEvents(t, expected, actual) +}