diff --git a/pkg/kubelet/dockertools/manager.go b/pkg/kubelet/dockertools/manager.go index c1050bbd64e..4198514e914 100644 --- a/pkg/kubelet/dockertools/manager.go +++ b/pkg/kubelet/dockertools/manager.go @@ -2110,10 +2110,22 @@ func (dm *DockerManager) GetPodStatus(uid types.UID, name, namespace string) (*k if dockerName.PodUID != uid { continue } - result, ip, err := dm.inspectContainer(c.ID, name, namespace) if err != nil { - return podStatus, err + if _, ok := err.(*docker.NoSuchContainer); ok { + // https://github.com/kubernetes/kubernetes/issues/22541 + // Sometimes when docker's state is corrupt, a container can be listed + // but couldn't be inspected. We fake a status for this container so + // that we can still return a status for the pod to sync. + result = &kubecontainer.ContainerStatus{ + ID: kubecontainer.DockerID(c.ID).ContainerID(), + Name: dockerName.ContainerName, + State: kubecontainer.ContainerStateUnknown, + } + glog.Errorf("Unable to inspect container %q: %v", c.ID, err) + } else { + return podStatus, err + } } containerStatuses = append(containerStatuses, result) if ip != "" { diff --git a/pkg/kubelet/dockertools/manager_test.go b/pkg/kubelet/dockertools/manager_test.go index ff6bcd4fd48..09e4826fde8 100644 --- a/pkg/kubelet/dockertools/manager_test.go +++ b/pkg/kubelet/dockertools/manager_test.go @@ -1887,3 +1887,55 @@ func TestCheckVersionCompatibility(t *testing.T) { } } } + +func TestGetPodStatusNoSuchContainer(t *testing.T) { + const ( + noSuchContainerID = "nosuchcontainer" + infraContainerID = "9876" + ) + dm, fakeDocker := newTestDockerManager() + pod := &api.Pod{ + ObjectMeta: api.ObjectMeta{ + UID: "12345678", + Name: "foo", + Namespace: "new", + }, + Spec: api.PodSpec{ + Containers: []api.Container{{Name: "nosuchcontainer"}}, + }, + } + + fakeDocker.SetFakeContainers([]*docker.Container{ + { + ID: noSuchContainerID, + Name: "/k8s_nosuchcontainer_foo_new_12345678_42", + State: docker.State{ + ExitCode: 0, + StartedAt: time.Now(), + FinishedAt: time.Now(), + Running: false, + }, + }, + { + ID: infraContainerID, + Name: "/k8s_POD." + strconv.FormatUint(generatePodInfraContainerHash(pod), 16) + "_foo_new_12345678_42", + State: docker.State{ + ExitCode: 0, + StartedAt: time.Now(), + FinishedAt: time.Now(), + Running: false, + }, + }}) + + fakeDocker.Errors = map[string]error{"inspect": &docker.NoSuchContainer{}} + runSyncPod(t, dm, fakeDocker, pod, nil, false) + + // Verify that we will try to start new contrainers even if the inspections + // failed. + verifyCalls(t, fakeDocker, []string{ + // Start a new infra container. + "create", "start", "inspect_container", "inspect_container", + // Start a new container. + "create", "start", "inspect_container", + }) +}