Garbage collect unidentified Kubernetes containers.

These containers may be caused by a change in the Kubernetes naming
convention. The old containers are killed, the new ones started, but the
old ones are never GC'd. This change makes Kubelet GC all Kubernetes
containers, old and new.

Fixes #5372.

pkg/kubelet/kubelet.go

@@ -466,12 +466,48 @@ func (kl *Kubelet) GarbageCollectContainers() error {
 	if err != nil {
 		return err
 	}
+
+	type unidentifiedContainer struct {
+		// Docker ID.
+		id string
+
+		// Docker container name
+		name string
+	}
+
+	unidentifiedContainers := make([]unidentifiedContainer, 0)
 	uidToIDMap := map[string][]string{}
 	for _, container := range containers {
 		_, uid, name, _ := dockertools.ParseDockerName(container.Names[0])
+		if uid == "" && name == "" {
+			unidentifiedContainers = append(unidentifiedContainers, unidentifiedContainer{
+				id:   container.ID,
+				name: container.Names[0],
+			})
+			continue
+		}
 		uidName := string(uid) + "." + name
 		uidToIDMap[uidName] = append(uidToIDMap[uidName], container.ID)
 	}
+
+	// Remove all non-running unidentified containers.
+	for _, container := range unidentifiedContainers {
+		data, err := kl.dockerClient.InspectContainer(container.id)
+		if err != nil {
+			return err
+		}
+		if data.State.Running {
+			continue
+		}
+
+		glog.Infof("Removing unidentified dead container %q with ID %q", container.name, container.id)
+		err = kl.dockerClient.RemoveContainer(docker.RemoveContainerOptions{ID: container.id})
+		if err != nil {
+			return err
+		}
+	}
+
+	// Evict dead containers according to our policies.
 	for _, list := range uidToIDMap {
 		if len(list) <= kl.maxContainerCount {
 			continue
@@ -480,6 +516,7 @@ func (kl *Kubelet) GarbageCollectContainers() error {
 			return err
 		}
 	}
+
 	return nil
 }
 

pkg/kubelet/kubelet_test.go

@@ -1741,6 +1741,43 @@ func TestKubeletGarbageCollection(t *testing.T) {
 			},
 			expectedRemoved: []string{"1706", "1876"},
 		},
+		// Remove non-running unidentified Kubernetes containers.
+		{
+			containers: []docker.APIContainers{
+				{
+					// Unidentified Kubernetes container.
+					Names: []string{"/k8s_unidentified"},
+					ID:    "1876",
+				},
+				{
+					// Unidentified (non-running) Kubernetes container.
+					Names: []string{"/k8s_unidentified"},
+					ID:    "2309",
+				},
+				{
+					// Regular Kubernetes container.
+					Names: []string{"/k8s_POD_foo_new_.deadbeef_42"},
+					ID:    "3876",
+				},
+			},
+			containerDetails: map[string]*docker.Container{
+				"1876": {
+					State: docker.State{
+						Running: false,
+					},
+					ID:      "1876",
+					Created: time.Now(),
+				},
+				"2309": {
+					State: docker.State{
+						Running: true,
+					},
+					ID:      "2309",
+					Created: time.Now(),
+				},
+			},
+			expectedRemoved: []string{"1876"},
+		},
 	}
 	for _, test := range tests {
 		kubelet, fakeDocker, _, _ := newTestKubelet(t)