continuously delete pods on nodes that don't exist

pkg/controller/node/nodecontroller.go

@@ -37,6 +37,7 @@ import (
 	"k8s.io/kubernetes/pkg/controller/framework"
 	"k8s.io/kubernetes/pkg/fields"
 	"k8s.io/kubernetes/pkg/kubelet/util/format"
+	"k8s.io/kubernetes/pkg/labels"
 	"k8s.io/kubernetes/pkg/runtime"
 	"k8s.io/kubernetes/pkg/types"
 	"k8s.io/kubernetes/pkg/util"
@@ -284,6 +285,8 @@ func (nc *NodeController) Run(period time.Duration) {
 			return false, remaining
 		})
 	}, nodeEvictionPeriod, wait.NeverStop)
+
+	go wait.Until(nc.cleanupOrphanedPods, 30*time.Second, wait.NeverStop)
 }
 
 // Generates num pod CIDRs that could be assigned to nodes.
@@ -368,6 +371,28 @@ func (nc *NodeController) maybeDeleteTerminatingPod(obj interface{}) {
 	}
 }
 
+// cleanupOrphanedPods deletes pods that are bound to nodes that don't
+// exist.
+func (nc *NodeController) cleanupOrphanedPods() {
+	pods, err := nc.podStore.List(labels.Everything())
+	if err != nil {
+		utilruntime.HandleError(err)
+		return
+	}
+
+	for _, pod := range pods {
+		if pod.Spec.NodeName == "" {
+			continue
+		}
+		if _, exists, _ := nc.nodeStore.Store.GetByKey(pod.Spec.NodeName); exists {
+			continue
+		}
+		if err := nc.forcefullyDeletePod(pod); err != nil {
+			utilruntime.HandleError(err)
+		}
+	}
+}
+
 func forcefullyDeletePod(c clientset.Interface, pod *api.Pod) error {
 	var zero int64
 	err := c.Core().Pods(pod.Namespace).Delete(pod.Name, &api.DeleteOptions{GracePeriodSeconds: &zero})
@@ -759,18 +784,6 @@ func (nc *NodeController) tryUpdateNodeStatus(node *api.Node) (time.Duration, ap
 	return gracePeriod, lastReadyCondition, readyCondition, err
 }
 
-// returns true if the provided node still has pods scheduled to it, or an error if
-// the server could not be contacted.
-func (nc *NodeController) hasPods(nodeName string) (bool, error) {
-	selector := fields.OneTermEqualSelector(api.PodHostField, nodeName)
-	options := api.ListOptions{FieldSelector: selector}
-	pods, err := nc.kubeClient.Core().Pods(api.NamespaceAll).List(options)
-	if err != nil {
-		return false, err
-	}
-	return len(pods.Items) > 0, nil
-}
-
 // evictPods queues an eviction for the provided node name, and returns false if the node is already
 // queued for eviction.
 func (nc *NodeController) evictPods(nodeName string) bool {

pkg/controller/node/nodecontroller_test.go

@@ -1116,6 +1116,48 @@ func TestCheckPod(t *testing.T) {
 	}
 }
 
+func TestCleanupOrphanedPods(t *testing.T) {
+	newPod := func(name, node string) api.Pod {
+		return api.Pod{
+			ObjectMeta: api.ObjectMeta{
+				Name: name,
+			},
+			Spec: api.PodSpec{
+				NodeName: node,
+			},
+		}
+	}
+	pods := []api.Pod{
+		newPod("a", "foo"),
+		newPod("b", "bar"),
+		newPod("c", "gone"),
+	}
+	nc := NewNodeController(nil, nil, 0, nil, nil, 0, 0, 0, nil, false)
+
+	nc.nodeStore.Store.Add(newNode("foo"))
+	nc.nodeStore.Store.Add(newNode("bar"))
+	for _, pod := range pods {
+		p := pod
+		nc.podStore.Store.Add(&p)
+	}
+
+	var deleteCalls int
+	var deletedPodName string
+	nc.forcefullyDeletePod = func(p *api.Pod) error {
+		deleteCalls++
+		deletedPodName = p.ObjectMeta.Name
+		return nil
+	}
+	nc.cleanupOrphanedPods()
+
+	if deleteCalls != 1 {
+		t.Fatalf("expected one delete, got: %v", deleteCalls)
+	}
+	if deletedPodName != "c" {
+		t.Fatalf("expected deleted pod name to be 'c', but got: %q", deletedPodName)
+	}
+}
+
 func newNode(name string) *api.Node {
 	return &api.Node{
 		ObjectMeta: api.ObjectMeta{Name: name},