Handle deleted pods in replication and endpoint controllers

Pods that are slated for deletion should be excluded from
replication and endpoints immediately.

pkg/controller/controller_utils.go

@@ -294,7 +294,8 @@ func filterActivePods(pods []api.Pod) []*api.Pod {
 	var result []*api.Pod
 	for i := range pods {
 		if api.PodSucceeded != pods[i].Status.Phase &&
-			api.PodFailed != pods[i].Status.Phase {
+			api.PodFailed != pods[i].Status.Phase &&
+			pods[i].DeletionTimestamp == nil {
 			result = append(result, &pods[i])
 		}
 	}

pkg/controller/replication_controller.go

@@ -204,6 +204,12 @@ func (rm *ReplicationManager) getPodControllers(pod *api.Pod) *api.ReplicationCo
 // When a pod is created, enqueue the controller that manages it and update it's expectations.
 func (rm *ReplicationManager) addPod(obj interface{}) {
 	pod := obj.(*api.Pod)
+	if pod.DeletionTimestamp != nil {
+		// on a restart of the controller manager, it's possible a new pod shows up in a state that
+		// is already pending deletion. Prevent the pod from being a creation observation.
+		rm.deletePod(pod)
+		return
+	}
 	if rc := rm.getPodControllers(pod); rc != nil {
 		rm.expectations.CreationObserved(rc)
 		rm.enqueueController(rc)
@@ -220,6 +226,15 @@ func (rm *ReplicationManager) updatePod(old, cur interface{}) {
 	}
 	// TODO: Write a unittest for this case
 	curPod := cur.(*api.Pod)
+	if curPod.DeletionTimestamp != nil {
+		// when a pod is deleted gracefully it's deletion timestamp is first modified to reflect a grace period,
+		// and after such time has passed, the kubelet actually deletes it from the store. We receive an update
+		// for modification of the deletion timestamp and expect an rc to create more replicas asap, not wait
+		// until the kubelet actually deletes the pod. This is different from the Phase of a pod changing, because
+		// an rc never initiates a phase change, and so is never asleep waiting for the same.
+		rm.deletePod(curPod)
+		return
+	}
 	if rc := rm.getPodControllers(curPod); rc != nil {
 		rm.enqueueController(rc)
 	}

pkg/service/endpoints_controller.go

@@ -308,7 +308,11 @@ func (e *EndpointController) syncService(key string) {
 				continue
 			}
 			if len(pod.Status.PodIP) == 0 {
-				glog.V(4).Infof("Failed to find an IP for pod %s/%s", pod.Namespace, pod.Name)
+				glog.V(5).Infof("Failed to find an IP for pod %s/%s", pod.Namespace, pod.Name)
+				continue
+			}
+			if pod.DeletionTimestamp != nil {
+				glog.V(5).Infof("Pod is being deleted %s/%s", pod.Namespace, pod.Name)
 				continue
 			}