Merge pull request #37206 from gmarek/nodecontroller

Automatic merge from submit-queue

Add more logging around Pod deletion

After this PR we'll have at least V(2) level log near all Pod deletions.

@saad-ali - this is required by GKE to help with diagnosing possible problem.

cc @dchen1107 @wojtek-t

pkg/controller/controller_utils.go

@@ -520,11 +520,11 @@ func (r RealPodControl) DeletePod(namespace string, podID string, object runtime
 	if err != nil {
 		return fmt.Errorf("object does not have ObjectMeta, %v", err)
 	}
+	glog.V(2).Infof("Controller %v deleting pod %v/%v", accessor.GetName(), namespace, podID)
 	if err := r.KubeClient.Core().Pods(namespace).Delete(podID, nil); err != nil {
 		r.Recorder.Eventf(object, api.EventTypeWarning, FailedDeletePodReason, "Error deleting: %v", err)
 		return fmt.Errorf("unable to delete pods: %v", err)
 	} else {
-		glog.V(4).Infof("Controller %v deleted pod %v", accessor.GetName(), podID)
 		r.Recorder.Eventf(object, api.EventTypeNormal, SuccessfulDeletePodReason, "Deleted pod: %v", podID)
 	}
 	return nil

pkg/controller/cronjob/controller.go

@@ -231,6 +231,7 @@ func SyncOne(sj batch.CronJob, js []batch.Job, now time.Time, jc jobControlInter
 			}
 			errList := []error{}
 			for _, pod := range podList.Items {
+				glog.V(2).Infof("CronJob controller is deleting Pod %v/%v", pod.Namespace, pod.Name)
 				if err := pc.DeletePod(pod.Namespace, pod.Name); err != nil {
 					// ignores the error when the pod isn't found
 					if !errors.IsNotFound(err) {

pkg/controller/petset/pet.go

@@ -107,7 +107,7 @@ func (p *petSyncer) Sync(pet *pcb) error {
 	}
 	// if pet failed - we need to remove old one because of consistent naming
 	if exists && realPet.pod.Status.Phase == api.PodFailed {
-		glog.V(4).Infof("Delete evicted pod %v", realPet.pod.Name)
+		glog.V(2).Infof("Deleting evicted pod %v/%v", realPet.pod.Namespace, realPet.pod.Name)
 		if err := p.petClient.Delete(realPet); err != nil {
 			return err
 		}
@@ -156,7 +156,7 @@ func (p *petSyncer) Delete(pet *pcb) error {
 	// The returned error will force a requeue.
 	p.blockingPet = realPet
 	if !p.isDying(realPet.pod) {
-		glog.V(4).Infof("StatefulSet %v deleting pet %v", pet.parent.Name, pet.pod.Name)
+		glog.V(2).Infof("StatefulSet %v deleting pet %v/%v", pet.parent.Name, pet.pod.Namespace, pet.pod.Name)
 		return p.petClient.Delete(pet)
 	}
 	glog.V(4).Infof("StatefulSet %v waiting on pet %v to die in %v", pet.parent.Name, realPet.pod.Name, realPet.pod.DeletionTimestamp)

pkg/kubelet/pod/mirror_client.go

@@ -81,7 +81,7 @@ func (mc *basicMirrorClient) DeleteMirrorPod(podFullName string) error {
 		glog.Errorf("Failed to parse a pod full name %q", podFullName)
 		return err
 	}
-	glog.V(4).Infof("Deleting a mirror pod %q", podFullName)
+	glog.V(2).Infof("Deleting a mirror pod %q", podFullName)
 	// TODO(random-liu): Delete the mirror pod with uid precondition in mirror pod manager
 	if err := mc.apiserverClient.Core().Pods(namespace).Delete(name, api.NewDeleteOptions(0)); err != nil && !errors.IsNotFound(err) {
 		glog.Errorf("Failed deleting a mirror pod %q: %v", podFullName, err)

pkg/kubelet/status/status_manager.go

@@ -438,6 +438,7 @@ func (m *manager) syncPod(uid types.UID, status versionedPodStatus) {
 			deleteOptions := api.NewDeleteOptions(0)
 			// Use the pod UID as the precondition for deletion to prevent deleting a newly created pod with the same name and namespace.
 			deleteOptions.Preconditions = api.NewUIDPreconditions(string(pod.UID))
+			glog.V(2).Infof("Removing Pod %q from etcd", format.Pod(pod))
 			if err = m.kubeClient.Core().Pods(pod.Namespace).Delete(pod.Name, deleteOptions); err == nil {
 				glog.V(3).Infof("Pod %q fully terminated and removed from etcd", format.Pod(pod))
 				m.deletePodStatus(uid)

pkg/volume/util.go

@@ -84,10 +84,9 @@ func internalRecycleVolumeByWatchingPodUntilCompletion(pvName string, pod *api.P
 		}
 	}
 	defer func(pod *api.Pod) {
+		glog.V(2).Infof("deleting recycler pod %s/%s", pod.Namespace, pod.Name)
 		if err := recyclerClient.DeletePod(pod.Name, pod.Namespace); err != nil {
 			glog.Errorf("failed to delete recycler pod %s/%s: %v", pod.Namespace, pod.Name, err)
-		} else {
-			glog.V(4).Infof("deleted recycler pod %s/%s", pod.Namespace, pod.Name)
 		}
 	}(pod)