Fix recycler pod deletion race.

We should use clone of recycler pod template instead of reusing the same
one for two or more recyclers running in parallel.

Also add some logs to relevant places to spot the error easily next time.

pkg/volume/host_path/BUILD

@@ -19,6 +19,7 @@ go_library(
     tags = ["automanaged"],
     deps = [
         "//pkg/api:go_default_library",
+        "//pkg/conversion:go_default_library",
         "//pkg/types:go_default_library",
         "//pkg/util/uuid:go_default_library",
         "//pkg/volume:go_default_library",

pkg/volume/host_path/host_path.go

@@ -22,6 +22,7 @@ import (
 	"regexp"
 
 	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/conversion"
 	"k8s.io/kubernetes/pkg/types"
 	"k8s.io/kubernetes/pkg/util/uuid"
 	"k8s.io/kubernetes/pkg/volume"
@@ -244,7 +245,11 @@ func (r *hostPathRecycler) GetPath() string {
 // Recycle blocks until the pod has completed or any error occurs.
 // HostPath recycling only works in single node clusters and is meant for testing purposes only.
 func (r *hostPathRecycler) Recycle() error {
-	pod := r.config.RecyclerPodTemplate
+	templateClone, err := conversion.NewCloner().DeepCopy(r.config.RecyclerPodTemplate)
+	if err != nil {
+		return err
+	}
+	pod := templateClone.(*api.Pod)
 	// overrides
 	pod.Spec.ActiveDeadlineSeconds = &r.timeout
 	pod.Spec.Volumes[0].VolumeSource = api.VolumeSource{

pkg/volume/nfs/BUILD

@@ -19,6 +19,7 @@ go_library(
     tags = ["automanaged"],
     deps = [
         "//pkg/api:go_default_library",
+        "//pkg/conversion:go_default_library",
         "//pkg/types:go_default_library",
         "//pkg/util/exec:go_default_library",
         "//pkg/util/mount:go_default_library",

pkg/volume/nfs/nfs.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/golang/glog"
 	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/conversion"
 	"k8s.io/kubernetes/pkg/types"
 	"k8s.io/kubernetes/pkg/util/exec"
 	"k8s.io/kubernetes/pkg/util/mount"
@@ -332,7 +333,11 @@ func (r *nfsRecycler) GetPath() string {
 // Recycle recycles/scrubs clean an NFS volume.
 // Recycle blocks until the pod has completed or any error occurs.
 func (r *nfsRecycler) Recycle() error {
-	pod := r.config.RecyclerPodTemplate
+	templateClone, err := conversion.NewCloner().DeepCopy(r.config.RecyclerPodTemplate)
+	if err != nil {
+		return err
+	}
+	pod := templateClone.(*api.Pod)
 	// overrides
 	pod.Spec.ActiveDeadlineSeconds = &r.timeout
 	pod.GenerateName = "pv-recycler-nfs-"

pkg/volume/util.go

@@ -83,7 +83,13 @@ func internalRecycleVolumeByWatchingPodUntilCompletion(pvName string, pod *api.P
 			return fmt.Errorf("unexpected error creating recycler pod:  %+v\n", err)
 		}
 	}
-	defer recyclerClient.DeletePod(pod.Name, pod.Namespace)
+	defer func(pod *api.Pod) {
+		if err := recyclerClient.DeletePod(pod.Name, pod.Namespace); err != nil {
+			glog.Errorf("failed to delete recycler pod %s/%s: %v", pod.Namespace, pod.Name, err)
+		} else {
+			glog.V(4).Infof("deleted recycler pod %s/%s", pod.Namespace, pod.Name)
+		}
+	}(pod)
 
 	// Now only the old pod or the new pod run. Watch it until it finishes
 	// and send all events on the pod to the PV