From 17af029bc55fdf210a72f3ad7817b470b8d80bbc Mon Sep 17 00:00:00 2001
From: ymmt <ymmt2005@gmail.com>
Date: Sat, 28 Sep 2019 18:00:16 +0000
Subject: [PATCH] do not create StatefulSet pods when PVC is being deleted

Pod with PVC will not be scheduled if the PVC is being deleted.
This can happen when the PVC has finalizers of storage plugins.

Such a pod becomes pending.  Unfortunately, after the finalizer
finishes and PVC is deleted, the pod remains pending forever.
The StatefulSet controller does nothing for this pending pod.

This commit prevents the StatefulSet controller from creating
such pods when PVC is to be deleted.
---
 .../statefulset/stateful_pod_control.go       |  6 ++-
 .../statefulset/stateful_pod_control_test.go  | 38 +++++++++++++++++++
 2 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/pkg/controller/statefulset/stateful_pod_control.go b/pkg/controller/statefulset/stateful_pod_control.go
index eff625d7068..b995d0892f5 100644
--- a/pkg/controller/statefulset/stateful_pod_control.go
+++ b/pkg/controller/statefulset/stateful_pod_control.go
@@ -181,7 +181,7 @@ func (spc *realStatefulPodControl) recordClaimEvent(verb string, set *apps.State
 func (spc *realStatefulPodControl) createPersistentVolumeClaims(set *apps.StatefulSet, pod *v1.Pod) error {
 	var errs []error
 	for _, claim := range getPersistentVolumeClaims(set, pod) {
-		_, err := spc.pvcLister.PersistentVolumeClaims(claim.Namespace).Get(claim.Name)
+		pvc, err := spc.pvcLister.PersistentVolumeClaims(claim.Namespace).Get(claim.Name)
 		switch {
 		case apierrors.IsNotFound(err):
 			_, err := spc.client.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(context.TODO(), &claim, metav1.CreateOptions{})
@@ -194,6 +194,10 @@ func (spc *realStatefulPodControl) createPersistentVolumeClaims(set *apps.Statef
 		case err != nil:
 			errs = append(errs, fmt.Errorf("failed to retrieve PVC %s: %s", claim.Name, err))
 			spc.recordClaimEvent("create", set, pod, &claim, err)
+		case err == nil:
+			if pvc.DeletionTimestamp != nil {
+				errs = append(errs, fmt.Errorf("pvc %s is being deleted", claim.Name))
+			}
 		}
 		// TODO: Check resource requirements and accessmodes, update if necessary
 	}
diff --git a/pkg/controller/statefulset/stateful_pod_control_test.go b/pkg/controller/statefulset/stateful_pod_control_test.go
index 8c924e8c0a1..b4403b06a59 100644
--- a/pkg/controller/statefulset/stateful_pod_control_test.go
+++ b/pkg/controller/statefulset/stateful_pod_control_test.go
@@ -20,6 +20,7 @@ import (
 	"errors"
 	"strings"
 	"testing"
+	"time"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -29,6 +30,7 @@ import (
 	"k8s.io/client-go/tools/record"
 
 	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes/fake"
 	corelisters "k8s.io/client-go/listers/core/v1"
 	_ "k8s.io/kubernetes/pkg/apis/apps/install"
@@ -128,6 +130,42 @@ func TestStatefulPodControlCreatePodPvcCreateFailure(t *testing.T) {
 		}
 	}
 }
+func TestStatefulPodControlCreatePodPvcDeleting(t *testing.T) {
+	recorder := record.NewFakeRecorder(10)
+	set := newStatefulSet(3)
+	pod := newStatefulSetPod(set, 0)
+	fakeClient := &fake.Clientset{}
+	pvcs := getPersistentVolumeClaims(set, pod)
+	pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
+	deleteTime := time.Date(2019, time.January, 1, 0, 0, 0, 0, time.UTC)
+	for k := range pvcs {
+		pvc := pvcs[k]
+		pvc.DeletionTimestamp = &metav1.Time{Time: deleteTime}
+		pvcIndexer.Add(&pvc)
+	}
+	pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer)
+	control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder)
+	fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) {
+		create := action.(core.CreateAction)
+		return true, create.GetObject(), nil
+	})
+	fakeClient.AddReactor("create", "pods", func(action core.Action) (bool, runtime.Object, error) {
+		create := action.(core.CreateAction)
+		return true, create.GetObject(), nil
+	})
+	if err := control.CreateStatefulPod(set, pod); err == nil {
+		t.Error("Failed to produce error on deleting PVC")
+	}
+	events := collectEvents(recorder.Events)
+	if eventCount := len(events); eventCount != 1 {
+		t.Errorf("Deleting PVC: got %d events, but want 1", eventCount)
+	}
+	for i := range events {
+		if !strings.Contains(events[i], v1.EventTypeWarning) {
+			t.Errorf("Found unexpected non-warning event %s", events[i])
+		}
+	}
+}
 
 type fakeIndexer struct {
 	cache.Indexer