mirror of
https://github.com/k3s-io/kubernetes.git
synced 2026-01-05 07:27:21 +00:00
StatefulSet refactoring and semantics fix
1. pcb and pcb controller are removed and their functionality is encapsulated in StatefulPodControlInterface. 2. IdentityMappers has been removed to clarify what properties of a Pod are mutated by the controller. All mutations are performed in the UpdateStatefulPod method of the StatefulPodControlInterface. 3. The statefulSetIterator and petQueue classes are removed. These classes sorted Pods by CreationTimestamp. This is brittle and not resilient to clock skew. The current control loop, which implements the same logic, is in stateful_set_control.go. The Pods are now sorted and considered by their ordinal indices, as is outlined in the documentation. 4. StatefulSetController now checks to see if the Pods matching a StatefulSet's Selector also match the Name of the StatefulSet. This will make the controller resilient to overlapping, and will be enhanced by the addition of ControllerRefs.
This commit is contained in:
Kenneth Owens
@@ -17,316 +17,444 @@ limitations under the License.
|
||||
package statefulset
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"k8s.io/apimachinery/pkg/util/errors"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
"k8s.io/client-go/util/workqueue"
|
||||
|
||||
"k8s.io/kubernetes/pkg/api/v1"
|
||||
apps "k8s.io/kubernetes/pkg/apis/apps/v1beta1"
|
||||
fakeinternal "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/fake"
|
||||
"k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/apps/v1beta1"
|
||||
"k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/apps/v1beta1/fake"
|
||||
"k8s.io/kubernetes/pkg/client/legacylisters"
|
||||
"k8s.io/kubernetes/pkg/controller"
|
||||
)
|
||||
|
||||
func newFakeStatefulSetController() (*StatefulSetController, *fakePetClient) {
|
||||
fpc := newFakePetClient()
|
||||
return &StatefulSetController{
|
||||
kubeClient: nil,
|
||||
blockingPetStore: newUnHealthyPetTracker(fpc),
|
||||
podStoreSynced: func() bool { return true },
|
||||
psStore: listers.StoreToStatefulSetLister{Store: cache.NewStore(controller.KeyFunc)},
|
||||
podStore: listers.StoreToPodLister{Indexer: cache.NewIndexer(controller.KeyFunc, cache.Indexers{})},
|
||||
newSyncer: func(blockingPet *pcb) *petSyncer {
|
||||
return &petSyncer{fpc, blockingPet}
|
||||
},
|
||||
}, fpc
|
||||
}
|
||||
|
||||
func checkPets(ps *apps.StatefulSet, creates, deletes int, fc *fakePetClient, t *testing.T) {
|
||||
if fc.petsCreated != creates || fc.petsDeleted != deletes {
|
||||
t.Errorf("Found (creates: %d, deletes: %d), expected (creates: %d, deletes: %d)", fc.petsCreated, fc.petsDeleted, creates, deletes)
|
||||
}
|
||||
gotClaims := map[string]v1.PersistentVolumeClaim{}
|
||||
for _, pvc := range fc.claims {
|
||||
gotClaims[pvc.Name] = pvc
|
||||
}
|
||||
for i := range fc.pets {
|
||||
expectedPet, _ := newPCB(fmt.Sprintf("%v", i), ps)
|
||||
if identityHash(ps, fc.pets[i].pod) != identityHash(ps, expectedPet.pod) {
|
||||
t.Errorf("Unexpected pod at index %d", i)
|
||||
}
|
||||
for _, pvc := range expectedPet.pvcs {
|
||||
gotPVC, ok := gotClaims[pvc.Name]
|
||||
if !ok {
|
||||
t.Errorf("PVC %v not created for pod %v", pvc.Name, expectedPet.pod.Name)
|
||||
}
|
||||
if !reflect.DeepEqual(gotPVC.Spec, pvc.Spec) {
|
||||
t.Errorf("got PVC %v differs from created pvc", pvc.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func scaleStatefulSet(t *testing.T, ps *apps.StatefulSet, psc *StatefulSetController, fc *fakePetClient, scale int) error {
|
||||
errs := []error{}
|
||||
for i := 0; i < scale; i++ {
|
||||
pl := fc.getPodList()
|
||||
if len(pl) != i {
|
||||
t.Errorf("Unexpected number of pods, expected %d found %d", i, len(pl))
|
||||
}
|
||||
if _, syncErr := psc.syncStatefulSet(ps, pl); syncErr != nil {
|
||||
errs = append(errs, syncErr)
|
||||
}
|
||||
fc.setHealthy(i)
|
||||
checkPets(ps, i+1, 0, fc, t)
|
||||
}
|
||||
return errors.NewAggregate(errs)
|
||||
}
|
||||
|
||||
func saturateStatefulSet(t *testing.T, ps *apps.StatefulSet, psc *StatefulSetController, fc *fakePetClient) {
|
||||
err := scaleStatefulSet(t, ps, psc, fc, int(*(ps.Spec.Replicas)))
|
||||
if err != nil {
|
||||
t.Errorf("Error scaleStatefulSet: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatefulSetControllerCreates(t *testing.T) {
|
||||
psc, fc := newFakeStatefulSetController()
|
||||
replicas := 3
|
||||
ps := newStatefulSet(replicas)
|
||||
|
||||
saturateStatefulSet(t, ps, psc, fc)
|
||||
|
||||
podList := fc.getPodList()
|
||||
// Deleted pet gets recreated
|
||||
fc.pets = fc.pets[:replicas-1]
|
||||
if _, err := psc.syncStatefulSet(ps, podList); err != nil {
|
||||
t.Errorf("Error syncing StatefulSet: %v", err)
|
||||
ssc, spc := newFakeStatefulSetController()
|
||||
set := newStatefulSet(3)
|
||||
if err := scaleUpStatefulSetController(set, ssc, spc); err != nil {
|
||||
t.Errorf("Failed to turn up StatefulSet : %s", err)
|
||||
}
|
||||
if obj, _, err := spc.setsIndexer.Get(set); err != nil {
|
||||
t.Error(err)
|
||||
} else {
|
||||
set = obj.(*apps.StatefulSet)
|
||||
}
|
||||
if set.Status.Replicas != 3 {
|
||||
t.Error("Falied to scale statefulset to 3 replicas")
|
||||
}
|
||||
checkPets(ps, replicas+1, 0, fc, t)
|
||||
}
|
||||
|
||||
func TestStatefulSetControllerDeletes(t *testing.T) {
|
||||
psc, fc := newFakeStatefulSetController()
|
||||
replicas := 4
|
||||
ps := newStatefulSet(replicas)
|
||||
|
||||
saturateStatefulSet(t, ps, psc, fc)
|
||||
|
||||
// Drain
|
||||
errs := []error{}
|
||||
*(ps.Spec.Replicas) = 0
|
||||
knownPods := fc.getPodList()
|
||||
for i := replicas - 1; i >= 0; i-- {
|
||||
if len(fc.pets) != i+1 {
|
||||
t.Errorf("Unexpected number of pods, expected %d found %d", i+1, len(fc.pets))
|
||||
}
|
||||
if _, syncErr := psc.syncStatefulSet(ps, knownPods); syncErr != nil {
|
||||
errs = append(errs, syncErr)
|
||||
}
|
||||
ssc, spc := newFakeStatefulSetController()
|
||||
set := newStatefulSet(3)
|
||||
if err := scaleUpStatefulSetController(set, ssc, spc); err != nil {
|
||||
t.Errorf("Failed to turn up StatefulSet : %s", err)
|
||||
}
|
||||
if len(errs) != 0 {
|
||||
t.Errorf("Error syncing StatefulSet: %v", errors.NewAggregate(errs))
|
||||
if obj, _, err := spc.setsIndexer.Get(set); err != nil {
|
||||
t.Error(err)
|
||||
} else {
|
||||
set = obj.(*apps.StatefulSet)
|
||||
}
|
||||
if set.Status.Replicas != 3 {
|
||||
t.Error("Falied to scale statefulset to 3 replicas")
|
||||
}
|
||||
*set.Spec.Replicas = 0
|
||||
if err := scaleDownStatefulSetController(set, ssc, spc); err != nil {
|
||||
t.Errorf("Failed to turn down StatefulSet : %s", err)
|
||||
}
|
||||
if obj, _, err := spc.setsIndexer.Get(set); err != nil {
|
||||
t.Error(err)
|
||||
} else {
|
||||
set = obj.(*apps.StatefulSet)
|
||||
}
|
||||
if set.Status.Replicas != 0 {
|
||||
t.Error("Falied to scale statefulset to 3 replicas")
|
||||
}
|
||||
checkPets(ps, replicas, replicas, fc, t)
|
||||
}
|
||||
|
||||
func TestStatefulSetControllerRespectsTermination(t *testing.T) {
|
||||
psc, fc := newFakeStatefulSetController()
|
||||
replicas := 4
|
||||
ps := newStatefulSet(replicas)
|
||||
|
||||
saturateStatefulSet(t, ps, psc, fc)
|
||||
|
||||
fc.setDeletionTimestamp(replicas - 1)
|
||||
*(ps.Spec.Replicas) = 2
|
||||
_, err := psc.syncStatefulSet(ps, fc.getPodList())
|
||||
ssc, spc := newFakeStatefulSetController()
|
||||
set := newStatefulSet(3)
|
||||
if err := scaleUpStatefulSetController(set, ssc, spc); err != nil {
|
||||
t.Errorf("Failed to turn up StatefulSet : %s", err)
|
||||
}
|
||||
if obj, _, err := spc.setsIndexer.Get(set); err != nil {
|
||||
t.Error(err)
|
||||
} else {
|
||||
set = obj.(*apps.StatefulSet)
|
||||
}
|
||||
if set.Status.Replicas != 3 {
|
||||
t.Error("Falied to scale statefulset to 3 replicas")
|
||||
}
|
||||
pods, err := spc.addTerminatedPod(set, 3)
|
||||
if err != nil {
|
||||
t.Errorf("Error syncing StatefulSet: %v", err)
|
||||
t.Error(err)
|
||||
}
|
||||
// Finding a pod with the deletion timestamp will pause all deletions.
|
||||
knownPods := fc.getPodList()
|
||||
if len(knownPods) != 4 {
|
||||
t.Errorf("Pods deleted prematurely before deletion timestamp expired, len %d", len(knownPods))
|
||||
}
|
||||
fc.pets = fc.pets[:replicas-1]
|
||||
_, err = psc.syncStatefulSet(ps, fc.getPodList())
|
||||
pods, err = spc.addTerminatedPod(set, 4)
|
||||
if err != nil {
|
||||
t.Errorf("Error syncing StatefulSet: %v", err)
|
||||
t.Error(err)
|
||||
}
|
||||
checkPets(ps, replicas, 1, fc, t)
|
||||
}
|
||||
|
||||
func TestStatefulSetControllerRespectsOrder(t *testing.T) {
|
||||
psc, fc := newFakeStatefulSetController()
|
||||
replicas := 4
|
||||
ps := newStatefulSet(replicas)
|
||||
|
||||
saturateStatefulSet(t, ps, psc, fc)
|
||||
|
||||
errs := []error{}
|
||||
*(ps.Spec.Replicas) = 0
|
||||
// Shuffle known list and check that pets are deleted in reverse
|
||||
knownPods := fc.getPodList()
|
||||
for i := range knownPods {
|
||||
j := rand.Intn(i + 1)
|
||||
knownPods[i], knownPods[j] = knownPods[j], knownPods[i]
|
||||
ssc.syncStatefulSet(set, pods)
|
||||
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
for i := 0; i < replicas; i++ {
|
||||
if len(fc.pets) != replicas-i {
|
||||
t.Errorf("Unexpected number of pods, expected %d found %d", i, len(fc.pets))
|
||||
}
|
||||
if _, syncErr := psc.syncStatefulSet(ps, knownPods); syncErr != nil {
|
||||
errs = append(errs, syncErr)
|
||||
}
|
||||
checkPets(ps, replicas, i+1, fc, t)
|
||||
pods, err = spc.podsLister.Pods(set.Namespace).List(selector)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if len(errs) != 0 {
|
||||
t.Errorf("Error syncing StatefulSet: %v", errors.NewAggregate(errs))
|
||||
if len(pods) != 5 {
|
||||
t.Error("StatefulSet does not respect termination")
|
||||
}
|
||||
sort.Sort(ascendingOrdinal(pods))
|
||||
spc.DeleteStatefulPod(set, pods[3])
|
||||
spc.DeleteStatefulPod(set, pods[4])
|
||||
*set.Spec.Replicas = 0
|
||||
if err := scaleDownStatefulSetController(set, ssc, spc); err != nil {
|
||||
t.Errorf("Failed to turn down StatefulSet : %s", err)
|
||||
}
|
||||
if obj, _, err := spc.setsIndexer.Get(set); err != nil {
|
||||
t.Error(err)
|
||||
} else {
|
||||
set = obj.(*apps.StatefulSet)
|
||||
}
|
||||
if set.Status.Replicas != 0 {
|
||||
t.Error("Falied to scale statefulset to 3 replicas")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatefulSetControllerBlocksScaling(t *testing.T) {
|
||||
psc, fc := newFakeStatefulSetController()
|
||||
replicas := 5
|
||||
ps := newStatefulSet(replicas)
|
||||
scaleStatefulSet(t, ps, psc, fc, 3)
|
||||
|
||||
// Create 4th pet, then before flipping it to healthy, kill the first pet.
|
||||
// There should only be 1 not-healty pet at a time.
|
||||
pl := fc.getPodList()
|
||||
if _, err := psc.syncStatefulSet(ps, pl); err != nil {
|
||||
t.Errorf("Error syncing StatefulSet: %v", err)
|
||||
ssc, spc := newFakeStatefulSetController()
|
||||
set := newStatefulSet(3)
|
||||
if err := scaleUpStatefulSetController(set, ssc, spc); err != nil {
|
||||
t.Errorf("Failed to turn up StatefulSet : %s", err)
|
||||
}
|
||||
|
||||
deletedPod := pl[0]
|
||||
fc.deletePetAtIndex(0)
|
||||
pl = fc.getPodList()
|
||||
if _, err := psc.syncStatefulSet(ps, pl); err != nil {
|
||||
t.Errorf("Error syncing StatefulSet: %v", err)
|
||||
if obj, _, err := spc.setsIndexer.Get(set); err != nil {
|
||||
t.Error(err)
|
||||
} else {
|
||||
set = obj.(*apps.StatefulSet)
|
||||
}
|
||||
newPodList := fc.getPodList()
|
||||
for _, p := range newPodList {
|
||||
if p.Name == deletedPod.Name {
|
||||
t.Errorf("Deleted pod was created while existing pod was unhealthy")
|
||||
if set.Status.Replicas != 3 {
|
||||
t.Error("Falied to scale statefulset to 3 replicas")
|
||||
}
|
||||
*set.Spec.Replicas = 5
|
||||
fakeResourceVersion(set)
|
||||
spc.setsIndexer.Update(set)
|
||||
pods, err := spc.setPodTerminated(set, 0)
|
||||
if err != nil {
|
||||
t.Error("Failed to set pod terminated at ordinal 0")
|
||||
}
|
||||
ssc.enqueueStatefulSet(set)
|
||||
fakeWorker(ssc)
|
||||
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
pods, err = spc.podsLister.Pods(set.Namespace).List(selector)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if len(pods) != 3 {
|
||||
t.Error("StatefulSet does not block scaling")
|
||||
}
|
||||
sort.Sort(ascendingOrdinal(pods))
|
||||
spc.DeleteStatefulPod(set, pods[0])
|
||||
ssc.enqueueStatefulSet(set)
|
||||
fakeWorker(ssc)
|
||||
pods, err = spc.podsLister.Pods(set.Namespace).List(selector)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if len(pods) != 3 {
|
||||
t.Error("StatefulSet does not resume when terminated Pod is removed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStateSetControllerAddPod(t *testing.T) {
|
||||
ssc, spc := newFakeStatefulSetController()
|
||||
set := newStatefulSet(3)
|
||||
pod := newStatefulSetPod(set, 0)
|
||||
spc.setsIndexer.Add(set)
|
||||
ssc.addPod(pod)
|
||||
key, done := ssc.queue.Get()
|
||||
if key == nil || done {
|
||||
t.Error("Failed to enqueue StatefulSet")
|
||||
} else if key, ok := key.(string); !ok {
|
||||
t.Error("Key is not a string")
|
||||
} else if expectedKey, _ := controller.KeyFunc(set); expectedKey != key {
|
||||
t.Errorf("Expected StatefulSet key %s found %s", expectedKey, key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStateSetControllerAddPodNoSet(t *testing.T) {
|
||||
ssc, _ := newFakeStatefulSetController()
|
||||
set := newStatefulSet(3)
|
||||
pod := newStatefulSetPod(set, 0)
|
||||
ssc.addPod(pod)
|
||||
ssc.queue.ShutDown()
|
||||
key, _ := ssc.queue.Get()
|
||||
if key != nil {
|
||||
t.Errorf("StatefulSet enqueued key for Pod with no Set %s", key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatefulSetControllerUpdatePod(t *testing.T) {
|
||||
ssc, spc := newFakeStatefulSetController()
|
||||
set := newStatefulSet(3)
|
||||
pod := newStatefulSetPod(set, 0)
|
||||
spc.setsIndexer.Add(set)
|
||||
prev := *pod
|
||||
fakeResourceVersion(pod)
|
||||
ssc.updatePod(&prev, pod)
|
||||
key, done := ssc.queue.Get()
|
||||
if key == nil || done {
|
||||
t.Error("Failed to enqueue StatefulSet")
|
||||
} else if key, ok := key.(string); !ok {
|
||||
t.Error("Key is not a string")
|
||||
} else if expectedKey, _ := controller.KeyFunc(set); expectedKey != key {
|
||||
t.Errorf("Expected StatefulSet key %s found %s", expectedKey, key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatefulSetControllerUpdatePodWithNoSet(t *testing.T) {
|
||||
ssc, _ := newFakeStatefulSetController()
|
||||
set := newStatefulSet(3)
|
||||
pod := newStatefulSetPod(set, 0)
|
||||
prev := *pod
|
||||
fakeResourceVersion(pod)
|
||||
ssc.updatePod(&prev, pod)
|
||||
ssc.queue.ShutDown()
|
||||
key, _ := ssc.queue.Get()
|
||||
if key != nil {
|
||||
t.Errorf("StatefulSet enqueued key for Pod with no Set %s", key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatefulSetControllerUpdatePodWithSameVersion(t *testing.T) {
|
||||
ssc, spc := newFakeStatefulSetController()
|
||||
set := newStatefulSet(3)
|
||||
pod := newStatefulSetPod(set, 0)
|
||||
spc.setsIndexer.Add(set)
|
||||
ssc.updatePod(pod, pod)
|
||||
ssc.queue.ShutDown()
|
||||
key, _ := ssc.queue.Get()
|
||||
if key != nil {
|
||||
t.Errorf("StatefulSet enqueued key for Pod with no Set %s", key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatefulSetControllerUpdatePodWithNewLabels(t *testing.T) {
|
||||
ssc, spc := newFakeStatefulSetController()
|
||||
set := newStatefulSet(3)
|
||||
pod := newStatefulSetPod(set, 0)
|
||||
set2 := newStatefulSet(3)
|
||||
set2.Name = "foo2"
|
||||
set2.Spec.Selector.MatchLabels = map[string]string{"foo2": "bar2"}
|
||||
set2.Spec.Template.Labels = map[string]string{"foo2": "bar2"}
|
||||
spc.setsIndexer.Add(set)
|
||||
spc.setsIndexer.Add(set2)
|
||||
clone := *pod
|
||||
clone.Labels = map[string]string{"foo2": "bar2"}
|
||||
fakeResourceVersion(&clone)
|
||||
ssc.updatePod(pod, &clone)
|
||||
key, done := ssc.queue.Get()
|
||||
if key == nil || done {
|
||||
t.Error("Failed to enqueue StatefulSet")
|
||||
} else if key, ok := key.(string); !ok {
|
||||
t.Error("Key is not a string")
|
||||
} else if expectedKey, _ := controller.KeyFunc(set2); expectedKey != key {
|
||||
t.Errorf("Expected StatefulSet key %s found %s", expectedKey, key)
|
||||
}
|
||||
key, done = ssc.queue.Get()
|
||||
if key == nil || done {
|
||||
t.Error("Failed to enqueue StatefulSet")
|
||||
} else if key, ok := key.(string); !ok {
|
||||
t.Error("Key is not a string")
|
||||
} else if expectedKey, _ := controller.KeyFunc(set); expectedKey != key {
|
||||
t.Errorf("Expected StatefulSet key %s found %s", expectedKey, key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatefulSetControllerDeletePod(t *testing.T) {
|
||||
ssc, spc := newFakeStatefulSetController()
|
||||
set := newStatefulSet(3)
|
||||
pod := newStatefulSetPod(set, 0)
|
||||
spc.setsIndexer.Add(set)
|
||||
ssc.deletePod(pod)
|
||||
key, done := ssc.queue.Get()
|
||||
if key == nil || done {
|
||||
t.Error("Failed to enqueue StatefulSet")
|
||||
} else if key, ok := key.(string); !ok {
|
||||
t.Error("Key is not a string")
|
||||
} else if expectedKey, _ := controller.KeyFunc(set); expectedKey != key {
|
||||
t.Errorf("Expected StatefulSet key %s found %s", expectedKey, key)
|
||||
}
|
||||
}
|
||||
func TestStatefulSetControllerDeletePodTombstone(t *testing.T) {
|
||||
ssc, spc := newFakeStatefulSetController()
|
||||
set := newStatefulSet(3)
|
||||
pod := newStatefulSetPod(set, 0)
|
||||
spc.setsIndexer.Add(set)
|
||||
tombstoneKey, _ := controller.KeyFunc(pod)
|
||||
tombstone := cache.DeletedFinalStateUnknown{Key: tombstoneKey, Obj: pod}
|
||||
ssc.deletePod(tombstone)
|
||||
key, done := ssc.queue.Get()
|
||||
if key == nil || done {
|
||||
t.Error("Failed to enqueue StatefulSet")
|
||||
} else if key, ok := key.(string); !ok {
|
||||
t.Error("Key is not a string")
|
||||
} else if expectedKey, _ := controller.KeyFunc(set); expectedKey != key {
|
||||
t.Errorf("Expected StatefulSet key %s found %s", expectedKey, key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatefulSetControllerGetStatefulSetForPod(t *testing.T) {
|
||||
ssc, spc := newFakeStatefulSetController()
|
||||
set := newStatefulSet(3)
|
||||
pod := newStatefulSetPod(set, 0)
|
||||
spc.setsIndexer.Add(set)
|
||||
spc.podsIndexer.Add(pod)
|
||||
if set := ssc.getStatefulSetForPod(pod); set == nil {
|
||||
t.Error("Failed to get StatefulSet for Pod ")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatefulSetControllerGetStatefulSetForPodOverlapping(t *testing.T) {
|
||||
ssc, spc := newFakeStatefulSetController()
|
||||
set := newStatefulSet(3)
|
||||
pod := newStatefulSetPod(set, 0)
|
||||
set2 := newStatefulSet(3)
|
||||
set2.Name = "foo2"
|
||||
set3 := newStatefulSet(3)
|
||||
set3.Name = "foo3"
|
||||
set3.CreationTimestamp.Add(1 * time.Second)
|
||||
spc.setsIndexer.Add(set3)
|
||||
spc.setsIndexer.Add(set2)
|
||||
spc.setsIndexer.Add(set)
|
||||
spc.podsIndexer.Add(pod)
|
||||
if found := ssc.getStatefulSetForPod(pod); found == nil {
|
||||
t.Error("Failed to get StatefulSet for Pod")
|
||||
} else if found.Name != set.Name {
|
||||
t.Errorf("Returned wrong StatefulSet %s for Pod", set.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func newFakeStatefulSetController() (*StatefulSetController, *fakeStatefulPodControl) {
|
||||
fpc := newFakeStatefulPodControl()
|
||||
ssc := &StatefulSetController{
|
||||
kubeClient: nil,
|
||||
podStoreSynced: func() bool { return true },
|
||||
setStore: fpc.setsLister,
|
||||
podStore: fpc.podsLister,
|
||||
control: NewDefaultStatefulSetControl(fpc),
|
||||
queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "statefulset"),
|
||||
}
|
||||
return ssc, fpc
|
||||
}
|
||||
|
||||
func fakeWorker(ssc *StatefulSetController) {
|
||||
if obj, done := ssc.queue.Get(); !done {
|
||||
ssc.sync(obj.(string))
|
||||
ssc.queue.Done(obj)
|
||||
}
|
||||
}
|
||||
|
||||
func getPodAtOrdinal(pods []*v1.Pod, ordinal int) *v1.Pod {
|
||||
if 0 > ordinal || ordinal >= len(pods) {
|
||||
return nil
|
||||
}
|
||||
sort.Sort(ascendingOrdinal(pods))
|
||||
return pods[ordinal]
|
||||
}
|
||||
|
||||
func scaleUpStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetController, spc *fakeStatefulPodControl) error {
|
||||
spc.setsIndexer.Add(set)
|
||||
ssc.enqueueStatefulSet(set)
|
||||
fakeWorker(ssc)
|
||||
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for set.Status.Replicas < *set.Spec.Replicas {
|
||||
pods, err := spc.podsLister.Pods(set.Namespace).List(selector)
|
||||
ord := len(pods) - 1
|
||||
pod := getPodAtOrdinal(pods, ord)
|
||||
if pods, err = spc.setPodPending(set, ord); err != nil {
|
||||
return err
|
||||
}
|
||||
pod = getPodAtOrdinal(pods, ord)
|
||||
ssc.addPod(pod)
|
||||
fakeWorker(ssc)
|
||||
pod = getPodAtOrdinal(pods, ord)
|
||||
prev := *pod
|
||||
if pods, err = spc.setPodRunning(set, ord); err != nil {
|
||||
return err
|
||||
}
|
||||
pod = getPodAtOrdinal(pods, ord)
|
||||
ssc.updatePod(&prev, pod)
|
||||
fakeWorker(ssc)
|
||||
pod = getPodAtOrdinal(pods, ord)
|
||||
prev = *pod
|
||||
if pods, err = spc.setPodReady(set, ord); err != nil {
|
||||
return err
|
||||
}
|
||||
pod = getPodAtOrdinal(pods, ord)
|
||||
ssc.updatePod(&prev, pod)
|
||||
fakeWorker(ssc)
|
||||
if err := assertInvariants(set, spc); err != nil {
|
||||
return err
|
||||
}
|
||||
if obj, _, err := spc.setsIndexer.Get(set); err != nil {
|
||||
return err
|
||||
} else {
|
||||
set = obj.(*apps.StatefulSet)
|
||||
}
|
||||
|
||||
}
|
||||
return assertInvariants(set, spc)
|
||||
}
|
||||
|
||||
func scaleDownStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetController, spc *fakeStatefulPodControl) error {
|
||||
selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pods, err := spc.podsLister.Pods(set.Namespace).List(selector)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ord := len(pods) - 1
|
||||
pod := getPodAtOrdinal(pods, ord)
|
||||
prev := *pod
|
||||
fakeResourceVersion(set)
|
||||
spc.setsIndexer.Add(set)
|
||||
ssc.enqueueStatefulSet(set)
|
||||
fakeWorker(ssc)
|
||||
pods, err = spc.addTerminatedPod(set, ord)
|
||||
pod = getPodAtOrdinal(pods, ord)
|
||||
ssc.updatePod(&prev, pod)
|
||||
fakeWorker(ssc)
|
||||
spc.DeleteStatefulPod(set, pod)
|
||||
ssc.deletePod(pod)
|
||||
fakeWorker(ssc)
|
||||
for set.Status.Replicas > *set.Spec.Replicas {
|
||||
pods, err = spc.podsLister.Pods(set.Namespace).List(selector)
|
||||
ord := len(pods)
|
||||
pods, err = spc.addTerminatedPod(set, ord)
|
||||
pod = getPodAtOrdinal(pods, ord)
|
||||
ssc.updatePod(&prev, pod)
|
||||
fakeWorker(ssc)
|
||||
spc.DeleteStatefulPod(set, pod)
|
||||
ssc.deletePod(pod)
|
||||
fakeWorker(ssc)
|
||||
if obj, _, err := spc.setsIndexer.Get(set); err != nil {
|
||||
return err
|
||||
} else {
|
||||
set = obj.(*apps.StatefulSet)
|
||||
}
|
||||
}
|
||||
|
||||
fc.setHealthy(len(newPodList) - 1)
|
||||
if _, err := psc.syncStatefulSet(ps, pl); err != nil {
|
||||
t.Errorf("Error syncing StatefulSet: %v", err)
|
||||
}
|
||||
|
||||
found := false
|
||||
for _, p := range fc.getPodList() {
|
||||
if p.Name == deletedPod.Name {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("Deleted pod was not created after existing pods became healthy")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatefulSetBlockingPetIsCleared(t *testing.T) {
|
||||
psc, fc := newFakeStatefulSetController()
|
||||
ps := newStatefulSet(3)
|
||||
scaleStatefulSet(t, ps, psc, fc, 1)
|
||||
|
||||
if blocking, err := psc.blockingPetStore.Get(ps, fc.getPodList()); err != nil || blocking != nil {
|
||||
t.Errorf("Unexpected blocking pod %v, err %v", blocking, err)
|
||||
}
|
||||
|
||||
// 1 not yet healthy pet
|
||||
psc.syncStatefulSet(ps, fc.getPodList())
|
||||
|
||||
if blocking, err := psc.blockingPetStore.Get(ps, fc.getPodList()); err != nil || blocking == nil {
|
||||
t.Errorf("Expected blocking pod %v, err %v", blocking, err)
|
||||
}
|
||||
|
||||
// Deleting the statefulset should clear the blocking pet
|
||||
if err := psc.psStore.Store.Delete(ps); err != nil {
|
||||
t.Fatalf("Unable to delete pod %v from statefulset controller store.", ps.Name)
|
||||
}
|
||||
if err := psc.Sync(fmt.Sprintf("%v/%v", ps.Namespace, ps.Name)); err != nil {
|
||||
t.Errorf("Error during sync of deleted statefulset %v", err)
|
||||
}
|
||||
fc.pets = []*pcb{}
|
||||
fc.petsCreated = 0
|
||||
if blocking, err := psc.blockingPetStore.Get(ps, fc.getPodList()); err != nil || blocking != nil {
|
||||
t.Errorf("Unexpected blocking pod %v, err %v", blocking, err)
|
||||
}
|
||||
saturateStatefulSet(t, ps, psc, fc)
|
||||
|
||||
// Make sure we don't leak the final blockin pet in the store
|
||||
psc.syncStatefulSet(ps, fc.getPodList())
|
||||
if p, exists, err := psc.blockingPetStore.store.GetByKey(fmt.Sprintf("%v/%v", ps.Namespace, ps.Name)); err != nil || exists {
|
||||
t.Errorf("Unexpected blocking pod, err %v: %+v", err, p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncStatefulSetBlockedPet(t *testing.T) {
|
||||
psc, fc := newFakeStatefulSetController()
|
||||
ps := newStatefulSet(3)
|
||||
i, _ := psc.syncStatefulSet(ps, fc.getPodList())
|
||||
if i != len(fc.getPodList()) {
|
||||
t.Errorf("syncStatefulSet should return actual amount of pods")
|
||||
}
|
||||
}
|
||||
|
||||
type fakeClient struct {
|
||||
fakeinternal.Clientset
|
||||
statefulsetClient *fakeStatefulSetClient
|
||||
}
|
||||
|
||||
func (c *fakeClient) Apps() v1beta1.AppsV1beta1Interface {
|
||||
return &fakeApps{c, &fake.FakeAppsV1beta1{}}
|
||||
}
|
||||
|
||||
type fakeApps struct {
|
||||
*fakeClient
|
||||
*fake.FakeAppsV1beta1
|
||||
}
|
||||
|
||||
func (c *fakeApps) StatefulSets(namespace string) v1beta1.StatefulSetInterface {
|
||||
c.statefulsetClient.Namespace = namespace
|
||||
return c.statefulsetClient
|
||||
}
|
||||
|
||||
type fakeStatefulSetClient struct {
|
||||
*fake.FakeStatefulSets
|
||||
Namespace string
|
||||
replicas int32
|
||||
}
|
||||
|
||||
func (f *fakeStatefulSetClient) UpdateStatus(statefulset *apps.StatefulSet) (*apps.StatefulSet, error) {
|
||||
f.replicas = statefulset.Status.Replicas
|
||||
return statefulset, nil
|
||||
}
|
||||
|
||||
func TestStatefulSetReplicaCount(t *testing.T) {
|
||||
fpsc := &fakeStatefulSetClient{}
|
||||
psc, _ := newFakeStatefulSetController()
|
||||
psc.kubeClient = &fakeClient{
|
||||
statefulsetClient: fpsc,
|
||||
}
|
||||
|
||||
ps := newStatefulSet(3)
|
||||
psKey := fmt.Sprintf("%v/%v", ps.Namespace, ps.Name)
|
||||
psc.psStore.Store.Add(ps)
|
||||
|
||||
if err := psc.Sync(psKey); err != nil {
|
||||
t.Errorf("Error during sync of deleted statefulset %v", err)
|
||||
}
|
||||
|
||||
if fpsc.replicas != 1 {
|
||||
t.Errorf("Replicas count sent as status update for StatefulSet should be 1, is %d instead", fpsc.replicas)
|
||||
}
|
||||
return assertInvariants(set, spc)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user