github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-21 02:41:25 +00:00
Code Issues Projects Releases Wiki Activity

Fix statefulset test flakiness

Browse Source
This commit is contained in:
wojtekt 2020-05-08 16:19:37 +02:00
parent af67408c17
commit 238dde4bdb
1 changed files with 49 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
test/e2e/apps/statefulset.go
View File

@ -20,6 +20,7 @@ import (
"context" "context"
"fmt" "fmt"
"strings" "strings"
"sync"
"time" "time"
"github.com/onsi/ginkgo" "github.com/onsi/ginkgo"
@ -588,6 +589,29 @@ var _ = SIGDescribe("StatefulSet", func() {
}) })
framework.ExpectNoError(err) framework.ExpectNoError(err)
// Verify that statuful set will be scaled up in order.
wg := sync.WaitGroup{}
var orderErr error
wg.Add(1)
go func() {
defer wg.Done()
expectedOrder := []string{ssName + "-0", ssName + "-1", ssName + "-2"}
ctx, cancel := watchtools.ContextWithOptionalTimeout(context.Background(), statefulSetTimeout)
defer cancel()
_, orderErr = watchtools.Until(ctx, pl.ResourceVersion, w, func(event watch.Event) (bool, error) {
if event.Type != watch.Added {
return false, nil
}
pod := event.Object.(*v1.Pod)
if pod.Name == expectedOrder[0] {
expectedOrder = expectedOrder[1:]
}
return len(expectedOrder) == 0, nil
})
}()
ginkgo.By("Creating stateful set " + ssName + " in namespace " + ns) ginkgo.By("Creating stateful set " + ssName + " in namespace " + ns)
ss := e2estatefulset.NewStatefulSet(ssName, ns, headlessSvcName, 1, nil, nil, psLabels) ss := e2estatefulset.NewStatefulSet(ssName, ns, headlessSvcName, 1, nil, nil, psLabels)
setHTTPProbe(ss) setHTTPProbe(ss)
@ -609,11 +633,26 @@ var _ = SIGDescribe("StatefulSet", func() {
e2estatefulset.WaitForRunningAndReady(c, 3, ss) e2estatefulset.WaitForRunningAndReady(c, 3, ss)
ginkgo.By("Verifying that stateful set " + ssName + " was scaled up in order") ginkgo.By("Verifying that stateful set " + ssName + " was scaled up in order")
expectedOrder := []string{ssName + "-0", ssName + "-1", ssName + "-2"} wg.Wait()
framework.ExpectNoError(orderErr)
ginkgo.By("Scale down will halt with unhealthy stateful pod")
pl, err = f.ClientSet.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{
LabelSelector: psLabels.AsSelector().String(),
})
framework.ExpectNoError(err)
// Verify that statuful set will be scaled down in order.
wg.Add(1)
go func() {
defer wg.Done()
expectedOrder := []string{ssName + "-2", ssName + "-1", ssName + "-0"}
ctx, cancel := watchtools.ContextWithOptionalTimeout(context.Background(), statefulSetTimeout) ctx, cancel := watchtools.ContextWithOptionalTimeout(context.Background(), statefulSetTimeout)
defer cancel() defer cancel()
_, err = watchtools.Until(ctx, pl.ResourceVersion, w, func(event watch.Event) (bool, error) {
if event.Type != watch.Added { _, orderErr = watchtools.Until(ctx, pl.ResourceVersion, w, func(event watch.Event) (bool, error) {
if event.Type != watch.Deleted {
return false, nil return false, nil
} }
pod := event.Object.(*v1.Pod) pod := event.Object.(*v1.Pod)
@ -621,15 +660,8 @@ var _ = SIGDescribe("StatefulSet", func() {
expectedOrder = expectedOrder[1:] expectedOrder = expectedOrder[1:]
} }
return len(expectedOrder) == 0, nil return len(expectedOrder) == 0, nil
}) })
framework.ExpectNoError(err) }()
ginkgo.By("Scale down will halt with unhealthy stateful pod")
pl, err = f.ClientSet.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{
LabelSelector: psLabels.AsSelector().String(),
})
framework.ExpectNoError(err)
breakHTTPProbe(c, ss) breakHTTPProbe(c, ss)
e2estatefulset.WaitForStatusReadyReplicas(c, ss, 0) e2estatefulset.WaitForStatusReadyReplicas(c, ss, 0)
@ -642,21 +674,8 @@ var _ = SIGDescribe("StatefulSet", func() {
e2estatefulset.Scale(c, ss, 0) e2estatefulset.Scale(c, ss, 0)
ginkgo.By("Verifying that stateful set " + ssName + " was scaled down in reverse order") ginkgo.By("Verifying that stateful set " + ssName + " was scaled down in reverse order")
expectedOrder = []string{ssName + "-2", ssName + "-1", ssName + "-0"} wg.Wait()
ctx, cancel = watchtools.ContextWithOptionalTimeout(context.Background(), statefulSetTimeout) framework.ExpectNoError(orderErr)
defer cancel()
_, err = watchtools.Until(ctx, pl.ResourceVersion, w, func(event watch.Event) (bool, error) {
if event.Type != watch.Deleted {
return false, nil
}
pod := event.Object.(*v1.Pod)
if pod.Name == expectedOrder[0] {
expectedOrder = expectedOrder[1:]
}
return len(expectedOrder) == 0, nil
})
framework.ExpectNoError(err)
}) })
/* /*