github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-24 04:06:03 +00:00
Code Issues Projects Releases Wiki Activity

Run e2e tests in deterministic random order

Browse Source 
Currently, we run the e2e tests in whatever order readdir happens to
return, which is random on some filesystems, name sorted on others,
create order on others, etc. Eventually, we may want to be
automatically hermetic between e2e tests (especially as we introduce
more resource destructive tests), but until then, it would be useful
if we permute the test order randomly between runs to ensure that
developers don't accidentally rely on a particular order. This
introduces a form of forced hermeticism, since improper state cleanup
from one test may not perturb a given test, but there's probably *a*
test in the suite that the order will perturb, so the RNG will find
that order eventually.

Adds logging of the generated seed, and an --orderseed argument that
can be used to re-run in the same order. Also sorts the pass/fail list
now for easier human reading.
This commit is contained in:
Zach Loafman 2014-12-07 09:19:17 -08:00
parent 3910b2d6e1
commit f3a992aa08
1 changed files with 32 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
hack/e2e.go
View File

@ -22,12 +22,15 @@ import (
"flag"
"fmt"
"log"
"math/rand"
"os"
"os/exec"
"os/signal"
"path"
"path/filepath"
"sort"
"strings"
"time"
)
var (
@ -36,6 +39,7 @@ var (
up = flag.Bool("up", false, "If true, start the the e2e cluster. If cluster is already up, recreate it.")
push = flag.Bool("push", false, "If true, push to e2e cluster. Has no effect if -up is true.")
down = flag.Bool("down", false, "If true, tear down the cluster before exiting.")
orderseed = flag.Int64("orderseed", 0, "If non-zero, seed of random test shuffle order. (Otherwise random.)")
test = flag.Bool("test", false, "Run all tests in hack/e2e-suite.")
tests = flag.String("tests", "", "Run only tests in hack/e2e-suite matching this glob. Ignored if -test is set.")
root = flag.String("root", absOrDie(filepath.Clean(filepath.Join(path.Base(os.Args[0]), ".."))), "Root directory of kubernetes repository.")
@ -134,6 +138,14 @@ func tryUp() bool {
return runBash("up", path.Join(*root, "/cluster/kube-up.sh; test-setup;"))
}
// Fisher-Yates shuffle using the given RNG r
func shuffleStrings(strings []string, r *rand.Rand) {
for i := len(strings) - 1; i > 0; i-- {
j := r.Intn(i + 1)
strings[i], strings[j] = strings[j], strings[i]
}
}
func Test() (failed, passed []string) {
defer runBashUntil("watchEvents", "$KUBECTL --watch-only get events")()
// run tests!
@ -147,6 +159,7 @@ func Test() (failed, passed []string) {
log.Fatal("Couldn't read names in e2e-suite dir")
}
toRun := make([]string, 0, len(names))
for i := range names {
name := names[i]
if name == "." || name == ".." {
@ -155,8 +168,24 @@ func Test() (failed, passed []string) {
if match, err := path.Match(*tests, name); !match && err == nil {
continue
}
if err != nil {
log.Fatal("Bad test pattern: %v", tests)
}
toRun = append(toRun, name)
}
if *orderseed == 0 {
// Use low order bits of NanoTime as the default seed. (Using
// all the bits makes for a long, very similar looking seed
// between runs.)
*orderseed = time.Now().UnixNano() & (1<<32 - 1)
}
sort.Strings(toRun)
shuffleStrings(toRun, rand.New(rand.NewSource(*orderseed)))
log.Printf("Running tests matching %v shuffled with seed %#x: %v", *tests, *orderseed, toRun)
for _, name := range toRun {
absName := filepath.Join(*root, "hack", "e2e-suite", name)
log.Printf("%v matches %v. Starting test.", name, *tests)
log.Printf("Starting test %v.", name)
if runBash(name, absName) {
log.Printf("%v passed", name)
passed = append(passed, name)
@ -166,6 +195,8 @@ func Test() (failed, passed []string) {
}
}
sort.Strings(passed)
sort.Strings(failed)
return
}