From f3a992aa08cebb8d430afc4fc4d714eeb2c6668e Mon Sep 17 00:00:00 2001
From: Zach Loafman <zml@google.com>
Date: Sun, 7 Dec 2014 09:19:17 -0800
Subject: [PATCH] Run e2e tests in deterministic random order

Currently, we run the e2e tests in whatever order readdir happens to
return, which is random on some filesystems, name sorted on others,
create order on others, etc. Eventually, we may want to be
automatically hermetic between e2e tests (especially as we introduce
more resource destructive tests), but until then, it would be useful
if we permute the test order randomly between runs to ensure that
developers don't accidentally rely on a particular order. This
introduces a form of forced hermeticism, since improper state cleanup
from one test may not perturb a given test, but there's probably *a*
test in the suite that the order will perturb, so the RNG will find
that order eventually.

Adds logging of the generated seed, and an --orderseed argument that
can be used to re-run in the same order. Also sorts the pass/fail list
now for easier human reading.
---
 hack/e2e.go | 33 ++++++++++++++++++++++++++++++++-
 1 file changed, 32 insertions(+), 1 deletion(-)

diff --git a/hack/e2e.go b/hack/e2e.go
index 30ca9e9cc8a..5042580565e 100644
--- a/hack/e2e.go
+++ b/hack/e2e.go
@@ -22,12 +22,15 @@ import (
 	"flag"
 	"fmt"
 	"log"
+	"math/rand"
 	"os"
 	"os/exec"
 	"os/signal"
 	"path"
 	"path/filepath"
+	"sort"
 	"strings"
+	"time"
 )
 
 var (
@@ -36,6 +39,7 @@ var (
 	up               = flag.Bool("up", false, "If true, start the the e2e cluster. If cluster is already up, recreate it.")
 	push             = flag.Bool("push", false, "If true, push to e2e cluster. Has no effect if -up is true.")
 	down             = flag.Bool("down", false, "If true, tear down the cluster before exiting.")
+	orderseed        = flag.Int64("orderseed", 0, "If non-zero, seed of random test shuffle order. (Otherwise random.)")
 	test             = flag.Bool("test", false, "Run all tests in hack/e2e-suite.")
 	tests            = flag.String("tests", "", "Run only tests in hack/e2e-suite matching this glob. Ignored if -test is set.")
 	root             = flag.String("root", absOrDie(filepath.Clean(filepath.Join(path.Base(os.Args[0]), ".."))), "Root directory of kubernetes repository.")
@@ -134,6 +138,14 @@ func tryUp() bool {
 	return runBash("up", path.Join(*root, "/cluster/kube-up.sh; test-setup;"))
 }
 
+// Fisher-Yates shuffle using the given RNG r
+func shuffleStrings(strings []string, r *rand.Rand) {
+	for i := len(strings) - 1; i > 0; i-- {
+		j := r.Intn(i + 1)
+		strings[i], strings[j] = strings[j], strings[i]
+	}
+}
+
 func Test() (failed, passed []string) {
 	defer runBashUntil("watchEvents", "$KUBECTL --watch-only get events")()
 	// run tests!
@@ -147,6 +159,7 @@ func Test() (failed, passed []string) {
 		log.Fatal("Couldn't read names in e2e-suite dir")
 	}
 
+	toRun := make([]string, 0, len(names))
 	for i := range names {
 		name := names[i]
 		if name == "." || name == ".." {
@@ -155,8 +168,24 @@ func Test() (failed, passed []string) {
 		if match, err := path.Match(*tests, name); !match && err == nil {
 			continue
 		}
+		if err != nil {
+			log.Fatal("Bad test pattern: %v", tests)
+		}
+		toRun = append(toRun, name)
+	}
+
+	if *orderseed == 0 {
+		// Use low order bits of NanoTime as the default seed. (Using
+		// all the bits makes for a long, very similar looking seed
+		// between runs.)
+		*orderseed = time.Now().UnixNano() & (1<<32 - 1)
+	}
+	sort.Strings(toRun)
+	shuffleStrings(toRun, rand.New(rand.NewSource(*orderseed)))
+	log.Printf("Running tests matching %v shuffled with seed %#x: %v", *tests, *orderseed, toRun)
+	for _, name := range toRun {
 		absName := filepath.Join(*root, "hack", "e2e-suite", name)
-		log.Printf("%v matches %v. Starting test.", name, *tests)
+		log.Printf("Starting test %v.", name)
 		if runBash(name, absName) {
 			log.Printf("%v passed", name)
 			passed = append(passed, name)
@@ -166,6 +195,8 @@ func Test() (failed, passed []string) {
 		}
 	}
 
+	sort.Strings(passed)
+	sort.Strings(failed)
 	return
 }