diff --git a/cmd/e2e/e2e.go b/cmd/e2e/e2e.go index 258718b0eb6..becbf9abd90 100644 --- a/cmd/e2e/e2e.go +++ b/cmd/e2e/e2e.go @@ -32,6 +32,8 @@ var ( host = flag.String("host", "", "The host to connect to") repoRoot = flag.String("repo_root", "./", "Root directory of kubernetes repository, for finding test files. Default assumes working directory is repository root") provider = flag.String("provider", "", "The name of the Kubernetes provider") + orderseed = flag.Int64("orderseed", 0, "If non-zero, seed of random test shuffle order. (Otherwise random.)") + times = flag.Int("times", 1, "Number of times each test is eligible to be run. Individual order is determined by shuffling --times instances of each test using --orderseed (like a multi-deck shoe of cards).") testList util.StringList ) @@ -46,5 +48,9 @@ func main() { glog.Error("e2e needs the have the --provider flag set") os.Exit(1) } - e2e.RunE2ETests(*authConfig, *certDir, *host, *repoRoot, *provider, testList) + if *times <= 0 { + glog.Error("Invalid --times (negative or no testing requested)!") + os.Exit(1) + } + e2e.RunE2ETests(*authConfig, *certDir, *host, *repoRoot, *provider, *orderseed, *times, testList) } diff --git a/test/e2e/driver.go b/test/e2e/driver.go index 0d5cae0429b..39cbc396684 100644 --- a/test/e2e/driver.go +++ b/test/e2e/driver.go @@ -17,6 +17,7 @@ limitations under the License. package e2e import ( + "math/rand" "time" "github.com/GoogleCloudPlatform/kubernetes/pkg/client" @@ -29,8 +30,6 @@ type testSpec struct { test func(c *client.Client) bool // The human readable name of this test name string - // The id for this test. It should be constant for the life of the test. - id int } type testInfo struct { @@ -42,18 +41,26 @@ type testInfo struct { // See http://testanything.org/ for more info func outputTAPSummary(infoList []testInfo) { glog.Infof("1..%d", len(infoList)) - for _, info := range infoList { + for i, info := range infoList { if info.passed { - glog.Infof("ok %d - %s", info.spec.id, info.spec.name) + glog.Infof("ok %d - %s", i+1, info.spec.name) } else { - glog.Infof("not ok %d - %s", info.spec.id, info.spec.name) + glog.Infof("not ok %d - %s", i+1, info.spec.name) } } } +// Fisher-Yates shuffle using the given RNG r +func shuffleTests(tests []testSpec, r *rand.Rand) { + for i := len(tests) - 1; i > 0; i-- { + j := r.Intn(i + 1) + tests[i], tests[j] = tests[j], tests[i] + } +} + // Run each Go end-to-end-test. This function assumes the // creation of a test cluster. -func RunE2ETests(authConfig, certDir, host, repoRoot, provider string, testList []string) { +func RunE2ETests(authConfig, certDir, host, repoRoot, provider string, orderseed int64, times int, testList []string) { testContext = testContextType{authConfig, certDir, host, repoRoot, provider} util.ReallyCrash = true util.InitLogs() @@ -69,23 +76,22 @@ func RunE2ETests(authConfig, certDir, host, repoRoot, provider string, testList c := loadClientOrDie() tests := []testSpec{ - {TestKubernetesROService, "TestKubernetesROService", 1}, - {TestKubeletSendsEvent, "TestKubeletSendsEvent", 2}, - {TestImportantURLs, "TestImportantURLs", 3}, - {TestPodUpdate, "TestPodUpdate", 4}, - {TestNetwork, "TestNetwork", 5}, - {TestClusterDNS, "TestClusterDNS", 6}, - {TestPodHasServiceEnvVars, "TestPodHasServiceEnvVars", 7}, - {TestBasic, "TestBasic", 8}, - {TestPrivate, "TestPrivate", 9}, + {TestKubernetesROService, "TestKubernetesROService"}, + {TestKubeletSendsEvent, "TestKubeletSendsEvent"}, + {TestImportantURLs, "TestImportantURLs"}, + {TestPodUpdate, "TestPodUpdate"}, + {TestNetwork, "TestNetwork"}, + {TestClusterDNS, "TestClusterDNS"}, + {TestPodHasServiceEnvVars, "TestPodHasServiceEnvVars"}, + {TestBasic, "TestBasic"}, + {TestPrivate, "TestPrivate"}, } + // Check testList for non-existent tests and populate a StringSet with tests to run. validTestNames := util.NewStringSet() for _, test := range tests { validTestNames.Insert(test.name) } - - // Check testList for non-existent tests and populate a StringSet with tests to run. runTestNames := util.NewStringSet() for _, testName := range testList { if validTestNames.Has(testName) { @@ -95,15 +101,39 @@ func RunE2ETests(authConfig, certDir, host, repoRoot, provider string, testList } } + // if testList was specified, filter down now before we expand and shuffle + if len(testList) > 0 { + newTests := make([]testSpec, 0) + for i, test := range tests { + // Check if this test is supposed to run, either if listed explicitly in + // a --test flag or if no --test flags were supplied. + if !runTestNames.Has(test.name) { + glog.Infof("Skipping test %d %s", i+1, test.name) + continue + } + newTests = append(newTests, test) + } + tests = newTests + } + if times != 1 { + newTests := make([]testSpec, 0, times*len(tests)) + for i := 0; i < times; i++ { + newTests = append(newTests, tests...) + } + tests = newTests + } + if orderseed == 0 { + // Use low order bits of NanoTime as the default seed. (Using + // all the bits makes for a long, very similar looking seed + // between runs.) + orderseed = time.Now().UnixNano() & (1<<32 - 1) + } + shuffleTests(tests, rand.New(rand.NewSource(orderseed))) + glog.Infof("Tests shuffled with orderseed %#x\n", orderseed) + info := []testInfo{} passed := true for i, test := range tests { - // Check if this test is supposed to run, either if listed explicitly in - // a --test flag or if no --test flags were supplied. - if len(testList) > 0 && !runTestNames.Has(test.name) { - glog.Infof("Skipping test %d %s", i+1, test.name) - continue - } glog.Infof("Running test %d %s", i+1, test.name) testPassed := test.test(c) if !testPassed {