From faf47b6f0ba0f6d124da2453933316f580822bbf Mon Sep 17 00:00:00 2001
From: Yu-Ju Hong <yjhong@google.com>
Date: Thu, 9 Apr 2015 15:05:30 -0700
Subject: [PATCH] Limit the number of concurrent tests in integration.go

Integration test often time out because the machine is loaded. Instead of
increasing timeout, this change hopes to address the issue by limiting the
number of tests running simultaneously.

Add a new flag in integration.go to specify the maximum number of concurrent
tests. Set the default in travis and shippable configurations to be 4.
---
 .travis.yml                    |  2 +-
 cmd/integration/integration.go | 32 +++++++++++++++++++++++---------
 hack/test-integration.sh       |  5 +++--
 shippable.yml                  |  2 +-
 4 files changed, 28 insertions(+), 13 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 7afb89a2ce6..c317e959725 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -24,7 +24,7 @@ install:
 script:
   - KUBE_RACE="-race" KUBE_COVER="y" KUBE_GOVERALLS_BIN="$HOME/gopath/bin/goveralls" KUBE_TIMEOUT='-timeout 300s' KUBE_COVERPROCS=8 KUBE_TEST_API_VERSIONS=$KUBE_TEST_API_VERSIONS ./hack/test-go.sh -- -p=2
   - PATH=$HOME/gopath/bin:./third_party/etcd:$PATH ./hack/test-cmd.sh
-  - PATH=$HOME/gopath/bin:./third_party/etcd:$PATH KUBE_TEST_API_VERSIONS=$KUBE_TEST_API_VERSIONS ./hack/test-integration.sh
+  - PATH=$HOME/gopath/bin:./third_party/etcd:$PATH KUBE_TEST_API_VERSIONS=$KUBE_TEST_API_VERSIONS KUBE_INTEGRATION_TEST_MAX_CONCURRENCY=4 ./hack/test-integration.sh
 
 notifications:
   irc: "chat.freenode.net#google-containers"
diff --git a/cmd/integration/integration.go b/cmd/integration/integration.go
index cb34bd8ecda..7d73ef79479 100644
--- a/cmd/integration/integration.go
+++ b/cmd/integration/integration.go
@@ -67,6 +67,8 @@ var (
 	fakeDocker1, fakeDocker2 dockertools.FakeDockerClient
 	// API version that should be used by the client to talk to the server.
 	apiVersion string
+	// Limit the number of concurrent tests.
+	maxConcurrency int
 )
 
 type fakeKubeletClient struct{}
@@ -947,6 +949,8 @@ type testFunc func(*client.Client)
 
 func addFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&apiVersion, "apiVersion", latest.Version, "API version that should be used by the client for communicating with the server")
+	fs.IntVar(
+		&maxConcurrency, "maxConcurrency", -1, "Maximum number of tests to be run simultaneously. Unlimited if set to negative.")
 }
 
 func main() {
@@ -994,16 +998,26 @@ func main() {
 		},
 	}
 
-	var wg sync.WaitGroup
-	wg.Add(len(testFuncs))
-	for i := range testFuncs {
-		f := testFuncs[i]
-		go func() {
-			f(kubeClient)
-			wg.Done()
-		}()
+	// Only run at most maxConcurrency tests in parallel.
+	numFinishedTests := 0
+	for numFinishedTests < len(testFuncs) {
+		numTestsToRun := len(testFuncs) - numFinishedTests
+		if maxConcurrency > 0 && numTestsToRun > maxConcurrency {
+			numTestsToRun = maxConcurrency
+		}
+		glog.Infof("Running %d tests in parallel.", numTestsToRun)
+		var wg sync.WaitGroup
+		wg.Add(numTestsToRun)
+		for i := 0; i < numTestsToRun; i++ {
+			f := testFuncs[i+numFinishedTests]
+			go func() {
+				f(kubeClient)
+				wg.Done()
+			}()
+		}
+		wg.Wait()
+		numFinishedTests += numTestsToRun
 	}
-	wg.Wait()
 
 	// Check that kubelet tried to make the containers.
 	// Using a set to list unique creation attempts. Our fake is
diff --git a/hack/test-integration.sh b/hack/test-integration.sh
index 7799673b17a..976d4927a54 100755
--- a/hack/test-integration.sh
+++ b/hack/test-integration.sh
@@ -26,7 +26,7 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 # Comma separated list of API Versions that should be tested.
 KUBE_TEST_API_VERSIONS=${KUBE_TEST_API_VERSIONS:-"v1beta1,v1beta3"}
-
+KUBE_INTEGRATION_TEST_MAX_CONCURRENCY=${KUBE_INTEGRATION_TEST_MAX_CONCURRENCY:-"-1"}
 
 cleanup() {
   kube::etcd::cleanup
@@ -44,7 +44,8 @@ runTests() {
 
   kube::log::status "Running integration test scenario"
 
-  "${KUBE_OUTPUT_HOSTBIN}/integration" --v=2 --apiVersion="$1"
+  "${KUBE_OUTPUT_HOSTBIN}/integration" --v=2 --apiVersion="$1" \
+  --maxConcurrency="${KUBE_INTEGRATION_TEST_MAX_CONCURRENCY}"
 
   cleanup
 }
diff --git a/shippable.yml b/shippable.yml
index 79d2d45faa3..555d49baac5 100644
--- a/shippable.yml
+++ b/shippable.yml
@@ -33,7 +33,7 @@ install:
 script:
   - KUBE_RACE="-race" KUBE_COVER="y" KUBE_GOVERALLS_BIN="$HOME/gopath/bin/goveralls" KUBE_TIMEOUT='-timeout 300s' KUBE_COVERPROCS=8 KUBE_TEST_API_VERSIONS=$KUBE_TEST_API_VERSIONS ./hack/test-go.sh -- -p=2
   - PATH=$HOME/gopath/bin:./third_party/etcd:$PATH ./hack/test-cmd.sh
-  - PATH=$HOME/gopath/bin:./third_party/etcd:$PATH KUBE_TEST_API_VERSIONS=$KUBE_TEST_API_VERSIONS ./hack/test-integration.sh
+  - PATH=$HOME/gopath/bin:./third_party/etcd:$PATH KUBE_TEST_API_VERSIONS=$KUBE_TEST_API_VERSIONS KUBE_INTEGRATION_TEST_MAX_CONCURRENCY=4 ./hack/test-integration.sh
 
 notifications:
   irc: "chat.freenode.net#google-containers"