Merge pull request #27416 from wojtek-t/longer_system_pods_timeout

Automatic merge from submit-queue

Make timeout for starting system pods configurable

Context: in 2000-node clusters (if only one node is big enough to fit heapster, which is our testing configuration), heapster won't be scheduled until that node has route. However, creating routes is pretty expensive and currently can take even 2 hours.

@zmerlynn @gmarek

hack/verify-flags/known-flags.txt

@@ -446,6 +446,7 @@ suicide-timeout
 sync-frequency
 system-cgroups
 system-container
+system-pods-startup-timeout
 system-reserved
 target-port
 tcp-services

test/e2e/e2e.go

@@ -40,11 +40,6 @@ import (
 )
 
 const (
-	// podStartupTimeout is the time to allow all pods in the cluster to become
-	// running and ready before any e2e tests run. It includes pulling all of
-	// the pods (as of 5/18/15 this is 8 pods).
-	podStartupTimeout = 10 * time.Minute
-
 	// imagePrePullingTimeout is the time we wait for the e2e-image-puller
 	// static pods to pull the list of seeded images. If they don't pull
 	// images within this time we simply log their output and carry on
@@ -129,6 +124,7 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
 	// cluster infrastructure pods that are being pulled or started can block
 	// test pods from running, and tests that ensure all pods are running and
 	// ready will fail).
+	podStartupTimeout := framework.TestContext.SystemPodsStartupTimeout
 	if err := framework.WaitForPodsRunningReady(c, api.NamespaceSystem, int32(framework.TestContext.MinStartupPods), podStartupTimeout, framework.ImagePullerLabels); err != nil {
 		framework.DumpAllNamespaceInfo(c, api.NamespaceSystem)
 		framework.LogFailedContainers(c, api.NamespaceSystem)
@@ -141,7 +137,7 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
 		// and we don't even run the image puller on all platforms (including GKE).
 		// We wait for it so we get an indication of failures in the logs, and to
 		// maximize benefit of image pre-pulling.
-		framework.Logf("WARNING: Image pulling pods failed to enter success in %v: %v", podStartupTimeout, err)
+		framework.Logf("WARNING: Image pulling pods failed to enter success in %v: %v", imagePrePullingTimeout, err)
 	}
 
 	return nil

test/e2e/framework/test_context.go

@@ -19,6 +19,7 @@ package framework
 import (
 	"flag"
 	"os"
+	"time"
 
 	"github.com/onsi/ginkgo/config"
 	"k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
@@ -41,6 +42,8 @@ type TestContextType struct {
 	ReportPrefix       string
 	Prefix             string
 	MinStartupPods     int
+	// Timeout for waiting for system pods to be running
+	SystemPodsStartupTimeout time.Duration
 	UpgradeTarget            string
 	PrometheusPushGateway    string
 	OSDistro                 string
@@ -118,6 +121,7 @@ func RegisterFlags() {
 
 	flag.StringVar(&cloudConfig.ClusterTag, "cluster-tag", "", "Tag used to identify resources.  Only required if provider is aws.")
 	flag.IntVar(&TestContext.MinStartupPods, "minStartupPods", 0, "The number of pods which we need to see in 'Running' state with a 'Ready' condition of true, before we try running tests. This is useful in any cluster which needs some base pod-based services running before it can be used.")
+	flag.DurationVar(&TestContext.SystemPodsStartupTimeout, "system-pods-startup-timeout", 10*time.Minute, "Timeout for waiting for all system pods to be running before starting tests.")
 	flag.StringVar(&TestContext.UpgradeTarget, "upgrade-target", "ci/latest", "Version to upgrade to (e.g. 'release/stable', 'release/latest', 'ci/latest', '0.19.1', '0.19.1-669-gabac8c8') if doing an upgrade test.")
 	flag.StringVar(&TestContext.PrometheusPushGateway, "prom-push-gateway", "", "The URL to prometheus gateway, so that metrics can be pushed during e2es and scraped by prometheus. Typically something like 127.0.0.1:9091.")
 	flag.BoolVar(&TestContext.VerifyServiceAccount, "e2e-verify-service-account", true, "If true tests will verify the service account before running.")