Merge pull request #27416 from wojtek-t/longer_system_pods_timeout

Automatic merge from submit-queue

Make timeout for starting system pods configurable

Context: in 2000-node clusters (if only one node is big enough to fit heapster, which is our testing configuration), heapster won't be scheduled until that node has route. However, creating routes is pretty expensive and currently can take even 2 hours.

@zmerlynn @gmarek

hack/verify-flags/known-flags.txt

@@ -446,6 +446,7 @@ suicide-timeout
 sync-frequency
 system-cgroups
 system-container
+system-pods-startup-timeout
 system-reserved
 target-port
 tcp-services

test/e2e/e2e.go

@@ -40,11 +40,6 @@ import (
 )
 
 const (
-	// podStartupTimeout is the time to allow all pods in the cluster to become
-	// running and ready before any e2e tests run. It includes pulling all of
-	// the pods (as of 5/18/15 this is 8 pods).
-	podStartupTimeout = 10 * time.Minute
-
 	// imagePrePullingTimeout is the time we wait for the e2e-image-puller
 	// static pods to pull the list of seeded images. If they don't pull
 	// images within this time we simply log their output and carry on
@@ -129,6 +124,7 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
 	// cluster infrastructure pods that are being pulled or started can block
 	// test pods from running, and tests that ensure all pods are running and
 	// ready will fail).
+	podStartupTimeout := framework.TestContext.SystemPodsStartupTimeout
 	if err := framework.WaitForPodsRunningReady(c, api.NamespaceSystem, int32(framework.TestContext.MinStartupPods), podStartupTimeout, framework.ImagePullerLabels); err != nil {
 		framework.DumpAllNamespaceInfo(c, api.NamespaceSystem)
 		framework.LogFailedContainers(c, api.NamespaceSystem)
@@ -141,7 +137,7 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
 		// and we don't even run the image puller on all platforms (including GKE).
 		// We wait for it so we get an indication of failures in the logs, and to
 		// maximize benefit of image pre-pulling.
-		framework.Logf("WARNING: Image pulling pods failed to enter success in %v: %v", podStartupTimeout, err)
+		framework.Logf("WARNING: Image pulling pods failed to enter success in %v: %v", imagePrePullingTimeout, err)
 	}
 
 	return nil

test/e2e/framework/test_context.go

@@ -19,6 +19,7 @@ package framework
 import (
 	"flag"
 	"os"
+	"time"
 
 	"github.com/onsi/ginkgo/config"
 	"k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
@@ -26,27 +27,29 @@ import (
 )
 
 type TestContextType struct {
-	KubeConfig            string
-	KubeContext           string
-	KubeAPIContentType    string
-	KubeVolumeDir         string
-	CertDir               string
-	Host                  string
-	RepoRoot              string
-	Provider              string
-	CloudConfig           CloudConfig
-	KubectlPath           string
-	OutputDir             string
-	ReportDir             string
-	ReportPrefix          string
-	Prefix                string
-	MinStartupPods        int
-	UpgradeTarget         string
-	PrometheusPushGateway string
-	OSDistro              string
-	VerifyServiceAccount  bool
-	DeleteNamespace       bool
-	CleanStart            bool
+	KubeConfig         string
+	KubeContext        string
+	KubeAPIContentType string
+	KubeVolumeDir      string
+	CertDir            string
+	Host               string
+	RepoRoot           string
+	Provider           string
+	CloudConfig        CloudConfig
+	KubectlPath        string
+	OutputDir          string
+	ReportDir          string
+	ReportPrefix       string
+	Prefix             string
+	MinStartupPods     int
+	// Timeout for waiting for system pods to be running
+	SystemPodsStartupTimeout time.Duration
+	UpgradeTarget            string
+	PrometheusPushGateway    string
+	OSDistro                 string
+	VerifyServiceAccount     bool
+	DeleteNamespace          bool
+	CleanStart               bool
 	// If set to 'true' or 'all' framework will start a goroutine monitoring resource usage of system add-ons.
 	// It will read the data every 30 seconds from all Nodes and print summary during afterEach. If set to 'master'
 	// only master Node will be monitored.
@@ -118,6 +121,7 @@ func RegisterFlags() {
 
 	flag.StringVar(&cloudConfig.ClusterTag, "cluster-tag", "", "Tag used to identify resources.  Only required if provider is aws.")
 	flag.IntVar(&TestContext.MinStartupPods, "minStartupPods", 0, "The number of pods which we need to see in 'Running' state with a 'Ready' condition of true, before we try running tests. This is useful in any cluster which needs some base pod-based services running before it can be used.")
+	flag.DurationVar(&TestContext.SystemPodsStartupTimeout, "system-pods-startup-timeout", 10*time.Minute, "Timeout for waiting for all system pods to be running before starting tests.")
 	flag.StringVar(&TestContext.UpgradeTarget, "upgrade-target", "ci/latest", "Version to upgrade to (e.g. 'release/stable', 'release/latest', 'ci/latest', '0.19.1', '0.19.1-669-gabac8c8') if doing an upgrade test.")
 	flag.StringVar(&TestContext.PrometheusPushGateway, "prom-push-gateway", "", "The URL to prometheus gateway, so that metrics can be pushed during e2es and scraped by prometheus. Typically something like 127.0.0.1:9091.")
 	flag.BoolVar(&TestContext.VerifyServiceAccount, "e2e-verify-service-account", true, "If true tests will verify the service account before running.")