[e2e ingress-gce] Scale test to measure ingress create/update latency

test/e2e/network/ingress_scale.go

@@ -0,0 +1,64 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package network
+
+import (
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/network/scale"
+
+	. "github.com/onsi/ginkgo"
+)
+
+var _ = SIGDescribe("Loadbalancing: L7 Scalability", func() {
+	defer GinkgoRecover()
+	var (
+		ns string
+	)
+	f := framework.NewDefaultFramework("ingress-scale")
+
+	BeforeEach(func() {
+		ns = f.Namespace.Name
+	})
+
+	Describe("GCE [Slow] [Serial] [Feature:IngressScale]", func() {
+		var (
+			scaleFramework *scale.IngressScaleFramework
+		)
+
+		BeforeEach(func() {
+			framework.SkipUnlessProviderIs("gce", "gke")
+
+			scaleFramework = scale.NewIngressScaleFramework(f.ClientSet, ns, framework.TestContext.CloudConfig)
+			if err := scaleFramework.PrepareScaleTest(); err != nil {
+				framework.Failf("Unexpected error while preraring ingress scale test: %v", err)
+			}
+		})
+
+		AfterEach(func() {
+			if errs := scaleFramework.CleanupScaleTest(); len(errs) != 0 {
+				framework.Failf("Unexpected error while cleaning up ingress scale test: %v", errs)
+			}
+		})
+
+		It("Creating and updating ingresses should happen promptly with small/medium/large amount of ingresses", func() {
+			if errs := scaleFramework.RunScaleTest(); len(errs) != 0 {
+				framework.Failf("Unexpected error while running ingress scale test: %v", errs)
+			}
+
+		})
+	})
+})

test/e2e/network/scale/BUILD

@@ -0,0 +1,33 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["ingress.go"],
+    importpath = "k8s.io/kubernetes/test/e2e/network/scale",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//test/e2e/framework:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//test/e2e/network/scale/localrun:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

test/e2e/network/scale/ingress.go

@@ -0,0 +1,461 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package scale
+
+import (
+	"fmt"
+	"io/ioutil"
+	"sync"
+	"time"
+
+	"k8s.io/api/core/v1"
+	extensions "k8s.io/api/extensions/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	clientset "k8s.io/client-go/kubernetes"
+
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+const (
+	numIngressesSmall  = 5
+	numIngressesMedium = 20
+	numIngressesLarge  = 45
+
+	scaleTestIngressNamePrefix = "ing-scale"
+	scaleTestBackendName       = "echoheaders-scale"
+	scaleTestSecretName        = "tls-secret-scale"
+	scaleTestHostname          = "scale.ingress.com"
+	scaleTestNumBackends       = 10
+	scaleTestPollInterval      = 15 * time.Second
+
+	// We don't expect waitForIngress to take longer
+	// than waitForIngressMaxTimeout.
+	waitForIngressMaxTimeout = 80 * time.Minute
+	ingressesCleanupTimeout  = 80 * time.Minute
+)
+
+var (
+	scaleTestLabels = map[string]string{
+		"app": scaleTestBackendName,
+	}
+)
+
+// IngressScaleFramework defines the framework for ingress scale testing.
+type IngressScaleFramework struct {
+	Clientset     clientset.Interface
+	Jig           *framework.IngressTestJig
+	GCEController *framework.GCEIngressController
+	CloudConfig   framework.CloudConfig
+	Logger        framework.TestLogger
+
+	Namespace        string
+	EnableTLS        bool
+	NumIngressesTest []int
+	OutputFile       string
+
+	ScaleTestDeploy *extensions.Deployment
+	ScaleTestSvcs   []*v1.Service
+	ScaleTestIngs   []*extensions.Ingress
+
+	// BatchCreateLatencies stores all ingress creation latencies, in different
+	// batches.
+	BatchCreateLatencies [][]time.Duration
+	// BatchDurations stores the total duration for each ingress batch creation.
+	BatchDurations []time.Duration
+	// StepCreateLatencies stores the single ingress creation latency, which happens
+	// after each ingress batch creation is complete.
+	StepCreateLatencies []time.Duration
+	// StepCreateLatencies stores the single ingress update latency, which happens
+	// after each ingress batch creation is complete.
+	StepUpdateLatencies []time.Duration
+}
+
+// NewIngressScaleFramework returns a new framework for ingress scale testing.
+func NewIngressScaleFramework(cs clientset.Interface, ns string, cloudConfig framework.CloudConfig) *IngressScaleFramework {
+	return &IngressScaleFramework{
+		Namespace:   ns,
+		Clientset:   cs,
+		CloudConfig: cloudConfig,
+		Logger:      &framework.E2ELogger{},
+		EnableTLS:   true,
+		NumIngressesTest: []int{
+			numIngressesSmall,
+			numIngressesMedium,
+			numIngressesLarge,
+		},
+	}
+}
+
+// PrepareScaleTest prepares framework for ingress scale testing.
+func (f *IngressScaleFramework) PrepareScaleTest() error {
+	f.Logger.Infof("Initializing ingress test suite and gce controller...")
+	f.Jig = framework.NewIngressTestJig(f.Clientset)
+	f.Jig.Logger = f.Logger
+	f.Jig.PollInterval = scaleTestPollInterval
+	f.GCEController = &framework.GCEIngressController{
+		Client: f.Clientset,
+		Cloud:  f.CloudConfig,
+	}
+	if err := f.GCEController.Init(); err != nil {
+		return fmt.Errorf("Failed to initialize GCE controller: %v", err)
+	}
+
+	f.ScaleTestSvcs = []*v1.Service{}
+	f.ScaleTestIngs = []*extensions.Ingress{}
+
+	return nil
+}
+
+// CleanupScaleTest cleans up framework for ingress scale testing.
+func (f *IngressScaleFramework) CleanupScaleTest() []error {
+	var errs []error
+
+	f.Logger.Infof("Cleaning up ingresses...")
+	for _, ing := range f.ScaleTestIngs {
+		if ing != nil {
+			if err := f.Clientset.ExtensionsV1beta1().Ingresses(ing.Namespace).Delete(ing.Name, nil); err != nil {
+				errs = append(errs, fmt.Errorf("Error while deleting ingress %s/%s: %v", ing.Namespace, ing.Name, err))
+			}
+		}
+	}
+	f.Logger.Infof("Cleaning up services...")
+	for _, svc := range f.ScaleTestSvcs {
+		if svc != nil {
+			if err := f.Clientset.CoreV1().Services(svc.Namespace).Delete(svc.Name, nil); err != nil {
+				errs = append(errs, fmt.Errorf("Error while deleting service %s/%s: %v", svc.Namespace, svc.Name, err))
+			}
+		}
+	}
+	if f.ScaleTestDeploy != nil {
+		f.Logger.Infof("Cleaning up deployment %s...", f.ScaleTestDeploy.Name)
+		if err := f.Clientset.ExtensionsV1beta1().Deployments(f.ScaleTestDeploy.Namespace).Delete(f.ScaleTestDeploy.Name, nil); err != nil {
+			errs = append(errs, fmt.Errorf("Error while delting deployment %s/%s: %v", f.ScaleTestDeploy.Namespace, f.ScaleTestDeploy.Name, err))
+		}
+	}
+
+	f.Logger.Infof("Cleaning up cloud resources...")
+	if err := f.GCEController.CleanupGCEIngressControllerWithTimeout(ingressesCleanupTimeout); err != nil {
+		errs = append(errs, err)
+	}
+
+	return errs
+}
+
+// RunScaleTest runs ingress scale testing.
+func (f *IngressScaleFramework) RunScaleTest() []error {
+	var errs []error
+
+	testDeploy := generateScaleTestBackendDeploymentSpec(scaleTestNumBackends)
+	f.Logger.Infof("Creating deployment %s...", testDeploy.Name)
+	testDeploy, err := f.Jig.Client.ExtensionsV1beta1().Deployments(f.Namespace).Create(testDeploy)
+	if err != nil {
+		errs = append(errs, fmt.Errorf("Failed to create deployment %s: %v", testDeploy.Name, err))
+		return errs
+	}
+	f.ScaleTestDeploy = testDeploy
+
+	if f.EnableTLS {
+		f.Logger.Infof("Ensuring TLS secret %s...", scaleTestSecretName)
+		if err := f.Jig.PrepareTLSSecret(f.Namespace, scaleTestSecretName, scaleTestHostname); err != nil {
+			errs = append(errs, fmt.Errorf("Failed to prepare TLS secret %s: %v", scaleTestSecretName, err))
+			return errs
+		}
+	}
+
+	// currentNum keeps track of how many ingresses have been created.
+	currentNum := new(int)
+
+	prepareIngsFunc := func(goalNum int) {
+		var ingWg sync.WaitGroup
+		numToCreate := goalNum - *currentNum
+		ingWg.Add(numToCreate)
+		errQueue := make(chan error, numToCreate)
+		latencyQueue := make(chan time.Duration, numToCreate)
+		start := time.Now()
+		for ; *currentNum < goalNum; *currentNum++ {
+			suffix := fmt.Sprintf("%d", *currentNum)
+			go func() {
+				defer ingWg.Done()
+
+				start := time.Now()
+				svcCreated, ingCreated, err := f.createScaleTestServiceIngress(suffix, f.EnableTLS)
+				f.ScaleTestSvcs = append(f.ScaleTestSvcs, svcCreated)
+				f.ScaleTestIngs = append(f.ScaleTestIngs, ingCreated)
+				if err != nil {
+					errQueue <- err
+					return
+				}
+				f.Logger.Infof("Waiting for ingress %s to come up...", ingCreated.Name)
+				if err := f.Jig.WaitForGivenIngressWithTimeout(ingCreated, false, waitForIngressMaxTimeout); err != nil {
+					errQueue <- err
+					return
+				}
+				elapsed := time.Since(start)
+				f.Logger.Infof("Spent %s for ingress %s to come up", elapsed, ingCreated.Name)
+				latencyQueue <- elapsed
+			}()
+		}
+
+		// Wait until all ingress creations are complete.
+		f.Logger.Infof("Waiting for %d ingresses to come up...", numToCreate)
+		ingWg.Wait()
+		close(errQueue)
+		close(latencyQueue)
+		elapsed := time.Since(start)
+		var createLatencies []time.Duration
+		for latency := range latencyQueue {
+			createLatencies = append(createLatencies, latency)
+		}
+		f.BatchCreateLatencies = append(f.BatchCreateLatencies, createLatencies)
+		if len(errQueue) != 0 {
+			f.Logger.Errorf("Failed while creating services and ingresses, spent %v", elapsed)
+			for err := range errQueue {
+				errs = append(errs, err)
+			}
+			return
+		}
+		f.Logger.Infof("Spent %s for %d ingresses to come up", elapsed, numToCreate)
+		f.BatchDurations = append(f.BatchDurations, elapsed)
+	}
+
+	measureCreateUpdateFunc := func() {
+		f.Logger.Infof("Create one more ingress and wait for it to come up")
+		start := time.Now()
+		svcCreated, ingCreated, err := f.createScaleTestServiceIngress(fmt.Sprintf("%d", *currentNum), f.EnableTLS)
+		*currentNum = *currentNum + 1
+		f.ScaleTestSvcs = append(f.ScaleTestSvcs, svcCreated)
+		f.ScaleTestIngs = append(f.ScaleTestIngs, ingCreated)
+		if err != nil {
+			errs = append(errs, err)
+			return
+		}
+
+		f.Logger.Infof("Waiting for ingress %s to come up...", ingCreated.Name)
+		if err := f.Jig.WaitForGivenIngressWithTimeout(ingCreated, false, waitForIngressMaxTimeout); err != nil {
+			errs = append(errs, err)
+			return
+		}
+		elapsed := time.Since(start)
+		f.Logger.Infof("Spent %s for ingress %s to come up", elapsed, ingCreated.Name)
+		f.StepCreateLatencies = append(f.StepCreateLatencies, elapsed)
+
+		f.Logger.Infof("Updating ingress and wait for change to take effect")
+		ingToUpdate, err := f.Clientset.ExtensionsV1beta1().Ingresses(f.Namespace).Get(ingCreated.Name, metav1.GetOptions{})
+		if err != nil {
+			errs = append(errs, err)
+			return
+		}
+		addTestPathToIngress(ingToUpdate)
+		start = time.Now()
+		ingToUpdate, err = f.Clientset.ExtensionsV1beta1().Ingresses(f.Namespace).Update(ingToUpdate)
+		if err != nil {
+			errs = append(errs, err)
+			return
+		}
+
+		if err := f.Jig.WaitForGivenIngressWithTimeout(ingToUpdate, false, waitForIngressMaxTimeout); err != nil {
+			errs = append(errs, err)
+			return
+		}
+		elapsed = time.Since(start)
+		f.Logger.Infof("Spent %s for updating ingress %s", elapsed, ingToUpdate.Name)
+		f.StepUpdateLatencies = append(f.StepUpdateLatencies, elapsed)
+	}
+
+	defer f.dumpLatencies()
+
+	for _, num := range f.NumIngressesTest {
+		f.Logger.Infof("Create more ingresses until we reach %d ingresses", num)
+		prepareIngsFunc(num)
+		f.Logger.Infof("Measure create and update latency with %d ingresses", num)
+		measureCreateUpdateFunc()
+
+		if len(errs) != 0 {
+			return errs
+		}
+	}
+
+	return errs
+}
+
+func (f *IngressScaleFramework) dumpLatencies() {
+	f.Logger.Infof("Dumping scale test latencies...")
+	formattedData := f.GetFormattedLatencies()
+	if f.OutputFile != "" {
+		f.Logger.Infof("Dumping scale test latencies to file %s...", f.OutputFile)
+		ioutil.WriteFile(f.OutputFile, []byte(formattedData), 0644)
+		return
+	}
+	f.Logger.Infof("\n%v", formattedData)
+}
+
+// GetFormattedLatencies returns the formatted latencies output.
+// TODO: Need a better way/format for data output.
+func (f *IngressScaleFramework) GetFormattedLatencies() string {
+	if len(f.NumIngressesTest) == 0 ||
+		len(f.NumIngressesTest) != len(f.BatchCreateLatencies) ||
+		len(f.NumIngressesTest) != len(f.BatchDurations) ||
+		len(f.NumIngressesTest) != len(f.StepCreateLatencies) ||
+		len(f.NumIngressesTest) != len(f.StepUpdateLatencies) {
+		return "Failed to construct latencies output."
+	}
+
+	res := "--- Procedure logs ---\n"
+	for i, latencies := range f.BatchCreateLatencies {
+		res += fmt.Sprintf("Create %d ingresses parallelly, each of them takes below amount of time before starts serving traffic:\n", len(latencies))
+		for _, latency := range latencies {
+			res = res + fmt.Sprintf("- %v\n", latency)
+		}
+		res += fmt.Sprintf("Total duration for completing %d ingress creations: %v\n", len(latencies), f.BatchDurations[i])
+		res += fmt.Sprintf("Duration to create one more ingress with %d ingresses existing: %v\n", f.NumIngressesTest[i], f.StepCreateLatencies[i])
+		res += fmt.Sprintf("Duration to update one ingress with %d ingresses existing: %v\n", f.NumIngressesTest[i]+1, f.StepUpdateLatencies[i])
+	}
+	res = res + "--- Summary ---\n"
+	var batchTotalStr, batchAvgStr, singleCreateStr, singleUpdateStr string
+	for i, latencies := range f.BatchCreateLatencies {
+		batchTotalStr += fmt.Sprintf("Batch creation total latency for %d ingresses with %d ingresses existing: %v\n", len(latencies), f.NumIngressesTest[i]-len(latencies), f.BatchDurations[i])
+		var avgLatency time.Duration
+		for _, latency := range latencies {
+			avgLatency = avgLatency + latency
+		}
+		avgLatency /= time.Duration(len(latencies))
+		batchAvgStr += fmt.Sprintf("Batch creation average latency for %d ingresses with %d ingresses existing: %v\n", len(latencies), f.NumIngressesTest[i]-len(latencies), avgLatency)
+		singleCreateStr += fmt.Sprintf("Single ingress creation latency with %d ingresses existing: %v\n", f.NumIngressesTest[i], f.StepCreateLatencies[i])
+		singleUpdateStr += fmt.Sprintf("Single ingress update latency with %d ingresses existing: %v\n", f.NumIngressesTest[i]+1, f.StepUpdateLatencies[i])
+	}
+	res += batchTotalStr + batchAvgStr + singleCreateStr + singleUpdateStr
+	return res
+}
+
+func addTestPathToIngress(ing *extensions.Ingress) {
+	ing.Spec.Rules[0].IngressRuleValue.HTTP.Paths = append(
+		ing.Spec.Rules[0].IngressRuleValue.HTTP.Paths,
+		extensions.HTTPIngressPath{
+			Path:    "/test",
+			Backend: ing.Spec.Rules[0].IngressRuleValue.HTTP.Paths[0].Backend,
+		})
+}
+
+func (f *IngressScaleFramework) createScaleTestServiceIngress(suffix string, enableTLS bool) (*v1.Service, *extensions.Ingress, error) {
+	svcCreated, err := f.Clientset.CoreV1().Services(f.Namespace).Create(generateScaleTestServiceSpec(suffix))
+	if err != nil {
+		return nil, nil, err
+	}
+	ingCreated, err := f.Clientset.ExtensionsV1beta1().Ingresses(f.Namespace).Create(generateScaleTestIngressSpec(suffix, enableTLS))
+	if err != nil {
+		return nil, nil, err
+	}
+	return svcCreated, ingCreated, nil
+}
+
+func generateScaleTestIngressSpec(suffix string, enableTLS bool) *extensions.Ingress {
+	ing := &extensions.Ingress{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: fmt.Sprintf("%s-%s", scaleTestIngressNamePrefix, suffix),
+		},
+		Spec: extensions.IngressSpec{
+			TLS: []extensions.IngressTLS{
+				{SecretName: scaleTestSecretName},
+			},
+			Rules: []extensions.IngressRule{
+				{
+					Host: scaleTestHostname,
+					IngressRuleValue: extensions.IngressRuleValue{
+						HTTP: &extensions.HTTPIngressRuleValue{
+							Paths: []extensions.HTTPIngressPath{
+								{
+									Path: "/scale",
+									Backend: extensions.IngressBackend{
+										ServiceName: fmt.Sprintf("%s-%s", scaleTestBackendName, suffix),
+										ServicePort: intstr.IntOrString{
+											Type:   intstr.Int,
+											IntVal: 80,
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	if enableTLS {
+		ing.Spec.TLS = []extensions.IngressTLS{
+			{SecretName: scaleTestSecretName},
+		}
+	}
+	return ing
+}
+
+func generateScaleTestServiceSpec(suffix string) *v1.Service {
+	return &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   fmt.Sprintf("%s-%s", scaleTestBackendName, suffix),
+			Labels: scaleTestLabels,
+		},
+		Spec: v1.ServiceSpec{
+			Ports: []v1.ServicePort{{
+				Name:       "http",
+				Protocol:   v1.ProtocolTCP,
+				Port:       80,
+				TargetPort: intstr.FromInt(8080),
+			}},
+			Selector: scaleTestLabels,
+			Type:     v1.ServiceTypeNodePort,
+		},
+	}
+}
+
+func generateScaleTestBackendDeploymentSpec(numReplicas int32) *extensions.Deployment {
+	return &extensions.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: scaleTestBackendName,
+		},
+		Spec: extensions.DeploymentSpec{
+			Replicas: &numReplicas,
+			Selector: &metav1.LabelSelector{MatchLabels: scaleTestLabels},
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: scaleTestLabels,
+				},
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+						{
+							Name:  scaleTestBackendName,
+							Image: "gcr.io/google_containers/echoserver:1.6",
+							Ports: []v1.ContainerPort{{ContainerPort: 8080}},
+							ReadinessProbe: &v1.Probe{
+								Handler: v1.Handler{
+									HTTPGet: &v1.HTTPGetAction{
+										Port: intstr.FromInt(8080),
+										Path: "/healthz",
+									},
+								},
+								FailureThreshold: 10,
+								PeriodSeconds:    1,
+								SuccessThreshold: 1,
+								TimeoutSeconds:   1,
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}

test/e2e/network/scale/localrun/BUILD

@@ -0,0 +1,39 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["ingress_scale.go"],
+    importpath = "k8s.io/kubernetes/test/e2e/network/scale/localrun",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//pkg/cloudprovider/providers/gce:go_default_library",
+        "//test/e2e/framework:go_default_library",
+        "//test/e2e/network/scale:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
+    ],
+)
+
+go_binary(
+    name = "localrun",
+    embed = [":go_default_library"],
+    importpath = "k8s.io/kubernetes/test/e2e/network/scale/localrun",
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

test/e2e/network/scale/localrun/ingress_scale.go

@@ -0,0 +1,185 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+
+	"github.com/golang/glog"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
+
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/network/scale"
+)
+
+var (
+	kubeconfig       string
+	enableTLS        bool
+	numIngressesTest numIngressesSlice
+	testNamespace    string
+	cloudConfig      framework.CloudConfig
+	outputFile       string
+	cleanup          bool
+)
+
+type numIngressesSlice []int
+
+func (i *numIngressesSlice) String() string {
+	return fmt.Sprintf("%d", *i)
+}
+
+func (i *numIngressesSlice) Set(value string) error {
+	v, err := strconv.Atoi(value)
+	if err != nil {
+		return err
+	}
+	*i = append(*i, v)
+	sort.Ints(*i)
+	return nil
+}
+
+func registerFlags() {
+	if home := os.Getenv("HOME"); home != "" {
+		flag.StringVar(&kubeconfig, "kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) Absolute path to the kubeconfig file")
+	} else {
+		flag.StringVar(&kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig file")
+	}
+	flag.StringVar(&cloudConfig.ProjectID, "project", "", "GCE project being used")
+	flag.StringVar(&cloudConfig.Zone, "zone", "", "GCE zone being used")
+	flag.StringVar(&cloudConfig.Region, "region", "", "GCE region being used")
+	flag.Var(&numIngressesTest, "num-ingresses", "The number of ingresses to test, specify multiple times for step testing (e.g. 5 ingresses -> 20 ingresses -> 100 ingresses)")
+	flag.BoolVar(&enableTLS, "enable-tls", true, "Whether to enable TLS on ingress")
+	flag.StringVar(&testNamespace, "namespace", "ingress-test-scale", "Namespace for testing")
+	flag.StringVar(&outputFile, "output", "", "If specify, dump latencies to the specified file")
+	flag.BoolVar(&cleanup, "cleanup", true, "Whether to cleanup resources after test")
+}
+
+func verifyFlags() error {
+	if cloudConfig.ProjectID == "" || cloudConfig.Zone == "" || cloudConfig.Region == "" {
+		return fmt.Errorf("must set all of --project, --zone and --region")
+	}
+	return nil
+}
+
+func main() {
+	registerFlags()
+	flag.Parse()
+	if err := verifyFlags(); err != nil {
+		glog.Errorf("Failed to verify flags: %v", err)
+		os.Exit(1)
+	}
+
+	// Initializing a k8s client.
+	config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
+	if err != nil {
+		glog.Errorf("Failed to build kubeconfig: %v", err)
+		os.Exit(1)
+	}
+	cs, err := clientset.NewForConfig(config)
+	if err != nil {
+		glog.Errorf("Failed to create kubeclient: %v", err)
+		os.Exit(1)
+	}
+
+	// Initializing a GCE client.
+	gceAlphaFeatureGate, err := gcecloud.NewAlphaFeatureGate([]string{})
+	if err != nil {
+		glog.Errorf("Encountered error for creating alpha feature gate: %v", err)
+		os.Exit(1)
+	}
+	gceCloud, err := gcecloud.CreateGCECloud(&gcecloud.CloudConfig{
+		ProjectID:        cloudConfig.ProjectID,
+		Region:           cloudConfig.Region,
+		Zone:             cloudConfig.Zone,
+		AlphaFeatureGate: gceAlphaFeatureGate,
+	})
+	if err != nil {
+		glog.Errorf("Error building GCE provider: %v", err)
+		os.Exit(1)
+	}
+	cloudConfig.Provider = gceCloud
+
+	testSuccessFlag := true
+	defer func() {
+		if !testSuccessFlag {
+			glog.Errorf("Ingress scale test failed.")
+			os.Exit(1)
+		}
+	}()
+
+	ns := &v1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: testNamespace,
+		},
+	}
+	glog.Infof("Creating namespace %s...", ns.Name)
+	if _, err := cs.CoreV1().Namespaces().Create(ns); err != nil {
+		glog.Errorf("Failed to create namespace %s: %v", ns.Name, err)
+		testSuccessFlag = false
+		return
+	}
+	if cleanup {
+		defer func() {
+			glog.Infof("Deleting namespace %s...", ns.Name)
+			if err := cs.CoreV1().Namespaces().Delete(ns.Name, nil); err != nil {
+				glog.Errorf("Failed to delete namespace %s: %v", ns.Name, err)
+				testSuccessFlag = false
+			}
+		}()
+	}
+
+	// Setting up a localized scale test framework.
+	f := scale.NewIngressScaleFramework(cs, ns.Name, cloudConfig)
+	f.Logger = &framework.GLogger{}
+	// Customizing scale test.
+	f.EnableTLS = enableTLS
+	f.OutputFile = outputFile
+	if len(numIngressesTest) != 0 {
+		f.NumIngressesTest = numIngressesTest
+	}
+
+	// Real test begins.
+	if cleanup {
+		defer func() {
+			if errs := f.CleanupScaleTest(); len(errs) != 0 {
+				glog.Errorf("Failed to cleanup scale test: %v", errs)
+				testSuccessFlag = false
+			}
+		}()
+	}
+	err = f.PrepareScaleTest()
+	if err != nil {
+		glog.Errorf("Failed to prepare scale test: %v", err)
+		testSuccessFlag = false
+		return
+	}
+
+	if errs := f.RunScaleTest(); len(errs) != 0 {
+		glog.Errorf("Failed while running scale test: %v", errs)
+		testSuccessFlag = false
+	}
+}