Merge pull request #61143 from satyasm/ipam-perf-cloud-mock

Automatic merge from submit-queue (batch tested with PRs 61402, 61143, 61427, 60592). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Performance tests and fix for IPAM controller. Tests the four modes of allocations. Can be run using ./test-performance.sh under tests/integration/ipamperf directory. See ./test-performance.sh -h for supported flags. **What this PR does / why we need it**: **Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*: Fixes # **Special notes for your reviewer**: Please see the implementation notes comment block in cloud.go for core details of how the mocking works. README.md has details on how the tests can be run on the command line. **Release note**: ```release-note Performance test framework and basic tests for the IPAM controller, to simulate behavior of the four supported modes under lightly loaded and loaded conditions, where load is defined as the number of operations to perform as against the configured kubernetes API server QPS. ```
2025-07-23 03:41:45 +00:00 · 2018-03-27 15:33:09 -07:00 · 2018-03-27 15:33:09 -07:00 · 2bc231eb57
commit 2bc231eb57
parent 66ecee9cc0 77da96c7f6
12 changed files with 925 additions and 0 deletions
--- a/pkg/cloudprovider/providers/gce/support.go
+++ b/pkg/cloudprovider/providers/gce/support.go
@ -64,3 +64,14 @@ func (l *gceRateLimiter) Accept(ctx context.Context, key *cloud.RateLimitKey) er
 	}
 	return nil
 }
+
+// CreateGCECloudWithCloud is a helper function to create an instance of GCECloud with the
+// given Cloud interface implementation. Typical usage is to use cloud.NewMockGCE to get a
+// handle to a mock Cloud instance and then use that for testing.
+func CreateGCECloudWithCloud(config *CloudConfig, c cloud.Cloud) (*GCECloud, error) {
+	gceCloud, err := CreateGCECloud(config)
+	if err == nil {
+		gceCloud.c = c
+	}
+	return gceCloud, err
+}
--- a/test/integration/BUILD
+++ b/test/integration/BUILD
@ -48,6 +48,7 @@ filegroup(
        "//test/integration/examples:all-srcs",
        "//test/integration/framework:all-srcs",
        "//test/integration/garbagecollector:all-srcs",
+        "//test/integration/ipamperf:all-srcs",
        "//test/integration/master:all-srcs",
        "//test/integration/metrics:all-srcs",
        "//test/integration/objectmeta:all-srcs",
--- a/test/integration/ipamperf/BUILD
+++ b/test/integration/ipamperf/BUILD
@ -0,0 +1,70 @@
+package(default_visibility = ["//visibility:public"])
+
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_test(
+    name = "go_default_test",
+    size = "large",
+    srcs = [
+        "ipam_test.go",
+        "main_test.go",
+    ],
+    embed = [":go_default_library"],
+    tags = ["integration"],
+    deps = [
+        "//pkg/api/testapi:go_default_library",
+        "//pkg/controller/nodeipam:go_default_library",
+        "//pkg/controller/nodeipam/ipam:go_default_library",
+        "//test/integration/framework:go_default_library",
+        "//test/integration/util:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/client-go/informers:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/rest:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "cloud.go",
+        "results.go",
+        "util.go",
+    ],
+    importpath = "k8s.io/kubernetes/test/integration/ipamperf",
+    deps = [
+        "//pkg/api/testapi:go_default_library",
+        "//pkg/cloudprovider:go_default_library",
+        "//pkg/cloudprovider/providers/gce/cloud:go_default_library",
+        "//pkg/cloudprovider/providers/gce/cloud/meta:go_default_library",
+        "//pkg/controller/nodeipam/ipam:go_default_library",
+        "//pkg/controller/nodeipam/ipam/cidrset:go_default_library",
+        "//pkg/controller/util/node:go_default_library",
+        "//test/integration/util:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/google.golang.org/api/compute/v0.beta:go_default_library",
+        "//vendor/google.golang.org/api/compute/v1:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/client-go/informers:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/rest:go_default_library",
+        "//vendor/k8s.io/client-go/tools/cache:go_default_library",
+    ],
+)
--- a/test/integration/ipamperf/README.md
+++ b/test/integration/ipamperf/README.md
@ -0,0 +1,64 @@
+IPAM Performance Test
+=====
+
+Motivation
+-----
+We wanted to be able to test the behavior of the IPAM controller's under various scenarios, 
+by mocking and monitoring the edges that the controller interacts with. This has the following goals:
+
+- Save time on testing
+- To simulate various behaviors cheaply
+- To observe and model the ideal behavior of the IPAM controller code
+
+Currently the test runs through the 4 different IPAM controller modes for cases where the kube API QPS is a) 
+equal to and b) significantly less than the number of nodes being added to observe and quantify behavior.
+
+How to run
+-------
+
+```shell
+# In kubernetes root path
+make generated_files
+
+cd test/integration/ipamperf
+./test-performance.sh
+```
+
+The runner scripts support a few different options:
+
+```shell
+./test-performance.sh -h
+usage: ./test-performance.sh [-h] [-d] [-r <pattern>] [-o <filename>]
+ -h display this help message
+ -d enable debug logs in tests
+ -r <pattern> regex pattern to match for tests
+ -o <filename> file to write JSON formatted results to
+```
+
+The tests follow the pattern TestPerformance/{AllocatorType}-KubeQPS{X}-Nodes{Y}, where AllocatorType 
+is one of 
+
+- RangeAllocator
+- IPAMFromCluster
+- CloudAllocator
+- IPAMFromCloud
+
+and X represents the QPS configured for the kubernetes API client, and Y is the number of nodes to create.
+
+The -d flags set the -v level for glog to 6, enabling nearly all of the debug logs in the code.
+
+So to run the test for CloudAllocator with 10 nodes, one can run
+
+```shell
+./test-performance.sh -r /CloudAllocator.*Nodes10$
+```
+
+At the end of the test, a JSON format of the results for all the tests run is printed. Passing the -o option 
+allows for also saving this JSON to a named file.
+
+Code Organization
+-----
+The core of the tests are defined in [ipam_test.go](ipam_test.go), using the t.Run() helper to control parallelism
+as we want to able to start the master once. [cloud.go](cloud.go) contains the mock of the cloud server endpoint
+and can be configured to behave differently as needed by the various modes. The tracking of the node behavior and
+creation of the test results data is in [results.go](results.go).
--- a/test/integration/ipamperf/cloud.go
+++ b/test/integration/ipamperf/cloud.go
@ -0,0 +1,154 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ipamperf
+
+import (
+	"context"
+	"net"
+	"sync"
+
+	beta "google.golang.org/api/compute/v0.beta"
+	ga "google.golang.org/api/compute/v1"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/meta"
+	"k8s.io/kubernetes/pkg/controller/nodeipam/ipam/cidrset"
+	"k8s.io/kubernetes/test/integration/util"
+)
+
+// implemntation note:
+// ------------------
+// cloud.go implements hooks and handler functions for the MockGCE cloud in order to meet expectations
+// of cloud behavior from the IPAM controllers. The key constraint is that the IPAM code is spread
+// across both GA and Beta instances, which are distinct objects in the mock. We need to solve for
+//
+// 1. When a GET is called on an instance, we lazy create the instance with or without an assigned
+//    ip alias as needed by the IPAM controller type
+// 2. When we assign an IP alias for an instance, both the GA and Beta instance have to agree on the
+//    assigned alias range
+//
+// We solve both the problems by using a baseInstanceList which maintains a list of known instances,
+// and their pre-assigned ip-alias ranges (if needed). We then create GetHook for GA and Beta GetInstance
+// calls as closures over this betaInstanceList that can lookup base instance data.
+//
+// This has the advantage that once the Get hook pouplates the GCEMock with the base data, we then let the
+// rest of the mock code run as is.
+
+// baseInstance tracks basic instance data needed by the IPAM controllers
+type baseInstance struct {
+	name       string
+	zone       string
+	aliasRange string
+}
+
+// baseInstanceList tracks a set of base instances
+type baseInstanceList struct {
+	allocateCIDR   bool
+	clusterCIDR    *net.IPNet
+	subnetMaskSize int
+	cidrSet        *cidrset.CidrSet
+
+	lock      sync.Mutex // protect access to instances
+	instances map[meta.Key]*baseInstance
+}
+
+// toGA is an utility method to return the baseInstance data as a GA Instance object
+func (bi *baseInstance) toGA() *ga.Instance {
+	inst := &ga.Instance{Name: bi.name, Zone: bi.zone, NetworkInterfaces: []*ga.NetworkInterface{{}}}
+	if bi.aliasRange != "" {
+		inst.NetworkInterfaces[0].AliasIpRanges = []*ga.AliasIpRange{
+			{IpCidrRange: bi.aliasRange, SubnetworkRangeName: util.TestSecondaryRangeName},
+		}
+	}
+	return inst
+}
+
+// toGA is an utility method to return the baseInstance data as a beta Instance object
+func (bi *baseInstance) toBeta() *beta.Instance {
+	inst := &beta.Instance{Name: bi.name, Zone: bi.zone, NetworkInterfaces: []*beta.NetworkInterface{{}}}
+	if bi.aliasRange != "" {
+		inst.NetworkInterfaces[0].AliasIpRanges = []*beta.AliasIpRange{
+			{IpCidrRange: bi.aliasRange, SubnetworkRangeName: util.TestSecondaryRangeName},
+		}
+	}
+	return inst
+}
+
+// newBaseInstanceList is the baseInstanceList constructor
+func newBaseInstanceList(allocateCIDR bool, clusterCIDR *net.IPNet, subnetMaskSize int) *baseInstanceList {
+	cidrSet, _ := cidrset.NewCIDRSet(clusterCIDR, subnetMaskSize)
+	return &baseInstanceList{
+		allocateCIDR:   allocateCIDR,
+		clusterCIDR:    clusterCIDR,
+		subnetMaskSize: subnetMaskSize,
+		cidrSet:        cidrSet,
+		instances:      make(map[meta.Key]*baseInstance),
+	}
+}
+
+// getOrCreateBaseInstance lazily creates a new base instance, assigning if allocateCIDR is true
+func (bil *baseInstanceList) getOrCreateBaseInstance(key *meta.Key) *baseInstance {
+	bil.lock.Lock()
+	defer bil.lock.Unlock()
+
+	inst, found := bil.instances[*key]
+	if !found {
+		inst = &baseInstance{name: key.Name, zone: key.Zone}
+		if bil.allocateCIDR {
+			nextRange, _ := bil.cidrSet.AllocateNext()
+			inst.aliasRange = nextRange.String()
+		}
+		bil.instances[*key] = inst
+	}
+	return inst
+}
+
+// newGAGetHook creates a new closure with the current baseInstanceList to be used as a MockInstances.GetHook
+func (bil *baseInstanceList) newGAGetHook() func(ctx context.Context, key *meta.Key, m *cloud.MockInstances) (bool, *ga.Instance, error) {
+	return func(ctx context.Context, key *meta.Key, m *cloud.MockInstances) (bool, *ga.Instance, error) {
+		m.Lock.Lock()
+		defer m.Lock.Unlock()
+
+		if _, found := m.Objects[*key]; !found {
+			m.Objects[*key] = &cloud.MockInstancesObj{Obj: bil.getOrCreateBaseInstance(key).toGA()}
+		}
+		return false, nil, nil
+	}
+}
+
+// newBetaGetHook creates a new closure with the current baseInstanceList to be used as a MockBetaInstances.GetHook
+func (bil *baseInstanceList) newBetaGetHook() func(ctx context.Context, key *meta.Key, m *cloud.MockBetaInstances) (bool, *beta.Instance, error) {
+	return func(ctx context.Context, key *meta.Key, m *cloud.MockBetaInstances) (bool, *beta.Instance, error) {
+		m.Lock.Lock()
+		defer m.Lock.Unlock()
+
+		if _, found := m.Objects[*key]; !found {
+			m.Objects[*key] = &cloud.MockInstancesObj{Obj: bil.getOrCreateBaseInstance(key).toBeta()}
+		}
+		return false, nil, nil
+	}
+}
+
+// newMockCloud returns a mock GCE instance with the appropriate handlers hooks
+func (bil *baseInstanceList) newMockCloud() cloud.Cloud {
+	c := cloud.NewMockGCE(nil)
+
+	// insert hooks to lazy create a instance when needed
+	c.MockInstances.GetHook = bil.newGAGetHook()
+	c.MockBetaInstances.GetHook = bil.newBetaGetHook()
+
+	return c
+}
--- a/test/integration/ipamperf/ipam_test.go
+++ b/test/integration/ipamperf/ipam_test.go
@ -0,0 +1,150 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ipamperf
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/golang/glog"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/client-go/informers"
+	clientset "k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/kubernetes/pkg/api/testapi"
+	"k8s.io/kubernetes/pkg/controller/nodeipam"
+	"k8s.io/kubernetes/pkg/controller/nodeipam/ipam"
+	"k8s.io/kubernetes/test/integration/util"
+)
+
+func setupAllocator(apiURL string, config *Config, clusterCIDR, serviceCIDR *net.IPNet, subnetMaskSize int) (*clientset.Clientset, util.ShutdownFunc, error) {
+	controllerStopChan := make(chan struct{})
+	shutdownFunc := func() {
+		close(controllerStopChan)
+	}
+
+	clientSet := clientset.NewForConfigOrDie(&restclient.Config{
+		Host:          apiURL,
+		ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Groups[v1.GroupName].GroupVersion()},
+		QPS:           float32(config.KubeQPS),
+		Burst:         config.KubeQPS,
+	})
+
+	sharedInformer := informers.NewSharedInformerFactory(clientSet, 1*time.Hour)
+	ipamController, err := nodeipam.NewNodeIpamController(
+		sharedInformer.Core().V1().Nodes(), config.Cloud, clientSet,
+		clusterCIDR, serviceCIDR, subnetMaskSize, true, config.AllocatorType,
+	)
+	if err != nil {
+		return nil, shutdownFunc, err
+	}
+	go ipamController.Run(controllerStopChan)
+	sharedInformer.Start(controllerStopChan)
+
+	return clientSet, shutdownFunc, nil
+}
+
+func runTest(t *testing.T, apiURL string, config *Config, clusterCIDR, serviceCIDR *net.IPNet, subnetMaskSize int) (*Results, error) {
+	t.Helper()
+	glog.Infof("Running test %s", t.Name())
+
+	defer deleteNodes(apiURL, config) // cleanup nodes on after controller shutdown
+
+	clientSet, shutdownFunc, err := setupAllocator(apiURL, config, clusterCIDR, serviceCIDR, subnetMaskSize)
+	if err != nil {
+		t.Fatalf("Error starting IPAM allocator: %v", err)
+	}
+	defer shutdownFunc()
+
+	o := NewObserver(clientSet, config.NumNodes)
+	if err := o.StartObserving(); err != nil {
+		t.Fatalf("Could not start test observer: %v", err)
+	}
+
+	if err := createNodes(apiURL, config); err != nil {
+		t.Fatalf("Could not create nodes: %v", err)
+	}
+
+	results := o.Results(t.Name(), config)
+	glog.Infof("Results: %s", results)
+	if !results.Succeeded {
+		t.Errorf("%s: Not allocations succeeded", t.Name())
+	}
+	return results, nil
+}
+
+func logResults(allResults []*Results) {
+	jStr, err := json.MarshalIndent(allResults, "", "  ")
+	if err != nil {
+		glog.Errorf("Error formating results: %v", err)
+		return
+	}
+	if resultsLogFile != "" {
+		glog.Infof("Logging results to %s", resultsLogFile)
+		if err := ioutil.WriteFile(resultsLogFile, jStr, os.FileMode(0644)); err != nil {
+			glog.Errorf("Error logging results to %s: %v", resultsLogFile, err)
+		}
+	}
+	glog.Infof("AllResults:\n%s", string(jStr))
+}
+
+func TestPerformance(t *testing.T) {
+	apiURL, masterShutdown := util.StartApiserver()
+	defer masterShutdown()
+
+	_, clusterCIDR, _ := net.ParseCIDR("10.96.0.0/11") // allows up to 8K nodes
+	_, serviceCIDR, _ := net.ParseCIDR("10.94.0.0/24") // does not matter for test - pick upto  250 services
+	subnetMaskSize := 24
+
+	var (
+		allResults []*Results
+		tests      []*Config
+	)
+
+	for _, numNodes := range []int{10, 100} {
+		for _, alloc := range []ipam.CIDRAllocatorType{ipam.RangeAllocatorType, ipam.CloudAllocatorType, ipam.IPAMFromClusterAllocatorType, ipam.IPAMFromCloudAllocatorType} {
+			tests = append(tests, &Config{AllocatorType: alloc, NumNodes: numNodes, CreateQPS: numNodes, KubeQPS: 10, CloudQPS: 10})
+		}
+	}
+
+	for _, test := range tests {
+		testName := fmt.Sprintf("%s-KubeQPS%d-Nodes%d", test.AllocatorType, test.KubeQPS, test.NumNodes)
+		t.Run(testName, func(t *testing.T) {
+			allocateCIDR := false
+			if test.AllocatorType == ipam.IPAMFromCloudAllocatorType || test.AllocatorType == ipam.CloudAllocatorType {
+				allocateCIDR = true
+			}
+			bil := newBaseInstanceList(allocateCIDR, clusterCIDR, subnetMaskSize)
+			cloud, err := util.NewMockGCECloud(bil.newMockCloud())
+			if err != nil {
+				t.Fatalf("Unable to create mock cloud: %v", err)
+			}
+			test.Cloud = cloud
+			if results, err := runTest(t, apiURL, test, clusterCIDR, serviceCIDR, subnetMaskSize); err == nil {
+				allResults = append(allResults, results)
+			}
+		})
+	}
+
+	logResults(allResults)
+}
--- a/test/integration/ipamperf/main_test.go
+++ b/test/integration/ipamperf/main_test.go
@ -0,0 +1,32 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ipamperf
+
+import (
+	"flag"
+	"testing"
+
+	"k8s.io/kubernetes/test/integration/framework"
+)
+
+var resultsLogFile string
+
+func TestMain(m *testing.M) {
+	flag.StringVar(&resultsLogFile, "log", "", "log file to write JSON results to")
+	flag.Parse()
+	framework.EtcdMain(m.Run)
+}
--- a/test/integration/ipamperf/results.go
+++ b/test/integration/ipamperf/results.go
@ -0,0 +1,218 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ipamperf
+
+import (
+	"bytes"
+	"fmt"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/golang/glog"
+	"k8s.io/api/core/v1"
+	"k8s.io/client-go/informers"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+	"k8s.io/kubernetes/pkg/controller/nodeipam/ipam"
+	nodeutil "k8s.io/kubernetes/pkg/controller/util/node"
+)
+
+// Config represents the test configuration that is being run
+type Config struct {
+	CreateQPS     int                     // rate at which nodes are created
+	KubeQPS       int                     // rate for communication with kubernetes API
+	CloudQPS      int                     // rate for communication with cloud endpoint
+	NumNodes      int                     // number of nodes to created and monitored
+	AllocatorType ipam.CIDRAllocatorType  // type of allocator to run
+	Cloud         cloudprovider.Interface // cloud provider
+}
+
+type nodeTime struct {
+	added     time.Time // observed time for when node was added
+	allocated time.Time // observed time for when node was assigned podCIDR
+	podCIDR   string    // the allocated podCIDR range
+}
+
+// Observer represents the handle to test observer that watches for node changes
+// and tracks behavior
+type Observer struct {
+	numAdded     int                  // number of nodes observed added
+	numAllocated int                  // number of nodes observed allocated podCIDR
+	timing       map[string]*nodeTime // per node timing
+	numNodes     int                  // the number of nodes to expect
+	stopChan     chan struct{}        // for the shared informer
+	wg           sync.WaitGroup
+	clientSet    *clientset.Clientset
+}
+
+// JSONDuration is an alias of time.Duration to support custom Marshal code
+type JSONDuration time.Duration
+
+// NodeDuration represents the CIDR allocation time for each node
+type NodeDuration struct {
+	Name     string       // node name
+	PodCIDR  string       // the podCIDR that was assigned to the node
+	Duration JSONDuration // how long it took to assign podCIDR
+}
+
+// Results represents the observed test results.
+type Results struct {
+	Name           string         // name for the test
+	Config         *Config        // handle to the test config
+	Succeeded      bool           // whether all nodes were assigned podCIDR
+	MaxAllocTime   JSONDuration   // the maximum time take for assignment per node
+	TotalAllocTime JSONDuration   // duration between first addition and last assignment
+	NodeAllocTime  []NodeDuration // assignment time by node name
+}
+
+// NewObserver creates a new observer given a handle to the Clientset
+func NewObserver(clientSet *clientset.Clientset, numNodes int) *Observer {
+	o := &Observer{
+		timing:    map[string]*nodeTime{},
+		numNodes:  numNodes,
+		clientSet: clientSet,
+		stopChan:  make(chan struct{}),
+	}
+	return o
+}
+
+// StartObserving starts an asynchronous loop to monitor for node changes.
+// Call Results() to get the test results after starting observer.
+func (o *Observer) StartObserving() error {
+	go o.monitor()
+	glog.Infof("Test observer started")
+	return nil
+}
+
+// Results returns the test results. It waits for the observer to finish
+// and returns the computed results of the observations.
+func (o *Observer) Results(name string, config *Config) *Results {
+	var (
+		firstAdd       time.Time // earliest time any node was added (first node add)
+		lastAssignment time.Time // latest time any node was assignged CIDR (last node assignment)
+	)
+	o.wg.Wait()
+	close(o.stopChan) // shutdown the shared informer
+
+	results := &Results{
+		Name:          name,
+		Config:        config,
+		Succeeded:     o.numAdded == o.numNodes && o.numAllocated == o.numNodes,
+		MaxAllocTime:  0,
+		NodeAllocTime: []NodeDuration{},
+	}
+	for name, nTime := range o.timing {
+		addFound := !nTime.added.IsZero()
+		if addFound && (firstAdd.IsZero() || nTime.added.Before(firstAdd)) {
+			firstAdd = nTime.added
+		}
+		cidrFound := !nTime.allocated.IsZero()
+		if cidrFound && nTime.allocated.After(lastAssignment) {
+			lastAssignment = nTime.allocated
+		}
+		if addFound && cidrFound {
+			allocTime := nTime.allocated.Sub(nTime.added)
+			if allocTime > time.Duration(results.MaxAllocTime) {
+				results.MaxAllocTime = JSONDuration(allocTime)
+			}
+			results.NodeAllocTime = append(results.NodeAllocTime, NodeDuration{
+				Name: name, PodCIDR: nTime.podCIDR, Duration: JSONDuration(allocTime),
+			})
+		}
+	}
+	results.TotalAllocTime = JSONDuration(lastAssignment.Sub(firstAdd))
+	sort.Slice(results.NodeAllocTime, func(i, j int) bool {
+		return results.NodeAllocTime[i].Duration > results.NodeAllocTime[j].Duration
+	})
+	return results
+}
+
+func (o *Observer) monitor() {
+	o.wg.Add(1)
+
+	sharedInformer := informers.NewSharedInformerFactory(o.clientSet, 1*time.Second)
+	nodeInformer := sharedInformer.Core().V1().Nodes().Informer()
+
+	nodeInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc: nodeutil.CreateAddNodeHandler(func(node *v1.Node) (err error) {
+			name := node.GetName()
+			if node.Spec.PodCIDR != "" {
+				// ignore nodes that have PodCIDR (might be hold over from previous runs that did not get cleaned up)
+				return
+			}
+			nTime := &nodeTime{}
+			o.timing[name] = nTime
+			nTime.added = time.Now()
+			o.numAdded = o.numAdded + 1
+			return
+		}),
+		UpdateFunc: nodeutil.CreateUpdateNodeHandler(func(oldNode, newNode *v1.Node) (err error) {
+			name := newNode.GetName()
+			nTime, found := o.timing[name]
+			if !found {
+				return // consistency check - ignore nodes we have not seen the add event for
+			}
+			// check if CIDR assigned and ignore redundant updates
+			if newNode.Spec.PodCIDR != "" && nTime.podCIDR == "" {
+				nTime.allocated = time.Now()
+				nTime.podCIDR = newNode.Spec.PodCIDR
+				o.numAllocated++
+				// do following check only if numAllocated is modified, as otherwise, redundant updates
+				// can cause wg.Done() to be called multiple times, causing a panic
+				if o.numAdded == o.numNodes && o.numAllocated == o.numNodes {
+					glog.Info("All nodes assigned podCIDR")
+					o.wg.Done()
+				}
+			}
+			return
+		}),
+	})
+	sharedInformer.Start(o.stopChan)
+}
+
+// String implements the Stringer interface and returns a multi-line representation
+// of the test results.
+func (results *Results) String() string {
+	var b bytes.Buffer
+	fmt.Fprintf(&b, "\n  TestName: %s", results.Name)
+	fmt.Fprintf(&b, "\n  NumNodes: %d, CreateQPS: %d, KubeQPS: %d, CloudQPS: %d, Allocator: %v",
+		results.Config.NumNodes, results.Config.CreateQPS, results.Config.KubeQPS,
+		results.Config.CloudQPS, results.Config.AllocatorType)
+	fmt.Fprintf(&b, "\n  Succeeded: %v, TotalAllocTime: %v, MaxAllocTime: %v",
+		results.Succeeded, time.Duration(results.TotalAllocTime), time.Duration(results.MaxAllocTime))
+	fmt.Fprintf(&b, "\n  %5s %-20s %-20s %s", "Num", "Node", "PodCIDR", "Duration (s)")
+	for i, d := range results.NodeAllocTime {
+		fmt.Fprintf(&b, "\n  %5d %-20s %-20s %10.3f", i+1, d.Name, d.PodCIDR, time.Duration(d.Duration).Seconds())
+	}
+	return b.String()
+}
+
+// MarshalJSON implements the json.Marshaler interface
+func (jDuration *JSONDuration) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf("\"%s\"", time.Duration(*jDuration).String())), nil
+}
+
+// UnmarshalJSON implements the json.Unmarshaler interface
+func (jDuration *JSONDuration) UnmarshalJSON(b []byte) (err error) {
+	var d time.Duration
+	if d, err = time.ParseDuration(string(b[1 : len(b)-1])); err == nil {
+		*jDuration = JSONDuration(d)
+	}
+	return
+}
--- a/test/integration/ipamperf/test-performance.sh
+++ b/test/integration/ipamperf/test-performance.sh
@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+TEST_ARGS=""
+RUN_PATTERN=".*"
+
+function usage() {
+  echo "usage: $0 [-h] [-d] [-r <pattern>] [-o <filename>]"
+  echo " -h display this help message"
+  echo " -d enable debug logs in tests"
+  echo " -r <pattern> regex pattern to match for tests"
+  echo " -o <filename> file to write JSON formatted results to"
+  exit 1
+}
+
+while getopts ":hdr:o:" opt; do
+  case ${opt} in
+    d) TEST_ARGS="${TEST_ARGS} -v=6"
+      ;;
+    r) RUN_PATTERN="${OPTARG}"
+      ;;
+    o) TEST_ARGS="${TEST_ARGS} -log ${OPTARG}"
+      ;;
+    h) ::usage
+      ;;
+    \?) ::usage
+      ;;
+  esac
+done
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../../../
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
+kube::golang::setup_env
+
+DIR_BASENAME=$(dirname "${BASH_SOURCE}")
+pushd ${DIR_BASENAME}
+
+cleanup() {
+  popd 2> /dev/null
+  kube::etcd::cleanup
+  kube::log::status "performance test cleanup complete"
+}
+
+trap cleanup EXIT
+
+kube::etcd::start
+
+# Running IPAM tests. It might take a long time.
+kube::log::status "performance test (IPAM) start"
+go test -test.run=${RUN_PATTERN} -test.timeout=60m -test.short=false -v -args ${TEST_ARGS}
+kube::log::status "... IPAM tests finished."
--- a/test/integration/ipamperf/util.go
+++ b/test/integration/ipamperf/util.go
@ -0,0 +1,80 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ipamperf
+
+import (
+	"github.com/golang/glog"
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/kubernetes/pkg/api/testapi"
+)
+
+var (
+	baseNodeTemplate = &v1.Node{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "sample-node-",
+		},
+		Spec: v1.NodeSpec{
+			ExternalID: "foo",
+		},
+		Status: v1.NodeStatus{
+			Capacity: v1.ResourceList{
+				v1.ResourcePods:   *resource.NewQuantity(110, resource.DecimalSI),
+				v1.ResourceCPU:    resource.MustParse("4"),
+				v1.ResourceMemory: resource.MustParse("32Gi"),
+			},
+			Phase: v1.NodeRunning,
+			Conditions: []v1.NodeCondition{
+				{Type: v1.NodeReady, Status: v1.ConditionTrue},
+			},
+		},
+	}
+)
+
+func deleteNodes(apiURL string, config *Config) {
+	glog.Info("Deleting nodes")
+	clientSet := clientset.NewForConfigOrDie(&restclient.Config{
+		Host:          apiURL,
+		ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Groups[v1.GroupName].GroupVersion()},
+		QPS:           float32(config.CreateQPS),
+		Burst:         config.CreateQPS,
+	})
+	noGrace := int64(0)
+	if err := clientSet.CoreV1().Nodes().DeleteCollection(&metav1.DeleteOptions{GracePeriodSeconds: &noGrace}, metav1.ListOptions{}); err != nil {
+		glog.Errorf("Error deleting node: %v", err)
+	}
+}
+
+func createNodes(apiURL string, config *Config) error {
+	clientSet := clientset.NewForConfigOrDie(&restclient.Config{
+		Host:          apiURL,
+		ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Groups[v1.GroupName].GroupVersion()},
+		QPS:           float32(config.CreateQPS),
+		Burst:         config.CreateQPS,
+	})
+	glog.Infof("Creating %d nodes", config.NumNodes)
+	for i := 0; i < config.NumNodes; i++ {
+		if _, err := clientSet.CoreV1().Nodes().Create(baseNodeTemplate); err != nil {
+			return err
+		}
+	}
+	glog.Infof("%d nodes created", config.NumNodes)
+	return nil
+}
--- a/test/integration/util/BUILD
+++ b/test/integration/util/BUILD
@ -8,15 +8,19 @@ load(
 go_library(
    name = "go_default_library",
    srcs = [
+        "cloud.go",
        "util.go",
    ],
    importpath = "k8s.io/kubernetes/test/integration/util",
    deps = [
        "//pkg/api/legacyscheme:go_default_library",
+        "//pkg/cloudprovider/providers/gce:go_default_library",
+        "//pkg/cloudprovider/providers/gce/cloud:go_default_library",
        "//pkg/scheduler:go_default_library",
        "//pkg/scheduler/factory:go_default_library",
        "//test/integration/framework:go_default_library",
        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/golang.org/x/oauth2:go_default_library",
        "//vendor/k8s.io/api/core/v1:go_default_library",
        "//vendor/k8s.io/client-go/informers:go_default_library",
        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
--- a/test/integration/util/cloud.go
+++ b/test/integration/util/cloud.go
@ -0,0 +1,72 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"time"
+
+	"golang.org/x/oauth2"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
+)
+
+const (
+	// TestProjectID is the project id used for creating NewMockGCECloud
+	TestProjectID = "test-project"
+	// TestNetworkProjectID is the network project id for creating NewMockGCECloud
+	TestNetworkProjectID = "net-test-project"
+	// TestRegion is the region for creating NewMockGCECloud
+	TestRegion = "test-region"
+	// TestZone is the zone for creating NewMockGCECloud
+	TestZone = "test-zone"
+	// TestNetworkName is the network name for creating NewMockGCECloud
+	TestNetworkName = "test-network"
+	// TestSubnetworkName is the sub network name for creating NewMockGCECloud
+	TestSubnetworkName = "test-sub-network"
+	// TestSecondaryRangeName is the secondary range name for creating NewMockGCECloud
+	TestSecondaryRangeName = "test-secondary-range"
+)
+
+type mockTokenSource struct{}
+
+func (*mockTokenSource) Token() (*oauth2.Token, error) {
+	return &oauth2.Token{
+		AccessToken:  "access",
+		TokenType:    "Bearer",
+		RefreshToken: "refresh",
+		Expiry:       time.Now().Add(1 * time.Hour),
+	}, nil
+}
+
+// NewMockGCECloud returns a handle to a GCECloud instance that is
+// served by a mock http server
+func NewMockGCECloud(cloud cloud.Cloud) (*gce.GCECloud, error) {
+	config := &gce.CloudConfig{
+		ProjectID:          TestProjectID,
+		NetworkProjectID:   TestNetworkProjectID,
+		Region:             TestRegion,
+		Zone:               TestZone,
+		ManagedZones:       []string{TestZone},
+		NetworkName:        TestNetworkName,
+		SubnetworkName:     TestSubnetworkName,
+		SecondaryRangeName: TestSecondaryRangeName,
+		NodeTags:           []string{},
+		UseMetadataServer:  false,
+		TokenSource:        &mockTokenSource{},
+	}
+	return gce.CreateGCECloudWithCloud(config, cloud)
+}