diff --git a/test/integration/ipamperf/BUILD b/test/integration/ipamperf/BUILD index ea3a8bd287f..525fc4bca38 100644 --- a/test/integration/ipamperf/BUILD +++ b/test/integration/ipamperf/BUILD @@ -60,6 +60,7 @@ go_library( "//vendor/google.golang.org/api/compute/v0.beta:go_default_library", "//vendor/google.golang.org/api/compute/v1:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/client-go/informers:go_default_library", diff --git a/test/integration/ipamperf/README.md b/test/integration/ipamperf/README.md index 4c1e78720ac..532dcdc8c10 100644 --- a/test/integration/ipamperf/README.md +++ b/test/integration/ipamperf/README.md @@ -29,10 +29,18 @@ The runner scripts support a few different options: ```shell ./test-performance.sh -h usage: ./test-performance.sh [-h] [-d] [-r ] [-o ] +usage: ./test-performance.sh -h display this help message -d enable debug logs in tests -r regex pattern to match for tests -o file to write JSON formatted results to + -p enable cpu and memory profiles, output written to mem-.out and cpu-.out + -c enable custom test configuration + -a allocator name, one of RangeAllocator, CloudAllocator, IPAMFromCluster, IPAMFromCloud + -k api server qps for allocator + -n number of nodes to simulate + -m api server qps for node creation + -l gce cloud endpoint qps ``` The tests follow the pattern TestPerformance/{AllocatorType}-KubeQPS{X}-Nodes{Y}, where AllocatorType @@ -56,6 +64,19 @@ So to run the test for CloudAllocator with 10 nodes, one can run At the end of the test, a JSON format of the results for all the tests run is printed. Passing the -o option allows for also saving this JSON to a named file. +### Profiling the code +It's possible to get the CPU and memory profiles of code during test execution by using the ```-p``` option. +The CPU and memory profiles are generated in the same directory with the file names set to ```cpu-.out``` +and ```cpu-.out```, where `````` is the argument value. Typicall pattern is to put in the number +of nodes being simulated as the id, or 'all' in case running the full suite. + +### Custom Test Configuration +It's also possible to run a custom test configuration by passing the -c option. With this option, it then +possible to specify the number of nodes to simulate and the API server qps values for creation, +IPAM allocation and cloud endpoint, along with the allocator name to run. The defaults values for the +qps parmeters are 30 for IPAM allocation, 100 for node creation and 30 for the cloud endpoint, and the +default allocator is the RangeAllocator. + Code Organization ----- The core of the tests are defined in [ipam_test.go](ipam_test.go), using the t.Run() helper to control parallelism diff --git a/test/integration/ipamperf/ipam_test.go b/test/integration/ipamperf/ipam_test.go index 6125f00e6eb..1b82ef8721b 100644 --- a/test/integration/ipamperf/ipam_test.go +++ b/test/integration/ipamperf/ipam_test.go @@ -126,9 +126,13 @@ func TestPerformance(t *testing.T) { tests []*Config ) - for _, numNodes := range []int{10, 100} { - for _, alloc := range []ipam.CIDRAllocatorType{ipam.RangeAllocatorType, ipam.CloudAllocatorType, ipam.IPAMFromClusterAllocatorType, ipam.IPAMFromCloudAllocatorType} { - tests = append(tests, &Config{AllocatorType: alloc, NumNodes: numNodes, CreateQPS: numNodes, KubeQPS: 10, CloudQPS: 10}) + if isCustom { + tests = append(tests, customConfig) + } else { + for _, numNodes := range []int{10, 100} { + for _, alloc := range []ipam.CIDRAllocatorType{ipam.RangeAllocatorType, ipam.CloudAllocatorType, ipam.IPAMFromClusterAllocatorType, ipam.IPAMFromCloudAllocatorType} { + tests = append(tests, &Config{AllocatorType: alloc, NumNodes: numNodes, CreateQPS: numNodes, KubeQPS: 10, CloudQPS: 10}) + } } } diff --git a/test/integration/ipamperf/main_test.go b/test/integration/ipamperf/main_test.go index 6ab7fb68aca..401ad452836 100644 --- a/test/integration/ipamperf/main_test.go +++ b/test/integration/ipamperf/main_test.go @@ -20,13 +20,47 @@ import ( "flag" "testing" + "github.com/golang/glog" + "k8s.io/kubernetes/pkg/controller/nodeipam/ipam" "k8s.io/kubernetes/test/integration/framework" ) -var resultsLogFile string +var ( + resultsLogFile string + isCustom bool + customConfig = &Config{ + NumNodes: 10, + KubeQPS: 30, + CloudQPS: 30, + CreateQPS: 100, + AllocatorType: ipam.RangeAllocatorType, + } +) func TestMain(m *testing.M) { + allocator := string(ipam.RangeAllocatorType) + flag.StringVar(&resultsLogFile, "log", "", "log file to write JSON results to") + flag.BoolVar(&isCustom, "custom", false, "enable custom test configuration") + flag.StringVar(&allocator, "allocator", allocator, "allocator to use") + flag.IntVar(&customConfig.KubeQPS, "kube-qps", customConfig.KubeQPS, "API server qps for allocations") + flag.IntVar(&customConfig.NumNodes, "num-nodes", 10, "number of nodes to simulate") + flag.IntVar(&customConfig.CreateQPS, "create-qps", customConfig.CreateQPS, "API server qps for node creation") + flag.IntVar(&customConfig.CloudQPS, "cloud-qps", customConfig.CloudQPS, "GCE Cloud qps limit") flag.Parse() + + switch allocator { + case string(ipam.RangeAllocatorType): + customConfig.AllocatorType = ipam.RangeAllocatorType + case string(ipam.CloudAllocatorType): + customConfig.AllocatorType = ipam.CloudAllocatorType + case string(ipam.IPAMFromCloudAllocatorType): + customConfig.AllocatorType = ipam.IPAMFromCloudAllocatorType + case string(ipam.IPAMFromClusterAllocatorType): + customConfig.AllocatorType = ipam.IPAMFromClusterAllocatorType + default: + glog.Fatalf("Unknown allocator type: %s", allocator) + } + framework.EtcdMain(m.Run) } diff --git a/test/integration/ipamperf/results.go b/test/integration/ipamperf/results.go index 26452354e8b..c5971e14ddb 100644 --- a/test/integration/ipamperf/results.go +++ b/test/integration/ipamperf/results.go @@ -95,7 +95,7 @@ func NewObserver(clientSet *clientset.Clientset, numNodes int) *Observer { // StartObserving starts an asynchronous loop to monitor for node changes. // Call Results() to get the test results after starting observer. func (o *Observer) StartObserving() error { - go o.monitor() + o.monitor() glog.Infof("Test observer started") return nil } @@ -173,6 +173,9 @@ func (o *Observer) monitor() { nTime.allocated = time.Now() nTime.podCIDR = newNode.Spec.PodCIDR o.numAllocated++ + if o.numAllocated%10 == 0 { + glog.Infof("progress: %d/%d - %.2d%%", o.numAllocated, o.numNodes, (o.numAllocated * 100.0 / o.numNodes)) + } // do following check only if numAllocated is modified, as otherwise, redundant updates // can cause wg.Done() to be called multiple times, causing a panic if o.numAdded == o.numNodes && o.numAllocated == o.numNodes { diff --git a/test/integration/ipamperf/test-performance.sh b/test/integration/ipamperf/test-performance.sh index 766b19b2107..0540978c090 100755 --- a/test/integration/ipamperf/test-performance.sh +++ b/test/integration/ipamperf/test-performance.sh @@ -20,17 +20,25 @@ set -o pipefail TEST_ARGS="" RUN_PATTERN=".*" +PROFILE_OPTS="" function usage() { - echo "usage: $0 [-h] [-d] [-r ] [-o ]" + echo "usage: $0 " echo " -h display this help message" echo " -d enable debug logs in tests" echo " -r regex pattern to match for tests" echo " -o file to write JSON formatted results to" + echo " -p enable cpu and memory profiles, output written to mem-.out and cpu-.out" + echo " -c enable custom test configuration" + echo " -a allocator name, one of RangeAllocator, CloudAllocator, IPAMFromCluster, IPAMFromCloud" + echo " -k api server qps for allocator" + echo " -n number of nodes to simulate" + echo " -m api server qps for node creation" + echo " -l gce cloud endpoint qps" exit 1 } -while getopts ":hdr:o:" opt; do +while getopts ":hdr:o:p:ca:k:n:m:l:" opt; do case ${opt} in d) TEST_ARGS="${TEST_ARGS} -v=6" ;; @@ -38,9 +46,23 @@ while getopts ":hdr:o:" opt; do ;; o) TEST_ARGS="${TEST_ARGS} -log ${OPTARG}" ;; - h) ::usage + p) PROFILE_OPTS="-memprofile mem-${OPTARG}.out -cpuprofile cpu-${OPTARG}.out" ;; - \?) ::usage + c) TEST_ARGS="${TEST_ARGS} -custom" + ;; + a) TEST_ARGS="${TEST_ARGS} -allocator ${OPTARG}" + ;; + k) TEST_ARGS="${TEST_ARGS} -kube-qps ${OPTARG}" + ;; + n) TEST_ARGS="${TEST_ARGS} -num-nodes ${OPTARG}" + ;; + m) TEST_ARGS="${TEST_ARGS} -create-qps ${OPTARG}" + ;; + l) TEST_ARGS="${TEST_ARGS} -cloud-qps ${OPTARG}" + ;; + h) usage + ;; + \?) usage ;; esac done @@ -65,5 +87,5 @@ kube::etcd::start # Running IPAM tests. It might take a long time. kube::log::status "performance test (IPAM) start" -go test -test.run=${RUN_PATTERN} -test.timeout=60m -test.short=false -v -args ${TEST_ARGS} +go test ${PROFILE_OPTS} -test.run=${RUN_PATTERN} -test.timeout=60m -test.short=false -v -args ${TEST_ARGS} kube::log::status "... IPAM tests finished." diff --git a/test/integration/ipamperf/util.go b/test/integration/ipamperf/util.go index d45f21515ee..fa34cd712f1 100644 --- a/test/integration/ipamperf/util.go +++ b/test/integration/ipamperf/util.go @@ -17,8 +17,11 @@ limitations under the License. package ipamperf import ( + "time" + "github.com/golang/glog" "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" @@ -26,6 +29,11 @@ import ( "k8s.io/kubernetes/pkg/api/testapi" ) +const ( + maxCreateRetries = 10 + retryDelay = 10 * time.Second +) + var ( baseNodeTemplate = &v1.Node{ ObjectMeta: metav1.ObjectMeta{ @@ -71,7 +79,17 @@ func createNodes(apiURL string, config *Config) error { }) glog.Infof("Creating %d nodes", config.NumNodes) for i := 0; i < config.NumNodes; i++ { - if _, err := clientSet.CoreV1().Nodes().Create(baseNodeTemplate); err != nil { + var err error + for j := 0; j < maxCreateRetries; j++ { + if _, err = clientSet.CoreV1().Nodes().Create(baseNodeTemplate); err != nil && errors.IsServerTimeout(err) { + glog.Infof("Server timeout creating nodes, retrying after %v", retryDelay) + time.Sleep(retryDelay) + continue + } + break + } + if err != nil { + glog.Errorf("Error creating nodes: %v", err) return err } }