From bb71f9473f9dd9c3802c73e072a146b74f577563 Mon Sep 17 00:00:00 2001
From: gmarek <gmarek@google.com>
Date: Thu, 6 Oct 2016 16:36:58 +0200
Subject: [PATCH] Small update to scheduler benchmark

---
 plugin/pkg/scheduler/generic_scheduler.go     |  2 +-
 .../scheduler_perf/scheduler_test.go          | 48 +++++++++++++++++--
 test/integration/scheduler_perf/util.go       |  2 -
 3 files changed, 44 insertions(+), 8 deletions(-)

diff --git a/plugin/pkg/scheduler/generic_scheduler.go b/plugin/pkg/scheduler/generic_scheduler.go
index fe8172f50fe..169592066f3 100644
--- a/plugin/pkg/scheduler/generic_scheduler.go
+++ b/plugin/pkg/scheduler/generic_scheduler.go
@@ -83,7 +83,7 @@ func (g *genericScheduler) Schedule(pod *api.Pod, nodeLister algorithm.NodeListe
 	} else {
 		trace = util.NewTrace("Scheduling <nil> pod")
 	}
-	defer trace.LogIfLong(20 * time.Millisecond)
+	defer trace.LogIfLong(100 * time.Millisecond)
 
 	nodes, err := nodeLister.List()
 	if err != nil {
diff --git a/test/integration/scheduler_perf/scheduler_test.go b/test/integration/scheduler_perf/scheduler_test.go
index 50b9a9e886a..e8d3081e203 100644
--- a/test/integration/scheduler_perf/scheduler_test.go
+++ b/test/integration/scheduler_perf/scheduler_test.go
@@ -18,16 +18,27 @@ package benchmark
 
 import (
 	"fmt"
+	"math"
 	"testing"
 	"time"
 )
 
+const (
+	threshold3K  = 100
+	threshold30K = 30
+	threshold60K = 30
+)
+
 // TestSchedule100Node3KPods schedules 3k pods on 100 nodes.
 func TestSchedule100Node3KPods(t *testing.T) {
 	if testing.Short() {
 		t.Skip("Skipping because we want to run short tests")
 	}
-	schedulePods(100, 3000)
+	if min := schedulePods(100, 3000); min < threshold3K {
+		t.Errorf("To small pod scheduling throughput for 3k pods. Expected %v got %v", threshold3K, min)
+	} else {
+		fmt.Printf("Minimal observed throughput for 3k pod test: %v\n", min)
+	}
 }
 
 // TestSchedule1000Node30KPods schedules 30k pods on 1000 nodes.
@@ -35,14 +46,32 @@ func TestSchedule1000Node30KPods(t *testing.T) {
 	if testing.Short() {
 		t.Skip("Skipping because we want to run short tests")
 	}
-	schedulePods(1000, 30000)
+	if min := schedulePods(1000, 30000); min < threshold30K {
+		t.Errorf("To small pod scheduling throughput for 30k pods. Expected %v got %v", threshold30K, min)
+	} else {
+		fmt.Printf("Minimal observed throughput for 30k pod test: %v\n", min)
+	}
 }
 
+// TestSchedule2000Node60KPods schedules 60k pods on 2000 nodes.
+// This test won't fit in normal 10 minutes time window.
+// func TestSchedule2000Node60KPods(t *testing.T) {
+// 	if testing.Short() {
+// 		t.Skip("Skipping because we want to run short tests")
+// 	}
+// 	if min := schedulePods(2000, 60000); min < threshold60K {
+// 		t.Errorf("To small pod scheduling throughput for 60k pods. Expected %v got %v", threshold60K, min)
+// 	} else {
+// 		fmt.Printf("Minimal observed throughput for 60k pod test: %v\n", min)
+// 	}
+// }
+
 // schedulePods schedules specific number of pods on specific number of nodes.
 // This is used to learn the scheduling throughput on various
 // sizes of cluster and changes as more and more pods are scheduled.
 // It won't stop until all pods are scheduled.
-func schedulePods(numNodes, numPods int) {
+// It retruns the minimum of throughput over whole run.
+func schedulePods(numNodes, numPods int) int32 {
 	schedulerConfigFactory, destroyFunc := mustSetupScheduler()
 	defer destroyFunc()
 	c := schedulerConfigFactory.Client
@@ -51,16 +80,25 @@ func schedulePods(numNodes, numPods int) {
 	makePodsFromRC(c, "rc1", numPods)
 
 	prev := 0
+	minQps := int32(math.MaxInt32)
 	start := time.Now()
 	for {
 		// This can potentially affect performance of scheduler, since List() is done under mutex.
 		// Listing 10000 pods is an expensive operation, so running it frequently may impact scheduler.
 		// TODO: Setup watch on apiserver and wait until all pods scheduled.
 		scheduled := schedulerConfigFactory.ScheduledPodLister.Indexer.List()
-		fmt.Printf("%ds\trate: %d\ttotal: %d\n", time.Since(start)/time.Second, len(scheduled)-prev, len(scheduled))
 		if len(scheduled) >= numPods {
-			return
+			fmt.Printf("Scheduled %v Pods in %v seconds (%v per second on average).\n",
+				numPods, int(time.Since(start)/time.Second), numPods/int(time.Since(start)/time.Second))
+			return minQps
 		}
+		// There's no point in printing it for the last iteration, as the value is random
+		qps := len(scheduled) - prev
+		if int32(qps) < minQps {
+			minQps = int32(qps)
+		}
+
+		fmt.Printf("%ds\trate: %d\ttotal: %d\n", time.Since(start)/time.Second, qps, len(scheduled))
 		prev = len(scheduled)
 		time.Sleep(1 * time.Second)
 	}
diff --git a/test/integration/scheduler_perf/util.go b/test/integration/scheduler_perf/util.go
index a7041b8b441..fe3673b3d52 100644
--- a/test/integration/scheduler_perf/util.go
+++ b/test/integration/scheduler_perf/util.go
@@ -44,8 +44,6 @@ import (
 // Notes on rate limiter:
 //   - client rate limit is set to 5000.
 func mustSetupScheduler() (schedulerConfigFactory *factory.ConfigFactory, destroyFunc func()) {
-	// framework.DeleteAllEtcdKeys()
-
 	var m *master.Master
 	masterConfig := framework.NewIntegrationTestMasterConfig()
 	m, err := masterConfig.Complete().New()