Merge pull request #64521 from shyamjvs/compute-scheduler-throughput-avg

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Compute avg and quantiles of scheduler throughput in density test

Based on my comment here - https://github.com/kubernetes/kubernetes/pull/64266#issuecomment-393189953

/sig scheduling
/kind cleanup
/priority important-soon
/milestone v1.11
/cc @wojtek-t 

```release-note
NONE
```

test/e2e/framework/metrics_util.go

@@ -213,7 +213,10 @@ type SchedulingMetrics struct {
 	SchedulingLatency LatencyMetric `json:"schedulingLatency"`
 	BindingLatency    LatencyMetric `json:"bindingLatency"`
 	E2ELatency        LatencyMetric `json:"e2eLatency"`
-	ThroughputSamples []float64     `json:"throughputSamples"`
+	ThroughputAverage float64       `json:"throughputAverage"`
+	ThroughputPerc50  float64       `json:"throughputPerc50"`
+	ThroughputPerc90  float64       `json:"throughputPerc90"`
+	ThroughputPerc99  float64       `json:"throughputPerc99"`
 }
 
 func (l *SchedulingMetrics) SummaryKind() string {

test/e2e/scalability/density.go

@@ -222,6 +222,24 @@ func density30AddonResourceVerifier(numNodes int) map[string]framework.ResourceC
 	return constraints
 }
 
+func computeAverage(sample []float64) float64 {
+	sum := 0.0
+	for _, value := range sample {
+		sum += value
+	}
+	return sum / float64(len(sample))
+}
+
+func computeQuantile(sample []float64, quantile float64) float64 {
+	Expect(sort.Float64sAreSorted(sample)).To(Equal(true))
+	Expect(quantile >= 0.0 && quantile <= 1.0).To(Equal(true))
+	index := int(quantile*float64(len(sample))) - 1
+	if index < 0 {
+		return math.NaN()
+	}
+	return sample[index]
+}
+
 func logPodStartupStatus(
 	c clientset.Interface,
 	expectedPods int,
@@ -398,7 +416,16 @@ var _ = SIGDescribe("Density", func() {
 		latency, err := framework.VerifySchedulerLatency(c)
 		framework.ExpectNoError(err)
 		if err == nil {
-			latency.ThroughputSamples = scheduleThroughputs
+			// Compute avg and quantiles of throughput (excluding last element, that's usually an outlier).
+			sampleSize := len(scheduleThroughputs)
+			if sampleSize > 1 {
+				scheduleThroughputs = scheduleThroughputs[:sampleSize-1]
+				sort.Float64s(scheduleThroughputs)
+				latency.ThroughputAverage = computeAverage(scheduleThroughputs)
+				latency.ThroughputPerc50 = computeQuantile(scheduleThroughputs, 0.5)
+				latency.ThroughputPerc90 = computeQuantile(scheduleThroughputs, 0.9)
+				latency.ThroughputPerc99 = computeQuantile(scheduleThroughputs, 0.99)
+			}
 			summaries = append(summaries, latency)
 		}
 		summaries = append(summaries, testPhaseDurations)