mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-29 14:37:00 +00:00
Collect some of scheduling metrics and scheduling throughput
In addition to getting overall performance measurements from golang benchmark, collect metrics that provides information about insides of the scheduler itself. This is a first step towards improving what we collect about the scheduler. Metrics in question: - scheduler_scheduling_algorithm_predicate_evaluation_seconds - scheduler_scheduling_algorithm_priority_evaluation_seconds - scheduler_binding_duration_seconds - scheduler_e2e_scheduling_duration_seconds Scheduling throughput is computed on the fly inside perfScheduling.
This commit is contained in:
Jan Chaloupka
parent
c5d75749c8
commit
7b5534021c
@ -13,6 +13,7 @@ go_library(
|
||||
"//staging/src/k8s.io/apimachinery/pkg/version:go_default_library",
|
||||
"//staging/src/k8s.io/component-base/metrics:go_default_library",
|
||||
"//vendor/github.com/prometheus/client_golang/prometheus/testutil:go_default_library",
|
||||
"//vendor/github.com/prometheus/client_model/go:go_default_library",
|
||||
"//vendor/github.com/prometheus/common/expfmt:go_default_library",
|
||||
"//vendor/github.com/prometheus/common/model:go_default_library",
|
||||
],
|
||||
@ -34,7 +35,14 @@ filegroup(
|
||||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = ["testutil_test.go"],
|
||||
srcs = [
|
||||
"metrics_test.go",
|
||||
"testutil_test.go",
|
||||
],
|
||||
embed = [":go_default_library"],
|
||||
deps = ["//staging/src/k8s.io/component-base/metrics:go_default_library"],
|
||||
deps = [
|
||||
"//staging/src/k8s.io/component-base/metrics:go_default_library",
|
||||
"//vendor/github.com/prometheus/client_model/go:go_default_library",
|
||||
"//vendor/k8s.io/utils/pointer:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
@ -19,11 +19,16 @@ package testutil
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
dto "github.com/prometheus/client_model/go"
|
||||
"github.com/prometheus/common/expfmt"
|
||||
"github.com/prometheus/common/model"
|
||||
|
||||
"k8s.io/component-base/metrics"
|
||||
)
|
||||
|
||||
var (
|
||||
@ -178,3 +183,140 @@ func ValidateMetrics(metrics Metrics, metricName string, expectedLabels ...strin
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Histogram wraps prometheus histogram DTO (data transfer object)
|
||||
type Histogram struct {
|
||||
*dto.Histogram
|
||||
}
|
||||
|
||||
// GetHistogramFromGatherer collects a metric from a gatherer implementing k8s.io/component-base/metrics.Gatherer interface.
|
||||
// Used only for testing purposes where we need to gather metrics directly from a running binary (without metrics endpoint).
|
||||
func GetHistogramFromGatherer(gatherer metrics.Gatherer, metricName string) (Histogram, error) {
|
||||
var metricFamily *dto.MetricFamily
|
||||
m, err := gatherer.Gather()
|
||||
if err != nil {
|
||||
return Histogram{}, err
|
||||
}
|
||||
for _, mFamily := range m {
|
||||
if mFamily.Name != nil && *mFamily.Name == metricName {
|
||||
metricFamily = mFamily
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if metricFamily == nil {
|
||||
return Histogram{}, fmt.Errorf("Metric %q not found", metricName)
|
||||
}
|
||||
|
||||
if metricFamily.GetMetric() == nil {
|
||||
return Histogram{}, fmt.Errorf("Metric %q is empty", metricName)
|
||||
}
|
||||
|
||||
if len(metricFamily.GetMetric()) == 0 {
|
||||
return Histogram{}, fmt.Errorf("Metric %q is empty", metricName)
|
||||
}
|
||||
|
||||
return Histogram{
|
||||
// Histograms are stored under the first index (based on observation).
|
||||
// Given there's only one histogram registered per each metric name, accessing
|
||||
// the first index is sufficient.
|
||||
metricFamily.GetMetric()[0].GetHistogram(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func uint64Ptr(u uint64) *uint64 {
|
||||
return &u
|
||||
}
|
||||
|
||||
// Bucket of a histogram
|
||||
type bucket struct {
|
||||
upperBound float64
|
||||
count float64
|
||||
}
|
||||
|
||||
func bucketQuantile(q float64, buckets []bucket) float64 {
|
||||
if q < 0 {
|
||||
return math.Inf(-1)
|
||||
}
|
||||
if q > 1 {
|
||||
return math.Inf(+1)
|
||||
}
|
||||
|
||||
if len(buckets) < 2 {
|
||||
return math.NaN()
|
||||
}
|
||||
|
||||
rank := q * buckets[len(buckets)-1].count
|
||||
b := sort.Search(len(buckets)-1, func(i int) bool { return buckets[i].count >= rank })
|
||||
|
||||
if b == 0 {
|
||||
return buckets[0].upperBound * (rank / buckets[0].count)
|
||||
}
|
||||
|
||||
// linear approximation of b-th bucket
|
||||
brank := rank - buckets[b-1].count
|
||||
bSize := buckets[b].upperBound - buckets[b-1].upperBound
|
||||
bCount := buckets[b].count - buckets[b-1].count
|
||||
|
||||
return buckets[b-1].upperBound + bSize*(brank/bCount)
|
||||
}
|
||||
|
||||
// Quantile computes q-th quantile of a cumulative histogram.
|
||||
// It's expected the histogram is valid (by calling Validate)
|
||||
func (hist *Histogram) Quantile(q float64) float64 {
|
||||
buckets := []bucket{}
|
||||
|
||||
for _, bckt := range hist.Bucket {
|
||||
buckets = append(buckets, bucket{
|
||||
count: float64(*bckt.CumulativeCount),
|
||||
upperBound: *bckt.UpperBound,
|
||||
})
|
||||
}
|
||||
|
||||
// bucketQuantile expects the upper bound of the last bucket to be +inf
|
||||
// buckets[len(buckets)-1].upperBound = math.Inf(+1)
|
||||
|
||||
return bucketQuantile(q, buckets)
|
||||
}
|
||||
|
||||
// Average computes histogram's average value
|
||||
func (hist *Histogram) Average() float64 {
|
||||
return *hist.SampleSum / float64(*hist.SampleCount)
|
||||
}
|
||||
|
||||
// Clear clears all fields of the wrapped histogram
|
||||
func (hist *Histogram) Clear() {
|
||||
if hist.SampleCount != nil {
|
||||
*hist.SampleCount = 0
|
||||
}
|
||||
if hist.SampleSum != nil {
|
||||
*hist.SampleSum = 0
|
||||
}
|
||||
for _, b := range hist.Bucket {
|
||||
if b.CumulativeCount != nil {
|
||||
*b.CumulativeCount = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Validate makes sure the wrapped histogram has all necessary fields set and with valid values.
|
||||
func (hist *Histogram) Validate() error {
|
||||
if hist.SampleCount == nil || *hist.SampleCount == 0 {
|
||||
return fmt.Errorf("nil or empty histogram SampleCount")
|
||||
}
|
||||
|
||||
if hist.SampleSum == nil || *hist.SampleSum == 0 {
|
||||
return fmt.Errorf("nil or empty histogram SampleSum")
|
||||
}
|
||||
|
||||
for _, bckt := range hist.Bucket {
|
||||
if bckt == nil {
|
||||
return fmt.Errorf("empty histogram bucket")
|
||||
}
|
||||
if bckt.UpperBound == nil || *bckt.UpperBound < 0 {
|
||||
return fmt.Errorf("nil or negative histogram bucket UpperBound")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -0,0 +1,255 @@
|
||||
/*
|
||||
Copyright 2020 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package testutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"k8s.io/utils/pointer"
|
||||
|
||||
dto "github.com/prometheus/client_model/go"
|
||||
)
|
||||
|
||||
func samples2Histogram(samples []float64, upperBounds []float64) Histogram {
|
||||
histogram := dto.Histogram{
|
||||
SampleCount: uint64Ptr(0),
|
||||
SampleSum: pointer.Float64Ptr(0.0),
|
||||
}
|
||||
|
||||
for _, ub := range upperBounds {
|
||||
histogram.Bucket = append(histogram.Bucket, &dto.Bucket{
|
||||
CumulativeCount: uint64Ptr(0),
|
||||
UpperBound: pointer.Float64Ptr(ub),
|
||||
})
|
||||
}
|
||||
|
||||
for _, sample := range samples {
|
||||
for i, bucket := range histogram.Bucket {
|
||||
if sample < *bucket.UpperBound {
|
||||
*histogram.Bucket[i].CumulativeCount++
|
||||
}
|
||||
}
|
||||
*histogram.SampleCount++
|
||||
*histogram.SampleSum += sample
|
||||
}
|
||||
return Histogram{
|
||||
&histogram,
|
||||
}
|
||||
}
|
||||
|
||||
func TestHistogramQuantile(t *testing.T) {
|
||||
tests := []struct {
|
||||
samples []float64
|
||||
bounds []float64
|
||||
q50 float64
|
||||
q90 float64
|
||||
q99 float64
|
||||
}{
|
||||
{
|
||||
// repeating numbers
|
||||
samples: []float64{0.5, 0.5, 0.5, 0.5, 1.5, 1.5, 1.5, 1.5, 3, 3, 3, 3, 6, 6, 6, 6},
|
||||
bounds: []float64{1, 2, 4, 8},
|
||||
q50: 2,
|
||||
q90: 6.4,
|
||||
q99: 7.84,
|
||||
},
|
||||
{
|
||||
// random numbers
|
||||
samples: []float64{11, 67, 61, 21, 40, 36, 52, 63, 8, 3, 67, 35, 61, 1, 36, 58},
|
||||
bounds: []float64{10, 20, 40, 80},
|
||||
q50: 40,
|
||||
q90: 72,
|
||||
q99: 79.2,
|
||||
},
|
||||
{
|
||||
// the last bucket is empty
|
||||
samples: []float64{6, 34, 30, 10, 20, 18, 26, 31, 4, 2, 33, 17, 30, 1, 18, 29},
|
||||
bounds: []float64{10, 20, 40, 80},
|
||||
q50: 20,
|
||||
q90: 36,
|
||||
q99: 39.6,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
h := samples2Histogram(test.samples, test.bounds)
|
||||
q50 := h.Quantile(0.5)
|
||||
q90 := h.Quantile(0.9)
|
||||
q99 := h.Quantile(0.99)
|
||||
q999999 := h.Quantile(0.999999)
|
||||
|
||||
if q50 != test.q50 {
|
||||
t.Errorf("Expected q50 to be %v, got %v instead", test.q50, q50)
|
||||
}
|
||||
if q90 != test.q90 {
|
||||
t.Errorf("Expected q90 to be %v, got %v instead", test.q90, q90)
|
||||
}
|
||||
if q99 != test.q99 {
|
||||
t.Errorf("Expected q99 to be %v, got %v instead", test.q99, q99)
|
||||
}
|
||||
lastUpperBound := test.bounds[len(test.bounds)-1]
|
||||
if !(q999999 < lastUpperBound) {
|
||||
t.Errorf("Expected q999999 to be less than %v, got %v instead", lastUpperBound, q999999)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHistogramClear(t *testing.T) {
|
||||
samples := []float64{0.5, 0.5, 0.5, 0.5, 1.5, 1.5, 1.5, 1.5, 3, 3, 3, 3, 6, 6, 6, 6}
|
||||
bounds := []float64{1, 2, 4, 8}
|
||||
h := samples2Histogram(samples, bounds)
|
||||
|
||||
if *h.SampleCount == 0 {
|
||||
t.Errorf("Expected histogram .SampleCount to be non-zero")
|
||||
}
|
||||
if *h.SampleSum == 0 {
|
||||
t.Errorf("Expected histogram .SampleSum to be non-zero")
|
||||
}
|
||||
|
||||
for _, b := range h.Bucket {
|
||||
if b.CumulativeCount != nil {
|
||||
if *b.CumulativeCount == 0 {
|
||||
t.Errorf("Expected histogram bucket to have non-zero comulative count")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
h.Clear()
|
||||
|
||||
if *h.SampleCount != 0 {
|
||||
t.Errorf("Expected histogram .SampleCount to be zero, have %v instead", *h.SampleCount)
|
||||
}
|
||||
|
||||
if *h.SampleSum != 0 {
|
||||
t.Errorf("Expected histogram .SampleSum to be zero, have %v instead", *h.SampleSum)
|
||||
}
|
||||
|
||||
for _, b := range h.Bucket {
|
||||
if b.CumulativeCount != nil {
|
||||
if *b.CumulativeCount != 0 {
|
||||
t.Errorf("Expected histogram bucket to have zero comulative count, have %v instead", *b.CumulativeCount)
|
||||
}
|
||||
}
|
||||
if b.UpperBound != nil {
|
||||
*b.UpperBound = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHistogramValidate(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
h Histogram
|
||||
err error
|
||||
}{
|
||||
{
|
||||
name: "nil SampleCount",
|
||||
h: Histogram{
|
||||
&dto.Histogram{},
|
||||
},
|
||||
err: fmt.Errorf("nil or empty histogram SampleCount"),
|
||||
},
|
||||
{
|
||||
name: "empty SampleCount",
|
||||
h: Histogram{
|
||||
&dto.Histogram{
|
||||
SampleCount: uint64Ptr(0),
|
||||
},
|
||||
},
|
||||
err: fmt.Errorf("nil or empty histogram SampleCount"),
|
||||
},
|
||||
{
|
||||
name: "nil SampleSum",
|
||||
h: Histogram{
|
||||
&dto.Histogram{
|
||||
SampleCount: uint64Ptr(1),
|
||||
},
|
||||
},
|
||||
err: fmt.Errorf("nil or empty histogram SampleSum"),
|
||||
},
|
||||
{
|
||||
name: "empty SampleSum",
|
||||
h: Histogram{
|
||||
&dto.Histogram{
|
||||
SampleCount: uint64Ptr(1),
|
||||
SampleSum: pointer.Float64Ptr(0.0),
|
||||
},
|
||||
},
|
||||
err: fmt.Errorf("nil or empty histogram SampleSum"),
|
||||
},
|
||||
{
|
||||
name: "nil bucket",
|
||||
h: Histogram{
|
||||
&dto.Histogram{
|
||||
SampleCount: uint64Ptr(1),
|
||||
SampleSum: pointer.Float64Ptr(1.0),
|
||||
Bucket: []*dto.Bucket{
|
||||
nil,
|
||||
},
|
||||
},
|
||||
},
|
||||
err: fmt.Errorf("empty histogram bucket"),
|
||||
},
|
||||
{
|
||||
name: "nil bucket UpperBound",
|
||||
h: Histogram{
|
||||
&dto.Histogram{
|
||||
SampleCount: uint64Ptr(1),
|
||||
SampleSum: pointer.Float64Ptr(1.0),
|
||||
Bucket: []*dto.Bucket{
|
||||
{},
|
||||
},
|
||||
},
|
||||
},
|
||||
err: fmt.Errorf("nil or negative histogram bucket UpperBound"),
|
||||
},
|
||||
{
|
||||
name: "negative bucket UpperBound",
|
||||
h: Histogram{
|
||||
&dto.Histogram{
|
||||
SampleCount: uint64Ptr(1),
|
||||
SampleSum: pointer.Float64Ptr(1.0),
|
||||
Bucket: []*dto.Bucket{
|
||||
{UpperBound: pointer.Float64Ptr(-1.0)},
|
||||
},
|
||||
},
|
||||
},
|
||||
err: fmt.Errorf("nil or negative histogram bucket UpperBound"),
|
||||
},
|
||||
{
|
||||
name: "valid histogram",
|
||||
h: samples2Histogram(
|
||||
[]float64{0.5, 0.5, 0.5, 0.5, 1.5, 1.5, 1.5, 1.5, 3, 3, 3, 3, 6, 6, 6, 6},
|
||||
[]float64{1, 2, 4, 8},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
err := test.h.Validate()
|
||||
if test.err != nil {
|
||||
if err == nil || err.Error() != test.err.Error() {
|
||||
t.Errorf("Expected %q error, got %q instead", test.err, err)
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
t.Errorf("Expected error to be nil, got %q instead", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -20,7 +20,10 @@ go_library(
|
||||
"//staging/src/k8s.io/client-go/informers/core/v1:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/rest:go_default_library",
|
||||
"//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library",
|
||||
"//staging/src/k8s.io/component-base/metrics/testutil:go_default_library",
|
||||
"//test/integration/util:go_default_library",
|
||||
"//vendor/k8s.io/klog:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@ -38,6 +38,15 @@ const (
|
||||
configFile = "config/performance-config.yaml"
|
||||
)
|
||||
|
||||
var (
|
||||
defaultMetrics = []string{
|
||||
"scheduler_scheduling_algorithm_predicate_evaluation_seconds",
|
||||
"scheduler_scheduling_algorithm_priority_evaluation_seconds",
|
||||
"scheduler_binding_duration_seconds",
|
||||
"scheduler_e2e_scheduling_duration_seconds",
|
||||
}
|
||||
)
|
||||
|
||||
// testCase configures a test case to run the scheduler performance test. Users should be able to
|
||||
// provide this via a YAML file.
|
||||
//
|
||||
@ -92,6 +101,7 @@ type testParams struct {
|
||||
}
|
||||
|
||||
func BenchmarkPerfScheduling(b *testing.B) {
|
||||
dataItems := DataItems{Version: "v1"}
|
||||
tests := getSimpleTestCases(configFile)
|
||||
|
||||
for _, test := range tests {
|
||||
@ -100,12 +110,15 @@ func BenchmarkPerfScheduling(b *testing.B) {
|
||||
for feature, flag := range test.FeatureGates {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, feature, flag)()
|
||||
}
|
||||
perfScheduling(test, b)
|
||||
dataItems.DataItems = append(dataItems.DataItems, perfScheduling(test, b)...)
|
||||
})
|
||||
}
|
||||
if err := dataItems2JSONFile(dataItems, b.Name()); err != nil {
|
||||
klog.Fatalf("%v: unable to write measured data: %v", b.Name(), err)
|
||||
}
|
||||
}
|
||||
|
||||
func perfScheduling(test testCase, b *testing.B) {
|
||||
func perfScheduling(test testCase, b *testing.B) []DataItem {
|
||||
var nodeStrategy testutils.PrepareNodeStrategy = &testutils.TrivialNodePrepareStrategy{}
|
||||
if test.Nodes.NodeAllocatableStrategy != nil {
|
||||
nodeStrategy = test.Nodes.NodeAllocatableStrategy
|
||||
@ -180,15 +193,45 @@ func perfScheduling(test testCase, b *testing.B) {
|
||||
|
||||
// start benchmark
|
||||
b.ResetTimer()
|
||||
|
||||
// Start measuring throughput
|
||||
stopCh := make(chan struct{})
|
||||
throughputCollector := newThroughputCollector(podInformer)
|
||||
go throughputCollector.run(stopCh)
|
||||
|
||||
// Scheduling the main workload
|
||||
config = testutils.NewTestPodCreatorConfig()
|
||||
config.AddStrategy(testNamespace, test.PodsToSchedule.Num, testPodStrategy)
|
||||
podCreator = testutils.NewTestPodCreator(clientset, config)
|
||||
podCreator.CreatePods()
|
||||
|
||||
<-completedCh
|
||||
close(stopCh)
|
||||
|
||||
// Note: without this line we're taking the overhead of defer() into account.
|
||||
b.StopTimer()
|
||||
|
||||
setNameLabel := func(dataItem *DataItem) DataItem {
|
||||
if dataItem.Labels == nil {
|
||||
dataItem.Labels = map[string]string{}
|
||||
}
|
||||
dataItem.Labels["Name"] = b.Name()
|
||||
return *dataItem
|
||||
}
|
||||
|
||||
dataItems := []DataItem{
|
||||
setNameLabel(throughputCollector.collect()),
|
||||
}
|
||||
|
||||
for _, metric := range defaultMetrics {
|
||||
dataItem := newMetricsCollector(metric).collect()
|
||||
if dataItem == nil {
|
||||
continue
|
||||
}
|
||||
dataItems = append(dataItems, setNameLabel(dataItem))
|
||||
}
|
||||
|
||||
return dataItems
|
||||
}
|
||||
|
||||
func getPodStrategy(pc podCase) testutils.TestPodCreateStrategy {
|
||||
|
||||
@ -17,15 +17,34 @@ limitations under the License.
|
||||
package benchmark
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"path"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
coreinformers "k8s.io/client-go/informers/core/v1"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
restclient "k8s.io/client-go/rest"
|
||||
"k8s.io/component-base/metrics/legacyregistry"
|
||||
"k8s.io/component-base/metrics/testutil"
|
||||
"k8s.io/klog"
|
||||
"k8s.io/kubernetes/test/integration/util"
|
||||
)
|
||||
|
||||
const (
|
||||
dateFormat = "2006-01-02T15:04:05Z"
|
||||
throughputSampleFrequency = time.Second
|
||||
)
|
||||
|
||||
var dataItemsDir = flag.String("data-items-dir", "", "destination directory for storing generated data items for perf dashboard")
|
||||
|
||||
// mustSetupScheduler starts the following components:
|
||||
// - k8s api server (a.k.a. master)
|
||||
// - scheduler
|
||||
@ -66,3 +85,145 @@ func getScheduledPods(podInformer coreinformers.PodInformer) ([]*v1.Pod, error)
|
||||
}
|
||||
return scheduled, nil
|
||||
}
|
||||
|
||||
// DataItem is the data point.
|
||||
type DataItem struct {
|
||||
// Data is a map from bucket to real data point (e.g. "Perc90" -> 23.5). Notice
|
||||
// that all data items with the same label combination should have the same buckets.
|
||||
Data map[string]float64 `json:"data"`
|
||||
// Unit is the data unit. Notice that all data items with the same label combination
|
||||
// should have the same unit.
|
||||
Unit string `json:"unit"`
|
||||
// Labels is the labels of the data item.
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
}
|
||||
|
||||
// DataItems is the data point set. It is the struct that perf dashboard expects.
|
||||
type DataItems struct {
|
||||
Version string `json:"version"`
|
||||
DataItems []DataItem `json:"dataItems"`
|
||||
}
|
||||
|
||||
func dataItems2JSONFile(dataItems DataItems, namePrefix string) error {
|
||||
b, err := json.Marshal(dataItems)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
destFile := fmt.Sprintf("%v_%v.json", namePrefix, time.Now().Format(dateFormat))
|
||||
if *dataItemsDir != "" {
|
||||
destFile = path.Join(*dataItemsDir, destFile)
|
||||
}
|
||||
|
||||
return ioutil.WriteFile(destFile, b, 0644)
|
||||
}
|
||||
|
||||
// metricsCollector collects metrics from legacyregistry.DefaultGatherer.Gather() endpoint.
|
||||
// Currently only Histrogram metrics are supported.
|
||||
type metricsCollector struct {
|
||||
metric string
|
||||
}
|
||||
|
||||
func newMetricsCollector(metric string) *metricsCollector {
|
||||
return &metricsCollector{
|
||||
metric: metric,
|
||||
}
|
||||
}
|
||||
|
||||
func (pc *metricsCollector) collect() *DataItem {
|
||||
hist, err := testutil.GetHistogramFromGatherer(legacyregistry.DefaultGatherer, pc.metric)
|
||||
if err != nil {
|
||||
klog.Error(err)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := hist.Validate(); err != nil {
|
||||
klog.Error(err)
|
||||
return nil
|
||||
}
|
||||
|
||||
q50 := hist.Quantile(0.50)
|
||||
q90 := hist.Quantile(0.90)
|
||||
q99 := hist.Quantile(0.95)
|
||||
avg := hist.Average()
|
||||
|
||||
// clear the metrics so that next test always starts with empty prometheus
|
||||
// metrics (since the metrics are shared among all tests run inside the same binary)
|
||||
hist.Clear()
|
||||
|
||||
msFactor := float64(time.Second) / float64(time.Millisecond)
|
||||
|
||||
return &DataItem{
|
||||
Labels: map[string]string{
|
||||
"Metric": pc.metric,
|
||||
},
|
||||
Data: map[string]float64{
|
||||
"Perc50": q50 * msFactor,
|
||||
"Perc90": q90 * msFactor,
|
||||
"Perc99": q99 * msFactor,
|
||||
"Average": avg * msFactor,
|
||||
},
|
||||
Unit: "ms",
|
||||
}
|
||||
}
|
||||
|
||||
type throughputCollector struct {
|
||||
podInformer coreinformers.PodInformer
|
||||
schedulingThroughputs []float64
|
||||
}
|
||||
|
||||
func newThroughputCollector(podInformer coreinformers.PodInformer) *throughputCollector {
|
||||
return &throughputCollector{
|
||||
podInformer: podInformer,
|
||||
}
|
||||
}
|
||||
|
||||
func (tc *throughputCollector) run(stopCh chan struct{}) {
|
||||
podsScheduled, err := getScheduledPods(tc.podInformer)
|
||||
if err != nil {
|
||||
klog.Fatalf("%v", err)
|
||||
}
|
||||
lastScheduledCount := len(podsScheduled)
|
||||
for {
|
||||
select {
|
||||
case <-stopCh:
|
||||
return
|
||||
case <-time.After(throughputSampleFrequency):
|
||||
podsScheduled, err := getScheduledPods(tc.podInformer)
|
||||
if err != nil {
|
||||
klog.Fatalf("%v", err)
|
||||
}
|
||||
|
||||
scheduled := len(podsScheduled)
|
||||
samplingRatioSeconds := float64(throughputSampleFrequency) / float64(time.Second)
|
||||
throughput := float64(scheduled-lastScheduledCount) / samplingRatioSeconds
|
||||
tc.schedulingThroughputs = append(tc.schedulingThroughputs, throughput)
|
||||
lastScheduledCount = scheduled
|
||||
|
||||
klog.Infof("%d pods scheduled", lastScheduledCount)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (tc *throughputCollector) collect() *DataItem {
|
||||
throughputSummary := &DataItem{}
|
||||
if length := len(tc.schedulingThroughputs); length > 0 {
|
||||
sort.Float64s(tc.schedulingThroughputs)
|
||||
sum := 0.0
|
||||
for i := range tc.schedulingThroughputs {
|
||||
sum += tc.schedulingThroughputs[i]
|
||||
}
|
||||
|
||||
throughputSummary.Labels = map[string]string{
|
||||
"Metric": "SchedulingThroughput",
|
||||
}
|
||||
throughputSummary.Data = map[string]float64{
|
||||
"Average": sum / float64(length),
|
||||
"Perc50": tc.schedulingThroughputs[int(math.Ceil(float64(length*50)/100))-1],
|
||||
"Perc90": tc.schedulingThroughputs[int(math.Ceil(float64(length*90)/100))-1],
|
||||
"Perc99": tc.schedulingThroughputs[int(math.Ceil(float64(length*99)/100))-1],
|
||||
}
|
||||
throughputSummary.Unit = "pods/s"
|
||||
}
|
||||
return throughputSummary
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user