mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 04:06:03 +00:00
Merge pull request #87923 from ingvagabund/move-direct-prometheus-metrics-under-component-base-metrics
Collect some of scheduling metrics and scheduling throughput (vol. 2)
This commit is contained in:
Kubernetes Prow Robot
GitHub
commit
a53b25ae92
@ -13,6 +13,7 @@ go_library(
|
||||
"//staging/src/k8s.io/apimachinery/pkg/version:go_default_library",
|
||||
"//staging/src/k8s.io/component-base/metrics:go_default_library",
|
||||
"//vendor/github.com/prometheus/client_golang/prometheus/testutil:go_default_library",
|
||||
"//vendor/github.com/prometheus/client_model/go:go_default_library",
|
||||
"//vendor/github.com/prometheus/common/expfmt:go_default_library",
|
||||
"//vendor/github.com/prometheus/common/model:go_default_library",
|
||||
],
|
||||
@ -34,7 +35,14 @@ filegroup(
|
||||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = ["testutil_test.go"],
|
||||
srcs = [
|
||||
"metrics_test.go",
|
||||
"testutil_test.go",
|
||||
],
|
||||
embed = [":go_default_library"],
|
||||
deps = ["//staging/src/k8s.io/component-base/metrics:go_default_library"],
|
||||
deps = [
|
||||
"//staging/src/k8s.io/component-base/metrics:go_default_library",
|
||||
"//vendor/github.com/prometheus/client_model/go:go_default_library",
|
||||
"//vendor/k8s.io/utils/pointer:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
@ -19,11 +19,16 @@ package testutil
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
dto "github.com/prometheus/client_model/go"
|
||||
"github.com/prometheus/common/expfmt"
|
||||
"github.com/prometheus/common/model"
|
||||
|
||||
"k8s.io/component-base/metrics"
|
||||
)
|
||||
|
||||
var (
|
||||
@ -178,3 +183,140 @@ func ValidateMetrics(metrics Metrics, metricName string, expectedLabels ...strin
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Histogram wraps prometheus histogram DTO (data transfer object)
|
||||
type Histogram struct {
|
||||
*dto.Histogram
|
||||
}
|
||||
|
||||
// GetHistogramFromGatherer collects a metric from a gatherer implementing k8s.io/component-base/metrics.Gatherer interface.
|
||||
// Used only for testing purposes where we need to gather metrics directly from a running binary (without metrics endpoint).
|
||||
func GetHistogramFromGatherer(gatherer metrics.Gatherer, metricName string) (Histogram, error) {
|
||||
var metricFamily *dto.MetricFamily
|
||||
m, err := gatherer.Gather()
|
||||
if err != nil {
|
||||
return Histogram{}, err
|
||||
}
|
||||
for _, mFamily := range m {
|
||||
if mFamily.Name != nil && *mFamily.Name == metricName {
|
||||
metricFamily = mFamily
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if metricFamily == nil {
|
||||
return Histogram{}, fmt.Errorf("Metric %q not found", metricName)
|
||||
}
|
||||
|
||||
if metricFamily.GetMetric() == nil {
|
||||
return Histogram{}, fmt.Errorf("Metric %q is empty", metricName)
|
||||
}
|
||||
|
||||
if len(metricFamily.GetMetric()) == 0 {
|
||||
return Histogram{}, fmt.Errorf("Metric %q is empty", metricName)
|
||||
}
|
||||
|
||||
return Histogram{
|
||||
// Histograms are stored under the first index (based on observation).
|
||||
// Given there's only one histogram registered per each metric name, accessing
|
||||
// the first index is sufficient.
|
||||
metricFamily.GetMetric()[0].GetHistogram(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func uint64Ptr(u uint64) *uint64 {
|
||||
return &u
|
||||
}
|
||||
|
||||
// Bucket of a histogram
|
||||
type bucket struct {
|
||||
upperBound float64
|
||||
count float64
|
||||
}
|
||||
|
||||
func bucketQuantile(q float64, buckets []bucket) float64 {
|
||||
if q < 0 {
|
||||
return math.Inf(-1)
|
||||
}
|
||||
if q > 1 {
|
||||
return math.Inf(+1)
|
||||
}
|
||||
|
||||
if len(buckets) < 2 {
|
||||
return math.NaN()
|
||||
}
|
||||
|
||||
rank := q * buckets[len(buckets)-1].count
|
||||
b := sort.Search(len(buckets)-1, func(i int) bool { return buckets[i].count >= rank })
|
||||
|
||||
if b == 0 {
|
||||
return buckets[0].upperBound * (rank / buckets[0].count)
|
||||
}
|
||||
|
||||
// linear approximation of b-th bucket
|
||||
brank := rank - buckets[b-1].count
|
||||
bSize := buckets[b].upperBound - buckets[b-1].upperBound
|
||||
bCount := buckets[b].count - buckets[b-1].count
|
||||
|
||||
return buckets[b-1].upperBound + bSize*(brank/bCount)
|
||||
}
|
||||
|
||||
// Quantile computes q-th quantile of a cumulative histogram.
|
||||
// It's expected the histogram is valid (by calling Validate)
|
||||
func (hist *Histogram) Quantile(q float64) float64 {
|
||||
buckets := []bucket{}
|
||||
|
||||
for _, bckt := range hist.Bucket {
|
||||
buckets = append(buckets, bucket{
|
||||
count: float64(*bckt.CumulativeCount),
|
||||
upperBound: *bckt.UpperBound,
|
||||
})
|
||||
}
|
||||
|
||||
// bucketQuantile expects the upper bound of the last bucket to be +inf
|
||||
// buckets[len(buckets)-1].upperBound = math.Inf(+1)
|
||||
|
||||
return bucketQuantile(q, buckets)
|
||||
}
|
||||
|
||||
// Average computes histogram's average value
|
||||
func (hist *Histogram) Average() float64 {
|
||||
return *hist.SampleSum / float64(*hist.SampleCount)
|
||||
}
|
||||
|
||||
// Clear clears all fields of the wrapped histogram
|
||||
func (hist *Histogram) Clear() {
|
||||
if hist.SampleCount != nil {
|
||||
*hist.SampleCount = 0
|
||||
}
|
||||
if hist.SampleSum != nil {
|
||||
*hist.SampleSum = 0
|
||||
}
|
||||
for _, b := range hist.Bucket {
|
||||
if b.CumulativeCount != nil {
|
||||
*b.CumulativeCount = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Validate makes sure the wrapped histogram has all necessary fields set and with valid values.
|
||||
func (hist *Histogram) Validate() error {
|
||||
if hist.SampleCount == nil || *hist.SampleCount == 0 {
|
||||
return fmt.Errorf("nil or empty histogram SampleCount")
|
||||
}
|
||||
|
||||
if hist.SampleSum == nil || *hist.SampleSum == 0 {
|
||||
return fmt.Errorf("nil or empty histogram SampleSum")
|
||||
}
|
||||
|
||||
for _, bckt := range hist.Bucket {
|
||||
if bckt == nil {
|
||||
return fmt.Errorf("empty histogram bucket")
|
||||
}
|
||||
if bckt.UpperBound == nil || *bckt.UpperBound < 0 {
|
||||
return fmt.Errorf("nil or negative histogram bucket UpperBound")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -0,0 +1,255 @@
|
||||
/*
|
||||
Copyright 2020 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package testutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"k8s.io/utils/pointer"
|
||||
|
||||
dto "github.com/prometheus/client_model/go"
|
||||
)
|
||||
|
||||
func samples2Histogram(samples []float64, upperBounds []float64) Histogram {
|
||||
histogram := dto.Histogram{
|
||||
SampleCount: uint64Ptr(0),
|
||||
SampleSum: pointer.Float64Ptr(0.0),
|
||||
}
|
||||
|
||||
for _, ub := range upperBounds {
|
||||
histogram.Bucket = append(histogram.Bucket, &dto.Bucket{
|
||||
CumulativeCount: uint64Ptr(0),
|
||||
UpperBound: pointer.Float64Ptr(ub),
|
||||
})
|
||||
}
|
||||
|
||||
for _, sample := range samples {
|
||||
for i, bucket := range histogram.Bucket {
|
||||
if sample < *bucket.UpperBound {
|
||||
*histogram.Bucket[i].CumulativeCount++
|
||||
}
|
||||
}
|
||||
*histogram.SampleCount++
|
||||
*histogram.SampleSum += sample
|
||||
}
|
||||
return Histogram{
|
||||
&histogram,
|
||||
}
|
||||
}
|
||||
|
||||
func TestHistogramQuantile(t *testing.T) {
|
||||
tests := []struct {
|
||||
samples []float64
|
||||
bounds []float64
|
||||
q50 float64
|
||||
q90 float64
|
||||
q99 float64
|
||||
}{
|
||||
{
|
||||
// repeating numbers
|
||||
samples: []float64{0.5, 0.5, 0.5, 0.5, 1.5, 1.5, 1.5, 1.5, 3, 3, 3, 3, 6, 6, 6, 6},
|
||||
bounds: []float64{1, 2, 4, 8},
|
||||
q50: 2,
|
||||
q90: 6.4,
|
||||
q99: 7.84,
|
||||
},
|
||||
{
|
||||
// random numbers
|
||||
samples: []float64{11, 67, 61, 21, 40, 36, 52, 63, 8, 3, 67, 35, 61, 1, 36, 58},
|
||||
bounds: []float64{10, 20, 40, 80},
|
||||
q50: 40,
|
||||
q90: 72,
|
||||
q99: 79.2,
|
||||
},
|
||||
{
|
||||
// the last bucket is empty
|
||||
samples: []float64{6, 34, 30, 10, 20, 18, 26, 31, 4, 2, 33, 17, 30, 1, 18, 29},
|
||||
bounds: []float64{10, 20, 40, 80},
|
||||
q50: 20,
|
||||
q90: 36,
|
||||
q99: 39.6,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
h := samples2Histogram(test.samples, test.bounds)
|
||||
q50 := h.Quantile(0.5)
|
||||
q90 := h.Quantile(0.9)
|
||||
q99 := h.Quantile(0.99)
|
||||
q999999 := h.Quantile(0.999999)
|
||||
|
||||
if q50 != test.q50 {
|
||||
t.Errorf("Expected q50 to be %v, got %v instead", test.q50, q50)
|
||||
}
|
||||
if q90 != test.q90 {
|
||||
t.Errorf("Expected q90 to be %v, got %v instead", test.q90, q90)
|
||||
}
|
||||
if q99 != test.q99 {
|
||||
t.Errorf("Expected q99 to be %v, got %v instead", test.q99, q99)
|
||||
}
|
||||
lastUpperBound := test.bounds[len(test.bounds)-1]
|
||||
if !(q999999 < lastUpperBound) {
|
||||
t.Errorf("Expected q999999 to be less than %v, got %v instead", lastUpperBound, q999999)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHistogramClear(t *testing.T) {
|
||||
samples := []float64{0.5, 0.5, 0.5, 0.5, 1.5, 1.5, 1.5, 1.5, 3, 3, 3, 3, 6, 6, 6, 6}
|
||||
bounds := []float64{1, 2, 4, 8}
|
||||
h := samples2Histogram(samples, bounds)
|
||||
|
||||
if *h.SampleCount == 0 {
|
||||
t.Errorf("Expected histogram .SampleCount to be non-zero")
|
||||
}
|
||||
if *h.SampleSum == 0 {
|
||||
t.Errorf("Expected histogram .SampleSum to be non-zero")
|
||||
}
|
||||
|
||||
for _, b := range h.Bucket {
|
||||
if b.CumulativeCount != nil {
|
||||
if *b.CumulativeCount == 0 {
|
||||
t.Errorf("Expected histogram bucket to have non-zero comulative count")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
h.Clear()
|
||||
|
||||
if *h.SampleCount != 0 {
|
||||
t.Errorf("Expected histogram .SampleCount to be zero, have %v instead", *h.SampleCount)
|
||||
}
|
||||
|
||||
if *h.SampleSum != 0 {
|
||||
t.Errorf("Expected histogram .SampleSum to be zero, have %v instead", *h.SampleSum)
|
||||
}
|
||||
|
||||
for _, b := range h.Bucket {
|
||||
if b.CumulativeCount != nil {
|
||||
if *b.CumulativeCount != 0 {
|
||||
t.Errorf("Expected histogram bucket to have zero comulative count, have %v instead", *b.CumulativeCount)
|
||||
}
|
||||
}
|
||||
if b.UpperBound != nil {
|
||||
*b.UpperBound = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHistogramValidate(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
h Histogram
|
||||
err error
|
||||
}{
|
||||
{
|
||||
name: "nil SampleCount",
|
||||
h: Histogram{
|
||||
&dto.Histogram{},
|
||||
},
|
||||
err: fmt.Errorf("nil or empty histogram SampleCount"),
|
||||
},
|
||||
{
|
||||
name: "empty SampleCount",
|
||||
h: Histogram{
|
||||
&dto.Histogram{
|
||||
SampleCount: uint64Ptr(0),
|
||||
},
|
||||
},
|
||||
err: fmt.Errorf("nil or empty histogram SampleCount"),
|
||||
},
|
||||
{
|
||||
name: "nil SampleSum",
|
||||
h: Histogram{
|
||||
&dto.Histogram{
|
||||
SampleCount: uint64Ptr(1),
|
||||
},
|
||||
},
|
||||
err: fmt.Errorf("nil or empty histogram SampleSum"),
|
||||
},
|
||||
{
|
||||
name: "empty SampleSum",
|
||||
h: Histogram{
|
||||
&dto.Histogram{
|
||||
SampleCount: uint64Ptr(1),
|
||||
SampleSum: pointer.Float64Ptr(0.0),
|
||||
},
|
||||
},
|
||||
err: fmt.Errorf("nil or empty histogram SampleSum"),
|
||||
},
|
||||
{
|
||||
name: "nil bucket",
|
||||
h: Histogram{
|
||||
&dto.Histogram{
|
||||
SampleCount: uint64Ptr(1),
|
||||
SampleSum: pointer.Float64Ptr(1.0),
|
||||
Bucket: []*dto.Bucket{
|
||||
nil,
|
||||
},
|
||||
},
|
||||
},
|
||||
err: fmt.Errorf("empty histogram bucket"),
|
||||
},
|
||||
{
|
||||
name: "nil bucket UpperBound",
|
||||
h: Histogram{
|
||||
&dto.Histogram{
|
||||
SampleCount: uint64Ptr(1),
|
||||
SampleSum: pointer.Float64Ptr(1.0),
|
||||
Bucket: []*dto.Bucket{
|
||||
{},
|
||||
},
|
||||
},
|
||||
},
|
||||
err: fmt.Errorf("nil or negative histogram bucket UpperBound"),
|
||||
},
|
||||
{
|
||||
name: "negative bucket UpperBound",
|
||||
h: Histogram{
|
||||
&dto.Histogram{
|
||||
SampleCount: uint64Ptr(1),
|
||||
SampleSum: pointer.Float64Ptr(1.0),
|
||||
Bucket: []*dto.Bucket{
|
||||
{UpperBound: pointer.Float64Ptr(-1.0)},
|
||||
},
|
||||
},
|
||||
},
|
||||
err: fmt.Errorf("nil or negative histogram bucket UpperBound"),
|
||||
},
|
||||
{
|
||||
name: "valid histogram",
|
||||
h: samples2Histogram(
|
||||
[]float64{0.5, 0.5, 0.5, 0.5, 1.5, 1.5, 1.5, 1.5, 3, 3, 3, 3, 6, 6, 6, 6},
|
||||
[]float64{1, 2, 4, 8},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
err := test.h.Validate()
|
||||
if test.err != nil {
|
||||
if err == nil || err.Error() != test.err.Error() {
|
||||
t.Errorf("Expected %q error, got %q instead", test.err, err)
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
t.Errorf("Expected error to be nil, got %q instead", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -20,7 +20,10 @@ go_library(
|
||||
"//staging/src/k8s.io/client-go/informers/core/v1:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/rest:go_default_library",
|
||||
"//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library",
|
||||
"//staging/src/k8s.io/component-base/metrics/testutil:go_default_library",
|
||||
"//test/integration/util:go_default_library",
|
||||
"//vendor/k8s.io/klog:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@ -38,6 +38,15 @@ const (
|
||||
configFile = "config/performance-config.yaml"
|
||||
)
|
||||
|
||||
var (
|
||||
defaultMetrics = []string{
|
||||
"scheduler_scheduling_algorithm_predicate_evaluation_seconds",
|
||||
"scheduler_scheduling_algorithm_priority_evaluation_seconds",
|
||||
"scheduler_binding_duration_seconds",
|
||||
"scheduler_e2e_scheduling_duration_seconds",
|
||||
}
|
||||
)
|
||||
|
||||
// testCase configures a test case to run the scheduler performance test. Users should be able to
|
||||
// provide this via a YAML file.
|
||||
//
|
||||
@ -92,6 +101,7 @@ type testParams struct {
|
||||
}
|
||||
|
||||
func BenchmarkPerfScheduling(b *testing.B) {
|
||||
dataItems := DataItems{Version: "v1"}
|
||||
tests := getSimpleTestCases(configFile)
|
||||
|
||||
for _, test := range tests {
|
||||
@ -100,12 +110,15 @@ func BenchmarkPerfScheduling(b *testing.B) {
|
||||
for feature, flag := range test.FeatureGates {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, feature, flag)()
|
||||
}
|
||||
perfScheduling(test, b)
|
||||
dataItems.DataItems = append(dataItems.DataItems, perfScheduling(test, b)...)
|
||||
})
|
||||
}
|
||||
if err := dataItems2JSONFile(dataItems, b.Name()); err != nil {
|
||||
klog.Fatalf("%v: unable to write measured data: %v", b.Name(), err)
|
||||
}
|
||||
}
|
||||
|
||||
func perfScheduling(test testCase, b *testing.B) {
|
||||
func perfScheduling(test testCase, b *testing.B) []DataItem {
|
||||
var nodeStrategy testutils.PrepareNodeStrategy = &testutils.TrivialNodePrepareStrategy{}
|
||||
if test.Nodes.NodeAllocatableStrategy != nil {
|
||||
nodeStrategy = test.Nodes.NodeAllocatableStrategy
|
||||
@ -180,15 +193,45 @@ func perfScheduling(test testCase, b *testing.B) {
|
||||
|
||||
// start benchmark
|
||||
b.ResetTimer()
|
||||
|
||||
// Start measuring throughput
|
||||
stopCh := make(chan struct{})
|
||||
throughputCollector := newThroughputCollector(podInformer)
|
||||
go throughputCollector.run(stopCh)
|
||||
|
||||
// Scheduling the main workload
|
||||
config = testutils.NewTestPodCreatorConfig()
|
||||
config.AddStrategy(testNamespace, test.PodsToSchedule.Num, testPodStrategy)
|
||||
podCreator = testutils.NewTestPodCreator(clientset, config)
|
||||
podCreator.CreatePods()
|
||||
|
||||
<-completedCh
|
||||
close(stopCh)
|
||||
|
||||
// Note: without this line we're taking the overhead of defer() into account.
|
||||
b.StopTimer()
|
||||
|
||||
setNameLabel := func(dataItem *DataItem) DataItem {
|
||||
if dataItem.Labels == nil {
|
||||
dataItem.Labels = map[string]string{}
|
||||
}
|
||||
dataItem.Labels["Name"] = b.Name()
|
||||
return *dataItem
|
||||
}
|
||||
|
||||
dataItems := []DataItem{
|
||||
setNameLabel(throughputCollector.collect()),
|
||||
}
|
||||
|
||||
for _, metric := range defaultMetrics {
|
||||
dataItem := newMetricsCollector(metric).collect()
|
||||
if dataItem == nil {
|
||||
continue
|
||||
}
|
||||
dataItems = append(dataItems, setNameLabel(dataItem))
|
||||
}
|
||||
|
||||
return dataItems
|
||||
}
|
||||
|
||||
func getPodStrategy(pc podCase) testutils.TestPodCreateStrategy {
|
||||
|
||||
@ -17,15 +17,34 @@ limitations under the License.
|
||||
package benchmark
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"path"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
coreinformers "k8s.io/client-go/informers/core/v1"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
restclient "k8s.io/client-go/rest"
|
||||
"k8s.io/component-base/metrics/legacyregistry"
|
||||
"k8s.io/component-base/metrics/testutil"
|
||||
"k8s.io/klog"
|
||||
"k8s.io/kubernetes/test/integration/util"
|
||||
)
|
||||
|
||||
const (
|
||||
dateFormat = "2006-01-02T15:04:05Z"
|
||||
throughputSampleFrequency = time.Second
|
||||
)
|
||||
|
||||
var dataItemsDir = flag.String("data-items-dir", "", "destination directory for storing generated data items for perf dashboard")
|
||||
|
||||
// mustSetupScheduler starts the following components:
|
||||
// - k8s api server (a.k.a. master)
|
||||
// - scheduler
|
||||
@ -66,3 +85,145 @@ func getScheduledPods(podInformer coreinformers.PodInformer) ([]*v1.Pod, error)
|
||||
}
|
||||
return scheduled, nil
|
||||
}
|
||||
|
||||
// DataItem is the data point.
|
||||
type DataItem struct {
|
||||
// Data is a map from bucket to real data point (e.g. "Perc90" -> 23.5). Notice
|
||||
// that all data items with the same label combination should have the same buckets.
|
||||
Data map[string]float64 `json:"data"`
|
||||
// Unit is the data unit. Notice that all data items with the same label combination
|
||||
// should have the same unit.
|
||||
Unit string `json:"unit"`
|
||||
// Labels is the labels of the data item.
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
}
|
||||
|
||||
// DataItems is the data point set. It is the struct that perf dashboard expects.
|
||||
type DataItems struct {
|
||||
Version string `json:"version"`
|
||||
DataItems []DataItem `json:"dataItems"`
|
||||
}
|
||||
|
||||
func dataItems2JSONFile(dataItems DataItems, namePrefix string) error {
|
||||
b, err := json.Marshal(dataItems)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
destFile := fmt.Sprintf("%v_%v.json", namePrefix, time.Now().Format(dateFormat))
|
||||
if *dataItemsDir != "" {
|
||||
destFile = path.Join(*dataItemsDir, destFile)
|
||||
}
|
||||
|
||||
return ioutil.WriteFile(destFile, b, 0644)
|
||||
}
|
||||
|
||||
// metricsCollector collects metrics from legacyregistry.DefaultGatherer.Gather() endpoint.
|
||||
// Currently only Histrogram metrics are supported.
|
||||
type metricsCollector struct {
|
||||
metric string
|
||||
}
|
||||
|
||||
func newMetricsCollector(metric string) *metricsCollector {
|
||||
return &metricsCollector{
|
||||
metric: metric,
|
||||
}
|
||||
}
|
||||
|
||||
func (pc *metricsCollector) collect() *DataItem {
|
||||
hist, err := testutil.GetHistogramFromGatherer(legacyregistry.DefaultGatherer, pc.metric)
|
||||
if err != nil {
|
||||
klog.Error(err)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := hist.Validate(); err != nil {
|
||||
klog.Error(err)
|
||||
return nil
|
||||
}
|
||||
|
||||
q50 := hist.Quantile(0.50)
|
||||
q90 := hist.Quantile(0.90)
|
||||
q99 := hist.Quantile(0.95)
|
||||
avg := hist.Average()
|
||||
|
||||
// clear the metrics so that next test always starts with empty prometheus
|
||||
// metrics (since the metrics are shared among all tests run inside the same binary)
|
||||
hist.Clear()
|
||||
|
||||
msFactor := float64(time.Second) / float64(time.Millisecond)
|
||||
|
||||
return &DataItem{
|
||||
Labels: map[string]string{
|
||||
"Metric": pc.metric,
|
||||
},
|
||||
Data: map[string]float64{
|
||||
"Perc50": q50 * msFactor,
|
||||
"Perc90": q90 * msFactor,
|
||||
"Perc99": q99 * msFactor,
|
||||
"Average": avg * msFactor,
|
||||
},
|
||||
Unit: "ms",
|
||||
}
|
||||
}
|
||||
|
||||
type throughputCollector struct {
|
||||
podInformer coreinformers.PodInformer
|
||||
schedulingThroughputs []float64
|
||||
}
|
||||
|
||||
func newThroughputCollector(podInformer coreinformers.PodInformer) *throughputCollector {
|
||||
return &throughputCollector{
|
||||
podInformer: podInformer,
|
||||
}
|
||||
}
|
||||
|
||||
func (tc *throughputCollector) run(stopCh chan struct{}) {
|
||||
podsScheduled, err := getScheduledPods(tc.podInformer)
|
||||
if err != nil {
|
||||
klog.Fatalf("%v", err)
|
||||
}
|
||||
lastScheduledCount := len(podsScheduled)
|
||||
for {
|
||||
select {
|
||||
case <-stopCh:
|
||||
return
|
||||
case <-time.After(throughputSampleFrequency):
|
||||
podsScheduled, err := getScheduledPods(tc.podInformer)
|
||||
if err != nil {
|
||||
klog.Fatalf("%v", err)
|
||||
}
|
||||
|
||||
scheduled := len(podsScheduled)
|
||||
samplingRatioSeconds := float64(throughputSampleFrequency) / float64(time.Second)
|
||||
throughput := float64(scheduled-lastScheduledCount) / samplingRatioSeconds
|
||||
tc.schedulingThroughputs = append(tc.schedulingThroughputs, throughput)
|
||||
lastScheduledCount = scheduled
|
||||
|
||||
klog.Infof("%d pods scheduled", lastScheduledCount)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (tc *throughputCollector) collect() *DataItem {
|
||||
throughputSummary := &DataItem{}
|
||||
if length := len(tc.schedulingThroughputs); length > 0 {
|
||||
sort.Float64s(tc.schedulingThroughputs)
|
||||
sum := 0.0
|
||||
for i := range tc.schedulingThroughputs {
|
||||
sum += tc.schedulingThroughputs[i]
|
||||
}
|
||||
|
||||
throughputSummary.Labels = map[string]string{
|
||||
"Metric": "SchedulingThroughput",
|
||||
}
|
||||
throughputSummary.Data = map[string]float64{
|
||||
"Average": sum / float64(length),
|
||||
"Perc50": tc.schedulingThroughputs[int(math.Ceil(float64(length*50)/100))-1],
|
||||
"Perc90": tc.schedulingThroughputs[int(math.Ceil(float64(length*90)/100))-1],
|
||||
"Perc99": tc.schedulingThroughputs[int(math.Ceil(float64(length*99)/100))-1],
|
||||
}
|
||||
throughputSummary.Unit = "pods/s"
|
||||
}
|
||||
return throughputSummary
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user