From 54218a7988f34097266ed2cab2ebfa0236593cb0 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Tue, 19 Mar 2019 23:53:33 -0700 Subject: [PATCH] scheduler: add unit test for pending pods metrics --- .../internal/queue/scheduling_queue_test.go | 157 +++++++++++++++++- 1 file changed, 151 insertions(+), 6 deletions(-) diff --git a/pkg/scheduler/internal/queue/scheduling_queue_test.go b/pkg/scheduler/internal/queue/scheduling_queue_test.go index 6b0d763af0e..d9ba6790eaf 100644 --- a/pkg/scheduler/internal/queue/scheduling_queue_test.go +++ b/pkg/scheduler/internal/queue/scheduling_queue_test.go @@ -23,11 +23,13 @@ import ( "testing" "time" + dto "github.com/prometheus/client_model/go" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/clock" podutil "k8s.io/kubernetes/pkg/api/v1/pod" + "k8s.io/kubernetes/pkg/scheduler/metrics" "k8s.io/kubernetes/pkg/scheduler/util" ) @@ -1066,7 +1068,7 @@ func TestPodTimestamp(t *testing.T) { tests := []struct { name string operations []operation - operants []*podInfo + operands []*podInfo expected []*podInfo }{ { @@ -1075,7 +1077,7 @@ func TestPodTimestamp(t *testing.T) { addPodActiveQ, addPodActiveQ, }, - operants: []*podInfo{pInfo2, pInfo1}, + operands: []*podInfo{pInfo2, pInfo1}, expected: []*podInfo{pInfo1, pInfo2}, }, { @@ -1084,7 +1086,7 @@ func TestPodTimestamp(t *testing.T) { updatePodActiveQ, updatePodActiveQ, }, - operants: []*podInfo{pInfo2, pInfo1}, + operands: []*podInfo{pInfo2, pInfo1}, expected: []*podInfo{pInfo1, pInfo2}, }, { @@ -1094,7 +1096,7 @@ func TestPodTimestamp(t *testing.T) { addPodUnschedulableQ, moveAllToActiveQ, }, - operants: []*podInfo{pInfo2, pInfo1, nil}, + operands: []*podInfo{pInfo2, pInfo1, nil}, expected: []*podInfo{pInfo1, pInfo2}, }, { @@ -1106,7 +1108,7 @@ func TestPodTimestamp(t *testing.T) { flushBackoffQ, moveAllToActiveQ, }, - operants: []*podInfo{pInfo2, pInfo1, pInfo1, nil, nil}, + operands: []*podInfo{pInfo2, pInfo1, pInfo1, nil, nil}, expected: []*podInfo{pInfo1, pInfo2}, }, } @@ -1117,7 +1119,7 @@ func TestPodTimestamp(t *testing.T) { var podInfoList []*podInfo for i, op := range test.operations { - op(queue, test.operants[i]) + op(queue, test.operands[i]) } for i := 0; i < len(test.expected); i++ { @@ -1135,3 +1137,146 @@ func TestPodTimestamp(t *testing.T) { }) } } + +// TestPendingPodsMetric tests Prometheus metrics related with pending pods +func TestPendingPodsMetric(t *testing.T) { + total := 50 + timestamp := time.Now() + var pInfos = make([]*podInfo, 0, total) + for i := 1; i <= total; i++ { + p := &podInfo{ + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("test-pod-%d", i), + Namespace: fmt.Sprintf("ns%d", i), + UID: types.UID(fmt.Sprintf("tp-%d", i)), + }, + }, + timestamp: timestamp, + } + pInfos = append(pInfos, p) + } + tests := []struct { + name string + operations []operation + operands [][]*podInfo + expected []int64 + }{ + { + name: "add pods to activeQ and unschedulableQ", + operations: []operation{ + addPodActiveQ, + addPodUnschedulableQ, + }, + operands: [][]*podInfo{ + pInfos[:30], + pInfos[30:], + }, + expected: []int64{30, 0, 20}, + }, + { + name: "add pods to all kinds of queues", + operations: []operation{ + addPodActiveQ, + backoffPod, + addPodBackoffQ, + addPodUnschedulableQ, + }, + operands: [][]*podInfo{ + pInfos[:15], + pInfos[15:40], + pInfos[15:40], + pInfos[40:], + }, + expected: []int64{15, 25, 10}, + }, + { + name: "add pods to unschedulableQ and then move all to activeQ", + operations: []operation{ + addPodUnschedulableQ, + moveAllToActiveQ, + }, + operands: [][]*podInfo{ + pInfos[:total], + {nil}, + }, + expected: []int64{int64(total), 0, 0}, + }, + { + name: "make some pods subject to backoff, add pods to unschedulableQ, and then move all to activeQ", + operations: []operation{ + backoffPod, + addPodUnschedulableQ, + moveAllToActiveQ, + }, + operands: [][]*podInfo{ + pInfos[:20], + pInfos[:total], + {nil}, + }, + expected: []int64{int64(total - 20), 20, 0}, + }, + { + name: "make some pods subject to backoff, add pods to unschedulableQ/activeQ, move all to activeQ, and finally flush backoffQ", + operations: []operation{ + backoffPod, + addPodUnschedulableQ, + addPodActiveQ, + moveAllToActiveQ, + flushBackoffQ, + }, + operands: [][]*podInfo{ + pInfos[:20], + pInfos[:40], + pInfos[40:], + {nil}, + {nil}, + }, + expected: []int64{int64(total), 0, 0}, + }, + } + + resetMetrics := func() { + metrics.ActivePods.Set(0) + metrics.BackoffPods.Set(0) + metrics.UnschedulablePods.Set(0) + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + resetMetrics() + queue := NewPriorityQueueWithClock(nil, clock.NewFakeClock(timestamp)) + for i, op := range test.operations { + for _, pInfo := range test.operands[i] { + op(queue, pInfo) + } + } + + var activeNum, backoffNum, unschedulableNum float64 + metricProto := &dto.Metric{} + if err := metrics.ActivePods.Write(metricProto); err != nil { + t.Errorf("error writing ActivePods metric: %v", err) + } + activeNum = metricProto.Gauge.GetValue() + if int64(activeNum) != test.expected[0] { + t.Errorf("ActivePods: Expected %v, got %v", test.expected[0], activeNum) + } + + if err := metrics.BackoffPods.Write(metricProto); err != nil { + t.Errorf("error writing BackoffPods metric: %v", err) + } + backoffNum = metricProto.Gauge.GetValue() + if int64(backoffNum) != test.expected[1] { + t.Errorf("BackoffPods: Expected %v, got %v", test.expected[1], backoffNum) + } + + if err := metrics.UnschedulablePods.Write(metricProto); err != nil { + t.Errorf("error writing UnschedulablePods metric: %v", err) + } + unschedulableNum = metricProto.Gauge.GetValue() + if int64(unschedulableNum) != test.expected[2] { + t.Errorf("UnschedulablePods: Expected %v, got %v", test.expected[2], unschedulableNum) + } + }) + } +}