From 7df9bfcfefd7659e76b69c1dc686da912ff5e0e1 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Fri, 5 Aug 2022 18:30:09 -0700 Subject: [PATCH] Expose a pending pods summary in scheudler's dummper output --- pkg/scheduler/internal/cache/debugger/comparer.go | 2 +- pkg/scheduler/internal/cache/debugger/dumper.go | 4 ++-- pkg/scheduler/internal/queue/scheduling_queue.go | 13 ++++++++----- .../internal/queue/scheduling_queue_test.go | 14 +++++++++++--- pkg/scheduler/scheduler_test.go | 2 +- test/integration/scheduler/filters/filters_test.go | 6 +++--- test/integration/scheduler/queue_test.go | 6 ++++-- 7 files changed, 30 insertions(+), 17 deletions(-) diff --git a/pkg/scheduler/internal/cache/debugger/comparer.go b/pkg/scheduler/internal/cache/debugger/comparer.go index f348977b4f9..bf8cafb7844 100644 --- a/pkg/scheduler/internal/cache/debugger/comparer.go +++ b/pkg/scheduler/internal/cache/debugger/comparer.go @@ -54,7 +54,7 @@ func (c *CacheComparer) Compare() error { dump := c.Cache.Dump() - pendingPods := c.PodQueue.PendingPods() + pendingPods, _ := c.PodQueue.PendingPods() if missed, redundant := c.CompareNodes(nodes, dump.Nodes); len(missed)+len(redundant) != 0 { klog.InfoS("Cache mismatch", "missedNodes", missed, "redundantNodes", redundant) diff --git a/pkg/scheduler/internal/cache/debugger/dumper.go b/pkg/scheduler/internal/cache/debugger/dumper.go index b2424b67888..d95c234eed7 100644 --- a/pkg/scheduler/internal/cache/debugger/dumper.go +++ b/pkg/scheduler/internal/cache/debugger/dumper.go @@ -54,12 +54,12 @@ func (d *CacheDumper) dumpNodes() { // dumpSchedulingQueue writes pods in the scheduling queue to the scheduler logs. func (d *CacheDumper) dumpSchedulingQueue() { - pendingPods := d.podQueue.PendingPods() + pendingPods, s := d.podQueue.PendingPods() var podData strings.Builder for _, p := range pendingPods { podData.WriteString(printPod(p)) } - klog.InfoS("Dump of scheduling queue", "pods", podData.String()) + klog.InfoS("Dump of scheduling queue", "summary", s, "pods", podData.String()) } // printNodeInfo writes parts of NodeInfo to a string. diff --git a/pkg/scheduler/internal/queue/scheduling_queue.go b/pkg/scheduler/internal/queue/scheduling_queue.go index 8de1a09c03b..a7f4ce47a3b 100644 --- a/pkg/scheduler/internal/queue/scheduling_queue.go +++ b/pkg/scheduler/internal/queue/scheduling_queue.go @@ -102,7 +102,7 @@ type SchedulingQueue interface { MoveAllToActiveOrBackoffQueue(event framework.ClusterEvent, preCheck PreEnqueueCheck) AssignedPodAdded(pod *v1.Pod) AssignedPodUpdated(pod *v1.Pod) - PendingPods() []*v1.Pod + PendingPods() ([]*v1.Pod, string) // Close closes the SchedulingQueue so that the goroutine which is // waiting to pop items can exit gracefully. Close() @@ -678,9 +678,12 @@ func (p *PriorityQueue) getUnschedulablePodsWithMatchingAffinityTerm(pod *v1.Pod return podsToMove } -// PendingPods returns all the pending pods in the queue. This function is -// used for debugging purposes in the scheduler cache dumper and comparer. -func (p *PriorityQueue) PendingPods() []*v1.Pod { +var pendingPodsSummary = "activeQ:%v; backoffQ:%v; unschedulablePods:%v" + +// PendingPods returns all the pending pods in the queue; accompanied by a debugging string +// recording showing the number of pods in each queue respectively. +// This function is used for debugging purposes in the scheduler cache dumper and comparer. +func (p *PriorityQueue) PendingPods() ([]*v1.Pod, string) { p.lock.RLock() defer p.lock.RUnlock() var result []*v1.Pod @@ -693,7 +696,7 @@ func (p *PriorityQueue) PendingPods() []*v1.Pod { for _, pInfo := range p.unschedulablePods.podInfoMap { result = append(result, pInfo.Pod) } - return result + return result, fmt.Sprintf(pendingPodsSummary, p.activeQ.Len(), p.podBackoffQ.Len(), len(p.unschedulablePods.podInfoMap)) } // Close closes the priority queue. diff --git a/pkg/scheduler/internal/queue/scheduling_queue_test.go b/pkg/scheduler/internal/queue/scheduling_queue_test.go index 8333ea476c7..b2e5c335095 100644 --- a/pkg/scheduler/internal/queue/scheduling_queue_test.go +++ b/pkg/scheduler/internal/queue/scheduling_queue_test.go @@ -745,13 +745,21 @@ func TestPriorityQueue_PendingPods(t *testing.T) { q.AddUnschedulableIfNotPresent(q.newQueuedPodInfo(highPriorityPodInfo.Pod), q.SchedulingCycle()) expectedSet := makeSet([]*v1.Pod{medPriorityPodInfo.Pod, unschedulablePodInfo.Pod, highPriorityPodInfo.Pod}) - if !reflect.DeepEqual(expectedSet, makeSet(q.PendingPods())) { + gotPods, gotSummary := q.PendingPods() + if !reflect.DeepEqual(expectedSet, makeSet(gotPods)) { t.Error("Unexpected list of pending Pods.") } + if wantSummary := fmt.Sprintf(pendingPodsSummary, 1, 0, 2); wantSummary != gotSummary { + t.Errorf("Unexpected pending pods summary: want %v, but got %v.", wantSummary, gotSummary) + } // Move all to active queue. We should still see the same set of pods. q.MoveAllToActiveOrBackoffQueue(TestEvent, nil) - if !reflect.DeepEqual(expectedSet, makeSet(q.PendingPods())) { - t.Error("Unexpected list of pending Pods...") + gotPods, gotSummary = q.PendingPods() + if !reflect.DeepEqual(expectedSet, makeSet(gotPods)) { + t.Error("Unexpected list of pending Pods.") + } + if wantSummary := fmt.Sprintf(pendingPodsSummary, 1, 2, 0); wantSummary != gotSummary { + t.Errorf("Unexpected pending pods summary: want %v, but got %v.", wantSummary, gotSummary) } } diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index 839175488b8..ca07ff260e1 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -418,7 +418,7 @@ func TestFailureHandler_PodAlreadyBound(t *testing.T) { // getPodFromPriorityQueue is the function used in the TestDefaultErrorFunc test to get // the specific pod from the given priority queue. It returns the found pod in the priority queue. func getPodFromPriorityQueue(queue *internalqueue.PriorityQueue, pod *v1.Pod) *v1.Pod { - podList := queue.PendingPods() + podList, _ := queue.PendingPods() if len(podList) == 0 { return nil } diff --git a/test/integration/scheduler/filters/filters_test.go b/test/integration/scheduler/filters/filters_test.go index edd3241e236..96b1d1e839d 100644 --- a/test/integration/scheduler/filters/filters_test.go +++ b/test/integration/scheduler/filters/filters_test.go @@ -1713,9 +1713,9 @@ func TestUnschedulablePodBecomesSchedulable(t *testing.T) { t.Errorf("Pod %v was not scheduled: %v", pod.Name, err) } // Make sure pending queue is empty. - pendingPods := len(testCtx.Scheduler.SchedulingQueue.PendingPods()) - if pendingPods != 0 { - t.Errorf("pending pods queue is not empty, size is: %d", pendingPods) + pendingPods, s := testCtx.Scheduler.SchedulingQueue.PendingPods() + if len(pendingPods) != 0 { + t.Errorf("pending pods queue is not empty, size is: %d, summary is: %s", len(pendingPods), s) } }) } diff --git a/test/integration/scheduler/queue_test.go b/test/integration/scheduler/queue_test.go index 5efaf06e484..831431a60ec 100644 --- a/test/integration/scheduler/queue_test.go +++ b/test/integration/scheduler/queue_test.go @@ -88,7 +88,8 @@ func TestCoreResourceEnqueue(t *testing.T) { // Wait for the three pods to be present in the scheduling queue. if err := wait.Poll(time.Millisecond*200, wait.ForeverTestTimeout, func() (bool, error) { - return len(testCtx.Scheduler.SchedulingQueue.PendingPods()) == 3, nil + pendingPods, _ := testCtx.Scheduler.SchedulingQueue.PendingPods() + return len(pendingPods) == 3, nil }); err != nil { t.Fatal(err) } @@ -263,7 +264,8 @@ func TestCustomResourceEnqueue(t *testing.T) { // Wait for the testing Pod to be present in the scheduling queue. if err := wait.Poll(time.Millisecond*200, wait.ForeverTestTimeout, func() (bool, error) { - return len(testCtx.Scheduler.SchedulingQueue.PendingPods()) == 1, nil + pendingPods, _ := testCtx.Scheduler.SchedulingQueue.PendingPods() + return len(pendingPods) == 1, nil }); err != nil { t.Fatal(err) }