mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-25 20:53:33 +00:00
Pop from the backoffQ when the activeQ is empty
This commit is contained in:
parent
6b8e5a9457
commit
c7919f5e22
@ -113,14 +113,22 @@ func (uaq *unlockedActiveQueue) has(pInfo *framework.QueuedPodInfo) bool {
|
|||||||
return uaq.queue.Has(pInfo)
|
return uaq.queue.Has(pInfo)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// backoffQPopper defines method that is used to pop from the backoffQ when the activeQ is empty.
|
||||||
|
type backoffQPopper interface {
|
||||||
|
// popBackoff pops the pInfo from the podBackoffQ.
|
||||||
|
popBackoff() (*framework.QueuedPodInfo, error)
|
||||||
|
// len returns length of the podBackoffQ queue.
|
||||||
|
lenBackoff() int
|
||||||
|
}
|
||||||
|
|
||||||
// activeQueue implements activeQueuer. All of the fields have to be protected using the lock.
|
// activeQueue implements activeQueuer. All of the fields have to be protected using the lock.
|
||||||
type activeQueue struct {
|
type activeQueue struct {
|
||||||
// lock synchronizes all operations related to activeQ.
|
// lock synchronizes all operations related to activeQ.
|
||||||
// It protects activeQ, inFlightPods, inFlightEvents, schedulingCycle and closed fields.
|
// It protects activeQ, inFlightPods, inFlightEvents, schedulingCycle and closed fields.
|
||||||
// Caution: DO NOT take "SchedulingQueue.lock" after taking "lock".
|
// Caution: DO NOT take "SchedulingQueue.lock" after taking "lock".
|
||||||
// You should always take "SchedulingQueue.lock" first, otherwise the queue could end up in deadlock.
|
// You should always take "SchedulingQueue.lock" first, otherwise the queue could end up in deadlock.
|
||||||
// "lock" should not be taken after taking "nLock".
|
// "lock" should not be taken after taking "backoffQueue.lock" or "nominator.nLock".
|
||||||
// Correct locking order is: SchedulingQueue.lock > lock > nominator.nLock.
|
// Correct locking order is: SchedulingQueue.lock > lock > backoffQueue.lock > nominator.nLock.
|
||||||
lock sync.RWMutex
|
lock sync.RWMutex
|
||||||
|
|
||||||
// activeQ is heap structure that scheduler actively looks at to find pods to
|
// activeQ is heap structure that scheduler actively looks at to find pods to
|
||||||
@ -132,6 +140,8 @@ type activeQueue struct {
|
|||||||
unlockedQueue *unlockedActiveQueue
|
unlockedQueue *unlockedActiveQueue
|
||||||
|
|
||||||
// cond is a condition that is notified when the pod is added to activeQ.
|
// cond is a condition that is notified when the pod is added to activeQ.
|
||||||
|
// When SchedulerPopFromBackoffQ feature is enabled,
|
||||||
|
// condition is also notified when the pod is added to backoffQ.
|
||||||
// It is used with lock.
|
// It is used with lock.
|
||||||
cond sync.Cond
|
cond sync.Cond
|
||||||
|
|
||||||
@ -171,9 +181,13 @@ type activeQueue struct {
|
|||||||
isSchedulingQueueHintEnabled bool
|
isSchedulingQueueHintEnabled bool
|
||||||
|
|
||||||
metricsRecorder metrics.MetricAsyncRecorder
|
metricsRecorder metrics.MetricAsyncRecorder
|
||||||
|
|
||||||
|
// backoffQPopper is used to pop from backoffQ when activeQ is empty.
|
||||||
|
// It is non-nil only when SchedulerPopFromBackoffQ feature is enabled.
|
||||||
|
backoffQPopper backoffQPopper
|
||||||
}
|
}
|
||||||
|
|
||||||
func newActiveQueue(queue *heap.Heap[*framework.QueuedPodInfo], isSchedulingQueueHintEnabled bool, metricRecorder metrics.MetricAsyncRecorder) *activeQueue {
|
func newActiveQueue(queue *heap.Heap[*framework.QueuedPodInfo], isSchedulingQueueHintEnabled bool, metricRecorder metrics.MetricAsyncRecorder, backoffQPopper backoffQPopper) *activeQueue {
|
||||||
aq := &activeQueue{
|
aq := &activeQueue{
|
||||||
queue: queue,
|
queue: queue,
|
||||||
inFlightPods: make(map[types.UID]*list.Element),
|
inFlightPods: make(map[types.UID]*list.Element),
|
||||||
@ -181,6 +195,7 @@ func newActiveQueue(queue *heap.Heap[*framework.QueuedPodInfo], isSchedulingQueu
|
|||||||
isSchedulingQueueHintEnabled: isSchedulingQueueHintEnabled,
|
isSchedulingQueueHintEnabled: isSchedulingQueueHintEnabled,
|
||||||
metricsRecorder: metricRecorder,
|
metricsRecorder: metricRecorder,
|
||||||
unlockedQueue: newUnlockedActiveQueue(queue),
|
unlockedQueue: newUnlockedActiveQueue(queue),
|
||||||
|
backoffQPopper: backoffQPopper,
|
||||||
}
|
}
|
||||||
aq.cond.L = &aq.lock
|
aq.cond.L = &aq.lock
|
||||||
|
|
||||||
@ -238,7 +253,13 @@ func (aq *activeQueue) pop(logger klog.Logger) (*framework.QueuedPodInfo, error)
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (aq *activeQueue) unlockedPop(logger klog.Logger) (*framework.QueuedPodInfo, error) {
|
func (aq *activeQueue) unlockedPop(logger klog.Logger) (*framework.QueuedPodInfo, error) {
|
||||||
|
var pInfo *framework.QueuedPodInfo
|
||||||
for aq.queue.Len() == 0 {
|
for aq.queue.Len() == 0 {
|
||||||
|
// backoffQPopper is non-nil only if SchedulerPopFromBackoffQ feature is enabled.
|
||||||
|
// In case of non-empty backoffQ, try popping from there.
|
||||||
|
if aq.backoffQPopper != nil && aq.backoffQPopper.lenBackoff() != 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
// When the queue is empty, invocation of Pop() is blocked until new item is enqueued.
|
// When the queue is empty, invocation of Pop() is blocked until new item is enqueued.
|
||||||
// When Close() is called, the p.closed is set and the condition is broadcast,
|
// When Close() is called, the p.closed is set and the condition is broadcast,
|
||||||
// which causes this loop to continue and return from the Pop().
|
// which causes this loop to continue and return from the Pop().
|
||||||
@ -250,7 +271,15 @@ func (aq *activeQueue) unlockedPop(logger klog.Logger) (*framework.QueuedPodInfo
|
|||||||
}
|
}
|
||||||
pInfo, err := aq.queue.Pop()
|
pInfo, err := aq.queue.Pop()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
if aq.backoffQPopper == nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Try to pop from backoffQ when activeQ is empty.
|
||||||
|
pInfo, err = aq.backoffQPopper.popBackoff()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
metrics.SchedulerQueueIncomingPods.WithLabelValues("active", framework.PopFromBackoffQ).Inc()
|
||||||
}
|
}
|
||||||
pInfo.Attempts++
|
pInfo.Attempts++
|
||||||
pInfo.BackoffExpiration = time.Time{}
|
pInfo.BackoffExpiration = time.Time{}
|
||||||
|
@ -30,7 +30,7 @@ import (
|
|||||||
func TestClose(t *testing.T) {
|
func TestClose(t *testing.T) {
|
||||||
logger, ctx := ktesting.NewTestContext(t)
|
logger, ctx := ktesting.NewTestContext(t)
|
||||||
rr := metrics.NewMetricsAsyncRecorder(10, time.Second, ctx.Done())
|
rr := metrics.NewMetricsAsyncRecorder(10, time.Second, ctx.Done())
|
||||||
aq := newActiveQueue(heap.NewWithRecorder(podInfoKeyFunc, heap.LessFunc[*framework.QueuedPodInfo](newDefaultQueueSort()), metrics.NewActivePodsRecorder()), true, *rr)
|
aq := newActiveQueue(heap.NewWithRecorder(podInfoKeyFunc, heap.LessFunc[*framework.QueuedPodInfo](newDefaultQueueSort()), metrics.NewActivePodsRecorder()), true, *rr, nil)
|
||||||
|
|
||||||
aq.underLock(func(unlockedActiveQ unlockedActiveQueuer) {
|
aq.underLock(func(unlockedActiveQ unlockedActiveQueuer) {
|
||||||
unlockedActiveQ.add(&framework.QueuedPodInfo{PodInfo: &framework.PodInfo{Pod: st.MakePod().Namespace("foo").Name("p1").UID("p1").Obj()}}, framework.EventUnscheduledPodAdd.Label())
|
unlockedActiveQ.add(&framework.QueuedPodInfo{PodInfo: &framework.PodInfo{Pod: st.MakePod().Namespace("foo").Name("p1").UID("p1").Obj()}}, framework.EventUnscheduledPodAdd.Label())
|
||||||
|
@ -17,6 +17,7 @@ limitations under the License.
|
|||||||
package queue
|
package queue
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
v1 "k8s.io/api/core/v1"
|
v1 "k8s.io/api/core/v1"
|
||||||
@ -35,13 +36,14 @@ import (
|
|||||||
const backoffQOrderingWindowDuration = time.Second
|
const backoffQOrderingWindowDuration = time.Second
|
||||||
|
|
||||||
// backoffQueuer is a wrapper for backoffQ related operations.
|
// backoffQueuer is a wrapper for backoffQ related operations.
|
||||||
|
// Its methods that relies on the queues, take the lock inside.
|
||||||
type backoffQueuer interface {
|
type backoffQueuer interface {
|
||||||
// isPodBackingoff returns true if a pod is still waiting for its backoff timer.
|
// isPodBackingoff returns true if a pod is still waiting for its backoff timer.
|
||||||
// If this returns true, the pod should not be re-tried.
|
// If this returns true, the pod should not be re-tried.
|
||||||
// If the pod backoff time is in the actual ordering window, it should still be backing off.
|
// If the pod backoff time is in the actual ordering window, it should still be backing off.
|
||||||
isPodBackingoff(podInfo *framework.QueuedPodInfo) bool
|
isPodBackingoff(podInfo *framework.QueuedPodInfo) bool
|
||||||
// popEachBackoffCompleted run fn for all pods from podBackoffQ and podErrorBackoffQ that completed backoff while popping them.
|
// popAllBackoffCompleted pops all pods from podBackoffQ and podErrorBackoffQ that completed backoff.
|
||||||
popEachBackoffCompleted(logger klog.Logger, fn func(pInfo *framework.QueuedPodInfo))
|
popAllBackoffCompleted(logger klog.Logger) []*framework.QueuedPodInfo
|
||||||
|
|
||||||
// podInitialBackoffDuration returns initial backoff duration that pod can get.
|
// podInitialBackoffDuration returns initial backoff duration that pod can get.
|
||||||
podInitialBackoffDuration() time.Duration
|
podInitialBackoffDuration() time.Duration
|
||||||
@ -61,7 +63,8 @@ type backoffQueuer interface {
|
|||||||
// It returns new pod info if updated, nil otherwise.
|
// It returns new pod info if updated, nil otherwise.
|
||||||
update(newPod *v1.Pod, oldPodInfo *framework.QueuedPodInfo) *framework.QueuedPodInfo
|
update(newPod *v1.Pod, oldPodInfo *framework.QueuedPodInfo) *framework.QueuedPodInfo
|
||||||
// delete deletes the pInfo from backoffQueue.
|
// delete deletes the pInfo from backoffQueue.
|
||||||
delete(pInfo *framework.QueuedPodInfo)
|
// It returns true if the pod was deleted.
|
||||||
|
delete(pInfo *framework.QueuedPodInfo) bool
|
||||||
// get returns the pInfo matching given pInfoLookup, if exists.
|
// get returns the pInfo matching given pInfoLookup, if exists.
|
||||||
get(pInfoLookup *framework.QueuedPodInfo) (*framework.QueuedPodInfo, bool)
|
get(pInfoLookup *framework.QueuedPodInfo) (*framework.QueuedPodInfo, bool)
|
||||||
// has inform if pInfo exists in the queue.
|
// has inform if pInfo exists in the queue.
|
||||||
@ -75,6 +78,14 @@ type backoffQueuer interface {
|
|||||||
// backoffQueue implements backoffQueuer and wraps two queues inside,
|
// backoffQueue implements backoffQueuer and wraps two queues inside,
|
||||||
// providing seamless access as if it were one queue.
|
// providing seamless access as if it were one queue.
|
||||||
type backoffQueue struct {
|
type backoffQueue struct {
|
||||||
|
// lock synchronizes all operations related to backoffQ.
|
||||||
|
// It protects both podBackoffQ and podErrorBackoffQ.
|
||||||
|
// Caution: DO NOT take "SchedulingQueue.lock" or "activeQueue.lock" after taking "lock".
|
||||||
|
// You should always take "SchedulingQueue.lock" and "activeQueue.lock" first, otherwise the queue could end up in deadlock.
|
||||||
|
// "lock" should not be taken after taking "nominator.nLock".
|
||||||
|
// Correct locking order is: SchedulingQueue.lock > activeQueue.lock > lock > nominator.nLock.
|
||||||
|
lock sync.RWMutex
|
||||||
|
|
||||||
clock clock.WithTicker
|
clock clock.WithTicker
|
||||||
|
|
||||||
// podBackoffQ is a heap ordered by backoff expiry. Pods which have completed backoff
|
// podBackoffQ is a heap ordered by backoff expiry. Pods which have completed backoff
|
||||||
@ -239,7 +250,8 @@ func (bq *backoffQueue) calculateBackoffDuration(podInfo *framework.QueuedPodInf
|
|||||||
return duration
|
return duration
|
||||||
}
|
}
|
||||||
|
|
||||||
func (bq *backoffQueue) popEachBackoffCompletedWithQueue(logger klog.Logger, fn func(pInfo *framework.QueuedPodInfo), queue *heap.Heap[*framework.QueuedPodInfo]) {
|
func (bq *backoffQueue) popAllBackoffCompletedWithQueue(logger klog.Logger, queue *heap.Heap[*framework.QueuedPodInfo]) []*framework.QueuedPodInfo {
|
||||||
|
var poppedPods []*framework.QueuedPodInfo
|
||||||
for {
|
for {
|
||||||
pInfo, ok := queue.Peek()
|
pInfo, ok := queue.Peek()
|
||||||
if !ok || pInfo == nil {
|
if !ok || pInfo == nil {
|
||||||
@ -254,23 +266,27 @@ func (bq *backoffQueue) popEachBackoffCompletedWithQueue(logger klog.Logger, fn
|
|||||||
logger.Error(err, "Unable to pop pod from backoff queue despite backoff completion", "pod", klog.KObj(pod))
|
logger.Error(err, "Unable to pop pod from backoff queue despite backoff completion", "pod", klog.KObj(pod))
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
if fn != nil {
|
poppedPods = append(poppedPods, pInfo)
|
||||||
fn(pInfo)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
return poppedPods
|
||||||
}
|
}
|
||||||
|
|
||||||
// popEachBackoffCompleted run fn for all pods from podBackoffQ and podErrorBackoffQ that completed backoff while popping them.
|
// popAllBackoffCompleted pops all pods from podBackoffQ and podErrorBackoffQ that completed backoff.
|
||||||
func (bq *backoffQueue) popEachBackoffCompleted(logger klog.Logger, fn func(pInfo *framework.QueuedPodInfo)) {
|
func (bq *backoffQueue) popAllBackoffCompleted(logger klog.Logger) []*framework.QueuedPodInfo {
|
||||||
|
bq.lock.Lock()
|
||||||
|
defer bq.lock.Unlock()
|
||||||
|
|
||||||
// Ensure both queues are called
|
// Ensure both queues are called
|
||||||
bq.popEachBackoffCompletedWithQueue(logger, fn, bq.podBackoffQ)
|
return append(bq.popAllBackoffCompletedWithQueue(logger, bq.podBackoffQ), bq.popAllBackoffCompletedWithQueue(logger, bq.podErrorBackoffQ)...)
|
||||||
bq.popEachBackoffCompletedWithQueue(logger, fn, bq.podErrorBackoffQ)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// add adds the pInfo to backoffQueue.
|
// add adds the pInfo to backoffQueue.
|
||||||
// The event should show which event triggered this addition and is used for the metric recording.
|
// The event should show which event triggered this addition and is used for the metric recording.
|
||||||
// It also ensures that pInfo is not in both queues.
|
// It also ensures that pInfo is not in both queues.
|
||||||
func (bq *backoffQueue) add(logger klog.Logger, pInfo *framework.QueuedPodInfo, event string) {
|
func (bq *backoffQueue) add(logger klog.Logger, pInfo *framework.QueuedPodInfo, event string) {
|
||||||
|
bq.lock.Lock()
|
||||||
|
defer bq.lock.Unlock()
|
||||||
|
|
||||||
// If pod has empty both unschedulable plugins and pending plugins,
|
// If pod has empty both unschedulable plugins and pending plugins,
|
||||||
// it means that it failed because of error and should be moved to podErrorBackoffQ.
|
// it means that it failed because of error and should be moved to podErrorBackoffQ.
|
||||||
if pInfo.UnschedulablePlugins.Len() == 0 && pInfo.PendingPlugins.Len() == 0 {
|
if pInfo.UnschedulablePlugins.Len() == 0 && pInfo.PendingPlugins.Len() == 0 {
|
||||||
@ -297,6 +313,9 @@ func (bq *backoffQueue) add(logger klog.Logger, pInfo *framework.QueuedPodInfo,
|
|||||||
// update updates the pod in backoffQueue if oldPodInfo is already in the queue.
|
// update updates the pod in backoffQueue if oldPodInfo is already in the queue.
|
||||||
// It returns new pod info if updated, nil otherwise.
|
// It returns new pod info if updated, nil otherwise.
|
||||||
func (bq *backoffQueue) update(newPod *v1.Pod, oldPodInfo *framework.QueuedPodInfo) *framework.QueuedPodInfo {
|
func (bq *backoffQueue) update(newPod *v1.Pod, oldPodInfo *framework.QueuedPodInfo) *framework.QueuedPodInfo {
|
||||||
|
bq.lock.Lock()
|
||||||
|
defer bq.lock.Unlock()
|
||||||
|
|
||||||
// If the pod is in the backoff queue, update it there.
|
// If the pod is in the backoff queue, update it there.
|
||||||
if pInfo, exists := bq.podBackoffQ.Get(oldPodInfo); exists {
|
if pInfo, exists := bq.podBackoffQ.Get(oldPodInfo); exists {
|
||||||
_ = pInfo.Update(newPod)
|
_ = pInfo.Update(newPod)
|
||||||
@ -313,13 +332,32 @@ func (bq *backoffQueue) update(newPod *v1.Pod, oldPodInfo *framework.QueuedPodIn
|
|||||||
}
|
}
|
||||||
|
|
||||||
// delete deletes the pInfo from backoffQueue.
|
// delete deletes the pInfo from backoffQueue.
|
||||||
func (bq *backoffQueue) delete(pInfo *framework.QueuedPodInfo) {
|
// It returns true if the pod was deleted.
|
||||||
_ = bq.podBackoffQ.Delete(pInfo)
|
func (bq *backoffQueue) delete(pInfo *framework.QueuedPodInfo) bool {
|
||||||
_ = bq.podErrorBackoffQ.Delete(pInfo)
|
bq.lock.Lock()
|
||||||
|
defer bq.lock.Unlock()
|
||||||
|
|
||||||
|
if bq.podBackoffQ.Delete(pInfo) == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return bq.podErrorBackoffQ.Delete(pInfo) == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// popBackoff pops the pInfo from the podBackoffQ.
|
||||||
|
// It returns error if the queue is empty.
|
||||||
|
// This doesn't pop the pods from the podErrorBackoffQ.
|
||||||
|
func (bq *backoffQueue) popBackoff() (*framework.QueuedPodInfo, error) {
|
||||||
|
bq.lock.Lock()
|
||||||
|
defer bq.lock.Unlock()
|
||||||
|
|
||||||
|
return bq.podBackoffQ.Pop()
|
||||||
}
|
}
|
||||||
|
|
||||||
// get returns the pInfo matching given pInfoLookup, if exists.
|
// get returns the pInfo matching given pInfoLookup, if exists.
|
||||||
func (bq *backoffQueue) get(pInfoLookup *framework.QueuedPodInfo) (*framework.QueuedPodInfo, bool) {
|
func (bq *backoffQueue) get(pInfoLookup *framework.QueuedPodInfo) (*framework.QueuedPodInfo, bool) {
|
||||||
|
bq.lock.RLock()
|
||||||
|
defer bq.lock.RUnlock()
|
||||||
|
|
||||||
pInfo, exists := bq.podBackoffQ.Get(pInfoLookup)
|
pInfo, exists := bq.podBackoffQ.Get(pInfoLookup)
|
||||||
if exists {
|
if exists {
|
||||||
return pInfo, true
|
return pInfo, true
|
||||||
@ -329,11 +367,17 @@ func (bq *backoffQueue) get(pInfoLookup *framework.QueuedPodInfo) (*framework.Qu
|
|||||||
|
|
||||||
// has inform if pInfo exists in the queue.
|
// has inform if pInfo exists in the queue.
|
||||||
func (bq *backoffQueue) has(pInfo *framework.QueuedPodInfo) bool {
|
func (bq *backoffQueue) has(pInfo *framework.QueuedPodInfo) bool {
|
||||||
|
bq.lock.RLock()
|
||||||
|
defer bq.lock.RUnlock()
|
||||||
|
|
||||||
return bq.podBackoffQ.Has(pInfo) || bq.podErrorBackoffQ.Has(pInfo)
|
return bq.podBackoffQ.Has(pInfo) || bq.podErrorBackoffQ.Has(pInfo)
|
||||||
}
|
}
|
||||||
|
|
||||||
// list returns all pods that are in the queue.
|
// list returns all pods that are in the queue.
|
||||||
func (bq *backoffQueue) list() []*v1.Pod {
|
func (bq *backoffQueue) list() []*v1.Pod {
|
||||||
|
bq.lock.RLock()
|
||||||
|
defer bq.lock.RUnlock()
|
||||||
|
|
||||||
var result []*v1.Pod
|
var result []*v1.Pod
|
||||||
for _, pInfo := range bq.podBackoffQ.List() {
|
for _, pInfo := range bq.podBackoffQ.List() {
|
||||||
result = append(result, pInfo.Pod)
|
result = append(result, pInfo.Pod)
|
||||||
@ -346,5 +390,16 @@ func (bq *backoffQueue) list() []*v1.Pod {
|
|||||||
|
|
||||||
// len returns length of the queue.
|
// len returns length of the queue.
|
||||||
func (bq *backoffQueue) len() int {
|
func (bq *backoffQueue) len() int {
|
||||||
|
bq.lock.RLock()
|
||||||
|
defer bq.lock.RUnlock()
|
||||||
|
|
||||||
return bq.podBackoffQ.Len() + bq.podErrorBackoffQ.Len()
|
return bq.podBackoffQ.Len() + bq.podErrorBackoffQ.Len()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// lenBackoff returns length of the podBackoffQ.
|
||||||
|
func (bq *backoffQueue) lenBackoff() int {
|
||||||
|
bq.lock.RLock()
|
||||||
|
defer bq.lock.RUnlock()
|
||||||
|
|
||||||
|
return bq.podBackoffQ.Len()
|
||||||
|
}
|
||||||
|
@ -78,7 +78,7 @@ func TestBackoffQueue_calculateBackoffDuration(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBackoffQueue_popEachBackoffCompleted(t *testing.T) {
|
func TestBackoffQueue_popAllBackoffCompleted(t *testing.T) {
|
||||||
fakeClock := testingclock.NewFakeClock(time.Now())
|
fakeClock := testingclock.NewFakeClock(time.Now())
|
||||||
podInfos := map[string]*framework.QueuedPodInfo{
|
podInfos := map[string]*framework.QueuedPodInfo{
|
||||||
"pod0": {
|
"pod0": {
|
||||||
@ -156,10 +156,11 @@ func TestBackoffQueue_popEachBackoffCompleted(t *testing.T) {
|
|||||||
for _, podName := range tt.podsInBackoff {
|
for _, podName := range tt.podsInBackoff {
|
||||||
bq.add(logger, podInfos[podName], framework.EventUnscheduledPodAdd.Label())
|
bq.add(logger, podInfos[podName], framework.EventUnscheduledPodAdd.Label())
|
||||||
}
|
}
|
||||||
|
gotPodInfos := bq.popAllBackoffCompleted(logger)
|
||||||
var gotPods []string
|
var gotPods []string
|
||||||
bq.popEachBackoffCompleted(logger, func(pInfo *framework.QueuedPodInfo) {
|
for _, pInfo := range gotPodInfos {
|
||||||
gotPods = append(gotPods, pInfo.Pod.Name)
|
gotPods = append(gotPods, pInfo.Pod.Name)
|
||||||
})
|
}
|
||||||
if diff := cmp.Diff(tt.wantPods, gotPods); diff != "" {
|
if diff := cmp.Diff(tt.wantPods, gotPods); diff != "" {
|
||||||
t.Errorf("Unexpected pods moved (-want, +got):\n%s", diff)
|
t.Errorf("Unexpected pods moved (-want, +got):\n%s", diff)
|
||||||
}
|
}
|
||||||
@ -248,10 +249,11 @@ func TestBackoffQueueOrdering(t *testing.T) {
|
|||||||
for _, podInfo := range podInfos {
|
for _, podInfo := range podInfos {
|
||||||
bq.add(logger, podInfo, framework.EventUnscheduledPodAdd.Label())
|
bq.add(logger, podInfo, framework.EventUnscheduledPodAdd.Label())
|
||||||
}
|
}
|
||||||
|
gotPodInfos := bq.popAllBackoffCompleted(logger)
|
||||||
var gotPods []string
|
var gotPods []string
|
||||||
bq.popEachBackoffCompleted(logger, func(pInfo *framework.QueuedPodInfo) {
|
for _, pInfo := range gotPodInfos {
|
||||||
gotPods = append(gotPods, pInfo.Pod.Name)
|
gotPods = append(gotPods, pInfo.Pod.Name)
|
||||||
})
|
}
|
||||||
if diff := cmp.Diff(tt.wantPods, gotPods); diff != "" {
|
if diff := cmp.Diff(tt.wantPods, gotPods); diff != "" {
|
||||||
t.Errorf("Unexpected pods moved (-want, +got):\n%s", diff)
|
t.Errorf("Unexpected pods moved (-want, +got):\n%s", diff)
|
||||||
}
|
}
|
||||||
|
@ -35,10 +35,10 @@ import (
|
|||||||
type nominator struct {
|
type nominator struct {
|
||||||
// nLock synchronizes all operations related to nominator.
|
// nLock synchronizes all operations related to nominator.
|
||||||
// It should not be used anywhere else.
|
// It should not be used anywhere else.
|
||||||
// Caution: DO NOT take ("SchedulingQueue.lock" or "activeQueue.lock") after taking "nLock".
|
// Caution: DO NOT take ("SchedulingQueue.lock" or "activeQueue.lock" or "backoffQueue.lock") after taking "nLock".
|
||||||
// You should always take "SchedulingQueue.lock" and "activeQueue.lock" first,
|
// You should always take "SchedulingQueue.lock" and "activeQueue.lock" and "backoffQueue.lock" first,
|
||||||
// otherwise the nominator could end up in deadlock.
|
// otherwise the nominator could end up in deadlock.
|
||||||
// Correct locking order is: SchedulingQueue.lock > activeQueue.lock > nLock.
|
// Correct locking order is: SchedulingQueue.lock > activeQueue.lock = backoffQueue.lock > nLock.
|
||||||
nLock sync.RWMutex
|
nLock sync.RWMutex
|
||||||
|
|
||||||
// podLister is used to verify if the given pod is alive.
|
// podLister is used to verify if the given pod is alive.
|
||||||
|
@ -160,8 +160,8 @@ type PriorityQueue struct {
|
|||||||
clock clock.WithTicker
|
clock clock.WithTicker
|
||||||
|
|
||||||
// lock takes precedence and should be taken first,
|
// lock takes precedence and should be taken first,
|
||||||
// before any other locks in the queue (activeQueue.lock or nominator.nLock).
|
// before any other locks in the queue (activeQueue.lock or backoffQueue.lock or nominator.nLock).
|
||||||
// Correct locking order is: lock > activeQueue.lock > nominator.nLock.
|
// Correct locking order is: lock > activeQueue.lock > backoffQueue.lock > nominator.nLock.
|
||||||
lock sync.RWMutex
|
lock sync.RWMutex
|
||||||
|
|
||||||
// the maximum time a pod can stay in the unschedulablePods.
|
// the maximum time a pod can stay in the unschedulablePods.
|
||||||
@ -331,12 +331,12 @@ func NewPriorityQueue(
|
|||||||
isSchedulingQueueHintEnabled := utilfeature.DefaultFeatureGate.Enabled(features.SchedulerQueueingHints)
|
isSchedulingQueueHintEnabled := utilfeature.DefaultFeatureGate.Enabled(features.SchedulerQueueingHints)
|
||||||
isPopFromBackoffQEnabled := utilfeature.DefaultFeatureGate.Enabled(features.SchedulerPopFromBackoffQ)
|
isPopFromBackoffQEnabled := utilfeature.DefaultFeatureGate.Enabled(features.SchedulerPopFromBackoffQ)
|
||||||
|
|
||||||
|
backoffQ := newBackoffQueue(options.clock, options.podInitialBackoffDuration, options.podMaxBackoffDuration, lessFn, isPopFromBackoffQEnabled)
|
||||||
pq := &PriorityQueue{
|
pq := &PriorityQueue{
|
||||||
clock: options.clock,
|
clock: options.clock,
|
||||||
stop: make(chan struct{}),
|
stop: make(chan struct{}),
|
||||||
podMaxInUnschedulablePodsDuration: options.podMaxInUnschedulablePodsDuration,
|
podMaxInUnschedulablePodsDuration: options.podMaxInUnschedulablePodsDuration,
|
||||||
activeQ: newActiveQueue(heap.NewWithRecorder(podInfoKeyFunc, heap.LessFunc[*framework.QueuedPodInfo](lessFn), metrics.NewActivePodsRecorder()), isSchedulingQueueHintEnabled, options.metricsRecorder),
|
backoffQ: backoffQ,
|
||||||
backoffQ: newBackoffQueue(options.clock, options.podInitialBackoffDuration, options.podMaxBackoffDuration, lessFn, isPopFromBackoffQEnabled),
|
|
||||||
unschedulablePods: newUnschedulablePods(metrics.NewUnschedulablePodsRecorder(), metrics.NewGatedPodsRecorder()),
|
unschedulablePods: newUnschedulablePods(metrics.NewUnschedulablePodsRecorder(), metrics.NewGatedPodsRecorder()),
|
||||||
preEnqueuePluginMap: options.preEnqueuePluginMap,
|
preEnqueuePluginMap: options.preEnqueuePluginMap,
|
||||||
queueingHintMap: options.queueingHintMap,
|
queueingHintMap: options.queueingHintMap,
|
||||||
@ -346,6 +346,11 @@ func NewPriorityQueue(
|
|||||||
isSchedulingQueueHintEnabled: isSchedulingQueueHintEnabled,
|
isSchedulingQueueHintEnabled: isSchedulingQueueHintEnabled,
|
||||||
isPopFromBackoffQEnabled: isPopFromBackoffQEnabled,
|
isPopFromBackoffQEnabled: isPopFromBackoffQEnabled,
|
||||||
}
|
}
|
||||||
|
var backoffQPopper backoffQPopper
|
||||||
|
if isPopFromBackoffQEnabled {
|
||||||
|
backoffQPopper = backoffQ
|
||||||
|
}
|
||||||
|
pq.activeQ = newActiveQueue(heap.NewWithRecorder(podInfoKeyFunc, heap.LessFunc[*framework.QueuedPodInfo](lessFn), metrics.NewActivePodsRecorder()), isSchedulingQueueHintEnabled, options.metricsRecorder, backoffQPopper)
|
||||||
pq.nsLister = informerFactory.Core().V1().Namespaces().Lister()
|
pq.nsLister = informerFactory.Core().V1().Namespaces().Lister()
|
||||||
pq.nominator = newPodNominator(options.podLister)
|
pq.nominator = newPodNominator(options.podLister)
|
||||||
|
|
||||||
@ -672,6 +677,12 @@ func (p *PriorityQueue) activate(logger klog.Logger, pod *v1.Pod) bool {
|
|||||||
if !exists {
|
if !exists {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
// Delete pod from the backoffQ now to make sure it won't be popped from the backoffQ
|
||||||
|
// just before moving it to the activeQ
|
||||||
|
if deleted := p.backoffQ.delete(pInfo); !deleted {
|
||||||
|
// Pod was popped from the backoffQ in the meantime. Don't activate it.
|
||||||
|
return false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if pInfo == nil {
|
if pInfo == nil {
|
||||||
@ -756,7 +767,11 @@ func (p *PriorityQueue) addUnschedulableWithoutQueueingHint(logger klog.Logger,
|
|||||||
// - No unschedulable plugins are associated with this Pod,
|
// - No unschedulable plugins are associated with this Pod,
|
||||||
// meaning something unusual (a temporal failure on kube-apiserver, etc) happened and this Pod gets moved back to the queue.
|
// meaning something unusual (a temporal failure on kube-apiserver, etc) happened and this Pod gets moved back to the queue.
|
||||||
// In this case, we should retry scheduling it because this Pod may not be retried until the next flush.
|
// In this case, we should retry scheduling it because this Pod may not be retried until the next flush.
|
||||||
_ = p.moveToBackoffQ(logger, pInfo, framework.ScheduleAttemptFailure)
|
if added := p.moveToBackoffQ(logger, pInfo, framework.ScheduleAttemptFailure); added {
|
||||||
|
if p.isPopFromBackoffQEnabled {
|
||||||
|
p.activeQ.broadcast()
|
||||||
|
}
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
p.unschedulablePods.addOrUpdate(pInfo, framework.ScheduleAttemptFailure)
|
p.unschedulablePods.addOrUpdate(pInfo, framework.ScheduleAttemptFailure)
|
||||||
logger.V(5).Info("Pod moved to an internal scheduling queue", "pod", klog.KObj(pod), "event", framework.ScheduleAttemptFailure, "queue", unschedulablePods)
|
logger.V(5).Info("Pod moved to an internal scheduling queue", "pod", klog.KObj(pod), "event", framework.ScheduleAttemptFailure, "queue", unschedulablePods)
|
||||||
@ -809,7 +824,7 @@ func (p *PriorityQueue) AddUnschedulableIfNotPresent(logger klog.Logger, pInfo *
|
|||||||
// In this case, we try to requeue this Pod to activeQ/backoffQ.
|
// In this case, we try to requeue this Pod to activeQ/backoffQ.
|
||||||
queue := p.requeuePodViaQueueingHint(logger, pInfo, schedulingHint, framework.ScheduleAttemptFailure)
|
queue := p.requeuePodViaQueueingHint(logger, pInfo, schedulingHint, framework.ScheduleAttemptFailure)
|
||||||
logger.V(3).Info("Pod moved to an internal scheduling queue", "pod", klog.KObj(pod), "event", framework.ScheduleAttemptFailure, "queue", queue, "schedulingCycle", podSchedulingCycle, "hint", schedulingHint, "unschedulable plugins", rejectorPlugins)
|
logger.V(3).Info("Pod moved to an internal scheduling queue", "pod", klog.KObj(pod), "event", framework.ScheduleAttemptFailure, "queue", queue, "schedulingCycle", podSchedulingCycle, "hint", schedulingHint, "unschedulable plugins", rejectorPlugins)
|
||||||
if queue == activeQ {
|
if queue == activeQ || (p.isPopFromBackoffQEnabled && queue == backoffQ) {
|
||||||
// When the Pod is moved to activeQ, need to let p.cond know so that the Pod will be pop()ed out.
|
// When the Pod is moved to activeQ, need to let p.cond know so that the Pod will be pop()ed out.
|
||||||
p.activeQ.broadcast()
|
p.activeQ.broadcast()
|
||||||
}
|
}
|
||||||
@ -822,11 +837,12 @@ func (p *PriorityQueue) flushBackoffQCompleted(logger klog.Logger) {
|
|||||||
p.lock.Lock()
|
p.lock.Lock()
|
||||||
defer p.lock.Unlock()
|
defer p.lock.Unlock()
|
||||||
activated := false
|
activated := false
|
||||||
p.backoffQ.popEachBackoffCompleted(logger, func(pInfo *framework.QueuedPodInfo) {
|
podsCompletedBackoff := p.backoffQ.popAllBackoffCompleted(logger)
|
||||||
|
for _, pInfo := range podsCompletedBackoff {
|
||||||
if added := p.moveToActiveQ(logger, pInfo, framework.BackoffComplete); added {
|
if added := p.moveToActiveQ(logger, pInfo, framework.BackoffComplete); added {
|
||||||
activated = true
|
activated = true
|
||||||
}
|
}
|
||||||
})
|
}
|
||||||
if activated {
|
if activated {
|
||||||
p.activeQ.broadcast()
|
p.activeQ.broadcast()
|
||||||
}
|
}
|
||||||
@ -954,7 +970,7 @@ func (p *PriorityQueue) Update(logger klog.Logger, oldPod, newPod *v1.Pod) {
|
|||||||
logger.V(5).Info("Pod moved to an internal scheduling queue because the Pod is updated", "pod", klog.KObj(newPod), "event", evt.Label(), "queue", queue)
|
logger.V(5).Info("Pod moved to an internal scheduling queue because the Pod is updated", "pod", klog.KObj(newPod), "event", evt.Label(), "queue", queue)
|
||||||
p.unschedulablePods.delete(pInfo.Pod, gated)
|
p.unschedulablePods.delete(pInfo.Pod, gated)
|
||||||
}
|
}
|
||||||
if queue == activeQ {
|
if queue == activeQ || (p.isPopFromBackoffQEnabled && queue == backoffQ) {
|
||||||
p.activeQ.broadcast()
|
p.activeQ.broadcast()
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
@ -967,6 +983,9 @@ func (p *PriorityQueue) Update(logger klog.Logger, oldPod, newPod *v1.Pod) {
|
|||||||
if p.backoffQ.isPodBackingoff(pInfo) {
|
if p.backoffQ.isPodBackingoff(pInfo) {
|
||||||
if added := p.moveToBackoffQ(logger, pInfo, framework.EventUnscheduledPodUpdate.Label()); added {
|
if added := p.moveToBackoffQ(logger, pInfo, framework.EventUnscheduledPodUpdate.Label()); added {
|
||||||
p.unschedulablePods.delete(pInfo.Pod, gated)
|
p.unschedulablePods.delete(pInfo.Pod, gated)
|
||||||
|
if p.isPopFromBackoffQEnabled {
|
||||||
|
p.activeQ.broadcast()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@ -995,12 +1014,14 @@ func (p *PriorityQueue) Delete(pod *v1.Pod) {
|
|||||||
defer p.lock.Unlock()
|
defer p.lock.Unlock()
|
||||||
p.DeleteNominatedPodIfExists(pod)
|
p.DeleteNominatedPodIfExists(pod)
|
||||||
pInfo := newQueuedPodInfoForLookup(pod)
|
pInfo := newQueuedPodInfoForLookup(pod)
|
||||||
if err := p.activeQ.delete(pInfo); err != nil {
|
if err := p.activeQ.delete(pInfo); err == nil {
|
||||||
// The item was probably not found in the activeQ.
|
return
|
||||||
p.backoffQ.delete(pInfo)
|
}
|
||||||
if pInfo = p.unschedulablePods.get(pod); pInfo != nil {
|
if deleted := p.backoffQ.delete(pInfo); deleted {
|
||||||
p.unschedulablePods.delete(pod, pInfo.Gated)
|
return
|
||||||
}
|
}
|
||||||
|
if pInfo = p.unschedulablePods.get(pod); pInfo != nil {
|
||||||
|
p.unschedulablePods.delete(pod, pInfo.Gated)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1127,7 +1148,7 @@ func (p *PriorityQueue) movePodsToActiveOrBackoffQueue(logger klog.Logger, podIn
|
|||||||
p.unschedulablePods.delete(pInfo.Pod, pInfo.Gated)
|
p.unschedulablePods.delete(pInfo.Pod, pInfo.Gated)
|
||||||
queue := p.requeuePodViaQueueingHint(logger, pInfo, schedulingHint, event.Label())
|
queue := p.requeuePodViaQueueingHint(logger, pInfo, schedulingHint, event.Label())
|
||||||
logger.V(4).Info("Pod moved to an internal scheduling queue", "pod", klog.KObj(pInfo.Pod), "event", event.Label(), "queue", queue, "hint", schedulingHint)
|
logger.V(4).Info("Pod moved to an internal scheduling queue", "pod", klog.KObj(pInfo.Pod), "event", event.Label(), "queue", queue, "hint", schedulingHint)
|
||||||
if queue == activeQ {
|
if queue == activeQ || (p.isPopFromBackoffQEnabled && queue == backoffQ) {
|
||||||
activated = true
|
activated = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -1222,11 +1243,13 @@ func (p *PriorityQueue) PendingPods() ([]*v1.Pod, string) {
|
|||||||
defer p.lock.RUnlock()
|
defer p.lock.RUnlock()
|
||||||
result := p.activeQ.list()
|
result := p.activeQ.list()
|
||||||
activeQLen := len(result)
|
activeQLen := len(result)
|
||||||
result = append(result, p.backoffQ.list()...)
|
backoffQPods := p.backoffQ.list()
|
||||||
|
backoffQLen := len(backoffQPods)
|
||||||
|
result = append(result, backoffQPods...)
|
||||||
for _, pInfo := range p.unschedulablePods.podInfoMap {
|
for _, pInfo := range p.unschedulablePods.podInfoMap {
|
||||||
result = append(result, pInfo.Pod)
|
result = append(result, pInfo.Pod)
|
||||||
}
|
}
|
||||||
return result, fmt.Sprintf(pendingPodsSummary, activeQLen, p.backoffQ.len(), len(p.unschedulablePods.podInfoMap))
|
return result, fmt.Sprintf(pendingPodsSummary, activeQLen, backoffQLen, len(p.unschedulablePods.podInfoMap))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Note: this function assumes the caller locks both p.lock.RLock and p.activeQ.getLock().RLock.
|
// Note: this function assumes the caller locks both p.lock.RLock and p.activeQ.getLock().RLock.
|
||||||
|
@ -1010,25 +1010,88 @@ func TestPriorityQueue_AddUnschedulableIfNotPresent_Backoff(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPriorityQueue_Pop(t *testing.T) {
|
// tryPop tries to pop one pod from the queue and returns it.
|
||||||
objs := []runtime.Object{medPriorityPodInfo.Pod}
|
// It waits 5 seconds before timing out, assuming the queue is then empty.
|
||||||
logger, ctx := ktesting.NewTestContext(t)
|
func tryPop(t *testing.T, logger klog.Logger, q *PriorityQueue) *framework.QueuedPodInfo {
|
||||||
ctx, cancel := context.WithCancel(ctx)
|
t.Helper()
|
||||||
defer cancel()
|
|
||||||
q := NewTestQueueWithObjects(ctx, newDefaultQueueSort(), objs)
|
var gotPod *framework.QueuedPodInfo
|
||||||
wg := sync.WaitGroup{}
|
popped := make(chan struct{}, 1)
|
||||||
wg.Add(1)
|
|
||||||
go func() {
|
go func() {
|
||||||
defer wg.Done()
|
pod, err := q.Pop(logger)
|
||||||
if p, err := q.Pop(logger); err != nil || p.Pod != medPriorityPodInfo.Pod {
|
if err != nil {
|
||||||
t.Errorf("Expected: %v after Pop, but got: %v", medPriorityPodInfo.Pod.Name, p.Pod.Name)
|
t.Errorf("Failed to pop pod from scheduling queue: %s", err)
|
||||||
}
|
}
|
||||||
if len(q.nominator.nominatedPods["node1"]) != 1 {
|
if pod != nil {
|
||||||
t.Errorf("Expected medPriorityPodInfo to be present in nominatedPods: %v", q.nominator.nominatedPods["node1"])
|
gotPod = pod
|
||||||
}
|
}
|
||||||
|
popped <- struct{}{}
|
||||||
}()
|
}()
|
||||||
q.Add(logger, medPriorityPodInfo.Pod)
|
|
||||||
wg.Wait()
|
timer := time.NewTimer(5 * time.Second)
|
||||||
|
select {
|
||||||
|
case <-timer.C:
|
||||||
|
q.Close()
|
||||||
|
case <-popped:
|
||||||
|
timer.Stop()
|
||||||
|
}
|
||||||
|
return gotPod
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPriorityQueue_Pop(t *testing.T) {
|
||||||
|
highPriorityPodInfo2 := mustNewPodInfo(
|
||||||
|
st.MakePod().Name("hpp2").Namespace("ns1").UID("hpp2ns1").Priority(highPriority).Obj(),
|
||||||
|
)
|
||||||
|
objs := []runtime.Object{medPriorityPodInfo.Pod, highPriorityPodInfo.Pod, highPriorityPodInfo2.Pod, unschedulablePodInfo.Pod}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
popFromBackoffQEnabled bool
|
||||||
|
wantPods []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "Pop pods from both activeQ and backoffQ when PopFromBackoffQ is enabled",
|
||||||
|
popFromBackoffQEnabled: true,
|
||||||
|
wantPods: []string{medPriorityPodInfo.Pod.Name, highPriorityPodInfo.Pod.Name},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Pop pod only from activeQ when PopFromBackoffQ is disabled",
|
||||||
|
popFromBackoffQEnabled: false,
|
||||||
|
wantPods: []string{medPriorityPodInfo.Pod.Name},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SchedulerPopFromBackoffQ, tt.popFromBackoffQEnabled)
|
||||||
|
logger, ctx := ktesting.NewTestContext(t)
|
||||||
|
ctx, cancel := context.WithCancel(ctx)
|
||||||
|
defer cancel()
|
||||||
|
q := NewTestQueueWithObjects(ctx, newDefaultQueueSort(), objs)
|
||||||
|
|
||||||
|
// Add medium priority pod to the activeQ
|
||||||
|
q.Add(logger, medPriorityPodInfo.Pod)
|
||||||
|
// Add high priority pod to the backoffQ
|
||||||
|
backoffPodInfo := q.newQueuedPodInfo(highPriorityPodInfo.Pod, "plugin")
|
||||||
|
q.backoffQ.add(logger, backoffPodInfo, framework.EventUnscheduledPodAdd.Label())
|
||||||
|
// Add high priority pod to the errorBackoffQ
|
||||||
|
errorBackoffPodInfo := q.newQueuedPodInfo(highPriorityPodInfo2.Pod)
|
||||||
|
q.backoffQ.add(logger, errorBackoffPodInfo, framework.EventUnscheduledPodAdd.Label())
|
||||||
|
// Add pod to the unschedulablePods
|
||||||
|
unschedulablePodInfo := q.newQueuedPodInfo(unschedulablePodInfo.Pod, "plugin")
|
||||||
|
q.unschedulablePods.addOrUpdate(unschedulablePodInfo, framework.EventUnscheduledPodAdd.Label())
|
||||||
|
|
||||||
|
var gotPods []string
|
||||||
|
for i := 0; i < len(tt.wantPods)+1; i++ {
|
||||||
|
gotPod := tryPop(t, logger, q)
|
||||||
|
if gotPod == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
gotPods = append(gotPods, gotPod.Pod.Name)
|
||||||
|
}
|
||||||
|
if diff := cmp.Diff(tt.wantPods, gotPods); diff != "" {
|
||||||
|
t.Errorf("Unexpected popped pods (-want, +got): %s", diff)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPriorityQueue_Update(t *testing.T) {
|
func TestPriorityQueue_Update(t *testing.T) {
|
||||||
@ -1951,7 +2014,7 @@ func TestPriorityQueue_MoveAllToActiveOrBackoffQueue(t *testing.T) {
|
|||||||
// pop out the pods in the backoffQ.
|
// pop out the pods in the backoffQ.
|
||||||
// This doesn't make them in-flight pods.
|
// This doesn't make them in-flight pods.
|
||||||
c.Step(q.backoffQ.podMaxBackoffDuration())
|
c.Step(q.backoffQ.podMaxBackoffDuration())
|
||||||
q.backoffQ.popEachBackoffCompleted(logger, nil)
|
_ = q.backoffQ.popAllBackoffCompleted(logger)
|
||||||
expectInFlightPods(t, q, medPriorityPodInfo.Pod.UID)
|
expectInFlightPods(t, q, medPriorityPodInfo.Pod.UID)
|
||||||
|
|
||||||
q.Add(logger, unschedulablePodInfo.Pod)
|
q.Add(logger, unschedulablePodInfo.Pod)
|
||||||
@ -2074,7 +2137,7 @@ func TestPriorityQueue_MoveAllToActiveOrBackoffQueueWithOutQueueingHint(t *testi
|
|||||||
// pop out the pods in the backoffQ.
|
// pop out the pods in the backoffQ.
|
||||||
// This doesn't make them in-flight pods.
|
// This doesn't make them in-flight pods.
|
||||||
c.Step(q.backoffQ.podMaxBackoffDuration())
|
c.Step(q.backoffQ.podMaxBackoffDuration())
|
||||||
q.backoffQ.popEachBackoffCompleted(logger, nil)
|
_ = q.backoffQ.popAllBackoffCompleted(logger)
|
||||||
|
|
||||||
unschedulableQueuedPodInfo := attemptQueuedPodInfo(q.newQueuedPodInfo(unschedulablePodInfo.Pod, "fooPlugin"))
|
unschedulableQueuedPodInfo := attemptQueuedPodInfo(q.newQueuedPodInfo(unschedulablePodInfo.Pod, "fooPlugin"))
|
||||||
highPriorityQueuedPodInfo := attemptQueuedPodInfo(q.newQueuedPodInfo(highPriorityPodInfo.Pod, "fooPlugin"))
|
highPriorityQueuedPodInfo := attemptQueuedPodInfo(q.newQueuedPodInfo(highPriorityPodInfo.Pod, "fooPlugin"))
|
||||||
@ -3883,9 +3946,10 @@ func TestMoveAllToActiveOrBackoffQueue_PreEnqueueChecks(t *testing.T) {
|
|||||||
q.MoveAllToActiveOrBackoffQueue(logger, tt.event, nil, nil, tt.preEnqueueCheck)
|
q.MoveAllToActiveOrBackoffQueue(logger, tt.event, nil, nil, tt.preEnqueueCheck)
|
||||||
got := sets.New[string]()
|
got := sets.New[string]()
|
||||||
c.Step(2 * q.backoffQ.podMaxBackoffDuration())
|
c.Step(2 * q.backoffQ.podMaxBackoffDuration())
|
||||||
q.backoffQ.popEachBackoffCompleted(logger, func(pInfo *framework.QueuedPodInfo) {
|
gotPodInfos := q.backoffQ.popAllBackoffCompleted(logger)
|
||||||
|
for _, pInfo := range gotPodInfos {
|
||||||
got.Insert(pInfo.Pod.Name)
|
got.Insert(pInfo.Pod.Name)
|
||||||
})
|
}
|
||||||
if diff := cmp.Diff(tt.want, got); diff != "" {
|
if diff := cmp.Diff(tt.want, got); diff != "" {
|
||||||
t.Errorf("Unexpected diff (-want, +got):\n%s", diff)
|
t.Errorf("Unexpected diff (-want, +got):\n%s", diff)
|
||||||
}
|
}
|
||||||
|
@ -31,6 +31,8 @@ const (
|
|||||||
ScheduleAttemptFailure = "ScheduleAttemptFailure"
|
ScheduleAttemptFailure = "ScheduleAttemptFailure"
|
||||||
// BackoffComplete is the event when a pod finishes backoff.
|
// BackoffComplete is the event when a pod finishes backoff.
|
||||||
BackoffComplete = "BackoffComplete"
|
BackoffComplete = "BackoffComplete"
|
||||||
|
// PopFromBackoffQ is the event when a pod is popped from backoffQ when activeQ is empty.
|
||||||
|
PopFromBackoffQ = "PopFromBackoffQ"
|
||||||
// ForceActivate is the event when a pod is moved from unschedulablePods/backoffQ
|
// ForceActivate is the event when a pod is moved from unschedulablePods/backoffQ
|
||||||
// to activeQ. Usually it's triggered by plugin implementations.
|
// to activeQ. Usually it's triggered by plugin implementations.
|
||||||
ForceActivate = "ForceActivate"
|
ForceActivate = "ForceActivate"
|
||||||
|
@ -212,6 +212,8 @@ func TestUpdateNominatedNodeName(t *testing.T) {
|
|||||||
for _, qHintEnabled := range []bool{false, true} {
|
for _, qHintEnabled := range []bool{false, true} {
|
||||||
t.Run(fmt.Sprintf("%s, with queuehint(%v)", tt.name, qHintEnabled), func(t *testing.T) {
|
t.Run(fmt.Sprintf("%s, with queuehint(%v)", tt.name, qHintEnabled), func(t *testing.T) {
|
||||||
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SchedulerQueueingHints, qHintEnabled)
|
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SchedulerQueueingHints, qHintEnabled)
|
||||||
|
// Set the SchedulerPopFromBackoffQ feature to false, because when it's enabled, we can't be sure the pod won't be popped from the backoffQ.
|
||||||
|
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SchedulerPopFromBackoffQ, false)
|
||||||
|
|
||||||
testCtx, teardown := schedulerutils.InitTestSchedulerForFrameworkTest(t, testContext, 0, true,
|
testCtx, teardown := schedulerutils.InitTestSchedulerForFrameworkTest(t, testContext, 0, true,
|
||||||
scheduler.WithClock(fakeClock),
|
scheduler.WithClock(fakeClock),
|
||||||
|
@ -309,14 +309,18 @@ var _ framework.PreFilterPlugin = &PreFilterPlugin{}
|
|||||||
var _ framework.PostFilterPlugin = &PostFilterPlugin{}
|
var _ framework.PostFilterPlugin = &PostFilterPlugin{}
|
||||||
var _ framework.ScorePlugin = &ScorePlugin{}
|
var _ framework.ScorePlugin = &ScorePlugin{}
|
||||||
var _ framework.FilterPlugin = &FilterPlugin{}
|
var _ framework.FilterPlugin = &FilterPlugin{}
|
||||||
|
var _ framework.EnqueueExtensions = &FilterPlugin{}
|
||||||
var _ framework.ScorePlugin = &ScorePlugin{}
|
var _ framework.ScorePlugin = &ScorePlugin{}
|
||||||
var _ framework.ScorePlugin = &ScoreWithNormalizePlugin{}
|
var _ framework.ScorePlugin = &ScoreWithNormalizePlugin{}
|
||||||
|
var _ framework.EnqueueExtensions = &ScorePlugin{}
|
||||||
var _ framework.ReservePlugin = &ReservePlugin{}
|
var _ framework.ReservePlugin = &ReservePlugin{}
|
||||||
var _ framework.PreScorePlugin = &PreScorePlugin{}
|
var _ framework.PreScorePlugin = &PreScorePlugin{}
|
||||||
var _ framework.PreBindPlugin = &PreBindPlugin{}
|
var _ framework.PreBindPlugin = &PreBindPlugin{}
|
||||||
|
var _ framework.EnqueueExtensions = &PreBindPlugin{}
|
||||||
var _ framework.BindPlugin = &BindPlugin{}
|
var _ framework.BindPlugin = &BindPlugin{}
|
||||||
var _ framework.PostBindPlugin = &PostBindPlugin{}
|
var _ framework.PostBindPlugin = &PostBindPlugin{}
|
||||||
var _ framework.PermitPlugin = &PermitPlugin{}
|
var _ framework.PermitPlugin = &PermitPlugin{}
|
||||||
|
var _ framework.EnqueueExtensions = &PermitPlugin{}
|
||||||
var _ framework.QueueSortPlugin = &QueueSortPlugin{}
|
var _ framework.QueueSortPlugin = &QueueSortPlugin{}
|
||||||
|
|
||||||
func (ep *QueueSortPlugin) Name() string {
|
func (ep *QueueSortPlugin) Name() string {
|
||||||
@ -377,6 +381,10 @@ func (sp *ScorePlugin) ScoreExtensions() framework.ScoreExtensions {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (sp *ScorePlugin) EventsToRegister(_ context.Context) ([]framework.ClusterEventWithHint, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
// Name returns name of the score plugin.
|
// Name returns name of the score plugin.
|
||||||
func (sp *ScoreWithNormalizePlugin) Name() string {
|
func (sp *ScoreWithNormalizePlugin) Name() string {
|
||||||
return scoreWithNormalizePluginName
|
return scoreWithNormalizePluginName
|
||||||
@ -427,6 +435,12 @@ func (fp *FilterPlugin) Filter(ctx context.Context, state *framework.CycleState,
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (fp *FilterPlugin) EventsToRegister(_ context.Context) ([]framework.ClusterEventWithHint, error) {
|
||||||
|
return []framework.ClusterEventWithHint{
|
||||||
|
{Event: framework.ClusterEvent{Resource: framework.Pod, ActionType: framework.Delete}},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
// Name returns name of the plugin.
|
// Name returns name of the plugin.
|
||||||
func (rp *ReservePlugin) Name() string {
|
func (rp *ReservePlugin) Name() string {
|
||||||
return rp.name
|
return rp.name
|
||||||
@ -491,6 +505,10 @@ func (pp *PreBindPlugin) PreBind(ctx context.Context, state *framework.CycleStat
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (pp *PreBindPlugin) EventsToRegister(_ context.Context) ([]framework.ClusterEventWithHint, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
const bindPluginAnnotation = "bindPluginName"
|
const bindPluginAnnotation = "bindPluginName"
|
||||||
|
|
||||||
func (bp *BindPlugin) Name() string {
|
func (bp *BindPlugin) Name() string {
|
||||||
@ -651,6 +669,10 @@ func (pp *PermitPlugin) rejectAllPods() {
|
|||||||
pp.fh.IterateOverWaitingPods(func(wp framework.WaitingPod) { wp.Reject(pp.name, "rejectAllPods") })
|
pp.fh.IterateOverWaitingPods(func(wp framework.WaitingPod) { wp.Reject(pp.name, "rejectAllPods") })
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (pp *PermitPlugin) EventsToRegister(_ context.Context) ([]framework.ClusterEventWithHint, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
// TestPreFilterPlugin tests invocation of prefilter plugins.
|
// TestPreFilterPlugin tests invocation of prefilter plugins.
|
||||||
func TestPreFilterPlugin(t *testing.T) {
|
func TestPreFilterPlugin(t *testing.T) {
|
||||||
testContext := testutils.InitTestAPIServer(t, "prefilter-plugin", nil)
|
testContext := testutils.InitTestAPIServer(t, "prefilter-plugin", nil)
|
||||||
|
@ -581,3 +581,38 @@ func (p *fakePermitPlugin) EventsToRegister(_ context.Context) ([]framework.Clus
|
|||||||
{Event: framework.ClusterEvent{Resource: framework.Node, ActionType: framework.UpdateNodeLabel}, QueueingHintFn: p.schedulingHint},
|
{Event: framework.ClusterEvent{Resource: framework.Node, ActionType: framework.UpdateNodeLabel}, QueueingHintFn: p.schedulingHint},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPopFromBackoffQWhenActiveQEmpty(t *testing.T) {
|
||||||
|
// Set initial backoff to 1000s to make sure pod won't go to the activeQ after being requeued.
|
||||||
|
testCtx := testutils.InitTestSchedulerWithNS(t, "pop-from-backoffq", scheduler.WithPodInitialBackoffSeconds(1000), scheduler.WithPodMaxBackoffSeconds(1000))
|
||||||
|
cs, ns, ctx := testCtx.ClientSet, testCtx.NS.Name, testCtx.Ctx
|
||||||
|
|
||||||
|
// Create node, so we can schedule pods.
|
||||||
|
node := st.MakeNode().Name("node").Obj()
|
||||||
|
if _, err := cs.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}); err != nil {
|
||||||
|
t.Fatal("Failed to create node")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a pod that will be unschedulable.
|
||||||
|
pod := st.MakePod().Namespace(ns).Name("pod").NodeAffinityIn("foo", []string{"bar"}, st.NodeSelectorTypeMatchExpressions).Container(imageutils.GetPauseImageName()).Obj()
|
||||||
|
if _, err := cs.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}); err != nil {
|
||||||
|
t.Fatalf("Failed to create pod: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err := wait.PollUntilContextTimeout(ctx, 200*time.Millisecond, wait.ForeverTestTimeout, false, testutils.PodUnschedulable(cs, ns, pod.Name))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Expected pod to be unschedulable: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create node with label to make the pod schedulable.
|
||||||
|
node2 := st.MakeNode().Name("node-schedulable").Label("foo", "bar").Obj()
|
||||||
|
if _, err := cs.CoreV1().Nodes().Create(ctx, node2, metav1.CreateOptions{}); err != nil {
|
||||||
|
t.Fatal("Failed to create node-schedulable")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pod should be scheduled, even if it was in the backoffQ, because PopFromBackoffQ feature is enabled.
|
||||||
|
err = wait.PollUntilContextTimeout(ctx, 200*time.Millisecond, wait.ForeverTestTimeout, false, testutils.PodScheduled(cs, ns, pod.Name))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Expected pod to be scheduled: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user