mirror of
https://github.com/k3s-io/kubernetes.git
synced 2026-07-24 21:13:16 +00:00
265 lines
8.3 KiB
Go
265 lines
8.3 KiB
Go
/*
|
|
Copyright 2022 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package scheduler
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"testing"
|
|
"time"
|
|
|
|
v1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/util/wait"
|
|
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
|
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
|
"k8s.io/klog/v2"
|
|
fwk "k8s.io/kube-scheduler/framework"
|
|
"k8s.io/kubernetes/pkg/features"
|
|
"k8s.io/kubernetes/pkg/scheduler"
|
|
st "k8s.io/kubernetes/pkg/scheduler/testing"
|
|
testutils "k8s.io/kubernetes/test/integration/util"
|
|
)
|
|
|
|
var _ fwk.PermitPlugin = &PermitPlugin{}
|
|
var _ fwk.EnqueueExtensions = &PermitPlugin{}
|
|
var _ fwk.ReservePlugin = &ReservePlugin{}
|
|
var _ fwk.EnqueueExtensions = &ReservePlugin{}
|
|
|
|
type ResettablePlugin interface {
|
|
fwk.Plugin
|
|
Reset()
|
|
}
|
|
|
|
type ReservePlugin struct {
|
|
name string
|
|
statusCode fwk.Code
|
|
numReserveCalled int
|
|
numUnreserveCalled int
|
|
}
|
|
|
|
func (rp *ReservePlugin) Name() string {
|
|
return rp.name
|
|
}
|
|
|
|
func (rp *ReservePlugin) Reserve(ctx context.Context, state fwk.CycleState, p *v1.Pod, nodeName string) *fwk.Status {
|
|
rp.numReserveCalled += 1
|
|
|
|
if rp.statusCode == fwk.Error {
|
|
return fwk.NewStatus(fwk.Error, "failed to reserve")
|
|
}
|
|
|
|
if rp.statusCode == fwk.Unschedulable {
|
|
if rp.numReserveCalled <= 1 {
|
|
return fwk.NewStatus(fwk.Unschedulable, "reject to reserve")
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (rp *ReservePlugin) Unreserve(ctx context.Context, state fwk.CycleState, p *v1.Pod, nodeName string) {
|
|
rp.numUnreserveCalled += 1
|
|
}
|
|
|
|
func (rp *ReservePlugin) EventsToRegister(_ context.Context) ([]fwk.ClusterEventWithHint, error) {
|
|
return []fwk.ClusterEventWithHint{
|
|
{
|
|
Event: fwk.ClusterEvent{Resource: fwk.Node, ActionType: fwk.Add},
|
|
QueueingHintFn: func(logger klog.Logger, pod *v1.Pod, oldObj, newObj interface{}) (fwk.QueueingHint, error) {
|
|
return fwk.Queue, nil
|
|
},
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
func (rp *ReservePlugin) Reset() {
|
|
rp.numReserveCalled = 0
|
|
rp.numUnreserveCalled = 0
|
|
}
|
|
|
|
type PermitPlugin struct {
|
|
name string
|
|
statusCode fwk.Code
|
|
numPermitCalled int
|
|
}
|
|
|
|
func (pp *PermitPlugin) Name() string {
|
|
return pp.name
|
|
}
|
|
|
|
func (pp *PermitPlugin) Permit(ctx context.Context, state fwk.CycleState, p *v1.Pod, nodeName string) (*fwk.Status, time.Duration) {
|
|
pp.numPermitCalled += 1
|
|
|
|
if pp.statusCode == fwk.Error {
|
|
return fwk.NewStatus(fwk.Error, "failed to permit"), 0
|
|
}
|
|
|
|
if pp.statusCode == fwk.Unschedulable {
|
|
if pp.numPermitCalled <= 1 {
|
|
return fwk.NewStatus(fwk.Unschedulable, "reject to permit"), 0
|
|
}
|
|
}
|
|
|
|
return nil, 0
|
|
}
|
|
|
|
func (pp *PermitPlugin) EventsToRegister(_ context.Context) ([]fwk.ClusterEventWithHint, error) {
|
|
return []fwk.ClusterEventWithHint{
|
|
{
|
|
Event: fwk.ClusterEvent{Resource: fwk.Node, ActionType: fwk.Add},
|
|
QueueingHintFn: func(logger klog.Logger, pod *v1.Pod, oldObj, newObj interface{}) (fwk.QueueingHint, error) {
|
|
return fwk.Queue, nil
|
|
},
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
func (pp *PermitPlugin) Reset() {
|
|
pp.numPermitCalled = 0
|
|
}
|
|
|
|
func TestReScheduling(t *testing.T) {
|
|
testContext := testutils.InitTestAPIServer(t, "permit-plugin", nil)
|
|
tests := []struct {
|
|
name string
|
|
plugin ResettablePlugin
|
|
action func() error
|
|
// The first time for pod scheduling, we make pod scheduled error or unschedulable on purpose.
|
|
// This is controlled by wantFirstSchedulingError. By default, pod is unschedulable.
|
|
wantFirstSchedulingError bool
|
|
|
|
// wantScheduled/wantError means the final expected scheduling result.
|
|
wantScheduled bool
|
|
wantError bool
|
|
}{
|
|
{
|
|
name: "Rescheduling pod rejected by Permit Plugin",
|
|
plugin: &PermitPlugin{name: "permit", statusCode: fwk.Unschedulable},
|
|
action: func() error {
|
|
_, err := testutils.CreateNode(testContext.ClientSet, st.MakeNode().Name("fake-node").Obj())
|
|
return err
|
|
},
|
|
wantScheduled: true,
|
|
},
|
|
{
|
|
name: "Rescheduling pod rejected by Permit Plugin with unrelated event",
|
|
plugin: &PermitPlugin{name: "permit", statusCode: fwk.Unschedulable},
|
|
action: func() error {
|
|
_, err := testutils.CreatePausePod(testContext.ClientSet,
|
|
testutils.InitPausePod(&testutils.PausePodConfig{Name: "test-pod-2", Namespace: testContext.NS.Name}))
|
|
return err
|
|
},
|
|
wantScheduled: false,
|
|
},
|
|
{
|
|
name: "Rescheduling pod failed by Permit Plugin",
|
|
plugin: &PermitPlugin{name: "permit", statusCode: fwk.Error},
|
|
action: func() error {
|
|
_, err := testutils.CreateNode(testContext.ClientSet, st.MakeNode().Name("fake-node").Obj())
|
|
return err
|
|
},
|
|
wantFirstSchedulingError: true,
|
|
wantError: true,
|
|
},
|
|
{
|
|
name: "Rescheduling pod rejected by Reserve Plugin",
|
|
plugin: &ReservePlugin{name: "reserve", statusCode: fwk.Unschedulable},
|
|
action: func() error {
|
|
_, err := testutils.CreateNode(testContext.ClientSet, st.MakeNode().Name("fake-node").Obj())
|
|
return err
|
|
},
|
|
wantScheduled: true,
|
|
},
|
|
{
|
|
name: "Rescheduling pod rejected by Reserve Plugin with unrelated event",
|
|
plugin: &ReservePlugin{name: "reserve", statusCode: fwk.Unschedulable},
|
|
action: func() error {
|
|
_, err := testutils.CreatePausePod(testContext.ClientSet,
|
|
testutils.InitPausePod(&testutils.PausePodConfig{Name: "test-pod-2", Namespace: testContext.NS.Name}))
|
|
return err
|
|
},
|
|
wantScheduled: false,
|
|
},
|
|
{
|
|
name: "Rescheduling pod failed by Reserve Plugin",
|
|
plugin: &ReservePlugin{name: "reserve", statusCode: fwk.Error},
|
|
action: func() error {
|
|
_, err := testutils.CreateNode(testContext.ClientSet, st.MakeNode().Name("fake-node").Obj())
|
|
return err
|
|
},
|
|
wantFirstSchedulingError: true,
|
|
wantError: true,
|
|
},
|
|
}
|
|
|
|
for _, asyncAPICallsEnabled := range []bool{true, false} {
|
|
for _, test := range tests {
|
|
t.Run(fmt.Sprintf("%s (Async API calls enabled: %v)", test.name, asyncAPICallsEnabled), func(t *testing.T) {
|
|
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SchedulerAsyncAPICalls, asyncAPICallsEnabled)
|
|
|
|
// Create a plugin registry for testing. Register only a permit plugin.
|
|
registry, prof := InitRegistryAndConfig(t, nil, test.plugin)
|
|
t.Cleanup(test.plugin.Reset)
|
|
|
|
testCtx, teardown := InitTestSchedulerForFrameworkTest(t, testContext, 2, true,
|
|
scheduler.WithProfiles(prof),
|
|
scheduler.WithFrameworkOutOfTreeRegistry(registry))
|
|
defer teardown()
|
|
|
|
pod, err := testutils.CreatePausePod(testCtx.ClientSet,
|
|
testutils.InitPausePod(&testutils.PausePodConfig{Name: "test-pod", Namespace: testCtx.NS.Name}))
|
|
if err != nil {
|
|
t.Errorf("Error while creating a test pod: %v", err)
|
|
}
|
|
|
|
// The first time for scheduling, pod is error or unschedulable, controlled by wantFirstSchedulingError
|
|
if test.wantFirstSchedulingError {
|
|
if err = wait.PollUntilContextTimeout(testCtx.Ctx, 10*time.Millisecond, 30*time.Second, false,
|
|
testutils.PodSchedulingError(testCtx.ClientSet, pod.Namespace, pod.Name)); err != nil {
|
|
t.Errorf("Expected a scheduling error, but got: %v", err)
|
|
}
|
|
} else {
|
|
if err = testutils.WaitForPodUnschedulable(testCtx.Ctx, testCtx.ClientSet, pod); err != nil {
|
|
t.Errorf("Didn't expect the pod to be scheduled. error: %v", err)
|
|
}
|
|
}
|
|
|
|
if test.action != nil {
|
|
if err = test.action(); err != nil {
|
|
t.Errorf("Perform action() error: %v", err)
|
|
}
|
|
}
|
|
|
|
if test.wantScheduled {
|
|
if err = testutils.WaitForPodToSchedule(testCtx.Ctx, testCtx.ClientSet, pod); err != nil {
|
|
t.Errorf("Didn't expect the pod to be unschedulable. error: %v", err)
|
|
}
|
|
} else if test.wantError {
|
|
if err = wait.PollUntilContextTimeout(testCtx.Ctx, 10*time.Millisecond, 30*time.Second, false,
|
|
testutils.PodSchedulingError(testCtx.ClientSet, pod.Namespace, pod.Name)); err != nil {
|
|
t.Errorf("Expected a scheduling error, but got: %v", err)
|
|
}
|
|
} else {
|
|
if err = testutils.WaitForPodUnschedulable(testCtx.Ctx, testCtx.ClientSet, pod); err != nil {
|
|
t.Errorf("Didn't expect the pod to be scheduled. error: %v", err)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|