mirror of
https://github.com/k3s-io/kubernetes.git
synced 2026-01-05 15:37:24 +00:00
feature(scheduler): implement ClusterEventWithHint to filter out useless events
This commit is contained in:
Kensei Nakada
@@ -39,7 +39,9 @@ import (
|
||||
"k8s.io/klog/v2"
|
||||
"k8s.io/klog/v2/ktesting"
|
||||
schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config"
|
||||
"k8s.io/kubernetes/pkg/scheduler/apis/config/testing/defaults"
|
||||
"k8s.io/kubernetes/pkg/scheduler/framework"
|
||||
"k8s.io/kubernetes/pkg/scheduler/framework/plugins"
|
||||
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder"
|
||||
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/queuesort"
|
||||
frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime"
|
||||
@@ -643,3 +645,440 @@ func indexByPodAnnotationNodeName(obj interface{}) ([]string, error) {
|
||||
}
|
||||
return []string{nodeName}, nil
|
||||
}
|
||||
|
||||
const (
|
||||
fakeNoop = "fakeNoop"
|
||||
fakeNode = "fakeNode"
|
||||
fakePod = "fakePod"
|
||||
fakeNoopRuntime = "fakeNoopRuntime"
|
||||
queueSort = "no-op-queue-sort-plugin"
|
||||
fakeBind = "bind-plugin"
|
||||
)
|
||||
|
||||
func Test_buildQueueingHintMap(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
plugins []framework.Plugin
|
||||
want map[framework.ClusterEvent][]*internalqueue.QueueingHintFunction
|
||||
assertFn func(t *testing.T, got map[framework.ClusterEvent][]*internalqueue.QueueingHintFunction) bool
|
||||
}{
|
||||
{
|
||||
name: "no-op plugin",
|
||||
plugins: []framework.Plugin{&fakeNoopPlugin{}},
|
||||
want: map[framework.ClusterEvent][]*internalqueue.QueueingHintFunction{
|
||||
{Resource: framework.Pod, ActionType: framework.All}: {
|
||||
{PluginName: fakeBind, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: fakeNoop, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: queueSort, QueueingHintFn: defaultQueueingHintFn},
|
||||
},
|
||||
{Resource: framework.Node, ActionType: framework.All}: {
|
||||
{PluginName: fakeBind, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: fakeNoop, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: queueSort, QueueingHintFn: defaultQueueingHintFn},
|
||||
},
|
||||
{Resource: framework.CSINode, ActionType: framework.All}: {
|
||||
{PluginName: fakeBind, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: fakeNoop, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: queueSort, QueueingHintFn: defaultQueueingHintFn},
|
||||
},
|
||||
{Resource: framework.CSIDriver, ActionType: framework.All}: {
|
||||
{PluginName: fakeBind, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: fakeNoop, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: queueSort, QueueingHintFn: defaultQueueingHintFn},
|
||||
},
|
||||
{Resource: framework.CSIStorageCapacity, ActionType: framework.All}: {
|
||||
{PluginName: fakeBind, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: fakeNoop, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: queueSort, QueueingHintFn: defaultQueueingHintFn},
|
||||
},
|
||||
{Resource: framework.PersistentVolume, ActionType: framework.All}: {
|
||||
{PluginName: fakeBind, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: fakeNoop, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: queueSort, QueueingHintFn: defaultQueueingHintFn},
|
||||
},
|
||||
{Resource: framework.StorageClass, ActionType: framework.All}: {
|
||||
{PluginName: fakeBind, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: fakeNoop, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: queueSort, QueueingHintFn: defaultQueueingHintFn},
|
||||
},
|
||||
{Resource: framework.PersistentVolumeClaim, ActionType: framework.All}: {
|
||||
{PluginName: fakeBind, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: fakeNoop, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: queueSort, QueueingHintFn: defaultQueueingHintFn},
|
||||
},
|
||||
{Resource: framework.PodSchedulingContext, ActionType: framework.All}: {
|
||||
{PluginName: fakeBind, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: fakeNoop, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: queueSort, QueueingHintFn: defaultQueueingHintFn},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "node and pod plugin",
|
||||
plugins: []framework.Plugin{&fakeNodePlugin{}, &fakePodPlugin{}},
|
||||
want: map[framework.ClusterEvent][]*internalqueue.QueueingHintFunction{
|
||||
{Resource: framework.Pod, ActionType: framework.All}: {
|
||||
{PluginName: fakeBind, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: queueSort, QueueingHintFn: defaultQueueingHintFn},
|
||||
},
|
||||
{Resource: framework.Pod, ActionType: framework.Add}: {
|
||||
{PluginName: fakePod, QueueingHintFn: fakePodPluginQueueingFn},
|
||||
},
|
||||
{Resource: framework.Node, ActionType: framework.All}: {
|
||||
{PluginName: fakeBind, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: queueSort, QueueingHintFn: defaultQueueingHintFn},
|
||||
},
|
||||
{Resource: framework.Node, ActionType: framework.Add}: {
|
||||
{PluginName: fakeNode, QueueingHintFn: fakeNodePluginQueueingFn},
|
||||
},
|
||||
{Resource: framework.CSINode, ActionType: framework.All}: {
|
||||
{PluginName: fakeBind, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: queueSort, QueueingHintFn: defaultQueueingHintFn},
|
||||
},
|
||||
{Resource: framework.CSIDriver, ActionType: framework.All}: {
|
||||
{PluginName: fakeBind, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: queueSort, QueueingHintFn: defaultQueueingHintFn},
|
||||
},
|
||||
{Resource: framework.CSIStorageCapacity, ActionType: framework.All}: {
|
||||
{PluginName: fakeBind, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: queueSort, QueueingHintFn: defaultQueueingHintFn},
|
||||
},
|
||||
{Resource: framework.PersistentVolume, ActionType: framework.All}: {
|
||||
{PluginName: fakeBind, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: queueSort, QueueingHintFn: defaultQueueingHintFn},
|
||||
},
|
||||
{Resource: framework.StorageClass, ActionType: framework.All}: {
|
||||
{PluginName: fakeBind, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: queueSort, QueueingHintFn: defaultQueueingHintFn},
|
||||
},
|
||||
{Resource: framework.PersistentVolumeClaim, ActionType: framework.All}: {
|
||||
{PluginName: fakeBind, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: queueSort, QueueingHintFn: defaultQueueingHintFn},
|
||||
},
|
||||
{Resource: framework.PodSchedulingContext, ActionType: framework.All}: {
|
||||
{PluginName: fakeBind, QueueingHintFn: defaultQueueingHintFn},
|
||||
{PluginName: queueSort, QueueingHintFn: defaultQueueingHintFn},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
registry := frameworkruntime.Registry{}
|
||||
cfgPls := &schedulerapi.Plugins{}
|
||||
plugins := append(tt.plugins, &fakebindPlugin{}, &fakeQueueSortPlugin{})
|
||||
for _, pl := range plugins {
|
||||
tmpPl := pl
|
||||
if err := registry.Register(pl.Name(), func(_ runtime.Object, _ framework.Handle) (framework.Plugin, error) {
|
||||
return tmpPl, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("fail to register filter plugin (%s)", pl.Name())
|
||||
}
|
||||
cfgPls.MultiPoint.Enabled = append(cfgPls.MultiPoint.Enabled, schedulerapi.Plugin{Name: pl.Name()})
|
||||
}
|
||||
|
||||
profile := schedulerapi.KubeSchedulerProfile{Plugins: cfgPls}
|
||||
stopCh := make(chan struct{})
|
||||
defer close(stopCh)
|
||||
fwk, err := newFramework(registry, profile, stopCh)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
exts := fwk.EnqueueExtensions()
|
||||
// need to sort to make the test result stable.
|
||||
sort.Slice(exts, func(i, j int) bool {
|
||||
return exts[i].Name() < exts[j].Name()
|
||||
})
|
||||
|
||||
got := buildQueueingHintMap(exts)
|
||||
|
||||
for e, fns := range got {
|
||||
wantfns, ok := tt.want[e]
|
||||
if !ok {
|
||||
t.Errorf("got unexpected event %v", e)
|
||||
continue
|
||||
}
|
||||
if len(fns) != len(wantfns) {
|
||||
t.Errorf("got %v queueing hint functions, want %v", len(fns), len(wantfns))
|
||||
continue
|
||||
}
|
||||
for i, fn := range fns {
|
||||
if fn.PluginName != wantfns[i].PluginName {
|
||||
t.Errorf("got plugin name %v, want %v", fn.PluginName, wantfns[i].PluginName)
|
||||
continue
|
||||
}
|
||||
if fn.QueueingHintFn(nil, nil, nil) != wantfns[i].QueueingHintFn(nil, nil, nil) {
|
||||
t.Errorf("got queueing hint function (%v) returning %v, expect it to return %v", fn.PluginName, fn.QueueingHintFn(nil, nil, nil), wantfns[i].QueueingHintFn(nil, nil, nil))
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Test_UnionedGVKs tests UnionedGVKs worked with buildQueueingHintMap.
|
||||
func Test_UnionedGVKs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
plugins schedulerapi.PluginSet
|
||||
want map[framework.GVK]framework.ActionType
|
||||
}{
|
||||
{
|
||||
name: "no-op plugin",
|
||||
plugins: schedulerapi.PluginSet{
|
||||
Enabled: []schedulerapi.Plugin{
|
||||
{Name: fakeNoop},
|
||||
{Name: queueSort},
|
||||
{Name: fakeBind},
|
||||
},
|
||||
Disabled: []schedulerapi.Plugin{{Name: "*"}}, // disable default plugins
|
||||
},
|
||||
want: map[framework.GVK]framework.ActionType{
|
||||
framework.Pod: framework.All,
|
||||
framework.Node: framework.All,
|
||||
framework.CSINode: framework.All,
|
||||
framework.CSIDriver: framework.All,
|
||||
framework.CSIStorageCapacity: framework.All,
|
||||
framework.PersistentVolume: framework.All,
|
||||
framework.PersistentVolumeClaim: framework.All,
|
||||
framework.StorageClass: framework.All,
|
||||
framework.PodSchedulingContext: framework.All,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "node plugin",
|
||||
plugins: schedulerapi.PluginSet{
|
||||
Enabled: []schedulerapi.Plugin{
|
||||
{Name: fakeNode},
|
||||
{Name: queueSort},
|
||||
{Name: fakeBind},
|
||||
},
|
||||
Disabled: []schedulerapi.Plugin{{Name: "*"}}, // disable default plugins
|
||||
},
|
||||
want: map[framework.GVK]framework.ActionType{
|
||||
framework.Pod: framework.All,
|
||||
framework.Node: framework.All,
|
||||
framework.CSINode: framework.All,
|
||||
framework.CSIDriver: framework.All,
|
||||
framework.CSIStorageCapacity: framework.All,
|
||||
framework.PersistentVolume: framework.All,
|
||||
framework.PersistentVolumeClaim: framework.All,
|
||||
framework.StorageClass: framework.All,
|
||||
framework.PodSchedulingContext: framework.All,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "pod plugin",
|
||||
plugins: schedulerapi.PluginSet{
|
||||
Enabled: []schedulerapi.Plugin{
|
||||
{Name: fakePod},
|
||||
{Name: queueSort},
|
||||
{Name: fakeBind},
|
||||
},
|
||||
Disabled: []schedulerapi.Plugin{{Name: "*"}}, // disable default plugins
|
||||
},
|
||||
want: map[framework.GVK]framework.ActionType{
|
||||
framework.Pod: framework.All,
|
||||
framework.Node: framework.All,
|
||||
framework.CSINode: framework.All,
|
||||
framework.CSIDriver: framework.All,
|
||||
framework.CSIStorageCapacity: framework.All,
|
||||
framework.PersistentVolume: framework.All,
|
||||
framework.PersistentVolumeClaim: framework.All,
|
||||
framework.StorageClass: framework.All,
|
||||
framework.PodSchedulingContext: framework.All,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "node and pod plugin",
|
||||
plugins: schedulerapi.PluginSet{
|
||||
Enabled: []schedulerapi.Plugin{
|
||||
{Name: fakePod},
|
||||
{Name: fakeNode},
|
||||
{Name: queueSort},
|
||||
{Name: fakeBind},
|
||||
},
|
||||
Disabled: []schedulerapi.Plugin{{Name: "*"}}, // disable default plugins
|
||||
},
|
||||
want: map[framework.GVK]framework.ActionType{
|
||||
framework.Pod: framework.All,
|
||||
framework.Node: framework.All,
|
||||
framework.CSINode: framework.All,
|
||||
framework.CSIDriver: framework.All,
|
||||
framework.CSIStorageCapacity: framework.All,
|
||||
framework.PersistentVolume: framework.All,
|
||||
framework.PersistentVolumeClaim: framework.All,
|
||||
framework.StorageClass: framework.All,
|
||||
framework.PodSchedulingContext: framework.All,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "no-op runtime plugin",
|
||||
plugins: schedulerapi.PluginSet{
|
||||
Enabled: []schedulerapi.Plugin{
|
||||
{Name: fakeNoopRuntime},
|
||||
{Name: queueSort},
|
||||
{Name: fakeBind},
|
||||
},
|
||||
Disabled: []schedulerapi.Plugin{{Name: "*"}}, // disable default plugins
|
||||
},
|
||||
want: map[framework.GVK]framework.ActionType{
|
||||
framework.Pod: framework.All,
|
||||
framework.Node: framework.All,
|
||||
framework.CSINode: framework.All,
|
||||
framework.CSIDriver: framework.All,
|
||||
framework.CSIStorageCapacity: framework.All,
|
||||
framework.PersistentVolume: framework.All,
|
||||
framework.PersistentVolumeClaim: framework.All,
|
||||
framework.StorageClass: framework.All,
|
||||
framework.PodSchedulingContext: framework.All,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "plugins with default profile",
|
||||
plugins: schedulerapi.PluginSet{Enabled: defaults.PluginsV1.MultiPoint.Enabled},
|
||||
want: map[framework.GVK]framework.ActionType{
|
||||
framework.Pod: framework.All,
|
||||
framework.Node: framework.All,
|
||||
framework.CSINode: framework.All,
|
||||
framework.CSIDriver: framework.All,
|
||||
framework.CSIStorageCapacity: framework.All,
|
||||
framework.PersistentVolume: framework.All,
|
||||
framework.PersistentVolumeClaim: framework.All,
|
||||
framework.StorageClass: framework.All,
|
||||
framework.PodSchedulingContext: framework.All,
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
registry := plugins.NewInTreeRegistry()
|
||||
|
||||
cfgPls := &schedulerapi.Plugins{MultiPoint: tt.plugins}
|
||||
plugins := []framework.Plugin{&fakeNodePlugin{}, &fakePodPlugin{}, &fakeNoopPlugin{}, &fakeNoopRuntimePlugin{}, &fakeQueueSortPlugin{}, &fakebindPlugin{}}
|
||||
for _, pl := range plugins {
|
||||
tmpPl := pl
|
||||
if err := registry.Register(pl.Name(), func(_ runtime.Object, _ framework.Handle) (framework.Plugin, error) {
|
||||
return tmpPl, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("fail to register filter plugin (%s)", pl.Name())
|
||||
}
|
||||
}
|
||||
|
||||
profile := schedulerapi.KubeSchedulerProfile{Plugins: cfgPls, PluginConfig: defaults.PluginConfigsV1}
|
||||
stopCh := make(chan struct{})
|
||||
defer close(stopCh)
|
||||
fwk, err := newFramework(registry, profile, stopCh)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
queueingHintsPerProfile := internalqueue.QueueingHintMapPerProfile{
|
||||
"default": buildQueueingHintMap(fwk.EnqueueExtensions()),
|
||||
}
|
||||
got := unionedGVKs(queueingHintsPerProfile)
|
||||
|
||||
if diff := cmp.Diff(tt.want, got); diff != "" {
|
||||
t.Errorf("Unexpected eventToPlugin map (-want,+got):%s", diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func newFramework(r frameworkruntime.Registry, profile schedulerapi.KubeSchedulerProfile, stopCh <-chan struct{}) (framework.Framework, error) {
|
||||
return frameworkruntime.NewFramework(context.Background(), r, &profile,
|
||||
frameworkruntime.WithSnapshotSharedLister(internalcache.NewSnapshot(nil, nil)),
|
||||
frameworkruntime.WithInformerFactory(informers.NewSharedInformerFactory(fake.NewSimpleClientset(), 0)),
|
||||
)
|
||||
}
|
||||
|
||||
var _ framework.QueueSortPlugin = &fakeQueueSortPlugin{}
|
||||
|
||||
// fakeQueueSortPlugin is a no-op implementation for QueueSort extension point.
|
||||
type fakeQueueSortPlugin struct{}
|
||||
|
||||
func (pl *fakeQueueSortPlugin) Name() string {
|
||||
return queueSort
|
||||
}
|
||||
|
||||
func (pl *fakeQueueSortPlugin) Less(_, _ *framework.QueuedPodInfo) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
var _ framework.BindPlugin = &fakebindPlugin{}
|
||||
|
||||
// fakebindPlugin is a no-op implementation for Bind extension point.
|
||||
type fakebindPlugin struct{}
|
||||
|
||||
func (t *fakebindPlugin) Name() string {
|
||||
return fakeBind
|
||||
}
|
||||
|
||||
func (t *fakebindPlugin) Bind(ctx context.Context, state *framework.CycleState, p *v1.Pod, nodeName string) *framework.Status {
|
||||
return nil
|
||||
}
|
||||
|
||||
// fakeNoopPlugin doesn't implement interface framework.EnqueueExtensions.
|
||||
type fakeNoopPlugin struct{}
|
||||
|
||||
func (*fakeNoopPlugin) Name() string { return fakeNoop }
|
||||
|
||||
func (*fakeNoopPlugin) Filter(_ context.Context, _ *framework.CycleState, _ *v1.Pod, _ *framework.NodeInfo) *framework.Status {
|
||||
return nil
|
||||
}
|
||||
|
||||
var hintFromFakeNode = framework.QueueingHint(100)
|
||||
|
||||
type fakeNodePlugin struct{}
|
||||
|
||||
var fakeNodePluginQueueingFn = func(_ *v1.Pod, _, _ interface{}) framework.QueueingHint {
|
||||
return hintFromFakeNode
|
||||
}
|
||||
|
||||
func (*fakeNodePlugin) Name() string { return fakeNode }
|
||||
|
||||
func (*fakeNodePlugin) Filter(_ context.Context, _ *framework.CycleState, _ *v1.Pod, _ *framework.NodeInfo) *framework.Status {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (*fakeNodePlugin) EventsToRegister() []framework.ClusterEventWithHint {
|
||||
return []framework.ClusterEventWithHint{
|
||||
{Event: framework.ClusterEvent{Resource: framework.Node, ActionType: framework.Add}, QueueingHintFn: fakeNodePluginQueueingFn},
|
||||
}
|
||||
}
|
||||
|
||||
var hintFromFakePod = framework.QueueingHint(101)
|
||||
|
||||
type fakePodPlugin struct{}
|
||||
|
||||
var fakePodPluginQueueingFn = func(_ *v1.Pod, _, _ interface{}) framework.QueueingHint {
|
||||
return hintFromFakePod
|
||||
}
|
||||
|
||||
func (*fakePodPlugin) Name() string { return fakePod }
|
||||
|
||||
func (*fakePodPlugin) Filter(_ context.Context, _ *framework.CycleState, _ *v1.Pod, _ *framework.NodeInfo) *framework.Status {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (*fakePodPlugin) EventsToRegister() []framework.ClusterEventWithHint {
|
||||
return []framework.ClusterEventWithHint{
|
||||
{Event: framework.ClusterEvent{Resource: framework.Pod, ActionType: framework.Add}, QueueingHintFn: fakePodPluginQueueingFn},
|
||||
}
|
||||
}
|
||||
|
||||
// fakeNoopRuntimePlugin implement interface framework.EnqueueExtensions, but returns nil
|
||||
// at runtime. This can simulate a plugin registered at scheduler setup, but does nothing
|
||||
// due to some disabled feature gate.
|
||||
type fakeNoopRuntimePlugin struct{}
|
||||
|
||||
func (*fakeNoopRuntimePlugin) Name() string { return fakeNoopRuntime }
|
||||
|
||||
func (*fakeNoopRuntimePlugin) Filter(_ context.Context, _ *framework.CycleState, _ *v1.Pod, _ *framework.NodeInfo) *framework.Status {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (*fakeNoopRuntimePlugin) EventsToRegister() []framework.ClusterEventWithHint { return nil }
|
||||
|
||||
Reference in New Issue
Block a user