diff --git a/pkg/scheduler/framework/preemption/preemption.go b/pkg/scheduler/framework/preemption/preemption.go index d3744396455..5c0a35134a2 100644 --- a/pkg/scheduler/framework/preemption/preemption.go +++ b/pkg/scheduler/framework/preemption/preemption.go @@ -570,7 +570,7 @@ func (ev *Evaluator) DryRunPreemption(ctx context.Context, pod *v1.Pod, potentia var statusesLock sync.Mutex var errs []error checkNode := func(i int) { - nodeInfoCopy := potentialNodes[(int(offset)+i)%len(potentialNodes)].Clone() + nodeInfoCopy := potentialNodes[(int(offset)+i)%len(potentialNodes)].Snapshot() stateCopy := ev.State.Clone() pods, numPDBViolations, status := ev.SelectVictimsOnNode(ctx, stateCopy, pod, nodeInfoCopy, pdbs) if status.IsSuccess() && len(pods) != 0 { diff --git a/pkg/scheduler/framework/runtime/framework.go b/pkg/scheduler/framework/runtime/framework.go index 4d4588ef5b8..9817d7b3de1 100644 --- a/pkg/scheduler/framework/runtime/framework.go +++ b/pkg/scheduler/framework/runtime/framework.go @@ -942,7 +942,7 @@ func addNominatedPods(ctx context.Context, fh framework.Handle, pod *v1.Pod, sta if len(nominatedPodInfos) == 0 { return false, state, nodeInfo, nil } - nodeInfoOut := nodeInfo.Clone() + nodeInfoOut := nodeInfo.Snapshot() stateOut := state.Clone() podsAdded := false for _, pi := range nominatedPodInfos { diff --git a/pkg/scheduler/framework/types.go b/pkg/scheduler/framework/types.go index c7269890055..6431211cbc8 100644 --- a/pkg/scheduler/framework/types.go +++ b/pkg/scheduler/framework/types.go @@ -462,8 +462,20 @@ func getNamespacesFromPodAffinityTerm(pod *v1.Pod, podAffinityTerm *v1.PodAffini type ImageStateSummary struct { // Size of the image Size int64 - // Used to track how many nodes have this image + // Used to track how many nodes have this image, it is computed from the Nodes field below + // during the execution of Snapshot. NumNodes int + // A set of node names for nodes having this image present. This field is used for + // keeping track of the nodes during update/add/remove events. + Nodes sets.Set[string] +} + +// Snapshot returns a copy without Nodes field of ImageStateSummary +func (iss *ImageStateSummary) Snapshot() *ImageStateSummary { + return &ImageStateSummary{ + Size: iss.Size, + NumNodes: iss.Nodes.Len(), + } } // NodeInfo is node level aggregated information. @@ -640,15 +652,15 @@ func (n *NodeInfo) Node() *v1.Node { return n.node } -// Clone returns a copy of this node. -func (n *NodeInfo) Clone() *NodeInfo { +// Snapshot returns a copy of this node, Except that ImageStates is copied without the Nodes field. +func (n *NodeInfo) Snapshot() *NodeInfo { clone := &NodeInfo{ node: n.node, Requested: n.Requested.Clone(), NonZeroRequested: n.NonZeroRequested.Clone(), Allocatable: n.Allocatable.Clone(), UsedPorts: make(HostPortInfo), - ImageStates: n.ImageStates, + ImageStates: make(map[string]*ImageStateSummary), PVCRefCounts: make(map[string]int), Generation: n.Generation, } @@ -671,6 +683,13 @@ func (n *NodeInfo) Clone() *NodeInfo { if len(n.PodsWithRequiredAntiAffinity) > 0 { clone.PodsWithRequiredAntiAffinity = append([]*PodInfo(nil), n.PodsWithRequiredAntiAffinity...) } + if len(n.ImageStates) > 0 { + state := make(map[string]*ImageStateSummary, len(n.ImageStates)) + for imageName, imageState := range n.ImageStates { + state[imageName] = imageState.Snapshot() + } + clone.ImageStates = state + } for key, value := range n.PVCRefCounts { clone.PVCRefCounts[key] = value } diff --git a/pkg/scheduler/framework/types_test.go b/pkg/scheduler/framework/types_test.go index 7449d9e4cb1..74f9695b28f 100644 --- a/pkg/scheduler/framework/types_test.go +++ b/pkg/scheduler/framework/types_test.go @@ -494,7 +494,7 @@ func TestNodeInfoClone(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("case_%d", i), func(t *testing.T) { - ni := test.nodeInfo.Clone() + ni := test.nodeInfo.Snapshot() // Modify the field to check if the result is a clone of the origin one. test.nodeInfo.Generation += 10 test.nodeInfo.UsedPorts.Remove("127.0.0.1", "TCP", 80) diff --git a/pkg/scheduler/internal/cache/cache.go b/pkg/scheduler/internal/cache/cache.go index 4e94b4b3baa..83b57d30eb0 100644 --- a/pkg/scheduler/internal/cache/cache.go +++ b/pkg/scheduler/internal/cache/cache.go @@ -71,8 +71,8 @@ type cacheImpl struct { // head of the linked list. headNode *nodeInfoListItem nodeTree *nodeTree - // A map from image name to its imageState. - imageStates map[string]*imageState + // A map from image name to its ImageStateSummary. + imageStates map[string]*framework.ImageStateSummary } type podState struct { @@ -84,21 +84,6 @@ type podState struct { bindingFinished bool } -type imageState struct { - // Size of the image - size int64 - // A set of node names for nodes having this image present - nodes sets.Set[string] -} - -// createImageStateSummary returns a summarizing snapshot of the given image's state. -func (cache *cacheImpl) createImageStateSummary(state *imageState) *framework.ImageStateSummary { - return &framework.ImageStateSummary{ - Size: state.size, - NumNodes: len(state.nodes), - } -} - func newCache(ctx context.Context, ttl, period time.Duration) *cacheImpl { logger := klog.FromContext(ctx) return &cacheImpl{ @@ -110,7 +95,7 @@ func newCache(ctx context.Context, ttl, period time.Duration) *cacheImpl { nodeTree: newNodeTree(logger, nil), assumedPods: sets.New[string](), podStates: make(map[string]*podState), - imageStates: make(map[string]*imageState), + imageStates: make(map[string]*framework.ImageStateSummary), } } @@ -182,7 +167,7 @@ func (cache *cacheImpl) Dump() *Dump { nodes := make(map[string]*framework.NodeInfo, len(cache.nodes)) for k, v := range cache.nodes { - nodes[k] = v.info.Clone() + nodes[k] = v.info.Snapshot() } return &Dump{ @@ -233,7 +218,7 @@ func (cache *cacheImpl) UpdateSnapshot(logger klog.Logger, nodeSnapshot *Snapsho existing = &framework.NodeInfo{} nodeSnapshot.nodeInfoMap[np.Name] = existing } - clone := node.info.Clone() + clone := node.info.Snapshot() // We track nodes that have pods with affinity, here we check if this node changed its // status from having pods with affinity to NOT having pods with affinity or the other // way around. @@ -629,13 +614,12 @@ func (cache *cacheImpl) AddNode(logger klog.Logger, node *v1.Node) *framework.No cache.nodeTree.addNode(logger, node) cache.addNodeImageStates(node, n.info) n.info.SetNode(node) - return n.info.Clone() + return n.info.Snapshot() } func (cache *cacheImpl) UpdateNode(logger klog.Logger, oldNode, newNode *v1.Node) *framework.NodeInfo { cache.mu.Lock() defer cache.mu.Unlock() - n, ok := cache.nodes[newNode.Name] if !ok { n = newNodeInfoListItem(framework.NewNodeInfo()) @@ -649,7 +633,7 @@ func (cache *cacheImpl) UpdateNode(logger klog.Logger, oldNode, newNode *v1.Node cache.nodeTree.updateNode(logger, oldNode, newNode) cache.addNodeImageStates(newNode, n.info) n.info.SetNode(newNode) - return n.info.Clone() + return n.info.Snapshot() } // RemoveNode removes a node from the cache's tree. @@ -693,17 +677,17 @@ func (cache *cacheImpl) addNodeImageStates(node *v1.Node, nodeInfo *framework.No // update the entry in imageStates state, ok := cache.imageStates[name] if !ok { - state = &imageState{ - size: image.SizeBytes, - nodes: sets.New(node.Name), + state = &framework.ImageStateSummary{ + Size: image.SizeBytes, + Nodes: sets.New(node.Name), } cache.imageStates[name] = state } else { - state.nodes.Insert(node.Name) + state.Nodes.Insert(node.Name) } - // create the imageStateSummary for this image + // create the ImageStateSummary for this image if _, ok := newSum[name]; !ok { - newSum[name] = cache.createImageStateSummary(state) + newSum[name] = state } } } @@ -722,8 +706,8 @@ func (cache *cacheImpl) removeNodeImageStates(node *v1.Node) { for _, name := range image.Names { state, ok := cache.imageStates[name] if ok { - state.nodes.Delete(node.Name) - if len(state.nodes) == 0 { + state.Nodes.Delete(node.Name) + if state.Nodes.Len() == 0 { // Remove the unused image to make sure the length of // imageStates represents the total number of different // images on all nodes diff --git a/pkg/scheduler/internal/cache/cache_test.go b/pkg/scheduler/internal/cache/cache_test.go index 28e2a738ac5..ccc3d9b2b49 100644 --- a/pkg/scheduler/internal/cache/cache_test.go +++ b/pkg/scheduler/internal/cache/cache_test.go @@ -1037,7 +1037,7 @@ func TestForgetPod(t *testing.T) { } // buildNodeInfo creates a NodeInfo by simulating node operations in cache. -func buildNodeInfo(node *v1.Node, pods []*v1.Pod) *framework.NodeInfo { +func buildNodeInfo(node *v1.Node, pods []*v1.Pod, imageStates map[string]*framework.ImageStateSummary) *framework.NodeInfo { expected := framework.NewNodeInfo() expected.SetNode(node) expected.Allocatable = framework.NewResource(node.Status.Allocatable) @@ -1045,48 +1045,83 @@ func buildNodeInfo(node *v1.Node, pods []*v1.Pod) *framework.NodeInfo { for _, pod := range pods { expected.AddPod(pod) } + for _, image := range node.Status.Images { + for _, name := range image.Names { + if state, ok := imageStates[name]; ok { + expected.ImageStates[name] = state + } + } + } return expected } +// buildImageStates creates ImageStateSummary of image from nodes that will be added in cache. +func buildImageStates(nodes []*v1.Node) map[string]*framework.ImageStateSummary { + imageStates := make(map[string]*framework.ImageStateSummary) + for _, item := range nodes { + for _, image := range item.Status.Images { + for _, name := range image.Names { + if state, ok := imageStates[name]; !ok { + state = &framework.ImageStateSummary{ + Size: image.SizeBytes, + Nodes: sets.New[string](item.Name), + } + imageStates[name] = state + } else { + state.Nodes.Insert(item.Name) + } + } + } + } + return imageStates +} + // TestNodeOperators tests node operations of cache, including add, update // and remove. func TestNodeOperators(t *testing.T) { // Test data - nodeName := "test-node" - cpu1 := resource.MustParse("1000m") - mem100m := resource.MustParse("100m") cpuHalf := resource.MustParse("500m") mem50m := resource.MustParse("50m") - resourceFooName := "example.com/foo" - resourceFoo := resource.MustParse("1") - + resourceList1 := map[v1.ResourceName]string{ + v1.ResourceCPU: "1000m", + v1.ResourceMemory: "100m", + v1.ResourceName("example.com/foo"): "1", + } + resourceList2 := map[v1.ResourceName]string{ + v1.ResourceCPU: "500m", + v1.ResourceMemory: "50m", + v1.ResourceName("example.com/foo"): "2", + } + taints := []v1.Taint{ + { + Key: "test-key", + Value: "test-value", + Effect: v1.TaintEffectPreferNoSchedule, + }, + } + imageStatus1 := map[string]int64{ + "gcr.io/80:latest": 80 * mb, + "gcr.io/80:v1": 80 * mb, + "gcr.io/300:latest": 300 * mb, + "gcr.io/300:v1": 300 * mb, + } + imageStatus2 := map[string]int64{ + "gcr.io/600:latest": 600 * mb, + "gcr.io/80:latest": 80 * mb, + "gcr.io/900:latest": 900 * mb, + } tests := []struct { - name string - node *v1.Node - pods []*v1.Pod + name string + nodes []*v1.Node + pods []*v1.Pod }{ { name: "operate the node with one pod", - node: &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: nodeName, - }, - Status: v1.NodeStatus{ - Allocatable: v1.ResourceList{ - v1.ResourceCPU: cpu1, - v1.ResourceMemory: mem100m, - v1.ResourceName(resourceFooName): resourceFoo, - }, - }, - Spec: v1.NodeSpec{ - Taints: []v1.Taint{ - { - Key: "test-key", - Value: "test-value", - Effect: v1.TaintEffectPreferNoSchedule, - }, - }, - }, + nodes: []*v1.Node{ + &st.MakeNode().Name("test-node-1").Capacity(resourceList1).Taints(taints).Images(imageStatus1).Node, + &st.MakeNode().Name("test-node-2").Capacity(resourceList2).Taints(taints).Images(imageStatus2).Node, + &st.MakeNode().Name("test-node-3").Capacity(resourceList1).Taints(taints).Images(imageStatus1).Node, + &st.MakeNode().Name("test-node-4").Capacity(resourceList2).Taints(taints).Images(imageStatus2).Node, }, pods: []*v1.Pod{ { @@ -1095,7 +1130,7 @@ func TestNodeOperators(t *testing.T) { UID: types.UID("pod1"), }, Spec: v1.PodSpec{ - NodeName: nodeName, + NodeName: "test-node-1", Containers: []v1.Container{ { Resources: v1.ResourceRequirements{ @@ -1119,26 +1154,10 @@ func TestNodeOperators(t *testing.T) { }, { name: "operate the node with two pods", - node: &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: nodeName, - }, - Status: v1.NodeStatus{ - Allocatable: v1.ResourceList{ - v1.ResourceCPU: cpu1, - v1.ResourceMemory: mem100m, - v1.ResourceName(resourceFooName): resourceFoo, - }, - }, - Spec: v1.NodeSpec{ - Taints: []v1.Taint{ - { - Key: "test-key", - Value: "test-value", - Effect: v1.TaintEffectPreferNoSchedule, - }, - }, - }, + nodes: []*v1.Node{ + &st.MakeNode().Name("test-node-1").Capacity(resourceList1).Taints(taints).Images(imageStatus1).Node, + &st.MakeNode().Name("test-node-2").Capacity(resourceList2).Taints(taints).Images(imageStatus2).Node, + &st.MakeNode().Name("test-node-3").Capacity(resourceList1).Taints(taints).Images(imageStatus1).Node, }, pods: []*v1.Pod{ { @@ -1147,7 +1166,7 @@ func TestNodeOperators(t *testing.T) { UID: types.UID("pod1"), }, Spec: v1.PodSpec{ - NodeName: nodeName, + NodeName: "test-node-1", Containers: []v1.Container{ { Resources: v1.ResourceRequirements{ @@ -1166,7 +1185,7 @@ func TestNodeOperators(t *testing.T) { UID: types.UID("pod2"), }, Spec: v1.PodSpec{ - NodeName: nodeName, + NodeName: "test-node-1", Containers: []v1.Container{ { Resources: v1.ResourceRequirements{ @@ -1188,16 +1207,24 @@ func TestNodeOperators(t *testing.T) { logger, ctx := ktesting.NewTestContext(t) ctx, cancel := context.WithCancel(ctx) defer cancel() - expected := buildNodeInfo(tc.node, tc.pods) - node := tc.node + node := tc.nodes[0] + + imageStates := buildImageStates(tc.nodes) + expected := buildNodeInfo(node, tc.pods, imageStates) cache := newCache(ctx, time.Second, time.Second) - cache.AddNode(logger, node) + for _, nodeItem := range tc.nodes { + cache.AddNode(logger, nodeItem) + } for _, pod := range tc.pods { if err := cache.AddPod(logger, pod); err != nil { t.Fatal(err) } } + nodes := map[string]*framework.NodeInfo{} + for nodeItem := cache.headNode; nodeItem != nil; nodeItem = nodeItem.next { + nodes[nodeItem.info.Node().Name] = nodeItem.info + } // Step 1: the node was added into cache successfully. got, found := cache.nodes[node.Name] @@ -1208,14 +1235,20 @@ func TestNodeOperators(t *testing.T) { if err != nil { t.Fatal(err) } - if cache.nodeTree.numNodes != 1 || nodesList[len(nodesList)-1] != node.Name { - t.Errorf("cache.nodeTree is not updated correctly after adding node: %v", node.Name) + if cache.nodeTree.numNodes != len(tc.nodes) || len(nodesList) != len(tc.nodes) { + t.Errorf("cache.nodeTree is not updated correctly after adding node got: %d, expected: %d", + cache.nodeTree.numNodes, len(tc.nodes)) } // Generations are globally unique. We check in our unit tests that they are incremented correctly. expected.Generation = got.info.Generation if diff := cmp.Diff(expected, got.info, cmp.AllowUnexported(framework.NodeInfo{})); diff != "" { - t.Errorf("Unexpected node info from cache (-want, +got):\n%s", diff) + t.Errorf("Failed to add node into scheduler cache (-want,+got):\n%s", diff) + } + + // check imageState of NodeInfo with specific image when node added + if !checkImageStateSummary(nodes, "gcr.io/80:latest", "gcr.io/300:latest") { + t.Error("image have different ImageStateSummary") } // Step 2: dump cached nodes successfully. @@ -1224,12 +1257,16 @@ func TestNodeOperators(t *testing.T) { t.Error(err) } newNode, found := cachedNodes.nodeInfoMap[node.Name] - if !found || len(cachedNodes.nodeInfoMap) != 1 { - t.Errorf("failed to dump cached nodes:\n got: %v \nexpected: %v", cachedNodes, cache.nodes) + if !found || len(cachedNodes.nodeInfoMap) != len(tc.nodes) { + t.Errorf("failed to dump cached nodes:\n got: %v \nexpected: %v", cachedNodes.nodeInfoMap, tc.nodes) } expected.Generation = newNode.Generation - if diff := cmp.Diff(expected, newNode, cmp.AllowUnexported(framework.NodeInfo{})); diff != "" { - t.Errorf("Unexpected clone node info (-want, +got):\n%s", diff) + if diff := cmp.Diff(newNode, expected.Snapshot(), cmp.AllowUnexported(framework.NodeInfo{})); diff != "" { + t.Errorf("Failed to clone node:\n%s", diff) + } + // check imageState of NodeInfo with specific image when update snapshot + if !checkImageStateSummary(cachedNodes.nodeInfoMap, "gcr.io/80:latest", "gcr.io/300:latest") { + t.Error("image have different ImageStateSummary") } // Step 3: update node attribute successfully. @@ -1249,13 +1286,17 @@ func TestNodeOperators(t *testing.T) { if diff := cmp.Diff(expected, got.info, cmp.AllowUnexported(framework.NodeInfo{})); diff != "" { t.Errorf("Unexpected schedulertypes after updating node (-want, +got):\n%s", diff) } + // check imageState of NodeInfo with specific image when update node + if !checkImageStateSummary(nodes, "gcr.io/80:latest", "gcr.io/300:latest") { + t.Error("image have different ImageStateSummary") + } // Check nodeTree after update nodesList, err = cache.nodeTree.list() if err != nil { t.Fatal(err) } - if cache.nodeTree.numNodes != 1 || nodesList[len(nodesList)-1] != node.Name { - t.Errorf("unexpected cache.nodeTree after updating node: %v", node.Name) + if cache.nodeTree.numNodes != len(tc.nodes) || len(nodesList) != len(tc.nodes) { + t.Errorf("unexpected cache.nodeTree after updating node") } // Step 4: the node can be removed even if it still has pods. @@ -1278,9 +1319,13 @@ func TestNodeOperators(t *testing.T) { if err != nil { t.Fatal(err) } - if cache.nodeTree.numNodes != 0 || len(nodesList) != 0 { + if cache.nodeTree.numNodes != len(tc.nodes)-1 || len(nodesList) != len(tc.nodes)-1 { t.Errorf("unexpected cache.nodeTree after removing node: %v", node.Name) } + // check imageState of NodeInfo with specific image when delete node + if !checkImageStateSummary(nodes, "gcr.io/80:latest", "gcr.io/300:latest") { + t.Error("image have different ImageStateSummary after removing node") + } // Pods are still in the pods cache. for _, p := range tc.pods { if _, err := cache.GetPod(p); err != nil { @@ -1976,6 +2021,28 @@ func makeBasePod(t testingMode, nodeName, objName, cpu, mem, extended string, po return podWrapper.Obj() } +// checkImageStateSummary collect ImageStateSummary of image traverse nodes, +// the collected ImageStateSummary should be equal +func checkImageStateSummary(nodes map[string]*framework.NodeInfo, imageNames ...string) bool { + for _, imageName := range imageNames { + var imageState *framework.ImageStateSummary + for _, node := range nodes { + state, ok := node.ImageStates[imageName] + if !ok { + continue + } + if imageState == nil { + imageState = state + continue + } + if diff := cmp.Diff(imageState, state); diff != "" { + return false + } + } + } + return true +} + func setupCacheOf1kNodes30kPods(b *testing.B) Cache { logger, ctx := ktesting.NewTestContext(b) ctx, cancel := context.WithCancel(ctx) diff --git a/pkg/scheduler/internal/cache/snapshot.go b/pkg/scheduler/internal/cache/snapshot.go index abd79312a2d..164f1510c6d 100644 --- a/pkg/scheduler/internal/cache/snapshot.go +++ b/pkg/scheduler/internal/cache/snapshot.go @@ -130,7 +130,7 @@ func getNodeImageStates(node *v1.Node, imageExistenceMap map[string]sets.Set[str for _, name := range image.Names { imageStates[name] = &framework.ImageStateSummary{ Size: image.SizeBytes, - NumNodes: len(imageExistenceMap[name]), + NumNodes: imageExistenceMap[name].Len(), } } } diff --git a/pkg/scheduler/schedule_one_test.go b/pkg/scheduler/schedule_one_test.go index bce0284a366..fd032aee63c 100644 --- a/pkg/scheduler/schedule_one_test.go +++ b/pkg/scheduler/schedule_one_test.go @@ -53,6 +53,7 @@ import ( "k8s.io/kubernetes/pkg/scheduler/framework" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/imagelocality" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeports" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/podtopologyspread" @@ -70,7 +71,8 @@ import ( ) const ( - testSchedulerName = "test-scheduler" + testSchedulerName = "test-scheduler" + mb int64 = 1024 * 1024 ) var ( @@ -2676,6 +2678,59 @@ func TestZeroRequest(t *testing.T) { } func Test_prioritizeNodes(t *testing.T) { + imageStatus1 := []v1.ContainerImage{ + { + Names: []string{ + "gcr.io/40:latest", + "gcr.io/40:v1", + }, + SizeBytes: int64(80 * mb), + }, + { + Names: []string{ + "gcr.io/300:latest", + "gcr.io/300:v1", + }, + SizeBytes: int64(300 * mb), + }, + } + + imageStatus2 := []v1.ContainerImage{ + { + Names: []string{ + "gcr.io/300:latest", + }, + SizeBytes: int64(300 * mb), + }, + { + Names: []string{ + "gcr.io/40:latest", + "gcr.io/40:v1", + }, + SizeBytes: int64(80 * mb), + }, + } + + imageStatus3 := []v1.ContainerImage{ + { + Names: []string{ + "gcr.io/600:latest", + }, + SizeBytes: int64(600 * mb), + }, + { + Names: []string{ + "gcr.io/40:latest", + }, + SizeBytes: int64(80 * mb), + }, + { + Names: []string{ + "gcr.io/900:latest", + }, + SizeBytes: int64(900 * mb), + }, + } tests := []struct { name string pod *v1.Pod @@ -2862,6 +2917,115 @@ func Test_prioritizeNodes(t *testing.T) { {Name: "node2", Scores: []framework.PluginScore{}}, }, }, + { + name: "the score from Image Locality plugin with image in all nodes", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Image: "gcr.io/40", + }, + }, + }, + }, + nodes: []*v1.Node{ + makeNode("node1", 1000, schedutil.DefaultMemoryRequest*10, imageStatus1...), + makeNode("node2", 1000, schedutil.DefaultMemoryRequest*10, imageStatus2...), + makeNode("node3", 1000, schedutil.DefaultMemoryRequest*10, imageStatus3...), + }, + pluginRegistrations: []tf.RegisterPluginFunc{ + tf.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), + tf.RegisterScorePlugin(imagelocality.Name, imagelocality.New, 1), + tf.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), + }, + extenders: nil, + want: []framework.NodePluginScores{ + { + Name: "node1", + Scores: []framework.PluginScore{ + { + Name: "ImageLocality", + Score: 5, + }, + }, + TotalScore: 5, + }, + { + Name: "node2", + Scores: []framework.PluginScore{ + { + Name: "ImageLocality", + Score: 5, + }, + }, + TotalScore: 5, + }, + { + Name: "node3", + Scores: []framework.PluginScore{ + { + Name: "ImageLocality", + Score: 5, + }, + }, + TotalScore: 5, + }, + }, + }, + { + name: "the score from Image Locality plugin with image in partial nodes", + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Image: "gcr.io/300", + }, + }, + }, + }, + nodes: []*v1.Node{makeNode("node1", 1000, schedutil.DefaultMemoryRequest*10, imageStatus1...), + makeNode("node2", 1000, schedutil.DefaultMemoryRequest*10, imageStatus2...), + makeNode("node3", 1000, schedutil.DefaultMemoryRequest*10, imageStatus3...), + }, + pluginRegistrations: []tf.RegisterPluginFunc{ + tf.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), + tf.RegisterScorePlugin(imagelocality.Name, imagelocality.New, 1), + tf.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), + }, + extenders: nil, + want: []framework.NodePluginScores{ + { + Name: "node1", + Scores: []framework.PluginScore{ + { + Name: "ImageLocality", + Score: 18, + }, + }, + TotalScore: 18, + }, + { + Name: "node2", + Scores: []framework.PluginScore{ + { + Name: "ImageLocality", + Score: 18, + }, + }, + TotalScore: 18, + }, + { + Name: "node3", + Scores: []framework.PluginScore{ + { + Name: "ImageLocality", + Score: 0, + }, + }, + TotalScore: 0, + }, + }, + }, } for _, test := range tests { @@ -2869,9 +3033,16 @@ func Test_prioritizeNodes(t *testing.T) { client := clientsetfake.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(client, 0) - snapshot := internalcache.NewSnapshot(test.pods, test.nodes) ctx, cancel := context.WithCancel(context.Background()) defer cancel() + cache := internalcache.New(ctx, time.Duration(0)) + for _, node := range test.nodes { + cache.AddNode(klog.FromContext(ctx), node) + } + snapshot := internalcache.NewEmptySnapshot() + if err := cache.UpdateSnapshot(klog.FromContext(ctx), snapshot); err != nil { + t.Fatal(err) + } fwk, err := tf.NewFramework( ctx, test.pluginRegistrations, "", @@ -3151,7 +3322,7 @@ func makeScheduler(ctx context.Context, nodes []*v1.Node) *Scheduler { return sched } -func makeNode(node string, milliCPU, memory int64) *v1.Node { +func makeNode(node string, milliCPU, memory int64, images ...v1.ContainerImage) *v1.Node { return &v1.Node{ ObjectMeta: metav1.ObjectMeta{Name: node}, Status: v1.NodeStatus{ @@ -3166,6 +3337,7 @@ func makeNode(node string, milliCPU, memory int64) *v1.Node { v1.ResourceMemory: *resource.NewQuantity(memory, resource.BinarySI), "pods": *resource.NewQuantity(100, resource.DecimalSI), }, + Images: images, }, } } diff --git a/pkg/scheduler/testing/framework/fake_extender.go b/pkg/scheduler/testing/framework/fake_extender.go index 42d8dfad948..755f95ed3c8 100644 --- a/pkg/scheduler/testing/framework/fake_extender.go +++ b/pkg/scheduler/testing/framework/fake_extender.go @@ -232,7 +232,7 @@ func (f *FakeExtender) selectVictimsOnNodeByExtender(pod *v1.Pod, node *v1.Node) // Otherwise, as a extender support preemption and have cached node info, we will assume cachedNodeNameToInfo is available // and get cached node info by given node name. - nodeInfoCopy := f.CachedNodeNameToInfo[node.GetName()].Clone() + nodeInfoCopy := f.CachedNodeNameToInfo[node.GetName()].Snapshot() var potentialVictims []*v1.Pod